"""Read only the ledger in the first worksheet; never import dashboard cells.""" from collections import Counter from datetime import date, datetime from decimal import Decimal, ROUND_HALF_UP import hashlib import json from database import connect, ensure_asset, initialize from models import canonical_name, cents, name_key, valid_date HEADERS = { 'date': {'datum', 'date'}, 'name': {'artdesertrags', 'position', 'asset'}, 'amount': {'betrag', 'betrageur', 'amount'}, 'category': {'kategorie', 'category'}, 'note': {'notiz', 'note'}, 'expected': {'erwartet', 'expected'}, 'received': {'erhalten', 'received'}, } CATEGORY_MAP = {'dividende': 'dividend', 'dividenden': 'dividend', 'dividend': 'dividend', 'dividendenausschüttungen': 'dividend', 'ausschüttung': 'distribution', 'ausschüttungen': 'distribution', 'distribution': 'distribution', 'zinsen': 'interest', 'zins': 'interest', 'interest': 'interest', 'sonstiges': 'other', 'sonstige': 'other', 'other': 'other'} def boolean(value, default): if value is None or value == '': return default key = str(value).strip().casefold() if key in {'1', 'true', 'ja', 'yes', 'wahr'}: return 1 if key in {'0', 'false', 'nein', 'no', 'falsch'}: return 0 raise ValueError('Ungültiger Erwartet-/Erhalten-Wert.') def read_ledger(filename): # Only the standalone importer needs openpyxl, never the running web app. from openpyxl import load_workbook from openpyxl.utils.datetime import from_excel book = load_workbook(filename, read_only=True, data_only=False) records, mapping, warnings = [], None, [] try: sheet = book.worksheets[0] for row_index, row in enumerate(sheet.iter_rows(), 1): values = [cell.value for cell in row] if mapping is None: found = {} for index, value in enumerate(values): key = name_key(str(value or '')) for field, aliases in HEADERS.items(): if key in aliases: found[field] = index if {'date', 'name', 'amount', 'category'} <= found.keys(): mapping = found continue def value(field): index = mapping.get(field) return values[index] if index is not None and index < len(values) else None if all(value(field) in (None, '') for field in ('date', 'name', 'amount', 'category')): continue # A summary/header row is not a transaction. if value('date') in (None, '') and name_key(str(value('name') or '')) in {'', 'gesamt', 'summe'}: continue try: if any(row[mapping[field]].data_type == 'f' for field in ('date', 'name', 'amount', 'category')): raise ValueError('Formel innerhalb einer Buchung; bitte als echte Buchungswerte bereitstellen.') raw_date = value('date') if isinstance(raw_date, (int, float)): raw_date = from_excel(raw_date, book.epoch) if isinstance(raw_date, datetime): raw_date = raw_date.date() if isinstance(raw_date, date): day = raw_date.isoformat() else: text = str(raw_date).strip() try: day = valid_date(text) except ValueError: day = datetime.strptime(text, '%d.%m.%Y').date().isoformat() if value('name') is None: raise ValueError('Position fehlt.') name = canonical_name(value('name')) category = CATEGORY_MAP.get(name_key(str(value('category') or ''))) if category is None: raise ValueError('Unbekannte Kategorie.') note = str(value('note') or '').strip() # airBaltic is a bond: historical combined dividend label is inaccurate. if name == 'airBaltic' and category != 'interest': category = 'interest' note = (note + ' | ' if note else '') + 'Excel-Kategorie fachlich korrigiert: airBaltic-Anleihezinsen.' warnings.append(f'Zeile {row_index}: airBaltic als Zinsen normalisiert.') raw_amount = value('amount') if isinstance(raw_amount, (int, float)): decimal = Decimal(str(raw_amount)) rounded = decimal.quantize(Decimal('.01'), rounding=ROUND_HALF_UP) if abs(decimal - rounded) > Decimal('0.000001'): raise ValueError('Betrag hat mehr als zwei Nachkommastellen.') amount = cents(format(rounded, '.2f')) else: amount = cents(raw_amount) expected = boolean(value('expected'), 0) received = boolean(value('received'), 1) if len(note) > 2000: raise ValueError('Notiz zu lang.') kind = 'bond' if name == 'airBaltic' else 'etf' if name == 'STOXX Global Select Dividend 100' else 'interest' if category == 'interest' else 'stock' if category in {'dividend','distribution'} else 'other' records.append(dict(date=day, name=name, amount=amount, category=category, note=note, expected=expected, received=received, kind=kind)) except (ValueError, TypeError, OverflowError, ArithmeticError) as error: raise ValueError(f'Zeile {row_index}: {error}') from error if mapping is None: raise ValueError('Kein Ertragsbuch mit Datum, Position/Art des Ertrags, Betrag und Kategorie im ersten Blatt gefunden.') if not records: raise ValueError('Das Ertragsbuch enthält keine Buchungen.') return records, warnings finally: book.close() def import_excel(filename, path=None, dry_run=False): records, warnings = read_ledger(filename) initialize(path) added = skipped = 0 occurrences = Counter() with connect(path) as db: db.execute('BEGIN IMMEDIATE') for record in records: asset_id = ensure_asset(db, record['name'], record['kind']) identity = (record['date'], asset_id, record['category'], record['amount'], record['expected'], record['received']) occurrences[identity] += 1 occurrence = occurrences[identity] fingerprint = hashlib.sha256(json.dumps([*identity, occurrence], separators=(',', ':')).encode()).hexdigest() if db.execute('SELECT 1 FROM import_records WHERE fingerprint=?', (fingerprint,)).fetchone(): skipped += 1 continue # Reuse matching manual entries as well. Preserve legitimate identical payments by occurrence. existing = db.execute('SELECT id FROM income_entries WHERE date=? AND asset_id=? AND category=? AND amount=? AND expected=? AND received=? ORDER BY id LIMIT 1 OFFSET ?', (*identity, occurrence-1)).fetchone() if existing: entry_id = existing['id'] skipped += 1 else: entry_id = db.execute('INSERT INTO income_entries (date,asset_id,category,amount,expected,received,note) VALUES (?,?,?,?,?,?,?)', (*identity, record['note'] or None)).lastrowid added += 1 db.execute('INSERT INTO import_records (fingerprint,entry_id) VALUES (?,?)', (fingerprint, entry_id)) september = db.execute("SELECT COALESCE(SUM(amount),0) FROM income_entries WHERE date >= '2026-09-01' AND date < '2026-10-01' AND received=1").fetchone()[0] enbridge = db.execute("SELECT COUNT(*) FROM income_entries i JOIN assets a ON a.id=i.asset_id WHERE date='2026-09-02' AND a.normalized_name='enbridge' AND amount=4 AND received=1").fetchone()[0] if dry_run: db.rollback() return dict(rows=len(records), added=added, skipped=skipped, warnings=warnings, september_2026=september, enbridge_check=bool(enbridge), dry_run=dry_run)