Add SQLite passive income dashboard with Excel import and deployment tools
This commit is contained in:
@@ -0,0 +1,147 @@
|
||||
"""Read only the ledger in the first worksheet; never import dashboard cells."""
|
||||
from collections import Counter
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from database import connect, ensure_asset, initialize
|
||||
from models import canonical_name, cents, name_key, valid_date
|
||||
|
||||
HEADERS = {
|
||||
'date': {'datum', 'date'}, 'name': {'artdesertrags', 'position', 'asset'},
|
||||
'amount': {'betrag', 'betrageur', 'amount'}, 'category': {'kategorie', 'category'},
|
||||
'note': {'notiz', 'note'}, 'expected': {'erwartet', 'expected'}, 'received': {'erhalten', 'received'},
|
||||
}
|
||||
CATEGORY_MAP = {'dividende': 'dividend', 'dividenden': 'dividend', 'dividend': 'dividend',
|
||||
'dividendenausschüttungen': 'dividend', 'ausschüttung': 'distribution',
|
||||
'ausschüttungen': 'distribution', 'distribution': 'distribution',
|
||||
'zinsen': 'interest', 'zins': 'interest', 'interest': 'interest',
|
||||
'sonstiges': 'other', 'sonstige': 'other', 'other': 'other'}
|
||||
|
||||
|
||||
def boolean(value, default):
|
||||
if value is None or value == '':
|
||||
return default
|
||||
key = str(value).strip().casefold()
|
||||
if key in {'1', 'true', 'ja', 'yes', 'wahr'}:
|
||||
return 1
|
||||
if key in {'0', 'false', 'nein', 'no', 'falsch'}:
|
||||
return 0
|
||||
raise ValueError('Ungültiger Erwartet-/Erhalten-Wert.')
|
||||
|
||||
|
||||
def read_ledger(filename):
|
||||
# Only the standalone importer needs openpyxl, never the running web app.
|
||||
from openpyxl import load_workbook
|
||||
from openpyxl.utils.datetime import from_excel
|
||||
book = load_workbook(filename, read_only=True, data_only=False)
|
||||
records, mapping, warnings = [], None, []
|
||||
try:
|
||||
sheet = book.worksheets[0]
|
||||
for row_index, row in enumerate(sheet.iter_rows(), 1):
|
||||
values = [cell.value for cell in row]
|
||||
if mapping is None:
|
||||
found = {}
|
||||
for index, value in enumerate(values):
|
||||
key = name_key(str(value or ''))
|
||||
for field, aliases in HEADERS.items():
|
||||
if key in aliases:
|
||||
found[field] = index
|
||||
if {'date', 'name', 'amount', 'category'} <= found.keys():
|
||||
mapping = found
|
||||
continue
|
||||
def value(field):
|
||||
index = mapping.get(field)
|
||||
return values[index] if index is not None and index < len(values) else None
|
||||
if all(value(field) in (None, '') for field in ('date', 'name', 'amount', 'category')):
|
||||
continue
|
||||
# A summary/header row is not a transaction.
|
||||
if value('date') in (None, '') and name_key(str(value('name') or '')) in {'', 'gesamt', 'summe'}:
|
||||
continue
|
||||
try:
|
||||
if any(row[mapping[field]].data_type == 'f' for field in ('date', 'name', 'amount', 'category')):
|
||||
raise ValueError('Formel innerhalb einer Buchung; bitte als echte Buchungswerte bereitstellen.')
|
||||
raw_date = value('date')
|
||||
if isinstance(raw_date, (int, float)):
|
||||
raw_date = from_excel(raw_date, book.epoch)
|
||||
if isinstance(raw_date, datetime):
|
||||
raw_date = raw_date.date()
|
||||
if isinstance(raw_date, date):
|
||||
day = raw_date.isoformat()
|
||||
else:
|
||||
text = str(raw_date).strip()
|
||||
try:
|
||||
day = valid_date(text)
|
||||
except ValueError:
|
||||
day = datetime.strptime(text, '%d.%m.%Y').date().isoformat()
|
||||
if value('name') is None:
|
||||
raise ValueError('Position fehlt.')
|
||||
name = canonical_name(value('name'))
|
||||
category = CATEGORY_MAP.get(name_key(str(value('category') or '')))
|
||||
if category is None:
|
||||
raise ValueError('Unbekannte Kategorie.')
|
||||
note = str(value('note') or '').strip()
|
||||
# airBaltic is a bond: historical combined dividend label is inaccurate.
|
||||
if name == 'airBaltic' and category != 'interest':
|
||||
category = 'interest'
|
||||
note = (note + ' | ' if note else '') + 'Excel-Kategorie fachlich korrigiert: airBaltic-Anleihezinsen.'
|
||||
warnings.append(f'Zeile {row_index}: airBaltic als Zinsen normalisiert.')
|
||||
raw_amount = value('amount')
|
||||
if isinstance(raw_amount, (int, float)):
|
||||
decimal = Decimal(str(raw_amount))
|
||||
rounded = decimal.quantize(Decimal('.01'), rounding=ROUND_HALF_UP)
|
||||
if abs(decimal - rounded) > Decimal('0.000001'):
|
||||
raise ValueError('Betrag hat mehr als zwei Nachkommastellen.')
|
||||
amount = cents(format(rounded, '.2f'))
|
||||
else:
|
||||
amount = cents(raw_amount)
|
||||
expected = boolean(value('expected'), 0)
|
||||
received = boolean(value('received'), 1)
|
||||
if len(note) > 2000:
|
||||
raise ValueError('Notiz zu lang.')
|
||||
kind = 'bond' if name == 'airBaltic' else 'etf' if name == 'STOXX Global Select Dividend 100' else 'interest' if category == 'interest' else 'stock' if category in {'dividend','distribution'} else 'other'
|
||||
records.append(dict(date=day, name=name, amount=amount, category=category, note=note,
|
||||
expected=expected, received=received, kind=kind))
|
||||
except (ValueError, TypeError, OverflowError, ArithmeticError) as error:
|
||||
raise ValueError(f'Zeile {row_index}: {error}') from error
|
||||
if mapping is None:
|
||||
raise ValueError('Kein Ertragsbuch mit Datum, Position/Art des Ertrags, Betrag und Kategorie im ersten Blatt gefunden.')
|
||||
if not records:
|
||||
raise ValueError('Das Ertragsbuch enthält keine Buchungen.')
|
||||
return records, warnings
|
||||
finally:
|
||||
book.close()
|
||||
|
||||
|
||||
def import_excel(filename, path=None, dry_run=False):
|
||||
records, warnings = read_ledger(filename)
|
||||
initialize(path)
|
||||
added = skipped = 0
|
||||
occurrences = Counter()
|
||||
with connect(path) as db:
|
||||
db.execute('BEGIN IMMEDIATE')
|
||||
for record in records:
|
||||
asset_id = ensure_asset(db, record['name'], record['kind'])
|
||||
identity = (record['date'], asset_id, record['category'], record['amount'], record['expected'], record['received'])
|
||||
occurrences[identity] += 1
|
||||
occurrence = occurrences[identity]
|
||||
fingerprint = hashlib.sha256(json.dumps([*identity, occurrence], separators=(',', ':')).encode()).hexdigest()
|
||||
if db.execute('SELECT 1 FROM import_records WHERE fingerprint=?', (fingerprint,)).fetchone():
|
||||
skipped += 1
|
||||
continue
|
||||
# Reuse matching manual entries as well. Preserve legitimate identical payments by occurrence.
|
||||
existing = db.execute('SELECT id FROM income_entries WHERE date=? AND asset_id=? AND category=? AND amount=? AND expected=? AND received=? ORDER BY id LIMIT 1 OFFSET ?', (*identity, occurrence-1)).fetchone()
|
||||
if existing:
|
||||
entry_id = existing['id']
|
||||
skipped += 1
|
||||
else:
|
||||
entry_id = db.execute('INSERT INTO income_entries (date,asset_id,category,amount,expected,received,note) VALUES (?,?,?,?,?,?,?)', (*identity, record['note'] or None)).lastrowid
|
||||
added += 1
|
||||
db.execute('INSERT INTO import_records (fingerprint,entry_id) VALUES (?,?)', (fingerprint, entry_id))
|
||||
september = db.execute("SELECT COALESCE(SUM(amount),0) FROM income_entries WHERE date >= '2026-09-01' AND date < '2026-10-01' AND received=1").fetchone()[0]
|
||||
enbridge = db.execute("SELECT COUNT(*) FROM income_entries i JOIN assets a ON a.id=i.asset_id WHERE date='2026-09-02' AND a.normalized_name='enbridge' AND amount=4 AND received=1").fetchone()[0]
|
||||
if dry_run:
|
||||
db.rollback()
|
||||
return dict(rows=len(records), added=added, skipped=skipped, warnings=warnings,
|
||||
september_2026=september, enbridge_check=bool(enbridge), dry_run=dry_run)
|
||||
Reference in New Issue
Block a user