Add SQLite passive income dashboard with Excel import and deployment tools
This commit is contained in:
@@ -0,0 +1,147 @@
|
||||
"""Read only the ledger in the first worksheet; never import dashboard cells."""
|
||||
from collections import Counter
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from database import connect, ensure_asset, initialize
|
||||
from models import canonical_name, cents, name_key, valid_date
|
||||
|
||||
HEADERS = {
|
||||
'date': {'datum', 'date'}, 'name': {'artdesertrags', 'position', 'asset'},
|
||||
'amount': {'betrag', 'betrageur', 'amount'}, 'category': {'kategorie', 'category'},
|
||||
'note': {'notiz', 'note'}, 'expected': {'erwartet', 'expected'}, 'received': {'erhalten', 'received'},
|
||||
}
|
||||
CATEGORY_MAP = {'dividende': 'dividend', 'dividenden': 'dividend', 'dividend': 'dividend',
|
||||
'dividendenausschüttungen': 'dividend', 'ausschüttung': 'distribution',
|
||||
'ausschüttungen': 'distribution', 'distribution': 'distribution',
|
||||
'zinsen': 'interest', 'zins': 'interest', 'interest': 'interest',
|
||||
'sonstiges': 'other', 'sonstige': 'other', 'other': 'other'}
|
||||
|
||||
|
||||
def boolean(value, default):
|
||||
if value is None or value == '':
|
||||
return default
|
||||
key = str(value).strip().casefold()
|
||||
if key in {'1', 'true', 'ja', 'yes', 'wahr'}:
|
||||
return 1
|
||||
if key in {'0', 'false', 'nein', 'no', 'falsch'}:
|
||||
return 0
|
||||
raise ValueError('Ungültiger Erwartet-/Erhalten-Wert.')
|
||||
|
||||
|
||||
def read_ledger(filename):
|
||||
# Only the standalone importer needs openpyxl, never the running web app.
|
||||
from openpyxl import load_workbook
|
||||
from openpyxl.utils.datetime import from_excel
|
||||
book = load_workbook(filename, read_only=True, data_only=False)
|
||||
records, mapping, warnings = [], None, []
|
||||
try:
|
||||
sheet = book.worksheets[0]
|
||||
for row_index, row in enumerate(sheet.iter_rows(), 1):
|
||||
values = [cell.value for cell in row]
|
||||
if mapping is None:
|
||||
found = {}
|
||||
for index, value in enumerate(values):
|
||||
key = name_key(str(value or ''))
|
||||
for field, aliases in HEADERS.items():
|
||||
if key in aliases:
|
||||
found[field] = index
|
||||
if {'date', 'name', 'amount', 'category'} <= found.keys():
|
||||
mapping = found
|
||||
continue
|
||||
def value(field):
|
||||
index = mapping.get(field)
|
||||
return values[index] if index is not None and index < len(values) else None
|
||||
if all(value(field) in (None, '') for field in ('date', 'name', 'amount', 'category')):
|
||||
continue
|
||||
# A summary/header row is not a transaction.
|
||||
if value('date') in (None, '') and name_key(str(value('name') or '')) in {'', 'gesamt', 'summe'}:
|
||||
continue
|
||||
try:
|
||||
if any(row[mapping[field]].data_type == 'f' for field in ('date', 'name', 'amount', 'category')):
|
||||
raise ValueError('Formel innerhalb einer Buchung; bitte als echte Buchungswerte bereitstellen.')
|
||||
raw_date = value('date')
|
||||
if isinstance(raw_date, (int, float)):
|
||||
raw_date = from_excel(raw_date, book.epoch)
|
||||
if isinstance(raw_date, datetime):
|
||||
raw_date = raw_date.date()
|
||||
if isinstance(raw_date, date):
|
||||
day = raw_date.isoformat()
|
||||
else:
|
||||
text = str(raw_date).strip()
|
||||
try:
|
||||
day = valid_date(text)
|
||||
except ValueError:
|
||||
day = datetime.strptime(text, '%d.%m.%Y').date().isoformat()
|
||||
if value('name') is None:
|
||||
raise ValueError('Position fehlt.')
|
||||
name = canonical_name(value('name'))
|
||||
category = CATEGORY_MAP.get(name_key(str(value('category') or '')))
|
||||
if category is None:
|
||||
raise ValueError('Unbekannte Kategorie.')
|
||||
note = str(value('note') or '').strip()
|
||||
# airBaltic is a bond: historical combined dividend label is inaccurate.
|
||||
if name == 'airBaltic' and category != 'interest':
|
||||
category = 'interest'
|
||||
note = (note + ' | ' if note else '') + 'Excel-Kategorie fachlich korrigiert: airBaltic-Anleihezinsen.'
|
||||
warnings.append(f'Zeile {row_index}: airBaltic als Zinsen normalisiert.')
|
||||
raw_amount = value('amount')
|
||||
if isinstance(raw_amount, (int, float)):
|
||||
decimal = Decimal(str(raw_amount))
|
||||
rounded = decimal.quantize(Decimal('.01'), rounding=ROUND_HALF_UP)
|
||||
if abs(decimal - rounded) > Decimal('0.000001'):
|
||||
raise ValueError('Betrag hat mehr als zwei Nachkommastellen.')
|
||||
amount = cents(format(rounded, '.2f'))
|
||||
else:
|
||||
amount = cents(raw_amount)
|
||||
expected = boolean(value('expected'), 0)
|
||||
received = boolean(value('received'), 1)
|
||||
if len(note) > 2000:
|
||||
raise ValueError('Notiz zu lang.')
|
||||
kind = 'bond' if name == 'airBaltic' else 'etf' if name == 'STOXX Global Select Dividend 100' else 'interest' if category == 'interest' else 'stock' if category in {'dividend','distribution'} else 'other'
|
||||
records.append(dict(date=day, name=name, amount=amount, category=category, note=note,
|
||||
expected=expected, received=received, kind=kind))
|
||||
except (ValueError, TypeError, OverflowError, ArithmeticError) as error:
|
||||
raise ValueError(f'Zeile {row_index}: {error}') from error
|
||||
if mapping is None:
|
||||
raise ValueError('Kein Ertragsbuch mit Datum, Position/Art des Ertrags, Betrag und Kategorie im ersten Blatt gefunden.')
|
||||
if not records:
|
||||
raise ValueError('Das Ertragsbuch enthält keine Buchungen.')
|
||||
return records, warnings
|
||||
finally:
|
||||
book.close()
|
||||
|
||||
|
||||
def import_excel(filename, path=None, dry_run=False):
|
||||
records, warnings = read_ledger(filename)
|
||||
initialize(path)
|
||||
added = skipped = 0
|
||||
occurrences = Counter()
|
||||
with connect(path) as db:
|
||||
db.execute('BEGIN IMMEDIATE')
|
||||
for record in records:
|
||||
asset_id = ensure_asset(db, record['name'], record['kind'])
|
||||
identity = (record['date'], asset_id, record['category'], record['amount'], record['expected'], record['received'])
|
||||
occurrences[identity] += 1
|
||||
occurrence = occurrences[identity]
|
||||
fingerprint = hashlib.sha256(json.dumps([*identity, occurrence], separators=(',', ':')).encode()).hexdigest()
|
||||
if db.execute('SELECT 1 FROM import_records WHERE fingerprint=?', (fingerprint,)).fetchone():
|
||||
skipped += 1
|
||||
continue
|
||||
# Reuse matching manual entries as well. Preserve legitimate identical payments by occurrence.
|
||||
existing = db.execute('SELECT id FROM income_entries WHERE date=? AND asset_id=? AND category=? AND amount=? AND expected=? AND received=? ORDER BY id LIMIT 1 OFFSET ?', (*identity, occurrence-1)).fetchone()
|
||||
if existing:
|
||||
entry_id = existing['id']
|
||||
skipped += 1
|
||||
else:
|
||||
entry_id = db.execute('INSERT INTO income_entries (date,asset_id,category,amount,expected,received,note) VALUES (?,?,?,?,?,?,?)', (*identity, record['note'] or None)).lastrowid
|
||||
added += 1
|
||||
db.execute('INSERT INTO import_records (fingerprint,entry_id) VALUES (?,?)', (fingerprint, entry_id))
|
||||
september = db.execute("SELECT COALESCE(SUM(amount),0) FROM income_entries WHERE date >= '2026-09-01' AND date < '2026-10-01' AND received=1").fetchone()[0]
|
||||
enbridge = db.execute("SELECT COUNT(*) FROM income_entries i JOIN assets a ON a.id=i.asset_id WHERE date='2026-09-02' AND a.normalized_name='enbridge' AND amount=4 AND received=1").fetchone()[0]
|
||||
if dry_run:
|
||||
db.rollback()
|
||||
return dict(rows=len(records), added=added, skipped=skipped, warnings=warnings,
|
||||
september_2026=september, enbridge_check=bool(enbridge), dry_run=dry_run)
|
||||
@@ -0,0 +1,107 @@
|
||||
from datetime import date
|
||||
from fastapi import HTTPException
|
||||
from database import connect
|
||||
from models import CATEGORIES, MONTHS, cents, valid_date, percent
|
||||
|
||||
|
||||
def assets():
|
||||
with connect() as db:
|
||||
return db.execute('SELECT * FROM assets ORDER BY name COLLATE NOCASE').fetchall()
|
||||
|
||||
|
||||
def get_entry(entry_id):
|
||||
if not 1 <= entry_id <= 9223372036854775807:
|
||||
raise HTTPException(404, 'Zahlung nicht gefunden.')
|
||||
with connect() as db:
|
||||
row = db.execute('SELECT * FROM income_entries WHERE id = ?', (entry_id,)).fetchone()
|
||||
if row is None:
|
||||
raise HTTPException(404, 'Zahlung nicht gefunden.')
|
||||
return dict(row)
|
||||
|
||||
|
||||
def save_entry(data, entry_id=None):
|
||||
if entry_id is not None and not 1 <= entry_id <= 9223372036854775807:
|
||||
raise HTTPException(404, 'Zahlung nicht gefunden.')
|
||||
day = valid_date(data.get('date', ''))
|
||||
amount = cents(data.get('amount', ''))
|
||||
category = data.get('category', '')
|
||||
if category not in CATEGORIES:
|
||||
raise ValueError('Bitte eine gültige Kategorie auswählen.')
|
||||
try:
|
||||
asset_id = int(data.get('asset_id', ''))
|
||||
if not 1 <= asset_id <= 9223372036854775807:
|
||||
raise ValueError
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError('Bitte eine Position auswählen.') from None
|
||||
note = data.get('note', '').strip()
|
||||
if len(note) > 2000:
|
||||
raise ValueError('Notiz darf maximal 2000 Zeichen enthalten.')
|
||||
expected, received = int(data.get('expected') == '1'), int(data.get('received') == '1')
|
||||
with connect() as db:
|
||||
db.execute('BEGIN IMMEDIATE')
|
||||
existing = db.execute('SELECT * FROM income_entries WHERE id = ?', (entry_id,)).fetchone() if entry_id else None
|
||||
if entry_id and existing is None:
|
||||
raise HTTPException(404, 'Zahlung nicht gefunden.')
|
||||
asset = db.execute('SELECT * FROM assets WHERE id = ?', (asset_id,)).fetchone()
|
||||
if asset is None or (not asset['active'] and (existing is None or existing['asset_id'] != asset_id)):
|
||||
raise ValueError('Diese Position ist nicht mehr verfügbar.')
|
||||
values = (day, asset_id, category, amount, note or None, expected, received)
|
||||
if entry_id:
|
||||
db.execute("UPDATE income_entries SET date=?, asset_id=?, category=?, amount=?, note=?, expected=?, received=?, updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id=?", (*values, entry_id))
|
||||
else:
|
||||
entry_id = db.execute('INSERT INTO income_entries (date,asset_id,category,amount,note,expected,received) VALUES (?,?,?,?,?,?,?)', values).lastrowid
|
||||
return entry_id
|
||||
|
||||
|
||||
def list_entries(year=None, month=None, asset_id=None, category=None, limit=None, offset=0):
|
||||
# SQL fragments are constants. All filter values remain bound parameters.
|
||||
clauses, args = [], []
|
||||
for sql, value in [("substr(i.date,1,4) = ?", str(year) if year else None),
|
||||
("substr(i.date,6,2) = ?", f'{month:02}' if month else None),
|
||||
('i.asset_id = ?', asset_id), ('i.category = ?', category)]:
|
||||
if value is not None:
|
||||
clauses.append(sql)
|
||||
args.append(value)
|
||||
query = 'SELECT i.*, a.name FROM income_entries i JOIN assets a ON a.id=i.asset_id'
|
||||
if clauses:
|
||||
query += ' WHERE ' + ' AND '.join(clauses)
|
||||
query += ' ORDER BY i.date DESC, i.id DESC'
|
||||
if limit is not None:
|
||||
query += ' LIMIT ? OFFSET ?'
|
||||
args.extend([limit, offset])
|
||||
with connect() as db:
|
||||
return db.execute(query, args).fetchall()
|
||||
|
||||
|
||||
def available_years():
|
||||
with connect() as db:
|
||||
return [int(row[0]) for row in db.execute('SELECT DISTINCT substr(date,1,4) FROM income_entries ORDER BY 1')]
|
||||
|
||||
|
||||
def dashboard(today=None):
|
||||
today = today or date.today()
|
||||
with connect() as db:
|
||||
grouped = db.execute("SELECT substr(date,1,4) year, substr(date,6,2) month, SUM(amount) amount, COUNT(*) count FROM income_entries WHERE received=1 GROUP BY year, month").fetchall()
|
||||
shares = [dict(r) for r in db.execute('SELECT a.name, SUM(i.amount) amount FROM income_entries i JOIN assets a ON a.id=i.asset_id WHERE received=1 GROUP BY a.id ORDER BY amount DESC')]
|
||||
kinds = {r['category']: r['amount'] for r in db.execute('SELECT category, SUM(amount) amount FROM income_entries WHERE received=1 GROUP BY category')}
|
||||
pending = db.execute('SELECT COUNT(*) count, COALESCE(SUM(amount),0) amount FROM income_entries WHERE expected=1 AND received=0').fetchone()
|
||||
years_found = available_years()
|
||||
years = list(range(min(years_found + [today.year]), max(years_found + [today.year + 1]) + 1))
|
||||
monthly = {year: [0] * 12 for year in years}
|
||||
count = 0
|
||||
for row in grouped:
|
||||
year = int(row['year'])
|
||||
monthly[year][int(row['month']) - 1] = row['amount']
|
||||
if year == today.year:
|
||||
count += row['count']
|
||||
totals = {year: sum(values) for year, values in monthly.items()}
|
||||
current_month = monthly[today.year][today.month - 1]
|
||||
prior_month = monthly.get(today.year - 1, [0] * 12)[today.month - 1]
|
||||
prior_year = totals.get(today.year - 1, 0)
|
||||
return dict(years=years, monthly=monthly, totals=totals, month=current_month, prior_month=prior_month,
|
||||
month_change=percent(current_month, prior_month), year=totals[today.year], prior_year=prior_year,
|
||||
year_change=percent(totals[today.year], prior_year), all_time=sum(totals.values()), count=count,
|
||||
pending=dict(pending), today=today, shares=shares,
|
||||
chart={'months': MONTHS, 'years': [{'label': str(y), 'data': monthly[y]} for y in years],
|
||||
'shares': shares, 'kinds': [{'name': 'Dividenden / Ausschüttungen', 'amount': kinds.get('dividend',0)+kinds.get('distribution',0)},
|
||||
{'name': 'Zinsen', 'amount': kinds.get('interest',0)}, {'name': 'Sonstiges', 'amount': kinds.get('other',0)}]})
|
||||
Reference in New Issue
Block a user