From bbd75400d272c12858c9b0e80e8ceac1c0f0c32f Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah Date: Tue, 16 Jun 2026 02:15:20 -0700 Subject: [PATCH] Add financial intelligence foundation schema --- .ai-context/ARCHITECTURE.md | 7 +- .ai-context/STATUS.md | 4 +- CHANGELOG.md | 3 + docs/financial_intelligence_architecture.md | 82 ++-- src/bankbuddy/financial/__init__.py | 1 + src/bankbuddy/financial/dao.py | 342 +++++++++++++++++ src/bankbuddy/financial/records.py | 156 ++++++++ ...0010_financial_intelligence_foundation.sql | 350 ++++++++++++++++++ tests/test_database.py | 43 +-- tests/test_financial_foundation.py | 155 ++++++++ 10 files changed, 1068 insertions(+), 75 deletions(-) create mode 100644 src/bankbuddy/financial/__init__.py create mode 100644 src/bankbuddy/financial/dao.py create mode 100644 src/bankbuddy/financial/records.py create mode 100644 src/bankbuddy/migrations/0010_financial_intelligence_foundation.sql create mode 100644 tests/test_financial_foundation.py diff --git a/.ai-context/ARCHITECTURE.md b/.ai-context/ARCHITECTURE.md index 5c342b2..39b2dd3 100644 --- a/.ai-context/ARCHITECTURE.md +++ b/.ai-context/ARCHITECTURE.md @@ -87,6 +87,7 @@ and `Relationship` foundations. New schema proposals use `BB_`-prefixed singular table names and prefer normalized relational tables over JSON blobs. TaxBuddy issue #100 is paused until the v2 document/entity/observation model -exists. Early v2 work should focus on a schema reset path, generic document -storage/import, provenance, and inspect/report foundations before adding a -user-facing infer workflow. +exists. Early v2 work should add `BB_` foundation tables beside existing legacy +tables, keep new SQL centralized behind DAO modules, and focus on generic +document storage/import, provenance, and inspect/report foundations before +adding a user-facing infer workflow. diff --git a/.ai-context/STATUS.md b/.ai-context/STATUS.md index 3a4417d..bc6992f 100644 --- a/.ai-context/STATUS.md +++ b/.ai-context/STATUS.md @@ -40,8 +40,8 @@ section is `Unreleased`. TaxBuddy expected-form gap detection and annual readiness summaries are paused until the financial intelligence v2 document/entity/observation foundation -exists. The next architectural implementation area should be issue-backed v2 -foundation work, not issue #100 as originally scoped. +exists. The active architectural implementation area is additive v2 foundation +work with DAO-based database access, not issue #100 as originally scoped. ## Validation Snapshot diff --git a/CHANGELOG.md b/CHANGELOG.md index ea02a8e..0b6e270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,9 @@ and versions are tracked in the repo-root `VERSION` file. ### Added +- Added additive financial intelligence v2 foundation tables, seed type + dictionaries, indexes, and DAO-based read/write helpers without dropping + existing banking or tax tables. - Added the reviewed financial intelligence architecture documents that define BankBuddy's v2 direction around documents, entities, observations, relationships, managed document storage, and native-currency reporting. diff --git a/docs/financial_intelligence_architecture.md b/docs/financial_intelligence_architecture.md index 45f4768..06bc8e4 100644 --- a/docs/financial_intelligence_architecture.md +++ b/docs/financial_intelligence_architecture.md @@ -25,10 +25,10 @@ be retained: local SQLite, canonical storage, dry-run imports, parser staging, file hashing, import attempt history, integer money storage, Base-style CLI runtime, environment-specific data homes, and a growing parser test suite. -The current architecture should not constrain the next design. Because existing -data can be discarded, the cleanest path is a deliberate schema reset into a -versioned v2 model, not a long compatibility migration that preserves every old -bank/account assumption. +The current architecture should not constrain the next design. The v2 model +should be introduced as an additive foundation beside the existing banking and +tax tables, then adopted workflow by workflow. Existing legacy tables do not +need to be discarded just to make room for the new architecture. ## Architecture Decisions @@ -37,7 +37,8 @@ The following decisions define the current v2 direction: - Keep the product and CLI name as `bankbuddy`. - Use `Document`, `Entity`, `Observation`, and `Relationship` as the conceptual foundation, while allowing details to evolve during implementation. -- Use a v2 schema reset rather than compatibility migrations. +- Introduce the v2 foundation schema additively beside the existing legacy + tables. Do not silently discard existing tables or data. - Move imported documents into managed storage when an import activity finishes, whether the import succeeds or fails. - Treat future `infer` work as read-only with respect to source documents unless @@ -61,6 +62,8 @@ The schema should also follow these conventions: filters as part of each migration. - Attribute values should be typed through `BB_ENTITY_ATTRIBUTE_TYPE` rather than stored as arbitrary key names. +- New v2 read/write paths should use Data Access Objects so SQL stays + centralized instead of being buried throughout command and service code. ## 1. Current-State Assessment @@ -151,10 +154,10 @@ schema. persistence, archive planning, and repair-sensitive legacy behavior. - TaxBuddy is currently a separate package slice, but the new platform wants tax documents to be one document family inside the same evidence graph. -- The database migration chain assumes additive evolution. The new direction is - a semantic schema reset. -- There is no repository/service layer boundary. Many modules open SQLite - connections directly and issue SQL inline. +- The database migration chain supports additive evolution, which is suitable + for introducing v2 foundation tables beside legacy tables. +- There is no DAO/service layer boundary. Many modules open SQLite connections + directly and issue SQL inline. ## 3. Proposed Target Architecture @@ -432,10 +435,6 @@ BB_IMPORT_ATTEMPT( import_status, started_at, finished_at, - parser_id, - rows_parsed, - rows_imported, - rows_duplicate, error_message, created_at, updated_at @@ -444,11 +443,10 @@ BB_IMPORT_ATTEMPT( BB_PARSER( parser_id, parser_name, - parser_family, - document_type, file_type, - institution_id, - enabled, + document_family, + default_document_type, + active, created_at, updated_at ) @@ -457,9 +455,11 @@ BB_EXTRACTION_RUN( extraction_run_id, document_id, parser_id, - extraction_method, - confidence, - status, + extraction_status, + raw_text_stored, + started_at, + finished_at, + error_message, created_at, updated_at ) @@ -1051,31 +1051,21 @@ The highest-value family UI is likely: ### Recommendation -Use a deliberate v2 schema reset. - -Because backward compatibility is not required and existing data can be -discarded, avoid a fragile migration that tries to preserve old semantics. -However, do not silently destroy existing data homes. The migration path should -be explicit. - -Possible command: - -```text -bankbuddy storage reset-schema --to financial-intelligence-v2 --dry-run -bankbuddy storage reset-schema --to financial-intelligence-v2 --apply -``` +Introduce the v2 foundation schema additively. -The dry run should report: +Backward-compatible behavior is not the primary design constraint, but existing +tables do not need to be dropped just to begin the v2 architecture. The safer +path is to add `BB_` foundation tables beside the current schema, then migrate +or replace workflows deliberately as the v2 document/entity/observation model +becomes usable. -- current schema version; -- tables that will be replaced; -- files left untouched; -- database backup/export recommendation; -- new schema version. +The first v2 database slice should: -Because compatibility is explicitly not required, this reset can discard -transaction/account rows. The command must still avoid accidental data loss by -requiring an explicit `--apply` mode and clear user-facing output. +- leave existing banking and tax tables intact; +- create `BB_` foundation tables through an ordinary ordered migration; +- seed minimal type dictionaries; +- add indexes with the migration; +- introduce DAO-based read/write helpers for new v2 code. ### Retain @@ -1083,7 +1073,7 @@ requiring an explicit `--apply` mode and clear user-facing output. - `pyproject.toml` / `uv` workflow. - Base-style runtime. - `paths.py` environment semantics. -- SQLite migration mechanism, possibly moved under `db/`. +- SQLite migration mechanism, with v2 SQL kept in ordered migrations. - Current parser test fixtures and parser-specific knowledge. - Currency minor-unit discipline. - CLI rendering style. @@ -1138,7 +1128,7 @@ Build: Validation: - migration tests; -- repository tests; +- DAO tests; - document import dry-run and real import tests using synthetic fixtures. - index presence tests for key report and join paths. @@ -1239,7 +1229,7 @@ Build: |---|---|---|---| | Runtime | Base-style CLI setup, logging, env selection | Nothing material | Move to `core/` later. | | Paths | Environment homes, canonical layout | Bank/tax-only assumption as final shape | Add generic `documents/` roots. | -| Database | Migration runner, SQLite connection discipline | Old schema as canonical | Introduce explicit v2 reset/migrations. | +| Database | Migration runner, SQLite connection discipline | Old schema as canonical | Introduce additive `BB_` v2 migrations and DAO boundaries. | | Imports | Parsers, dry-run, hash, staging, duplicate logic | Parser-to-account direct coupling | Split into document service, parser service, inference service. | | Accounts | Account setup, account refs, masking | Bank as root concept | Generalize bank to institution and account to typed entity. | | Transactions | Minor units, categories, filters, reports | Account-only worldview | Reference documents and v2 accounts. | @@ -1289,7 +1279,7 @@ Resolved decisions: 1. Keep the user-facing product and CLI name as `BankBuddy` / `bankbuddy`. 2. Use `Document/Entity/Observation/Relationship` as the foundation, with details refined during implementation. -3. Use a schema reset path instead of compatibility migrations. +3. Introduce the v2 foundation schema additively beside existing legacy tables. 4. Move documents into managed storage after completed import attempts, including failures. 5. Pause issue #100 until the v2 document/entity model exists. diff --git a/src/bankbuddy/financial/__init__.py b/src/bankbuddy/financial/__init__.py new file mode 100644 index 0000000..6a9c4cb --- /dev/null +++ b/src/bankbuddy/financial/__init__.py @@ -0,0 +1 @@ +"""Financial intelligence v2 foundation helpers.""" diff --git a/src/bankbuddy/financial/dao.py b/src/bankbuddy/financial/dao.py new file mode 100644 index 0000000..c3a762f --- /dev/null +++ b/src/bankbuddy/financial/dao.py @@ -0,0 +1,342 @@ +"""Data access objects for financial intelligence v2 tables.""" + +from __future__ import annotations + +import sqlite3 + +from bankbuddy.financial.records import ( + DocumentCreate, + DocumentRecord, + EntityAttributeCreate, + EntityAttributeRecord, + EntityAttributeTypeRecord, + EntityCreate, + EntityRecord, + ObservationCreate, + ObservationRecord, + ObservationEvidenceCreate, + ObservationEvidenceRecord, + ObservationTypeRecord, + RelationshipTypeRecord, +) + + +class FinancialReferenceTypeNotFoundError(ValueError): + """Raised when a requested v2 reference/type code is not seeded.""" + + +class FinancialIntelligenceDAO: + """Persistence boundary for v2 financial intelligence tables.""" + + def __init__(self, conn: sqlite3.Connection) -> None: + self._conn = conn + + def list_entity_attribute_types(self) -> list[EntityAttributeTypeRecord]: + """Return seeded entity attribute types ordered by code.""" + + rows = self._conn.execute( + """ + select + entity_attribute_type_id, + attribute_code, + display_name, + value_kind, + sensitivity, + is_system + from BB_ENTITY_ATTRIBUTE_TYPE + order by attribute_code + """ + ).fetchall() + return [ + EntityAttributeTypeRecord( + entity_attribute_type_id=int(row["entity_attribute_type_id"]), + attribute_code=str(row["attribute_code"]), + display_name=str(row["display_name"]), + value_kind=str(row["value_kind"]), + sensitivity=str(row["sensitivity"]), + is_system=bool(row["is_system"]), + ) + for row in rows + ] + + def list_relationship_types(self) -> list[RelationshipTypeRecord]: + """Return seeded relationship types ordered by code.""" + + rows = self._conn.execute( + """ + select + relationship_type_id, + relationship_type_code, + display_name, + is_system + from BB_RELATIONSHIP_TYPE + order by relationship_type_code + """ + ).fetchall() + return [ + RelationshipTypeRecord( + relationship_type_id=int(row["relationship_type_id"]), + relationship_type_code=str(row["relationship_type_code"]), + display_name=str(row["display_name"]), + is_system=bool(row["is_system"]), + ) + for row in rows + ] + + def list_observation_types(self) -> list[ObservationTypeRecord]: + """Return seeded observation types ordered by code.""" + + rows = self._conn.execute( + """ + select + observation_type_id, + observation_type_code, + display_name, + value_kind, + is_system + from BB_OBSERVATION_TYPE + order by observation_type_code + """ + ).fetchall() + return [ + ObservationTypeRecord( + observation_type_id=int(row["observation_type_id"]), + observation_type_code=str(row["observation_type_code"]), + display_name=str(row["display_name"]), + value_kind=str(row["value_kind"]), + is_system=bool(row["is_system"]), + ) + for row in rows + ] + + def create_document(self, record: DocumentCreate) -> DocumentRecord: + """Create a v2 document row.""" + + cursor = self._conn.execute( + """ + insert into BB_DOCUMENT ( + file_hash, + original_file_name, + canonical_file_name, + storage_path, + source_uri, + document_type, + jurisdiction_code, + tax_year, + document_status + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + record.file_hash, + record.original_file_name, + record.canonical_file_name, + record.storage_path, + record.source_uri, + record.document_type, + record.jurisdiction_code, + record.tax_year, + record.document_status, + ), + ) + return DocumentRecord(document_id=int(cursor.lastrowid), **record.__dict__) + + def find_document_by_hash(self, file_hash: str) -> DocumentRecord | None: + """Return a v2 document by file hash.""" + + row = self._conn.execute( + """ + select + document_id, + file_hash, + original_file_name, + canonical_file_name, + storage_path, + source_uri, + document_type, + jurisdiction_code, + tax_year, + document_status + from BB_DOCUMENT + where file_hash = ? + """, + (file_hash,), + ).fetchone() + if row is None: + return None + return _document_from_row(row) + + def create_entity(self, record: EntityCreate) -> EntityRecord: + """Create a v2 entity row.""" + + cursor = self._conn.execute( + """ + insert into BB_ENTITY ( + entity_type, + display_name, + status + ) values (?, ?, ?) + """, + (record.entity_type, record.display_name, record.status), + ) + return EntityRecord(entity_id=int(cursor.lastrowid), **record.__dict__) + + def add_entity_attribute( + self, + record: EntityAttributeCreate, + ) -> EntityAttributeRecord: + """Create a typed attribute for a v2 entity.""" + + attribute_type_id = self._type_id_for_code( + table_name="BB_ENTITY_ATTRIBUTE_TYPE", + id_column="entity_attribute_type_id", + code_column="attribute_code", + code=record.attribute_type_code, + ) + cursor = self._conn.execute( + """ + insert into BB_ENTITY_ATTRIBUTE ( + entity_id, + entity_attribute_type_id, + value_text, + value_integer, + value_decimal, + value_date, + value_boolean, + source_document_id, + valid_from, + valid_to + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + record.entity_id, + attribute_type_id, + record.value_text, + record.value_integer, + record.value_decimal, + record.value_date, + _bool_to_int(record.value_boolean), + record.source_document_id, + record.valid_from, + record.valid_to, + ), + ) + return EntityAttributeRecord( + entity_attribute_id=int(cursor.lastrowid), + entity_attribute_type_id=attribute_type_id, + **record.__dict__, + ) + + def create_observation(self, record: ObservationCreate) -> ObservationRecord: + """Create a typed observation.""" + + observation_type_id = self._type_id_for_code( + table_name="BB_OBSERVATION_TYPE", + id_column="observation_type_id", + code_column="observation_type_code", + code=record.observation_type_code, + ) + cursor = self._conn.execute( + """ + insert into BB_OBSERVATION ( + observation_type_id, + document_id, + subject_entity_id, + value_text, + value_integer, + value_decimal, + value_date, + value_boolean, + confidence, + review_status, + observed_at + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + observation_type_id, + record.document_id, + record.subject_entity_id, + record.value_text, + record.value_integer, + record.value_decimal, + record.value_date, + _bool_to_int(record.value_boolean), + record.confidence, + record.review_status, + record.observed_at, + ), + ) + return ObservationRecord( + observation_id=int(cursor.lastrowid), + observation_type_id=observation_type_id, + **record.__dict__, + ) + + def add_observation_evidence( + self, + record: ObservationEvidenceCreate, + ) -> ObservationEvidenceRecord: + """Create evidence for an observation.""" + + cursor = self._conn.execute( + """ + insert into BB_OBSERVATION_EVIDENCE ( + observation_id, + document_id, + extraction_run_id, + evidence_text, + page_number, + location_text + ) values (?, ?, ?, ?, ?, ?) + """, + ( + record.observation_id, + record.document_id, + record.extraction_run_id, + record.evidence_text, + record.page_number, + record.location_text, + ), + ) + return ObservationEvidenceRecord( + observation_evidence_id=int(cursor.lastrowid), + **record.__dict__, + ) + + def _type_id_for_code( + self, + *, + table_name: str, + id_column: str, + code_column: str, + code: str, + ) -> int: + row = self._conn.execute( + f"select {id_column} from {table_name} where {code_column} = ?", + (code,), + ).fetchone() + if row is None: + raise FinancialReferenceTypeNotFoundError( + f"Unknown financial reference type: {code}" + ) + return int(row[id_column]) + + +def _document_from_row(row: sqlite3.Row) -> DocumentRecord: + return DocumentRecord( + document_id=int(row["document_id"]), + file_hash=str(row["file_hash"]), + original_file_name=str(row["original_file_name"]), + canonical_file_name=row["canonical_file_name"], + storage_path=row["storage_path"], + source_uri=row["source_uri"], + document_type=row["document_type"], + jurisdiction_code=row["jurisdiction_code"], + tax_year=row["tax_year"], + document_status=str(row["document_status"]), + ) + + +def _bool_to_int(value: bool | None) -> int | None: + if value is None: + return None + return int(value) diff --git a/src/bankbuddy/financial/records.py b/src/bankbuddy/financial/records.py new file mode 100644 index 0000000..9b330fa --- /dev/null +++ b/src/bankbuddy/financial/records.py @@ -0,0 +1,156 @@ +"""Typed records used by financial intelligence DAOs.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ReferenceTypeRecord: + """A seeded reference/type row.""" + + type_id: int + code: str + display_name: str + value_kind: str | None = None + sensitivity: str | None = None + is_system: bool = True + + +@dataclass(frozen=True) +class EntityAttributeTypeRecord: + """A typed entity attribute definition.""" + + entity_attribute_type_id: int + attribute_code: str + display_name: str + value_kind: str + sensitivity: str + is_system: bool + + +@dataclass(frozen=True) +class RelationshipTypeRecord: + """A typed relationship definition.""" + + relationship_type_id: int + relationship_type_code: str + display_name: str + is_system: bool + + +@dataclass(frozen=True) +class ObservationTypeRecord: + """A typed observation definition.""" + + observation_type_id: int + observation_type_code: str + display_name: str + value_kind: str + is_system: bool + + +@dataclass(frozen=True) +class DocumentCreate: + """Input data for creating a v2 document record.""" + + file_hash: str + original_file_name: str + canonical_file_name: str | None = None + storage_path: str | None = None + source_uri: str | None = None + document_type: str | None = None + jurisdiction_code: str | None = None + tax_year: int | None = None + document_status: str = "active" + + +@dataclass(frozen=True) +class DocumentRecord(DocumentCreate): + """Stored v2 document record.""" + + document_id: int = 0 + + +@dataclass(frozen=True) +class EntityCreate: + """Input data for creating a v2 entity record.""" + + entity_type: str + display_name: str | None = None + status: str = "active" + + +@dataclass(frozen=True) +class EntityRecord(EntityCreate): + """Stored v2 entity record.""" + + entity_id: int = 0 + + +@dataclass(frozen=True) +class EntityAttributeCreate: + """Input data for creating a typed entity attribute.""" + + entity_id: int + attribute_type_code: str + value_text: str | None = None + value_integer: int | None = None + value_decimal: str | None = None + value_date: str | None = None + value_boolean: bool | None = None + source_document_id: int | None = None + valid_from: str | None = None + valid_to: str | None = None + + +@dataclass(frozen=True) +class EntityAttributeRecord(EntityAttributeCreate): + """Stored typed entity attribute.""" + + entity_attribute_id: int = 0 + entity_attribute_type_id: int = 0 + + +@dataclass(frozen=True) +class ObservationCreate: + """Input data for creating a typed observation.""" + + observation_type_code: str + document_id: int | None = None + subject_entity_id: int | None = None + value_text: str | None = None + value_integer: int | None = None + value_decimal: str | None = None + value_date: str | None = None + value_boolean: bool | None = None + confidence: float = 1.0 + review_status: str = "needs_review" + observed_at: str | None = None + + +@dataclass(frozen=True) +class ObservationRecord(ObservationCreate): + """Stored typed observation.""" + + observation_id: int = 0 + observation_type_id: int = 0 + + +@dataclass(frozen=True) +class ObservationEvidenceCreate: + """Input data for creating observation evidence.""" + + observation_id: int + document_id: int + extraction_run_id: int | None = None + evidence_text: str | None = None + page_number: int | None = None + location_text: str | None = None + + +@dataclass(frozen=True) +class ObservationEvidenceRecord(ObservationEvidenceCreate): + """Stored observation evidence.""" + + observation_evidence_id: int = 0 diff --git a/src/bankbuddy/migrations/0010_financial_intelligence_foundation.sql b/src/bankbuddy/migrations/0010_financial_intelligence_foundation.sql new file mode 100644 index 0000000..b4ad514 --- /dev/null +++ b/src/bankbuddy/migrations/0010_financial_intelligence_foundation.sql @@ -0,0 +1,350 @@ +create table BB_CURRENCY ( + currency_code text primary key, + display_name text not null, + minor_unit_exponent integer not null check (minor_unit_exponent >= 0), + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +insert into BB_CURRENCY ( + currency_code, + display_name, + minor_unit_exponent +) values + ('INR', 'Indian Rupee', 2), + ('USD', 'US Dollar', 2); + +create table BB_JURISDICTION ( + jurisdiction_code text primary key, + display_name text not null, + country_code text not null, + region_code text, + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +insert into BB_JURISDICTION ( + jurisdiction_code, + display_name, + country_code, + region_code +) values + ('IN', 'India', 'IN', null), + ('US', 'United States', 'US', null); + +create table BB_DOCUMENT ( + document_id integer primary key, + file_hash text not null unique, + original_file_name text not null, + canonical_file_name text, + storage_path text, + source_uri text, + document_type text, + jurisdiction_code text references BB_JURISDICTION(jurisdiction_code), + tax_year integer, + document_status text not null default 'active' check ( + document_status in ('active', 'duplicate', 'failed', 'archived') + ), + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +create table BB_IMPORT_ATTEMPT ( + import_attempt_id integer primary key, + document_id integer references BB_DOCUMENT(document_id), + source_path text not null, + import_status text not null check ( + import_status in ( + 'planned', + 'success', + 'failed', + 'duplicate', + 'review_needed' + ) + ), + started_at text not null default current_timestamp, + finished_at text, + error_message text, + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +create table BB_PARSER ( + parser_id integer primary key, + parser_name text not null unique, + file_type text not null, + document_family text not null, + default_document_type text, + active integer not null default 1 check (active in (0, 1)), + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +insert into BB_PARSER ( + parser_name, + file_type, + document_family, + default_document_type, + active +) values + ('apple_card_pdf', 'pdf', 'bank', 'credit_card_statement', 1), + ('boa_csv', 'csv', 'bank', 'bank_statement', 1), + ('boa_pdf', 'pdf', 'bank', 'bank_statement', 1), + ('hdfc_xls', 'xls', 'bank', 'bank_statement', 1), + ('icici_xls', 'xls', 'bank', 'bank_statement', 1), + ('tax_pdf', 'pdf', 'tax', 'tax_document', 1); + +create table BB_EXTRACTION_RUN ( + extraction_run_id integer primary key, + document_id integer not null references BB_DOCUMENT(document_id), + parser_id integer not null references BB_PARSER(parser_id), + extraction_status text not null check ( + extraction_status in ('success', 'failed', 'partial') + ), + raw_text_stored integer not null default 0 check (raw_text_stored in (0, 1)), + started_at text not null default current_timestamp, + finished_at text, + error_message text, + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +create table BB_ENTITY ( + entity_id integer primary key, + entity_type text not null, + display_name text, + status text not null default 'active' check ( + status in ('active', 'inactive', 'candidate', 'merged', 'deleted') + ), + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +create table BB_ENTITY_ATTRIBUTE_TYPE ( + entity_attribute_type_id integer primary key, + attribute_code text not null unique, + display_name text not null, + value_kind text not null check ( + value_kind in ('text', 'integer', 'decimal', 'date', 'boolean') + ), + sensitivity text not null check ( + sensitivity in ('public', 'private', 'sensitive', 'secret') + ), + is_system integer not null default 1 check (is_system in (0, 1)), + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +insert into BB_ENTITY_ATTRIBUTE_TYPE ( + attribute_code, + display_name, + value_kind, + sensitivity, + is_system +) values + ('FULL_NAME', 'Full name', 'text', 'sensitive', 1), + ('DISPLAY_NAME', 'Display name', 'text', 'private', 1), + ('ACCOUNT_NUMBER', 'Account number', 'text', 'secret', 1), + ('ACCOUNT_LAST4', 'Account last four digits', 'text', 'sensitive', 1), + ('STATEMENT_ACCOUNT_REF', 'Statement account reference', 'text', 'sensitive', 1), + ('ADDRESS_LINE_1', 'Address line 1', 'text', 'sensitive', 1), + ('ADDRESS_LINE_2', 'Address line 2', 'text', 'sensitive', 1), + ('ADDRESS_LINE_3', 'Address line 3', 'text', 'sensitive', 1), + ('ADDRESS_CITY', 'Address city', 'text', 'sensitive', 1), + ('ADDRESS_STATE', 'Address state or province', 'text', 'sensitive', 1), + ('ADDRESS_POSTAL_CODE', 'Address postal code', 'text', 'sensitive', 1), + ('ADDRESS_COUNTRY', 'Address country', 'text', 'sensitive', 1), + ('JURISDICTION', 'Jurisdiction', 'text', 'private', 1), + ('CURRENCY', 'Currency', 'text', 'private', 1); + +create table BB_ENTITY_ATTRIBUTE ( + entity_attribute_id integer primary key, + entity_id integer not null references BB_ENTITY(entity_id), + entity_attribute_type_id integer not null + references BB_ENTITY_ATTRIBUTE_TYPE(entity_attribute_type_id), + value_text text, + value_integer integer, + value_decimal text, + value_date text, + value_boolean integer check (value_boolean in (0, 1)), + source_document_id integer references BB_DOCUMENT(document_id), + valid_from text, + valid_to text, + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +create table BB_RELATIONSHIP_TYPE ( + relationship_type_id integer primary key, + relationship_type_code text not null unique, + display_name text not null, + is_system integer not null default 1 check (is_system in (0, 1)), + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +insert into BB_RELATIONSHIP_TYPE ( + relationship_type_code, + display_name, + is_system +) values + ('OWNS', 'Owns', 1), + ('MEMBER_OF_HOUSEHOLD', 'Member of household', 1), + ('HELD_AT_INSTITUTION', 'Held at institution', 1), + ('ISSUED_BY', 'Issued by', 1), + ('EVIDENCES', 'Evidences', 1); + +create table BB_RELATIONSHIP ( + relationship_id integer primary key, + relationship_type_id integer not null + references BB_RELATIONSHIP_TYPE(relationship_type_id), + source_entity_id integer not null references BB_ENTITY(entity_id), + target_entity_id integer not null references BB_ENTITY(entity_id), + source_document_id integer references BB_DOCUMENT(document_id), + valid_from text, + valid_to text, + confidence real not null default 1.0 check ( + confidence >= 0.0 and confidence <= 1.0 + ), + review_status text not null default 'accepted' check ( + review_status in ('accepted', 'needs_review', 'rejected', 'superseded') + ), + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +create table BB_OBSERVATION_TYPE ( + observation_type_id integer primary key, + observation_type_code text not null unique, + display_name text not null, + value_kind text not null check ( + value_kind in ('text', 'integer', 'decimal', 'date', 'boolean') + ), + is_system integer not null default 1 check (is_system in (0, 1)), + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +insert into BB_OBSERVATION_TYPE ( + observation_type_code, + display_name, + value_kind, + is_system +) values + ('DOCUMENT_TYPE', 'Document type', 'text', 1), + ('STATEMENT_PERIOD', 'Statement period', 'text', 1), + ('ACCOUNT_BALANCE', 'Account balance', 'decimal', 1), + ('TRANSACTION', 'Transaction', 'text', 1), + ('ENTITY_IDENTIFIER', 'Entity identifier', 'text', 1), + ('TAX_YEAR', 'Tax year', 'integer', 1), + ('JURISDICTION', 'Jurisdiction', 'text', 1); + +create table BB_OBSERVATION ( + observation_id integer primary key, + observation_type_id integer not null + references BB_OBSERVATION_TYPE(observation_type_id), + document_id integer references BB_DOCUMENT(document_id), + subject_entity_id integer references BB_ENTITY(entity_id), + value_text text, + value_integer integer, + value_decimal text, + value_date text, + value_boolean integer check (value_boolean in (0, 1)), + confidence real not null default 1.0 check ( + confidence >= 0.0 and confidence <= 1.0 + ), + review_status text not null default 'needs_review' check ( + review_status in ('accepted', 'needs_review', 'rejected', 'superseded') + ), + observed_at text, + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +create table BB_OBSERVATION_EVIDENCE ( + observation_evidence_id integer primary key, + observation_id integer not null references BB_OBSERVATION(observation_id), + document_id integer not null references BB_DOCUMENT(document_id), + extraction_run_id integer references BB_EXTRACTION_RUN(extraction_run_id), + evidence_text text, + page_number integer, + location_text text, + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +create table BB_PERSON ( + person_id integer primary key, + entity_id integer not null unique references BB_ENTITY(entity_id), + preferred_name text, + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +create table BB_HOUSEHOLD ( + household_id integer primary key, + entity_id integer not null unique references BB_ENTITY(entity_id), + household_name text not null, + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp +); + +create table BB_HOUSEHOLD_MEMBER ( + household_member_id integer primary key, + household_id integer not null references BB_HOUSEHOLD(household_id), + person_id integer not null references BB_PERSON(person_id), + relationship_label text, + valid_from text, + valid_to text, + created_at text not null default current_timestamp, + updated_at text not null default current_timestamp, + unique (household_id, person_id, valid_from) +); + +create index idx_BB_DOCUMENT_file_hash +on BB_DOCUMENT(file_hash); + +create index idx_BB_DOCUMENT_type_year +on BB_DOCUMENT(document_type, tax_year); + +create index idx_BB_IMPORT_ATTEMPT_document_status +on BB_IMPORT_ATTEMPT(document_id, import_status); + +create index idx_BB_PARSER_file_type +on BB_PARSER(file_type, active); + +create index idx_BB_EXTRACTION_RUN_document_parser +on BB_EXTRACTION_RUN(document_id, parser_id); + +create index idx_BB_ENTITY_type_status +on BB_ENTITY(entity_type, status); + +create index idx_BB_ENTITY_ATTRIBUTE_entity_type +on BB_ENTITY_ATTRIBUTE(entity_id, entity_attribute_type_id); + +create index idx_BB_ENTITY_ATTRIBUTE_source_document +on BB_ENTITY_ATTRIBUTE(source_document_id); + +create index idx_BB_RELATIONSHIP_source_type +on BB_RELATIONSHIP(source_entity_id, relationship_type_id); + +create index idx_BB_RELATIONSHIP_target_type +on BB_RELATIONSHIP(target_entity_id, relationship_type_id); + +create index idx_BB_OBSERVATION_subject_type +on BB_OBSERVATION(subject_entity_id, observation_type_id); + +create index idx_BB_OBSERVATION_document +on BB_OBSERVATION(document_id); + +create index idx_BB_OBSERVATION_review_status +on BB_OBSERVATION(review_status); + +create index idx_BB_OBSERVATION_EVIDENCE_observation +on BB_OBSERVATION_EVIDENCE(observation_id); + +create index idx_BB_HOUSEHOLD_MEMBER_household +on BB_HOUSEHOLD_MEMBER(household_id); + +create index idx_BB_HOUSEHOLD_MEMBER_person +on BB_HOUSEHOLD_MEMBER(person_id); diff --git a/tests/test_database.py b/tests/test_database.py index 52089d4..e3b602d 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -13,6 +13,20 @@ from bankbuddy.paths import resolve_app_paths +EXPECTED_MIGRATION_VERSIONS = [ + "0001_core_schema", + "0002_import_file_metadata", + "0003_import_attempt_account", + "0004_duplicate_import_attempts", + "0005_account_balances_and_value_dates", + "0006_normalize_bank_country_codes", + "0007_add_rental_income_category", + "0008_account_statement_refs", + "0009_tax_documents", + "0010_financial_intelligence_foundation", +] + + def test_initialize_database_creates_directories_and_schema_table(tmp_path) -> None: paths = resolve_app_paths(tmp_path) @@ -69,17 +83,7 @@ def test_initialize_database_applies_core_schema_and_seed_categories(tmp_path) - "account_statement_refs", "tax_documents", }.issubset(table_names) - assert migration_versions == [ - "0001_core_schema", - "0002_import_file_metadata", - "0003_import_attempt_account", - "0004_duplicate_import_attempts", - "0005_account_balances_and_value_dates", - "0006_normalize_bank_country_codes", - "0007_add_rental_income_category", - "0008_account_statement_refs", - "0009_tax_documents", - ] + assert migration_versions == EXPECTED_MIGRATION_VERSIONS assert categories == { "Dining": "expense", "Dividends": "income", @@ -112,7 +116,7 @@ def test_initialize_database_is_idempotent(tmp_path) -> None: ).fetchone()[0] category_count = conn.execute("select count(*) from categories").fetchone()[0] - assert migration_count == 9 + assert migration_count == len(EXPECTED_MIGRATION_VERSIONS) assert category_count == 16 @@ -221,17 +225,7 @@ def test_schema_tracks_value_date_and_latest_account_balance(tmp_path) -> None: assert account_columns["latest_balance_currency"] == "TEXT" assert account_columns["latest_balance_as_of_date"] == "TEXT" assert account_columns["latest_balance_source_file_id"] == "INTEGER" - assert migration_versions == [ - "0001_core_schema", - "0002_import_file_metadata", - "0003_import_attempt_account", - "0004_duplicate_import_attempts", - "0005_account_balances_and_value_dates", - "0006_normalize_bank_country_codes", - "0007_add_rental_income_category", - "0008_account_statement_refs", - "0009_tax_documents", - ] + assert migration_versions == EXPECTED_MIGRATION_VERSIONS def test_schema_tracks_account_statement_refs(tmp_path) -> None: @@ -331,7 +325,8 @@ def test_tax_documents_schema_tracks_imported_document_metadata(tmp_path) -> Non "account_ref": "TEXT", "imported_at": "TEXT", }.items() <= columns.items() - assert migration_versions[-1] == "0009_tax_documents" + assert "0009_tax_documents" in migration_versions + assert migration_versions[-1] == "0010_financial_intelligence_foundation" assert dict(row) == { "document_type": "1099-INT", "jurisdiction": "US", diff --git a/tests/test_financial_foundation.py b/tests/test_financial_foundation.py new file mode 100644 index 0000000..15ddb7f --- /dev/null +++ b/tests/test_financial_foundation.py @@ -0,0 +1,155 @@ +from bankbuddy.database import connect_database, initialize_database +from bankbuddy.financial.dao import FinancialIntelligenceDAO +from bankbuddy.financial.records import ( + DocumentCreate, + EntityAttributeCreate, + EntityCreate, + ObservationCreate, + ObservationEvidenceCreate, +) +from bankbuddy.paths import resolve_app_paths + + +def test_financial_foundation_schema_is_additive_and_seeded(tmp_path) -> None: + paths = resolve_app_paths(tmp_path) + + initialize_database(paths) + + with connect_database(paths) as conn: + table_names = { + row["name"] + for row in conn.execute( + "select name from sqlite_master where type = 'table'" + ).fetchall() + } + index_names = { + row["name"] + for row in conn.execute( + "select name from sqlite_master where type = 'index'" + ).fetchall() + } + dao = FinancialIntelligenceDAO(conn) + + entity_attribute_codes = { + row.attribute_code for row in dao.list_entity_attribute_types() + } + relationship_type_codes = { + row.relationship_type_code for row in dao.list_relationship_types() + } + observation_type_codes = { + row.observation_type_code for row in dao.list_observation_types() + } + + assert {"banks", "accounts", "transactions", "tax_documents"}.issubset( + table_names + ) + assert { + "BB_DOCUMENT", + "BB_IMPORT_ATTEMPT", + "BB_PARSER", + "BB_EXTRACTION_RUN", + "BB_ENTITY", + "BB_ENTITY_ATTRIBUTE_TYPE", + "BB_ENTITY_ATTRIBUTE", + "BB_RELATIONSHIP_TYPE", + "BB_RELATIONSHIP", + "BB_OBSERVATION_TYPE", + "BB_OBSERVATION", + "BB_OBSERVATION_EVIDENCE", + "BB_PERSON", + "BB_HOUSEHOLD", + "BB_HOUSEHOLD_MEMBER", + "BB_CURRENCY", + "BB_JURISDICTION", + }.issubset(table_names) + assert { + "idx_BB_DOCUMENT_file_hash", + "idx_BB_PARSER_file_type", + "idx_BB_ENTITY_type_status", + "idx_BB_ENTITY_ATTRIBUTE_entity_type", + "idx_BB_OBSERVATION_subject_type", + "idx_BB_RELATIONSHIP_source_type", + }.issubset(index_names) + assert { + "ACCOUNT_NUMBER", + "ACCOUNT_LAST4", + "ADDRESS_LINE_1", + "ADDRESS_CITY", + "ADDRESS_COUNTRY", + "JURISDICTION", + }.issubset(entity_attribute_codes) + assert {"OWNS", "MEMBER_OF_HOUSEHOLD", "HELD_AT_INSTITUTION"}.issubset( + relationship_type_codes + ) + assert { + "DOCUMENT_TYPE", + "STATEMENT_PERIOD", + "ACCOUNT_BALANCE", + "TAX_YEAR", + }.issubset(observation_type_codes) + + +def test_financial_dao_round_trips_records_without_inline_sql_callers( + tmp_path, +) -> None: + paths = resolve_app_paths(tmp_path) + initialize_database(paths) + + with connect_database(paths) as conn: + dao = FinancialIntelligenceDAO(conn) + document = dao.create_document( + DocumentCreate( + file_hash="a" * 64, + original_file_name="statement.pdf", + canonical_file_name="bank_1234_2026-01-01_2026-01-31.pdf", + storage_path=( + "bank/processed/bank/2026/01/" + "bank_1234_2026-01-01_2026-01-31.pdf" + ), + document_type="bank_statement", + jurisdiction_code="US", + tax_year=2026, + ) + ) + fetched_document = dao.find_document_by_hash("a" * 64) + entity = dao.create_entity( + EntityCreate(entity_type="account", display_name="Everyday Checking") + ) + attribute = dao.add_entity_attribute( + EntityAttributeCreate( + entity_id=entity.entity_id, + attribute_type_code="ACCOUNT_LAST4", + value_text="1234", + source_document_id=document.document_id, + ) + ) + observation = dao.create_observation( + ObservationCreate( + observation_type_code="STATEMENT_PERIOD", + document_id=document.document_id, + subject_entity_id=entity.entity_id, + value_text="2026-01-01/2026-01-31", + confidence=1.0, + review_status="accepted", + ) + ) + evidence = dao.add_observation_evidence( + ObservationEvidenceCreate( + observation_id=observation.observation_id, + document_id=document.document_id, + evidence_text="Statement Period 01/01/26 to 01/31/26", + page_number=1, + location_text="page header", + ) + ) + + assert fetched_document is not None + assert fetched_document.document_id == document.document_id + assert fetched_document.file_hash == "a" * 64 + assert document.document_status == "active" + assert entity.entity_type == "account" + assert attribute.attribute_type_code == "ACCOUNT_LAST4" + assert attribute.value_text == "1234" + assert observation.observation_type_code == "STATEMENT_PERIOD" + assert observation.review_status == "accepted" + assert evidence.observation_id == observation.observation_id