diff --git a/tests/helpers/memdb.ts b/tests/helpers/memdb.ts new file mode 100644 index 0000000..4e867d4 --- /dev/null +++ b/tests/helpers/memdb.ts @@ -0,0 +1,17 @@ +// Test helper: fresh in-memory database with the full Recall schema. +// +// Property-based suites create a database per generated case (dozens per +// test), so the file-backed setupTestDb harness is too heavy — this builds +// the same tables/indexes in memory in ~1ms. FTS virtual tables and triggers +// are omitted: the export/dedup logic under property test never touches them. +import { Database } from 'bun:sqlite'; +import { CREATE_TABLES, CREATE_INDEXES, CREATE_VECTOR_TABLES } from '../../src/db/schema'; + +export function createMemoryDb(): Database { + const db = new Database(':memory:'); + db.exec('PRAGMA foreign_keys = ON'); // matches production getDb() + db.exec(CREATE_TABLES); + db.exec(CREATE_INDEXES); + db.exec(CREATE_VECTOR_TABLES); + return db; +} diff --git a/tests/lib/dedup.property.test.ts b/tests/lib/dedup.property.test.ts new file mode 100644 index 0000000..22209c3 --- /dev/null +++ b/tests/lib/dedup.property.test.ts @@ -0,0 +1,461 @@ +// Property-based tests for dedup invariants (issue #44 phase 2; dedup from +// issue #45, merged in PR #60). +// +// Every oracle is computed from generator inputs only (the tuple-multiset +// pattern from PR #57). Duplicate groups are built structurally: each group +// has a canonical lowercase single-spaced text and members vary only by +// case/whitespace/quoting, so expected grouping is generator-known without +// calling normalizeText. The survivor comparator and cosine similarity are +// restated from the issue #45 spec (provenance > richness > importance > +// recency > lowest id; cos of the angle difference), never read back from +// the implementation. +// +// Scope note: the one-hop lineage guarantee is WITHIN a single plan ("a +// planned survivor never re-marked" — the PR #60 blocker fix). Across runs, +// a prior survivor may legitimately be re-marked by a later semantic pass; +// the repeated-runs property therefore asserts auditability (uniqueness, +// record preservation, no re-marking of marked records), not cross-run +// one-hop. +// +// Generator sizes are bounded (≤5 groups × ≤4 members) so the suite stays +// practical under normal `bun test` runs. + +import { describe, test, expect } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import fc from 'fast-check'; +import { + applyDedupPlan, + MIN_DEDUP_TEXT_LENGTH, + planDedup, + type DedupPlan, +} from '../../src/lib/dedup'; +import { embeddingToBlob } from '../../src/lib/embeddings'; +import { createMemoryDb } from '../helpers/memdb'; + +type Prov = 'user_authored' | 'verbatim' | 'extracted' | 'derived' | null; +type DedupTestTable = 'breadcrumbs' | 'decisions'; + +// Spec restated (issue #45 / ADR-0001): survivor priority order, unknown +// (NULL) last. Deliberately not imported from src/types so a reordering +// regression fails here instead of silently reordering the oracle too. +const SURVIVOR_PRIORITY = ['user_authored', 'verbatim', 'extracted', 'derived'] as const; +const specRank = (p: Prov): number => + p === null ? SURVIVOR_PRIORITY.length : SURVIVOR_PRIORITY.indexOf(p); + +interface InsRecord { + table: DedupTestTable; + id: number; + project: string | null; + provenance: Prov; + importance: number; + createdAt: string; + /** Generator-known normalized text (the group canonical). */ + normalized: string; + angleDeg: number; +} + +// Spec comparator: provenance rank, richness (longer normalized text), +// importance, recency (lexicographic created_at desc), lowest id. +function specCompare(a: InsRecord, b: InsRecord): number { + const rank = specRank(a.provenance) - specRank(b.provenance); + if (rank !== 0) return rank; + if (a.normalized.length !== b.normalized.length) return b.normalized.length - a.normalized.length; + if (a.importance !== b.importance) return b.importance - a.importance; + if (a.createdAt !== b.createdAt) return a.createdAt > b.createdAt ? -1 : 1; + return a.id - b.id; +} + +// Cosine of two unit vectors at the generated angles — pure math over +// generator inputs. Stored vectors round-trip through float32 blobs, so +// comparisons against implementation output use F32_TOLERANCE. +const specCos = (aDeg: number, bDeg: number): number => + Math.cos(((aDeg - bDeg) * Math.PI) / 180); +const F32_TOLERANCE = 1e-4; + +const provArb = fc.constantFrom('user_authored', 'verbatim', 'extracted', 'derived', null); +const importanceArb = fc.integer({ min: 1, max: 10 }); +const dayArb = fc.integer({ min: 1, max: 28 }); +const angleArb = fc.integer({ min: 0, max: 359 }); + +const createdAt = (day: number): string => `2026-01-${String(day).padStart(2, '0')} 12:00:00`; + +// Canonical group text: lowercase, single-spaced — its own normalized form. +const canonical = (groupIdx: number): string => `dedup group ${groupIdx} canonical body text`; + +interface GenMember { + table: DedupTestTable; + project: 'p-a' | 'p-b' | null; + provenance: Prov; + importance: number; + day: number; + angleDeg: number; + upper: boolean; + pad: boolean; + quote: boolean; +} + +// Members vary only by case, whitespace, and quoting — all of which the +// documented normalization erases — so every member of a group shares the +// canonical normalized text by construction. +function decorate(text: string, m: GenMember): string { + let t = text; + if (m.upper) t = t.toUpperCase(); + if (m.pad) t = ` ${t.split(' ').join(' ')} `; + if (m.quote) t = `"${t}"`; + return t; +} + +const memberArb: fc.Arbitrary = fc.record({ + table: fc.constantFrom('breadcrumbs', 'decisions'), + project: fc.constantFrom<'p-a' | 'p-b' | null>('p-a', 'p-b', null), + provenance: provArb, + importance: importanceArb, + day: dayArb, + angleDeg: angleArb, + upper: fc.boolean(), + pad: fc.boolean(), + quote: fc.boolean(), +}); + +interface GenCorpus { + groups: GenMember[][]; + /** Records below MIN_DEDUP_TEXT_LENGTH — must never become candidates. */ + shorts: number; +} + +const corpusArb: fc.Arbitrary = fc.record({ + groups: fc.array(fc.array(memberArb, { minLength: 1, maxLength: 4 }), { minLength: 1, maxLength: 5 }), + shorts: fc.integer({ min: 0, max: 2 }), +}); + +function insertRecord( + db: Database, + table: DedupTestTable, + text: string, + m: Pick +): number { + const sql = table === 'breadcrumbs' + ? `INSERT INTO breadcrumbs (content, project, importance, provenance, created_at) VALUES (?, ?, ?, ?, ?)` + : `INSERT INTO decisions (decision, project, importance, provenance, created_at) VALUES (?, ?, ?, ?, ?)`; + const result = db.prepare(sql).run(text, m.project, m.importance, m.provenance, createdAt(m.day)); + return result.lastInsertRowid as number; +} + +function insertCorpus(db: Database, corpus: GenCorpus): InsRecord[] { + const records: InsRecord[] = []; + corpus.groups.forEach((members, groupIdx) => { + const base = canonical(groupIdx); + expect(base.length).toBeGreaterThanOrEqual(MIN_DEDUP_TEXT_LENGTH); // generator validity guard + for (const m of members) { + const id = insertRecord(db, m.table, decorate(base, m), m); + records.push({ + table: m.table, + id, + project: m.project, + provenance: m.provenance, + importance: m.importance, + createdAt: createdAt(m.day), + normalized: base, + angleDeg: m.angleDeg, + }); + } + }); + for (let i = 0; i < corpus.shorts; i++) { + insertRecord(db, 'breadcrumbs', 'tiny note', { project: null, provenance: null, importance: 5, day: 1 }); + } + return records; +} + +function insertEmbeddings(db: Database, records: InsRecord[]): void { + const stmt = db.prepare( + `INSERT INTO embeddings (source_table, source_id, model, dimensions, embedding) VALUES (?, ?, 'test', 2, ?)` + ); + for (const r of records) { + const rad = (r.angleDeg * Math.PI) / 180; + stmt.run(r.table, r.id, embeddingToBlob([Math.cos(rad), Math.sin(rad)])); + } +} + +type LineageTuple = [string, number, string, number, string, number | null]; + +const sortedTuples = (tuples: LineageTuple[]): string[] => + tuples.map(t => JSON.stringify(t)).sort(); + +function plannedTuples(plan: DedupPlan): LineageTuple[] { + return plan.tables.flatMap(t => t.planned).map(e => + [e.survivor_table, e.survivor_id, e.duplicate_table, e.duplicate_id, e.reason, e.similarity] as LineageTuple + ); +} + +// Generator oracle for the exact pass: group by (table, project, canonical), +// survivor by the spec comparator, one lineage tuple per other member. +function expectedExactTuples(records: InsRecord[]): LineageTuple[] { + const byKey = new Map(); + for (const r of records) { + const k = `${r.table}|${r.project ?? ''}|${r.normalized}`; + const group = byKey.get(k); + if (group) group.push(r); + else byKey.set(k, [r]); + } + const tuples: LineageTuple[] = []; + for (const group of byKey.values()) { + if (group.length < 2) continue; + const sorted = [...group].sort(specCompare); + for (const dup of sorted.slice(1)) { + tuples.push([sorted[0].table, sorted[0].id, dup.table, dup.id, 'exact', 1]); + } + } + return tuples; +} + +const SNAPSHOT_TABLES = [ + 'sessions', 'messages', 'decisions', 'learnings', 'breadcrumbs', 'loa_entries', 'dedup_lineage', 'embeddings', +]; + +function snapshot(db: Database): string { + return JSON.stringify( + SNAPSHOT_TABLES.map(t => [t, db.prepare(`SELECT * FROM ${t} ORDER BY id`).all()]) + ); +} + +function markedLineageTuples(db: Database): LineageTuple[] { + const rows = db.prepare( + `SELECT survivor_table, survivor_id, duplicate_table, duplicate_id, reason, similarity + FROM dedup_lineage WHERE status = 'marked'` + ).all() as Array<{ + survivor_table: string; survivor_id: number; + duplicate_table: string; duplicate_id: number; + reason: string; similarity: number | null; + }>; + return rows.map(r => + [r.survivor_table, r.survivor_id, r.duplicate_table, r.duplicate_id, r.reason, r.similarity] as LineageTuple + ); +} + +function withDb(fn: (db: Database) => T): T { + const db = createMemoryDb(); + try { + return fn(db); + } finally { + db.close(); + } +} + +describe('dedup exact-pass properties', () => { + test('planned lineage is exactly the generator-expected (survivor, duplicate) tuple multiset; non-duplicates and too-short records never appear', () => { + fc.assert( + fc.property(corpusArb, corpus => { + withDb(db => { + const records = insertCorpus(db, corpus); + const plan = planDedup(db, { semantic: false }); + + // Tuple-multiset equality pins survivor order under arbitrary + // provenance mixes AND non-duplicate preservation in one oracle: + // singletons and short records appear in no expected tuple. + expect(sortedTuples(plannedTuples(plan))).toEqual(sortedTuples(expectedExactTuples(records))); + + for (const table of ['breadcrumbs', 'decisions'] as const) { + const report = plan.tables.find(t => t.table === table)!; + const inserted = records.filter(r => r.table === table).length + + (table === 'breadcrumbs' ? corpus.shorts : 0); + expect(report.scanned).toBe(inserted); + if (table === 'breadcrumbs') expect(report.tooShort).toBe(corpus.shorts); + } + }); + }) + ); + }); +}); + +describe('dedup semantic threshold properties', () => { + interface SemRecord { + angleDeg: number; + extraLen: number; + provenance: Prov; + importance: number; + day: number; + } + const semRecordArb: fc.Arbitrary = fc.record({ + angleDeg: angleArb, + extraLen: fc.integer({ min: 0, max: 30 }), + provenance: provArb, + importance: importanceArb, + day: dayArb, + }); + const semCorpusArb = fc.array(semRecordArb, { minLength: 2, maxLength: 7 }); + + // Unique texts (no exact groups), varied lengths (richness tie-breaker is + // live, unlike exact groups where normalized lengths are equal by + // definition), one table + project so every pair is actionable. + function insertSemCorpus(db: Database, corpus: SemRecord[]): InsRecord[] { + const records: InsRecord[] = corpus.map((r, i) => { + const text = `semantic record ${i} body text${'x'.repeat(r.extraLen)}`; + const id = insertRecord(db, 'breadcrumbs', text, { project: null, ...r }); + return { + table: 'breadcrumbs' as const, + id, + project: null, + provenance: r.provenance, + importance: r.importance, + createdAt: createdAt(r.day), + normalized: text, + angleDeg: r.angleDeg, + }; + }); + insertEmbeddings(db, records); + return records; + } + + test('a threshold above every pairwise similarity plans nothing — never merge below the threshold', () => { + fc.assert( + fc.property(semCorpusArb, fc.integer({ min: 1, max: 10 }), (corpus, gapPct) => { + withDb(db => { + insertSemCorpus(db, corpus); + let maxSim = -1; + for (let i = 0; i < corpus.length; i++) { + for (let j = i + 1; j < corpus.length; j++) { + maxSim = Math.max(maxSim, specCos(corpus[i].angleDeg, corpus[j].angleDeg)); + } + } + const plan = planDedup(db, { semantic: true, threshold: maxSim + gapPct / 100 }); + expect(plannedTuples(plan)).toEqual([]); + }); + }) + ); + }); + + test('every planned semantic pair is at/above the threshold, records its true similarity, and keeps the spec-ordered survivor', () => { + fc.assert( + fc.property(semCorpusArb, fc.integer({ min: 30, max: 99 }), (corpus, thresholdPct) => { + withDb(db => { + const records = insertSemCorpus(db, corpus); + const byId = new Map(records.map(r => [r.id, r])); + const threshold = thresholdPct / 100; + const plan = planDedup(db, { semantic: true, threshold }); + + for (const entry of plan.tables.flatMap(t => t.planned)) { + expect(entry.reason).toBe('semantic'); + const survivor = byId.get(entry.survivor_id)!; + const duplicate = byId.get(entry.duplicate_id)!; + const expectedSim = specCos(survivor.angleDeg, duplicate.angleDeg); + expect(expectedSim).toBeGreaterThanOrEqual(threshold - F32_TOLERANCE); + expect(Math.abs((entry.similarity ?? 0) - expectedSim)).toBeLessThanOrEqual(F32_TOLERANCE); + expect(specCompare(survivor, duplicate)).toBeLessThan(0); + } + }); + }) + ); + }); +}); + +describe('dedup one-hop lineage and write-freeness', () => { + const thresholdArb = fc.integer({ min: 50, max: 99 }).map(n => n / 100); + + test('within a plan, survivor and duplicate key sets are disjoint and every duplicate is marked at most once', () => { + fc.assert( + fc.property(corpusArb, thresholdArb, (corpus, threshold) => { + withDb(db => { + const records = insertCorpus(db, corpus); + insertEmbeddings(db, records); + const entries = planDedup(db, { semantic: true, threshold }).tables.flatMap(t => t.planned); + + const duplicateKeys = entries.map(e => `${e.duplicate_table}:${e.duplicate_id}`); + const duplicateSet = new Set(duplicateKeys); + expect(duplicateKeys.length).toBe(duplicateSet.size); + for (const e of entries) { + // A planned survivor is never re-marked — lineage stays one hop. + expect(duplicateSet.has(`${e.survivor_table}:${e.survivor_id}`)).toBe(false); + } + }); + }) + ); + }); + + test('planDedup is write-free: the database is byte-identical before and after planning', () => { + fc.assert( + fc.property(corpusArb, thresholdArb, (corpus, threshold) => { + withDb(db => { + const records = insertCorpus(db, corpus); + insertEmbeddings(db, records); + const before = snapshot(db); + planDedup(db, { semantic: true, threshold }); + expect(snapshot(db)).toBe(before); + }); + }) + ); + }); +}); + +describe('dedup apply, idempotence, and repeated-run auditability', () => { + test('apply marks exactly the planned tuples, preserves every record, and a re-plan finds nothing (exact pass)', () => { + fc.assert( + fc.property(corpusArb, corpus => { + withDb(db => { + const records = insertCorpus(db, corpus); + const expected = sortedTuples(expectedExactTuples(records)); + + const plan1 = planDedup(db, { semantic: false }); + const result1 = applyDedupPlan(db, plan1); + expect(result1).toEqual({ marked: expected.length, deleted: 0, fkProtected: 0 }); + expect(sortedTuples(markedLineageTuples(db))).toEqual(expected); + + // Non-destructive default: every generated record is still there. + for (const table of ['breadcrumbs', 'decisions'] as const) { + const inserted = records.filter(r => r.table === table).length + + (table === 'breadcrumbs' ? corpus.shorts : 0); + const { c } = db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get() as { c: number }; + expect(c).toBe(inserted); + } + + // Idempotence: marked duplicates are excluded, surviving groups + // are singletons, so the second plan is empty and changes nothing. + const plan2 = planDedup(db, { semantic: false }); + expect(plannedTuples(plan2)).toEqual([]); + for (const table of ['breadcrumbs', 'decisions'] as const) { + const report = plan2.tables.find(t => t.table === table)!; + const expectedMarked = expectedExactTuples(records) + .filter(t => t[2] === table).length; + expect(report.alreadyMarked).toBe(expectedMarked); + } + expect(applyDedupPlan(db, plan2)).toEqual({ marked: 0, deleted: 0, fkProtected: 0 }); + expect(sortedTuples(markedLineageTuples(db))).toEqual(expected); + }); + }) + ); + }); + + test('repeated semantic plan/apply cycles stay auditable: unique active marks, no re-marking, all records preserved', () => { + fc.assert( + fc.property(corpusArb, fc.integer({ min: 50, max: 99 }).map(n => n / 100), (corpus, threshold) => { + withDb(db => { + const records = insertCorpus(db, corpus); + insertEmbeddings(db, records); + const totalRows = (table: DedupTestTable): number => + records.filter(r => r.table === table).length + (table === 'breadcrumbs' ? corpus.shorts : 0); + + applyDedupPlan(db, planDedup(db, { semantic: true, threshold })); + const markedAfterFirst = new Set( + markedLineageTuples(db).map(t => `${t[2]}:${t[3]}`) + ); + + const plan2 = planDedup(db, { semantic: true, threshold }); + // Already-marked records are excluded from later runs entirely. + for (const e of plan2.tables.flatMap(t => t.planned)) { + expect(markedAfterFirst.has(`${e.duplicate_table}:${e.duplicate_id}`)).toBe(false); + expect(markedAfterFirst.has(`${e.survivor_table}:${e.survivor_id}`)).toBe(false); + } + applyDedupPlan(db, plan2); + + // Each record is an actively marked duplicate at most once, and + // the non-destructive default never removes rows. + const allMarked = markedLineageTuples(db).map(t => `${t[2]}:${t[3]}`); + expect(allMarked.length).toBe(new Set(allMarked).size); + for (const table of ['breadcrumbs', 'decisions'] as const) { + const { c } = db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get() as { c: number }; + expect(c).toBe(totalRows(table)); + } + }); + }) + ); + }); +}); diff --git a/tests/lib/export.property.test.ts b/tests/lib/export.property.test.ts new file mode 100644 index 0000000..f631ba7 --- /dev/null +++ b/tests/lib/export.property.test.ts @@ -0,0 +1,291 @@ +// Property-based tests for export invariants (issue #44 phase 2; export +// surface from issue #43, merged in PR #59). +// +// Every oracle is computed from generator inputs only (the tuple-multiset +// pattern from PR #57): expected counts, provenance histograms, and +// (text, provenance) tuples all derive from the generated corpus, never from +// the implementation under test. The SQL round-trip additionally checks +// dump→restore as an inverse pair against the rows the generator inserted. +// +// Two distinct NULL-provenance contracts are pinned: +// - app-level (JSON/Markdown) exports render NULL as the literal 'unknown', +// never omitted, never guessed (issue #43 provenance contract); +// - the SQL dump carries raw rows, so NULL survives restore as NULL. +// +// Generator sizes are bounded (≤6 rows per table) so the suite stays +// practical under normal `bun test` runs. + +import { describe, test, expect } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import fc from 'fast-check'; +import { + buildManifest, + collectExportData, + EXPORT_TABLES, + renderJsonExport, + renderMarkdownExport, + renderSqlDump, +} from '../../src/lib/export'; +import { createMemoryDb } from '../helpers/memdb'; + +type Prov = 'user_authored' | 'verbatim' | 'extracted' | 'derived' | null; + +// Spec restated (issue #43 / ADR-0001), deliberately not imported from +// src/types: a regression that drops a table from PROVENANCE_TABLES or +// EXPORT_TABLES must fail here instead of silently weakening the oracle. +const PROV_TABLES = ['messages', 'decisions', 'learnings', 'breadcrumbs', 'loa_entries'] as const; +const ALL_TABLES = ['sessions', ...PROV_TABLES, 'dedup_lineage'] as const; +type ProvTable = typeof PROV_TABLES[number]; + +// Which column carries the generated text per table. +const TEXT_COLUMN: Record = { + messages: 'content', + decisions: 'decision', + learnings: 'problem', + breadcrumbs: 'content', + loa_entries: 'fabric_extract', +}; + +interface GenRecord { + text: string; + project: string | null; + provenance: Prov; + importance: number; +} + +interface GenCorpus { + sessions: number; + messages: GenRecord[]; + decisions: GenRecord[]; + learnings: GenRecord[]; + breadcrumbs: GenRecord[]; + loa_entries: GenRecord[]; + lineage: number; +} + +// Apostrophes and newlines are injected with probability 1/2 each so SQL +// quoting and Markdown multi-line rendering are exercised on every run. +const textArb = fc + .tuple(fc.string({ maxLength: 30 }), fc.boolean(), fc.boolean()) + .map(([s, apostrophe, multiline]) => + `${apostrophe ? "it's " : ''}${s}${multiline ? '\nsecond "line"' : ''}`); + +const recordArb: fc.Arbitrary = fc.record({ + text: textArb, + project: fc.option(fc.constantFrom('proj-a', 'proj-b'), { nil: null }), + provenance: fc.constantFrom('user_authored', 'verbatim', 'extracted', 'derived', null), + importance: fc.integer({ min: 1, max: 10 }), +}); + +const rowsArb = fc.array(recordArb, { maxLength: 6 }); + +const corpusArb: fc.Arbitrary = fc.record({ + sessions: fc.integer({ min: 1, max: 3 }), + messages: rowsArb, + decisions: rowsArb, + learnings: rowsArb, + breadcrumbs: rowsArb, + loa_entries: rowsArb, + lineage: fc.integer({ min: 0, max: 3 }), +}); + +const CREATED_AT = new Date('2026-06-11T00:00:00Z'); + +function insertCorpus(db: Database, corpus: GenCorpus): void { + const session = db.prepare( + `INSERT INTO sessions (session_id, started_at) VALUES (?, '2026-06-01T00:00:00Z')` + ); + for (let i = 0; i < corpus.sessions; i++) session.run(`s-${i}`); + + const inserts: Record void> = { + messages: (r, i) => db.prepare( + `INSERT INTO messages (session_id, timestamp, role, content, project, importance, provenance) + VALUES (?, '2026-06-01T01:00:00Z', 'user', ?, ?, ?, ?)` + ).run(`s-${i % corpus.sessions}`, r.text, r.project, r.importance, r.provenance), + decisions: r => db.prepare( + `INSERT INTO decisions (decision, project, importance, provenance) VALUES (?, ?, ?, ?)` + ).run(r.text, r.project, r.importance, r.provenance), + learnings: r => db.prepare( + `INSERT INTO learnings (problem, project, importance, provenance) VALUES (?, ?, ?, ?)` + ).run(r.text, r.project, r.importance, r.provenance), + breadcrumbs: r => db.prepare( + `INSERT INTO breadcrumbs (content, project, importance, provenance) VALUES (?, ?, ?, ?)` + ).run(r.text, r.project, r.importance, r.provenance), + loa_entries: r => db.prepare( + `INSERT INTO loa_entries (title, fabric_extract, project, importance, provenance) + VALUES ('entry', ?, ?, ?, ?)` + ).run(r.text, r.project, r.importance, r.provenance), + }; + for (const table of PROV_TABLES) { + corpus[table].forEach((r, i) => inserts[table](r, i)); + } + + const lineage = db.prepare( + `INSERT INTO dedup_lineage (survivor_table, survivor_id, duplicate_table, duplicate_id, reason, similarity, status) + VALUES ('messages', ?, 'messages', ?, 'exact', 1.0, 'marked')` + ); + for (let i = 0; i < corpus.lineage; i++) lineage.run(1000 + i, 2000 + i); +} + +function expectedCount(corpus: GenCorpus, table: string): number { + if (table === 'sessions') return corpus.sessions; + if (table === 'dedup_lineage') return corpus.lineage; + return corpus[table as ProvTable].length; +} + +// Per-table provenance histogram from generator data: 'unknown' key always +// present (even at zero), other keys only when at least one record carries +// the value — the manifest contract from issue #43. +function expectedHistogram(records: GenRecord[]): Record { + const histogram: Record = { unknown: 0 }; + for (const r of records) { + const key = r.provenance ?? 'unknown'; + histogram[key] = (histogram[key] ?? 0) + 1; + } + return histogram; +} + +const sortedJson = (tuples: unknown[][]): string[] => + tuples.map(t => JSON.stringify(t)).sort(); + +function withCorpusDb(corpus: GenCorpus, fn: (db: Database) => T): T { + const db = createMemoryDb(); + try { + insertCorpus(db, corpus); + return fn(db); + } finally { + db.close(); + } +} + +describe('export properties', () => { + test('EXPORT_TABLES is the durable-table contract from issues #43/#45', () => { + expect([...EXPORT_TABLES]).toEqual([...ALL_TABLES]); + }); + + test('JSON export: manifest counts, (text, provenance) tuple multisets, NULL rendered as explicit unknown', () => { + fc.assert( + fc.property(corpusArb, corpus => { + withCorpusDb(corpus, db => { + const manifest = buildManifest(db, 'json', [...EXPORT_TABLES], { + includesEmbeddings: false, + createdAt: CREATED_AT, + }); + const parsed = JSON.parse(renderJsonExport(manifest, collectExportData(db))) as { + manifest: typeof manifest; + tables: Record>>; + }; + + expect(parsed.manifest.tables).toEqual([...ALL_TABLES]); + for (const table of ALL_TABLES) { + expect(parsed.manifest.counts[table]).toBe(expectedCount(corpus, table)); + expect(parsed.tables[table].length).toBe(expectedCount(corpus, table)); + } + + for (const table of PROV_TABLES) { + const actual = parsed.tables[table].map(row => [row[TEXT_COLUMN[table]], row.provenance]); + const expected = corpus[table].map(r => [r.text, r.provenance ?? 'unknown']); + expect(sortedJson(actual)).toEqual(sortedJson(expected)); + expect(parsed.manifest.provenance_counts[table]).toEqual(expectedHistogram(corpus[table])); + } + + // Tables without the column get no field invented. + for (const row of parsed.tables.sessions) { + expect('provenance' in row).toBe(false); + } + }); + }) + ); + }); + + test('Markdown export: one table heading with the generated count, one record heading per generated row', () => { + fc.assert( + fc.property(corpusArb, corpus => { + withCorpusDb(corpus, db => { + const manifest = buildManifest(db, 'markdown', [...EXPORT_TABLES], { + includesEmbeddings: false, + createdAt: CREATED_AT, + }); + const lines = renderMarkdownExport(manifest, collectExportData(db)).split('\n'); + + // Value lines can never start at column 0 with '#': single-line + // values render inline after '- **key:**' and multi-line values are + // indented blockquotes — so heading counts are sound oracles even + // for adversarial generated text. + expect(lines[0]).toBe('# Recall Memory Export'); + expect(lines.filter(l => l.startsWith('## '))).toEqual( + ALL_TABLES.map(t => `## ${t} (${expectedCount(corpus, t)} rows)`) + ); + for (const table of ALL_TABLES) { + expect(lines.filter(l => l.startsWith(`### ${table} #`)).length).toBe( + expectedCount(corpus, table) + ); + } + }); + }) + ); + }); + + test('SQL dump restores into an empty database with full row fidelity; NULL provenance stays NULL', () => { + fc.assert( + fc.property(corpusArb, corpus => { + withCorpusDb(corpus, db => { + const manifest = buildManifest(db, 'sql', [...EXPORT_TABLES], { + includesEmbeddings: false, + createdAt: CREATED_AT, + }); + const restored = new Database(':memory:'); + try { + restored.exec(renderSqlDump(db, manifest)); + + // dump ∘ restore is the identity on every exported table's rows + // (the source rows came from the generator's inserts). + for (const table of ALL_TABLES) { + const src = db.prepare(`SELECT * FROM ${table} ORDER BY id`).all(); + const back = restored.prepare(`SELECT * FROM ${table} ORDER BY id`).all(); + expect(back).toEqual(src); + expect(back.length).toBe(expectedCount(corpus, table)); + } + + // Generator-anchored raw tuples: unlike JSON, the dump carries + // stored values — legacy NULL provenance must restore as NULL, + // not as 'unknown'. + for (const table of PROV_TABLES) { + const rows = restored.prepare(`SELECT * FROM ${table}`).all() as Array>; + const actual = rows.map(row => [row[TEXT_COLUMN[table]], row.provenance]); + const expected = corpus[table].map(r => [r.text, r.provenance]); + expect(sortedJson(actual)).toEqual(sortedJson(expected)); + } + } finally { + restored.close(); + } + }); + }) + ); + }); + + test('manifest is accurate for any table subset; provenance histograms always cover every provenance-bearing table', () => { + fc.assert( + fc.property(corpusArb, fc.subarray([...ALL_TABLES], { minLength: 1 }), (corpus, subset) => { + withCorpusDb(corpus, db => { + const manifest = buildManifest(db, 'json', subset, { + includesEmbeddings: false, + createdAt: CREATED_AT, + }); + + expect(manifest.tables).toEqual(subset); + expect(Object.keys(manifest.counts).sort()).toEqual([...subset].sort()); + for (const table of subset) { + expect(manifest.counts[table]).toBe(expectedCount(corpus, table)); + } + expect(Object.keys(manifest.provenance_counts).sort()).toEqual([...PROV_TABLES].sort()); + for (const table of PROV_TABLES) { + expect(manifest.provenance_counts[table]).toEqual(expectedHistogram(corpus[table])); + } + expect(manifest.created_at).toBe(CREATED_AT.toISOString()); + expect(manifest.includes_embeddings).toBe(false); + }); + }) + ); + }); +});