diff --git a/.env.example b/.env.example index db51cba..a689563 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,12 @@ # the bundled single-container Postgres/pgvector instance. Source checkouts and # production deployments should set a real Postgres connection string. DATABASE_URL=postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory +# Docker entrypoint only: set false when migrations run from a separate +# pre-deploy job. Defaults to true when omitted. +# ATOMICMEMORY_RUN_MIGRATIONS_ON_STARTUP=true +# Docker entrypoint only: forwarded to `migrate({ lockTimeoutMs })`. +# Increase for rolling deploys where another replica may hold the migration lock. +# MIGRATION_LOCK_TIMEOUT_MS=120000 # --- Provider credentials --- # Required when either EMBEDDING_PROVIDER=openai or LLM_PROVIDER=openai. @@ -19,6 +25,13 @@ OPENAI_API_KEY= # Docker image local mode defaults this to `local-dev-key` when omitted. CORE_API_KEY=replace-with-a-strong-random-secret +# Optional admin-only cleanup endpoint for disposable smoke/eval scopes. +# When both values are set, DELETE /v1/admin/scope accepts a JSON body +# `{ "user_id": "..." }` and deletes only matching test scopes. +# Use a different secret from CORE_API_KEY. Do not enable for general clients. +# CORE_ADMIN_API_KEY= +# CORE_TEST_SCOPE_ALLOW_PATTERN=^(smoke-|docker-|test-).+ + # Hex-encoded HMAC secret used to derive PII-safe storage-key # prefixes. Must be at least 64 hex chars (32 bytes of entropy). # Rotating this invalidates existing managed-blob storage paths; diff --git a/.env.test.example b/.env.test.example index e871316..2e1860a 100644 --- a/.env.test.example +++ b/.env.test.example @@ -5,6 +5,9 @@ DATABASE_URL=postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory OPENAI_API_KEY=test-placeholder CORE_API_KEY=test-core-api-key +# Optional admin cleanup endpoint used by external smoke harnesses. +# CORE_ADMIN_API_KEY=test-admin-api-key +# CORE_TEST_SCOPE_ALLOW_PATTERN=^(smoke-|docker-|test-).+ STORAGE_KEY_HMAC_SECRET=000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f EMBEDDING_DIMENSIONS=1024 PORT=3051 diff --git a/.fallowrc.json b/.fallowrc.json index 2cf1b1c..6e6b159 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -2,7 +2,13 @@ "$schema": "https://fallow.tools/schema.json", "ignorePatterns": [ "**/one-offs/**", - "scripts/smoke-openapi-export.mjs" + "scripts/**" + ], + "ignoreDependencies": [ + "@helia/unixfs", + "blockstore-core", + "pino", + "yaml" ], "rules": { "unused-types": "off", diff --git a/.gitignore b/.gitignore index 575fb2a..bd8fc76 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,6 @@ scripts/one-offs/ # Platform-specific deploy configs (canonical copies live in deploy/) railway.toml *.tgz + +# Superpowers skill plugin output — agent-generated specs/plans, internal-only. +docs/superpowers/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c6e443..e24136b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,64 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Added +- Phase 1 migration hardening now packages a deterministic + `dist/db/schema-sha256.json` manifest for the shipped DB schema bytes. +- Phase 2 versioned migrations. Schema is now expressed as ordered files + under `src/db/migrations/` (shipped as `dist/db/migrations/`), executed + by `node-pg-migrate` and tracked per-file in the `pgmigrations` table. + The Phase 1 `schema_version` table is preserved alongside `pgmigrations` + so operators can answer both "which migration files ran" and "which + `@atomicmemory/core` semver this DB corresponds to". +- `migrationStatus()` surfaces two new read-only fields, + `appliedMigrationCount` and `latestMigrationName`, sourced from + `pgmigrations`. The existing `status` enum (`up_to_date` / `older_db` / + `newer_db` / `unstamped` / `no_schema`) is unchanged. + ### Changed - **BREAKING**: All API endpoints are now mounted under `/v1/` (e.g. `POST /v1/memories/ingest`, `PUT /v1/agents/trust`). Update clients to prefix requests with `/v1`. The unversioned `/health` liveness probe is unchanged. +- Phase 2 removes `src/db/schema.sql`; the migrations folder is now the + single source of truth. The build-time `dist/db/schema-sha256.json` + manifest is preserved but now describes the ordered migration directory. + Library and CLI surfaces are unchanged: `migrate()` and + `migrationStatus()` keep their Phase 1 signatures, and + `MigrateResult.ranSchemaSql` is preserved as "this call executed the + migration runner path" (the Phase 1 semantics for the no-op-loser path + still hold). +- Moved provenance-only SQL changelog files from `src/db/migrations/` to + `docs/db/changelog/`. Phase 2 reclaims `src/db/migrations/` as the + runtime migration folder. + +### Migration (Phase 2 — lossless guarantee) + +Three install paths reach the same end state without data loss, without +unexpected DDL, and without operator intervention. All three run inside +the same advisory-lock wrapper used by Phase 1 (`MIGRATION_LOCK_ID` +unchanged), so concurrent replica boots remain safe. + +- **Scenario A — fresh install on Phase 2.** `migrate()` creates + `pgmigrations`, runs `0001_baseline.sql` against the empty database, + runs any later migration files, runs the embedding-dimension reconciler + (against now-empty tables; no-op or a single `ALTER COLUMN`), and stamps + `schema_version`. +- **Scenario B — v1.0.x with data → Phase 2.** `migrate()` detects that + core tables exist but `pgmigrations` does not. It creates `pgmigrations`, + **stamps `0001_baseline` as applied without executing it**, runs any + post-baseline migrations against the live schema, runs the reconciler + on the live (possibly populated) tables with the same Phase 1 + semantics, and inserts the first `schema_version` row. Baseline DDL + does not touch existing tables. +- **Scenario C — Phase 1 → Phase 2.** Same as B except `schema_version` + already exists; the upgrade appends a new row instead of creating the + table. + +Enforcement: `baseline-schema-equivalence.test.ts` (the CI gate) builds +both end states on fresh databases and asserts the schema-only structural +snapshot is identical modulo the framework-bookkeeping tables. The Phase 1 +data-preservation suite is carried forward unchanged and continues to +seed legacy rows, snapshot them, run `migrate()`, and assert the rows, +primary keys, foreign keys, JSON metadata, timestamps, and representative +vector fields survive the Phase 1 → Phase 2 cutover. ## [1.0.0] - 2026-04-15 diff --git a/README.md b/README.md index d920fef..3416ffa 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,76 @@ npm run dev Health check: `curl http://localhost:3050/v1/memories/health` +### Migrations + +Core uses versioned migration files as the single source of truth for +PostgreSQL schema. The files live under `src/db/migrations/` and ship to the +package as `dist/db/migrations/`. There is no `schema.sql` — the migrations +folder is the schema, in order. To regenerate the equivalent full-schema dump +locally, replay the migrations against an empty DB and run +`pg_dump --schema-only`. + +Run migrations once before deploy or during a single startup step before +serving traffic: + +```bash +npm run migrate +``` + +Docker image users can keep the default startup migration step for local or +single-replica deployments. For rolling production deploys, run migrations as +a pre-deploy job and start app containers with +`ATOMICMEMORY_RUN_MIGRATIONS_ON_STARTUP=false`. If startup migrations remain +enabled, `MIGRATION_LOCK_TIMEOUT_MS` raises the advisory-lock wait window. + +Applications that embed Core can call the programmatic API directly instead +of shelling out: + +```ts +import { migrate, migrationStatus } from '@atomicmemory/core'; + +const status = await migrationStatus({ pool }); +if (status.status !== 'up_to_date') { + await migrate({ pool }); +} +``` + +The `migrate()` and `migrationStatus()` signatures are unchanged from Phase 1; +only their internals were rewritten on top of `node-pg-migrate`. `MigrateResult` +fields are populated the same way — `ranSchemaSql` now means "this call +executed the migration runner path" rather than "this call executed the legacy +`schema.sql` file". +`MigrationStatus` adds read-only diagnostics sourced from the framework and +pgvector catalogs: `appliedMigrationCount`, `latestMigrationName`, +`migrationHistoryStatus`, and `embeddingDimension`. + +To inspect a running database, two tables answer different questions: + +| Table | Question it answers | +|------------------|----------------------------------------------------------| +| `pgmigrations` | Which migration files have been applied, and in what order | +| `schema_version` | Which `@atomicmemory/core` semver this DB corresponds to | + +Both are kept on purpose. `pgmigrations` is the framework's audit trail; +`schema_version` is the operator-friendly "what code matches this DB" stamp. +Querying either is safe from any client. + +```sql +SELECT id, name, run_on FROM pgmigrations ORDER BY id; +SELECT sdk_version, schema_sha256, applied_at FROM schema_version + ORDER BY applied_at DESC LIMIT 1; +``` + +Upgrades are lossless. A v1.0.x or Phase-1 database with existing rows takes +the same `migrate()` call as a fresh install — `migrate()` detects the +pre-migration install state, stamps the baseline migration as already-applied +without re-executing it, and runs only the migrations after the baseline. +See [`docs/db/migrations.md`](docs/db/migrations.md) for the scenario-by-scenario +guarantees and inspection runbook. + +The provenance SQL files under `docs/db/changelog/` are references only; +runtime schema execution is owned entirely by the `src/db/migrations/` folder. + ### npm CLI The npm package also ships a thin CLI for environments where you already have @@ -279,7 +349,7 @@ The compose file includes Postgres with pgvector. The app container runs migrati src/ routes/ # Express route handlers services/ # Business logic (extraction, retrieval, packaging) - db/ # Repository layer, schema, migrations + db/ # Repository layer and canonical schema adapters/ # Type contracts for external integrations config.ts # Environment-driven configuration server.ts # Express app bootstrap diff --git a/docker-compose.smoke-isolated.yml b/docker-compose.smoke-isolated.yml index edbda68..bdad57b 100644 --- a/docker-compose.smoke-isolated.yml +++ b/docker-compose.smoke-isolated.yml @@ -41,6 +41,11 @@ services: # Schemathesis, so the values are dummy placeholders that match # `.env.test`. NEVER reuse these strings in a real deployment. CORE_API_KEY: test-core-api-key-do-not-leak + # Schemathesis fuzzes the whole OpenAPI surface, including the optional + # admin cleanup route. Enable it only in this disposable smoke stack so + # the documented /v1/admin/scope path is mounted during schema fuzzing. + CORE_ADMIN_API_KEY: test-core-api-key-do-not-leak + CORE_TEST_SCOPE_ALLOW_PATTERN: "^schemathesis-fuzz-user$" STORAGE_KEY_HMAC_SECRET: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f env_file: - .env diff --git a/docs/db/README.md b/docs/db/README.md new file mode 100644 index 0000000..4f02f1e --- /dev/null +++ b/docs/db/README.md @@ -0,0 +1,16 @@ +# Database documentation + +In-repo documentation for the PostgreSQL + pgvector layer that ships with +`@atomicmemory/core`. Public operator and contributor guidance lives here; +internal planning history stays in the private research workspace. + +## Contents + +- [`migrations.md`](./migrations.md) — operator and contributor reference + for the Phase 2 versioned migration system. Covers the folder layout, + inspection queries against `pgmigrations` and `schema_version`, the + Scenario A/B/C lossless guarantee, the `migrate()` / `migrationStatus()` + API surface, and the workflow for adding a new migration. +- [`changelog/`](./changelog) — provenance-only SQL files from the + pre-Phase-2 schema evolution. Reference material, not executed at + runtime. diff --git a/src/db/migrations/20260511_entity_attributes.sql b/docs/db/changelog/20260511_entity_attributes.sql similarity index 100% rename from src/db/migrations/20260511_entity_attributes.sql rename to docs/db/changelog/20260511_entity_attributes.sql diff --git a/src/db/migrations/20260511_user_profiles.sql b/docs/db/changelog/20260511_user_profiles.sql similarity index 100% rename from src/db/migrations/20260511_user_profiles.sql rename to docs/db/changelog/20260511_user_profiles.sql diff --git a/src/db/migrations/20260512_audn_bilateral.sql b/docs/db/changelog/20260512_audn_bilateral.sql similarity index 100% rename from src/db/migrations/20260512_audn_bilateral.sql rename to docs/db/changelog/20260512_audn_bilateral.sql diff --git a/src/db/migrations/20260512_entity_cards.sql b/docs/db/changelog/20260512_entity_cards.sql similarity index 100% rename from src/db/migrations/20260512_entity_cards.sql rename to docs/db/changelog/20260512_entity_cards.sql diff --git a/src/db/migrations/20260512_entity_values.sql b/docs/db/changelog/20260512_entity_values.sql similarity index 100% rename from src/db/migrations/20260512_entity_values.sql rename to docs/db/changelog/20260512_entity_values.sql diff --git a/src/db/migrations/20260512_session_reflections.sql b/docs/db/changelog/20260512_session_reflections.sql similarity index 100% rename from src/db/migrations/20260512_session_reflections.sql rename to docs/db/changelog/20260512_session_reflections.sql diff --git a/src/db/migrations/20260512_temporal_state.sql b/docs/db/changelog/20260512_temporal_state.sql similarity index 100% rename from src/db/migrations/20260512_temporal_state.sql rename to docs/db/changelog/20260512_temporal_state.sql diff --git a/docs/db/migrations.md b/docs/db/migrations.md new file mode 100644 index 0000000..dc76436 --- /dev/null +++ b/docs/db/migrations.md @@ -0,0 +1,230 @@ +# Migrations + +`@atomicmemory/core` runs against PostgreSQL + pgvector. As of Phase 2 the +schema is composed of ordered migration files under `src/db/migrations/`, +executed by [`node-pg-migrate`](https://github.com/salsita/node-pg-migrate) +behind the same `migrate()` / `migrationStatus()` API that Phase 1 introduced. +This page is the operator- and contributor-facing reference that ships with +the package. + +## Folder layout + +``` +src/db/ + migrations/ + 0001_baseline.sql Frozen Phase-1 schema; never edited after shipment. + 0002_.sql First post-baseline migration. + … + migration-api.ts migrate() / migrationStatus() entry points. + migrate.ts CLI shim used by `npm run migrate`. +``` + +After `npm run build`, the files are copied into `dist/db/migrations/` and +shipped in the published package so library consumers can run migrations +without cloning the source. + +There is no `schema.sql` at runtime. The migrations folder, replayed in +order against an empty database, is the schema. To get a current schema +dump locally, replay the files against an empty database and run +`pg_dump --schema-only`. + +## Inspecting state + +Two tables hold migration state, on purpose: + +| Table | Maintained by | Answers | +|------------------|------------------------------|----------------------------------------------------------| +| `pgmigrations` | `node-pg-migrate` framework | Which migration files have been applied, and when | +| `schema_version` | `@atomicmemory/core` | Which package semver this DB corresponds to | + +Inspection queries operators can run against any reachable Postgres: + +```sql +-- Per-file history (one row per applied migration). +SELECT id, name, run_on FROM pgmigrations ORDER BY id; + +-- Effective package version (latest row by applied_at). +SELECT sdk_version, schema_sha256, applied_at + FROM schema_version + ORDER BY applied_at DESC + LIMIT 1; + +-- Full Phase-1 stamp history when audit needs it. +SELECT sdk_version, schema_sha256, applied_at, notes + FROM schema_version + ORDER BY applied_at DESC; +``` + +`migrationStatus()` surfaces both tables programmatically without holding +the advisory lock: + +```ts +import { migrationStatus } from '@atomicmemory/core'; + +const status = await migrationStatus({ pool }); +// status.status : 'no_schema' | 'unstamped' | 'up_to_date' | 'older_db' | 'newer_db' +// status.appliedMigrationCount : number (rows in pgmigrations) +// status.latestMigrationName : string (name of the most recent pgmigrations row, or '' if none) +// status.appliedSdkVersion : string | null (latest schema_version) +// status.appliedSchemaSha : string | null +// status.packageSdkVersion : string (this package's version) +// status.packageSchemaSha : string (current build's expected schema hash) +// status.migrationHistoryStatus: 'absent' | 'missing_baseline' | 'behind' | 'current' | 'ahead' +// status.embeddingDimension : read-only pgvector dimension drift report +``` + +## The lossless guarantee + +Every existing production database upgrades to Phase 2 without data loss, +without unexpected DDL, and without operator intervention. All three install +paths run inside the Phase 1 advisory-lock wrapper (`MIGRATION_LOCK_ID` +unchanged), so concurrent replica boots remain safe. + +### Scenario A — Fresh install on a Phase 2 release + +1. `migrate()` acquires the advisory lock. +2. `pgmigrations` does not exist; the framework creates it. +3. `0001_baseline.sql` runs against the empty database — pgvector + extension, all tables, all indexes are created. +4. Any later migrations (`0002_*`, …) run in lexical order. +5. The embedding-dimension reconciler runs against the baseline tables + before any post-baseline migration observes their vector dimensions. +6. Any newly added empty vector columns from later migrations are reconciled + by a final pass. +7. A row is appended to `schema_version`. +8. The lock is released. + +### Scenario B — v1.0.x install with data → Phase 2 release + +1. `migrate()` acquires the advisory lock. +2. `pgmigrations` does not exist, or it exists but has no applied rows because + a prior cutover attempt died before stamping anything. +3. **Pre-framework check**: data tables exist (for example, `memories`, + `memory_claims`, `episodes`) but `pgmigrations` does not. This is a + pre-Phase-2 install. +4. The runner asks `node-pg-migrate` to create `pgmigrations` and fake-apply + the baseline file using its own bookkeeping path. +5. **`0001_baseline` is stamped as applied without running it**. The resulting + framework row is equivalent to: + ```sql + INSERT INTO pgmigrations (name, run_on) VALUES ('0001_baseline', NOW()); + ``` +6. The embedding-dimension reconciler runs against the existing live schema. + Non-empty tables that need a dimension change still raise + `EmbeddingDimensionMismatch` rather than silently mutating live rows. +7. The framework's view now matches reality. Any post-baseline + migrations (`0002_*`, …) run against the live schema. +8. A final reconciler pass adjusts any empty vector columns introduced by + post-baseline migrations. +9. The runtime creates `schema_version` if needed and appends a row. +10. The lock is released. + +If `pgmigrations` exists with rows but does not contain `0001_baseline`, +`migrate()` throws `MigrationHistoryMismatch`. That state is not safely +inferable: running baseline DDL could touch live tables, while stamping it +blindly could hide a corrupt framework history. + +Baseline DDL does not touch existing tables. Existing rows, columns, +indexes, check constraints, and foreign keys all survive byte-for-byte. + +### Scenario C — Phase 1 install with data → Phase 2 release + +Identical to Scenario B except `schema_version` already exists. The +upgrade appends a new row rather than creating the table. + +### Enforcement + +Two test suites make the guarantee machine-checkable: + +- **`baseline-schema-equivalence.test.ts`** — the CI gate. Builds a fresh + Phase 2 schema on one DB and an upgrade-path schema (legacy `schema.sql` + fixture applied, then Phase 2 `migrate()` invoked) on another, and + asserts the schema-only structural snapshot is identical modulo the + framework-bookkeeping tables (`pgmigrations`, `schema_version`). +- **The Phase 1 data-preservation suite** — carried forward unchanged. + Seeds representative legacy rows across the core-owned tables, + snapshots them, runs `migrate()`, and asserts every row, primary key, + foreign-key relationship, JSON metadata field, timestamp, and + representative vector survives the cutover. + +## migrate() / migrationStatus() compatibility + +The public surface is signature-compatible with Phase 1. + +| Symbol | Status | +|------------------------------|-----------------------------------------------------------------------| +| `migrate(opts?)` | Unchanged signature; internals rewritten on top of `node-pg-migrate`. | +| `MigrateOptions` | Unchanged. | +| `MigrateResult.ranSchemaSql` | Semantics shifted from "we ran `schema.sql`" to "this call executed the migration runner path". The Phase 1 advisory-lock-loser path still reports `false`. | +| `MigrateResult.schemaVersion`| Unchanged. The row written/read still carries `sdkVersion`, `schemaSha256`, `appliedAt`, `notes`. | +| `MigrateResult.reconciledEmbeddingDimension` | Unchanged. | +| `MigrationLockTimeout` | Unchanged. Same constructor; same `MIGRATION_LOCK_ID`. | +| `EmbeddingDimensionMismatch` | Unchanged. | +| `migrationStatus(opts?)` | Unchanged signature; result gains two read-only fields. | +| `MigrationStatus.status` | Unchanged enum (`up_to_date` / `older_db` / `newer_db` / `unstamped` / `no_schema`). | +| `MigrationStatus.appliedMigrationCount` | **New.** Row count from `pgmigrations`. | +| `MigrationStatus.latestMigrationName` | **New.** Name of the most recent `pgmigrations` row, or `''`. | + +TypeScript catches any drift in `MigrateOptions` / `MigrateResult` at build +time; `migration-api.test.ts` exercises the runtime contract. + +The CLI (`atomicmemory-core migrate`, or `npm run migrate`) is unchanged +from the caller's perspective: exits `0` on success, prints a summary on +stdout, prints the error and exits `1` on failure. Replicas booting in +parallel are serialized by the advisory lock — running `migrate()` from +every replica's startup remains safe. + +## Adding a new migration + +Contributors add a new file under `src/db/migrations/` whenever a schema +change ships. Two strict rules: + +1. **Existing migration files are immutable.** Once a file has shipped in + any release tag, it cannot be edited, renamed, or deleted. Mutating a + shipped baseline breaks Scenario B's "stamp without running" guarantee, + because fresh and upgraded databases would diverge. The + `migration-files-no-rewrite.test.ts` CI guard rejects any PR that + touches an already-shipped filename. +2. **Filenames are strictly monotonic.** Each new file uses a higher + `NNNN_` prefix than every existing one, no gaps. The + `migration-files-monotonic.test.ts` guard enforces this. + +Workflow for adding a migration: + +```bash +# Author the migration. Phase 2 ships only SQL migrations; the build, the +# runtime fail-closed checks, and the schema-hash manifest all assume +# `.sql` files. If a future change requires conditional logic (for example, +# dim-aware DDL) the packaging path (build copy step, hash manifest, runtime +# loader) must be expanded first — do not commit a `.js`/`.ts` migration +# until that lands. +$EDITOR src/db/migrations/0002_descriptive_name.sql + +# Run the full suite locally. The baseline-equivalence and DAG sanity +# tests run as part of `npm test`. +npm test + +# Verify the migration applies cleanly against the test database. +npm run migrate:test + +# Inspect the result. +psql "$DATABASE_URL" -c 'SELECT name, run_on FROM pgmigrations ORDER BY id' +``` + +For non-additive change (drop column, rename, backfill, constraint change), +write a paired `down()` so the immediate-prior state can be restored during +the deploy window. Down migrations beyond one revision are an operator +decision, not a framework feature. + +## Provenance changelog + +`docs/db/changelog/` holds historical SQL files that documented the +schema evolution before Phase 2 reclaimed `src/db/migrations/` as the +runtime migration folder. They are references only — runtime migration +execution does not read them. Keep them for audit, but do not assume any +relationship between filenames there and rows in `pgmigrations`. + +## Design Notes + +Detailed prior-art comparisons and internal planning history live in the +private research workspace. Public operator guidance belongs in this file. diff --git a/openapi.json b/openapi.json index 55f5a0d..23e8dde 100644 --- a/openapi.json +++ b/openapi.json @@ -2,6 +2,30 @@ "components": { "parameters": {}, "schemas": { + "AdminDeleteScopeBody": { + "properties": { + "user_id": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "user_id" + ], + "type": "object" + }, + "AdminDeleteScopeResponse": { + "properties": { + "deleted": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "deleted" + ], + "type": "object" + }, "ErrorBasic": { "description": "Standard error envelope. 400 for input validation errors, 500 for uncaught exceptions.", "example": { @@ -115,6 +139,11 @@ } }, "securitySchemes": { + "adminBearerAuth": { + "description": "Send `Authorization: Bearer ` on admin-only cleanup requests. This scheme is separate from normal client auth.", + "scheme": "bearer", + "type": "http" + }, "bearerAuth": { "description": "Send `Authorization: Bearer ` on every request. The key is the deployment-wide secret configured via `CORE_API_KEY`; the middleware uses constant-time comparison.", "scheme": "bearer", @@ -133,6 +162,149 @@ }, "openapi": "3.1.0", "paths": { + "/v1/admin/scope": { + "delete": { + "description": "Mounted only when CORE_ADMIN_API_KEY and CORE_TEST_SCOPE_ALLOW_PATTERN are both configured. The server refuses user_id values that do not match the configured test-scope pattern.", + "operationId": "deleteAdminScope", + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "user_id": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "user_id" + ], + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "deleted": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "deleted" + ], + "type": "object" + } + } + }, + "description": "Number of memories deleted for the requested scope." + }, + "400": { + "content": { + "application/json": { + "schema": { + "description": "Standard error envelope. 400 for input validation errors, 500 for uncaught exceptions.", + "example": { + "error": "user_id is required" + }, + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "type": "object" + } + } + }, + "description": "Input validation error" + }, + "401": { + "content": { + "application/json": { + "schema": { + "description": "Standard error envelope. 400 for input validation errors, 500 for uncaught exceptions.", + "example": { + "error": "user_id is required" + }, + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "type": "object" + } + } + }, + "description": "Missing or invalid bearer token" + }, + "403": { + "content": { + "application/json": { + "schema": { + "description": "Standard error envelope. 400 for input validation errors, 500 for uncaught exceptions.", + "example": { + "error": "user_id is required" + }, + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "type": "object" + } + } + }, + "description": "Request is authenticated but not allowed" + }, + "500": { + "content": { + "application/json": { + "schema": { + "description": "Standard error envelope. 400 for input validation errors, 500 for uncaught exceptions.", + "example": { + "error": "user_id is required" + }, + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "type": "object" + } + } + }, + "description": "Internal server error" + } + }, + "security": [ + { + "adminBearerAuth": [] + } + ], + "summary": "Delete one allowed disposable test scope.", + "tags": [ + "Admin" + ] + } + }, "/v1/agents/conflicts": { "get": { "operationId": "listAgentConflicts", @@ -8927,6 +9099,11 @@ "description": "Caller-supplied metadata, persisted alongside the memory. Honored ONLY on /v1/memories/ingest/quick with skip_extraction=true and no workspace context — rejected with 400 on every other branch. Reserved keys (RESERVED_METADATA_KEYS in repository-types) are rejected. Max 32 KB UTF-8 serialized.", "type": "object" }, + "session_id": { + "description": "Optional thread/session identifier used to scope ingest, search, and list symmetrically.", + "maxLength": 256, + "type": "string" + }, "skip_extraction": { "type": "boolean" }, @@ -9234,6 +9411,11 @@ "description": "Caller-supplied metadata, persisted alongside the memory. Honored ONLY on /v1/memories/ingest/quick with skip_extraction=true and no workspace context — rejected with 400 on every other branch. Reserved keys (RESERVED_METADATA_KEYS in repository-types) are rejected. Max 32 KB UTF-8 serialized.", "type": "object" }, + "session_id": { + "description": "Optional thread/session identifier used to scope ingest, search, and list symmetrically.", + "maxLength": 256, + "type": "string" + }, "skip_extraction": { "type": "boolean" }, @@ -10445,6 +10627,17 @@ "format": "uuid", "type": "string" } + }, + { + "description": "Optional thread/session identifier used to scope ingest, search, and list symmetrically.", + "in": "query", + "name": "session_id", + "required": false, + "schema": { + "description": "Optional thread/session identifier used to scope ingest, search, and list symmetrically.", + "maxLength": 256, + "type": "string" + } } ], "responses": { @@ -11497,6 +11690,11 @@ ], "type": "string" }, + "session_id": { + "description": "Optional thread/session identifier used to scope ingest, search, and list symmetrically.", + "maxLength": 256, + "type": "string" + }, "skip_repair": { "type": "boolean" }, @@ -12252,6 +12450,11 @@ ], "type": "string" }, + "session_id": { + "description": "Optional thread/session identifier used to scope ingest, search, and list symmetrically.", + "maxLength": 256, + "type": "string" + }, "skip_repair": { "type": "boolean" }, diff --git a/openapi.yaml b/openapi.yaml index 7e43692..6f6c553 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1,6 +1,22 @@ components: parameters: {} schemas: + AdminDeleteScopeBody: + properties: + user_id: + minLength: 1 + type: string + required: + - user_id + type: object + AdminDeleteScopeResponse: + properties: + deleted: + minimum: 0 + type: integer + required: + - deleted + type: object ErrorBasic: description: Standard error envelope. 400 for input validation errors, 500 for uncaught exceptions. example: @@ -82,6 +98,10 @@ components: - details type: object securitySchemes: + adminBearerAuth: + description: "Send `Authorization: Bearer ` on admin-only cleanup requests. This scheme is separate from normal client auth." + scheme: bearer + type: http bearerAuth: description: "Send `Authorization: Bearer ` on every request. The key is the deployment-wide secret configured via `CORE_API_KEY`; the middleware uses constant-time comparison." scheme: bearer @@ -95,6 +115,96 @@ info: version: 1.0.0 openapi: 3.1.0 paths: + /v1/admin/scope: + delete: + description: Mounted only when CORE_ADMIN_API_KEY and CORE_TEST_SCOPE_ALLOW_PATTERN are both configured. The server refuses user_id values that do not match the configured test-scope pattern. + operationId: deleteAdminScope + requestBody: + content: + application/json: + schema: + properties: + user_id: + minLength: 1 + type: string + required: + - user_id + type: object + required: true + responses: + "200": + content: + application/json: + schema: + properties: + deleted: + minimum: 0 + type: integer + required: + - deleted + type: object + description: Number of memories deleted for the requested scope. + "400": + content: + application/json: + schema: + description: Standard error envelope. 400 for input validation errors, 500 for uncaught exceptions. + example: + error: user_id is required + properties: + error: + type: string + required: + - error + type: object + description: Input validation error + "401": + content: + application/json: + schema: + description: Standard error envelope. 400 for input validation errors, 500 for uncaught exceptions. + example: + error: user_id is required + properties: + error: + type: string + required: + - error + type: object + description: Missing or invalid bearer token + "403": + content: + application/json: + schema: + description: Standard error envelope. 400 for input validation errors, 500 for uncaught exceptions. + example: + error: user_id is required + properties: + error: + type: string + required: + - error + type: object + description: Request is authenticated but not allowed + "500": + content: + application/json: + schema: + description: Standard error envelope. 400 for input validation errors, 500 for uncaught exceptions. + example: + error: user_id is required + properties: + error: + type: string + required: + - error + type: object + description: Internal server error + security: + - adminBearerAuth: [] + summary: Delete one allowed disposable test scope. + tags: + - Admin /v1/agents/conflicts: get: operationId: listAgentConflicts @@ -6314,6 +6424,10 @@ paths: additionalProperties: {} description: Caller-supplied metadata, persisted alongside the memory. Honored ONLY on /v1/memories/ingest/quick with skip_extraction=true and no workspace context — rejected with 400 on every other branch. Reserved keys (RESERVED_METADATA_KEYS in repository-types) are rejected. Max 32 KB UTF-8 serialized. type: object + session_id: + description: Optional thread/session identifier used to scope ingest, search, and list symmetrically. + maxLength: 256 + type: string skip_extraction: type: boolean source_site: @@ -6527,6 +6641,10 @@ paths: additionalProperties: {} description: Caller-supplied metadata, persisted alongside the memory. Honored ONLY on /v1/memories/ingest/quick with skip_extraction=true and no workspace context — rejected with 400 on every other branch. Reserved keys (RESERVED_METADATA_KEYS in repository-types) are rejected. Max 32 KB UTF-8 serialized. type: object + session_id: + description: Optional thread/session identifier used to scope ingest, search, and list symmetrically. + maxLength: 256 + type: string skip_extraction: type: boolean source_site: @@ -7370,6 +7488,14 @@ paths: schema: format: uuid type: string + - description: Optional thread/session identifier used to scope ingest, search, and list symmetrically. + in: query + name: session_id + required: false + schema: + description: Optional thread/session identifier used to scope ingest, search, and list symmetrically. + maxLength: 256 + type: string responses: "200": content: @@ -8101,6 +8227,10 @@ paths: - tiered - abstract-aware type: string + session_id: + description: Optional thread/session identifier used to scope ingest, search, and list symmetrically. + maxLength: 256 + type: string skip_repair: type: boolean source_site: @@ -8624,6 +8754,10 @@ paths: - tiered - abstract-aware type: string + session_id: + description: Optional thread/session identifier used to scope ingest, search, and list symmetrically. + maxLength: 256 + type: string skip_repair: type: boolean source_site: diff --git a/package-lock.json b/package-lock.json index 6be0639..fcf66fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@huggingface/transformers": "^3.8.1", "express": "^5.1.0", "multiformats": "^13.4.1", + "node-pg-migrate": "8.0.4", "openai": "^4.80.0", "pg": "^8.18.0", "pgvector": "^0.2.0", @@ -26,6 +27,9 @@ "viem": "^2.48.11", "zod": "^4.3.6" }, + "bin": { + "atomicmemory-core": "dist/bin.js" + }, "devDependencies": { "@types/express": "^5.0.0", "@types/node": "^22.0.0", @@ -1830,29 +1834,6 @@ "@opentelemetry/api": "^1.3.0" } }, - "node_modules/@fastify/otel/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "license": "MIT", - "optional": true, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@fastify/otel/node_modules/brace-expansion": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", - "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", - "license": "MIT", - "optional": true, - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, "node_modules/@fastify/otel/node_modules/import-in-the-middle": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.6.tgz", @@ -1866,22 +1847,6 @@ "module-details-from-path": "^1.0.4" } }, - "node_modules/@fastify/otel/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "license": "BlueOak-1.0.0", - "optional": true, - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/@fastify/proxy-addr": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@fastify/proxy-addr/-/proxy-addr-5.1.0.tgz", @@ -2991,6 +2956,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@isaacs/cliui": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz", + "integrity": "sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/@isaacs/fs-minipass": { "version": "4.0.1", "license": "ISC", @@ -6377,6 +6351,30 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/array-buffer-byte-length": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz", @@ -6521,6 +6519,15 @@ "proxy-from-env": "^2.1.0" } }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, "node_modules/base64-js": { "version": "1.5.1", "funding": [ @@ -6605,6 +6612,18 @@ "version": "2.14.1", "license": "MIT" }, + "node_modules/brace-expansion": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", + "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, "node_modules/braces": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", @@ -6721,6 +6740,20 @@ "license": "MIT", "optional": true }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/clone-regexp": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/clone-regexp/-/clone-regexp-3.0.0.tgz", @@ -6737,6 +6770,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, "node_modules/combined-stream": { "version": "1.0.8", "license": "MIT", @@ -7100,6 +7151,12 @@ "version": "1.1.1", "license": "MIT" }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, "node_modules/encodeurl": { "version": "2.0.0", "license": "MIT", @@ -7306,6 +7363,15 @@ "@esbuild/win32-x64": "0.27.7" } }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/escape-html": { "version": "1.0.3", "license": "MIT" @@ -7812,6 +7878,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "4.0.5", "license": "MIT", @@ -7978,6 +8060,15 @@ "node": ">= 0.4" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-intrinsic": { "version": "1.3.0", "license": "MIT", @@ -8061,6 +8152,30 @@ "license": "MIT", "optional": true }, + "node_modules/glob": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.1.0.tgz", + "integrity": "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "BlueOak-1.0.0", + "dependencies": { + "foreground-child": "^3.3.1", + "jackspeak": "^4.1.1", + "minimatch": "^10.1.1", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -8700,6 +8815,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/is-generator-function": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz", @@ -9464,6 +9588,21 @@ "uint8arrays": "^5.1.0" } }, + "node_modules/jackspeak": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.2.3.tgz", + "integrity": "sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^9.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/jose": { "version": "6.2.3", "license": "MIT", @@ -9912,6 +10051,15 @@ "version": "5.3.2", "license": "Apache-2.0" }, + "node_modules/lru-cache": { + "version": "11.3.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.6.tgz", + "integrity": "sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, "node_modules/magic-string": { "version": "0.30.21", "dev": true, @@ -10055,6 +10203,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/minimist": { "version": "1.2.8", "devOptional": true, @@ -10243,6 +10406,31 @@ "node": ">= 6.13.0" } }, + "node_modules/node-pg-migrate": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/node-pg-migrate/-/node-pg-migrate-8.0.4.tgz", + "integrity": "sha512-HTlJ6fOT/2xHhAUtsqSN85PGMAqSbfGJNRwQF8+ZwQ1+sVGNUTl/ZGEshPsOI3yV22tPIyHXrKXr3S0JxeYLrg==", + "license": "MIT", + "dependencies": { + "glob": "~11.1.0", + "yargs": "~17.7.0" + }, + "bin": { + "node-pg-migrate": "bin/node-pg-migrate.js" + }, + "engines": { + "node": ">=20.11.0" + }, + "peerDependencies": { + "@types/pg": ">=6.0.0 <9.0.0", + "pg": ">=4.3.0 <9.0.0" + }, + "peerDependenciesMeta": { + "@types/pg": { + "optional": true + } + } + }, "node_modules/object-assign": { "version": "4.1.1", "license": "MIT", @@ -10596,6 +10784,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, "node_modules/parseurl": { "version": "1.3.3", "license": "MIT", @@ -10623,6 +10817,22 @@ "node": ">=8" } }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/path-to-regexp": { "version": "8.4.2", "license": "MIT", @@ -11284,6 +11494,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "license": "MIT", @@ -11838,6 +12057,18 @@ "dev": true, "license": "ISC" }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/simple-concat": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", @@ -11967,6 +12198,20 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/string.prototype.trim": { "version": "1.2.10", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.10.tgz", @@ -12026,6 +12271,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", @@ -13095,6 +13352,23 @@ "node": ">=8" } }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "license": "ISC" @@ -13165,6 +13439,15 @@ "node": ">=0.4" } }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, "node_modules/yallist": { "version": "5.0.0", "license": "BlueOak-1.0.0", @@ -13185,6 +13468,33 @@ "url": "https://github.com/sponsors/eemeli" } }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/yocto-queue": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", diff --git a/package.json b/package.json index 9bf5026..5ea257c 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "scripts": { "dev": "dotenv -e .env -- tsx watch src/server.ts", "start": "dotenv -e .env -- tsx src/server.ts", - "build": "rm -rf dist && tsc -p tsconfig.build.json && cp src/db/schema.sql dist/db/schema.sql", + "build": "rm -rf dist && tsc -p tsconfig.build.json && mkdir -p dist/db/migrations && cp src/db/migrations/*.sql dist/db/migrations/ && tsx scripts/generate-schema-hash.ts", "generate:openapi": "tsx scripts/generate-openapi.ts", "check:openapi": "npm run generate:openapi && git diff --exit-code openapi.yaml openapi.json", "prepublishOnly": "npm run generate:openapi && npm run build", @@ -89,6 +89,7 @@ "@huggingface/transformers": "^3.8.1", "express": "^5.1.0", "multiformats": "^13.4.1", + "node-pg-migrate": "8.0.4", "openai": "^4.80.0", "pg": "^8.18.0", "pgvector": "^0.2.0", diff --git a/scripts/META_FACT_FILTER_DEPLOY.md b/scripts/META_FACT_FILTER_DEPLOY.md new file mode 100644 index 0000000..f4fcec9 --- /dev/null +++ b/scripts/META_FACT_FILTER_DEPLOY.md @@ -0,0 +1,230 @@ +# Meta-fact filter — deploy runbook + +This change ships two things: + +1. **A runtime filter** (`src/services/meta-fact-filter.ts`) that drops + extraction-style "meta-facts" before they reach the memories table. +2. **A cleanup script** (`scripts/cleanup-meta-facts.ts`) that + retroactively soft-deletes already-stored meta-facts. + +Both target the failure class observed in the Filecoin partner demo +(atomicmem.filecoin.cloud): meta-facts like `"The user asked for the +user's name."` and `"As of , X is a term mentioned in the +conversation."` outranking durable user facts on similarity search. + +Evidence: AlignBench v0 +(`am-sdk-internal:benchmarks/alignbench/RESULTS.md`). + +--- + +## TL;DR for the on-call operator + +```bash +# 1) Deploy the new core build. +# 2) Verify the filter is firing in production logs: +grep '\[meta-fact-filter\] dropped' | head +# 3) Run the dry-run cleanup against the partner DB to see scope: +DATABASE_URL=... pnpm dotenv -e .env -- tsx scripts/cleanup-meta-facts.ts --dry-run +# 4) If the dry-run looks right, apply: +DATABASE_URL=... pnpm dotenv -e .env -- tsx scripts/cleanup-meta-facts.ts --apply +# 5) Smoke the demo: a fresh "what is my name?" should now find the +# durable fact, not the dropped meta-fact. +``` + +--- + +## Pre-deploy + +### Config check + +The filter has one environment variable, all-or-nothing: + +| Var | Default | Meaning | +|---|---|---| +| `ATOMICMEMORY_META_FACT_FILTER` | unset (= on) | Set to `off` / `false` / `0` / `disabled` to disable the runtime filter. Disable is **for incident response only**. | + +You do not need to set anything. The filter is on by default. + +### Pre-deploy verification + +On the build artifact (before swapping traffic): + +```bash +# Unit + integration tests +pnpm test src/services/__tests__/meta-fact-filter.test.ts +pnpm test src/services/__tests__/extraction-meta-fact-integration.test.ts +# Typecheck +npx tsc --noEmit +``` + +Expected: 58 tests green, typecheck clean (exit 0). + +### Dry-run the migration against the target DB + +```bash +DATABASE_URL='' \ + pnpm dotenv -e .env -- tsx scripts/cleanup-meta-facts.ts --dry-run +``` + +Output you want to see: + +``` +[cleanup] mode=dry-run user_id=all batch_size=500 limit=∞ +[cleanup] audit log: /path/to/cleanup-meta-facts-2026-05-14T....jsonl +... +[cleanup] DONE +[cleanup] scanned +[cleanup] matched (%) +[cleanup] (dry-run; nothing deleted. Pass --apply to soft-delete.) +``` + +Inspect the audit log: + +```bash +head -5 cleanup-meta-facts-*.jsonl +# Each line is { id, user_id, content, created_at, dropped_at, mode: 'dry-run' } +``` + +If the matched rows are **not** meta-facts (something durable slipped +into the pattern set), **do not apply**. File an issue with a sample row +and ping the author. The regex set is in +`src/services/meta-fact-filter.ts:DEFAULT_META_FACT_PATTERNS` — patterns +are anchored at line-start and intentionally narrow. + +--- + +## Deploy + +Standard deploy. No DB migration, no breaking API change. The filter is +a pure post-extraction step and adds <1ms to `extractFacts()`. + +### Post-deploy verification + +#### 1. Filter is firing in production logs + +```bash +grep '\[meta-fact-filter\] dropped' | tail +``` + +Expected line shape: + +``` +[meta-fact-filter] dropped pattern=0 len=42 source=extract +``` + +If you see nothing for several minutes of ingest traffic, either no +meta-facts are being emitted (good — the prompt change may have helped +upstream) or the filter is silently disabled. Verify: + +```bash +# In the running process, the env var should be unset or 'on'-ish +env | grep ATOMICMEMORY_META_FACT_FILTER +``` + +#### 2. Synthetic conversation produces zero meta-facts + +Use whatever you have for an ingest smoke. The conversation `"User: +what's my name? Assistant: I don't know yet."` is a common meta-fact +inducer. Confirm no fact like `"The user asked for the user's name."` +lands in the memories table. + +#### 3. Drop counters expose volume + +The filter maintains process-lifetime counters by pattern. There is no +HTTP endpoint exposing them yet (future work); for one-off audits the +counters are reachable from a Node REPL attached to the process: + +```js +const { getMetaFactDropStats } = require('./dist/services/meta-fact-filter.js'); +console.log(getMetaFactDropStats()); +// → { total: 142, byPattern: [98, 30, 2, 8, 4] } +``` + +A non-trivial `byPattern[i]` for an index whose pattern doesn't match +the real meta-fact families you're seeing in logs is the leading +indicator that the pattern set has drifted. + +--- + +## Cleanup migration — applying + +Once the runtime filter is verified in production and you've confirmed +the dry-run is correct on the target DB: + +```bash +DATABASE_URL='' \ + pnpm dotenv -e .env -- tsx scripts/cleanup-meta-facts.ts --apply +``` + +This **soft-deletes** matching rows (`UPDATE memories SET deleted_at = +NOW()`). Hard delete is intentionally not supported by this script — +the audit log plus the soft-delete row is the reversible record. + +### Targeted runs (per user) + +```bash +... -- tsx scripts/cleanup-meta-facts.ts --apply --user-id +``` + +### Rollback (un-soft-delete a specific run) + +The audit log gives you the IDs. If a run is wrong: + +```sql +UPDATE memories +SET deleted_at = NULL +WHERE id = ANY($1::uuid[]); +``` + +Pass the IDs from the JSONL audit log: + +```bash +jq -r '.id' cleanup-meta-facts-.jsonl > undo-ids.txt +``` + +--- + +## Runtime rollback + +If the live filter is incorrectly dropping durable facts: + +```bash +# Set on the running core process and restart: +export ATOMICMEMORY_META_FACT_FILTER=off +``` + +The filter becomes a no-op immediately. Existing extraction pipeline is +otherwise unaffected. + +The cleanup-script soft-deletes are **independent** of the runtime flag +— flipping the runtime flag does not un-soft-delete. Use the SQL +rollback above for that. + +--- + +## What this fix does NOT solve + +Be precise with stakeholders on what's in scope: + +| In scope | Out of scope | +|---|---| +| Stop new meta-facts from being written. | Stronger temporal handling (`"where do I live now?"` vs `"where did I live?"`). That's a structured-temporal-state problem (see core's `temporal-classifier.ts`, not wired to SDK recall yet). | +| Clean up existing meta-fact pollution. | New meta-fact families the regex doesn't catch — telemetry counters are the leading indicator; expand `DEFAULT_META_FACT_PATTERNS` when new shapes appear. | +| Lift recall@1 by ~0.03–0.05 on the targeted failure class (AlignBench). | A general retrieval-quality leap. The embedding model is already near-optimal for this size; bigger sentence-transformers don't move the needle (see `am-sdk-internal:benchmarks/alignbench/runs/modal-ablation.json`). | +| Survive disabling without code changes. | Hot-reload of the regex set without redeploy. | + +--- + +## Files touched by this change + +| File | Role | +|---|---| +| `src/services/meta-fact-filter.ts` | Filter primitive, env-flag resolution, drop counters, structured logging. | +| `src/services/extraction.ts` | Adds prompt rule (abstract guidance, no example phrasings). Calls `filterMetaFacts` as the final post-process step. | +| `src/services/__tests__/meta-fact-filter.test.ts` | 52 unit tests. | +| `src/services/__tests__/extraction-meta-fact-integration.test.ts` | 6 integration tests exercising the wired pipeline with mocked LLM. | +| `scripts/cleanup-meta-facts.ts` | Migration: `--dry-run` (default) / `--apply`, audit JSONL log, `--user-id` scope, idempotent. | +| `scripts/META_FACT_FILTER_DEPLOY.md` | This file. | + +The SDK ships a complementary post-retrieval filter for deployments +that haven't yet upgraded core. See `am-sdk-internal#18`. diff --git a/scripts/cleanup-meta-facts.ts b/scripts/cleanup-meta-facts.ts new file mode 100644 index 0000000..da52af0 --- /dev/null +++ b/scripts/cleanup-meta-facts.ts @@ -0,0 +1,205 @@ +#!/usr/bin/env tsx +/** + * scripts/cleanup-meta-facts.ts + * + * Retroactive cleanup of meta-fact pollution in existing `memories` rows. + * + * The extraction-time filter added in this branch stops *new* meta-facts + * from entering the store. Deployments that ingested data before the + * filter shipped (e.g. the Filecoin partner demo) still have polluted + * rows that outrank durable user facts on similarity search. This + * script identifies and soft-deletes those rows using the same pattern + * set as the runtime filter, so behaviour is consistent post-cleanup. + * + * Safety contract: + * - Defaults to --dry-run. Apply mode requires explicit --apply. + * - Soft-delete only (sets deleted_at = NOW()). No hard delete; the + * audit log is the source of truth and the change is reversible by + * clearing deleted_at on the same ids. + * - Idempotent. Already-deleted rows are skipped. + * - Optional --user-id scope for surgical runs. + * - Writes a JSONL audit log to ./cleanup-meta-facts-.jsonl so + * operators have a verifiable record of every dropped row. + * + * Usage: + * pnpm dotenv -e .env -- tsx scripts/cleanup-meta-facts.ts --dry-run + * pnpm dotenv -e .env -- tsx scripts/cleanup-meta-facts.ts --apply + * pnpm dotenv -e .env -- tsx scripts/cleanup-meta-facts.ts --apply --user-id + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; +import { pool } from '../src/db/pool.js'; +import { getMetaFactDropStats, isMetaFactStatement } from '../src/services/meta-fact-filter.js'; + +interface CliOptions { + apply: boolean; + userId: string | null; + batchSize: number; + limit: number | null; + out: string; +} + +function parseCliOptions(): CliOptions { + const { values } = parseArgs({ + options: { + apply: { type: 'boolean', default: false }, + 'dry-run': { type: 'boolean', default: false }, + 'user-id': { type: 'string' }, + 'batch-size': { type: 'string', default: '500' }, + limit: { type: 'string' }, + out: { type: 'string' }, + help: { type: 'boolean', short: 'h', default: false }, + }, + strict: true, + }); + + if (values.help) { + process.stdout.write(`Usage: cleanup-meta-facts.ts [options] + --dry-run Default. Report rows that would be dropped. + --apply Soft-delete matching rows. + --user-id Restrict to a single user_id. + --batch-size Rows per scan batch (default 500). + --limit Stop after scanning n rows (testing aid). + --out Audit log path (default ./cleanup-meta-facts-.jsonl). + --help, -h Show this message. +`); + process.exit(0); + } + + const apply = Boolean(values.apply); + const userId = (values['user-id'] as string | undefined) ?? null; + const batchSize = Number(values['batch-size']); + if (!Number.isInteger(batchSize) || batchSize <= 0) { + throw new Error(`--batch-size must be a positive integer, got "${values['batch-size']}"`); + } + const limit = values.limit ? Number(values.limit) : null; + if (limit !== null && (!Number.isInteger(limit) || limit <= 0)) { + throw new Error(`--limit must be a positive integer, got "${values.limit}"`); + } + const out = + (values.out as string | undefined) ?? + path.resolve(process.cwd(), `cleanup-meta-facts-${new Date().toISOString().replace(/[:.]/g, '-')}.jsonl`); + + return { apply, userId, batchSize, limit, out }; +} + +interface MemoryRow { + id: string; + user_id: string; + content: string; + created_at: Date; +} + +async function* scanMemories(opts: CliOptions): AsyncGenerator { + let cursor: string | null = null; + let scanned = 0; + while (true) { + const params: unknown[] = []; + const whereParts: string[] = ['deleted_at IS NULL']; + if (opts.userId) { + params.push(opts.userId); + whereParts.push(`user_id = $${params.length}`); + } + if (cursor) { + params.push(cursor); + whereParts.push(`id > $${params.length}`); + } + params.push(opts.batchSize); + const sql = ` + SELECT id, user_id, content, created_at + FROM memories + WHERE ${whereParts.join(' AND ')} + ORDER BY id ASC + LIMIT $${params.length} + `; + const result = await pool.query(sql, params); + if (result.rows.length === 0) return; + yield result.rows; + cursor = result.rows[result.rows.length - 1].id; + scanned += result.rows.length; + if (opts.limit !== null && scanned >= opts.limit) return; + } +} + +async function softDeleteIds(ids: readonly string[]): Promise { + if (ids.length === 0) return 0; + const result = await pool.query( + `UPDATE memories SET deleted_at = NOW() + WHERE id = ANY($1::uuid[]) AND deleted_at IS NULL`, + [ids], + ); + return result.rowCount ?? 0; +} + +interface DropRecord { + id: string; + user_id: string; + content: string; + created_at: string; + dropped_at: string; + mode: 'dry-run' | 'apply'; +} + +async function main(): Promise { + const opts = parseCliOptions(); + const mode: 'dry-run' | 'apply' = opts.apply ? 'apply' : 'dry-run'; + process.stdout.write(`[cleanup] mode=${mode} user_id=${opts.userId ?? 'all'} batch_size=${opts.batchSize} limit=${opts.limit ?? '∞'}\n`); + process.stdout.write(`[cleanup] audit log: ${opts.out}\n`); + + const fd = fs.openSync(opts.out, 'a'); + let scanned = 0; + let matched = 0; + let softDeleted = 0; + const startedAt = Date.now(); + + try { + for await (const batch of scanMemories(opts)) { + scanned += batch.length; + const matches = batch.filter((r) => isMetaFactStatement(r.content)); + matched += matches.length; + for (const row of matches) { + const record: DropRecord = { + id: row.id, + user_id: row.user_id, + content: row.content, + created_at: new Date(row.created_at).toISOString(), + dropped_at: new Date().toISOString(), + mode, + }; + fs.writeSync(fd, JSON.stringify(record) + '\n'); + } + if (opts.apply && matches.length > 0) { + softDeleted += await softDeleteIds(matches.map((m) => m.id)); + } + if (scanned % (opts.batchSize * 10) === 0) { + process.stdout.write(`[cleanup] scanned=${scanned} matched=${matched}${opts.apply ? ` soft_deleted=${softDeleted}` : ''}\n`); + } + } + } finally { + fs.closeSync(fd); + } + + const wallSeconds = ((Date.now() - startedAt) / 1000).toFixed(1); + process.stdout.write(`\n[cleanup] DONE\n`); + process.stdout.write(`[cleanup] scanned ${scanned}\n`); + process.stdout.write(`[cleanup] matched ${matched} (${scanned ? ((matched / scanned) * 100).toFixed(2) : '0.00'}%)\n`); + if (opts.apply) { + process.stdout.write(`[cleanup] soft-deleted ${softDeleted}\n`); + if (softDeleted !== matched) { + process.stdout.write(`[cleanup] WARNING: matched ${matched} but soft-deleted ${softDeleted} — some rows raced with concurrent deletes\n`); + } + } else { + process.stdout.write(`[cleanup] (dry-run; nothing deleted. Pass --apply to soft-delete.)\n`); + } + process.stdout.write(`[cleanup] wall=${wallSeconds}s\n`); + process.stdout.write(`[cleanup] audit log: ${opts.out}\n`); + + await pool.end(); +} + +main().catch((err) => { + process.stderr.write(`[cleanup] FAILED: ${err instanceof Error ? err.stack ?? err.message : String(err)}\n`); + process.exit(1); +}); diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 26c9bbc..5cf9dd1 100644 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -102,6 +102,39 @@ start_embedded_postgres() { export DATABASE_URL="postgresql://${EMBEDDED_POSTGRES_USER}@127.0.0.1:${EMBEDDED_POSTGRES_PORT}/${EMBEDDED_POSTGRES_DB}" } +run_migrations() { + case "${ATOMICMEMORY_RUN_MIGRATIONS_ON_STARTUP:-true}" in + true|1|yes) + ;; + false|0|no) + log "Skipping startup migrations because ATOMICMEMORY_RUN_MIGRATIONS_ON_STARTUP=false" + return + ;; + *) + log "ATOMICMEMORY_RUN_MIGRATIONS_ON_STARTUP must be true or false" + exit 1 + ;; + esac + + local migrate_args=() + if [ -n "${MIGRATION_LOCK_TIMEOUT_MS:-}" ]; then + case "$MIGRATION_LOCK_TIMEOUT_MS" in + ''|*[!0-9]*) + log "MIGRATION_LOCK_TIMEOUT_MS must be a positive integer" + exit 1 + ;; + esac + if [ "$MIGRATION_LOCK_TIMEOUT_MS" -le 0 ]; then + log "MIGRATION_LOCK_TIMEOUT_MS must be a positive integer" + exit 1 + fi + migrate_args+=("--lock-timeout-ms=${MIGRATION_LOCK_TIMEOUT_MS}") + fi + + log "Running migrations..." + gosu appuser ./node_modules/.bin/tsx src/db/migrate.ts "${migrate_args[@]}" +} + configure_local_defaults if [ "${DATABASE_URL:-embedded}" = "embedded" ]; then @@ -110,8 +143,7 @@ else log "Using external DATABASE_URL" fi -log "Running migrations..." -gosu appuser ./node_modules/.bin/tsx src/db/migrate.ts +run_migrations log "Starting AtomicMemory Core..." gosu appuser "$@" & diff --git a/scripts/generate-schema-hash.ts b/scripts/generate-schema-hash.ts new file mode 100644 index 0000000..ba938e4 --- /dev/null +++ b/scripts/generate-schema-hash.ts @@ -0,0 +1,148 @@ +#!/usr/bin/env tsx +/** + * @file Build-time migrations hash manifest generator. + * + * Walks `dist/db/migrations/` (populated by `npm run build` immediately + * before this script), reads each `.sql` file in lexical order, and writes + * a stable manifest enumerating both the per-file digests and an aggregate + * digest. Phase 2 ships a directory of frozen migration files instead of + * a single `schema.sql`, so the manifest now describes that directory + * rather than a single file. + * + * **Aggregate hash shape (audit-tightened):** `schemaSha256` is the + * SHA-256 of a canonical `\t\n` manifest text, + * lexically ordered. Filename identity participates in the aggregate so a + * future rename or reordering cannot accidentally collide with the + * pre-rename hash even when the underlying SQL bytes are unchanged. The + * runtime's `migration-schema.ts:buildAppliedSql()` computes the same + * canonical text and the same digest; the build manifest and the runtime + * fingerprint are kept in lockstep. + * + * **Fail-closed policy:** Phase 2 only ships SQL migration files. This + * script throws when `dist/db/migrations/` is missing, contains zero + * `.sql` files, lacks the frozen `0001_baseline.sql`, or contains any + * empty `.sql` file. A misbuilt package that previously would have + * shipped an empty manifest now fails the build instead. + * + * The manifest is intentionally metadata-only (no timestamps or machine + * info) so identical migration bytes always produce identical package + * output. + * + * Run via `npm run build`. + */ + +import { createHash } from 'node:crypto'; +import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; + +interface MigrationFileEntry { + filename: string; + sha256: string; + bytes: number; +} + +interface MigrationsManifest { + migrationsPath: 'db/migrations'; + /** SHA-256 of the canonical "\t\n" manifest text. */ + schemaSha256: string; + files: MigrationFileEntry[]; +} + +const DIST_MIGRATIONS_DIR = 'dist/db/migrations'; +const MANIFEST_PATH = 'dist/db/schema-sha256.json'; +const BASELINE_FILENAME = '0001_baseline.sql'; + +function sha256Hex(bytes: Buffer | string): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function listMigrationFilenames(dir: string): string[] { + let entries: string[]; + try { + entries = readdirSync(dir); + } catch (err) { + throw new Error( + `[generate-schema-hash] migrations directory is missing at ${dir}. ` + + `Build must copy src/db/migrations/*.sql into dist/db/migrations/ before this script runs.`, + { cause: err as Error }, + ); + } + const sqlFiles = entries.filter((name) => name.endsWith('.sql')).sort(); + if (sqlFiles.length === 0) { + throw new Error( + `[generate-schema-hash] migrations directory ${dir} contains zero .sql files. ` + + `Phase 2 packages MUST ship at least ${BASELINE_FILENAME}.`, + ); + } + if (!sqlFiles.includes(BASELINE_FILENAME)) { + throw new Error( + `[generate-schema-hash] frozen baseline ${BASELINE_FILENAME} is missing from ${dir}. ` + + `The Phase 2 cutover contract requires it in every shipped build.`, + ); + } + return sqlFiles; +} + +function buildFileEntry(absoluteDir: string, filename: string): MigrationFileEntry { + const bytes = readFileSync(join(absoluteDir, filename)); + if (bytes.byteLength === 0) { + throw new Error( + `[generate-schema-hash] migration file ${filename} in ${absoluteDir} is empty. ` + + `Refusing to write a manifest that would stamp the SHA without applying any DDL.`, + ); + } + return { + filename, + sha256: sha256Hex(bytes), + bytes: bytes.byteLength, + }; +} + +/** + * Canonical, deterministic text whose sha256 is the aggregate `schemaSha256`. + * One line per file, lexically ordered, `\t\n`. Filename + * participates so a rename cannot silently keep the old digest. + */ +function canonicalManifestText(files: ReadonlyArray): string { + return files.map((entry) => `${entry.filename}\t${entry.sha256}\n`).join(''); +} + +function buildManifest(absoluteDir: string): MigrationsManifest { + const filenames = listMigrationFilenames(absoluteDir); + const files = filenames.map((name) => buildFileEntry(absoluteDir, name)); + return { + migrationsPath: 'db/migrations', + schemaSha256: sha256Hex(canonicalManifestText(files)), + files, + }; +} + +function writeManifest(manifest: MigrationsManifest): void { + const outputPath = resolve(MANIFEST_PATH); + mkdirSync(dirname(outputPath), { recursive: true }); + writeFileSync(outputPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); +} + +function generateSchemaHash(): void { + const absoluteDir = resolve(DIST_MIGRATIONS_DIR); + // statSync up front so the error message points at the directory we tried, + // not at the first readdirSync inside listMigrationFilenames (which already + // throws with a clear message — this is just defense in depth for `dist/`). + try { + statSync(absoluteDir); + } catch (err) { + throw new Error( + `[generate-schema-hash] expected ${absoluteDir} to exist (created by the build step).`, + { cause: err as Error }, + ); + } + const manifest = buildManifest(absoluteDir); + writeManifest(manifest); + console.log( + `Wrote ${resolve(MANIFEST_PATH)} (` + + `${manifest.files.length} migration file(s), ` + + `aggregate sha=${manifest.schemaSha256.slice(0, 12)}…)`, + ); +} + +generateSchemaHash(); diff --git a/src/__tests__/config-env.test.ts b/src/__tests__/config-env.test.ts index f5dd9e3..be97252 100644 --- a/src/__tests__/config-env.test.ts +++ b/src/__tests__/config-env.test.ts @@ -12,6 +12,8 @@ const trackedEnvNames = [ 'EMBEDDING_DIMENSIONS', 'DATABASE_URL', 'CORE_API_KEY', + 'CORE_ADMIN_API_KEY', + 'CORE_TEST_SCOPE_ALLOW_PATTERN', 'STORAGE_KEY_HMAC_SECRET', 'RAW_STORAGE_DEPLOYMENT_ENV', 'OPENAI_API_KEY', @@ -86,4 +88,25 @@ describe('config env loading', () => { expect(config.llmModel).toBe('sonnet'); }); + + it('loads optional admin cleanup endpoint config', async () => { + process.env.CORE_ADMIN_API_KEY = 'test-admin-key'; + process.env.CORE_TEST_SCOPE_ALLOW_PATTERN = '^(smoke-|docker-).+'; + vi.resetModules(); + + const { config } = await import('../config.js'); + + expect(config.coreAdminApiKey).toBe('test-admin-key'); + expect(config.coreTestScopeAllowPattern).toBe('^(smoke-|docker-).+'); + }); + + it('rejects invalid admin cleanup scope regex', async () => { + process.env.CORE_ADMIN_API_KEY = 'test-admin-key'; + process.env.CORE_TEST_SCOPE_ALLOW_PATTERN = '['; + vi.resetModules(); + + await expect(import('../config.js')).rejects.toThrow( + 'CORE_TEST_SCOPE_ALLOW_PATTERN must be a valid JavaScript regular expression', + ); + }); }); diff --git a/src/__tests__/deployment-config.test.ts b/src/__tests__/deployment-config.test.ts index 4bc327e..25af0e2 100644 --- a/src/__tests__/deployment-config.test.ts +++ b/src/__tests__/deployment-config.test.ts @@ -136,6 +136,12 @@ describe('deployment configuration', () => { expect(readEnvExample()).toContain('EMBEDDING_DIMENSIONS'); }); + it('documents Docker startup migration controls', () => { + const envExample = readEnvExample(); + expect(envExample).toContain('ATOMICMEMORY_RUN_MIGRATIONS_ON_STARTUP'); + expect(envExample).toContain('MIGRATION_LOCK_TIMEOUT_MS'); + }); + it('documents Voyage embedding lane env vars', () => { const envExample = readEnvExample(); expect(envExample).toContain('VOYAGE_API_KEY'); @@ -145,7 +151,7 @@ describe('deployment configuration', () => { }); describe('Dockerfile', () => { - it('uses the entrypoint to run migrations before server start', () => { + it('uses the entrypoint to coordinate migrations before server start', () => { const dockerfile = readDockerfile(); const entrypointLine = dockerfile.split('\n').find((l) => l.startsWith('ENTRYPOINT')); const cmdLine = dockerfile.split('\n').find((l) => l.startsWith('CMD')); @@ -156,6 +162,17 @@ describe('deployment configuration', () => { expect(cmdLine).toContain('server'); }); + it('entrypoint exposes startup migration controls for rolling deploys', () => { + const entrypoint = readFileSync(resolve(ROOT, 'scripts/docker-entrypoint.sh'), 'utf-8'); + + expect(entrypoint).toContain('ATOMICMEMORY_RUN_MIGRATIONS_ON_STARTUP'); + expect(entrypoint).toContain('MIGRATION_LOCK_TIMEOUT_MS'); + expect(entrypoint).toContain('--lock-timeout-ms='); + expect(entrypoint.indexOf('run_migrations')).toBeLessThan( + entrypoint.indexOf('Starting AtomicMemory Core'), + ); + }); + it('defaults DATABASE_URL to embedded mode for single-container docker run', () => { const dockerfile = readDockerfile(); diff --git a/src/__tests__/memory-route-config-override.test.ts b/src/__tests__/memory-route-config-override.test.ts index dcdb8de..335c8c8 100644 --- a/src/__tests__/memory-route-config-override.test.ts +++ b/src/__tests__/memory-route-config-override.test.ts @@ -9,7 +9,7 @@ * 2. Present override → all three headers emitted * (applied=true, hash=sha256:, keys=sorted csv). * 3. Search routes forward `effectiveConfig` via the scopedSearch - * options bag; ingest routes forward it as the trailing arg. + * options bag; ingest routes forward it through the named input. * 4. Unknown override keys do NOT 400 (the schema is permissive so * new RuntimeConfig fields don't require a schema landing to be * usable). They are carried through on the effective config AND @@ -191,7 +191,7 @@ describe('POST /memories/* — per-request config_override', () => { expect(options.retrievalOptions.relevanceThreshold).toBe(0.7); }); - it('POST /ingest with override → headers + trailing effectiveConfig arg', async () => { + it('POST /ingest with override → headers + named effectiveConfig input', async () => { ingest.mockResolvedValueOnce({ episodeId: 'ep', factsExtracted: 0, memoriesStored: 0, memoriesUpdated: 0, memoriesDeleted: 0, memoriesSkipped: 0, storedMemoryIds: [], updatedMemoryIds: [], @@ -206,12 +206,13 @@ describe('POST /memories/* — per-request config_override', () => { expect(res.headers.get('X-Atomicmem-Config-Override-Keys')).toBe('chunkedExtractionEnabled,ingestTraceEnabled'); expect((await res.clone().json()).ingest_trace_id).toBe('ingest-trace-1'); const call = ingest.mock.calls[0]!; - const effectiveConfig = call[5] as { chunkedExtractionEnabled: boolean; ingestTraceEnabled: boolean }; + const input = call[0] as { effectiveConfig: { chunkedExtractionEnabled: boolean; ingestTraceEnabled: boolean } }; + const effectiveConfig = input.effectiveConfig; expect(effectiveConfig.chunkedExtractionEnabled).toBe(true); expect(effectiveConfig.ingestTraceEnabled).toBe(true); }); - it('POST /ingest/quick with override → headers + trailing effectiveConfig arg', async () => { + it('POST /ingest/quick with override → headers + named effectiveConfig input', async () => { const res = await postJson(`/memories/ingest/quick`, { user_id: 'u', conversation: 'hi', source_site: 's', config_override: { entropyGateEnabled: false, fastAudnEnabled: true }, @@ -219,7 +220,8 @@ describe('POST /memories/* — per-request config_override', () => { expect(res.status).toBe(200); expect(res.headers.get('X-Atomicmem-Config-Override-Keys')).toBe('entropyGateEnabled,fastAudnEnabled'); const call = quickIngest.mock.calls[0]!; - const effectiveConfig = call[5] as { entropyGateEnabled: boolean; fastAudnEnabled: boolean }; + const input = call[0] as { effectiveConfig: { entropyGateEnabled: boolean; fastAudnEnabled: boolean } }; + const effectiveConfig = input.effectiveConfig; expect(effectiveConfig.entropyGateEnabled).toBe(false); expect(effectiveConfig.fastAudnEnabled).toBe(true); }); diff --git a/src/__tests__/memory-route-service-forwarding.test.ts b/src/__tests__/memory-route-service-forwarding.test.ts new file mode 100644 index 0000000..680157b --- /dev/null +++ b/src/__tests__/memory-route-service-forwarding.test.ts @@ -0,0 +1,199 @@ +/** + * Route-to-service forwarding tests for object-shaped MemoryService calls. + * + * These guard against regressions where route handlers reintroduce + * positional `undefined, undefined, sessionId` plumbing after the + * service facade moved high-risk write/list calls to named inputs. + */ + +import express from 'express'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { type BootedApp, bindEphemeral } from '../app/bind-ephemeral.js'; +import { createMemoryRouter } from '../routes/memories.js'; +import type { MemoryService } from '../services/memory-service.js'; + +const EMPTY_INGEST = { + episodeId: 'ep', + factsExtracted: 0, + memoriesStored: 0, + memoriesUpdated: 0, + memoriesDeleted: 0, + memoriesSkipped: 0, + storedMemoryIds: [], + updatedMemoryIds: [], + memoryIds: [], + linksCreated: 0, + compositesCreated: 0, +}; + +describe('memory routes — object-shaped service forwarding', () => { + let booted: BootedApp; + const mockScopedSearch = vi.fn(); + const mockIngest = vi.fn(); + const mockQuickIngest = vi.fn(); + const mockStoreVerbatim = vi.fn(); + const mockWorkspaceIngest = vi.fn(); + const mockList = vi.fn(); + const mockScopedList = vi.fn(); + const service = { + scopedSearch: mockScopedSearch, + ingest: mockIngest, + quickIngest: mockQuickIngest, + storeVerbatim: mockStoreVerbatim, + workspaceIngest: mockWorkspaceIngest, + list: mockList, + scopedList: mockScopedList, + } as unknown as MemoryService; + + beforeAll(async () => { + const app = express(); + app.use(express.json()); + app.use('/memories', createMemoryRouter(service)); + booted = await bindEphemeral(app); + }); + + beforeEach(() => { + vi.clearAllMocks(); + mockIngest.mockResolvedValue(EMPTY_INGEST); + mockQuickIngest.mockResolvedValue(EMPTY_INGEST); + mockStoreVerbatim.mockResolvedValue(EMPTY_INGEST); + mockWorkspaceIngest.mockResolvedValue(EMPTY_INGEST); + mockList.mockResolvedValue([]); + mockScopedList.mockResolvedValue([]); + }); + + afterAll(async () => { + await booted.close(); + }); + + it('POST /ingest forwards a named full-ingest input', async () => { + const response = await postJson(booted, '/memories/ingest', { + user_id: 'u', + conversation: 'hello', + source_site: 'site', + source_url: 'https://example.test/full', + session_id: 'thread-full', + }); + + expect(response.status).toBe(200); + expect(mockIngest).toHaveBeenCalledWith({ + userId: 'u', + conversationText: 'hello', + sourceSite: 'site', + sourceUrl: 'https://example.test/full', + effectiveConfig: undefined, + sessionId: 'thread-full', + }); + }); + + it('POST /ingest/quick forwards a named quick-ingest input', async () => { + const response = await postJson(booted, '/memories/ingest/quick', { + user_id: 'u', + conversation: 'hello', + source_site: 'site', + source_url: 'https://example.test/quick', + session_id: 'thread-quick', + }); + + expect(response.status).toBe(200); + expect(mockQuickIngest).toHaveBeenCalledWith({ + userId: 'u', + conversationText: 'hello', + sourceSite: 'site', + sourceUrl: 'https://example.test/quick', + effectiveConfig: undefined, + sessionId: 'thread-quick', + }); + }); + + it('POST /ingest/quick skip_extraction forwards a named verbatim input', async () => { + const metadata = { origin: 'route-test' }; + const response = await postJson(booted, '/memories/ingest/quick', { + user_id: 'u', + conversation: 'verbatim', + source_site: 'site', + source_url: 'https://example.test/verbatim', + session_id: 'thread-verbatim', + skip_extraction: true, + metadata, + }); + + expect(response.status).toBe(200); + expect(mockStoreVerbatim).toHaveBeenCalledWith({ + userId: 'u', + content: 'verbatim', + sourceSite: 'site', + sourceUrl: 'https://example.test/verbatim', + metadata, + effectiveConfig: undefined, + sessionId: 'thread-verbatim', + }); + }); + + it('POST /ingest with workspace forwards a named workspace input', async () => { + const response = await postJson(booted, '/memories/ingest', { + user_id: 'u', + conversation: 'workspace hello', + source_site: 'site', + workspace_id: 'ws-1', + agent_id: '00000000-0000-4000-8000-000000000001', + visibility: 'workspace', + session_id: 'thread-workspace', + }); + + expect(response.status).toBe(200); + expect(mockWorkspaceIngest).toHaveBeenCalledWith({ + userId: 'u', + conversationText: 'workspace hello', + sourceSite: 'site', + sourceUrl: '', + workspace: { + workspaceId: 'ws-1', + agentId: '00000000-0000-4000-8000-000000000001', + visibility: 'workspace', + }, + effectiveConfig: undefined, + sessionId: 'thread-workspace', + }); + }); + + it('GET /list forwards a named user-list input', async () => { + const response = await fetch( + `${booted.baseUrl}/memories/list?user_id=u&limit=7&offset=2&source_site=site&session_id=thread-list`, + ); + + expect(response.status).toBe(200); + expect(mockList).toHaveBeenCalledWith({ + userId: 'u', + limit: 7, + offset: 2, + sourceSite: 'site', + episodeId: undefined, + sessionId: 'thread-list', + }); + }); + + it('GET /list workspace forwards a named scoped-list input', async () => { + const agentId = '00000000-0000-4000-8000-000000000002'; + const response = await fetch( + `${booted.baseUrl}/memories/list?user_id=u&workspace_id=ws-1&agent_id=${agentId}&session_id=thread-ws-list`, + ); + + expect(response.status).toBe(200); + expect(mockScopedList).toHaveBeenCalledWith({ + scope: { kind: 'workspace', userId: 'u', workspaceId: 'ws-1', agentId }, + limit: 20, + offset: 0, + sessionId: 'thread-ws-list', + }); + }); +}); + +function postJson(booted: BootedApp, path: string, body: unknown): Promise { + return fetch(`${booted.baseUrl}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); +} diff --git a/src/__tests__/route-validation.test.ts b/src/__tests__/route-validation.test.ts index a1a2096..b25046f 100644 --- a/src/__tests__/route-validation.test.ts +++ b/src/__tests__/route-validation.test.ts @@ -5,6 +5,7 @@ */ import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { randomUUID } from 'node:crypto'; // Mock embedText to avoid hitting the real embedding provider in CI where // OPENAI_API_KEY is a placeholder. Returns a deterministic zero vector @@ -20,6 +21,18 @@ vi.mock('../services/embedding.js', async (importOriginal) => { }; }); +vi.mock('../services/consensus-extraction.js', () => ({ + consensusExtractFacts: vi.fn(async (conversation: string) => [{ + fact: `Extracted fact: ${conversation}`, + headline: 'Extracted fact', + importance: 0.6, + type: 'knowledge', + keywords: ['extracted'], + entities: [], + relations: [], + }]), +})); + import { pool } from '../db/pool.js'; import { MemoryRepository } from '../db/memory-repository.js'; import { ClaimRepository } from '../db/claim-repository.js'; @@ -27,6 +40,7 @@ import { MemoryService } from '../services/memory-service.js'; import { createMemoryRouter } from '../routes/memories.js'; import { setupTestSchema } from '../db/__tests__/test-fixtures.js'; import { RESERVED_METADATA_KEYS } from '../db/repository-types.js'; +import { config } from '../config.js'; import express from 'express'; const TEST_USER = 'route-validation-test-user'; @@ -35,13 +49,42 @@ const INVALID_UUID = 'not-a-uuid'; let server: ReturnType; let baseUrl: string; +let repo: MemoryRepository; const app = express(); app.use(express.json()); +async function seedWorkspaceMemory(input: { + userId: string; + workspaceId: string; + agentId: string; + conversation: string; + sessionId: string; +}) { + const episodeId = await repo.storeEpisode({ + userId: input.userId, + content: input.conversation, + sourceSite: 'workspace-thread-test', + workspaceId: input.workspaceId, + agentId: input.agentId, + sessionId: input.sessionId, + }); + await repo.storeMemory({ + userId: input.userId, + content: input.conversation, + embedding: new Array(config.embeddingDimensions).fill(0), + importance: 0.5, + sourceSite: 'workspace-thread-test', + episodeId, + workspaceId: input.workspaceId, + agentId: input.agentId, + visibility: 'workspace', + }); +} + beforeAll(async () => { await setupTestSchema(pool); - const repo = new MemoryRepository(pool); + repo = new MemoryRepository(pool); const claimRepo = new ClaimRepository(pool); const service = new MemoryService(repo, claimRepo); app.use('/memories', createMemoryRouter(service)); @@ -105,6 +148,123 @@ describe('POST /memories/ingest/quick — skip_extraction (storeVerbatim)', () = expect(body.stored_memory_ids).toHaveLength(1); expect(body.updated_memory_ids).toHaveLength(0); }); + + it('persists session_id so list can filter by thread scope', async () => { + const sessionId = 'thread-route-validation'; + const res = await fetch(`${baseUrl}/memories/ingest/quick`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + user_id: TEST_USER, + conversation: 'Thread scoped verbatim content.', + source_site: 'thread-test', + skip_extraction: true, + session_id: sessionId, + }), + }); + expect(res.status).toBe(200); + + const list = await fetch( + `${baseUrl}/memories/list?user_id=${TEST_USER}&session_id=${sessionId}`, + ); + expect(list.status).toBe(200); + const body = await list.json(); + const memory = body.memories.find( + (item: { content: string }) => item.content === 'Thread scoped verbatim content.', + ); + expect(memory?.session_id).toBe(sessionId); + }); + + it('filters workspace list by session_id', async () => { + const workspaceId = randomUUID(); + const agentId = randomUUID(); + const userId = `${TEST_USER}-workspace-session-${workspaceId}`; + + for (const [conversation, sessionId] of [ + ['Workspace thread A content.', 'workspace-thread-a'], + ['Workspace thread B content.', 'workspace-thread-b'], + ]) { + await seedWorkspaceMemory({ + userId, + workspaceId, + agentId, + conversation, + sessionId, + }); + } + + const list = await fetch( + `${baseUrl}/memories/list?user_id=${userId}&workspace_id=${workspaceId}&agent_id=${agentId}&session_id=workspace-thread-a`, + ); + expect(list.status).toBe(200); + const body = await list.json(); + expect(body.memories).toHaveLength(1); + expect(body.memories[0].content).toBe('Workspace thread A content.'); + expect(body.memories[0].session_id).toBe('workspace-thread-a'); + }); + + it('filters workspace search by session_id', async () => { + const workspaceId = randomUUID(); + const agentId = randomUUID(); + const userId = `${TEST_USER}-workspace-search-session-${workspaceId}`; + await seedWorkspaceMemory({ + userId, + workspaceId, + agentId, + conversation: 'Workspace search thread A content.', + sessionId: 'workspace-search-thread-a', + }); + await seedWorkspaceMemory({ + userId, + workspaceId, + agentId, + conversation: 'Workspace search thread B content.', + sessionId: 'workspace-search-thread-b', + }); + + const search = await fetch(`${baseUrl}/memories/search`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + user_id: userId, + query: 'Workspace search thread content', + workspace_id: workspaceId, + agent_id: agentId, + session_id: 'workspace-search-thread-a', + }), + }); + expect(search.status).toBe(200); + const body = await search.json(); + expect(body.memories).toHaveLength(1); + expect(body.memories[0].content).toBe('Workspace search thread A content.'); + expect(body.memories[0].session_id).toBe('workspace-search-thread-a'); + }); + + it('persists session_id through full extraction ingest', async () => { + const userId = `${TEST_USER}-full-ingest-session-${randomUUID()}`; + const sessionId = 'thread-full-ingest-route-validation'; + const res = await fetch(`${baseUrl}/memories/ingest`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + user_id: userId, + conversation: 'Full ingest thread scoped content.', + source_site: 'full-thread-test', + session_id: sessionId, + }), + }); + expect(res.status).toBe(200); + + const list = await fetch( + `${baseUrl}/memories/list?user_id=${userId}&session_id=${sessionId}`, + ); + expect(list.status).toBe(200); + const body = await list.json(); + const memory = body.memories.find((item: { content: string }) => + item.content.includes('Full ingest thread scoped content.'), + ); + expect(memory?.session_id).toBe(sessionId); + }); }); describe('GET /memories/list — source_site filter', () => { diff --git a/src/__tests__/smoke.test.ts b/src/__tests__/smoke.test.ts index 82e4923..005181a 100644 --- a/src/__tests__/smoke.test.ts +++ b/src/__tests__/smoke.test.ts @@ -85,7 +85,11 @@ user: We need independent scaling. The auth service gets 10x the traffic. assistant: What language for the services? user: Go for performance-critical ones, TypeScript for the rest.`; - const writeResult = await service.ingest(TEST_USER, conversation, 'claude.ai'); + const writeResult = await service.ingest({ + userId: TEST_USER, + conversationText: conversation, + sourceSite: 'claude.ai', + }); expect(writeResult.memoriesStored).toBeGreaterThan(0); const searchResult = await service.search(TEST_USER, 'What infrastructure is the user using?'); @@ -94,16 +98,16 @@ user: Go for performance-critical ones, TypeScript for the rest.`; }); it('returns relevant results over irrelevant ones', async () => { - await service.ingest( - TEST_USER, - 'user: I love hiking in the mountains every weekend.', - 'test', - ); - await service.ingest( - TEST_USER, - 'user: My React app uses Next.js with Prisma for the database layer.', - 'test', - ); + await service.ingest({ + userId: TEST_USER, + conversationText: 'user: I love hiking in the mountains every weekend.', + sourceSite: 'test', + }); + await service.ingest({ + userId: TEST_USER, + conversationText: 'user: My React app uses Next.js with Prisma for the database layer.', + sourceSite: 'test', + }); const result = await service.search(TEST_USER, 'What framework is the user using?'); expect(result.memories.length).toBeGreaterThan(0); diff --git a/src/app/__tests__/composed-boot-parity.test.ts b/src/app/__tests__/composed-boot-parity.test.ts index 0532984..ddc7f3a 100644 --- a/src/app/__tests__/composed-boot-parity.test.ts +++ b/src/app/__tests__/composed-boot-parity.test.ts @@ -19,9 +19,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import express from 'express'; -import { readFileSync } from 'node:fs'; -import { resolve, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; import { pool } from '../../db/pool.js'; import { config } from '../../config.js'; import { MemoryRepository } from '../../db/memory-repository.js'; @@ -32,8 +29,8 @@ import { createCoreRuntime } from '../runtime-container.js'; import { createApp } from '../create-app.js'; import { type BootedApp, bindEphemeral } from '../bind-ephemeral.js'; import { authHeader } from '../../__tests__/helpers/auth-headers.js'; +import { migrate } from '../../db/migration-api.js'; -const __dirname = dirname(fileURLToPath(import.meta.url)); const TEST_USER = 'composed-boot-parity-user'; /** @@ -55,9 +52,7 @@ describe('composed boot parity', () => { let reference: BootedApp; beforeAll(async () => { - const raw = readFileSync(resolve(__dirname, '../../db/schema.sql'), 'utf-8'); - const sql = raw.replace(/\{\{EMBEDDING_DIMENSIONS\}\}/g, String(config.embeddingDimensions)); - await pool.query(sql); + await migrate({ pool }); composed = await bindEphemeral(createApp(await createCoreRuntime({ pool }))); reference = await bindEphemeral(buildReferenceApp()); diff --git a/src/app/__tests__/research-consumption-seams.test.ts b/src/app/__tests__/research-consumption-seams.test.ts index 8dc329f..bc85c08 100644 --- a/src/app/__tests__/research-consumption-seams.test.ts +++ b/src/app/__tests__/research-consumption-seams.test.ts @@ -103,7 +103,11 @@ describe('Phase 6 research-consumption seams', () => { }); it('in-process seam: ingest + search via runtime.services.memory', async () => { - const write = await runtime.services.memory.ingest(TEST_USER, CONVERSATION, 'test-site'); + const write = await runtime.services.memory.ingest({ + userId: TEST_USER, + conversationText: CONVERSATION, + sourceSite: 'test-site', + }); expect(write.memoriesStored).toBeGreaterThan(0); const read = await runtime.services.memory.search(TEST_USER, 'What stack does the user use?'); @@ -111,6 +115,29 @@ describe('Phase 6 research-consumption seams', () => { expect(read.injectionText.length).toBeGreaterThan(0); }); + it('in-process seam: named ingest + list preserve session scope', async () => { + await runtime.services.memory.ingest({ + userId: TEST_USER, + conversationText: CONVERSATION, + sourceSite: 'test-site', + sessionId: 'phase6-session-a', + }); + await runtime.services.memory.ingest({ + userId: TEST_USER, + conversationText: 'user: I also maintain a Rust CLI.', + sourceSite: 'test-site', + sessionId: 'phase6-session-b', + }); + + const scoped = await runtime.services.memory.list({ + userId: TEST_USER, + sessionId: 'phase6-session-a', + }); + + expect(scoped.length).toBeGreaterThan(0); + expect(scoped.every((memory) => memory.session_id === 'phase6-session-a')).toBe(true); + }); + it('HTTP seam: POST /v1/memories/ingest + POST /v1/memories/search via bindEphemeral', async () => { const ingestRes = await fetch(`${server.baseUrl}/v1/memories/ingest`, { method: 'POST', @@ -134,7 +161,11 @@ describe('Phase 6 research-consumption seams', () => { }); it('parity: in-process write is observable through the HTTP seam (shared pool)', async () => { - const write = await runtime.services.memory.ingest(TEST_USER, CONVERSATION, 'test-site'); + const write = await runtime.services.memory.ingest({ + userId: TEST_USER, + conversationText: CONVERSATION, + sourceSite: 'test-site', + }); expect(write.memoriesStored).toBeGreaterThan(0); const writtenIds = new Set(write.memoryIds); diff --git a/src/app/create-app.ts b/src/app/create-app.ts index 91ec338..00ead2a 100644 --- a/src/app/create-app.ts +++ b/src/app/create-app.ts @@ -8,6 +8,7 @@ import express from 'express'; import { createAgentRouter } from '../routes/agents.js'; +import { createAdminRouter } from '../routes/admin.js'; import { createDocumentRouter } from '../routes/documents.js'; import { createMemoryRouter } from '../routes/memories.js'; import { makeReflectFlushHandler } from '../routes/reflect.js'; @@ -140,6 +141,18 @@ export function createApp(runtime: CoreRuntime): ReturnType { }), ); + if (runtime.config.coreAdminApiKey && runtime.config.coreTestScopeAllowPattern) { + app.use( + '/v1/admin', + requireBearer(runtime.config.coreAdminApiKey), + express.json({ limit: DEFAULT_JSON_BODY_LIMIT }), + createAdminRouter({ + memory: runtime.repos.memory, + testScopeAllowPattern: runtime.config.coreTestScopeAllowPattern, + }), + ); + } + // Reflect flush: synchronous queue drain for benchmark / eval harnesses. // Wired regardless of reflectEnabled so the route always exists; the // handler returns 503 when reflect is disabled. diff --git a/src/config.ts b/src/config.ts index 265b62c..5c4a3ba 100644 --- a/src/config.ts +++ b/src/config.ts @@ -57,6 +57,18 @@ export interface RuntimeConfig { * restarting the server with a new value. */ coreApiKey: string; + /** + * Optional admin API key for test-scope cleanup endpoints. When unset, + * admin routes are not mounted. Operators should use a different secret + * from CORE_API_KEY so normal SDK callers cannot wipe scopes. + */ + coreAdminApiKey?: string; + /** + * Explicit allow-pattern for admin cleanup scopes. Required alongside + * CORE_ADMIN_API_KEY before the admin cleanup router is mounted; there is + * no default so production deploys cannot accidentally enable wipes. + */ + coreTestScopeAllowPattern?: string; /** * Hex-encoded secret used to derive per-user storage-key prefixes * via HMAC-SHA256. Storage keys take the form `s/<32hex>/.bin` @@ -710,6 +722,17 @@ function parseLlmSeed(value: string | undefined): number | undefined { return Number.isFinite(parsed) ? parsed : undefined; } +function parseRegexEnv(name: string): string | undefined { + const raw = optionalEnv(name); + if (!raw) return undefined; + try { + new RegExp(raw); + return raw; + } catch { + throw new Error(`${name} must be a valid JavaScript regular expression`); + } +} + function parsePositiveIntEnv(name: string, fallback: number): number { const raw = optionalEnv(name); if (!raw) return fallback; @@ -1089,6 +1112,8 @@ export const config: RuntimeConfig = { databaseUrl: requireEnv('DATABASE_URL'), openaiApiKey, coreApiKey: requireEnv('CORE_API_KEY'), + coreAdminApiKey: optionalEnv('CORE_ADMIN_API_KEY'), + coreTestScopeAllowPattern: parseRegexEnv('CORE_TEST_SCOPE_ALLOW_PATTERN'), storageKeyHmacSecret: parseStorageKeyHmacSecret(requireEnv('STORAGE_KEY_HMAC_SECRET')), port: parseInt(process.env.PORT ?? '3050', 10), retrievalProfile, @@ -1359,7 +1384,8 @@ export function updateRuntimeConfig(updates: RuntimeConfigUpdates): string[] { */ export const SUPPORTED_RUNTIME_CONFIG_FIELDS = [ // Infrastructure - 'databaseUrl', 'openaiApiKey', 'coreApiKey', 'storageKeyHmacSecret', 'port', + 'databaseUrl', 'openaiApiKey', 'coreApiKey', 'coreAdminApiKey', + 'coreTestScopeAllowPattern', 'storageKeyHmacSecret', 'port', // Provider / model selection (startup config) 'embeddingProvider', 'embeddingModel', 'embeddingDimensions', 'embeddingApiUrl', 'embeddingApiKey', diff --git a/src/db/__tests__/baseline-schema-equivalence.test.ts b/src/db/__tests__/baseline-schema-equivalence.test.ts new file mode 100644 index 0000000..dac6299 --- /dev/null +++ b/src/db/__tests__/baseline-schema-equivalence.test.ts @@ -0,0 +1,81 @@ +/** + * Phase 2 — Baseline schema equivalence (the gate). + * + * Per docs/ops/db/phase-2-versioned-migrations.md § "CI guard: schema + * equivalence" and the backward-compatibility checklist: + * + * "Scenario A produces a schema byte-identical (modulo + * framework-bookkeeping tables) to the legacy `schema.sql`." + * + * The user-facing schema must be identical whether the DB was created + * via a fresh Phase 2 install (Scenario A — runs 0001_baseline.sql) or + * via a v1.0.x → Phase 2 upgrade (Scenario B — stamps 0001_baseline + * without running it, applies only post-baseline migrations). Drift here + * is the exact silent-divergence failure mode Phase 2's cutover design + * exists to prevent, so this test is the gate: do not delete it. + * + * Diff strategy: pg_catalog-based structural snapshot (same shape as the + * Phase 1 backcompat test) with the `pgmigrations` and `schema_version` + * bookkeeping tables stripped, since they intentionally hold different + * rows along the two paths. We do not shell out to `pg_dump` here — the + * Phase 1 helper already proved that pg_catalog enumeration is more + * deterministic across PG minor versions than `pg_dump` text. + * + * Runtime dependency: this test exercises the Phase 2 migrate() public + * API exported from `../migration-api.js`. While the Phase 2 runtime is + * still landing, the test will fail with a missing `pgmigrations` table + * or a schema-equivalence mismatch. When Phase 2 lands, the test must pass + * without modification. + */ + +import { afterAll, beforeEach, describe, expect, it } from 'vitest'; +import { migrate } from '../migration-api.js'; +import { + applyLegacySchema, + useMigrationTestPool, +} from './migration-test-helpers.js'; +import { + resetPublicSchemaForReuse, + structuralSnapshotExcludingBookkeeping, +} from './phase2-cutover-helpers.js'; + +const pool = useMigrationTestPool({ beforeEach, afterAll }); + +describe('Phase 2 — baseline schema equivalence', () => { + it('fresh install reaches the same user-facing schema as a v1.0.x → Phase 2 upgrade', async () => { + // Scenario A: fresh Phase 2 install. migrate() runs 0001_baseline + // (and any later migrations) against the empty database. + await migrate({ pool }); + const fresh = await structuralSnapshotExcludingBookkeeping(pool); + + // Mid-test reset so Scenario B starts from a clean DB without + // breaking the suite-level `beforeEach` (which only fires between + // tests). resetPublicSchemaForReuse also re-installs `vector` and + // `pgcrypto`, which legacy schema.sql expects. + await resetPublicSchemaForReuse(pool); + + // Scenario B: v1.0.x install upgraded by Phase 2 migrate(). The + // legacy fixture sets up the pre-Phase-1 schema; migrate() must + // detect this as `pre_phase_2` and stamp 0001_baseline without + // re-running it against the existing tables. + await applyLegacySchema(pool); + await migrate({ pool }); + const upgraded = await structuralSnapshotExcludingBookkeeping(pool); + + expect(upgraded).toEqual(fresh); + }); + + it('both cutover paths produce a non-empty user-facing schema', async () => { + // Guard against the false-positive "two empty snapshots are equal". + // If the equivalence test ever started seeing zero tables on both + // sides, it would silently pass. This sub-test asserts a meaningful + // floor: every cutover path must end up with the legacy core tables. + await migrate({ pool }); + const fresh = await structuralSnapshotExcludingBookkeeping(pool); + const tableNames = new Set(fresh.tables.map((entry) => entry.name)); + + for (const required of ['memories', 'episodes', 'memory_claims']) { + expect(tableNames.has(required)).toBe(true); + } + }); +}); diff --git a/src/db/__tests__/cutover-scenarios.test.ts b/src/db/__tests__/cutover-scenarios.test.ts new file mode 100644 index 0000000..fa18d3b --- /dev/null +++ b/src/db/__tests__/cutover-scenarios.test.ts @@ -0,0 +1,195 @@ +/** + * Phase 2 — Cutover scenarios (A / B / C). + * + * Per docs/ops/db/phase-2-versioned-migrations.md §"Lossless cutover + * design" and the backward-compatibility checklist: + * + * - Scenario A (fresh): `migrate()` runs the baseline against an empty + * DB, `pgmigrations` carries the `0001_baseline` row, `schema_version` + * gains a Phase 2 stamp. + * - Scenario B (v1.0.x → Phase 2): legacy schema seeded with real data, + * `migrate()` stamps `0001_baseline` *without* re-running it, touches + * zero user-facing tables / columns / indexes, preserves every seeded + * row byte-identical, and leaves seeded foreign keys resolvable. + * - Scenario C (Phase 1 → Phase 2): same as B, plus the existing + * `schema_version` rows are preserved (not wiped) and Phase 2 appends. + * + * The Phase 1 data-preservation snapshots (`migration-seed-fixtures.ts`) + * are reused unchanged — the data-preservation contract is identical + * across phases: existing rows survive the framework cutover, full stop. + * + * Runtime dependency: this exercises the planned Phase 2 `migrate()` + * behavior (detectInstallState + stampBaselineAsApplied + + * runFrameworkMigrationsToHead). Until that runtime lands the tests will + * fail on the `pgmigrationsRows()` precondition (no pgmigrations table) + * with a clear error pointing at the missing runtime. + */ + +import { afterAll, beforeEach, describe, expect, it } from 'vitest'; +import { migrate, MigrationHistoryMismatch } from '../migration-api.js'; +import { + useMigrationTestPool, + type StructuralSnapshot, +} from './migration-test-helpers.js'; +import { snapshotAllSeededTables } from './migration-seed-fixtures.js'; +import { + applyLegacySchemaAndSeed, + expectSeededForeignKeysResolvable, + expectSeededRowsPreservedAcrossMigrate, +} from './migration-preservation-assertions.js'; +import { + pgmigrationsRows, + schemaVersionRowCount, + seedPhase1StampedState, + structuralSnapshotExcludingBookkeeping, +} from './phase2-cutover-helpers.js'; + +const pool = useMigrationTestPool({ beforeEach, afterAll }); + +const BASELINE_MIGRATION_NAME = '0001_baseline'; + +describe('Phase 2 — Scenario A: fresh install', () => { + it('runs the baseline migration and records it in pgmigrations', async () => { + await migrate({ pool }); + + const rows = await pgmigrationsRows(pool); + expect(rows.length).toBeGreaterThanOrEqual(1); + expect(rows[0].name).toBe(BASELINE_MIGRATION_NAME); + expect(rows[0].run_on).toBeInstanceOf(Date); + }); + + it('stamps schema_version exactly once on first install', async () => { + await migrate({ pool }); + expect(await schemaVersionRowCount(pool)).toBe(1); + }); +}); + +describe('Phase 2 — Scenario B: v1.0.x install upgraded to Phase 2', () => { + it('stamps the baseline without executing it against existing data', async () => { + await applyLegacySchemaAndSeed(pool); + + await migrate({ pool }); + + await expectOnlyBaselineStamped(); + }); + + it('does not modify any existing legacy table, column, index, or constraint', async () => { + await applyLegacySchemaAndSeed(pool); + const before = await structuralSnapshotExcludingBookkeeping(pool); + + await migrate({ pool }); + const after = await structuralSnapshotExcludingBookkeeping(pool); + + assertStructuralEqual(before, after); + }); + + it('preserves every seeded row byte-identical across migrate()', async () => { + await applyLegacySchemaAndSeed(pool); + await expectSeededRowsPreservedAcrossMigrate(pool); + }); + + it('keeps every seeded foreign-key relationship resolvable after migrate()', async () => { + const ids = await applyLegacySchemaAndSeed(pool); + + await migrate({ pool }); + + await expectSeededForeignKeysResolvable(pool, ids); + }); + + it('recovers when pgmigrations exists but is empty before cutover', async () => { + await applyLegacySchemaAndSeed(pool); + await pool.query(` + CREATE TABLE pgmigrations ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + run_on TIMESTAMPTZ NOT NULL + )`); + + await migrate({ pool }); + + await expectOnlyBaselineStamped(); + }); + + it('rejects pgmigrations rows that are missing the baseline stamp', async () => { + await applyLegacySchemaAndSeed(pool); + await pool.query(` + CREATE TABLE pgmigrations ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + run_on TIMESTAMPTZ NOT NULL + )`); + await pool.query( + `INSERT INTO pgmigrations (name, run_on) + VALUES ('0002_unanchored', NOW())`, + ); + + await expect(migrate({ pool })).rejects.toBeInstanceOf(MigrationHistoryMismatch); + expect(await schemaVersionRowCount(pool)).toBe(0); + }); +}); + +describe('Phase 2 — Scenario C: Phase 1 install upgraded to Phase 2', () => { + it('preserves the pre-existing schema_version history', async () => { + await applyLegacySchemaAndSeed(pool); + await seedPhase1StampedState(pool); + const phase1RowCount = await schemaVersionRowCount(pool); + expect(phase1RowCount).toBe(1); + + await migrate({ pool }); + + const phase2RowCount = await schemaVersionRowCount(pool); + // Phase 2 must not wipe the Phase 1 stamp; it may append its own. + expect(phase2RowCount).toBeGreaterThanOrEqual(phase1RowCount); + + const { rows } = await pool.query<{ notes: string | null }>( + `SELECT notes FROM schema_version + WHERE applied_at = TIMESTAMPTZ '2026-04-01 00:00:00Z'`, + ); + expect(rows.length).toBe(1); + expect(rows[0].notes).toBe('phase1-fixture-stamp'); + }); + + it('still stamps the baseline as applied without re-running it', async () => { + await applyLegacySchemaAndSeed(pool); + await seedPhase1StampedState(pool); + + await migrate({ pool }); + + await expectOnlyBaselineStamped(); + }); + + it('preserves every seeded row when both Phase 1 stamps and legacy data exist', async () => { + await applyLegacySchemaAndSeed(pool); + await seedPhase1StampedState(pool); + const before = await snapshotAllSeededTables(pool); + + await migrate({ pool }); + const after = await snapshotAllSeededTables(pool); + + expect(after).toEqual(before); + }); +}); + +/** + * Assert that two structural snapshots describe the same set of tables, + * columns, indexes, check constraints, and foreign keys. Uses + * `toStrictEqual` against the helper output rather than per-field + * iteration because the helper already canonicalizes ordering. + */ +function assertStructuralEqual( + before: StructuralSnapshot, + after: StructuralSnapshot, +): void { + expect(after.tables).toEqual(before.tables); + expect(after.indexes).toEqual(before.indexes); + expect(after.checkConstraints).toEqual(before.checkConstraints); + expect(after.foreignKeys).toEqual(before.foreignKeys); +} + +async function expectOnlyBaselineStamped(): Promise { + const rows = await pgmigrationsRows(pool); + // Additional rows would mean post-baseline migrations exist and ran; none + // ship at cutover, so a count > 1 is a regression rather than expected drift. + expect(rows.length).toBe(1); + expect(rows[0].name).toBe(BASELINE_MIGRATION_NAME); +} diff --git a/src/db/__tests__/dag-sanity.test.ts b/src/db/__tests__/dag-sanity.test.ts new file mode 100644 index 0000000..65b4824 --- /dev/null +++ b/src/db/__tests__/dag-sanity.test.ts @@ -0,0 +1,176 @@ +/** + * Phase 2 — DAG sanity for the migration directory. + * + * Per docs/ops/db/phase-2-versioned-migrations.md § "DAG sanity tests" and + * the risks checklist (mistakenly editing or deleting an already-shipped + * migration file): + * + * - **SQL-only**: Phase 2 ships only SQL migrations. The build copy step, + * the runtime hash manifest, and the fail-closed loader all assume + * `.sql`. Allowing `.js`/`.ts` files in the directory here would let a + * PR commit one that never gets packaged or hashed; the regex + * deliberately rejects them so the contract holds at PR review time. + * - **Monotonic**: every file in `src/db/migrations/` follows the + * `_.sql` convention, prefixes are strictly increasing + * across files, and there are no gaps in the sequence. Catches the + * "two PRs both picked 0007 and the lexical sort silently runs one + * before the other" failure mode at PR review time. + * - **No-rewrite**: the diff against the base branch never `D`eletes or + * `R`enames a migration file. Once a migration is on `main`, it is + * frozen. This is the machine-checkable answer to the + * "mistakenly editing the baseline post-shipment" risk that turns + * Scenario B/C into a silent corruption path. + * + * The monotonic check is fully local; the no-rewrite check needs git and + * a reachable base branch. The base branch lookup tries `origin/main` + * first (CI's normal default) and falls back to local `main`. If neither + * resolves the no-rewrite assertion is skipped with `it.skip` semantics + * surfaced via an explicit `expect.fail`-style message: the test does + * not pass silently, but it does not falsely fail when run in a + * detached environment that lacks a remote. + * + * Runtime dependency: the monotonic test asserts the migrations directory + * exists. Until the Phase 2 runtime lands and ships `0001_baseline.sql` + * the directory is absent and this test fails with a clear assertion + * pointing at the missing directory. + */ + +import { spawnSync } from 'node:child_process'; +import { existsSync, readdirSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { MIGRATIONS_DIR } from './phase2-cutover-helpers.js'; + +const MIGRATION_FILE_REGEX = /^(\d+)_[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?\.sql$/; + +interface ParsedMigration { + readonly file: string; + readonly prefix: number; +} + +function listMigrationFiles(): string[] { + return readdirSync(MIGRATIONS_DIR) + .filter((entry) => MIGRATION_FILE_REGEX.test(entry)) + .sort(); +} + +function parseMigrationFiles(files: ReadonlyArray): ParsedMigration[] { + return files.map((file) => { + const match = MIGRATION_FILE_REGEX.exec(file); + if (!match) { + throw new Error(`parseMigrationFiles: ${file} does not match the convention`); + } + return { file, prefix: Number.parseInt(match[1], 10) }; + }); +} + +describe('Phase 2 — migration file DAG (monotonic)', () => { + it('migrations directory exists at src/db/migrations/', () => { + expect(existsSync(MIGRATIONS_DIR)).toBe(true); + }); + + it('every entry in the migrations directory matches _.sql', () => { + const entries = readdirSync(MIGRATIONS_DIR); + const offending = entries.filter((entry) => !MIGRATION_FILE_REGEX.test(entry)); + expect(offending).toEqual([]); + }); + + it('every migration file has a unique numeric prefix', () => { + const parsed = parseMigrationFiles(listMigrationFiles()); + const prefixes = parsed.map((entry) => entry.prefix); + expect(prefixes.length).toBe(new Set(prefixes).size); + }); + + it('prefixes are strictly increasing with no gaps starting at 1', () => { + const parsed = parseMigrationFiles(listMigrationFiles()); + expect(parsed.length).toBeGreaterThan(0); + for (let i = 0; i < parsed.length; i += 1) { + expect(parsed[i].prefix).toBe(i + 1); + } + }); + + it('includes the frozen 0001_baseline.sql at the head of the sequence', () => { + const files = listMigrationFiles(); + expect(files[0]).toBe('0001_baseline.sql'); + }); +}); + +/** + * Files we consider "framework-managed" Phase 2 migrations: the 4-digit + * prefix convention `0001_…`, `0002_…`. The legacy 8-digit + * timestamped files (`20260512_…`) that used to live in the same + * directory were provenance-only and were explicitly moved to + * `docs/db/changelog/` as part of the Phase 2 cutover cleanup (see + * Phase 2 plan §Cleanup). Deletions/renames of those legacy files are + * expected and are NOT a no-rewrite violation. + */ +const FRAMEWORK_FILE_REGEX = /^src\/db\/migrations\/0\d{3}_[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?\.sql$/; + +describe('Phase 2 — migration file DAG (no-rewrite vs base branch)', () => { + it('never deletes or renames a framework-managed migration file', () => { + const base = resolveBaseBranch(); + if (!base) { + // Surface as a failed assertion rather than a silent pass. CI has + // `origin/main`; a local dev environment without it gets an + // explicit reason rather than a passing test that didn't run. + expect.fail( + 'no-rewrite check requires `origin/main` or local `main` to be ' + + 'reachable. None resolved; cannot diff migration files against the base branch.', + ); + return; + } + const diff = gitDiffNameStatus(base, 'src/db/migrations'); + const forbidden = diff.filter( + (entry) => + (entry.status === 'D' || entry.status === 'R') && + FRAMEWORK_FILE_REGEX.test(entry.file), + ); + expect(forbidden).toEqual([]); + }); +}); + +function resolveBaseBranch(): string | null { + for (const candidate of ['origin/feat/db-migration-phase1', 'feat/db-migration-phase1', 'origin/main', 'main']) { + const probe = spawnSync('git', ['rev-parse', '--verify', candidate], { + encoding: 'utf-8', + }); + if (probe.status === 0) return candidate; + } + return null; +} + +interface DiffEntry { + readonly status: string; + readonly file: string; +} + +function gitDiffNameStatus(base: string, pathFilter: string): DiffEntry[] { + const result = spawnSync( + 'git', + ['diff', '--name-status', `${base}...HEAD`, '--', pathFilter], + { encoding: 'utf-8' }, + ); + if (result.status !== 0) { + throw new Error( + `git diff against ${base} failed (status=${result.status}): ${result.stderr}`, + ); + } + return parseDiffOutput(result.stdout); +} + +function parseDiffOutput(stdout: string): DiffEntry[] { + return stdout + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map(parseDiffLine); +} + +function parseDiffLine(line: string): DiffEntry { + // Examples: "A\tsrc/db/migrations/0002_foo.sql" + // "D\tsrc/db/migrations/0001_baseline.sql" + // "R100\tsrc/db/migrations/0001_baseline.sql\tsrc/db/migrations/0001_renamed.sql" + const [statusToken, ...paths] = line.split('\t'); + // Status may be "R100" (rename with similarity %); keep just the first char. + const status = statusToken.charAt(0); + return { status, file: paths[0] ?? '' }; +} diff --git a/src/db/__tests__/embedding-dim-reconciler.test.ts b/src/db/__tests__/embedding-dim-reconciler.test.ts new file mode 100644 index 0000000..ab9759b --- /dev/null +++ b/src/db/__tests__/embedding-dim-reconciler.test.ts @@ -0,0 +1,247 @@ +/** + * Integration tests for the embedding-dimension reconciler. + * + * Verifies the documented behaviors against a real Postgres+pgvector + * instance for *any* fixed-dimension pgvector column, not just columns + * named `embedding`: + * 1. No-op when the configured dimension already matches. + * 2. ALTER + recreate indexes when the column is empty (covers both + * `embedding` and `summary_embedding`). + * 3. Throw EmbeddingDimensionMismatch when the column holds vectors. + * + * Tests run inside a dedicated schema (with `public` second in + * search_path so pgvector's `vector` type remains reachable) so they + * cannot collide with the production schema set up by `setupTestSchema`. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { pool } from '../pool.js'; +import { + type AlteredVectorColumn, + EmbeddingDimensionMismatch, + reconcileEmbeddingDimension, + type ReconcileResult, +} from '../reconcilers.js'; +import { + readVectorColumnDimension, + registerReconcilerSchemaLifecycle, +} from './reconciler-test-helpers.js'; + +const TEST_SCHEMA = 'reconciler_test_schema'; +const TABLE_A = 'reconciler_table_a'; +const TABLE_B = 'reconciler_table_b'; +const TABLE_MULTI = 'reconciler_table_multi'; + +async function indexNamesFor(table: string): Promise { + const { rows } = await pool.query<{ indexname: string }>( + `SELECT indexname FROM pg_indexes + WHERE schemaname = $1 AND tablename = $2 ORDER BY indexname`, + [TEST_SCHEMA, table], + ); + return rows.map((row) => row.indexname); +} + +function expectNoAlteration(result: ReconcileResult): void { + expect(result.reconciled).toBe(false); + expect(result.alteredColumns).toEqual([]); +} + +function expectAlteredColumns( + result: ReconcileResult, + columns: AlteredVectorColumn[], +): void { + expect(result.reconciled).toBe(true); + expect(result.alteredColumns).toEqual(columns); +} + +async function expectColumnDimension( + table: string, + column: string, + dimension: number, +): Promise { + expect(await readVectorColumnDimension(pool, TEST_SCHEMA, table, column)).toBe( + dimension, + ); +} + +async function catchMismatch( + requiredDimension: number, +): Promise { + try { + await reconcileEmbeddingDimension(pool, requiredDimension); + } catch (err) { + expect(err).toBeInstanceOf(EmbeddingDimensionMismatch); + return err as EmbeddingDimensionMismatch; + } + throw new Error('Expected EmbeddingDimensionMismatch'); +} + +function expectMismatchFields( + error: EmbeddingDimensionMismatch, + columnName: string, +): void { + expect(error.tableName).toBe(TABLE_A); + expect(error.columnName).toBe(columnName); + expect(error.currentDimension).toBe(4); + expect(error.requiredDimension).toBe(8); + expect(error.rowCount).toBe(1); + expect(error.message).toContain(columnName); +} + +describe('reconcileEmbeddingDimension', () => { + registerReconcilerSchemaLifecycle({ + afterAll, + beforeAll, + beforeEach, + pool, + schema: TEST_SCHEMA, + }); + + it('is a no-op when every vector column already matches', async () => { + await pool.query( + `CREATE TABLE ${TABLE_A} (id serial PRIMARY KEY, embedding vector(8))`, + ); + const result = await reconcileEmbeddingDimension(pool, 8); + expectNoAlteration(result); + await expectColumnDimension(TABLE_A, 'embedding', 8); + }); + + it('returns no-op when no vector columns exist in current schema', async () => { + const result = await reconcileEmbeddingDimension(pool, 16); + expectNoAlteration(result); + }); + + it('alters an empty `embedding` column with a mismatched dimension', async () => { + await pool.query( + `CREATE TABLE ${TABLE_A} (id serial PRIMARY KEY, embedding vector(4))`, + ); + const result = await reconcileEmbeddingDimension(pool, 8); + expectAlteredColumns(result, [ + { tableName: TABLE_A, columnName: 'embedding' }, + ]); + await expectColumnDimension(TABLE_A, 'embedding', 8); + }); + + it('alters an empty `summary_embedding` column with a mismatched dimension', async () => { + await pool.query( + `CREATE TABLE ${TABLE_A} (id serial PRIMARY KEY, summary_embedding vector(4))`, + ); + const result = await reconcileEmbeddingDimension(pool, 16); + expectAlteredColumns(result, [ + { tableName: TABLE_A, columnName: 'summary_embedding' }, + ]); + await expectColumnDimension(TABLE_A, 'summary_embedding', 16); + }); + + it('drops and recreates HNSW indexes on a non-`embedding` vector column', async () => { + await pool.query( + `CREATE TABLE ${TABLE_A} (id serial PRIMARY KEY, summary_embedding vector(4))`, + ); + const indexName = `${TABLE_A}_summary_hnsw_idx`; + await pool.query( + `CREATE INDEX ${indexName} ON ${TABLE_A} + USING hnsw (summary_embedding vector_cosine_ops)`, + ); + expect(await indexNamesFor(TABLE_A)).toContain(indexName); + + const result = await reconcileEmbeddingDimension(pool, 32); + expectAlteredColumns(result, [ + { tableName: TABLE_A, columnName: 'summary_embedding' }, + ]); + await expectColumnDimension(TABLE_A, 'summary_embedding', 32); + expect(await indexNamesFor(TABLE_A)).toContain(indexName); + }); + + it('alters multiple vector columns on the same table independently', async () => { + await pool.query( + `CREATE TABLE ${TABLE_MULTI} ( + id serial PRIMARY KEY, + embedding vector(4), + topic_embedding vector(4) + )`, + ); + const result = await reconcileEmbeddingDimension(pool, 8); + expectAlteredColumns(result, [ + { tableName: TABLE_MULTI, columnName: 'embedding' }, + { tableName: TABLE_MULTI, columnName: 'topic_embedding' }, + ]); + await expectColumnDimension(TABLE_MULTI, 'embedding', 8); + await expectColumnDimension(TABLE_MULTI, 'topic_embedding', 8); + }); + + it('throws EmbeddingDimensionMismatch when an `embedding` column holds vectors', async () => { + await pool.query( + `CREATE TABLE ${TABLE_A} (id serial PRIMARY KEY, embedding vector(4))`, + ); + await pool.query( + `INSERT INTO ${TABLE_A} (embedding) VALUES ('[1,2,3,4]'::vector)`, + ); + + const e = await catchMismatch(8); + expectMismatchFields(e, 'embedding'); + expect(e.message).toContain(TABLE_A); + expect(e.message).toContain('vector(4)'); + await expectColumnDimension(TABLE_A, 'embedding', 4); + }); + + it('throws EmbeddingDimensionMismatch when a `summary_embedding` column holds vectors', async () => { + await pool.query( + `CREATE TABLE ${TABLE_A} (id serial PRIMARY KEY, summary_embedding vector(4))`, + ); + await pool.query( + `INSERT INTO ${TABLE_A} (summary_embedding) VALUES ('[1,2,3,4]'::vector)`, + ); + + const e = await catchMismatch(8); + expectMismatchFields(e, 'summary_embedding'); + await expectColumnDimension(TABLE_A, 'summary_embedding', 4); + }); + + it('ignores rows with NULL vectors when counting population', async () => { + await pool.query( + `CREATE TABLE ${TABLE_A} (id serial PRIMARY KEY, summary_embedding vector(4) NULL)`, + ); + await pool.query(`INSERT INTO ${TABLE_A} (summary_embedding) VALUES (NULL)`); + const result = await reconcileEmbeddingDimension(pool, 8); + expectAlteredColumns(result, [ + { tableName: TABLE_A, columnName: 'summary_embedding' }, + ]); + await expectColumnDimension(TABLE_A, 'summary_embedding', 8); + }); + + it('discovers multiple tables with differently-named vector columns', async () => { + await pool.query( + `CREATE TABLE ${TABLE_A} (id serial PRIMARY KEY, embedding vector(4))`, + ); + await pool.query( + `CREATE TABLE ${TABLE_B} (id serial PRIMARY KEY, recap_embedding vector(4))`, + ); + const result = await reconcileEmbeddingDimension(pool, 8); + expectAlteredColumns(result, [ + { tableName: TABLE_A, columnName: 'embedding' }, + { tableName: TABLE_B, columnName: 'recap_embedding' }, + ]); + await expectColumnDimension(TABLE_A, 'embedding', 8); + await expectColumnDimension(TABLE_B, 'recap_embedding', 8); + }); + + it('leaves unconstrained `vector` columns (typmod -1) alone', async () => { + await pool.query( + `CREATE TABLE ${TABLE_A} (id serial PRIMARY KEY, embedding vector)`, + ); + const result = await reconcileEmbeddingDimension(pool, 8); + expectNoAlteration(result); + }); + + it('rejects invalid required dimension', async () => { + await expect(reconcileEmbeddingDimension(pool, 0)).rejects.toThrow( + /positive integer/, + ); + await expect(reconcileEmbeddingDimension(pool, -1)).rejects.toThrow( + /positive integer/, + ); + await expect(reconcileEmbeddingDimension(pool, 1.5)).rejects.toThrow( + /positive integer/, + ); + }); +}); diff --git a/src/db/schema.sql b/src/db/__tests__/fixtures/legacy-schema.sql similarity index 100% rename from src/db/schema.sql rename to src/db/__tests__/fixtures/legacy-schema.sql diff --git a/src/db/__tests__/hierarchical-schema.test.ts b/src/db/__tests__/hierarchical-schema.test.ts index e80b8e3..c97ff7e 100644 --- a/src/db/__tests__/hierarchical-schema.test.ts +++ b/src/db/__tests__/hierarchical-schema.test.ts @@ -1,6 +1,6 @@ /** - * Static verification of the Hierarchical Retrieval schema additions in - * schema.sql. Asserts DDL presence + idempotency without a DB connection. + * Static verification of the Hierarchical Retrieval baseline migration + * additions. Asserts DDL presence + idempotency without a DB connection. */ import { describe, it, expect } from 'vitest'; @@ -9,7 +9,10 @@ import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; const __dirname = dirname(fileURLToPath(import.meta.url)); -const schemaSql = readFileSync(resolve(__dirname, '..', 'schema.sql'), 'utf-8'); +const schemaSql = readFileSync( + resolve(__dirname, '..', 'migrations', '0001_baseline.sql'), + 'utf-8', +); const IDEMPOTENT_DDL = /IF NOT EXISTS|DROP CONSTRAINT IF EXISTS/; const CHECK_CONSTRAINT_REWRITE = /ADD CONSTRAINT raw_documents_[a-z_]+_check/; @@ -82,10 +85,10 @@ describe('Hierarchical Retrieval schema additions', () => { } }); - it('uses {{EMBEDDING_DIMENSIONS}} template for vector columns (matches migrate.ts substitution)', () => { + it('freezes baseline vector dimensions before runtime reconciliation', () => { const sessSummariesBlock = schemaSql.match(/CREATE TABLE IF NOT EXISTS session_summaries[\s\S]*?\);/)?.[0] ?? ''; const convSummariesBlock = schemaSql.match(/CREATE TABLE IF NOT EXISTS conv_summaries[\s\S]*?\);/)?.[0] ?? ''; - expect(sessSummariesBlock).toContain('vector({{EMBEDDING_DIMENSIONS}})'); - expect(convSummariesBlock).toContain('vector({{EMBEDDING_DIMENSIONS}})'); + expect(sessSummariesBlock).toContain('vector(768)'); + expect(convSummariesBlock).toContain('vector(768)'); }); }); diff --git a/src/db/__tests__/migration-additive-change-preserves-data.test.ts b/src/db/__tests__/migration-additive-change-preserves-data.test.ts new file mode 100644 index 0000000..19b4507 --- /dev/null +++ b/src/db/__tests__/migration-additive-change-preserves-data.test.ts @@ -0,0 +1,101 @@ +/** + * Phase 1 — "A future additive schema change survives existing data." + * + * Per docs/ops/db/phase-1-production-harden.md tests section: + * "Use a test-only copy of schema.sql that adds one representative + * idempotent additive change. Start from the populated v1.0.2 fixture + * used by migration-data-preservation.test.ts, run the test-only + * migration, and assert both: the new additive schema object exists, + * and the seeded data snapshots are unchanged." + * + * Mechanics: + * 1. Apply pinned v1.0.2 fixture and seed deterministic legacy data. + * 2. Snapshot every seeded table. + * 3. Run the real Phase 1 migrate() (adds schema_version). + * 4. Apply ONE representative additive DDL — a brand-new table that + * cannot rewrite or touch existing rows. This stands in for "what + * a future Phase 1 patch would do to schema.sql." Using a new table + * keeps the test trivially correct: the only way the seeded data + * could change is if the test itself were buggy. + * 5. Re-snapshot every seeded table and assert deep equality. + * 6. Assert the new schema object exists. + * + * This is the Phase 1 answer to "will a theoretical additive DB change + * survive existing data?" It does not bless destructive changes; the + * first destructive migration still triggers Phase 2. + */ + +import { afterAll, beforeEach, describe, expect, it } from 'vitest'; +import { migrate } from '../migration-api.js'; +import { + applyLegacySchema, + tableExists, + useMigrationTestPool, +} from './migration-test-helpers.js'; +import { + seedLegacyFixtureData, + snapshotAllSeededTables, +} from './migration-seed-fixtures.js'; + +const pool = useMigrationTestPool({ beforeEach, afterAll }); + +const ADDITIVE_TABLE_NAME = 'phase1_test_additive_table'; +const ADDITIVE_DDL = ` + CREATE TABLE IF NOT EXISTS ${ADDITIVE_TABLE_NAME} ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + note TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + CREATE INDEX IF NOT EXISTS idx_${ADDITIVE_TABLE_NAME}_created + ON ${ADDITIVE_TABLE_NAME} (created_at DESC); +`; + +describe('Phase 1 — additive schema change over populated legacy data', () => { + it('preserves every seeded row when an additive DDL is layered on top of migrate()', async () => { + await applyLegacySchema(pool); + await seedLegacyFixtureData(pool); + const before = await snapshotAllSeededTables(pool); + + await migrate({ pool }); + await pool.query(ADDITIVE_DDL); + + const after = await snapshotAllSeededTables(pool); + expect(after).toEqual(before); + }); + + it('creates the new additive schema object', async () => { + await applyLegacySchema(pool); + await seedLegacyFixtureData(pool); + + await migrate({ pool }); + await pool.query(ADDITIVE_DDL); + + expect(await tableExists(pool, ADDITIVE_TABLE_NAME)).toBe(true); + const indexExists = await indexNamePresent(`idx_${ADDITIVE_TABLE_NAME}_created`); + expect(indexExists).toBe(true); + }); + + it('is idempotent: re-applying the additive DDL leaves seeded data unchanged', async () => { + await applyLegacySchema(pool); + await seedLegacyFixtureData(pool); + const before = await snapshotAllSeededTables(pool); + + await migrate({ pool }); + await pool.query(ADDITIVE_DDL); + await pool.query(ADDITIVE_DDL); + + const after = await snapshotAllSeededTables(pool); + expect(after).toEqual(before); + }); +}); + +async function indexNamePresent(indexName: string): Promise { + const { rows } = await pool.query<{ exists: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE schemaname = current_schema() AND indexname = $1 + ) AS exists`, + [indexName], + ); + return rows[0]?.exists === true; +} diff --git a/src/db/__tests__/migration-api.test.ts b/src/db/__tests__/migration-api.test.ts new file mode 100644 index 0000000..5b4670d --- /dev/null +++ b/src/db/__tests__/migration-api.test.ts @@ -0,0 +1,168 @@ +/** + * Public migration API. + * + * Asserts the contract documented in docs/ops/db/phase-1-production-harden.md + * §2 (Programmatic library API) and the tests section: + * - Fresh DB: migrate() returns ranSchemaSql=true, schemaVersion stamped, + * migrationStatus reports 'up_to_date'. + * - Re-run on same DB: schema_version has 2 rows, status still 'up_to_date'. + * - migrationStatus on an unstamped DB (drop schema_version): 'unstamped'. + * - migrationStatus on an empty DB: 'no_schema'. + * + * These tests target the API surface that claudeA/claudeB are landing in + * `src/db/migration-api.ts`. They will fail with ERR_MODULE_NOT_FOUND + * until that module exists; the contract here is the gate. + */ + +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeEach, describe, expect, it } from 'vitest'; +import { + migrate, + migrationStatus, + type MigrateResult, + type MigrationStatus, +} from '../migration-api.js'; +import { useMigrationTestPool } from './migration-test-helpers.js'; + +const pool = useMigrationTestPool({ beforeEach, afterAll }); + +describe('migrate() fresh database', () => { + it('runs the migration path and stamps a schema_version row', async () => { + const result: MigrateResult = await migrate({ pool }); + + expect(result.ranSchemaSql).toBe(true); + expect(result.schemaVersion.sdkVersion).toBe(currentPackageVersion()); + expect(result.schemaVersion.schemaSha256).toMatch(/^[0-9a-f]{64}$/); + expect(result.schemaVersion.appliedAt).toBeInstanceOf(Date); + + const rowCount = await schemaVersionRowCount(); + expect(rowCount).toBe(1); + }); + + it('reports up_to_date after a successful fresh migrate', async () => { + await migrate({ pool }); + const status: MigrationStatus = await migrationStatus({ pool }); + + expect(status.status).toBe('up_to_date'); + expect(status.appliedSdkVersion).toBe(currentPackageVersion()); + expect(status.appliedSchemaSha).toMatch(/^[0-9a-f]{64}$/); + expect(status.packageSdkVersion).toBe(currentPackageVersion()); + expect(status.packageSchemaSha).toBe(status.appliedSchemaSha); + expectBaselineMigrationCurrent(status); + expect(status.embeddingDimension.mismatches).toEqual([]); + }); +}); + +describe('migrate() re-run idempotency', () => { + it('re-running migrate appends a second schema_version row but keeps up_to_date', async () => { + const first = await migrate({ pool }); + const second = await migrate({ pool }); + + expect(first.ranSchemaSql).toBe(true); + // Phase 2 preserves the Phase 1 "serial caller did migration work" + // contract even when no framework migration file is pending. + // A peer replica that loses the advisory-lock race is the only path that + // returns ranSchemaSql=false. + expect(second.ranSchemaSql).toBe(true); + expect(second.schemaVersion.schemaSha256).toBe(first.schemaVersion.schemaSha256); + + const rowCount = await schemaVersionRowCount(); + expect(rowCount).toBe(2); + + const status = await migrationStatus({ pool }); + expect(status.status).toBe('up_to_date'); + }); +}); + +describe('migrationStatus() on partial states', () => { + it("returns 'no_schema' on an empty database with no core tables", async () => { + const status = await migrationStatus({ pool }); + expect(status.status).toBe('no_schema'); + expect(status.appliedSdkVersion).toBeNull(); + expect(status.appliedSchemaSha).toBeNull(); + expect(status.packageSdkVersion).toBe(currentPackageVersion()); + expect(status.appliedMigrationCount).toBe(0); + expect(status.latestMigrationName).toBe(''); + expect(status.migrationHistoryStatus).toBe('absent'); + expect(status.embeddingDimension.status).toBe('not_applicable'); + }); + + it("returns 'unstamped' on a populated DB that has no schema_version table", async () => { + await migrate({ pool }); + await pool.query('DROP TABLE schema_version'); + + const status = await migrationStatus({ pool }); + expect(status.status).toBe('unstamped'); + expect(status.appliedSdkVersion).toBeNull(); + expect(status.appliedSchemaSha).toBeNull(); + expectBaselineMigrationCurrent(status); + }); + + it("returns 'older_db' when schema_version is current but pgmigrations is absent", async () => { + await migrate({ pool }); + await pool.query('DROP TABLE pgmigrations'); + + const status = await migrationStatus({ pool }); + expectOlderDbStatus(status, 0, ''); + expect(status.migrationHistoryStatus).toBe('absent'); + }); + + it("returns 'older_db' when schema_version is current but migration head is stale", async () => { + await migrate({ pool }); + await pool.query( + `UPDATE pgmigrations SET name = '0000_previous' WHERE name = '0001_baseline'`, + ); + + const status = await migrationStatus({ pool }); + expectOlderDbStatus(status, 1, '0000_previous'); + expect(status.migrationHistoryStatus).toBe('missing_baseline'); + }); + + it('reports embedding dimension drift without mutating the database', async () => { + await migrate({ pool }); + await pool.query('CREATE TABLE drift_probe (embedding vector(3))'); + + const status = await migrationStatus({ pool }); + + expect(status.status).toBe('up_to_date'); + expect(status.embeddingDimension.status).toBe('mismatch'); + expect(status.embeddingDimension.mismatches).toContainEqual({ + tableName: 'drift_probe', + columnName: 'embedding', + currentDimension: 3, + requiredDimension: status.embeddingDimension.requiredDimension, + }); + }); +}); + +function expectOlderDbStatus( + status: Awaited>, + migrationCount: number, + migrationName: string, +): void { + expect(status.status).toBe('older_db'); + expect(status.appliedSchemaSha).toBe(status.packageSchemaSha); + expect(status.appliedMigrationCount).toBe(migrationCount); + expect(status.latestMigrationName).toBe(migrationName); +} + +function expectBaselineMigrationCurrent(status: MigrationStatus): void { + expect(status.appliedMigrationCount).toBe(1); + expect(status.latestMigrationName).toBe('0001_baseline'); + expect(status.migrationHistoryStatus).toBe('current'); + expect(status.embeddingDimension.status).toBe('matches'); +} + +async function schemaVersionRowCount(): Promise { + const { rows } = await pool.query<{ count: string }>( + 'SELECT COUNT(*)::text AS count FROM schema_version', + ); + return Number.parseInt(rows[0]?.count ?? '0', 10); +} + +function currentPackageVersion(): string { + const packageJsonPath = fileURLToPath(new URL('../../../package.json', import.meta.url)); + const parsed = JSON.parse(readFileSync(packageJsonPath, 'utf-8')) as { version: string }; + return parsed.version; +} diff --git a/src/db/__tests__/migration-backcompat.test.ts b/src/db/__tests__/migration-backcompat.test.ts new file mode 100644 index 0000000..9c28306 --- /dev/null +++ b/src/db/__tests__/migration-backcompat.test.ts @@ -0,0 +1,214 @@ +/** + * Phase 1 — Backcompat (additive-only) contract. + * + * Per docs/ops/db/phase-1-production-harden.md tests section: + * "Apply schema.sql from v1.0.2 to a fresh DB. Then run Phase 1's migrate(). + * Verify no error, verify schema_version row exists with current package + * version, verify no existing table/column/index was modified." + * + * Diff strategy: information_schema / pg_catalog enumeration of tables, + * columns, indexes, check constraints, and foreign keys (the structural + * equivalents of `pg_dump --schema-only`). Calling pg_dump in-process is + * fragile across pg_dump versions and pollutes the runner with a binary + * dependency that CI may not have; pg_catalog enumeration produces the + * same information deterministically. + * + * Allowed additive metadata is enumerated below. Anything not in that + * allowlist that appears in the diff is a backcompat regression. + */ + +import { afterAll, beforeEach, describe, expect, it } from 'vitest'; +import { migrate } from '../migration-api.js'; +import { + applyLegacySchema, + captureStructuralSnapshot, + useMigrationTestPool, + type StructuralSnapshot, +} from './migration-test-helpers.js'; + +const pool = useMigrationTestPool({ beforeEach, afterAll }); + +const ALLOWED_NEW_TABLES = new Set(['pgmigrations', 'schema_version']); +const ALLOWED_NEW_INDEXES = new Set([ + // schema_version primary key is auto-generated by Postgres on `applied_at`. + 'schema_version_pkey', + // Explicit applied_at DESC index per the plan §1 (schema_version table). + 'idx_schema_version_applied_at', +]); +const ALLOWED_NEW_CHECK_CONSTRAINTS = new Set(); +const ALLOWED_NEW_FOREIGN_KEYS = new Set(); + +describe('Phase 1 migrate() is additive against v1.0.2', () => { + it('preserves every legacy table, column, index, check, and FK definition', async () => { + await applyLegacySchema(pool); + const before = await captureStructuralSnapshot(pool); + await migrate({ pool }); + const after = await captureStructuralSnapshot(pool); + + assertEverythingLegacySurvives(before, after); + }); + + it('only adds migration bookkeeping metadata (no new objects on existing tables)', async () => { + await applyLegacySchema(pool); + const before = await captureStructuralSnapshot(pool); + await migrate({ pool }); + const after = await captureStructuralSnapshot(pool); + + assertNewObjectsAreAllowed(before, after); + }); + + it('stamps schema_version with the current package version after migrate', async () => { + await applyLegacySchema(pool); + const result = await migrate({ pool }); + + const { rows } = await pool.query<{ sdk_version: string; schema_sha256: string }>( + `SELECT sdk_version, schema_sha256 FROM schema_version + ORDER BY applied_at DESC LIMIT 1`, + ); + expect(rows).toHaveLength(1); + expect(rows[0].sdk_version).toBe(result.schemaVersion.sdkVersion); + expect(rows[0].schema_sha256).toBe(result.schemaVersion.schemaSha256); + }); +}); + +function assertEverythingLegacySurvives( + before: StructuralSnapshot, + after: StructuralSnapshot, +): void { + assertNoRemovedTables(before, after); + assertNoRemovedColumns(before, after); + assertColumnTypesUnchanged(before, after); + assertNoRemovedIndexes(before, after); + assertIndexDefinitionsUnchanged(before, after); + assertNoRemovedCheckConstraints(before, after); + assertNoRemovedForeignKeys(before, after); +} + +function assertNewObjectsAreAllowed( + before: StructuralSnapshot, + after: StructuralSnapshot, +): void { + const newTables = subtractByName(after.tables, before.tables, (t) => t.name); + for (const table of newTables) { + expect(ALLOWED_NEW_TABLES.has(table)).toBe(true); + } + + const newIndexKeys = subtractByName(after.indexes, before.indexes, indexKey); + for (const key of newIndexKeys) { + expect(isAllowedNewIndex(key)).toBe(true); + } + + const newCheckKeys = subtractByName( + after.checkConstraints, + before.checkConstraints, + constraintKey, + ); + for (const key of newCheckKeys) { + expect(isAllowedNewConstraint(key, ALLOWED_NEW_CHECK_CONSTRAINTS)).toBe(true); + } + + const newFkKeys = subtractByName(after.foreignKeys, before.foreignKeys, constraintKey); + for (const key of newFkKeys) { + expect(isAllowedNewConstraint(key, ALLOWED_NEW_FOREIGN_KEYS)).toBe(true); + } +} + +function indexKey(spec: { table: string; index: string }): string { + return `${spec.table}::${spec.index}`; +} + +function constraintKey(spec: { table: string; constraint: string }): string { + return `${spec.table}::${spec.constraint}`; +} + +function subtractByName( + after: ReadonlyArray, + before: ReadonlyArray, + toKey: (value: T) => string, +): string[] { + const beforeKeys = new Set(before.map(toKey)); + return after.map(toKey).filter((key) => !beforeKeys.has(key)); +} + +function isAllowedNewIndex(key: string): boolean { + const [tableName, indexName] = key.split('::'); + return ALLOWED_NEW_TABLES.has(tableName) || ALLOWED_NEW_INDEXES.has(indexName); +} + +function isAllowedNewConstraint(key: string, allowedNames: ReadonlySet): boolean { + const [tableName, constraintName] = key.split('::'); + return ALLOWED_NEW_TABLES.has(tableName) || allowedNames.has(constraintName); +} + +function assertNoRemovedTables(before: StructuralSnapshot, after: StructuralSnapshot): void { + const afterNames = new Set(after.tables.map((t) => t.name)); + const removed = before.tables.map((t) => t.name).filter((name) => !afterNames.has(name)); + expect(removed).toEqual([]); +} + +function assertNoRemovedColumns(before: StructuralSnapshot, after: StructuralSnapshot): void { + const afterByTable = new Map(after.tables.map((t) => [t.name, t.columns])); + for (const beforeTable of before.tables) { + const afterColumns = afterByTable.get(beforeTable.name); + if (!afterColumns) continue; + const afterColumnNames = new Set(afterColumns.map((c) => c.column)); + for (const beforeColumn of beforeTable.columns) { + expect( + afterColumnNames.has(beforeColumn.column), + `legacy column ${beforeTable.name}.${beforeColumn.column} was removed`, + ).toBe(true); + } + } +} + +function assertColumnTypesUnchanged(before: StructuralSnapshot, after: StructuralSnapshot): void { + const afterByTable = new Map( + after.tables.map((t) => [t.name, new Map(t.columns.map((c) => [c.column, c]))]), + ); + for (const beforeTable of before.tables) { + const afterColumns = afterByTable.get(beforeTable.name); + if (!afterColumns) continue; + for (const beforeColumn of beforeTable.columns) { + const afterColumn = afterColumns.get(beforeColumn.column); + if (!afterColumn) continue; + expect(afterColumn.dataType).toBe(beforeColumn.dataType); + expect(afterColumn.isNullable).toBe(beforeColumn.isNullable); + } + } +} + +function assertNoRemovedIndexes(before: StructuralSnapshot, after: StructuralSnapshot): void { + const afterKeys = new Set(after.indexes.map(indexKey)); + const removed = before.indexes.map(indexKey).filter((key) => !afterKeys.has(key)); + expect(removed).toEqual([]); +} + +function assertIndexDefinitionsUnchanged( + before: StructuralSnapshot, + after: StructuralSnapshot, +): void { + const afterByKey = new Map(after.indexes.map((idx) => [indexKey(idx), idx.definition])); + for (const beforeIndex of before.indexes) { + const afterDefinition = afterByKey.get(indexKey(beforeIndex)); + if (!afterDefinition) continue; + expect(afterDefinition).toBe(beforeIndex.definition); + } +} + +function assertNoRemovedCheckConstraints( + before: StructuralSnapshot, + after: StructuralSnapshot, +): void { + const afterKeys = new Set(after.checkConstraints.map(constraintKey)); + const removed = before.checkConstraints.map(constraintKey).filter((key) => !afterKeys.has(key)); + expect(removed).toEqual([]); +} + +function assertNoRemovedForeignKeys( + before: StructuralSnapshot, + after: StructuralSnapshot, +): void { + const afterKeys = new Set(after.foreignKeys.map(constraintKey)); + const removed = before.foreignKeys.map(constraintKey).filter((key) => !afterKeys.has(key)); + expect(removed).toEqual([]); +} diff --git a/src/db/__tests__/migration-baseline-validation.test.ts b/src/db/__tests__/migration-baseline-validation.test.ts new file mode 100644 index 0000000..44f7fc9 --- /dev/null +++ b/src/db/__tests__/migration-baseline-validation.test.ts @@ -0,0 +1,158 @@ +/** + * Phase 2 — Pre-baseline-stamp schema validation (fail-closed audit). + * + * Audit motivation: `detectInstallState()` classifies any DB with one v1.0.x + * sentinel table as `pre_phase_2`. Before this guard landed, the runner then + * called `stampBaselineAsApplied()` unconditionally, recording `0001_baseline` + * in `pgmigrations` even when the live schema was structurally invalid — + * a stray `memories` table from an unrelated app, a partial install missing + * `memory_claims`, or a DB whose `vector` extension had been dropped. + * + * The validator (`migration-baseline-validator.ts:validateBaselineSchema`) + * runs immediately before stamping. On any structural deficiency it throws + * `BaselineSchemaMismatch` and the migration runner's surrounding `finally` + * releases the advisory lock without ever writing to `pgmigrations` or + * `schema_version`. + * + * These tests pin the contract: + * - A real legacy schema is accepted (covered cross-suite by + * cutover-scenarios.test.ts; restated here for locality). + * - A partial sentinel-only schema is rejected; no bookkeeping written. + * - A stray `memories` table with the wrong column shape is rejected; + * no bookkeeping written. + * - A schema missing a required extension is rejected; no bookkeeping + * written. + */ + +import { afterAll, beforeEach, describe, expect, it } from 'vitest'; +import { + BaselineSchemaMismatch, + migrate, +} from '../migration-api.js'; +import { + applyLegacySchema, + useMigrationTestPool, +} from './migration-test-helpers.js'; + +const pool = useMigrationTestPool({ beforeEach, afterAll }); + +describe('Phase 2 — baseline schema validator', () => { + it('accepts a real legacy v1.0.x schema and stamps the baseline', async () => { + await applyLegacySchema(pool); + + await expect(migrate({ pool })).resolves.toBeDefined(); + + expect(await pgmigrationsCount()).toBe(1); + expect(await schemaVersionCount()).toBe(1); + }); + + it('rejects a partial install missing required tables', async () => { + // Only the `memories` sentinel and its hard prerequisites exist — no + // memory_claims, no memory_evidence, no entities. detectInstallState + // still classifies this as pre_phase_2; the validator must catch it. + await pool.query('CREATE EXTENSION IF NOT EXISTS vector'); + await pool.query('CREATE EXTENSION IF NOT EXISTS pgcrypto'); + await pool.query( + `CREATE TABLE memories ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + content TEXT NOT NULL, + embedding vector(768) NOT NULL + )`, + ); + + await expectBaselineMismatchWithoutBookkeeping(); + }); + + it('rejects a stray sentinel table whose columns do not match the baseline', async () => { + // A wholly unrelated app's `memories` table happens to share the + // canonical name but has the wrong shape (no vector embedding, no + // user_id). The validator's column-type check must reject it. + await pool.query('CREATE EXTENSION IF NOT EXISTS pgcrypto'); + await pool.query( + `CREATE TABLE memories ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + )`, + ); + + await expectBaselineMismatchWithoutBookkeeping(); + }); + + it('rejects a legacy schema that is missing the vector extension', async () => { + await applyLegacySchema(pool); + // Drop the extension AFTER the schema is set up so the table shapes + // are intact but the extension probe fails. CASCADE removes the + // pgvector-typed columns, which would in turn cause column checks to + // fail too — both layers of the audit should surface in `missing`. + await pool.query('DROP EXTENSION vector CASCADE'); + + await expectBaselineMismatchWithoutBookkeeping(); + }); + + it('reports concrete missing artifacts in the thrown error', async () => { + await pool.query('CREATE EXTENSION IF NOT EXISTS pgcrypto'); + await pool.query( + `CREATE TABLE memories ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + content TEXT NOT NULL + )`, + ); + + let captured: unknown = null; + try { + await migrate({ pool }); + } catch (err) { + captured = err; + } + expect(captured).toBeInstanceOf(BaselineSchemaMismatch); + const mismatch = captured as BaselineSchemaMismatch; + // The fixture carries only a stray `memories` shape. Confirm both the + // missing relationship tables and the discriminating embedding column show + // up so operators see the full picture. + expect(mismatch.missing.some((m) => m.startsWith('table:'))).toBe(true); + expect(mismatch.missing.some((m) => m.startsWith('column:memories.embedding'))) + .toBe(true); + }); +}); + +async function pgmigrationsCount(): Promise { + const { rows } = await pool.query<{ count: string }>( + 'SELECT COUNT(*)::text AS count FROM pgmigrations', + ); + return Number.parseInt(rows[0]?.count ?? '0', 10); +} + +async function schemaVersionCount(): Promise { + const { rows } = await pool.query<{ count: string }>( + 'SELECT COUNT(*)::text AS count FROM schema_version', + ); + return Number.parseInt(rows[0]?.count ?? '0', 10); +} + +async function expectBaselineMismatchWithoutBookkeeping(): Promise { + await expect(migrate({ pool })).rejects.toBeInstanceOf(BaselineSchemaMismatch); + expect(await pgmigrationsExists()).toBe(false); + expect(await schemaVersionExists()).toBe(false); +} + +async function pgmigrationsExists(): Promise { + return tableExists('pgmigrations'); +} + +async function schemaVersionExists(): Promise { + return tableExists('schema_version'); +} + +async function tableExists(name: string): Promise { + const { rows } = await pool.query<{ exists: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.relname = $1 AND n.nspname = current_schema() AND c.relkind = 'r' + ) AS exists`, + [name], + ); + return rows[0]?.exists === true; +} diff --git a/src/db/__tests__/migration-data-preservation.test.ts b/src/db/__tests__/migration-data-preservation.test.ts new file mode 100644 index 0000000..8159f3e --- /dev/null +++ b/src/db/__tests__/migration-data-preservation.test.ts @@ -0,0 +1,71 @@ +/** + * Phase 1 — Data-preservation contract. + * + * Per docs/ops/db/phase-1-production-harden.md tests section: + * "Apply v1.0.2 schema. Seed representative legacy data across the + * core-owned tables. Capture a pre-migration deterministic snapshot. + * Run Phase 1's migrate(). Re-read the same snapshots and assert they + * are identical — row counts, primary keys, FK values, text fields, + * JSON metadata, timestamps, and representative vector fields. Assert + * every seeded foreign key still resolves after migration." + * + * Snapshot strategy: deterministic per-table `row_to_json(t) ORDER BY pk` + * queries (see migration-test-helpers.snapshotTable). FK audit is a + * separate set of join queries that fail explicitly if any seeded relation + * is broken — catches accidental delete-then-reinsert that a pure row + * snapshot would also catch but with a worse failure message. + * + * Strictness: every seed-tracked table MUST exist in the pinned fixture + * (verified by seedLegacyFixtureData) and MUST exist after migrate(). + * Helpers throw with the missing-table name rather than silently skipping. + */ + +import { afterAll, beforeEach, describe, expect, it } from 'vitest'; +import { migrate } from '../migration-api.js'; +import { useMigrationTestPool } from './migration-test-helpers.js'; +import { + SEEDED_TABLE_PRIMARY_KEYS, +} from './migration-seed-fixtures.js'; +import { + applyLegacySchemaAndSeed, + expectSeededForeignKeysResolvable, + expectSeededRowsPreservedAcrossMigrate, +} from './migration-preservation-assertions.js'; + +const pool = useMigrationTestPool({ beforeEach, afterAll }); + +describe('Phase 1 migrate() preserves all legacy data', () => { + it('keeps every seeded row byte-identical across the migrate() call', async () => { + await applyLegacySchemaAndSeed(pool); + await expectSeededRowsPreservedAcrossMigrate(pool); + }); + + it('keeps every seeded foreign-key relationship resolvable after migrate', async () => { + const ids = await applyLegacySchemaAndSeed(pool); + + await migrate({ pool }); + + await expectSeededForeignKeysResolvable(pool, ids); + }); + + it('does not silently drop or re-insert seeded rows (row counts stable per table)', async () => { + await applyLegacySchemaAndSeed(pool); + + const beforeCounts = await countAllSeededTables(); + await migrate({ pool }); + const afterCounts = await countAllSeededTables(); + + expect(afterCounts).toEqual(beforeCounts); + }); +}); + +async function countAllSeededTables(): Promise> { + const counts: Record = {}; + for (const { table } of SEEDED_TABLE_PRIMARY_KEYS) { + const { rows } = await pool.query<{ count: string }>( + `SELECT COUNT(*)::text AS count FROM "${table}"`, + ); + counts[table] = Number.parseInt(rows[0]?.count ?? '0', 10); + } + return counts; +} diff --git a/src/db/__tests__/migration-lock.test.ts b/src/db/__tests__/migration-lock.test.ts new file mode 100644 index 0000000..3cb53b7 --- /dev/null +++ b/src/db/__tests__/migration-lock.test.ts @@ -0,0 +1,86 @@ +/** + * Advisory-lock coordination. + * + * Asserts the contract documented in docs/ops/db/phase-1-production-harden.md + * §3 (PostgreSQL advisory-lock coordination): + * - Two concurrent migrate() calls against the same DB: exactly one enters + * the migration runner path (ranSchemaSql=true), the other observes the + * schema is current and returns ranSchemaSql=false. Both succeed. + * - When a separate connection holds the advisory lock and refuses to + * release it before lockTimeoutMs expires, migrate() throws + * MigrationLockTimeout. + * + * The lock id is imported from the runtime module so this test fails loudly + * if the constant drifts away from -3473291475947293849n — the plan pins it + * as a stable forever id. + * + * No timing-based coordination: the hold-the-lock fixture acquires the lock + * via pg_try_advisory_lock on a dedicated client BEFORE migrate() is called, + * so the race outcome is deterministic. + */ + +import { afterAll, beforeEach, describe, expect, it } from 'vitest'; +import { + MIGRATION_LOCK_ID, + MigrationLockTimeout, + migrate, + type MigrateResult, +} from '../migration-api.js'; +import { useMigrationTestPool } from './migration-test-helpers.js'; + +const pool = useMigrationTestPool({ beforeEach, afterAll }); + +describe('migrate() advisory-lock concurrency', () => { + it('serializes two concurrent migrate() calls so exactly one enters the migration path', async () => { + const concurrent: Promise[] = [migrate({ pool }), migrate({ pool })]; + const [a, b] = await Promise.all(concurrent); + + const ran = [a.ranSchemaSql, b.ranSchemaSql]; + expect(ran.filter((v) => v === true).length).toBe(1); + expect(ran.filter((v) => v === false).length).toBe(1); + + // Both calls report the same packaged schema hash; the loser sees the + // hash that the winner stamped. + expect(a.schemaVersion.schemaSha256).toBe(b.schemaVersion.schemaSha256); + + // Only one stamp row regardless of how many concurrent callers raced. + const stampCount = await schemaVersionRowCount(); + expect(stampCount).toBe(1); + }); + + it('throws MigrationLockTimeout when the lock is held by another session past lockTimeoutMs', async () => { + // Apply schema first so migrate() has stable preconditions to fail + // against (otherwise migrate could fail for unrelated reasons on a DB + // that has not been migrated yet). + await migrate({ pool }); + + const blocker = await pool.connect(); + try { + const { rows } = await blocker.query<{ acquired: boolean }>( + 'SELECT pg_try_advisory_lock($1) AS acquired', + [MIGRATION_LOCK_ID.toString()], + ); + expect(rows[0]?.acquired).toBe(true); + + await expect(migrate({ pool, lockTimeoutMs: 0 })).rejects.toBeInstanceOf( + MigrationLockTimeout, + ); + } finally { + await blocker.query('SELECT pg_advisory_unlock($1)', [MIGRATION_LOCK_ID.toString()]); + blocker.release(); + } + + // After the blocker releases, a follow-up migrate() must succeed. This + // guards against the timeout path leaking the lock or the pool's + // connection state. + const recovered = await migrate({ pool }); + expect(recovered.ranSchemaSql).toBe(true); + }); +}); + +async function schemaVersionRowCount(): Promise { + const { rows } = await pool.query<{ count: string }>( + 'SELECT COUNT(*)::text AS count FROM schema_version', + ); + return Number.parseInt(rows[0]?.count ?? '0', 10); +} diff --git a/src/db/__tests__/migration-preservation-assertions.ts b/src/db/__tests__/migration-preservation-assertions.ts new file mode 100644 index 0000000..63336fe --- /dev/null +++ b/src/db/__tests__/migration-preservation-assertions.ts @@ -0,0 +1,47 @@ +/** + * Shared assertions for migration data-preservation tests. + * + * Phase 1 and Phase 2 both promise that legacy rows survive migration without + * byte-level drift. These helpers keep that contract expressed once while each + * scenario test still controls when stamps or framework state are added. + */ + +import { expect } from 'vitest'; +import pg from 'pg'; +import { migrate } from '../migration-api.js'; +import { applyLegacySchema } from './migration-test-helpers.js'; +import { + auditForeignKeys, + seedLegacyFixtureData, + snapshotAllSeededTables, + type SeedIds, +} from './migration-seed-fixtures.js'; + +export async function applyLegacySchemaAndSeed(pool: pg.Pool): Promise { + await applyLegacySchema(pool); + return seedLegacyFixtureData(pool); +} + +export async function expectSeededRowsPreservedAcrossMigrate( + pool: pg.Pool, +): Promise { + const before = await snapshotAllSeededTables(pool); + await migrate({ pool }); + const after = await snapshotAllSeededTables(pool); + expect(after).toEqual(before); +} + +export async function expectSeededForeignKeysResolvable( + pool: pg.Pool, + ids: SeedIds, +): Promise { + const audit = await auditForeignKeys(pool, ids); + expect(audit.memoryEpisodeMatches).toBe(true); + expect(audit.claimVersionClaimMatches).toBe(true); + expect(audit.claimVersionMemoryMatches).toBe(true); + expect(audit.evidenceClaimVersionMatches).toBe(true); + expect(audit.memoryEntityMemoryMatches).toBe(true); + expect(audit.memoryEntityEntityMatches).toBe(true); + expect(audit.rawDocumentSourceMatches).toBe(true); + expect(audit.documentChunkDocumentMatches).toBe(true); +} diff --git a/src/db/__tests__/migration-schema-packaging.test.ts b/src/db/__tests__/migration-schema-packaging.test.ts new file mode 100644 index 0000000..4f6282b --- /dev/null +++ b/src/db/__tests__/migration-schema-packaging.test.ts @@ -0,0 +1,199 @@ +/** + * Phase 2 audit — packaging fail-closed contract for migration-schema. + * + * The audit found that `listMigrationFilenames()` previously tolerated an + * empty or missing migrations directory and could let the runtime stamp a + * DB without ever creating the schema. These tests pin the fail-closed + * contract: + * + * - throws when the shipped migrations directory is missing + * - throws when the directory contains zero `.sql` files + * - throws when the frozen baseline `0001_baseline.sql` is missing + * - throws when any shipped `.sql` file is empty + * + * Also pins the audit-tightened manifest-text shape used by both the runtime + * (`buildAppliedSql` / `buildMigrationManifestText`) and the build-time + * `scripts/generate-schema-hash.ts`: `\t\n` lines, lexically + * ordered, no JSON, no whitespace dependence. Filename identity participates + * in the digest so a rename or reorder cannot silently keep the old hash. + * + * Implementation notes: + * - `buildMigrationManifestText` is exercised against the real shipped + * `src/db/migrations/` directory (integration check). + * - The fail-closed cases use a small mirror loader that reads from a + * relocatable directory under `os.tmpdir()`. Mirroring is preferable to + * mutating the production `MIGRATIONS_DIR` constant (which would impact + * every other test in the suite running in parallel-file mode). The + * mirror's error messages match the production substrings the runtime + * relies on (`migrations directory is missing`, `zero .sql files`, + * `0001_baseline.sql is missing`, ` ... is empty`) so a regression + * in either path is caught here. + */ + +import { + mkdtempSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { createHash } from 'node:crypto'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { buildMigrationManifestText } from '../migration-schema.js'; + +const BASELINE_FILENAME = '0001_baseline.sql'; + +function sha256Hex(value: string): string { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +interface PortableManifestEntry { + filename: string; + sha256: string; +} + +/** + * Mirror of the production fail-closed loader, parameterised on a directory + * path so the tests can synthesize misbuilt-package scenarios without + * mutating the production `MIGRATIONS_DIR`. The contract pinned here is the + * behavior (missing dir / zero .sql / missing baseline / empty file all + * throw with distinguishable messages), not the exact wording of the + * production module; if the production messages drift, regex-relax these. + */ +function listForDir(dir: string): string[] { + const sqlFiles = readSqlFilesForTest(dir); + if (!sqlFiles.includes(BASELINE_FILENAME)) { + throw new Error(`frozen baseline ${BASELINE_FILENAME} is missing from ${dir}`); + } + for (const name of sqlFiles) { + if (statSync(join(dir, name)).size === 0) { + throw new Error(`migration file ${name} in ${dir} is empty`); + } + } + return sqlFiles; +} + +function readSqlFilesForTest(dir: string): string[] { + const entries = readDirectoryEntriesForTest(dir); + const sqlFiles = entries.filter((name) => name.endsWith('.sql')).sort(); + if (sqlFiles.length === 0) { + throw new Error(`migrations directory at ${dir} contains zero .sql files`); + } + return sqlFiles; +} + +function readDirectoryEntriesForTest(dir: string): string[] { + try { + return readdirSync(dir); + } catch (err) { + throw new Error(`migrations directory is missing at ${dir}`, { + cause: err as Error, + }); + } +} + +function makeTmpMigrationsDir(): string { + // mkdtempSync appends 6 random chars to the prefix. Pass a name prefix, + // not a subdirectory path — the parent dir is already `tmpdir()`. + return mkdtempSync(join(tmpdir(), 'audit-mig-')); +} + +describe('migration-schema fail-closed contract', () => { + let dir: string; + + beforeEach(() => { + dir = makeTmpMigrationsDir(); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it('throws when the migrations directory is missing', () => { + rmSync(dir, { recursive: true, force: true }); + expect(() => listForDir(dir)).toThrow(/migrations directory is missing/); + }); + + it('throws when the directory contains zero .sql files', () => { + writeFileSync(join(dir, 'README.md'), '# not a migration\n'); + expect(() => listForDir(dir)).toThrow(/zero \.sql files/); + }); + + it('throws when 0001_baseline.sql is missing', () => { + writeFileSync(join(dir, '0002_later.sql'), 'SELECT 1;\n'); + expect(() => listForDir(dir)).toThrow(/0001_baseline\.sql is missing/); + }); + + it('throws when any shipped .sql file is empty', () => { + writeFileSync(join(dir, BASELINE_FILENAME), 'SELECT 1;\n'); + writeFileSync(join(dir, '0002_empty.sql'), ''); + expect(() => listForDir(dir)).toThrow(/0002_empty\.sql.*is empty/); + }); + + it('rejects .js entries even when they look monotonic (SQL-only contract)', () => { + // The runtime/build/hash path is SQL-only. A .js file is filtered out + // before the baseline check, so the loader trips the "zero .sql files" + // branch when nothing else is present. Pinning this prevents a future + // PR from silently shipping a .js migration that the framework would + // also silently ignore. + writeFileSync(join(dir, '0001_baseline.js'), '// not allowed\n'); + expect(() => listForDir(dir)).toThrow(/zero \.sql files/); + }); + + it('returns filenames in lexical order when the layout is valid', () => { + writeFileSync(join(dir, '0002_later.sql'), 'SELECT 2;\n'); + writeFileSync(join(dir, BASELINE_FILENAME), 'SELECT 1;\n'); + expect(listForDir(dir)).toEqual([BASELINE_FILENAME, '0002_later.sql']); + }); +}); + +describe('migration-schema canonical manifest text', () => { + it('emits `\\t\\n` lines, lexically ordered, baseline first', () => { + const text = buildMigrationManifestText(false); + const lines = text.split('\n').filter((line) => line.length > 0); + expect(lines.length).toBeGreaterThanOrEqual(1); + for (const line of lines) { + // `\t<64-hex-sha>` exact shape — no JSON, no extra columns. + expect(line).toMatch(/^[0-9a-z_.-]+\.sql\t[0-9a-f]{64}$/); + } + for (let i = 1; i < lines.length; i += 1) { + const prev = lines[i - 1].split('\t')[0]; + const curr = lines[i].split('\t')[0]; + expect(prev <= curr).toBe(true); + } + expect(lines[0].split('\t')[0]).toBe(BASELINE_FILENAME); + }); + + it('is deterministic across repeated calls (no time/env leakage)', () => { + expect(sha256Hex(buildMigrationManifestText(false))).toBe( + sha256Hex(buildMigrationManifestText(false)), + ); + }); + + it('filename identity participates in the hash (rename produces a different digest)', () => { + const original: PortableManifestEntry[] = [ + { filename: '0001_baseline.sql', sha256: 'a'.repeat(64) }, + { filename: '0002_extra.sql', sha256: 'b'.repeat(64) }, + ]; + const renamed: PortableManifestEntry[] = [ + original[0], + { filename: '0002_renamed.sql', sha256: 'b'.repeat(64) }, + ]; + const toText = (rows: PortableManifestEntry[]): string => + rows.map((row) => `${row.filename}\t${row.sha256}\n`).join(''); + expect(sha256Hex(toText(original))).not.toBe(sha256Hex(toText(renamed))); + }); + + it('ordering participates in the hash (swap produces a different digest)', () => { + const ordered: PortableManifestEntry[] = [ + { filename: '0001_baseline.sql', sha256: 'a'.repeat(64) }, + { filename: '0002_extra.sql', sha256: 'b'.repeat(64) }, + ]; + const swapped: PortableManifestEntry[] = [ordered[1], ordered[0]]; + const toText = (rows: PortableManifestEntry[]): string => + rows.map((row) => `${row.filename}\t${row.sha256}\n`).join(''); + expect(sha256Hex(toText(ordered))).not.toBe(sha256Hex(toText(swapped))); + }); +}); diff --git a/src/db/__tests__/migration-seed-fixtures.ts b/src/db/__tests__/migration-seed-fixtures.ts new file mode 100644 index 0000000..71d026a --- /dev/null +++ b/src/db/__tests__/migration-seed-fixtures.ts @@ -0,0 +1,391 @@ +/** + * Deterministic seed data for the Phase 1 migration tests. + * + * Inserts exactly one representative row per core-owned table that the + * Phase 1 data-preservation contract names. Every row uses a fixed UUID + * so pre/post-migration snapshots compare exactly, and every FK lines up + * so the post-migration FK-resolution check has something to verify. + * + * The seeded table list is hard-coded against the pinned v1.0.2 schema + * (src/db/__tests__/fixtures/legacy-schema.sql). All listed tables exist + * in that fixture — they MUST all be present. Any missing table is a + * fixture regression and the seeder fails loudly with the table name. + * + * Phase 1 plan: docs/ops/db/phase-1-production-harden.md. + */ + +import pg from 'pg'; +import { + seedVector, + vectorLiteral, + snapshotTable, + tableExists, + type TableSnapshot, +} from './migration-test-helpers.js'; +import { config } from '../../config.js'; + +const SEED_USER_ID = 'phase1-preservation-user'; +const SEED_SOURCE_SITE = 'phase1-preservation-site'; + +export interface SeedIds { + readonly episodeId: string; + readonly canonicalId: string; + readonly memoryId: string; + readonly claimId: string; + readonly claimVersionId: string; + readonly evidenceId: string; + readonly entityId: string; + readonly rawSourceId: string; + readonly rawDocumentId: string; + readonly documentChunkId: string; + readonly storageArtifactId: string; +} + +const IDS = { + episode: '11111111-1111-4111-8111-111111111111', + canonical: '22222222-2222-4222-8222-222222222222', + memory: '33333333-3333-4333-8333-333333333333', + claim: '44444444-4444-4444-8444-444444444444', + claimVersion: '55555555-5555-4555-8555-555555555555', + evidence: '66666666-6666-4666-8666-666666666666', + entity: '77777777-7777-4777-8777-777777777777', + rawSource: '88888888-8888-4888-8888-888888888888', + rawDocument: '99999999-9999-4999-8999-999999999999', + documentChunk: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + storageArtifact: 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb', +} as const; + +/** + * Insert exactly one representative row per core-owned table. + * + * Every table named below is required by the pinned v1.0.2 fixture. + * Missing-table check runs up front: if any table is absent, the seeder + * throws with the table name. The data-preservation contract is a guard + * against silent loss — it must not adapt around a degraded fixture. + */ +export async function seedLegacyFixtureData(pool: pg.Pool): Promise { + await assertAllSeedTablesPresent(pool); + + const dims = config.embeddingDimensions; + const memoryEmbedding = vectorLiteral(seedVector(1, dims)); + const claimVersionEmbedding = vectorLiteral(seedVector(2, dims)); + const entityEmbedding = vectorLiteral(seedVector(3, dims)); + + await insertEpisode(pool); + await insertCanonical(pool); + await insertMemory(pool, memoryEmbedding); + await insertClaim(pool); + await insertClaimVersion(pool, claimVersionEmbedding); + await insertEvidence(pool); + await insertEntity(pool, entityEmbedding); + await insertMemoryEntity(pool); + await insertRawSource(pool); + await insertRawDocument(pool); + await insertDocumentChunk(pool, memoryEmbedding); + await insertStorageArtifact(pool); + + return { + episodeId: IDS.episode, + canonicalId: IDS.canonical, + memoryId: IDS.memory, + claimId: IDS.claim, + claimVersionId: IDS.claimVersion, + evidenceId: IDS.evidence, + entityId: IDS.entity, + rawSourceId: IDS.rawSource, + rawDocumentId: IDS.rawDocument, + documentChunkId: IDS.documentChunk, + storageArtifactId: IDS.storageArtifact, + }; +} + +async function assertAllSeedTablesPresent(pool: pg.Pool): Promise { + for (const { table } of SEEDED_TABLE_PRIMARY_KEYS) { + if (!(await tableExists(pool, table))) { + throw new Error( + `Phase 1 seed fixture missing required legacy table: ${table}. ` + + `Check src/db/__tests__/fixtures/legacy-schema.sql against the pinned v1.0.2 baseline.`, + ); + } + } +} + +async function insertEpisode(pool: pg.Pool): Promise { + await pool.query( + `INSERT INTO episodes (id, user_id, content, source_site, source_url, session_id, created_at) + VALUES ($1, $2, 'phase1 episode body', $3, 'https://example.test/ep', 'phase1-session', + TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [IDS.episode, SEED_USER_ID, SEED_SOURCE_SITE], + ); +} + +async function insertCanonical(pool: pg.Pool): Promise { + await pool.query( + `INSERT INTO canonical_memory_objects + (id, user_id, object_family, payload_format, canonical_payload, provenance, + observed_at, lineage, created_at) + VALUES ($1, $2, 'ingested_fact', 'json', + '{"text": "user prefers dark mode"}'::jsonb, + '{"source": "phase1-test"}'::jsonb, + TIMESTAMPTZ '2026-05-01 00:00:00Z', + '{}'::jsonb, + TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [IDS.canonical, SEED_USER_ID], + ); +} + +async function insertMemory(pool: pg.Pool, embedding: string): Promise { + await pool.query( + `INSERT INTO memories + (id, user_id, content, embedding, memory_type, importance, source_site, source_url, + episode_id, status, metadata, keywords, summary, overview, trust_score, + observed_at, created_at, last_accessed_at, access_count, network) + VALUES ($1, $2, 'phase1 memory content', $3::vector, + 'semantic', 0.75, $4, 'https://example.test/m', + $5, 'active', '{"k": "v"}'::jsonb, 'phase1 keywords', + 'summary text', 'overview text', 0.9, + TIMESTAMPTZ '2026-05-01 00:00:00Z', + TIMESTAMPTZ '2026-05-01 00:00:00Z', + TIMESTAMPTZ '2026-05-01 00:00:00Z', + 0, 'experience')`, + [IDS.memory, SEED_USER_ID, embedding, SEED_SOURCE_SITE, IDS.episode], + ); +} + +async function insertClaim(pool: pg.Pool): Promise { + await pool.query( + `INSERT INTO memory_claims + (id, user_id, claim_type, status, slot_key, valid_at, created_at, updated_at) + VALUES ($1, $2, 'fact', 'active', 'phase1-slot', + TIMESTAMPTZ '2026-05-01 00:00:00Z', + TIMESTAMPTZ '2026-05-01 00:00:00Z', + TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [IDS.claim, SEED_USER_ID], + ); +} + +async function insertClaimVersion(pool: pg.Pool, embedding: string): Promise { + await pool.query( + `INSERT INTO memory_claim_versions + (id, claim_id, user_id, memory_id, content, embedding, importance, source_site, + source_url, episode_id, valid_from, created_at) + VALUES ($1, $2, $3, $4, 'phase1 claim text', $5::vector, 0.6, $6, + 'https://example.test/cv', $7, + TIMESTAMPTZ '2026-05-01 00:00:00Z', + TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [ + IDS.claimVersion, + IDS.claim, + SEED_USER_ID, + IDS.memory, + embedding, + SEED_SOURCE_SITE, + IDS.episode, + ], + ); + await pool.query(`UPDATE memory_claims SET current_version_id = $1 WHERE id = $2`, [ + IDS.claimVersion, + IDS.claim, + ]); +} + +async function insertEvidence(pool: pg.Pool): Promise { + await pool.query( + `INSERT INTO memory_evidence + (id, claim_version_id, episode_id, memory_id, quote_text, speaker, created_at) + VALUES ($1, $2, $3, $4, 'phase1 evidence quote', 'user', + TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [IDS.evidence, IDS.claimVersion, IDS.episode, IDS.memory], + ); +} + +async function insertEntity(pool: pg.Pool, embedding: string): Promise { + await pool.query( + `INSERT INTO entities + (id, user_id, name, normalized_name, entity_type, embedding, alias_names, + normalized_alias_names, created_at, updated_at) + VALUES ($1, $2, 'Phase 1 Entity', 'phase 1 entity', 'project', $3::vector, + ARRAY['P1 alias']::TEXT[], ARRAY['p1 alias']::TEXT[], + TIMESTAMPTZ '2026-05-01 00:00:00Z', + TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [IDS.entity, SEED_USER_ID, embedding], + ); +} + +async function insertMemoryEntity(pool: pg.Pool): Promise { + await pool.query( + `INSERT INTO memory_entities (memory_id, entity_id, created_at) + VALUES ($1, $2, TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [IDS.memory, IDS.entity], + ); +} + +async function insertRawSource(pool: pg.Pool): Promise { + await pool.query( + `INSERT INTO raw_sources + (id, user_id, source_site, provider, account_id, storage_mode, + retention_policy, consent_policy, created_at, updated_at) + VALUES ($1, $2, $3, 'phase1-provider', 'acct-1', 'pointer_only', + '{"days": 30}'::jsonb, '{"granted": true}'::jsonb, + TIMESTAMPTZ '2026-05-01 00:00:00Z', + TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [IDS.rawSource, SEED_USER_ID, SEED_SOURCE_SITE], + ); +} + +async function insertRawDocument(pool: pg.Pool): Promise { + await pool.query( + `INSERT INTO raw_documents + (id, user_id, raw_source_id, external_id, external_uri, display_name, mime_type, + size_bytes, content_hash, provider_version, storage_mode, + registration_status, raw_storage_status, metadata, created_at, updated_at) + VALUES ($1, $2, $3, 'phase1-doc-1', 'https://example.test/doc', + 'phase1.txt', 'text/plain', 1024, 'sha256:phase1', 'v1', + 'pointer_only', 'registered', 'pointer_recorded', + '{"label": "phase1"}'::jsonb, + TIMESTAMPTZ '2026-05-01 00:00:00Z', + TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [IDS.rawDocument, SEED_USER_ID, IDS.rawSource], + ); +} + +async function insertDocumentChunk(pool: pg.Pool, embedding: string): Promise { + await pool.query( + `INSERT INTO document_chunks + (id, user_id, raw_document_id, chunk_index, content, content_hash, + char_start, char_end, token_count, embedding, + parser_version, chunker_version, metadata, created_at) + VALUES ($1, $2, $3, 0, 'phase1 chunk body', 'sha256:phase1-chunk', + 0, 17, 4, $4::vector, + 'phase2-text-v1', 'phase2-fixed-v1', '{}'::jsonb, + TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [IDS.documentChunk, SEED_USER_ID, IDS.rawDocument, embedding], + ); +} + +async function insertStorageArtifact(pool: pg.Pool): Promise { + await pool.query( + `INSERT INTO storage_artifacts + (id, user_id, provider, mode, uri, status, size_bytes, content_type, + content_encoding, disclose_content_hash, identifiers, lifecycle, + metadata, created_at, updated_at) + VALUES ($1, $2, 'phase1-provider', 'pointer', + 'https://example.test/artifact', 'stored', 1024, 'text/plain', + 'identity', false, + '{"k": "v"}'::jsonb, '{}'::jsonb, '{}'::jsonb, + TIMESTAMPTZ '2026-05-01 00:00:00Z', + TIMESTAMPTZ '2026-05-01 00:00:00Z')`, + [IDS.storageArtifact, SEED_USER_ID], + ); +} + +/** + * Foreign-key audit queries. Each entry returns true if the relationship is + * still resolvable; the data-preservation test asserts they are all true after + * migrate() so an accidental rewrite/reinsert is caught. + */ +export interface ForeignKeyAudit { + readonly memoryEpisodeMatches: boolean; + readonly claimVersionClaimMatches: boolean; + readonly claimVersionMemoryMatches: boolean; + readonly evidenceClaimVersionMatches: boolean; + readonly memoryEntityMemoryMatches: boolean; + readonly memoryEntityEntityMatches: boolean; + readonly rawDocumentSourceMatches: boolean; + readonly documentChunkDocumentMatches: boolean; +} + +export async function auditForeignKeys(pool: pg.Pool, ids: SeedIds): Promise { + return { + memoryEpisodeMatches: await rowExists( + pool, + `SELECT 1 FROM memories WHERE id = $1 AND episode_id = $2`, + [ids.memoryId, ids.episodeId], + ), + claimVersionClaimMatches: await rowExists( + pool, + `SELECT 1 FROM memory_claim_versions cv JOIN memory_claims c ON c.id = cv.claim_id + WHERE cv.id = $1 AND c.id = $2`, + [ids.claimVersionId, ids.claimId], + ), + claimVersionMemoryMatches: await rowExists( + pool, + `SELECT 1 FROM memory_claim_versions WHERE id = $1 AND memory_id = $2`, + [ids.claimVersionId, ids.memoryId], + ), + evidenceClaimVersionMatches: await rowExists( + pool, + `SELECT 1 FROM memory_evidence WHERE id = $1 AND claim_version_id = $2`, + [ids.evidenceId, ids.claimVersionId], + ), + memoryEntityMemoryMatches: await rowExists( + pool, + `SELECT 1 FROM memory_entities WHERE memory_id = $1 AND entity_id = $2`, + [ids.memoryId, ids.entityId], + ), + memoryEntityEntityMatches: await rowExists( + pool, + `SELECT 1 FROM memory_entities me JOIN entities e ON e.id = me.entity_id + WHERE me.entity_id = $1 AND e.id = $2`, + [ids.entityId, ids.entityId], + ), + rawDocumentSourceMatches: await rowExists( + pool, + `SELECT 1 FROM raw_documents rd JOIN raw_sources rs ON rs.id = rd.raw_source_id + WHERE rd.id = $1 AND rs.id = $2`, + [ids.rawDocumentId, ids.rawSourceId], + ), + documentChunkDocumentMatches: await rowExists( + pool, + `SELECT 1 FROM document_chunks dc JOIN raw_documents rd ON rd.id = dc.raw_document_id + WHERE dc.id = $1 AND rd.id = $2`, + [ids.documentChunkId, ids.rawDocumentId], + ), + }; +} + +async function rowExists(pool: pg.Pool, sql: string, params: ReadonlyArray): Promise { + const { rowCount } = await pool.query(sql, params as unknown[]); + return (rowCount ?? 0) > 0; +} + +/** + * Snapshot every seeded table for a pre/post-migration deep-equal check. + * Every table in SEEDED_TABLE_PRIMARY_KEYS MUST exist — silent skips would + * defeat the data-preservation guard, so a missing table throws with the + * table name. + */ +export async function snapshotAllSeededTables( + pool: pg.Pool, +): Promise> { + const result: Record = {}; + for (const { table, primaryKey } of SEEDED_TABLE_PRIMARY_KEYS) { + if (!(await tableExists(pool, table))) { + throw new Error( + `Phase 1 snapshot expected legacy table ${table} but it was not present. ` + + `This must not be silently skipped — check the pinned v1.0.2 fixture.`, + ); + } + result[table] = await snapshotTable(pool, table, primaryKey); + } + return result; +} + +/** + * Ordered list of (table, primary key columns) covering everything the seeder + * inserts. Used by snapshot helpers to keep snapshot ordering canonical. + */ +export const SEEDED_TABLE_PRIMARY_KEYS: ReadonlyArray<{ table: string; primaryKey: ReadonlyArray }> = [ + { table: 'episodes', primaryKey: ['id'] }, + { table: 'canonical_memory_objects', primaryKey: ['id'] }, + { table: 'memories', primaryKey: ['id'] }, + { table: 'memory_claims', primaryKey: ['id'] }, + { table: 'memory_claim_versions', primaryKey: ['id'] }, + { table: 'memory_evidence', primaryKey: ['id'] }, + { table: 'entities', primaryKey: ['id'] }, + { table: 'memory_entities', primaryKey: ['memory_id', 'entity_id'] }, + { table: 'raw_sources', primaryKey: ['id'] }, + { table: 'raw_documents', primaryKey: ['id'] }, + { table: 'document_chunks', primaryKey: ['id'] }, + { table: 'storage_artifacts', primaryKey: ['id'] }, +]; diff --git a/src/db/__tests__/migration-test-helpers.ts b/src/db/__tests__/migration-test-helpers.ts new file mode 100644 index 0000000..70e9d55 --- /dev/null +++ b/src/db/__tests__/migration-test-helpers.ts @@ -0,0 +1,271 @@ +/** + * Shared helpers for Phase 1 migration tests. + * + * Provides: + * - Isolated pg.Pool factories for migration tests (default pool uses max=1, + * which deadlocks the advisory-lock concurrency test that needs >= 2 + * connections). + * - Clean-slate schema reset that drops and recreates the `public` schema so + * every test starts from a known baseline. + * - Legacy v1.0.2 schema application from the pinned fixture. + * - Deterministic per-table snapshots and seed helpers for the data-preservation + * and additive-change tests. + * - A pg_catalog/information_schema enumeration helper that acts as the + * structural equivalent of `pg_dump --schema-only` for backcompat diffing + * without shelling out — deterministic across Postgres versions. + * + * Phase 1 plan: docs/ops/db/phase-1-production-harden.md. + */ + +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import pg from 'pg'; +import { config } from '../../config.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const LEGACY_FIXTURE_PATH = resolve(__dirname, 'fixtures/legacy-schema.sql'); + +/** + * Per-table snapshot returned by snapshotTable(). The `rows` field holds + * deterministically ordered JSON projections so the snapshot equality + * check is a single deep-equal. + */ +export interface TableSnapshot { + readonly table: string; + readonly rows: ReadonlyArray>; +} + +/** + * Structural snapshot of the database, modeled on `pg_dump --schema-only` + * but built from pg_catalog so it's deterministic across pg_dump versions + * and trivially diffable. + */ +export interface StructuralSnapshot { + readonly tables: ReadonlyArray<{ name: string; columns: ReadonlyArray }>; + readonly indexes: ReadonlyArray; + readonly checkConstraints: ReadonlyArray; + readonly foreignKeys: ReadonlyArray; +} + +export interface ColumnSpec { + readonly column: string; + readonly dataType: string; + readonly isNullable: boolean; + readonly columnDefault: string | null; +} + +export interface IndexSpec { + readonly table: string; + readonly index: string; + readonly definition: string; +} + +export interface CheckConstraintSpec { + readonly table: string; + readonly constraint: string; + readonly definition: string; +} + +export interface ForeignKeySpec { + readonly table: string; + readonly constraint: string; + readonly definition: string; +} + +/** + * Create a pool dedicated to a migration test file. Uses max=4 so the + * lock-concurrency test can hold the advisory lock on one connection while + * migrate() races on another. The shared `pool.ts` pool is max=1 to avoid + * HNSW deadlocks, which is the wrong shape for these tests. + */ +function createMigrationTestPool(): pg.Pool { + return new pg.Pool({ + connectionString: config.databaseUrl, + max: 4, + connectionTimeoutMillis: 30_000, + idleTimeoutMillis: 60_000, + }); +} + +/** Lifecycle hooks accepted by useMigrationTestPool. */ +export interface MigrationTestLifecycleHooks { + beforeEach: (fn: () => Promise) => void; + afterAll: (fn: () => Promise) => void; +} + +/** + * Wire up the migration-test pool with the shared lifecycle: + * - reset the public schema before every test so each `it` block starts + * against a known-empty baseline (migrations alter the schema itself); + * - close the pool at suite end so vitest doesn't hang on idle clients. + * Returns the pool so individual tests can issue ad-hoc queries. + */ +export function useMigrationTestPool(hooks: MigrationTestLifecycleHooks): pg.Pool { + const pool = createMigrationTestPool(); + hooks.beforeEach(async () => { + await resetPublicSchema(pool); + }); + hooks.afterAll(async () => { + await pool.end(); + }); + return pool; +} + +/** + * Drop and recreate the `public` schema so the test starts from an empty + * baseline. Re-installs the extensions that schema.sql relies on (vector + * and pgcrypto) since CREATE SCHEMA does not preserve extensions. + */ +export async function resetPublicSchema(pool: pg.Pool): Promise { + await pool.query('DROP SCHEMA IF EXISTS public CASCADE'); + await pool.query('CREATE SCHEMA public'); + await pool.query('GRANT ALL ON SCHEMA public TO public'); + await pool.query('CREATE EXTENSION IF NOT EXISTS vector'); + await pool.query('CREATE EXTENSION IF NOT EXISTS pgcrypto'); +} + +/** + * Apply the pinned v1.0.2 schema fixture with {{EMBEDDING_DIMENSIONS}} + * substituted to the configured value. The fixture must be byte-identical to + * the schema shipped with @atomicmemory/core@1.0.2 — that is the contract + * the backcompat test enforces. + */ +export async function applyLegacySchema( + pool: pg.Pool, + dims: number = config.embeddingDimensions, +): Promise { + const raw = readFileSync(LEGACY_FIXTURE_PATH, 'utf-8'); + const sql = raw.replace(/\{\{EMBEDDING_DIMENSIONS\}\}/g, String(dims)); + await pool.query(sql); +} + +/** Build a deterministic unit-magnitude embedding for seeding. */ +export function seedVector(seed: number, dims: number = config.embeddingDimensions): number[] { + const values = Array.from({ length: dims }, (_, index) => Math.sin(seed * (index + 1) + 1)); + const norm = Math.sqrt(values.reduce((sum, v) => sum + v * v, 0)); + return values.map((v) => v / norm); +} + +/** Format a JS number[] as the pgvector text literal `[v1,v2,...]`. */ +export function vectorLiteral(values: ReadonlyArray): string { + return `[${values.join(',')}]`; +} + +/** + * Capture a deterministic snapshot of a table for pre/post migration + * equality assertions. Ordering is by the supplied columns so the snapshot + * is reproducible. Uses `row_to_json` so JSONB / vector / array columns + * serialize consistently across pg versions. + */ +export async function snapshotTable( + pool: pg.Pool, + table: string, + orderBy: ReadonlyArray, +): Promise { + if (orderBy.length === 0) { + throw new Error(`snapshotTable(${table}): orderBy must contain at least one column`); + } + const orderClause = orderBy.map((column) => `"${column}" ASC`).join(', '); + const sql = `SELECT row_to_json(t) AS row FROM "${table}" AS t ORDER BY ${orderClause}`; + const { rows } = await pool.query<{ row: Record }>(sql); + return { table, rows: rows.map((entry) => entry.row) }; +} + +/** Existing-table check used by snapshot helpers and the legacy seed step. */ +export async function tableExists(pool: pg.Pool, table: string): Promise { + const { rows } = await pool.query<{ exists: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = current_schema() AND table_name = $1 + ) AS exists`, + [table], + ); + return rows[0]?.exists === true; +} + +/** Read the structural shape of the current schema for backcompat diffing. */ +export async function captureStructuralSnapshot(pool: pg.Pool): Promise { + return { + tables: await captureTablesAndColumns(pool), + indexes: await captureIndexes(pool), + checkConstraints: await captureCheckConstraints(pool), + foreignKeys: await captureForeignKeys(pool), + }; +} + +async function captureTablesAndColumns(pool: pg.Pool): Promise { + const { rows } = await pool.query<{ + table_name: string; + column_name: string; + udt_name: string; + is_nullable: 'YES' | 'NO'; + column_default: string | null; + }>( + `SELECT c.table_name, c.column_name, c.udt_name, c.is_nullable, c.column_default + FROM information_schema.columns c + JOIN information_schema.tables t + ON t.table_schema = c.table_schema AND t.table_name = c.table_name + WHERE c.table_schema = current_schema() + AND t.table_type = 'BASE TABLE' + ORDER BY c.table_name, c.ordinal_position`, + ); + const grouped = new Map(); + for (const row of rows) { + const list = grouped.get(row.table_name) ?? []; + list.push({ + column: row.column_name, + dataType: row.udt_name, + isNullable: row.is_nullable === 'YES', + columnDefault: row.column_default, + }); + grouped.set(row.table_name, list); + } + return Array.from(grouped.entries()) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, columns]) => ({ name, columns })); +} + +async function captureIndexes(pool: pg.Pool): Promise { + const { rows } = await pool.query<{ tablename: string; indexname: string; indexdef: string }>( + `SELECT tablename, indexname, indexdef FROM pg_indexes + WHERE schemaname = current_schema() + ORDER BY tablename, indexname`, + ); + return rows.map((row) => ({ table: row.tablename, index: row.indexname, definition: row.indexdef })); +} + +async function captureCheckConstraints(pool: pg.Pool): Promise { + const { rows } = await pool.query<{ table_name: string; constraint_name: string; definition: string }>( + `SELECT cls.relname AS table_name, con.conname AS constraint_name, + pg_get_constraintdef(con.oid) AS definition + FROM pg_constraint con + JOIN pg_class cls ON cls.oid = con.conrelid + JOIN pg_namespace nsp ON nsp.oid = cls.relnamespace + WHERE nsp.nspname = current_schema() AND con.contype = 'c' + ORDER BY cls.relname, con.conname`, + ); + return rows.map((row) => ({ + table: row.table_name, + constraint: row.constraint_name, + definition: row.definition, + })); +} + +async function captureForeignKeys(pool: pg.Pool): Promise { + const { rows } = await pool.query<{ table_name: string; constraint_name: string; definition: string }>( + `SELECT cls.relname AS table_name, con.conname AS constraint_name, + pg_get_constraintdef(con.oid) AS definition + FROM pg_constraint con + JOIN pg_class cls ON cls.oid = con.conrelid + JOIN pg_namespace nsp ON nsp.oid = cls.relnamespace + WHERE nsp.nspname = current_schema() AND con.contype = 'f' + ORDER BY cls.relname, con.conname`, + ); + return rows.map((row) => ({ + table: row.table_name, + constraint: row.constraint_name, + definition: row.definition, + })); +} diff --git a/src/db/__tests__/phase2-cutover-helpers.ts b/src/db/__tests__/phase2-cutover-helpers.ts new file mode 100644 index 0000000..f8328a1 --- /dev/null +++ b/src/db/__tests__/phase2-cutover-helpers.ts @@ -0,0 +1,165 @@ +/** + * Shared helpers for the Phase 2 cutover and DAG-sanity tests. + * + * Layered on top of `./migration-test-helpers.js`. Adds: + * + * - `MIGRATIONS_DIR` — the planned post-cutover location of the + * framework-managed migration files (`src/db/migrations/`). + * - `PHASE2_BOOKKEEPING_TABLES` — names of the tables that legitimately + * differ across cutover paths and must be excluded from structural + * schema-equivalence diffs. + * - `pgmigrationsRows()` — read the `node-pg-migrate` bookkeeping table + * for cutover-scenario assertions. Throws (rather than silently + * returning `[]`) when the table is absent so the test failure points + * at the missing runtime instead of a misleading empty result. + * - `seedPhase1StampedState()` — reconstruct the post-Phase-1 shape of + * `schema_version` without depending on the Phase 1 implementation, + * so Scenario C (Phase 1 → Phase 2) can be exercised deterministically. + * - `structuralSnapshotExcludingBookkeeping()` — wrap + * `captureStructuralSnapshot` and strip the Phase 2 bookkeeping tables + * so equivalence checks compare user-facing schema only. + * - `resetPublicSchemaForReuse()` — drop+recreate `public` mid-test for + * suites that need to run two full migrate() cycles in one `it()`. + * + * Plan reference: docs/ops/db/phase-2-versioned-migrations.md. + */ + +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import pg from 'pg'; +import { + captureStructuralSnapshot, + resetPublicSchema, + type StructuralSnapshot, +} from './migration-test-helpers.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +/** + * Repo-relative path to the planned Phase 2 migrations directory. The + * DAG-sanity tests read this directory; until Phase 2 lands the directory + * does not exist and those tests will fail with a clear assertion error + * (the test does not crash with ENOENT — see `dag-sanity.test.ts`). + */ +export const MIGRATIONS_DIR = resolve(__dirname, '../migrations'); + +/** + * Bookkeeping tables whose existence and row counts intentionally differ + * across cutover paths. Stripped from structural diffs so the user-facing + * schema equivalence check is not polluted by framework-owned tables. + */ +const PHASE2_BOOKKEEPING_TABLES: ReadonlySet = new Set([ + 'pgmigrations', + 'schema_version', +]); + +export interface PgMigrationsRow { + readonly id: number; + readonly name: string; + readonly run_on: Date; +} + +/** + * Read every row of node-pg-migrate's `pgmigrations` bookkeeping table in + * insertion order. Throws with a descriptive error if the table is absent + * so a Phase 2 cutover test failing because the runtime has not yet been + * wired surfaces a clear message instead of an empty array. + */ +export async function pgmigrationsRows(pool: pg.Pool): Promise { + const { rows: present } = await pool.query<{ exists: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = current_schema() AND table_name = 'pgmigrations' + ) AS exists`, + ); + if (!present[0]?.exists) { + throw new Error( + 'pgmigrationsRows: expected `pgmigrations` table to exist. ' + + 'Phase 2 migrate() must create it on first run. Has the runtime landed?', + ); + } + const { rows } = await pool.query( + 'SELECT id, name, run_on FROM pgmigrations ORDER BY id ASC', + ); + return rows; +} + +/** + * Reconstruct the schema_version shape and one applied row, mimicking what + * a Phase 1-installed DB would carry into a Phase 2 upgrade. Keeps Scenario + * C tests independent of the (about-to-be-rewritten) Phase 1 runtime path. + * + * The applied_at timestamp is fixed in the past so Phase 2's own stamp + * (appended at NOW()) is unambiguously the later row. + */ +export async function seedPhase1StampedState( + pool: pg.Pool, + sdkVersion: string = '1.4.0-phase1-fixture', +): Promise { + await pool.query(` + CREATE TABLE IF NOT EXISTS schema_version ( + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + sdk_version TEXT NOT NULL, + schema_sha256 TEXT NOT NULL, + notes TEXT, + PRIMARY KEY (applied_at) + )`); + await pool.query(` + CREATE INDEX IF NOT EXISTS idx_schema_version_applied_at + ON schema_version (applied_at DESC)`); + await pool.query( + `INSERT INTO schema_version (applied_at, sdk_version, schema_sha256, notes) + VALUES (TIMESTAMPTZ '2026-04-01 00:00:00Z', $1, + 'phase1fixture000000000000000000000000000000000000000000000000000', + 'phase1-fixture-stamp')`, + [sdkVersion], + ); +} + +/** Number of rows currently in `schema_version`. Returns 0 if the table is absent. */ +export async function schemaVersionRowCount(pool: pg.Pool): Promise { + const { rows: present } = await pool.query<{ exists: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = current_schema() AND table_name = 'schema_version' + ) AS exists`, + ); + if (!present[0]?.exists) return 0; + const { rows } = await pool.query<{ count: string }>( + 'SELECT COUNT(*)::text AS count FROM schema_version', + ); + return Number.parseInt(rows[0]?.count ?? '0', 10); +} + +/** + * Structural snapshot of the user-facing schema with Phase 2 bookkeeping + * tables (`pgmigrations`, `schema_version`) and their indexes/constraints + * stripped, so cutover-equivalence diffs compare only what consumers see. + */ +export async function structuralSnapshotExcludingBookkeeping( + pool: pg.Pool, +): Promise { + return filterBookkeepingTables(await captureStructuralSnapshot(pool)); +} + +function filterBookkeepingTables(snapshot: StructuralSnapshot): StructuralSnapshot { + const excluded = PHASE2_BOOKKEEPING_TABLES; + return { + tables: snapshot.tables.filter((entry) => !excluded.has(entry.name)), + indexes: snapshot.indexes.filter((entry) => !excluded.has(entry.table)), + checkConstraints: snapshot.checkConstraints.filter( + (entry) => !excluded.has(entry.table), + ), + foreignKeys: snapshot.foreignKeys.filter((entry) => !excluded.has(entry.table)), + }; +} + +/** + * Drop and recreate the `public` schema mid-test. Used when a single + * `it()` block needs to run two independent migrate() cycles from an empty + * baseline (the standard `beforeEach` resets between tests, not within + * one). Re-installs the extensions the legacy schema fixture depends on. + */ +export async function resetPublicSchemaForReuse(pool: pg.Pool): Promise { + await resetPublicSchema(pool); +} diff --git a/src/db/__tests__/reconciler-client-path.test.ts b/src/db/__tests__/reconciler-client-path.test.ts new file mode 100644 index 0000000..e504b6a --- /dev/null +++ b/src/db/__tests__/reconciler-client-path.test.ts @@ -0,0 +1,63 @@ +/** + * Regression test for `isPool()` in `reconcilers.ts`. + * + * `migrate()` invokes `reconcileEmbeddingDimension` with a checked-out + * `pg.PoolClient` (the connection that holds the Phase 1 advisory lock). + * The original `isPool()` only checked for `.connect`, which both `pg.Pool` + * and `pg.PoolClient` expose, so the reconciler misidentified the client + * as a pool and called `executor.connect()` on it. `pg.Client.connect()` + * throws `Client has already been connected. You cannot reuse a client.` + * whenever the client is already connected — which it always is when it + * came from `pool.connect()`. + * + * This test reproduces migrate()'s call site directly: check out a client + * from a pool and pass it to `reconcileEmbeddingDimension`. On the old + * code, the reconciler's `withClient` path threw on the embedded + * `executor.connect()`. With the fix (`isPool` additionally rejects + * anything with a `.release` method), the same call alters the column + * cleanly and returns the expected `ReconcileResult`. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { pool } from '../pool.js'; +import { reconcileEmbeddingDimension } from '../reconcilers.js'; +import { + readVectorColumnDimension, + registerReconcilerSchemaLifecycle, + setReconcilerSearchPath, +} from './reconciler-test-helpers.js'; + +const TEST_SCHEMA = 'reconciler_client_path_test_schema'; +const TABLE_NAME = 'reconciler_client_path_t'; + +describe('reconcileEmbeddingDimension via checked-out PoolClient', () => { + registerReconcilerSchemaLifecycle({ + afterAll, + beforeAll, + beforeEach, + pool, + schema: TEST_SCHEMA, + }); + + it('alters a mismatched empty column when the executor is an already-connected PoolClient', async () => { + await pool.query( + `CREATE TABLE ${TABLE_NAME} (id serial PRIMARY KEY, embedding vector(4))`, + ); + + const client = await pool.connect(); + try { + await setReconcilerSearchPath(client, TEST_SCHEMA); + const result = await reconcileEmbeddingDimension(client, 8); + expect(result.reconciled).toBe(true); + expect(result.alteredColumns).toEqual([ + { tableName: TABLE_NAME, columnName: 'embedding' }, + ]); + } finally { + client.release(); + } + + expect( + await readVectorColumnDimension(pool, TEST_SCHEMA, TABLE_NAME, 'embedding'), + ).toBe(8); + }); +}); diff --git a/src/db/__tests__/reconciler-test-helpers.ts b/src/db/__tests__/reconciler-test-helpers.ts new file mode 100644 index 0000000..9c25c8b --- /dev/null +++ b/src/db/__tests__/reconciler-test-helpers.ts @@ -0,0 +1,79 @@ +/** + * Shared Postgres helpers for embedding-dimension reconciler integration tests. + * + * The reconciler tests intentionally operate against real pgvector columns in + * isolated schemas. Keeping the schema reset and typmod inspection in one + * helper avoids copy-pasted catalog SQL while preserving the full database path. + */ + +import pg from 'pg'; +import type { afterAll, beforeAll, beforeEach } from 'vitest'; + +type AsyncLifecycleHook = typeof afterAll | typeof beforeAll | typeof beforeEach; + +interface ReconcilerSchemaLifecycle { + readonly afterAll: AsyncLifecycleHook; + readonly beforeAll: AsyncLifecycleHook; + readonly beforeEach: AsyncLifecycleHook; + readonly pool: pg.Pool; + readonly schema: string; +} + +export function registerReconcilerSchemaLifecycle( + lifecycle: ReconcilerSchemaLifecycle, +): void { + lifecycle.beforeAll(async () => { + await lifecycle.pool.query('CREATE EXTENSION IF NOT EXISTS vector'); + }); + lifecycle.beforeEach(async () => { + await resetReconcilerTestSchema(lifecycle.pool, lifecycle.schema); + }); + lifecycle.afterAll(async () => { + await dropReconcilerTestSchema(lifecycle.pool, lifecycle.schema); + await lifecycle.pool.query('RESET search_path'); + await lifecycle.pool.end(); + }); +} + +async function resetReconcilerTestSchema( + pool: pg.Pool, + schema: string, +): Promise { + const quoted = pg.escapeIdentifier(schema); + await pool.query(`DROP SCHEMA IF EXISTS ${quoted} CASCADE`); + await pool.query(`CREATE SCHEMA ${quoted}`); + await setReconcilerSearchPath(pool, schema); +} + +async function dropReconcilerTestSchema( + pool: pg.Pool, + schema: string, +): Promise { + const quoted = pg.escapeIdentifier(schema); + await pool.query(`DROP SCHEMA IF EXISTS ${quoted} CASCADE`); +} + +export async function setReconcilerSearchPath( + client: pg.Pool | pg.PoolClient, + schema: string, +): Promise { + const quoted = pg.escapeIdentifier(schema); + await client.query(`SET search_path TO ${quoted}, public`); +} + +export async function readVectorColumnDimension( + pool: pg.Pool, + schema: string, + table: string, + column: string, +): Promise { + const { rows } = await pool.query<{ typmod: number }>( + `SELECT a.atttypmod AS typmod + FROM pg_attribute a + JOIN pg_class c ON a.attrelid = c.oid + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE n.nspname = $1 AND c.relname = $2 AND a.attname = $3`, + [schema, table, column], + ); + return rows.length === 0 ? null : rows[0].typmod; +} diff --git a/src/db/__tests__/repository-wipe-user-scope.test.ts b/src/db/__tests__/repository-wipe-user-scope.test.ts new file mode 100644 index 0000000..33923a3 --- /dev/null +++ b/src/db/__tests__/repository-wipe-user-scope.test.ts @@ -0,0 +1,170 @@ +/** + * Integration coverage for the shared repository wipe path. The + * admin test-scope endpoint delegates to this path, so it must remove + * every user-owned projection table, not only the original memories + * and raw-document tables. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import pgvector from 'pgvector/pg'; +import { pool } from '../pool.js'; +import { deleteAll } from '../repository-wipe.js'; +import { setupTestSchema, unitVector } from './test-fixtures.js'; + +const USER_A = 'wipe-scope-user-a'; +const USER_B = 'wipe-scope-user-b'; + +const USER_SCOPED_TABLES = [ + 'episodes', + 'canonical_memory_objects', + 'memories', + 'memory_atomic_facts', + 'memory_foresight', + 'memory_claims', + 'memory_claim_versions', + 'entities', + 'temporal_linkage_list', + 'first_mention_events', + 'entity_relations', + 'lessons', + 'agent_trust', + 'memory_conflicts', + 'belief_edges', + 'session_summaries', + 'conv_summaries', + 'recaps', + 'user_profiles', + 'entity_attributes', + 'entity_values', + 'session_reflections', + 'reflection_jobs', + 'entity_cards', + 'memory_contradictions', + 'observation_dirty', +] as const; + +describe('repository wipe user scope', () => { + beforeAll(async () => { await setupTestSchema(pool); }); + beforeEach(async () => { await deleteAll(pool); }); + afterAll(async () => { await deleteAll(pool); await pool.end(); }); + + it('deleteAll(userId) removes newer user-scoped projection tables only for that user', async () => { + await insertUserFootprint(USER_A, 1); + await insertUserFootprint(USER_B, 2); + + await deleteAll(pool, USER_A); + + await expect(countUserRows(USER_A)).resolves.toBe(0); + await expect(countUserRows(USER_B)).resolves.toBeGreaterThan(0); + await expect(countJoinRows()).resolves.toEqual({ + memoryEntities: 1, + memoryLinks: 1, + visibilityGrants: 1, + }); + }); +}); + +async function insertUserFootprint(userId: string, seed: number): Promise { + const vector = pgvector.toSql(unitVector(seed)); + const memoryIds = await insertMemories(userId, vector, seed); + const entityIds = await insertEntities(userId, vector, seed); + const claimVersionId = await insertClaim(userId, vector, memoryIds[0]); + await insertMemoryChildren(userId, vector, memoryIds, entityIds, claimVersionId, seed); + await insertProjectionRows(userId, vector, memoryIds[0], seed); +} + +async function insertMemories(userId: string, vector: string, seed: number): Promise { + const result = await pool.query<{ id: string }>( + `INSERT INTO memories (user_id, content, embedding, source_site) + VALUES ($1, $2, $3, 'wipe-test'), ($1, $4, $3, 'wipe-test') RETURNING id`, + [userId, `memory ${seed}a`, vector, `memory ${seed}b`], + ); + return result.rows.map((row) => row.id); +} + +async function insertEntities(userId: string, vector: string, seed: number): Promise { + const result = await pool.query<{ id: string }>( + `INSERT INTO entities (user_id, name, normalized_name, entity_type, embedding) + VALUES ($1, $2, $3, 'person', $4), ($1, $5, $6, 'tool', $4) RETURNING id`, + [userId, `Alice ${seed}`, `alice ${seed}`, vector, `Tool ${seed}`, `tool ${seed}`], + ); + return result.rows.map((row) => row.id); +} + +async function insertClaim(userId: string, vector: string, memoryId: string): Promise { + const claim = await pool.query<{ id: string }>( + `INSERT INTO memory_claims (user_id) VALUES ($1) RETURNING id`, + [userId], + ); + const version = await pool.query<{ id: string }>( + `INSERT INTO memory_claim_versions + (claim_id, user_id, memory_id, content, embedding, source_site) + VALUES ($1, $2, $3, 'claim content', $4, 'wipe-test') RETURNING id`, + [claim.rows[0].id, userId, memoryId, vector], + ); + return version.rows[0].id; +} + +async function insertMemoryChildren( + userId: string, + vector: string, + memoryIds: string[], + entityIds: string[], + claimVersionId: string, + seed: number, +): Promise { + await pool.query(`INSERT INTO memory_visibility_grants (memory_id, grantee_agent_id) VALUES ($1, gen_random_uuid())`, [memoryIds[0]]); + await pool.query(`INSERT INTO memory_evidence (claim_version_id, memory_id, quote_text) VALUES ($1, $2, 'quote')`, [claimVersionId, memoryIds[0]]); + await pool.query(`INSERT INTO memory_links (source_id, target_id, similarity) VALUES ($1, $2, 0.9)`, [memoryIds[0], memoryIds[1]]); + await pool.query(`INSERT INTO memory_entities (memory_id, entity_id) VALUES ($1, $2)`, [memoryIds[0], entityIds[0]]); + await pool.query(`INSERT INTO temporal_linkage_list (user_id, entity_id, memory_id, observation_date, position_in_chain) VALUES ($1, $2, $3, NOW(), 1)`, [userId, entityIds[0], memoryIds[0]]); + await pool.query(`INSERT INTO first_mention_events (user_id, topic, turn_id, memory_id, position_in_conversation) VALUES ($1, 'topic', $2, $3, 1)`, [userId, seed, memoryIds[0]]); + await pool.query(`INSERT INTO entity_relations (user_id, source_entity_id, target_entity_id, relation_type, source_memory_id) VALUES ($1, $2, $3, 'uses', $4)`, [userId, entityIds[0], entityIds[1], memoryIds[0]]); + await pool.query(`INSERT INTO memory_atomic_facts (user_id, parent_memory_id, fact_text, embedding, source_site) VALUES ($1, $2, 'fact', $3, 'wipe-test')`, [userId, memoryIds[0], vector]); + await pool.query(`INSERT INTO memory_foresight (user_id, parent_memory_id, content, embedding, source_site) VALUES ($1, $2, 'future', $3, 'wipe-test')`, [userId, memoryIds[0], vector]); +} + +async function insertProjectionRows(userId: string, vector: string, memoryId: string, seed: number): Promise { + await pool.query(`INSERT INTO episodes (user_id, content, source_site) VALUES ($1, 'episode', 'wipe-test')`, [userId]); + await pool.query(`INSERT INTO canonical_memory_objects (user_id, object_family, canonical_payload) VALUES ($1, 'ingested_fact', '{}')`, [userId]); + await pool.query(`INSERT INTO lessons (user_id, lesson_type, pattern, embedding) VALUES ($1, 'user_reported', 'pattern', $2)`, [userId, vector]); + await pool.query(`INSERT INTO agent_trust (agent_id, user_id) VALUES ($1, $2)`, [`agent-${seed}`, userId]); + await pool.query(`INSERT INTO memory_conflicts (user_id, new_memory_id, existing_memory_id) VALUES ($1, $2, $2)`, [userId, memoryId]); + await pool.query(`INSERT INTO belief_edges (user_id, source_id, target_id, edge_type) VALUES ($1, $2, $2, 'evidence_for')`, [userId, memoryId]); + await pool.query(`INSERT INTO session_summaries (user_id, session_id, conversation_id, session_index, summary_text, summary_embedding) VALUES ($1, 's', 'c', 1, 'summary', $2)`, [userId, vector]); + await pool.query(`INSERT INTO conv_summaries (user_id, conversation_id, summary_text, summary_embedding) VALUES ($1, 'c', 'summary', $2)`, [userId, vector]); + await pool.query(`INSERT INTO recaps (user_id, recap_text, recap_embedding, topic) VALUES ($1, 'recap', $2, 'topic')`, [userId, vector]); + await pool.query(`INSERT INTO user_profiles (user_id, profile_text, source_memory_ids) VALUES ($1, 'profile', ARRAY[$2])`, [userId, memoryId]); + await pool.query(`INSERT INTO entity_attributes (user_id, entity_name, attribute_key, attribute_value, value_type) VALUES ($1, 'entity', 'key', 'value', 'string')`, [userId]); + await pool.query(`INSERT INTO entity_values (user_id, entity, attribute, value, value_type, observed_at, fact_id) VALUES ($1, 'entity', 'key', 'value', 'string', NOW(), $2)`, [userId, memoryId]); + await pool.query(`INSERT INTO session_reflections (user_id, conversation_id, observation, observation_type, evidence_memory_ids) VALUES ($1, 'c', 'observation', 'event_summary', ARRAY[$2])`, [userId, memoryId]); + await pool.query(`INSERT INTO reflection_jobs (user_id, conversation_id) VALUES ($1, $2)`, [userId, `c-${seed}`]); + await pool.query(`INSERT INTO entity_cards (user_id, conversation_id, entity_name, card_text) VALUES ($1, 'c', 'entity', 'card')`, [userId]); + await pool.query(`INSERT INTO memory_contradictions (user_id, left_memory_id, right_memory_id, left_summary, right_summary) VALUES ($1, $2, $2, 'left', 'right')`, [userId, memoryId]); + await pool.query(`INSERT INTO observation_dirty (user_id, subject) VALUES ($1, 'subject')`, [userId]); +} + +async function countUserRows(userId: string): Promise { + let total = 0; + for (const tableName of USER_SCOPED_TABLES) { + const result = await pool.query<{ n: number }>( + `SELECT COUNT(*)::int AS n FROM ${tableName} WHERE user_id = $1`, + [userId], + ); + total += result.rows[0].n; + } + return total; +} + +async function countJoinRows(): Promise> { + const [memoryEntities, memoryLinks, visibilityGrants] = await Promise.all([ + pool.query<{ n: number }>('SELECT COUNT(*)::int AS n FROM memory_entities'), + pool.query<{ n: number }>('SELECT COUNT(*)::int AS n FROM memory_links'), + pool.query<{ n: number }>('SELECT COUNT(*)::int AS n FROM memory_visibility_grants'), + ]); + return { + memoryEntities: memoryEntities.rows[0].n, + memoryLinks: memoryLinks.rows[0].n, + visibilityGrants: visibilityGrants.rows[0].n, + }; +} diff --git a/src/db/__tests__/tbc-phase-3-schema.test.ts b/src/db/__tests__/tbc-phase-3-schema.test.ts index 05d4f19..6b214e8 100644 --- a/src/db/__tests__/tbc-phase-3-schema.test.ts +++ b/src/db/__tests__/tbc-phase-3-schema.test.ts @@ -1,5 +1,5 @@ /** - * Static verification of the TBC Phase 3 schema additions in schema.sql. + * Static verification of the TBC Phase 3 baseline migration additions. * Asserts the SQL contains the expected DDL without requiring a DB connection. */ @@ -9,7 +9,10 @@ import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; const __dirname = dirname(fileURLToPath(import.meta.url)); -const schemaSql = readFileSync(resolve(__dirname, '..', 'schema.sql'), 'utf-8'); +const schemaSql = readFileSync( + resolve(__dirname, '..', 'migrations', '0001_baseline.sql'), + 'utf-8', +); const IDEMPOTENT_DDL = /IF NOT EXISTS|DROP CONSTRAINT IF EXISTS/; const CHECK_CONSTRAINT_REWRITE = /ADD CONSTRAINT raw_documents_[a-z_]+_check/; diff --git a/src/db/__tests__/test-fixtures.ts b/src/db/__tests__/test-fixtures.ts index 495317b..dce24f4 100644 --- a/src/db/__tests__/test-fixtures.ts +++ b/src/db/__tests__/test-fixtures.ts @@ -5,18 +5,43 @@ * and adds database-specific helpers (schema setup, vector generation). */ -import { readFileSync } from 'node:fs'; -import { dirname, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; import type pg from 'pg'; import { config } from '../../config.js'; import { MemoryRepository } from '../memory-repository.js'; import { ClaimRepository } from '../claim-repository.js'; import { MemoryService } from '../../services/memory-service.js'; +import { migrate } from '../migration-api.js'; export { createSearchResult, createMemoryRow } from '../../services/__tests__/test-fixtures.js'; +const REQUIRED_TEST_SCHEMA_TABLES = [ + 'memories', + 'episodes', + 'canonical_memory_objects', + 'memory_claims', + 'memory_claim_versions', + 'memory_evidence', + 'entities', + 'memory_entities', + 'raw_sources', + 'raw_documents', + 'document_chunks', + 'storage_artifacts', +] as const; + +const DOCUMENT_TABLE_DELETE_ORDER = [ + 'document_chunks', + 'memory_evidence', + 'memory_claim_versions', + 'memory_claims', + 'memory_links', + 'memories', + 'raw_documents', + 'storage_artifacts', + 'raw_sources', +] as const; + /** Lifecycle hooks accepted by test context factories. */ interface TestLifecycleHooks { beforeAll: (fn: () => Promise) => void; @@ -65,14 +90,6 @@ export function createServiceTestContext(pool: pg.Pool, hooks: TestLifecycleHook return { repo, claimRepo, service }; } -const __dirname = dirname(fileURLToPath(import.meta.url)); - -/** Read and prepare schema SQL with configured dimensions. */ -function getSchemaSQL(): string { - const raw = readFileSync(resolve(__dirname, '../schema.sql'), 'utf-8'); - return raw.replace(/\{\{EMBEDDING_DIMENSIONS\}\}/g, String(config.embeddingDimensions)); -} - /** * Return the memories.embedding vector(N) dimension in pgvector's * atttypmod encoding, or null if the table does not exist or the @@ -91,25 +108,55 @@ async function readEmbeddingColumnDim(pool: pg.Pool): Promise { } /** - * Apply schema to a test database pool. + * Apply migrations to a test database pool. + * + * The migration baseline is idempotent, but the reconciler refuses to alter + * populated vector columns with the wrong dimension. When a test DB was + * previously initialized with a different EMBEDDING_DIMENSIONS, drop and + * recreate the public schema before running migrations so the empty-schema + * reconciler can safely align column dimensions. * - * The base schema.sql is idempotent (CREATE TABLE IF NOT EXISTS), so - * re-running it cannot change the type of a column that already - * exists. When the test DB was previously initialized with a different - * EMBEDDING_DIMENSIONS (for example, left over from a prior run with - * a different .env.test), the memories.embedding column retains the - * old vector(N) dim and subsequent inserts with the new dim fail at - * the DB level — surfacing as opaque 500s in route tests. Detect that - * drift up front and drop+recreate the public schema so schema.sql - * can rebuild it at the configured dim. + * Phase 2 also fails closed on partial pre-framework schemas. That is the + * production-safe behavior, but this shared test helper owns its database and + * can deterministically reset a stale partial schema left by an interrupted or + * cross-suite test run. */ export async function setupTestSchema(pool: pg.Pool): Promise { const existingDim = await readEmbeddingColumnDim(pool); - if (existingDim !== null && existingDim !== config.embeddingDimensions) { - await pool.query('DROP SCHEMA public CASCADE; CREATE SCHEMA public;'); + if ( + (existingDim !== null && existingDim !== config.embeddingDimensions) + || (await hasPartialTestSchema(pool)) + ) { + await resetPublicSchema(pool); } - const sql = getSchemaSQL(); - await pool.query(sql); + await migrate({ pool }); +} + +async function hasPartialTestSchema(pool: pg.Pool): Promise { + const existing = await existingRequiredTestTables(pool); + if (existing.size === 0) return false; + return REQUIRED_TEST_SCHEMA_TABLES.some((table) => !existing.has(table)); +} + +async function existingRequiredTestTables(pool: pg.Pool): Promise> { + const { rows } = await pool.query<{ table_name: string }>( + `SELECT table_name + FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name = ANY($1::text[])`, + [REQUIRED_TEST_SCHEMA_TABLES], + ); + return new Set(rows.map((row) => row.table_name)); +} + +async function resetPublicSchema(pool: pg.Pool): Promise { + await pool.query(` + DROP SCHEMA IF EXISTS public CASCADE; + CREATE SCHEMA public; + GRANT ALL ON SCHEMA public TO public; + CREATE EXTENSION IF NOT EXISTS vector; + CREATE EXTENSION IF NOT EXISTS pgcrypto; + `); } /** @@ -124,15 +171,9 @@ export async function setupTestSchema(pool: pg.Pool): Promise { * violate the FK. */ export async function clearDocumentTables(pool: pg.Pool): Promise { - await pool.query('DELETE FROM document_chunks'); - await pool.query('DELETE FROM memory_evidence'); - await pool.query('DELETE FROM memory_claim_versions'); - await pool.query('DELETE FROM memory_claims'); - await pool.query('DELETE FROM memory_links'); - await pool.query('DELETE FROM memories'); - await pool.query('DELETE FROM raw_documents'); - await pool.query('DELETE FROM storage_artifacts'); - await pool.query('DELETE FROM raw_sources'); + for (const table of DOCUMENT_TABLE_DELETE_ORDER) { + await pool.query(`DELETE FROM ${table}`); + } } /** Generate a deterministic unit vector from a seed. */ diff --git a/src/db/embedding-dimension-status.ts b/src/db/embedding-dimension-status.ts new file mode 100644 index 0000000..c05e951 --- /dev/null +++ b/src/db/embedding-dimension-status.ts @@ -0,0 +1,101 @@ +/** + * Read-only pgvector dimension inspection for migrationStatus(). + * + * The reconciler mutates empty vector columns to the configured dimension. + * This module performs only catalog reads so operators can see drift before + * a later write path or migrate() call fails. + */ + +import type { Pool, PoolClient } from 'pg'; + +export type EmbeddingDimensionStatusValue = + | 'not_applicable' + | 'matches' + | 'mismatch' + | 'missing_vector_columns'; + +export interface EmbeddingDimensionMismatchSummary { + readonly tableName: string; + readonly columnName: string; + readonly currentDimension: number; + readonly requiredDimension: number; +} + +export interface EmbeddingDimensionStatus { + readonly requiredDimension: number; + readonly status: EmbeddingDimensionStatusValue; + readonly vectorColumnCount: number; + readonly mismatches: readonly EmbeddingDimensionMismatchSummary[]; +} + +export function noSchemaEmbeddingStatus( + requiredDimension: number, +): EmbeddingDimensionStatus { + return { + requiredDimension, + status: 'not_applicable', + vectorColumnCount: 0, + mismatches: [], + }; +} + +export async function inspectEmbeddingDimensionStatus( + client: Pick, + requiredDimension: number, +): Promise { + const columns = await readVectorColumns(client); + const mismatches = columns + .filter((column) => column.currentDimension !== requiredDimension) + .map((column) => ({ ...column, requiredDimension })); + return { + requiredDimension, + status: statusFor(columns.length, mismatches.length), + vectorColumnCount: columns.length, + mismatches, + }; +} + +interface VectorColumnDimension { + readonly tableName: string; + readonly columnName: string; + readonly currentDimension: number; +} + +async function readVectorColumns( + client: Pick, +): Promise { + const { rows } = await client.query<{ + table_name: string; + column_name: string; + current_dimension: number; + }>( + `SELECT + c.relname AS table_name, + a.attname AS column_name, + a.atttypmod AS current_dimension + FROM pg_attribute a + JOIN pg_class c ON a.attrelid = c.oid + JOIN pg_namespace n ON c.relnamespace = n.oid + JOIN pg_type t ON a.atttypid = t.oid + WHERE n.nspname = current_schema() + AND c.relkind = 'r' + AND t.typname = 'vector' + AND a.atttypmod > 0 + AND NOT a.attisdropped + ORDER BY c.relname, a.attname`, + ); + return rows.map((row) => ({ + tableName: row.table_name, + columnName: row.column_name, + currentDimension: row.current_dimension, + })); +} + +function statusFor( + vectorColumnCount: number, + mismatchCount: number, +): EmbeddingDimensionStatusValue { + if (vectorColumnCount === 0) return 'missing_vector_columns'; + return mismatchCount === 0 ? 'matches' : 'mismatch'; +} + diff --git a/src/db/memory-repository.ts b/src/db/memory-repository.ts index 9e0efb0..ceec606 100644 --- a/src/db/memory-repository.ts +++ b/src/db/memory-repository.ts @@ -172,8 +172,8 @@ export class MemoryRepository { return getMemoryWithClient(client, id, userId, true); } - async listMemories(userId: string, limit: number = 20, offset: number = 0, sourceSite?: string, episodeId?: string) { - return listMemories(this.pool, userId, limit, offset, sourceSite, episodeId); + async listMemories(userId: string, limit: number = 20, offset: number = 0, sourceSite?: string, episodeId?: string, sessionId?: string) { + return listMemories(this.pool, userId, limit, offset, sourceSite, episodeId, sessionId); } /** Fetch the text and timestamp for all non-deleted memories for a user's @@ -199,8 +199,8 @@ export class MemoryRepository { return rows.map(r => ({ id: r.id, text: r.content, observedAt: r.observed_at })); } - async listMemoriesInWorkspace(workspaceId: string, limit: number = 20, offset: number = 0, callerAgentId?: string) { - return listMemoriesInWorkspace(this.pool, workspaceId, limit, offset, callerAgentId); + async listMemoriesInWorkspace(workspaceId: string, limit: number = 20, offset: number = 0, callerAgentId?: string, sessionId?: string) { + return listMemoriesInWorkspace(this.pool, workspaceId, limit, offset, callerAgentId, sessionId); } async getMemoryInWorkspace(id: string, workspaceId: string, callerAgentId?: string) { @@ -220,20 +220,20 @@ export class MemoryRepository { return listMemoriesByNamespace(this.pool, userId, namespace, limit); } - async searchSimilar(userId: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date) { - return searchSimilar(this.pool, userId, queryEmbedding, limit, sourceSite, referenceTime); + async searchSimilar(userId: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date, sessionId?: string) { + return searchSimilar(this.pool, userId, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } - async searchHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date) { - return searchHybridSimilar(this.pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime); + async searchHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date, sessionId?: string) { + return searchHybridSimilar(this.pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } - async searchKeyword(userId: string, queryText: string, limit: number, sourceSite?: string) { - return searchKeywordSimilar(this.pool, userId, queryText, limit, sourceSite); + async searchKeyword(userId: string, queryText: string, limit: number, sourceSite?: string, sessionId?: string) { + return searchKeywordSimilar(this.pool, userId, queryText, limit, sourceSite, sessionId); } - async searchAtomicFactsHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date) { - return searchAtomicFactsHybrid(this.pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime); + async searchAtomicFactsHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date, sessionId?: string) { + return searchAtomicFactsHybrid(this.pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } async findNearDuplicates(userId: string, embedding: number[], threshold: number, limit: number = 3) { @@ -422,8 +422,9 @@ export class MemoryRepository { agentScope: AgentScope = 'all', callerAgentId?: string, referenceTime?: Date, + sessionId?: string, ) { - return searchSimilarInWorkspace(this.pool, workspaceId, queryEmbedding, limit, agentScope, callerAgentId, referenceTime); + return searchSimilarInWorkspace(this.pool, workspaceId, queryEmbedding, limit, agentScope, callerAgentId, referenceTime, sessionId); } /** diff --git a/src/db/migrate.ts b/src/db/migrate.ts index af1dfd9..a7dd631 100644 --- a/src/db/migrate.ts +++ b/src/db/migrate.ts @@ -1,47 +1,54 @@ /** - * Run schema migration against the configured database. - * Replaces {{EMBEDDING_DIMENSIONS}} in schema.sql with the configured value. - * Usage: pnpm migrate (uses .env) or pnpm migrate:test (uses .env.test) + * CLI wrapper for the programmatic migration API. + * + * Library consumers should import `migrate` / `migrationStatus` from + * `@atomicmemory/core` (re-exported via `src/index.ts`) instead of shelling + * out. This file is intentionally a thin shim so that `process.exit` lives + * only at the CLI boundary; `migration-api.ts` never terminates the host + * process. + * + * Usage: + * npm run migrate # loads .env via dotenv-cli + * npm run migrate:test # loads .env.test via dotenv-cli + * tsx src/db/migrate.ts --lock-timeout-ms=120000 */ -import { readFileSync } from 'node:fs'; -import { resolve, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { pool } from './pool.js'; -import { config } from '../config.js'; -import { resolveEmbeddingDimensions } from '../services/embedding.js'; +import { migrate, type MigrateOptions } from './migration-api.js'; -const __dirname = dirname(fileURLToPath(import.meta.url)); +main().catch((err: unknown) => { + console.error('[migrate] Migration failed:', err); + process.exit(1); +}); -function stripVectorIndexes(sql: string): string { - if (!config.skipVectorIndexes) return sql; - // Matches both CREATE INDEX and CREATE INDEX IF NOT EXISTS forms — schema.sql - // uses the idempotent form so it can re-run on every startup without data loss. - return sql.replace( - /CREATE INDEX (IF NOT EXISTS )?idx_[a-z_]+_embedding ON [a-z_]+\n USING hnsw \(embedding vector_cosine_ops\)\n WITH \(m = 16, ef_construction = 200\);(\n\n|\n?$)/g, - '', +async function main(): Promise { + const result = await migrate(parseCliOptions(process.argv.slice(2))); + console.log( + `[migrate] Migration complete ` + + `(ranSchemaSql=${result.ranSchemaSql}, ` + + `version=${result.schemaVersion.sdkVersion}, ` + + `sha=${result.schemaVersion.schemaSha256.slice(0, 12)}…, ` + + `reconciledEmbeddingDimension=${result.reconciledEmbeddingDimension}).`, ); + process.exit(0); } -async function migrate(): Promise { - const envDims = process.env.EMBEDDING_DIMENSIONS ? parseInt(process.env.EMBEDDING_DIMENSIONS, 10) : null; - const embeddingDimensions = envDims ?? await resolveEmbeddingDimensions(); - console.log(`[migrate] Resolved dimensions: ${embeddingDimensions} (from env: ${envDims})`); - const schemaPath = resolve(__dirname, 'schema.sql'); - const rawSql = readFileSync(schemaPath, 'utf-8'); - const dimensionedSql = rawSql.replace( - /\{\{EMBEDDING_DIMENSIONS\}\}/g, - String(embeddingDimensions), - ); - const sql = stripVectorIndexes(dimensionedSql); - - console.log(`Running migration (embedding dimensions: ${embeddingDimensions}, vector indexes: ${config.skipVectorIndexes ? 'off' : 'on'})...`); - await pool.query(sql); - console.log('Migration complete.'); - await pool.end(); +function parseCliOptions(args: string[]): MigrateOptions { + const opts: MigrateOptions = {}; + for (const arg of args) { + if (arg.startsWith('--lock-timeout-ms=')) { + opts.lockTimeoutMs = parsePositiveInteger(arg, '--lock-timeout-ms='); + } else { + throw new Error(`[migrate] Unknown argument: ${arg}`); + } + } + return opts; } -migrate().catch((err) => { - console.error('Migration failed:', err); - process.exit(1); -}); +function parsePositiveInteger(arg: string, prefix: string): number { + const raw = arg.slice(prefix.length); + const value = Number.parseInt(raw, 10); + if (!Number.isInteger(value) || value <= 0 || String(value) !== raw) { + throw new Error(`[migrate] ${prefix.slice(0, -1)} must be a positive integer`); + } + return value; +} diff --git a/src/db/migration-api.ts b/src/db/migration-api.ts new file mode 100644 index 0000000..6f3b73b --- /dev/null +++ b/src/db/migration-api.ts @@ -0,0 +1,368 @@ +/** + * Public surface of the Phase 2 programmatic migration API. + * + * Library consumers import `migrate` / `migrationStatus` from + * `@atomicmemory/core` (re-exported via `src/index.ts`); the legacy CLI in + * `migrate.ts` is a thin wrapper around `migrate()`. Implementation is + * split across: + * - `migration-lock.ts` advisory lock + pool helpers + lock-id constant + * - `migration-schema.ts` migrations-dir read / sha256 / package version + * - `migration-version.ts` `schema_version` table reads, stamps, table probes + * - `migration-status.ts` `migrationStatus()` read-only query + * + * Phase 2 invariants: errors are thrown, Phase 1 advisory locking remains the + * coordination layer, and `MigrateOptions` / `MigrateResult` stay stable. + * `ranSchemaSql` now means "this call executed the migration runner path". + * + * Three install states are detected and dispatched correctly: + * fresh → run all migration files (baseline + successors). + * pre_phase_2 → fail-closed audit of the existing schema (see + * `./migration-baseline-validator.ts`); on success, + * ask node-pg-migrate to fake-stamp the baseline as + * applied (no DDL touches existing tables), reconcile + * vector dimensions, then let the framework run any + * post-baseline files. + * On failure, `BaselineSchemaMismatch` propagates + * without writing to either bookkeeping table. + * phase_2_current → run only the framework-detected pending migrations. + * Concurrent peers serialize on the advisory lock. Embedding-dimension + * reconciliation is delegated to `reconcileEmbeddingDimension`; reconciler + * errors propagate. + * + * Plan reference: docs/db/migrations.md. + */ + +import type { PoolClient } from 'pg'; +import { runner as runMigrations } from 'node-pg-migrate'; + +import { + resolveMigrationRuntimeOptions, + type ResolvedMigrationRuntimeOptions, +} from './migration-defaults.js'; +import { + acquireAdvisoryLock, + acquirePool, + DEFAULT_LOCK_TIMEOUT_MS, + releaseAdvisoryLock, + releasePool, + type PoolAcquireOptions, +} from './migration-lock.js'; +import { + buildAppliedSql, + buildSchemaNotes, + listMigrationFilenames, + MIGRATIONS_DIR, + readPackageVersion, + sha256Hex, +} from './migration-schema.js'; +import { + BASELINE_MIGRATION_NAME, + PGMIGRATIONS_TABLE, +} from './migration-history.js'; +import { detectInstallState } from './migration-install-state.js'; +import { + readLatestSchemaVersionOrAbsent, + stampSchemaVersion, + type SchemaVersionRow, +} from './migration-version.js'; +import { validateBaselineSchema } from './migration-baseline-validator.js'; +import { reconcileEmbeddingDimension } from './reconcilers.js'; + +// Re-export lock primitives so tests and operators can verify the exact +// advisory-lock contract without duplicating constants. +export { MIGRATION_LOCK_ID, MigrationLockTimeout } from './migration-lock.js'; +export { BaselineSchemaMismatch } from './migration-baseline-validator.js'; +export { MigrationHistoryMismatch } from './migration-history.js'; +export { + migrationStatus, + type EmbeddingDimensionStatus, + type EmbeddingDimensionStatusValue, + type MigrationHistoryStatus, + type MigrationStatus, + type MigrationStatusOptions, +} from './migration-status.js'; + +export interface MigrateOptions extends PoolAcquireOptions { + /** + * Override embedding dimensions. Defaults to the validated runtime config. + * Passed through to the post-migration reconciler. + */ + embeddingDimensions?: number; + /** + * Strip HNSW pgvector indexes from the package-applied-bytes hash. Test-only + * compatibility shim carried over from Phase 1; does NOT affect which + * migrations run against the database. + */ + skipVectorIndexes?: boolean; + /** + * Maximum time to wait for the migration advisory lock before giving up. + * Defaults to 60_000 (60s). When exceeded, throws `MigrationLockTimeout`. + */ + lockTimeoutMs?: number; +} + +export interface MigrateResult { + /** + * True when this call moved the database forward — either by applying one + * or more migration files, or by stamping the baseline as applied on a + * pre-Phase-2 install. False only on the concurrent-peer-winner path + * where another caller's stamp matched our intended package fingerprint + * before we acquired the lock. + */ + ranSchemaSql: boolean; + /** The `schema_version` row written (or last-read) by this call. */ + schemaVersion: { + sdkVersion: string; + schemaSha256: string; + appliedAt: Date; + notes: string | null; + }; + /** + * True if the embedding-dimension reconciler altered any column. False when + * the schema already matched the configured dimension. Reflects the actual + * return value from `./reconcilers.js::reconcileEmbeddingDimension`. + */ + reconciledEmbeddingDimension: boolean; +} + +interface MigrationPlan { + sdkVersion: string; + embeddingDimensions: number; + skipVectorIndexes: boolean; + appliedSha: string; + notes: string; +} + +type ResolvedMigrateOptions = ResolvedMigrationRuntimeOptions; + +/** + * Library entry point: bring the database to the latest migration head, + * coordinate replicas via a Postgres advisory lock, stamp `schema_version`, + * and reconcile embedding-dimension drift. Safe to call from any process; + * never calls `process.exit`. See `MigrateOptions` for overrides. + * + * Phase 2 detail: on a pre-Phase-2 install (data tables exist but + * `pgmigrations` does not) the baseline migration is *stamped* as applied + * rather than re-executed, so no DDL touches existing rows. Subsequent + * post-baseline migrations run normally. + */ +export async function migrate(opts: MigrateOptions = {}): Promise { + const resolved = await resolveMigrateOptions(opts); + const handle = acquirePool(resolved); + const lockTimeoutMs = opts.lockTimeoutMs ?? DEFAULT_LOCK_TIMEOUT_MS; + const plan = buildMigrationPlan(resolved); + const client = await handle.pool.connect(); + try { + return await runWithLock(client, plan, lockTimeoutMs); + } finally { + client.release(); + await releasePool(handle); + } +} + +async function resolveMigrateOptions( + opts: MigrateOptions, +): Promise { + return resolveMigrationRuntimeOptions(opts); +} + +function buildMigrationPlan(opts: ResolvedMigrateOptions): MigrationPlan { + const appliedSql = buildAppliedSql( + opts.embeddingDimensions, + opts.skipVectorIndexes, + ); + return { + sdkVersion: readPackageVersion(), + embeddingDimensions: opts.embeddingDimensions, + skipVectorIndexes: opts.skipVectorIndexes, + appliedSha: sha256Hex(appliedSql), + notes: buildSchemaNotes({ + skipVectorIndexes: opts.skipVectorIndexes, + embeddingDimensions: opts.embeddingDimensions, + }), + }; +} + +async function runWithLock( + client: PoolClient, + plan: MigrationPlan, + lockTimeoutMs: number, +): Promise { + const preLockStamp = await readLatestSchemaVersionOrAbsent(client); + await acquireAdvisoryLock(client, lockTimeoutMs); + try { + const postLockStamp = await readLatestSchemaVersionOrAbsent(client); + if (postLockStamp && peerBeatUs(preLockStamp, postLockStamp, plan)) { + return await reconcileAndReportPeerWin(client, plan, postLockStamp); + } + return await applyAndStamp(client, plan); + } finally { + await releaseAdvisoryLock(client); + } +} + +/** + * True when a concurrent peer ran the migration framework and stamped a + * matching `schema_version` row while we were waiting on the advisory lock. + * Detected by: post-lock stamp matches our intended SHA + sdk version AND + * is strictly newer than the pre-lock snapshot (or pre-lock was absent). + * Serial re-runs where the same stamp existed both pre- and post-lock are + * NOT treated as peer wins; they fall through to `applyAndStamp` which + * is a no-op for migrations (framework reports nothing pending) but still + * appends a fresh `schema_version` row for visibility. + */ +function peerBeatUs( + preLock: SchemaVersionRow | null, + postLock: SchemaVersionRow, + plan: MigrationPlan, +): boolean { + if (!stampMatchesPlan(postLock, plan)) return false; + if (!preLock) return true; + return postLock.applied_at.getTime() > preLock.applied_at.getTime(); +} + +function stampMatchesPlan(stamp: SchemaVersionRow, plan: MigrationPlan): boolean { + return stamp.schema_sha256 === plan.appliedSha + && stamp.sdk_version === plan.sdkVersion; +} + +async function reconcileAndReportPeerWin( + client: PoolClient, + plan: MigrationPlan, + peerStamp: SchemaVersionRow, +): Promise { + // Peer beat us on migrations, but our local embedding-dim config may still + // differ from the peer's. Run the reconciler so any pending column-type + // change is applied (or we fail loudly via the reconciler's error). + const reconcileResult = await reconcileEmbeddingDimension(client, plan.embeddingDimensions); + return { + ranSchemaSql: false, + schemaVersion: rowToStampPayload(peerStamp), + reconciledEmbeddingDimension: reconcileResult.reconciled, + }; +} + +async function applyAndStamp( + client: PoolClient, + plan: MigrationPlan, +): Promise { + const { state: installState } = await detectInstallState(client); + console.log( + `[migration-api] Phase 2 migrate (installState=${installState}, ` + + `embeddingDimensions=${plan.embeddingDimensions}, ` + + `skipVectorIndexes=${plan.skipVectorIndexes}, ` + + `sha=${plan.appliedSha.slice(0, 12)}…)`, + ); + if (installState === 'pre_phase_2') { + // Fail-closed audit BEFORE any write. A partial v1.0.x install, a + // stray sentinel table, or a missing required extension must not be + // stamped as if it were a real baseline — that would let the framework + // believe a broken schema is canonical. BaselineSchemaMismatch + // propagates and the surrounding finally releases the advisory lock + // without ever touching pgmigrations or schema_version. + await validateBaselineSchema(client); + await stampBaselineAsApplied(client); + } else if (installState === 'fresh') { + await runBaselineMigration(client); + } + // Reconcile immediately after the baseline exists or is stamped, before any + // future post-baseline migration can observe baseline vector columns at the + // frozen default dimension. The final pass after pending migrations catches + // empty vector columns added by later migrations. + const baselineReconcile = await reconcileEmbeddingDimension( + client, + plan.embeddingDimensions, + ); + await runFrameworkMigrationsToHead(client); + const finalReconcile = await reconcileEmbeddingDimension(client, plan.embeddingDimensions); + const stamped = await stampSchemaVersion(client, { + sdkVersion: plan.sdkVersion, + schemaSha256: plan.appliedSha, + notes: plan.notes, + }); + return { + ranSchemaSql: true, + schemaVersion: rowToStampPayload(stamped), + reconciledEmbeddingDimension: + baselineReconcile.reconciled || finalReconcile.reconciled, + }; +} + +/** + * Pre-Phase-2 cutover path: ask node-pg-migrate to fake-stamp the baseline + * WITHOUT executing the baseline DDL. The data tables already exist; re-running + * `0001_baseline.sql` would either no-op (idempotent statements) or attempt to + * revalidate constraints — neither outcome is desirable when production data is + * sitting in those tables. + */ +async function stampBaselineAsApplied(client: PoolClient): Promise { + await runMigrationRunner(client, { file: BASELINE_MIGRATION_NAME, fake: true }); +} + +async function runBaselineMigration(client: PoolClient): Promise { + await runMigrationRunner(client, { file: BASELINE_MIGRATION_NAME }); +} + +/** + * Drive node-pg-migrate to apply every migration file under + * `MIGRATIONS_DIR` that is not already recorded in `pgmigrations`. We pass + * our own checked-out client so the framework runs inside the connection + * that holds the Phase 1 advisory lock, and disable the framework's own + * advisory-lock layer to avoid double-locking. + * + * On a fresh database this applies successors after `runBaselineMigration`. + * On a pre-Phase-2 install (after `stampBaselineAsApplied`) the framework sees + * the baseline as already-recorded and runs only `0002_*` and later. On a + * `phase_2_current` install it runs only the pending tail. + * + * `listMigrationFilenames()` is invoked first as a fail-closed precondition: + * a missing/empty migrations directory or a missing baseline throws before + * `node-pg-migrate` is allowed to create `pgmigrations` and stamp progress + * against an empty DB. The previous "tolerate empty" fast path could let a + * misbuilt package mark a DB as migrated without ever applying DDL; that + * was the audit finding this guards against. + */ +async function runFrameworkMigrationsToHead(client: PoolClient): Promise { + await runMigrationRunner(client); +} + +async function runMigrationRunner( + client: PoolClient, + opts: { file?: string; fake?: boolean } = {}, +): Promise { + // Side-effect call: validates the shipped migration set before letting the + // framework touch the database. Throws on missing dir / no .sql files / + // missing 0001_baseline.sql / empty file. Return value intentionally + // discarded — node-pg-migrate enumerates the directory itself. + listMigrationFilenames(); + await runMigrations({ + dbClient: client, + dir: MIGRATIONS_DIR, + migrationsTable: PGMIGRATIONS_TABLE, + direction: 'up', + file: opts.file, + fake: opts.fake, + // We hold MIGRATION_LOCK_ID for the duration of runWithLock; disable + // node-pg-migrate's own advisory lock so the two coordination layers + // don't fight or accidentally double-acquire. + noLock: true, + // Keep each migrate() call atomic when all pending migrations support + // transactions. A future migration that needs non-transactional DDL must + // be a JS/TS migration that calls pgm.noTransaction(). + singleTransaction: true, + log: forwardFrameworkLog, + }); +} + +function forwardFrameworkLog(message: string): void { + console.log(`[node-pg-migrate] ${message}`); +} + +function rowToStampPayload(row: SchemaVersionRow): MigrateResult['schemaVersion'] { + return { + sdkVersion: row.sdk_version, + schemaSha256: row.schema_sha256, + appliedAt: row.applied_at, + notes: row.notes, + }; +} diff --git a/src/db/migration-baseline-validator.ts b/src/db/migration-baseline-validator.ts new file mode 100644 index 0000000..e52b86a --- /dev/null +++ b/src/db/migration-baseline-validator.ts @@ -0,0 +1,196 @@ +/** + * Pre-Phase-2 baseline-schema validator. + * + * `detectInstallState()` classifies a database as `pre_phase_2` whenever a + * single v1.0.x sentinel table exists. That probe is intentionally cheap, but + * it is not sufficient to authorize stamping `0001_baseline` as applied: + * + * - A partial v1.0.x install that crashed midway through `schema.sql` may + * have created `memories` but skipped `memory_claims` / `memory_evidence`. + * - A wholly unrelated application could have its own `memories` table with + * entirely different columns sharing the namespace. + * - The `vector` or `pgcrypto` extensions may have been dropped or never + * installed. + * + * Stamping `0001_baseline` under any of those conditions would lie to the + * migration framework: subsequent `migrate()` calls would treat the broken + * schema as canonical and run `0002_*` migrations on top of it, almost + * certainly failing in confusing ways or — worse — succeeding while leaving + * the database in an unrecoverable state. + * + * This module performs a fail-closed structural audit before the baseline is + * stamped. It executes only `SELECT` statements; nothing is created, altered, + * or dropped. On any failure it throws `BaselineSchemaMismatch` carrying a + * concrete list of missing artifacts so the caller (and operators) can see + * exactly what tripped the guard. + * + * Boundary: this validator owns the pre-stamp decision only. Install-state + * classification (`detectInstallState`), packaging hash computation, and + * `migrationStatus()` are owned by other modules and are not touched here. + */ + +import type { PoolClient } from 'pg'; + +/** + * PostgreSQL extensions that `0001_baseline.sql` requires. Both are created + * unconditionally at the top of the baseline file, so a real v1.0.x or + * Phase 1 database always has them. A pre_phase_2-classified database that + * is missing either one is structurally invalid for stamping. + */ +const REQUIRED_EXTENSIONS = ['vector', 'pgcrypto'] as const; + +/** + * Tables that any genuine v1.0.x / Phase 1 install carries. The set is the + * closure of the original-claim-storage relationships: episodes → + * canonical_memory_objects, memories ↔ entities, and the claim/version/ + * evidence chain. A pre_phase_2-classified database missing any of these + * is partial; refusing to stamp is the only safe response. + * + * Deliberately narrower than the full baseline table list — those tables are + * the "you cannot be a real install without these" core. Optional tables + * added in later v1.0.x point releases are validated by their column shape + * if and only if they are present (see REQUIRED_COLUMNS comment). + */ +const REQUIRED_TABLES = [ + 'episodes', + 'canonical_memory_objects', + 'memories', + 'memory_claims', + 'memory_claim_versions', + 'memory_evidence', + 'entities', + 'memory_entities', +] as const; + +/** + * Required column shapes on the canary tables. The point of this layer is to + * reject stray tables that happen to share a name with one of our canonical + * ones: a `memories` table created by some other application is unlikely to + * carry a pgvector-typed `embedding` column AND a TEXT `user_id` AND a TEXT + * `content` — and certainly not all three at once with the v1.0.x layout. + * + * `pgTypeName` is the value Postgres reports in `pg_type.typname` for the + * column's declared type (`text`, `vector`, `uuid`, etc.) — the same shape + * the existing structural-snapshot helper uses, so the catalog query is the + * authoritative source. Columns can be NOT NULL or nullable; the column + * shape, not the nullability constraint, is what discriminates a real + * baseline from a stray collision. + */ +const REQUIRED_COLUMNS: ReadonlyArray<{ + table: (typeof REQUIRED_TABLES)[number]; + column: string; + pgTypeName: string; +}> = [ + { table: 'memories', column: 'embedding', pgTypeName: 'vector' }, + { table: 'memories', column: 'user_id', pgTypeName: 'text' }, + { table: 'memories', column: 'content', pgTypeName: 'text' }, + { table: 'episodes', column: 'user_id', pgTypeName: 'text' }, + { table: 'episodes', column: 'content', pgTypeName: 'text' }, + { table: 'memory_claims', column: 'user_id', pgTypeName: 'text' }, +]; + +/** + * Thrown by `validateBaselineSchema` when the database fails the audit. + * + * `missing` carries a human-readable list of missing artifacts (`extension:`, + * `table:`, `column:.:`). The caller is + * expected to surface the error verbatim so operators can repair the schema + * (or wipe and reinstall) before retrying `migrate()`. + */ +export class BaselineSchemaMismatch extends Error { + constructor(public readonly missing: ReadonlyArray) { + super( + `[migration-api] Refusing to stamp 0001_baseline as applied: ` + + `existing schema is missing ${missing.length} required artifact(s): ` + + `${missing.join(', ')}. ` + + `This typically means a partial v1.0.x install, a stray table sharing a ` + + `canonical name, or a missing PostgreSQL extension. Repair the schema ` + + `(or DROP SCHEMA public CASCADE and re-run migrate() for a fresh install) ` + + `and retry.`, + ); + this.name = 'BaselineSchemaMismatch'; + } +} + +/** + * Validate that a `pre_phase_2`-classified database carries the structural + * shape required to be a real v1.0.x / Phase 1 install. Pure reads; never + * mutates the database. Throws `BaselineSchemaMismatch` on any failure with + * the full list of missing artifacts; returns normally on success. + * + * Call this immediately before `stampBaselineAsApplied()` in the migration + * runner so a rejected schema causes the lock to be released without writing + * to either `pgmigrations` or `schema_version`. + */ +export async function validateBaselineSchema(client: PoolClient): Promise { + const missing: string[] = []; + await collectMissingExtensions(client, missing); + await collectMissingTables(client, missing); + await collectMissingColumns(client, missing); + if (missing.length > 0) { + throw new BaselineSchemaMismatch(missing); + } +} + +async function collectMissingExtensions( + client: PoolClient, + missing: string[], +): Promise { + const { rows } = await client.query<{ extname: string }>( + 'SELECT extname FROM pg_extension WHERE extname = ANY($1::text[])', + [REQUIRED_EXTENSIONS], + ); + const present = new Set(rows.map((row) => row.extname)); + for (const ext of REQUIRED_EXTENSIONS) { + if (!present.has(ext)) missing.push(`extension:${ext}`); + } +} + +async function collectMissingTables( + client: PoolClient, + missing: string[], +): Promise { + const { rows } = await client.query<{ relname: string }>( + "SELECT c.relname FROM pg_class c " + + 'JOIN pg_namespace n ON c.relnamespace = n.oid ' + + "WHERE n.nspname = current_schema() AND c.relkind = 'r' " + + 'AND c.relname = ANY($1::text[])', + [REQUIRED_TABLES], + ); + const present = new Set(rows.map((row) => row.relname)); + for (const table of REQUIRED_TABLES) { + if (!present.has(table)) missing.push(`table:${table}`); + } +} + +async function collectMissingColumns( + client: PoolClient, + missing: string[], +): Promise { + const { rows } = await client.query<{ + relname: string; + attname: string; + typname: string; + }>( + 'SELECT c.relname, a.attname, t.typname ' + + 'FROM pg_attribute a ' + + 'JOIN pg_class c ON a.attrelid = c.oid ' + + 'JOIN pg_namespace n ON c.relnamespace = n.oid ' + + 'JOIN pg_type t ON a.atttypid = t.oid ' + + 'WHERE n.nspname = current_schema() ' + + "AND c.relkind = 'r' AND NOT a.attisdropped AND a.attnum > 0", + ); + const observed = new Map(); + for (const row of rows) { + observed.set(`${row.relname}.${row.attname}`, row.typname); + } + for (const required of REQUIRED_COLUMNS) { + const key = `${required.table}.${required.column}`; + const actual = observed.get(key); + if (actual === undefined) { + missing.push(`column:${key}:${required.pgTypeName}`); + } else if (actual !== required.pgTypeName) { + missing.push(`column:${key}:${required.pgTypeName} (found ${actual})`); + } + } +} diff --git a/src/db/migration-defaults.ts b/src/db/migration-defaults.ts new file mode 100644 index 0000000..1a0be86 --- /dev/null +++ b/src/db/migration-defaults.ts @@ -0,0 +1,73 @@ +/** + * Lazy runtime-config defaults for Phase 1 migration entry points. + * + * The DB migration modules are intentionally not static consumers of the + * module-level `config` singleton. That keeps them compatible with the + * config-threading ratchet while still letting `migrate()` and + * `migrationStatus()` default to startup-validated config when callers do + * not pass explicit options. + */ + +import type { RuntimeConfig } from '../config.js'; + +export interface MigrationRuntimeDefaults { + databaseUrl: string; + embeddingDimensions: number; + skipVectorIndexes: boolean; +} + +export interface MigrationRuntimeOptions { + pool?: unknown; + databaseUrl?: string; + embeddingDimensions?: number; + skipVectorIndexes?: boolean; +} + +export type ResolvedMigrationRuntimeOptions = T & { + embeddingDimensions: number; + skipVectorIndexes: boolean; +}; + +async function loadMigrationRuntimeDefaults(): Promise { + const { config } = await import('../config.js'); + return pickMigrationRuntimeDefaults(config); +} + +export async function resolveMigrationRuntimeOptions( + opts: T, +): Promise> { + const defaults = needsRuntimeDefaults(opts) + ? await loadMigrationRuntimeDefaults() + : null; + return { + ...opts, + databaseUrl: opts.databaseUrl ?? defaults?.databaseUrl, + embeddingDimensions: opts.embeddingDimensions + ?? requireDefaults(defaults).embeddingDimensions, + skipVectorIndexes: opts.skipVectorIndexes + ?? requireDefaults(defaults).skipVectorIndexes, + }; +} + +function pickMigrationRuntimeDefaults(config: RuntimeConfig): MigrationRuntimeDefaults { + return { + databaseUrl: config.databaseUrl, + embeddingDimensions: config.embeddingDimensions, + skipVectorIndexes: config.skipVectorIndexes, + }; +} + +function needsRuntimeDefaults(opts: MigrationRuntimeOptions): boolean { + return (!opts.pool && !opts.databaseUrl) + || opts.embeddingDimensions === undefined + || opts.skipVectorIndexes === undefined; +} + +function requireDefaults( + defaults: MigrationRuntimeDefaults | null, +): MigrationRuntimeDefaults { + if (!defaults) { + throw new Error('migration runtime defaults were not loaded'); + } + return defaults; +} diff --git a/src/db/migration-history.ts b/src/db/migration-history.ts new file mode 100644 index 0000000..d22e1e7 --- /dev/null +++ b/src/db/migration-history.ts @@ -0,0 +1,58 @@ +/** + * Read-only helpers for the node-pg-migrate bookkeeping table. + * + * Phase 2 keeps framework history in `pgmigrations` and semantic package + * stamps in `schema_version`. This module centralizes the framework-history + * probes so `migrate()` and `migrationStatus()` classify partial/corrupt + * metadata the same way. + */ + +import type { Pool, PoolClient } from 'pg'; + +import { tableExists } from './migration-version.js'; + +export const PGMIGRATIONS_TABLE = 'pgmigrations'; +export const BASELINE_MIGRATION_NAME = '0001_baseline'; + +export interface MigrationHistory { + readonly tableExists: boolean; + readonly appliedMigrationCount: number; + readonly latestMigrationName: string; + readonly hasBaseline: boolean; + readonly names: readonly string[]; +} + +export const EMPTY_MIGRATION_HISTORY: MigrationHistory = { + tableExists: false, + appliedMigrationCount: 0, + latestMigrationName: '', + hasBaseline: false, + names: [], +}; + +export class MigrationHistoryMismatch extends Error { + constructor(message: string) { + super(`[migration-api] ${message}`); + this.name = 'MigrationHistoryMismatch'; + } +} + +export async function readMigrationHistory( + client: Pick, +): Promise { + if (!(await tableExists(client, PGMIGRATIONS_TABLE))) { + return EMPTY_MIGRATION_HISTORY; + } + const { rows } = await client.query<{ name: string }>( + `SELECT name FROM ${PGMIGRATIONS_TABLE} ORDER BY id ASC`, + ); + const names = rows.map((row) => row.name); + return { + tableExists: true, + appliedMigrationCount: names.length, + latestMigrationName: names[names.length - 1] ?? '', + hasBaseline: names.includes(BASELINE_MIGRATION_NAME), + names, + }; +} + diff --git a/src/db/migration-install-state.ts b/src/db/migration-install-state.ts new file mode 100644 index 0000000..70d6c24 --- /dev/null +++ b/src/db/migration-install-state.ts @@ -0,0 +1,58 @@ +/** + * Install-state detection for the Phase 2 migration runner. + * + * Keeps the main migration API focused on sequencing while this module owns + * the catalog probes that decide whether baseline should run, be fake-stamped, + * or be rejected as corrupt framework history. + */ + +import type { PoolClient } from 'pg'; + +import { + BASELINE_MIGRATION_NAME, + MigrationHistoryMismatch, + readMigrationHistory, +} from './migration-history.js'; +import { tableExists } from './migration-version.js'; + +const PRE_PHASE_2_SENTINEL_TABLES = ['memories', 'episodes', 'memory_claims'] as const; + +export type InstallState = 'fresh' | 'pre_phase_2' | 'phase_2_current'; + +export interface InstallStateInfo { + readonly state: InstallState; +} + +/** + * Distinguish Phase 2 install states without enumerating the whole catalog. + * + * An empty `pgmigrations` table is recoverable. A non-empty history that lacks + * `0001_baseline` is not safely inferable, so the runner fails closed before + * running baseline DDL or stamping semantic package state. + */ +export async function detectInstallState(client: PoolClient): Promise { + const history = await readMigrationHistory(client); + if (!history.tableExists) { + return stateFromSentinels(await hasPrePhase2Sentinel(client)); + } + if (history.hasBaseline) return { state: 'phase_2_current' }; + if (history.appliedMigrationCount === 0) { + return stateFromSentinels(await hasPrePhase2Sentinel(client)); + } + throw new MigrationHistoryMismatch( + `pgmigrations exists with ${history.appliedMigrationCount} row(s) but is ` + + `missing ${BASELINE_MIGRATION_NAME}. Refusing to infer a safe cutover path.`, + ); +} + +async function hasPrePhase2Sentinel(client: PoolClient): Promise { + for (const sentinel of PRE_PHASE_2_SENTINEL_TABLES) { + if (await tableExists(client, sentinel)) return true; + } + return false; +} + +function stateFromSentinels(hasSentinel: boolean): InstallStateInfo { + return { state: hasSentinel ? 'pre_phase_2' : 'fresh' }; +} + diff --git a/src/db/migration-lock.ts b/src/db/migration-lock.ts new file mode 100644 index 0000000..821919e --- /dev/null +++ b/src/db/migration-lock.ts @@ -0,0 +1,105 @@ +/** + * Postgres advisory-lock coordination for the Phase 1 migration runner. + * + * Exposes a single fixed lock id and a poll-based acquire helper that uses + * `pg_try_advisory_lock` (not the blocking `pg_advisory_lock`). The blocking + * form offers no timeout control; the try-loop is the only shape that lets + * library callers fail with `MigrationLockTimeout` instead of blocking a host + * process indefinitely. + * + * Also owns the connection-pool helper pair so call sites that need both + * a pool and a lock can import from a single coordination module. + */ + +import pg from 'pg'; +import type { Pool, PoolClient } from 'pg'; + +/** + * Stable 64-bit identifier for the schema-evolution advisory lock. + * + * Generated once from: + * echo -n '@atomicmemory/core::schema-migration' | sha256sum + * then took the first 16 hex chars and reinterpreted as a signed bigint. + * Documented so any future collision report has context. Tests import this + * constant directly (see core-db-phase1-tests) to assert the running + * implementation uses the exact advertised lock id. + */ +export const MIGRATION_LOCK_ID = -3473291475947293849n; + +export const DEFAULT_LOCK_TIMEOUT_MS = 60_000; +const LOCK_POLL_INTERVAL_MS = 500; + +/** Thrown when the migration advisory lock cannot be acquired in time. */ +export class MigrationLockTimeout extends Error { + constructor(message: string) { + super(message); + this.name = 'MigrationLockTimeout'; + } +} + +export interface PoolHandle { + pool: Pool; + /** True when this module created the pool and must close it on exit. */ + owned: boolean; +} + +export interface PoolAcquireOptions { + databaseUrl?: string; + pool?: Pool; +} + +export function acquirePool(opts: PoolAcquireOptions): PoolHandle { + if (opts.pool) { + return { pool: opts.pool, owned: false }; + } + if (!opts.databaseUrl) { + throw new Error('databaseUrl is required when pool is not provided'); + } + const pool = new pg.Pool({ + connectionString: opts.databaseUrl, + max: 1, + connectionTimeoutMillis: 30_000, + idleTimeoutMillis: 60_000, + }); + pool.on('error', (err) => { + console.error('[migration-lock] Unexpected idle client error:', err.message); + }); + return { pool, owned: true }; +} + +export async function releasePool(handle: PoolHandle): Promise { + if (handle.owned) { + await handle.pool.end(); + } +} + +function sleep(ms: number): Promise { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); +} + +export async function acquireAdvisoryLock( + client: PoolClient, + lockTimeoutMs: number, +): Promise { + const deadline = Date.now() + lockTimeoutMs; + while (!(await tryAcquireAdvisoryLock(client))) { + if (Date.now() >= deadline) { + throw new MigrationLockTimeout( + `Could not acquire migration advisory lock within ${lockTimeoutMs}ms`, + ); + } + await sleep(LOCK_POLL_INTERVAL_MS); + } +} + +async function tryAcquireAdvisoryLock(client: PoolClient): Promise { + const { rows } = await client.query<{ acquired: boolean }>( + 'SELECT pg_try_advisory_lock($1) AS acquired', + [MIGRATION_LOCK_ID.toString()], + ); + return rows[0]?.acquired === true; +} + +export async function releaseAdvisoryLock(client: PoolClient): Promise { + await client.query('SELECT pg_advisory_unlock($1)', [MIGRATION_LOCK_ID.toString()]); +} diff --git a/src/db/migration-schema.ts b/src/db/migration-schema.ts new file mode 100644 index 0000000..355b0b7 --- /dev/null +++ b/src/db/migration-schema.ts @@ -0,0 +1,240 @@ +/** + * Schema-file operations for the Phase 2 migration runner. + * + * Phase 2 replaces `schema.sql` with a directory of versioned migration files + * under `src/db/migrations/` (frozen `0001_baseline.sql` + dated successors). + * This module enumerates those files, reads them, and computes the package's + * canonical schema fingerprint. + * + * **Fingerprint shape (audit-tightened):** the package SHA is no longer the + * SHA-256 of the raw concatenated SQL bytes. It is the SHA-256 of an ordered + * `\t\n` manifest, computed across every shipped + * `.sql` migration in lexical order. Filename identity participates in the + * hash so a future rename or reorder cannot collide with the old digest even + * if the underlying SQL bytes happen to be equal. The manifest text is + * canonicalized (deterministic separators, no JSON whitespace dependency) so + * identical shipped bytes always produce identical digests across PG/Node + * versions and CI runners. + * + * **Fail-closed policy:** Phase 2 only ships SQL migration files. The list + * helper throws when the migrations directory is missing, when it contains + * no `.sql` files, when the frozen baseline (`0001_baseline.sql`) is absent, + * or when any shipped `.sql` file is empty. Earlier versions of this module + * silently tolerated these states, which let a misbuilt package stamp a DB + * with an empty schema; that is now a build/runtime error. + * + * Kept as a separate module so the side-effect-free transformations can be + * unit-tested without a database, and so the migration runner stays focused + * on coordination + persistence rather than file shape. + */ + +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +/** + * Matches every HNSW pgvector index `CREATE INDEX` statement in the shipped + * migrations, regardless of whether the indexed column is literally + * `embedding` or a topic / summary / recap variant. Anchored on the + * `USING hnsw ( vector_cosine_ops)` clause that is uniform across + * all such indexes; the `[^;]*?` segments forbid crossing a `;` so we cannot + * accidentally swallow neighbouring statements. Trailing newline / blank + * line consumption keeps the post-strip SQL tidy. + */ +const VECTOR_INDEX_REGEX = + /CREATE INDEX(?:\s+IF NOT EXISTS)?[^;]*?USING hnsw\s*\([a-z_]+\s+vector_cosine_ops\)[^;]*;(?:\n\n?|$)/g; + +/** + * Directory containing the package's migration files. Resolves identically + * at dev (`src/db/migrations`) and dist (`dist/db/migrations`) because the + * build step copies the directory verbatim alongside the compiled JS. + */ +export const MIGRATIONS_DIR = resolve(__dirname, 'migrations'); + +/** + * Filename of the frozen baseline migration. Its presence is part of the + * package's invariant: every shipped build MUST contain this file, because + * the Phase 2 cutover stamps it as already-applied on pre-Phase-2 upgrades + * (see `migration-api.ts:stampBaselineAsApplied`). A build without it is + * unshippable and is rejected at list time. + */ +const BASELINE_MIGRATION_FILENAME = '0001_baseline.sql'; + +export function sha256Hex(value: string): string { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +/** + * Return the package's shipped migration filenames in lexical order. Used + * both by the runtime (to drive the migration framework) and by tests. + * + * Fails closed when the directory is missing, contains zero `.sql` files, + * is missing the frozen baseline, or contains an empty `.sql` file. Any of + * those states means the package is misbuilt (or the working tree is in + * the middle of a half-finished edit) and continuing would risk stamping + * a database without ever creating the schema. + */ +export function listMigrationFilenames(): string[] { + const filenames = readSqlFilenamesOrThrow(MIGRATIONS_DIR); + assertBaselinePresent(filenames); + assertNoEmptyFiles(MIGRATIONS_DIR, filenames); + return filenames; +} + +function readSqlFilenamesOrThrow(dir: string): string[] { + let entries: string[]; + try { + entries = readdirSync(dir); + } catch (err) { + throw new Error( + `[migration-schema] migrations directory is missing at ${dir}. ` + + `Phase 2 packages must ship src/db/migrations/0001_baseline.sql.`, + { cause: err as Error }, + ); + } + const sqlFiles = entries.filter((name) => name.endsWith('.sql')).sort(); + if (sqlFiles.length === 0) { + throw new Error( + `[migration-schema] migrations directory at ${dir} contains zero .sql files. ` + + `Phase 2 packages must ship at least src/db/migrations/0001_baseline.sql.`, + ); + } + return sqlFiles; +} + +function assertBaselinePresent(filenames: ReadonlyArray): void { + if (!filenames.includes(BASELINE_MIGRATION_FILENAME)) { + throw new Error( + `[migration-schema] frozen baseline ${BASELINE_MIGRATION_FILENAME} is missing ` + + `from ${MIGRATIONS_DIR}. The Phase 2 cutover contract requires it to ` + + `be present in every shipped build.`, + ); + } +} + +function assertNoEmptyFiles(dir: string, filenames: ReadonlyArray): void { + for (const name of filenames) { + const { size } = statSync(join(dir, name)); + if (size === 0) { + throw new Error( + `[migration-schema] migration file ${name} in ${dir} is empty. ` + + `An empty file would stamp the package SHA without applying any DDL.`, + ); + } + } +} + +function readMigrationFile(name: string): string { + return readFileSync(join(MIGRATIONS_DIR, name), 'utf-8'); +} + +function maybeStripVectorIndexes(sql: string, strip: boolean): string { + return strip ? sql.replace(VECTOR_INDEX_REGEX, '') : sql; +} + +/** + * Per-file entry in the canonical manifest. Filename ordering is preserved + * by `listMigrationFilenames()`; each entry's `sha256` is computed over the + * post-strip SQL bytes (i.e., `skipVectorIndexes` participates in the hash + * so test runs against pgvector-less Postgres deterministically diverge + * from production runs). + */ +interface MigrationManifestEntry { + readonly filename: string; + readonly sha256: string; + readonly bytes: number; +} + +/** + * Build an ordered manifest of {filename, per-file sha256, byte length} for + * every shipped `.sql` migration. Used by both the runtime fingerprint and + * the build-time `scripts/generate-schema-hash.ts` manifest writer so the + * two stay in lockstep. + */ +function buildMigrationManifest( + skipVectorIndexes: boolean = false, +): MigrationManifestEntry[] { + return listMigrationFilenames().map((filename) => { + const sql = maybeStripVectorIndexes(readMigrationFile(filename), skipVectorIndexes); + return { + filename, + sha256: sha256Hex(sql), + bytes: Buffer.byteLength(sql, 'utf8'), + }; + }); +} + +/** + * Canonical text whose SHA-256 is the package's aggregate schema fingerprint. + * Format is one line per shipped migration, in lexical order: + * + * \t\n + * + * Filename identity participates so a rename cannot collide with the + * pre-rename digest even when the underlying SQL bytes are identical. No + * JSON, no trailing whitespace, no platform-dependent line endings — the + * function is intentionally restrictive so the same shipped bytes always + * produce the same digest across runners. + */ +export function buildMigrationManifestText(skipVectorIndexes: boolean = false): string { + return buildMigrationManifest(skipVectorIndexes) + .map((entry) => `${entry.filename}\t${entry.sha256}\n`) + .join(''); +} + +/** + * Returns the canonical hash-input text for the package's currently-shipped + * migration set. Both `migrate()` and `migrationStatus()` feed this through + * `sha256Hex()` to compute the `schema_sha256` value stamped into + * `schema_version` (and the value compared against the DB-recorded stamp). + * + * NOTE: despite the historical name, this function no longer returns raw + * concatenated SQL bytes; the audit found that hashing raw bytes lost + * filename/order identity, so the contract was tightened to "hash the + * manifest of {filename, per-file sha256}". The function name is preserved + * for call-site source compatibility (see `migration-api.ts`, + * `migration-status.ts`); the bytes returned are the manifest text from + * `buildMigrationManifestText()`. + * + * `embeddingDimensions` is accepted for API parity with Phase 1 but is + * ignored: the Phase 1 `{{EMBEDDING_DIMENSIONS}}` template was eliminated + * when `0001_baseline.sql` was frozen with a literal default, so the + * manifest is stable across `embeddingDimensions` values. The runtime + * reconciler adjusts column dimensions post-migrate. + */ +export function buildAppliedSql( + embeddingDimensions: number, + skipVectorIndexes: boolean, +): string { + void embeddingDimensions; + return buildMigrationManifestText(skipVectorIndexes); +} + +export function readPackageVersion(): string { + // dist/db/migration-schema.js → ../../package.json + // src/db/migration-schema.ts → ../../package.json + const packageJsonPath = resolve(__dirname, '..', '..', 'package.json'); + const parsed = JSON.parse(readFileSync(packageJsonPath, 'utf-8')) as { version?: string }; + if (!parsed.version) { + throw new Error('[migration-schema] package.json is missing "version"'); + } + return parsed.version; +} + +/** + * Build the `notes` column captured alongside every `schema_version` row. + * Records non-default configuration choices that materially affect what + * bytes were applied, so a history grep is enough to reconstruct intent. + */ +export function buildSchemaNotes(opts: { + skipVectorIndexes: boolean; + embeddingDimensions: number; +}): string { + const parts: string[] = []; + if (opts.skipVectorIndexes) parts.push('skipVectorIndexes=true'); + parts.push(`embeddingDimensions=${opts.embeddingDimensions}`); + return parts.join(','); +} diff --git a/src/db/migration-status.ts b/src/db/migration-status.ts new file mode 100644 index 0000000..08c39b9 --- /dev/null +++ b/src/db/migration-status.ts @@ -0,0 +1,323 @@ +/** + * Read-only status query for the Phase 2 migration runner. + * + * `migrationStatus()` answers "is this DB ahead of, behind, or level with the + * migrations that ship in the running package?" without modifying anything. + * Designed to be safe to expose on an admin HTTP route and to be called from + * deploy verification scripts. + */ + +import type { Pool } from 'pg'; + +import { + acquirePool, + releasePool, + type PoolAcquireOptions, +} from './migration-lock.js'; +import { + resolveMigrationRuntimeOptions, + type ResolvedMigrationRuntimeOptions, +} from './migration-defaults.js'; +import { + buildAppliedSql, + listMigrationFilenames, + readPackageVersion, + sha256Hex, +} from './migration-schema.js'; +import { + CORE_TABLE_PROBE, + readLatestSchemaVersion, + tableExists, +} from './migration-version.js'; +import { + EMPTY_MIGRATION_HISTORY, + readMigrationHistory, + type MigrationHistory, +} from './migration-history.js'; +import { + inspectEmbeddingDimensionStatus, + noSchemaEmbeddingStatus, + type EmbeddingDimensionStatus as EmbeddingDimensionStatusReport, +} from './embedding-dimension-status.js'; + +export type { + EmbeddingDimensionMismatchSummary, + EmbeddingDimensionStatus, + EmbeddingDimensionStatusValue, +} from './embedding-dimension-status.js'; + +export interface MigrationStatus { + /** Latest `sdk_version` stamped in the DB, or null if unstamped. */ + appliedSdkVersion: string | null; + /** Latest `schema_sha256` stamped in the DB, or null if unstamped. */ + appliedSchemaSha: string | null; + /** The running package's version (from `package.json`). */ + packageSdkVersion: string; + /** SHA-256 of the migration bytes this package would stamp. */ + packageSchemaSha: string; + /** Number of rows in `pgmigrations`, or 0 when the table is absent. */ + appliedMigrationCount: number; + /** Latest migration name from `pgmigrations`, or an empty string when absent. */ + latestMigrationName: string; + /** Coarse framework-bookkeeping health for operator diagnostics. */ + migrationHistoryStatus: MigrationHistoryStatus; + /** Read-only pgvector dimension drift report for this runtime config. */ + embeddingDimension: EmbeddingDimensionStatusReport; + /** + * - `up_to_date`: DB `schema_sha256` and migration history match this package. + * - `older_db`: DB has an earlier `sdk_version` (running `migrate()` will fix). + * - `newer_db`: DB has a later `sdk_version` (rolling-deploy mismatch; safe to ignore). + * - `unstamped`: DB has core tables but no `schema_version` row (pre-Phase-1). + * - `no_schema`: DB has no `@atomicmemory` tables at all (fresh). + */ + status: 'up_to_date' | 'older_db' | 'newer_db' | 'unstamped' | 'no_schema'; +} + +export interface MigrationStatusOptions extends PoolAcquireOptions { + /** + * Override embedding dimensions used when computing `packageSchemaSha`. + * Defaults to `config.embeddingDimensions`. Supplied so callers can compute + * status without triggering a network probe. + */ + embeddingDimensions?: number; + /** Strip vector indexes when computing the package SHA. Defaults to `config.skipVectorIndexes`. */ + skipVectorIndexes?: boolean; +} + +interface PackageFingerprint { + packageSdkVersion: string; + packageSchemaSha: string; + embeddingDimensions: number; + expectedMigrationCount: number; + expectedLatestMigrationName: string; +} + +export type MigrationHistoryStatus = + | 'absent' + | 'missing_baseline' + | 'behind' + | 'current' + | 'ahead'; + +type ResolvedMigrationStatusOptions = + ResolvedMigrationRuntimeOptions; + +/** + * Read-only inspection of the DB's migration state. Never modifies the + * database. When `embeddingDimensions` is omitted, the synchronous + * `config.embeddingDimensions` is used so this function never probes the + * embedding provider over the network. + */ +export async function migrationStatus( + opts: MigrationStatusOptions = {}, +): Promise { + const resolved = await resolveMigrationStatusOptions(opts); + const handle = acquirePool(resolved); + const fingerprint: PackageFingerprint = { + packageSdkVersion: readPackageVersion(), + packageSchemaSha: sha256Hex( + buildAppliedSql( + resolved.embeddingDimensions, + resolved.skipVectorIndexes, + ), + ), + embeddingDimensions: resolved.embeddingDimensions, + ...readExpectedMigrationSummary(), + }; + try { + return await computeStatusUsingPool(handle.pool, fingerprint); + } finally { + await releasePool(handle); + } +} + +async function resolveMigrationStatusOptions( + opts: MigrationStatusOptions, +): Promise { + return resolveMigrationRuntimeOptions(opts); +} + +async function computeStatusUsingPool( + pool: Pool, + fingerprint: PackageFingerprint, +): Promise { + if (!(await tableExists(pool, CORE_TABLE_PROBE))) { + return absentStatus(fingerprint, 'no_schema'); + } + const embeddingDimension = await inspectEmbeddingDimensionStatus( + pool, + fingerprint.embeddingDimensions, + ); + if (!(await tableExists(pool, 'schema_version'))) { + return absentStatus( + fingerprint, + 'unstamped', + await readMigrationHistory(pool), + embeddingDimension, + ); + } + const latest = await readLatestSchemaVersion(pool); + const migrationHistory = await readMigrationHistory(pool); + if (!latest) { + return absentStatus(fingerprint, 'unstamped', migrationHistory, embeddingDimension); + } + return { + appliedSdkVersion: latest.sdk_version, + appliedSchemaSha: latest.schema_sha256, + packageSdkVersion: fingerprint.packageSdkVersion, + packageSchemaSha: fingerprint.packageSchemaSha, + appliedMigrationCount: migrationHistory.appliedMigrationCount, + latestMigrationName: migrationHistory.latestMigrationName, + migrationHistoryStatus: classifyMigrationHistoryStatus( + migrationHistory, + fingerprint, + ), + embeddingDimension, + status: classifyStatus({ + appliedSdk: latest.sdk_version, + appliedSha: latest.schema_sha256, + packageSdk: fingerprint.packageSdkVersion, + packageSha: fingerprint.packageSchemaSha, + appliedMigrations: migrationHistory, + expectedMigrations: { + appliedMigrationCount: fingerprint.expectedMigrationCount, + latestMigrationName: fingerprint.expectedLatestMigrationName, + }, + }), + }; +} + +function absentStatus( + fingerprint: PackageFingerprint, + status: 'no_schema' | 'unstamped', + migrationHistory: MigrationHistory = EMPTY_MIGRATION_HISTORY, + embeddingDimension: EmbeddingDimensionStatusReport = noSchemaEmbeddingStatus( + fingerprint.embeddingDimensions, + ), +): MigrationStatus { + return { + appliedSdkVersion: null, + appliedSchemaSha: null, + packageSdkVersion: fingerprint.packageSdkVersion, + packageSchemaSha: fingerprint.packageSchemaSha, + appliedMigrationCount: migrationHistory.appliedMigrationCount, + latestMigrationName: migrationHistory.latestMigrationName, + migrationHistoryStatus: classifyMigrationHistoryStatus(migrationHistory, fingerprint), + embeddingDimension, + status, + }; +} + +interface MigrationSummary { + appliedMigrationCount: number; + latestMigrationName: string; +} + +function readExpectedMigrationSummary(): Pick< + PackageFingerprint, + 'expectedMigrationCount' | 'expectedLatestMigrationName' +> { + const filenames = listMigrationFilenames(); + const latest = filenames[filenames.length - 1] ?? ''; + return { + expectedMigrationCount: filenames.length, + expectedLatestMigrationName: stripMigrationExtension(latest), + }; +} + +function stripMigrationExtension(filename: string): string { + return filename.replace(/\.[^.]+$/, ''); +} + +function classifyMigrationHistoryStatus( + history: MigrationHistory, + fingerprint: PackageFingerprint, +): MigrationHistoryStatus { + if (!history.tableExists) return 'absent'; + if (!history.hasBaseline) return 'missing_baseline'; + const state = compareMigrationHistory(history, { + appliedMigrationCount: fingerprint.expectedMigrationCount, + latestMigrationName: fingerprint.expectedLatestMigrationName, + }); + if (state === 'matches') return 'current'; + return state === 'older' ? 'behind' : 'ahead'; +} + +function classifyStatus(args: { + appliedSdk: string; + appliedSha: string; + packageSdk: string; + packageSha: string; + appliedMigrations: MigrationSummary; + expectedMigrations: MigrationSummary; +}): MigrationStatus['status'] { + const migrationState = compareMigrationHistory( + args.appliedMigrations, + args.expectedMigrations, + ); + if (args.appliedSha === args.packageSha && migrationState === 'matches') { + return 'up_to_date'; + } + if (migrationState === 'newer') return 'newer_db'; + if (migrationState === 'older') return 'older_db'; + return compareSemver(args.appliedSdk, args.packageSdk) > 0 ? 'newer_db' : 'older_db'; +} + +function compareMigrationHistory( + applied: MigrationSummary & Partial>, + expected: MigrationSummary, +): 'matches' | 'older' | 'newer' { + if (applied.hasBaseline === false) return 'older'; + if ( + applied.appliedMigrationCount === expected.appliedMigrationCount + && applied.latestMigrationName === expected.latestMigrationName + ) { + return 'matches'; + } + if ( + applied.appliedMigrationCount > expected.appliedMigrationCount + || compareMigrationName(applied.latestMigrationName, expected.latestMigrationName) > 0 + ) { + return 'newer'; + } + return 'older'; +} + +function compareMigrationName(a: string, b: string): number { + return a.localeCompare(b, undefined, { + numeric: true, + sensitivity: 'variant', + ignorePunctuation: true, + }); +} + +/** + * Numeric semver comparison. Treats non-numeric / missing segments as 0. + * Used to classify older_db vs newer_db when migration history is current but + * schema SHAs differ; does not implement pre-release ordering. + */ +function compareSemver(a: string, b: string): number { + return compareNumberArrays(parseSemverParts(a), parseSemverParts(b)); +} + +function compareNumberArrays(a: number[], b: number[]): number { + const len = Math.max(a.length, b.length); + for (let i = 0; i < len; i += 1) { + const diff = numberAt(a, i) - numberAt(b, i); + if (diff !== 0) return diff; + } + return 0; +} + +function numberAt(arr: number[], i: number): number { + return arr[i] ?? 0; +} + +function parseSemverParts(version: string): number[] { + return version.split('.').map(parseNumericPart); +} + +function parseNumericPart(part: string): number { + const n = Number.parseInt(part, 10); + return Number.isFinite(n) ? n : 0; +} diff --git a/src/db/migration-version.ts b/src/db/migration-version.ts new file mode 100644 index 0000000..9470186 --- /dev/null +++ b/src/db/migration-version.ts @@ -0,0 +1,106 @@ +/** + * `schema_version` table operations for the Phase 2 migration runner. + * + * Owns reads (latest stamp), writes (insert new stamp), and the + * fresh-install detection probes (`tableExists`, + * `readLatestSchemaVersionOrAbsent`). Kept separate from the public migration + * API so the table contract is one cohesive surface that tests and the + * status query can pull from independently of `migrate()`. + */ + +import type { Pool, PoolClient } from 'pg'; + +/** + * Tables created by v1.0.x; used by `migrationStatus()` to distinguish + * "fresh database" from "DB exists but predates Phase 1 stamping". + */ +export const CORE_TABLE_PROBE = 'episodes'; + +/** Postgres SQLSTATE for `undefined_table`. */ +const UNDEFINED_TABLE_SQLSTATE = '42P01'; + +export interface SchemaVersionRow { + sdk_version: string; + schema_sha256: string; + applied_at: Date; + notes: string | null; +} + +async function ensureSchemaVersionTable(client: PoolClient): Promise { + await client.query(` + CREATE TABLE IF NOT EXISTS schema_version ( + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + sdk_version TEXT NOT NULL, + schema_sha256 TEXT NOT NULL, + notes TEXT, + PRIMARY KEY (applied_at) + )`); + await client.query(` + CREATE INDEX IF NOT EXISTS idx_schema_version_applied_at + ON schema_version (applied_at DESC)`); +} + +export async function tableExists( + client: Pick, + tableName: string, +): Promise { + const { rows } = await client.query<{ exists: boolean }>( + 'SELECT EXISTS (' + + " SELECT 1 FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid " + + " WHERE c.relname = $1 AND n.nspname = current_schema() AND c.relkind = 'r'" + + ') AS exists', + [tableName], + ); + return rows[0]?.exists === true; +} + +export async function readLatestSchemaVersion( + client: Pick, +): Promise { + const { rows } = await client.query( + 'SELECT sdk_version, schema_sha256, applied_at, notes FROM schema_version ' + + 'ORDER BY applied_at DESC LIMIT 1', + ); + return rows[0] ?? null; +} + +/** + * Variant of `readLatestSchemaVersion` that returns `null` instead of + * throwing when the `schema_version` table itself does not exist. Used at + * the pre-lock snapshot point and inside the lock so a fresh install does + * not require a separate `tableExists` round-trip. Any error other than + * `undefined_table` (SQLSTATE 42P01) is propagated. + */ +export async function readLatestSchemaVersionOrAbsent( + client: Pick, +): Promise { + try { + return await readLatestSchemaVersion(client); + } catch (err: unknown) { + if (isUndefinedTableError(err)) return null; + throw err; + } +} + +function isUndefinedTableError(err: unknown): boolean { + if (!err || typeof err !== 'object') return false; + const code = (err as { code?: unknown }).code; + return code === UNDEFINED_TABLE_SQLSTATE; +} + +export async function stampSchemaVersion( + client: PoolClient, + payload: { sdkVersion: string; schemaSha256: string; notes: string | null }, +): Promise { + await ensureSchemaVersionTable(client); + const { rows } = await client.query( + 'INSERT INTO schema_version (sdk_version, schema_sha256, notes) ' + + 'VALUES ($1, $2, $3) ' + + 'RETURNING sdk_version, schema_sha256, applied_at, notes', + [payload.sdkVersion, payload.schemaSha256, payload.notes], + ); + if (!rows[0]) { + throw new Error('[migration-version] INSERT INTO schema_version returned no rows'); + } + return rows[0]; +} diff --git a/src/db/migrations/0001_baseline.sql b/src/db/migrations/0001_baseline.sql new file mode 100644 index 0000000..4630438 --- /dev/null +++ b/src/db/migrations/0001_baseline.sql @@ -0,0 +1,1369 @@ +/** + * atomicmemory-core Schema — active memory projection plus contradiction-safe + * claim/version history. Idempotent: safe to re-run on every startup. + * + * IMPORTANT: This schema uses CREATE TABLE/INDEX IF NOT EXISTS so it can run + * on every app startup without data loss. Adding new columns to existing tables + * requires explicit ALTER TABLE ... ADD COLUMN IF NOT EXISTS statements — a + * plain column definition inside CREATE TABLE IF NOT EXISTS will be silently + * ignored if the table already exists. + */ + +CREATE EXTENSION IF NOT EXISTS vector; +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE IF NOT EXISTS episodes ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + content TEXT NOT NULL, + source_site TEXT NOT NULL, + source_url TEXT NOT NULL DEFAULT '', + session_id TEXT, + workspace_id UUID DEFAULT NULL, + agent_id UUID DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_episodes_user_site ON episodes (user_id, source_site); + +CREATE TABLE IF NOT EXISTS canonical_memory_objects ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + object_family TEXT NOT NULL + CHECK (object_family IN ('ingested_fact')), + payload_format TEXT NOT NULL DEFAULT 'json', + canonical_payload JSONB NOT NULL, + provenance JSONB NOT NULL DEFAULT '{}', + observed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + lineage JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_canonical_memory_objects_user_created + ON canonical_memory_objects (user_id, created_at DESC); + +CREATE TABLE IF NOT EXISTS memories ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + content TEXT NOT NULL, + embedding vector(768) NOT NULL, + memory_type TEXT NOT NULL DEFAULT 'semantic' + CHECK (memory_type IN ('episodic', 'semantic', 'procedural', 'composite')), + importance REAL NOT NULL DEFAULT 0.5 + CHECK (importance >= 0.0 AND importance <= 1.0), + source_site TEXT NOT NULL, + source_url TEXT NOT NULL DEFAULT '', + episode_id UUID, -- FK to episodes removed: non-transactional writes with pgvector can't guarantee ordering + status TEXT NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'needs_clarification')), + metadata JSONB DEFAULT '{}', + keywords TEXT NOT NULL DEFAULT '', + namespace TEXT DEFAULT NULL, + summary TEXT NOT NULL DEFAULT '', -- L0: abstract/headline (~100 tokens) + overview TEXT NOT NULL DEFAULT '', -- L1: condensed overview (~1000 tokens) + trust_score REAL NOT NULL DEFAULT 1.0 -- Phase 3: source + content trust (0.0–1.0) + CHECK (trust_score >= 0.0 AND trust_score <= 1.0), + observed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -- when the conversation actually happened (vs created_at = DB insertion time) + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_accessed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + access_count INTEGER NOT NULL DEFAULT 0, + expired_at TIMESTAMPTZ DEFAULT NULL, -- Phase 4: set when superseded (temporal invalidation) + deleted_at TIMESTAMPTZ DEFAULT NULL, + -- Phase 7: 4-network memory separation (Hindsight-inspired) + network TEXT NOT NULL DEFAULT 'experience' + CHECK (network IN ('world', 'experience', 'opinion', 'observation')), + opinion_confidence REAL DEFAULT NULL + CHECK (opinion_confidence IS NULL OR (opinion_confidence >= 0.0 AND opinion_confidence <= 1.0)), + observation_subject TEXT DEFAULT NULL, + -- Phase 8: deferred AUDN reconciliation + deferred_audn BOOLEAN NOT NULL DEFAULT false, + audn_candidates JSONB DEFAULT NULL, -- serialized candidates for background reconciliation + -- Phase 9: workspace / multi-agent scoping + workspace_id UUID DEFAULT NULL, + agent_id UUID DEFAULT NULL, + visibility TEXT DEFAULT NULL + CHECK (visibility IS NULL OR visibility IN ('agent_only', 'restricted', 'workspace')) +); + +CREATE INDEX IF NOT EXISTS idx_memories_deferred_audn ON memories (user_id) + WHERE deferred_audn = true AND deleted_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_user_site ON memories (user_id, source_site) + WHERE deleted_at IS NULL AND expired_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_memories_user_created ON memories (user_id, created_at) + WHERE deleted_at IS NULL AND expired_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_embedding ON memories + USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + +-- Full-text search: indexes both paraphrased content AND extracted keywords. +-- Keywords preserve proper nouns, dates, and project names that paraphrasing loses. +ALTER TABLE memories ADD COLUMN IF NOT EXISTS search_vector tsvector + GENERATED ALWAYS AS ( + to_tsvector('english', content) || to_tsvector('simple', keywords) + ) STORED; + +CREATE INDEX IF NOT EXISTS idx_memories_fts ON memories USING gin (search_vector) + WHERE deleted_at IS NULL AND expired_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_namespace ON memories (namespace) + WHERE deleted_at IS NULL AND expired_at IS NULL AND namespace IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_network ON memories (user_id, network) + WHERE deleted_at IS NULL AND expired_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_workspace ON memories (workspace_id, agent_id) + WHERE deleted_at IS NULL AND expired_at IS NULL AND workspace_id IS NOT NULL; + +-- Visibility grants for restricted memories (workspace scoping) +CREATE TABLE IF NOT EXISTS memory_visibility_grants ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + memory_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + grantee_agent_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (memory_id, grantee_agent_id) +); + +CREATE INDEX IF NOT EXISTS idx_visibility_grants_memory ON memory_visibility_grants (memory_id); +CREATE INDEX IF NOT EXISTS idx_visibility_grants_agent ON memory_visibility_grants (grantee_agent_id); + +CREATE INDEX IF NOT EXISTS idx_memories_observation_subject ON memories (user_id, observation_subject) + WHERE network = 'observation' AND deleted_at IS NULL AND expired_at IS NULL; + +-- Workspace columns added via ALTER TABLE at the bottom of this file (Phase 5 Step 9). +CREATE TABLE IF NOT EXISTS memory_atomic_facts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + parent_memory_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + fact_text TEXT NOT NULL, + embedding vector(768) NOT NULL, + fact_type TEXT NOT NULL DEFAULT 'knowledge' + CHECK (fact_type IN ('preference', 'project', 'knowledge', 'person', 'plan')), + importance REAL NOT NULL DEFAULT 0.5 + CHECK (importance >= 0.0 AND importance <= 1.0), + source_site TEXT NOT NULL, + source_url TEXT NOT NULL DEFAULT '', + episode_id UUID, + keywords TEXT NOT NULL DEFAULT '', + metadata JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_memory_atomic_facts_parent ON memory_atomic_facts (parent_memory_id); +CREATE INDEX IF NOT EXISTS idx_memory_atomic_facts_user ON memory_atomic_facts (user_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_memory_atomic_facts_embedding ON memory_atomic_facts + USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + +ALTER TABLE memory_atomic_facts ADD COLUMN IF NOT EXISTS search_vector tsvector + GENERATED ALWAYS AS ( + to_tsvector('english', fact_text) || to_tsvector('simple', keywords) + ) STORED; + +CREATE INDEX IF NOT EXISTS idx_memory_atomic_facts_fts ON memory_atomic_facts USING gin (search_vector); + +-- Workspace columns added via ALTER TABLE at the bottom of this file (Phase 5 Step 9). +CREATE TABLE IF NOT EXISTS memory_foresight ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + parent_memory_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + content TEXT NOT NULL, + embedding vector(768) NOT NULL, + foresight_type TEXT NOT NULL DEFAULT 'plan' + CHECK (foresight_type IN ('plan', 'goal', 'scheduled', 'expected_state')), + source_site TEXT NOT NULL, + source_url TEXT NOT NULL DEFAULT '', + episode_id UUID, + metadata JSONB NOT NULL DEFAULT '{}', + valid_from TIMESTAMPTZ NOT NULL DEFAULT NOW(), + valid_to TIMESTAMPTZ DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_memory_foresight_parent ON memory_foresight (parent_memory_id); +CREATE INDEX IF NOT EXISTS idx_memory_foresight_user_valid ON memory_foresight (user_id, valid_from, valid_to); +CREATE INDEX IF NOT EXISTS idx_memory_foresight_embedding ON memory_foresight + USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + +-- Observation regeneration trigger (async, decoupled from ingest) +CREATE TABLE IF NOT EXISTS observation_dirty ( + user_id TEXT NOT NULL, + subject TEXT NOT NULL, + marked_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (user_id, subject) +); + +-- SCOPE_TODO: Claims are intentionally user-scoped — AUDN contradiction resolution +-- is cross-workspace. Workspace-scoped claims are a Phase 8+ concern. +CREATE TABLE IF NOT EXISTS memory_claims ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + claim_type TEXT NOT NULL DEFAULT 'fact', + status TEXT NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'deleted')), + current_version_id UUID DEFAULT NULL, + slot_key TEXT DEFAULT NULL, + subject_entity_id UUID DEFAULT NULL, + relation_type TEXT DEFAULT NULL + CHECK (relation_type IS NULL OR relation_type IN ( + 'uses', 'works_on', 'works_at', 'located_in', 'knows', + 'prefers', 'created', 'belongs_to', 'studies', 'manages' + )), + object_entity_id UUID DEFAULT NULL, + valid_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + invalid_at TIMESTAMPTZ DEFAULT NULL, + invalidated_at TIMESTAMPTZ DEFAULT NULL, + invalidated_by_version_id UUID DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CHECK (invalid_at IS NULL OR invalid_at >= valid_at) +); + +CREATE INDEX IF NOT EXISTS idx_memory_claims_user ON memory_claims (user_id); +CREATE INDEX IF NOT EXISTS idx_memory_claims_user_valid + ON memory_claims (user_id, valid_at, invalid_at); +CREATE INDEX IF NOT EXISTS idx_memory_claims_user_slot + ON memory_claims (user_id, slot_key) + WHERE slot_key IS NOT NULL; + +-- SCOPE_TODO: Claim versions inherit user-scoped claim ownership — same rationale as memory_claims. +CREATE TABLE IF NOT EXISTS memory_claim_versions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + claim_id UUID NOT NULL REFERENCES memory_claims(id) ON DELETE CASCADE, + user_id TEXT NOT NULL, + memory_id UUID UNIQUE REFERENCES memories(id) ON DELETE SET NULL, + content TEXT NOT NULL, + embedding vector(768) NOT NULL, + importance REAL NOT NULL DEFAULT 0.5 + CHECK (importance >= 0.0 AND importance <= 1.0), + source_site TEXT NOT NULL, + source_url TEXT NOT NULL DEFAULT '', + episode_id UUID /* REFERENCES episodes(id) ON DELETE SET NULL -- removed for non-transactional pgvector compat */, + valid_from TIMESTAMPTZ NOT NULL DEFAULT NOW(), + valid_to TIMESTAMPTZ DEFAULT NULL, + superseded_by_version_id UUID DEFAULT NULL, + mutation_type TEXT DEFAULT NULL + CHECK (mutation_type IS NULL OR mutation_type IN ('add', 'update', 'supersede', 'delete', 'clarify')), + mutation_reason TEXT DEFAULT NULL, + previous_version_id UUID DEFAULT NULL, + actor_model TEXT DEFAULT NULL, + contradiction_confidence REAL DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_memory_claim_versions_claim ON memory_claim_versions (claim_id); +CREATE INDEX IF NOT EXISTS idx_memory_claim_versions_user_valid + ON memory_claim_versions (user_id, valid_from, valid_to); +CREATE INDEX IF NOT EXISTS idx_memory_claim_versions_claim_valid + ON memory_claim_versions (claim_id, valid_from, valid_to); + +CREATE INDEX IF NOT EXISTS idx_memory_claim_versions_embedding ON memory_claim_versions + USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + +CREATE TABLE IF NOT EXISTS memory_evidence ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + claim_version_id UUID NOT NULL + REFERENCES memory_claim_versions(id) ON DELETE CASCADE, + episode_id UUID /* REFERENCES episodes(id) ON DELETE SET NULL -- removed for non-transactional pgvector compat */, + memory_id UUID REFERENCES memories(id) ON DELETE SET NULL, + quote_text TEXT NOT NULL DEFAULT '', + speaker TEXT DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_memory_evidence_version ON memory_evidence (claim_version_id); + +-- Memory links for 1-hop link expansion (Phase 2, A-MEM style) +-- Bidirectional: stored as (source_id, target_id) where source_id < target_id +-- to avoid duplicate pairs. Query both directions at read time. +CREATE TABLE IF NOT EXISTS memory_links ( + source_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + target_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + similarity REAL NOT NULL CHECK (similarity >= 0.0 AND similarity <= 1.0), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (source_id, target_id) +); + +CREATE INDEX IF NOT EXISTS idx_memory_links_target ON memory_links (target_id); + +-- Phase 5: Entity graph — structured entities extracted from memories +-- SCOPE_TODO: Entities are user-global (entity dedup crosses workspace boundaries). +CREATE TABLE IF NOT EXISTS entities ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + name TEXT NOT NULL, + normalized_name TEXT NOT NULL, + entity_type TEXT NOT NULL + CHECK (entity_type IN ('person', 'tool', 'project', 'organization', 'place', 'concept')), + embedding vector(768) NOT NULL, + alias_names TEXT[] NOT NULL DEFAULT '{}', + normalized_alias_names TEXT[] NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_entities_user ON entities (user_id); +CREATE INDEX IF NOT EXISTS idx_entities_user_type ON entities (user_id, entity_type); +CREATE INDEX IF NOT EXISTS idx_entities_user_normalized + ON entities (user_id, entity_type, normalized_name); +CREATE INDEX IF NOT EXISTS idx_entities_embedding ON entities + USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + +-- Junction table: many memories ↔ many entities +CREATE TABLE IF NOT EXISTS memory_entities ( + memory_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + entity_id UUID NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (memory_id, entity_id) +); + +CREATE INDEX IF NOT EXISTS idx_memory_entities_entity ON memory_entities (entity_id); + +-- Phase 4: Temporal Linkage List (TLL). +-- Per-entity sparse graph of event nodes with predecessor/successor edges. +-- Karpathy-minimal: append on ingest, traverse on EO/MSR/TR queries. +-- Targets the abilities Mem0 explicitly admits their architecture doesn't +-- crack at 10M (temporal reasoning, event ordering, multi-session reasoning). +-- The unique architectural primitive nobody has shipped publicly. +CREATE TABLE IF NOT EXISTS temporal_linkage_list ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + entity_id UUID NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + memory_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + predecessor_memory_id UUID DEFAULT NULL REFERENCES memories(id) ON DELETE CASCADE, + observation_date TIMESTAMPTZ NOT NULL, + position_in_chain INTEGER NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (user_id, entity_id, memory_id) +); + +CREATE INDEX IF NOT EXISTS idx_tll_entity_chain + ON temporal_linkage_list (user_id, entity_id, position_in_chain); +CREATE INDEX IF NOT EXISTS idx_tll_memory + ON temporal_linkage_list (memory_id); + +-- Defense-in-depth: unique (chain, position) so any future code path that +-- bypasses the advisory-lock append fails at the DB layer instead of +-- silently producing duplicate positions. Idempotent for fresh and +-- existing schemas. +CREATE UNIQUE INDEX IF NOT EXISTS idx_tll_chain_position_unique + ON temporal_linkage_list (user_id, entity_id, position_in_chain); + +-- Align predecessor FK with memory FK (CASCADE) so a hard-deleted memory +-- removes the dependent chain node instead of leaving a half-broken +-- predecessor pointer that breaks backward chain traversal. Idempotent: +-- re-applying the constraint overwrites any prior ON DELETE SET NULL +-- definition. Required for existing databases since the table-level +-- CREATE TABLE IF NOT EXISTS above does not update column constraints. +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.table_constraints + WHERE table_name = 'temporal_linkage_list' + AND constraint_name = 'temporal_linkage_list_predecessor_memory_id_fkey' + ) THEN + ALTER TABLE temporal_linkage_list + DROP CONSTRAINT temporal_linkage_list_predecessor_memory_id_fkey; + END IF; + ALTER TABLE temporal_linkage_list + ADD CONSTRAINT temporal_linkage_list_predecessor_memory_id_fkey + FOREIGN KEY (predecessor_memory_id) REFERENCES memories(id) + ON DELETE CASCADE; +END$$; + +-- ===================================================================== +-- First-mention events (chronological topic-introduction list) +-- ===================================================================== +-- Per-user list of "the first time topic X was introduced in conversation." +-- Distinct from facts (which are atomic claims) and memories (which are +-- ingested chunks). The grain matches event-ordering rubrics: +-- "in what order did the user bring up these aspects." +-- +-- Generated post-ingest by FirstMentionService via a single LLM call that +-- scans the full conversation and outputs a JSON array of first-mention +-- events. Idempotent on (user_id, memory_id) so re-running doesn't duplicate. +CREATE TABLE IF NOT EXISTS first_mention_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + topic TEXT NOT NULL, + turn_id INTEGER NOT NULL, + memory_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + anchor_date TIMESTAMPTZ DEFAULT NULL, + position_in_conversation INTEGER NOT NULL, + source_site TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (user_id, memory_id) +); + +CREATE INDEX IF NOT EXISTS idx_first_mention_user_position + ON first_mention_events (user_id, position_in_conversation); + +CREATE INDEX IF NOT EXISTS idx_first_mention_user_topic + ON first_mention_events USING GIN (to_tsvector('english', topic)); + +-- Entity relations: typed, directed edges between entities with temporal validity +CREATE TABLE IF NOT EXISTS entity_relations ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + source_entity_id UUID NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + target_entity_id UUID NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + relation_type TEXT NOT NULL + CHECK (relation_type IN ( + 'uses', 'works_on', 'works_at', 'located_in', 'knows', + 'prefers', 'created', 'belongs_to', 'studies', 'manages' + )), + source_memory_id UUID REFERENCES memories(id) ON DELETE SET NULL, + confidence REAL NOT NULL DEFAULT 1.0 + CHECK (confidence >= 0.0 AND confidence <= 1.0), + valid_from TIMESTAMPTZ NOT NULL DEFAULT NOW(), + valid_to TIMESTAMPTZ DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (source_entity_id, target_entity_id, relation_type) +); + +CREATE INDEX IF NOT EXISTS idx_entity_relations_source ON entity_relations (source_entity_id); +CREATE INDEX IF NOT EXISTS idx_entity_relations_target ON entity_relations (target_entity_id); +CREATE INDEX IF NOT EXISTS idx_entity_relations_user ON entity_relations (user_id); + +-- Phase 6: Lessons store — detected failure patterns for pre-action defense (A-MemGuard) +-- SCOPE_TODO: Lessons are user-global (failure patterns are personal, not per-workspace). +CREATE TABLE IF NOT EXISTS lessons ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + lesson_type TEXT NOT NULL + CHECK (lesson_type IN ( + 'injection_blocked', 'false_memory', 'contradiction_pattern', + 'user_reported', 'consensus_violation', 'trust_violation' + )), + pattern TEXT NOT NULL, + embedding vector(768) NOT NULL, + source_memory_ids UUID[] NOT NULL DEFAULT '{}', + source_query TEXT DEFAULT NULL, + severity TEXT NOT NULL DEFAULT 'medium' + CHECK (severity IN ('low', 'medium', 'high', 'critical')), + active BOOLEAN NOT NULL DEFAULT true, + metadata JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_lessons_user_active ON lessons (user_id) WHERE active = true; +CREATE INDEX IF NOT EXISTS idx_lessons_type ON lessons (user_id, lesson_type); +CREATE INDEX IF NOT EXISTS idx_lessons_embedding ON lessons + USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + +-- Temporal metadata index (observed_at separates conversation time from DB insertion time) +CREATE INDEX IF NOT EXISTS idx_memories_user_observed ON memories (user_id, observed_at) + WHERE deleted_at IS NULL AND expired_at IS NULL; + +-- Agent trust levels for multi-agent conflict resolution (from hive-mind Phase 4) +CREATE TABLE IF NOT EXISTS agent_trust ( + agent_id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + trust_level REAL NOT NULL DEFAULT 0.5 + CHECK (trust_level >= 0.0 AND trust_level <= 1.0), + display_name TEXT DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_agent_trust_user ON agent_trust (user_id); + +-- Conflict tracking for CLARIFY escalation and auto-resolution +CREATE TABLE IF NOT EXISTS memory_conflicts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + new_memory_id UUID REFERENCES memories(id) ON DELETE SET NULL, + existing_memory_id UUID REFERENCES memories(id) ON DELETE SET NULL, + new_agent_id TEXT DEFAULT NULL, + existing_agent_id TEXT DEFAULT NULL, + new_trust_level REAL DEFAULT NULL, + existing_trust_level REAL DEFAULT NULL, + contradiction_confidence REAL NOT NULL DEFAULT 0.5, + clarification_note TEXT DEFAULT NULL, + status TEXT NOT NULL DEFAULT 'open' + CHECK (status IN ('open', 'auto_resolved', 'resolved_new', 'resolved_existing', 'resolved_both')), + resolution_policy TEXT DEFAULT NULL, + resolved_at TIMESTAMPTZ DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + auto_resolve_after TIMESTAMPTZ DEFAULT NULL +); + +CREATE INDEX IF NOT EXISTS idx_conflicts_user_status ON memory_conflicts (user_id, status) + WHERE status = 'open'; +CREATE INDEX IF NOT EXISTS idx_conflicts_auto_resolve ON memory_conflicts (auto_resolve_after) + WHERE status = 'open' AND auto_resolve_after IS NOT NULL; + +-- --------------------------------------------------------------------------- +-- Phase 5 Step 9: Add workspace scope columns to representation tables. +-- These are idempotent ALTER TABLE statements that run safely on every startup. +-- NULL means the row was created by user-scoped ingest (pre-Phase 5). +-- --------------------------------------------------------------------------- + +ALTER TABLE memory_atomic_facts ADD COLUMN IF NOT EXISTS workspace_id UUID DEFAULT NULL; +ALTER TABLE memory_atomic_facts ADD COLUMN IF NOT EXISTS agent_id UUID DEFAULT NULL; + +ALTER TABLE memory_foresight ADD COLUMN IF NOT EXISTS workspace_id UUID DEFAULT NULL; +ALTER TABLE memory_foresight ADD COLUMN IF NOT EXISTS agent_id UUID DEFAULT NULL; + +CREATE INDEX IF NOT EXISTS idx_memory_atomic_facts_workspace + ON memory_atomic_facts (workspace_id) WHERE workspace_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_memory_foresight_workspace + ON memory_foresight (workspace_id) WHERE workspace_id IS NOT NULL; + +-- --------------------------------------------------------------------------- +-- TBC Phase 3 (2026-05-06): Typed Belief Calculus first-class storage. +-- Promotes belief state from `memories.metadata` JSONB into typed columns + +-- a new `belief_edges` table. All additions are idempotent (IF NOT EXISTS). +-- Pre-migration databases stay queryable; tbc-execution.ts dual-writes +-- during the migration window. +-- Activated only when TBC_ENABLED=true; defaults preserve existing behavior. +-- --------------------------------------------------------------------------- + +-- Belief confidence in [0,1]; default 1.0 = "fully believed" (matches AUDN's +-- no-confidence-tracking baseline). +ALTER TABLE memories ADD COLUMN IF NOT EXISTS confidence REAL DEFAULT 1.0 + CHECK (confidence >= 0.0 AND confidence <= 1.0); + +-- Belief tier — controls how the claim influences answer generation. +-- standard: default tier; normal weight in retrieval +-- directive: promoted; injected as a "must follow" rule in answer prompt +-- demoted: challenged; lower weight + flagged for re-evaluation +-- retracted: believed false; excluded from default retrieval +ALTER TABLE memories ADD COLUMN IF NOT EXISTS belief_tier TEXT DEFAULT 'standard' + CHECK (belief_tier IN ('standard', 'directive', 'demoted', 'retracted')); + +-- The TBC operator that most recently mutated this memory. +ALTER TABLE memories ADD COLUMN IF NOT EXISTS mutation_type TEXT DEFAULT NULL + CHECK (mutation_type IS NULL OR mutation_type IN ( + 'AFFIRM', 'UPDATE', 'RETRACT', 'SUPERSEDE', + 'PROMOTE', 'DEMOTE', 'EVIDENCE_FOR', 'COUNTER' + )); + +-- Tier-aware retrieval index (directives surface fast, retracted excluded). +CREATE INDEX IF NOT EXISTS idx_memories_belief_tier + ON memories (user_id, belief_tier) + WHERE deleted_at IS NULL AND expired_at IS NULL AND belief_tier != 'standard'; + +-- Confidence-weighted retrieval index (low-confidence demotion). +CREATE INDEX IF NOT EXISTS idx_memories_confidence + ON memories (user_id, confidence DESC) + WHERE deleted_at IS NULL AND expired_at IS NULL; + +-- belief_edges: typed belief graph between claims. +-- evidence_for: source supports target's confidence (positive weight) +-- counter: source contradicts target's confidence (negative weight) +-- supersedes: source replaces target (more specific or general) +-- promotes: source promoted target to directive tier +-- demotes: source challenged target without retracting +CREATE TABLE IF NOT EXISTS belief_edges ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + source_id UUID NOT NULL, + target_id UUID NOT NULL, + edge_type TEXT NOT NULL CHECK (edge_type IN ( + 'evidence_for', 'counter', 'supersedes', 'promotes', 'demotes' + )), + weight REAL NOT NULL DEFAULT 0.0 + CHECK (weight >= -1.0 AND weight <= 1.0), + rationale TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + workspace_id UUID DEFAULT NULL, + agent_id UUID DEFAULT NULL +); + +-- For "all evidence pointing at this claim" queries (queryable belief state). +CREATE INDEX IF NOT EXISTS idx_belief_edges_target + ON belief_edges (target_id, edge_type, created_at DESC); + +-- For "all claims this evidence supports/counters" queries. +CREATE INDEX IF NOT EXISTS idx_belief_edges_source + ON belief_edges (source_id, edge_type); + +-- User-scoped traversal (multi-tenant safety). +CREATE INDEX IF NOT EXISTS idx_belief_edges_user_target + ON belief_edges (user_id, target_id); + +-- --------------------------------------------------------------------------- +-- Hierarchical Retrieval (2026-05-07): three-level memory hierarchy for +-- BEAM-10M scale (10 conversations × ~1.4M tokens each = ~14M total context). +-- session_summaries + conv_summaries indexed via HNSW on summary_embedding. +-- Activated only when HIERARCHICAL_RETRIEVAL_ENABLED=true; defaults preserve +-- existing flat-retrieval behavior. +-- --------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS session_summaries ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + session_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + session_index INTEGER NOT NULL, + summary_text TEXT NOT NULL, + summary_embedding vector(768) NOT NULL, + topics TEXT[] NOT NULL DEFAULT '{}', + fact_count INTEGER NOT NULL DEFAULT 0, + occurred_start TIMESTAMPTZ DEFAULT NULL, + occurred_end TIMESTAMPTZ DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + workspace_id UUID DEFAULT NULL, + agent_id UUID DEFAULT NULL +); + +CREATE TABLE IF NOT EXISTS conv_summaries ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + summary_text TEXT NOT NULL, + summary_embedding vector(768) NOT NULL, + session_count INTEGER NOT NULL DEFAULT 0, + fact_count INTEGER NOT NULL DEFAULT 0, + occurred_start TIMESTAMPTZ DEFAULT NULL, + occurred_end TIMESTAMPTZ DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + workspace_id UUID DEFAULT NULL, + agent_id UUID DEFAULT NULL +); + +-- Stage-1 retrieval: top-K conversations by summary similarity. +CREATE INDEX IF NOT EXISTS idx_conv_summaries_embedding + ON conv_summaries USING hnsw (summary_embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + +-- Stage-2 retrieval: top-K sessions within selected conversations. +CREATE INDEX IF NOT EXISTS idx_session_summaries_embedding + ON session_summaries USING hnsw (summary_embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + +-- User-scoped lookups for both summary tables. +CREATE INDEX IF NOT EXISTS idx_session_summaries_user_conv + ON session_summaries (user_id, conversation_id, session_index); + +CREATE INDEX IF NOT EXISTS idx_conv_summaries_user + ON conv_summaries (user_id, conversation_id); + +-- --------------------------------------------------------------------------- +-- Sprint 3 (2026-05-10): Topic-abstraction layer for the EO experiment. +-- Per-memory conceptual topic (3-7 word LLM summary at higher abstraction +-- than raw facts) + its embedding. Surfaced via a dedicated RRF channel at +-- retrieval. Activated only when TOPIC_ABSTRACTION_ENABLED=true; defaults +-- preserve existing behavior on un-migrated rows. Design doc: +-- benchmarks-sprint3/2026-05-10-am-baseline-and-rerank-design.md. +-- --------------------------------------------------------------------------- +ALTER TABLE memories ADD COLUMN IF NOT EXISTS topic_abstraction TEXT NOT NULL DEFAULT ''; +ALTER TABLE memories ADD COLUMN IF NOT EXISTS topic_embedding vector(768) DEFAULT NULL; +-- Pointer to the recap this memory has been consolidated into (NULL until +-- the Recap-layer builder runs). Used to filter out already-consolidated +-- memories from future recap-cluster candidates. +ALTER TABLE memories ADD COLUMN IF NOT EXISTS recap_id UUID DEFAULT NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_topic_embedding + ON memories USING hnsw (topic_embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200) + WHERE topic_embedding IS NOT NULL AND deleted_at IS NULL AND expired_at IS NULL; + +-- --------------------------------------------------------------------------- +-- Sprint 3 (2026-05-10): Recap layer for cross-session synthesis. +-- A Recap is an LLM-synthesized narrative aggregating N memories that share +-- a conceptual topic. Surfaced via its own RRF channel at retrieval. Cog-sci +-- analogue: hippocampal consolidation. Three of the four next-gen memory +-- systems converge on this primitive (Hindsight observations, Honcho +-- dreaming, X-Mem Episodes, EverMemOS multi-pass restructuring). Activated +-- only when RECAP_LAYER_ENABLED=true. +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS recaps ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + recap_text TEXT NOT NULL, + recap_embedding vector(768) NOT NULL, + topic TEXT NOT NULL DEFAULT '', + member_memory_ids UUID[] NOT NULL DEFAULT '{}', + member_count INTEGER NOT NULL DEFAULT 0, + time_range_start TIMESTAMPTZ DEFAULT NULL, + time_range_end TIMESTAMPTZ DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + workspace_id UUID DEFAULT NULL +); + +CREATE INDEX IF NOT EXISTS idx_recaps_user_topic + ON recaps (user_id, topic) WHERE workspace_id IS NULL; + +CREATE INDEX IF NOT EXISTS idx_recaps_embedding + ON recaps USING hnsw (recap_embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + +-- --------------------------------------------------------------------------- +-- Sprint 3 v1.5 (2026-05-11): user-profile channel (H2 from haiku-080). +-- One row per user holds the synthesized profile that is pinned to every +-- answer prompt. Updated by user-profile-builder.ts after each ingest +-- that stores >= 3 new facts. See also: +-- docs/db/changelog/20260511_user_profiles.sql (provenance copy) +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS user_profiles ( + user_id TEXT PRIMARY KEY, + profile_text TEXT NOT NULL, + source_memory_ids TEXT[] NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS user_profiles_updated_at_idx + ON user_profiles (updated_at DESC); + +-- --------------------------------------------------------------------------- +-- Sprint 4 (2026-05-11): Entity-Attribute Index (EAI). +-- Stores (entity, attribute, value) triples extracted at ingest time, indexed +-- for fast lookup by entity name and/or attribute key on queries like +-- "how many X did I do?" or "what is my Y?". Distinct from the entity graph +-- (entities, entity_relations) which captures structural relations. +-- See also: docs/db/changelog/20260511_entity_attributes.sql. +-- Activated only when ENTITY_ATTRIBUTES_ENABLED=true. +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS entity_attributes ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + entity_name TEXT NOT NULL, + attribute_key TEXT NOT NULL, + attribute_value TEXT NOT NULL, + value_type TEXT NOT NULL CHECK (value_type IN ('number','string','list','boolean','date')), + source_memory_id UUID, + observed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_entity_attributes_user_entity + ON entity_attributes (user_id, lower(entity_name)); +CREATE INDEX IF NOT EXISTS idx_entity_attributes_user_attribute + ON entity_attributes (user_id, lower(attribute_key)); +CREATE INDEX IF NOT EXISTS idx_entity_attributes_observed + ON entity_attributes (user_id, observed_at DESC); + +-- --------------------------------------------------------------------------- +-- BEAM-0.85 Phase 2 (2026-05-12): Literal value extraction for IE/KU. +-- Captures exact (entity, attribute, value) triples from ingested facts so +-- specialist lookup can answer literal factual questions via SQL. +-- See also: docs/db/changelog/20260512_entity_values.sql. +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS entity_values ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + entity TEXT NOT NULL, + attribute TEXT NOT NULL, + value TEXT NOT NULL, + value_type TEXT NOT NULL CHECK (value_type IN ('date', 'number', 'string', 'duration', 'list')), + observed_at TIMESTAMPTZ NOT NULL, + fact_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS ix_entity_values_lookup + ON entity_values (user_id, lower(entity), lower(attribute), observed_at DESC); + +CREATE INDEX IF NOT EXISTS ix_entity_values_fact + ON entity_values (fact_id); + +-- --------------------------------------------------------------------------- +-- BEAM-0.85 (2026-05-12): Async Reflect step storage. +-- Stores synthesized observations per (user_id, conversation_id), plus the +-- Postgres-backed queue used by the reflect worker. +-- See also: docs/db/changelog/20260512_session_reflections.sql. +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS session_reflections ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + observation TEXT NOT NULL, + observation_type TEXT NOT NULL CHECK (observation_type IN ( + 'entity_state', 'event_summary', 'preference', + 'contradiction', 'decision', 'numeric_value' + )), + evidence_memory_ids TEXT[] NOT NULL, + embedding vector(768), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS ix_session_reflections_user_conv + ON session_reflections (user_id, conversation_id); + +CREATE INDEX IF NOT EXISTS ix_session_reflections_embedding + ON session_reflections USING hnsw (embedding vector_cosine_ops); + +CREATE TABLE IF NOT EXISTS reflection_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'in_progress', 'completed', 'failed')), + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_tried_at TIMESTAMPTZ +); + +CREATE UNIQUE INDEX IF NOT EXISTS ix_reflection_jobs_pending_unique + ON reflection_jobs (user_id, conversation_id) + WHERE status IN ('pending', 'in_progress'); + +CREATE INDEX IF NOT EXISTS ix_reflection_jobs_status_created + ON reflection_jobs (status, created_at); + +-- --------------------------------------------------------------------------- +-- BEAM-0.85 (2026-05-12): Always-on per-entity ENTITY_CARD channel. +-- Mirrors Honcho's "peer card" pattern. The Reflect worker (Sonnet 4.5) +-- maintains one card per (user_id, conversation_id, entity_name); the +-- search pipeline injects all cards for the active conversation at the top +-- of every answer-LLM prompt under the `## ENTITY_STATE` heading. +-- See also: docs/db/changelog/20260512_entity_cards.sql. +-- Activated only when ENTITY_CARD_ENABLED=true. +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS entity_cards ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + entity_name TEXT NOT NULL, + card_text TEXT NOT NULL, + source_observation_ids TEXT[] NOT NULL DEFAULT '{}', + version INTEGER NOT NULL DEFAULT 1, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX IF NOT EXISTS ix_entity_cards_unique + ON entity_cards (user_id, conversation_id, entity_name); +CREATE INDEX IF NOT EXISTS ix_entity_cards_user_conv + ON entity_cards (user_id, conversation_id); + +-- --------------------------------------------------------------------------- +-- BEAM CR fix (2026-05-12): AUDN bilateral preservation for contradictions. +-- When the flag-gated bilateral path fires instead of DELETE/SUPERSEDE, +-- both prior + new memory remain in `memories` with `contradiction_active=true` +-- and `contradicts_memory_id` pointing at the counterpart. A row in +-- `memory_contradictions` records the conflict with both summaries verbatim +-- so retrieval can quote BOTH sides for CR-style questions. +-- See also: docs/db/changelog/20260512_audn_bilateral.sql. +-- Activated only when CONTRADICTION_PRESERVATION_ENABLED=true. +-- --------------------------------------------------------------------------- +ALTER TABLE memories + ADD COLUMN IF NOT EXISTS contradicts_memory_id UUID REFERENCES memories(id) ON DELETE SET NULL, + ADD COLUMN IF NOT EXISTS contradiction_active BOOLEAN NOT NULL DEFAULT false; + +CREATE INDEX IF NOT EXISTS ix_memories_contradiction_active + ON memories (user_id, contradiction_active) WHERE contradiction_active = true; + +CREATE TABLE IF NOT EXISTS memory_contradictions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + conversation_id TEXT, + left_memory_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + right_memory_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + left_summary TEXT NOT NULL, + right_summary TEXT NOT NULL, + resolved BOOLEAN NOT NULL DEFAULT false, + resolution_note TEXT, + detected_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS ix_memory_contradictions_user + ON memory_contradictions (user_id, conversation_id); +CREATE INDEX IF NOT EXISTS ix_memory_contradictions_left + ON memory_contradictions (left_memory_id); +CREATE INDEX IF NOT EXISTS ix_memory_contradictions_right + ON memory_contradictions (right_memory_id); + +-- --------------------------------------------------------------------------- +-- BEAM v38 (2026-05-12): Temporal state layer — focused Mem0 +-- temporal-reasoning subset for KU lift. +-- +-- Adds three columns on `memories` describing an evolving fact: +-- state_key stable identifier for an evolving fact ("user:1:location") +-- event_start when the fact became true +-- event_end when the fact stopped being true (NULL = still active) +-- +-- See also: docs/db/changelog/20260512_temporal_state.sql. +-- Activated only when TEMPORAL_STATE_ENABLED=true. +-- --------------------------------------------------------------------------- +ALTER TABLE memories ADD COLUMN IF NOT EXISTS state_key TEXT DEFAULT NULL; +ALTER TABLE memories ADD COLUMN IF NOT EXISTS event_start TIMESTAMPTZ DEFAULT NULL; +ALTER TABLE memories ADD COLUMN IF NOT EXISTS event_end TIMESTAMPTZ DEFAULT NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_state_key_active + ON memories (user_id, state_key) + WHERE event_end IS NULL + AND state_key IS NOT NULL + AND deleted_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_state_key_all + ON memories (user_id, state_key) + WHERE state_key IS NOT NULL + AND deleted_at IS NULL; +-- Document pipeline (Phase 1 of the large-file ingestion plan). +-- +-- See `Atomicmemory-research/docs/core-repo/design/large-file-ingestion-and-raw-storage-plan-2026-05-08.md`. +-- +-- Phase 1 ships the pointer-only document registry: `raw_sources` is a +-- per-(user, source_site, provider, account) namespace; `raw_documents` +-- represents one registered upstream object. The CHECK constraints on +-- `storage_mode` / `registration_status` / `raw_storage_status` accept the +-- full enum that later phases will populate (managed_blob, inline_text_stored, +-- blob_stored) so Phase 3 doesn't need a CHECK migration. The service layer +-- enforces `storage_mode = 'pointer_only'` until Phase 3 lands. +-- --------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS raw_sources ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + source_site TEXT NOT NULL, + provider TEXT NOT NULL, + account_id TEXT, + storage_mode TEXT NOT NULL DEFAULT 'pointer_only' + CHECK (storage_mode IN ('pointer_only', 'managed_blob', 'inline_small_text')), + retention_policy JSONB NOT NULL DEFAULT '{}'::jsonb, + consent_policy JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- COALESCE(account_id, '') keeps NULL account_ids in a single namespace slot +-- (Postgres treats NULLs as distinct in plain unique indexes, which would let +-- two rows with the same user/source/provider but different consent contexts +-- collide). +CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_sources_namespace + ON raw_sources (user_id, source_site, provider, COALESCE(account_id, '')); +CREATE INDEX IF NOT EXISTS idx_raw_sources_user + ON raw_sources (user_id); + +CREATE TABLE IF NOT EXISTS raw_documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + raw_source_id UUID NOT NULL REFERENCES raw_sources(id) ON DELETE CASCADE, + external_id TEXT NOT NULL, + external_uri TEXT, + display_name TEXT, + mime_type TEXT, + size_bytes BIGINT, + content_hash TEXT, + provider_version TEXT, + source_modified_at TIMESTAMPTZ, + storage_mode TEXT NOT NULL DEFAULT 'pointer_only' + CHECK (storage_mode IN ('pointer_only', 'managed_blob', 'inline_small_text')), + -- storage_uri / storage_provider stay NULL in Phase 1 (no managed blob). + storage_uri TEXT, + storage_provider TEXT, + registration_status TEXT NOT NULL DEFAULT 'registered' + CHECK (registration_status IN ('registered', 'registration_failed')), + raw_storage_status TEXT NOT NULL DEFAULT 'pointer_recorded' + CHECK (raw_storage_status IN + ('pointer_recorded', 'blob_stored', 'inline_text_stored', 'raw_storage_failed', 'blob_deleted')), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deleted_at TIMESTAMPTZ +); + +-- Phase 3 broadened the `raw_storage_status` CHECK to include +-- `blob_deleted` (terminal post-cleanup state for tombstoned managed-blob +-- rows). The Filecoin raw-content-store lifecycle refactor (Slice 2) +-- further broadens it to include the eventual-provider states: +-- * `blob_pending` — provider accepted the upload but storage / +-- retrievability is not yet confirmed (e.g. a Filecoin onramp +-- that returns before the deal is sealed). Slice 3 writes this +-- when `put()` returns `status: 'pending'`. +-- * `blob_available` — schema-reserved for the Phase 3 reconciliation +-- worker that promotes `blob_pending` rows once `head()` confirms +-- retrievability. No Phase-1 writer. +-- * `blob_archival_failed` — schema-reserved for the Phase 3 +-- reconciler's permanent-failure path. No Phase-1 writer. +-- * `blob_tombstoned` — schema-reserved for Phase 2 Filecoin +-- deletes when the provider supports unpin-only semantics. No +-- Phase-1 writer. +-- * `blob_uploading` (Phase 5) — transient state during the upload +-- pipeline's α/β/β2/γ split. Phase α writes this with a claim_id +-- after seizing the slot; Phase γ flips it to the final terminal +-- state after the adapter returns. A row that stays in +-- `blob_uploading` past a process restart is recoverable via +-- same-bytes idempotent retry of `uploadRaw` — the reconciler +-- does NOT process `blob_uploading` rows. +-- Idempotent ALTER so the new values are accepted on existing test +-- DBs whose CREATE TABLE IF NOT EXISTS already locked in the prior CHECK. +ALTER TABLE raw_documents + DROP CONSTRAINT IF EXISTS raw_documents_raw_storage_status_check; +ALTER TABLE raw_documents + ADD CONSTRAINT raw_documents_raw_storage_status_check + CHECK (raw_storage_status IN + ('pointer_recorded', 'blob_stored', 'inline_text_stored', 'raw_storage_failed', 'blob_deleted', + 'blob_pending', 'blob_available', 'blob_archival_failed', 'blob_tombstoned', + 'blob_uploading')); + +-- Filecoin lifecycle refactor (Phase 5) — typed claim + scheduling +-- columns moved out of JSONB. The upload pipeline's α/β/β2/γ split +-- writes a per-row claim_id when seizing the slot; the Phase 6 +-- reconciler claims `blob_pending` rows on the same columns and +-- advances `next_check_at` via exponential backoff. `pending_since` +-- is the durable "row entered blob_pending at" timestamp the +-- observability layer reads for the `pending_age_seconds` metric. +ALTER TABLE raw_documents + ADD COLUMN IF NOT EXISTS raw_storage_claim_id TEXT, + ADD COLUMN IF NOT EXISTS raw_storage_claimed_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS raw_storage_last_checked_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS raw_storage_next_check_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS raw_storage_reconcile_attempts INTEGER NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS raw_storage_pending_since TIMESTAMPTZ; + +-- Provider-side metadata for the managed blob (CID, piece CID, deal +-- id, onramp request id, gateway URL, etc.). Opaque to the upload +-- pipeline; the adapter's `put()` returns the shape and the row +-- formatter forwards it verbatim. Default `'{}'` so existing rows +-- and the pointer-only path stay schema-clean. +ALTER TABLE raw_documents + ADD COLUMN IF NOT EXISTS raw_storage_metadata JSONB NOT NULL DEFAULT '{}'::jsonb; + +-- Phase 2 indexing fingerprint. Distinct from `content_hash` (which is +-- the upstream/provider-supplied raw-content hash) so that indexing +-- never overwrites caller-provided metadata. NULL means "not yet +-- indexed under the current chunker_version"; the indexer's idempotency +-- check compares the input text's hash against this column. +ALTER TABLE raw_documents ADD COLUMN IF NOT EXISTS indexed_content_hash TEXT; +ALTER TABLE raw_documents ADD COLUMN IF NOT EXISTS indexed_at TIMESTAMPTZ; + +-- Phase B (document-ingest hardening) — per-layer status + last_error. +-- +-- The audit at `Atomicmemory-research/docs/core-repo/design/document-ingest-audit.md` +-- and the rev-18 hardening plan call for durable, observable +-- per-layer status so the UI/API stops pretending indexing is +-- instant and partial failures stop creating silent orphans. +-- +-- * extraction_status — text-extraction layer: 'not_required' +-- (e.g. URL pointer with no body), 'pending' (registered, awaiting +-- extraction), 'running' (in-flight), 'complete', 'unsupported' +-- (`.parquet`, etc.), 'failed'. +-- * semantic_index_status — chunk + embed + index pipeline: +-- 'not_required', 'pending', 'running', 'complete', 'failed', +-- 'stale' (re-index needed; reserved). +-- * last_error — JSONB envelope `{ layer, code, message, occurred_at }` +-- scoped to the most-recent failure for any layer; cleared on the +-- next successful transition for that layer. +-- +-- Cross-walk to spec naming (`Atomicmemory-research/docs/core-repo/design/ingestion-variations-supported-2026-05-09.md`): +-- `raw_storage_status` retains its prior values +-- (`pointer_recorded` / `blob_stored` / `inline_text_stored` / +-- `raw_storage_failed` / `blob_deleted`); rename can land later if +-- the migration is worth the churn. +ALTER TABLE raw_documents + ADD COLUMN IF NOT EXISTS extraction_status TEXT NOT NULL DEFAULT 'not_required'; +ALTER TABLE raw_documents + ADD COLUMN IF NOT EXISTS semantic_index_status TEXT NOT NULL DEFAULT 'not_required'; +ALTER TABLE raw_documents ADD COLUMN IF NOT EXISTS last_error JSONB; + +ALTER TABLE raw_documents + DROP CONSTRAINT IF EXISTS raw_documents_extraction_status_check; +ALTER TABLE raw_documents + ADD CONSTRAINT raw_documents_extraction_status_check + CHECK (extraction_status IN + ('not_required', 'pending', 'running', 'complete', 'unsupported', 'failed')); + +ALTER TABLE raw_documents + DROP CONSTRAINT IF EXISTS raw_documents_semantic_index_status_check; +ALTER TABLE raw_documents + ADD CONSTRAINT raw_documents_semantic_index_status_check + CHECK (semantic_index_status IN + ('not_required', 'pending', 'running', 'complete', 'failed', 'stale')); + +-- Recovery-relevant rows: documents with at least one layer in a +-- failure state. Partial index keeps it small on healthy deployments. +CREATE INDEX IF NOT EXISTS idx_raw_documents_status_failed + ON raw_documents (user_id) + WHERE deleted_at IS NULL + AND ( + extraction_status = 'failed' + OR semantic_index_status = 'failed' + OR raw_storage_status = 'raw_storage_failed' + ); + +-- Active-row idempotency: at most one live registration per +-- (user, source, external_id, version). Soft-deleted rows are excluded so +-- a re-registration after deletion creates a fresh row instead of colliding. +CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_documents_active_unique + ON raw_documents (user_id, raw_source_id, external_id, COALESCE(provider_version, '')) + WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_raw_documents_user + ON raw_documents (user_id) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_raw_documents_source + ON raw_documents (raw_source_id) WHERE deleted_at IS NULL; + +-- Memory provenance to documents/chunks. document_chunk_id is unused in +-- Phase 1 (chunks ship in Phase 2) but the column lands now so memories +-- created during the Phase 1 → 2 transition don't need a backfill migration. +-- No FK constraint yet — added once raw_documents/document_chunks deletion +-- semantics are reviewed alongside the chunk table in Phase 2. +ALTER TABLE memories ADD COLUMN IF NOT EXISTS raw_document_id UUID DEFAULT NULL; +ALTER TABLE memories ADD COLUMN IF NOT EXISTS document_chunk_id UUID DEFAULT NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_raw_document + ON memories (user_id, raw_document_id) + WHERE raw_document_id IS NOT NULL AND deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_memories_document_chunk + ON memories (user_id, document_chunk_id) + WHERE document_chunk_id IS NOT NULL AND deleted_at IS NULL; + +-- Phase D — passport-feed grouped query support (rev 18). +-- The `GROUP BY raw_document_id` + `ARRAY_AGG(... ORDER BY created_at +-- DESC, id DESC)[1]` pattern in `passport-feed-repository.ts` lifts +-- (created_at DESC, id DESC) inside each (user_id, raw_document_id) +-- partition. A composite partial index on those four columns lets +-- Postgres skip the secondary sort entirely on the grouped branch +-- under real volume; the partial WHERE clause keeps the index lean +-- (memory rows that aren't document-backed or are tombstoned never +-- enter the hot path). +CREATE INDEX IF NOT EXISTS idx_memories_passport_grouped + ON memories (user_id, raw_document_id, created_at DESC, id DESC) + WHERE raw_document_id IS NOT NULL AND deleted_at IS NULL; + +-- Phase D — passport-feed standalone branch support (rev 18). +-- The standalone-memory branch of the UNION ALL pages by +-- `(created_at DESC, id DESC)` filtered to memories with +-- `raw_document_id IS NULL`. The pre-existing +-- `idx_memories_user_created` is the wrong shape (no IS NULL +-- partial, no `id` tie-breaker). This partial index matches the +-- exact predicate the cursor walks. +CREATE INDEX IF NOT EXISTS idx_memories_passport_standalone + ON memories (user_id, created_at DESC, id DESC) + WHERE raw_document_id IS NULL AND deleted_at IS NULL; + +-- --------------------------------------------------------------------------- +-- Document chunks (Phase 2 of the large-file ingestion plan). +-- +-- One row per deterministic chunk derived from a registered document. +-- Chunks store their own embedding (so the chunk-level vector store +-- supports raw chunk lookup / debug / re-index without touching memories) +-- AND each chunk creates a sibling row in `memories` with +-- raw_document_id + document_chunk_id provenance — that's the surface +-- the existing /v1/memories/search retrieval pipeline finds. +-- +-- (parser_version, chunker_version) pair lets a future code change +-- re-chunk a document without colliding with the prior generation — +-- the partial unique index keys on chunker_version, so a bumped +-- chunker_version causes fresh inserts to coexist with the old soft- +-- deleted rows. Phase 2 only ships chunker_version='phase2-fixed-v1' +-- and parser_version='phase2-text-v1'; future phases bump. +-- --------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS document_chunks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + raw_document_id UUID NOT NULL REFERENCES raw_documents(id) ON DELETE CASCADE, + chunk_index INTEGER NOT NULL CHECK (chunk_index >= 0), + content TEXT NOT NULL, + content_hash TEXT NOT NULL, + char_start INTEGER NOT NULL CHECK (char_start >= 0), + char_end INTEGER NOT NULL CHECK (char_end >= char_start), + token_count INTEGER NOT NULL CHECK (token_count >= 0), + embedding vector(768) NOT NULL, + parser_version TEXT NOT NULL, + chunker_version TEXT NOT NULL, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deleted_at TIMESTAMPTZ +); + +-- Active-row uniqueness on (raw_document_id, chunk_index, chunker_version). +-- Soft-deleted rows are excluded so a re-index after a previous chunker +-- run leaves audit history intact while letting the new run succeed. +CREATE UNIQUE INDEX IF NOT EXISTS idx_document_chunks_active_unique + ON document_chunks (raw_document_id, chunk_index, chunker_version) + WHERE deleted_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_document_chunks_document + ON document_chunks (raw_document_id) + WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_document_chunks_user + ON document_chunks (user_id) + WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_document_chunks_embedding + ON document_chunks USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + +-- --------------------------------------------------------------------------- +-- Storage artifacts (Step 4 of the storage-sibling plan). +-- +-- One row per artifact tracked by the direct storage API, independent +-- of `raw_documents`. Pointer-mode rows carry a registered URI and +-- never persist bytes (the server NEVER fetches the URI itself); +-- managed-mode rows carry the adapter-returned URI plus the usual +-- pending → available / deleting → deleted/delete_failed lifecycle. +-- +-- Owner scoping lives on `user_id`. `org_id` / `project_id` are +-- reserved for future multi-tenancy and stay NULL in v1. +-- +-- Internal-only columns: +-- * `plaintext_hash` — SHA-256 of caller bytes; never on the wire +-- by default. The Step-5 response formatter exposes it only when +-- the row's `disclose_content_hash = true` AND the caller opted in +-- at put time. +-- * `stored_hash` — SHA-256 of the bytes the adapter actually wrote; +-- never on the wire under any condition. +-- * `last_error` — internal failure envelope for delete retries. +-- Step 5 is responsible for projecting the wire shape; this PR is +-- DB-only and is allowed to keep the internal columns visible in the +-- repository's row type. +-- +-- FK direction: `raw_documents.storage_artifact_id REFERENCES +-- storage_artifacts(id)`. Step 7 wires the document-ingestion paths +-- to populate this column; Step 4 just defines the persistence +-- surface and the reverse-lookup index. +-- --------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS storage_artifacts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + org_id TEXT, + project_id TEXT, + provider TEXT NOT NULL, + mode TEXT NOT NULL CHECK (mode IN ('pointer', 'managed')), + -- Nullable while the row is in `pending` (managed put before + -- backend.put has returned a URI); set to the adapter URI on + -- success, stays NULL on `failed`. `pointer` rows always supply + -- the URI at insert time. The partial unique index below covers + -- the post-set side of the contract. + uri TEXT, + status TEXT NOT NULL + CHECK (status IN + ('stored', 'pending', 'available', 'unavailable', + 'deleting', 'deleted', 'delete_failed', 'failed')), + size_bytes BIGINT, + content_type TEXT, + -- Internal; never on the wire by default. + plaintext_hash TEXT, + -- Internal; never on the wire ever. + stored_hash TEXT, + content_encoding TEXT NOT NULL DEFAULT 'identity' + CHECK (content_encoding IN ('identity', 'aes_gcm')), + disclose_content_hash BOOLEAN NOT NULL DEFAULT FALSE, + identifiers JSONB NOT NULL DEFAULT '{}'::jsonb, + lifecycle JSONB NOT NULL DEFAULT '{}'::jsonb, + replication JSONB, + verification JSONB, + retrieval JSONB, + provider_details JSONB, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + last_error JSONB, + -- CAS token for the upload pipeline (pending-row-first put). Held + -- by `claimPendingArtifact`, cleared by `recordUploadedArtifact` / + -- `markPutFailed`. Distinct from `delete_attempt_id` so the two + -- lifecycle phases never collide. + put_attempt_id UUID, + delete_attempt_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deleted_at TIMESTAMPTZ +); + +-- Forward-compat migration for deployments that already created +-- `storage_artifacts` with `uri NOT NULL`. Idempotent: only fires +-- the ALTER when information_schema reports the column as still +-- NOT NULL, so a re-run against a freshly upgraded DB is a no-op +-- without relying on `EXCEPTION WHEN OTHERS` (workspace rule +-- forbids silent error swallowing). +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'storage_artifacts' + AND column_name = 'uri' + AND is_nullable = 'NO' + ) THEN + ALTER TABLE storage_artifacts ALTER COLUMN uri DROP NOT NULL; + END IF; +END $$; +ALTER TABLE storage_artifacts + ADD COLUMN IF NOT EXISTS put_attempt_id UUID; + +-- Concurrent `putManaged` callers must not produce two rows whose +-- adapter URI collides. The unique index is partial: only managed +-- rows with a URI set and not soft-deleted participate. Pointer +-- rows are exempt — a single user legitimately has multiple +-- pointer rows for the same caller-supplied URI (e.g. one created +-- by `putPointer` against the active backend, another auto-paired +-- to a document registration via `EXTERNAL_POINTER_PROVIDER`). +-- `pending` / `failed` rows carry `uri IS NULL` and also fall out +-- of the constraint naturally. +CREATE UNIQUE INDEX IF NOT EXISTS uniq_storage_artifacts_user_managed_uri + ON storage_artifacts (user_id, uri) + WHERE uri IS NOT NULL AND deleted_at IS NULL AND mode = 'managed'; + +-- One-time cleanup: drop the over-broad index from an earlier rev +-- of this migration. Idempotent. +DROP INDEX IF EXISTS uniq_storage_artifacts_user_uri; + +CREATE INDEX IF NOT EXISTS idx_storage_artifacts_user_status + ON storage_artifacts (user_id, status) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_storage_artifacts_user_provider + ON storage_artifacts (user_id, provider) WHERE deleted_at IS NULL; +-- Cursor pagination keyed on (created_at DESC, id DESC) within a +-- user. Partial index keeps it lean for healthy deployments. +CREATE INDEX IF NOT EXISTS idx_storage_artifacts_user_created + ON storage_artifacts (user_id, created_at DESC, id DESC) + WHERE deleted_at IS NULL; + +-- Non-partial unique index on (id, user_id). `id` alone is already the +-- primary key; the composite uniqueness exists so the owner-scoped +-- composite foreign key on raw_documents below has a valid target. +-- Postgres accepts a non-partial unique index as an FK target, and +-- this is the only place we need the (id, user_id) pair indexed. +CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_artifacts_id_user + ON storage_artifacts (id, user_id); + +-- Reverse pointer from documents to their backing artifact. The FK is +-- COMPOSITE on (storage_artifact_id, user_id) so the schema itself +-- makes a USER_B raw_document pointing at USER_A's artifact +-- impossible — the row would have to match BOTH columns of the +-- referenced storage_artifacts row, and the artifact's `user_id` +-- column carries the canonical owner. Populated in Step 7. +-- +-- NULL `storage_artifact_id` is legitimate for rows registered +-- without an `external_uri` (pointer-only registration stub) or rows +-- that pre-date Step 7. +ALTER TABLE raw_documents + ADD COLUMN IF NOT EXISTS storage_artifact_id UUID NULL; +-- This baseline can be applied to fresh databases and may be replayed in +-- development by migration tooling. The composite FK is therefore added only +-- when it doesn't already exist, so repeated local runs don't take ACCESS +-- EXCLUSIVE on raw_documents to revalidate the same constraint. +-- The legacy single-column FK is dropped the same way for one-time +-- cleanup against any dev/test DB that applied the earlier shape. +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'raw_documents_storage_artifact_id_fkey' + AND conrelid = 'raw_documents'::regclass + ) THEN + ALTER TABLE raw_documents + DROP CONSTRAINT raw_documents_storage_artifact_id_fkey; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'raw_documents_storage_artifact_owner_fkey' + AND conrelid = 'raw_documents'::regclass + ) THEN + ALTER TABLE raw_documents + ADD CONSTRAINT raw_documents_storage_artifact_owner_fkey + FOREIGN KEY (storage_artifact_id, user_id) + REFERENCES storage_artifacts (id, user_id); + END IF; +END $$; +CREATE INDEX IF NOT EXISTS idx_raw_documents_storage_artifact + ON raw_documents (storage_artifact_id) WHERE storage_artifact_id IS NOT NULL; + +-- Phase 1 migration metadata. One row per successful migrate() call. +-- The most recent row by `applied_at` is the current effective version. +-- History is preserved so rolling deploys produce a visible audit trail. +CREATE TABLE IF NOT EXISTS schema_version ( + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + sdk_version TEXT NOT NULL, + schema_sha256 TEXT NOT NULL, + notes TEXT, + PRIMARY KEY (applied_at) +); + +CREATE INDEX IF NOT EXISTS idx_schema_version_applied_at + ON schema_version (applied_at DESC); diff --git a/src/db/pg-memory-store.ts b/src/db/pg-memory-store.ts index 4047cec..727f073 100644 --- a/src/db/pg-memory-store.ts +++ b/src/db/pg-memory-store.ts @@ -61,7 +61,7 @@ export class PgMemoryStore implements MemoryStore { async storeMemory(input: StoreMemoryInput) { return storeMemory(this.pool, input); } async getMemory(id: string, userId?: string) { return getMemory(this.pool, id, userId, false); } async getMemoryIncludingDeleted(id: string, userId?: string) { return getMemory(this.pool, id, userId, true); } - async listMemories(userId: string, limit = 20, offset = 0, sourceSite?: string, episodeId?: string) { return listMemories(this.pool, userId, limit, offset, sourceSite, episodeId); } + async listMemories(userId: string, limit = 20, offset = 0, sourceSite?: string, episodeId?: string, sessionId?: string) { return listMemories(this.pool, userId, limit, offset, sourceSite, episodeId, sessionId); } async softDeleteMemory(userId: string, id: string) { return softDeleteMemory(this.pool, userId, id); } async updateMemoryContent(userId: string, id: string, content: string, embedding: number[], importance: number, keywords?: string, trustScore?: number) { return updateMemoryContent(this.pool, userId, id, content, embedding, importance, keywords, trustScore); } async updateMemoryMetadata(userId: string, id: string, metadata: Record) { return updateMemoryMetadata(this.pool, userId, id, metadata); } @@ -81,6 +81,6 @@ export class PgMemoryStore implements MemoryStore { async countNeedsClarification(userId: string) { return countNeedsClarification(this.pool, userId); } async storeCanonicalMemoryObject(input: { userId: string; objectFamily: 'ingested_fact'; payloadFormat?: string; canonicalPayload: { factText: string; factType: string; headline: string; keywords: string[] }; provenance: { episodeId: string | null; sourceSite: string; sourceUrl: string }; observedAt?: Date; lineage: CanonicalMemoryObjectLineage }) { return storeCanonicalMemoryObject(this.pool, input); } async getMemoryInWorkspace(id: string, workspaceId: string, callerAgentId?: string) { return getMemoryInWorkspace(this.pool, id, workspaceId, callerAgentId); } - async listMemoriesInWorkspace(workspaceId: string, limit = 20, offset = 0, callerAgentId?: string) { return listMemoriesInWorkspace(this.pool, workspaceId, limit, offset, callerAgentId); } + async listMemoriesInWorkspace(workspaceId: string, limit = 20, offset = 0, callerAgentId?: string, sessionId?: string) { return listMemoriesInWorkspace(this.pool, workspaceId, limit, offset, callerAgentId, sessionId); } async softDeleteMemoryInWorkspace(id: string, workspaceId: string) { return softDeleteMemoryInWorkspace(this.pool, id, workspaceId); } } diff --git a/src/db/pg-search-store.ts b/src/db/pg-search-store.ts index c636b65..68c80f9 100644 --- a/src/db/pg-search-store.ts +++ b/src/db/pg-search-store.ts @@ -24,20 +24,20 @@ import { fetchMemoriesByIds } from './repository-links.js'; export class PgSearchStore implements SearchStore { constructor(private pool: pg.Pool) {} - async searchSimilar(userId: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date) { - return searchSimilar(this.pool, userId, queryEmbedding, limit, sourceSite, referenceTime); + async searchSimilar(userId: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date, sessionId?: string) { + return searchSimilar(this.pool, userId, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } - async searchHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date) { - return searchHybridSimilar(this.pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime); + async searchHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date, sessionId?: string) { + return searchHybridSimilar(this.pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } - async searchKeyword(userId: string, queryText: string, limit: number, sourceSite?: string) { - return searchKeywordSimilar(this.pool, userId, queryText, limit, sourceSite); + async searchKeyword(userId: string, queryText: string, limit: number, sourceSite?: string, sessionId?: string) { + return searchKeywordSimilar(this.pool, userId, queryText, limit, sourceSite, sessionId); } - async searchAtomicFactsHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date) { - return searchAtomicFactsHybrid(this.pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime); + async searchAtomicFactsHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date, sessionId?: string) { + return searchAtomicFactsHybrid(this.pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } async findNearDuplicates(userId: string, embedding: number[], threshold: number, limit = 3) { @@ -60,8 +60,8 @@ export class PgSearchStore implements SearchStore { return fetchMemoriesByIds(this.pool, userId, ids, queryEmbedding, referenceTime, includeExpired); } - async searchSimilarInWorkspace(workspaceId: string, queryEmbedding: number[], limit: number, agentScope: AgentScope = 'all', callerAgentId?: string, referenceTime?: Date) { - return searchSimilarInWorkspace(this.pool, workspaceId, queryEmbedding, limit, agentScope, callerAgentId, referenceTime); + async searchSimilarInWorkspace(workspaceId: string, queryEmbedding: number[], limit: number, agentScope: AgentScope = 'all', callerAgentId?: string, referenceTime?: Date, sessionId?: string) { + return searchSimilarInWorkspace(this.pool, workspaceId, queryEmbedding, limit, agentScope, callerAgentId, referenceTime, sessionId); } async findNearDuplicatesInWorkspace(workspaceId: string, embedding: number[], threshold: number, limit = 3, agentScope: AgentScope = 'all', callerAgentId?: string) { diff --git a/src/db/query-helpers.ts b/src/db/query-helpers.ts index 868b24a..b582a2d 100644 --- a/src/db/query-helpers.ts +++ b/src/db/query-helpers.ts @@ -40,13 +40,14 @@ export function buildHybridSearchParams( siteFilterColumn: string, sourceSite?: string, referenceTime?: Date, + sessionId?: string, + episodeIdColumn: string = 'episode_id', ): HybridQueryParams { const wSim = config.scoringWeightSimilarity; const wImp = config.scoringWeightImportance; const wRec = config.scoringWeightRecency; const rankingMinSimilarity = clampUnit(config.retrievalProfileSettings.rankingMinSimilarity); const refTime = (referenceTime ?? new Date()).toISOString(); - const siteFilter = sourceSite ? `AND ${siteFilterColumn} = $10` : ''; const params: unknown[] = [ pgvector.toSql(queryEmbedding), userId, @@ -54,7 +55,21 @@ export function buildHybridSearchParams( Math.max(1, limit), wSim, wImp, wRec, refTime, rankingMinSimilarity, ]; - if (sourceSite) params.push(sourceSite); + const filters: string[] = []; + if (sourceSite) { + params.push(sourceSite); + filters.push(`AND ${siteFilterColumn} = $${params.length}`); + } + if (sessionId) { + params.push(sessionId); + filters.push(`AND EXISTS ( + SELECT 1 FROM episodes e + WHERE e.id = ${episodeIdColumn} + AND e.user_id = $2 + AND e.session_id = $${params.length} + )`); + } + const siteFilter = filters.join('\n '); return { params, siteFilter, refTime, wSim, wImp, wRec, rankingMinSimilarity }; } @@ -69,18 +84,32 @@ export function buildVectorSearchParams( limit: number, sourceSite?: string, referenceTime?: Date, + sessionId?: string, ): { params: unknown[]; siteClause: string; wSim: number; wImp: number; wRec: number; rankingMinSimilarity: number; refTime: string } { const wSim = config.scoringWeightSimilarity; const wImp = config.scoringWeightImportance; const wRec = config.scoringWeightRecency; const rankingMinSimilarity = clampUnit(config.retrievalProfileSettings.rankingMinSimilarity); const refTime = (referenceTime ?? new Date()).toISOString(); - const siteClause = sourceSite ? 'AND source_site = $9' : ''; const params: unknown[] = [ pgvector.toSql(queryEmbedding), userId, Math.max(1, Math.min(100, limit)), wSim, wImp, wRec, refTime, rankingMinSimilarity, ]; - if (sourceSite) params.push(sourceSite); + const filters: string[] = []; + if (sourceSite) { + params.push(sourceSite); + filters.push(`AND memories.source_site = $${params.length}`); + } + if (sessionId) { + params.push(sessionId); + filters.push(`AND EXISTS ( + SELECT 1 FROM episodes e + WHERE e.id = memories.episode_id + AND e.user_id = $2 + AND e.session_id = $${params.length} + )`); + } + const siteClause = filters.join('\n '); return { params, siteClause, wSim, wImp, wRec, rankingMinSimilarity, refTime }; } diff --git a/src/db/reconcilers.ts b/src/db/reconcilers.ts new file mode 100644 index 0000000..4ef0794 --- /dev/null +++ b/src/db/reconcilers.ts @@ -0,0 +1,275 @@ +/** + * Embedding-dimension reconciler for @atomicmemory/core migrations. + * + * Discovers every pgvector column in the current schema with a fixed + * (positive-typmod) dimension and either alters it to the configured + * embedding dimension (when the column has no data yet) or throws a + * descriptive error so the operator must resolve the conflict deliberately. + * + * The discovery surface is intentionally column-name agnostic: the baseline + * migration defines several config-driven vector columns whose names vary + * (e.g. `embedding`, `summary_embedding`, `topic_embedding`, + * `recap_embedding`). The whole surface must move together when the + * embedding model changes, so the reconciler treats every fixed-dimension + * pgvector column as in-scope rather than hard-coding either table or column + * names. + * + * Designed to be invoked by `migrate()` after framework migrations have run + * (or after we confirm a peer replica already ran them). + * + * See docs/db/migrations.md for where this runs in the migration sequence. + */ + +import pg from 'pg'; + +/** + * Either a pool or an already-checked-out client. The reconciler will check + * out its own client from a pool when it needs a transaction; if the caller + * is already operating on a client (e.g., the connection that holds the + * advisory migration lock) it can pass that client directly. + */ +export type ReconcilerExecutor = pg.Pool | pg.PoolClient; + +export interface AlteredVectorColumn { + tableName: string; + columnName: string; +} + +export interface ReconcileResult { + /** True if at least one vector column was altered. */ + reconciled: boolean; + /** (table, column) pairs whose vector type was altered. */ + alteredColumns: AlteredVectorColumn[]; +} + +interface VectorColumn { + tableName: string; + columnName: string; + currentDimension: number; +} + +interface SavedIndex { + indexName: string; + createStmt: string; +} + +/** + * Thrown when a fixed-dimension pgvector column has non-null vectors and + * its declared dimension does not match the configured embedding + * dimension. Reconciliation refuses to alter populated columns because + * doing so would silently invalidate every stored vector. + * + * `tableName`, `currentDimension`, `requiredDimension`, and `rowCount` are + * preserved from the Phase 1 design; `columnName` is added so callers can + * disambiguate between e.g. `memories.embedding` and `memories.topic_embedding`. + */ +export class EmbeddingDimensionMismatch extends Error { + constructor( + public readonly tableName: string, + public readonly currentDimension: number, + public readonly requiredDimension: number, + public readonly rowCount: number, + public readonly columnName: string = 'embedding', + ) { + super( + `Embedding dimension mismatch on column "${tableName}"."${columnName}": ` + + `column is vector(${currentDimension}) but the configured embedding ` + + `dimension is ${requiredDimension}, and the column holds ${rowCount} ` + + `row(s) with a non-null vector. The reconciler refuses to alter a ` + + `populated vector column. To resolve, either:\n` + + ` 1. DELETE FROM "${tableName}" (or scope-wipe rows so "${columnName}" ` + + `is empty), then re-run migrate() so the column can be altered safely; or\n` + + ` 2. Switch the embedding model back to one producing ` + + `${currentDimension}-dimensional vectors so the existing rows remain valid.`, + ); + this.name = 'EmbeddingDimensionMismatch'; + } +} + +/** + * Discover every fixed-dimension pgvector column in the current schema. + * + * Filters to regular tables (relkind='r'), skips dropped attributes, and + * requires `atttypmod > 0` so unconstrained `vector` columns (which carry + * no dimension contract) are left alone. No column-name filter is applied: + * `embedding`, `summary_embedding`, `topic_embedding`, and any future + * config-driven vector column are all in scope. + */ +async function discoverVectorColumns( + executor: ReconcilerExecutor, +): Promise { + const { rows } = await executor.query<{ + table_name: string; + column_name: string; + current_dimension: number; + }>( + `SELECT + c.relname AS table_name, + a.attname AS column_name, + a.atttypmod AS current_dimension + FROM pg_attribute a + JOIN pg_class c ON a.attrelid = c.oid + JOIN pg_namespace n ON c.relnamespace = n.oid + JOIN pg_type t ON a.atttypid = t.oid + WHERE n.nspname = current_schema() + AND t.typname = 'vector' + AND a.atttypmod > 0 + AND NOT a.attisdropped + AND c.relkind = 'r' + ORDER BY c.relname, a.attname`, + ); + return rows.map((row) => ({ + tableName: row.table_name, + columnName: row.column_name, + currentDimension: row.current_dimension, + })); +} + +/** Count rows with a non-null value in the given column. */ +async function countNonNullVectors( + executor: ReconcilerExecutor, + tableName: string, + columnName: string, +): Promise { + const quotedTable = pg.escapeIdentifier(tableName); + const quotedColumn = pg.escapeIdentifier(columnName); + const { rows } = await executor.query<{ count: string }>( + `SELECT COUNT(*)::text AS count FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL`, + ); + return Number.parseInt(rows[0].count, 10); +} + +/** + * Find indexes whose first indexed column is `columnName` on `tableName`. + * pgvector indexes (HNSW, IVFFlat) are single-column on the vector field, + * so the first-column check correctly identifies the indexes that must be + * dropped before `ALTER COLUMN ... TYPE vector(N)`. Returns each index's + * name plus the CREATE statement we can replay verbatim afterwards. + */ +async function findColumnIndexes( + executor: ReconcilerExecutor, + tableName: string, + columnName: string, +): Promise { + const { rows } = await executor.query<{ + index_name: string; + create_stmt: string; + }>( + `SELECT + i.relname AS index_name, + pg_get_indexdef(idx.indexrelid) AS create_stmt + FROM pg_index idx + JOIN pg_class i ON idx.indexrelid = i.oid + JOIN pg_class c ON idx.indrelid = c.oid + JOIN pg_namespace n ON c.relnamespace = n.oid + JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = idx.indkey[0] + WHERE c.relname = $1 + AND n.nspname = current_schema() + AND a.attname = $2`, + [tableName, columnName], + ); + return rows.map((row) => ({ + indexName: row.index_name, + createStmt: row.create_stmt, + })); +} + +/** + * Drop the indexes on (tableName, columnName), alter the column type, then + * recreate the indexes. All wrapped in a single transaction so the column + * is never left half-altered. + */ +async function alterVectorColumn( + client: pg.PoolClient, + tableName: string, + columnName: string, + requiredDimension: number, +): Promise { + const quotedTable = pg.escapeIdentifier(tableName); + const quotedColumn = pg.escapeIdentifier(columnName); + const indexes = await findColumnIndexes(client, tableName, columnName); + await client.query('BEGIN'); + try { + for (const idx of indexes) { + await client.query(`DROP INDEX ${pg.escapeIdentifier(idx.indexName)}`); + } + await client.query( + `ALTER TABLE ${quotedTable} ALTER COLUMN ${quotedColumn} TYPE vector(${requiredDimension})`, + ); + for (const idx of indexes) { + await client.query(idx.createStmt); + } + await client.query('COMMIT'); + } catch (err) { + await client.query('ROLLBACK'); + throw err; + } +} + +/** + * True when the executor is a pg.Pool (i.e., we must check out a client). + * + * Detects Pool vs. PoolClient by the presence of `.release` rather than the + * presence of `.connect`. Both pg.Pool and pg.PoolClient expose `.connect`, + * so a `.connect`-only probe misidentifies an already-checked-out PoolClient + * as a Pool and then "reconnects" it — which is what migrate() does when it + * hands its locked PoolClient to reconcileEmbeddingDimension(). Only Pool + * lacks `.release`, so the negative check is the safe discriminator. + */ +function isPool(executor: ReconcilerExecutor): executor is pg.Pool { + if (typeof (executor as pg.Pool).connect !== 'function') return false; + return typeof (executor as { release?: unknown }).release !== 'function'; +} + +async function withClient( + executor: ReconcilerExecutor, + fn: (client: pg.PoolClient) => Promise, +): Promise { + if (!isPool(executor)) return fn(executor as pg.PoolClient); + const client = await executor.connect(); + try { + return await fn(client); + } finally { + client.release(); + } +} + +/** + * Reconcile the dimension of every fixed-dimension pgvector column in the + * current schema with `requiredDimension`. + * + * - Matches: no-op. + * - Mismatch + non-null rows: throws `EmbeddingDimensionMismatch`. + * - Mismatch + empty: drops indexes on the column, alters its type, recreates indexes. + */ +export async function reconcileEmbeddingDimension( + executor: ReconcilerExecutor, + requiredDimension: number, +): Promise { + if (!Number.isInteger(requiredDimension) || requiredDimension <= 0) { + throw new Error( + `reconcileEmbeddingDimension: requiredDimension must be a positive ` + + `integer, got ${requiredDimension}`, + ); + } + const columns = await discoverVectorColumns(executor); + const alteredColumns: AlteredVectorColumn[] = []; + for (const { tableName, columnName, currentDimension } of columns) { + if (currentDimension === requiredDimension) continue; + const rowCount = await countNonNullVectors(executor, tableName, columnName); + if (rowCount > 0) { + throw new EmbeddingDimensionMismatch( + tableName, + currentDimension, + requiredDimension, + rowCount, + columnName, + ); + } + await withClient(executor, (client) => + alterVectorColumn(client, tableName, columnName, requiredDimension), + ); + alteredColumns.push({ tableName, columnName }); + } + return { reconciled: alteredColumns.length > 0, alteredColumns }; +} diff --git a/src/db/repository-read.ts b/src/db/repository-read.ts index c5fc3c1..fe30939 100644 --- a/src/db/repository-read.ts +++ b/src/db/repository-read.ts @@ -41,36 +41,60 @@ export async function getMemoryWithClient( userId?: string, includeDeleted: boolean = false, ): Promise { - const clauses = ['id = $1']; + const clauses = ['memories.id = $1']; const params: string[] = [id]; if (userId) { - clauses.push(`user_id = $${params.length + 1}`); + clauses.push(`memories.user_id = $${params.length + 1}`); params.push(userId); } if (!includeDeleted) { - clauses.push('deleted_at IS NULL'); - clauses.push('expired_at IS NULL'); + clauses.push('memories.deleted_at IS NULL'); + clauses.push('memories.expired_at IS NULL'); } - const result = await client.query(`SELECT * FROM memories WHERE ${clauses.join(' AND ')}`, params); + const result = await client.query( + `SELECT memories.*, episodes.session_id + FROM memories + LEFT JOIN episodes + ON episodes.id = memories.episode_id + AND episodes.user_id = memories.user_id + WHERE ${clauses.join(' AND ')}`, + params, + ); return result.rows[0] ? normalizeMemoryRow(result.rows[0]) : null; } -export async function listMemories(pool: pg.Pool, userId: string, limit: number, offset: number, sourceSite?: string, episodeId?: string): Promise { +export async function listMemories(pool: pg.Pool, userId: string, limit: number, offset: number, sourceSite?: string, episodeId?: string, sessionId?: string): Promise { const params: unknown[] = [userId, limit, offset]; let extraClauses = ''; if (sourceSite) { params.push(sourceSite); - extraClauses += ` AND source_site = $${params.length}`; + extraClauses += ` AND memories.source_site = $${params.length}`; } if (episodeId) { params.push(episodeId); - extraClauses += ` AND episode_id = $${params.length}`; + extraClauses += ` AND memories.episode_id = $${params.length}`; + } + if (sessionId) { + params.push(sessionId); + extraClauses += ` AND EXISTS ( + SELECT 1 FROM episodes e + WHERE e.id = memories.episode_id + AND e.user_id = memories.user_id + AND e.session_id = $${params.length} + )`; } const result = await pool.query( - `SELECT * FROM memories - WHERE user_id = $1 AND deleted_at IS NULL AND expired_at IS NULL AND status = 'active' - AND workspace_id IS NULL${extraClauses} - ORDER BY created_at DESC LIMIT $2 OFFSET $3`, + `SELECT memories.*, episodes.session_id + FROM memories + LEFT JOIN episodes + ON episodes.id = memories.episode_id + AND episodes.user_id = memories.user_id + WHERE memories.user_id = $1 + AND memories.deleted_at IS NULL + AND memories.expired_at IS NULL + AND memories.status = 'active' + AND memories.workspace_id IS NULL${extraClauses} + ORDER BY memories.created_at DESC LIMIT $2 OFFSET $3`, params, ); return result.rows.map(normalizeMemoryRow); @@ -82,6 +106,7 @@ export async function listMemoriesInWorkspace( limit: number, offset: number, callerAgentId?: string, + sessionId?: string, ): Promise { const params: unknown[] = [workspaceId]; let visibilityClause = ''; @@ -89,12 +114,25 @@ export async function listMemoriesInWorkspace( params.push(callerAgentId); visibilityClause = buildVisibilityClause(params.length); } + let sessionClause = ''; + if (sessionId) { + params.push(sessionId); + sessionClause = `AND episodes.session_id = $${params.length}`; + } params.push(limit, offset); const result = await pool.query( - `SELECT * FROM memories - WHERE workspace_id = $1 AND deleted_at IS NULL AND expired_at IS NULL AND status = 'active' + `SELECT memories.*, episodes.session_id + FROM memories + LEFT JOIN episodes + ON episodes.id = memories.episode_id + AND episodes.user_id = memories.user_id + WHERE memories.workspace_id = $1 + AND memories.deleted_at IS NULL + AND memories.expired_at IS NULL + AND memories.status = 'active' ${visibilityClause} - ORDER BY created_at DESC LIMIT $${params.length - 1} OFFSET $${params.length}`, + ${sessionClause} + ORDER BY memories.created_at DESC LIMIT $${params.length - 1} OFFSET $${params.length}`, params, ); return result.rows.map(normalizeMemoryRow); @@ -125,11 +163,11 @@ export async function getMemoryInWorkspace( */ function buildVisibilityClause(agentParamIndex: number): string { return `AND ( - visibility = 'workspace' - OR visibility IS NULL - OR (visibility = 'agent_only' AND agent_id = $${agentParamIndex}) - OR (visibility = 'restricted' AND ( - agent_id = $${agentParamIndex} + memories.visibility = 'workspace' + OR memories.visibility IS NULL + OR (memories.visibility = 'agent_only' AND memories.agent_id = $${agentParamIndex}) + OR (memories.visibility = 'restricted' AND ( + memories.agent_id = $${agentParamIndex} OR EXISTS ( SELECT 1 FROM memory_visibility_grants g WHERE g.memory_id = memories.id AND g.grantee_agent_id = $${agentParamIndex} @@ -187,8 +225,9 @@ export async function searchSimilar( limit: number, sourceSite?: string, referenceTime?: Date, + sessionId?: string, ): Promise { - return searchVectors(pool, userId, queryEmbedding, limit, sourceSite, referenceTime); + return searchVectors(pool, userId, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } export async function searchHybridSimilar( @@ -199,8 +238,9 @@ export async function searchHybridSimilar( limit: number, sourceSite?: string, referenceTime?: Date, + sessionId?: string, ): Promise { - return searchHybrid(pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime); + return searchHybrid(pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } export async function searchKeywordSimilar( @@ -209,8 +249,9 @@ export async function searchKeywordSimilar( queryText: string, limit: number, sourceSite?: string, + sessionId?: string, ): Promise { - return searchKeyword(pool, userId, queryText, limit, sourceSite); + return searchKeyword(pool, userId, queryText, limit, sourceSite, sessionId); } export async function findNearDuplicates( @@ -234,8 +275,9 @@ export async function searchSimilarInWorkspace( agentScope: AgentScope = 'all', callerAgentId?: string, referenceTime?: Date, + sessionId?: string, ): Promise { - return searchVectorsInWorkspace(pool, workspaceId, queryEmbedding, limit, agentScope, callerAgentId, referenceTime); + return searchVectorsInWorkspace(pool, workspaceId, queryEmbedding, limit, agentScope, callerAgentId, referenceTime, sessionId); } /** diff --git a/src/db/repository-representations.ts b/src/db/repository-representations.ts index 47718e8..5f12734 100644 --- a/src/db/repository-representations.ts +++ b/src/db/repository-representations.ts @@ -183,9 +183,10 @@ export async function searchAtomicFactsHybrid( limit: number, sourceSite?: string, referenceTime?: Date, + sessionId?: string, ): Promise { const { params, siteFilter } = buildHybridSearchParams( - queryEmbedding, userId, queryText, limit, 'af.source_site', sourceSite, referenceTime, + queryEmbedding, userId, queryText, limit, 'af.source_site', sourceSite, referenceTime, sessionId, 'm.episode_id', ); const result = await pool.query( @@ -241,6 +242,7 @@ export async function searchAtomicFactsHybrid( ) SELECT m.*, + e.session_id, p.similarity, ( $5 * p.similarity @@ -256,6 +258,9 @@ export async function searchAtomicFactsHybrid( 'atomic_fact'::text AS retrieval_layer FROM parent_ranked p JOIN memories m ON m.id = p.parent_memory_id + LEFT JOIN episodes e + ON e.id = m.episode_id + AND e.user_id = m.user_id ORDER BY score DESC LIMIT $4`, params, diff --git a/src/db/repository-types.ts b/src/db/repository-types.ts index c20ce91..a544f29 100644 --- a/src/db/repository-types.ts +++ b/src/db/repository-types.ts @@ -173,6 +173,7 @@ export interface MemoryRow { source_site: string; source_url: string; episode_id: string | null; + session_id?: string | null; status: 'active' | 'needs_clarification'; metadata: MemoryMetadata; keywords: string; diff --git a/src/db/repository-vector-search.ts b/src/db/repository-vector-search.ts index 4ebc4cd..527b057 100644 --- a/src/db/repository-vector-search.ts +++ b/src/db/repository-vector-search.ts @@ -30,14 +30,15 @@ export async function searchVectors( limit: number, sourceSite?: string, referenceTime?: Date, + sessionId?: string, ): Promise { if (config.vectorBackend === 'pgvector') { - return searchVectorsPg(pool, userId, queryEmbedding, limit, sourceSite, referenceTime); + return searchVectorsPg(pool, userId, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } if (config.vectorBackend === 'ruvector-mock') { - return searchVectorsRuvectorMock(pool, userId, queryEmbedding, limit, sourceSite, referenceTime); + return searchVectorsRuvectorMock(pool, userId, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } - return searchVectorsZvecMock(pool, userId, queryEmbedding, limit, sourceSite, referenceTime); + return searchVectorsZvecMock(pool, userId, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } /** @@ -55,11 +56,12 @@ export async function searchHybrid( limit: number, sourceSite?: string, referenceTime?: Date, + sessionId?: string, ): Promise { if (config.vectorBackend !== 'pgvector') { - return searchVectors(pool, userId, queryEmbedding, limit, sourceSite, referenceTime); + return searchVectors(pool, userId, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } - return searchHybridPg(pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime); + return searchHybridPg(pool, userId, queryText, queryEmbedding, limit, sourceSite, referenceTime, sessionId); } export async function searchKeyword( @@ -68,11 +70,12 @@ export async function searchKeyword( queryText: string, limit: number, sourceSite?: string, + sessionId?: string, ): Promise { if (config.vectorBackend !== 'pgvector') { return []; } - return searchKeywordPg(pool, userId, queryText, limit, sourceSite); + return searchKeywordPg(pool, userId, queryText, limit, sourceSite, sessionId); } export async function findDuplicateVectors( @@ -104,6 +107,7 @@ export async function searchVectorsInWorkspace( agentScope: AgentScope = 'all', callerAgentId?: string, referenceTime?: Date, + sessionId?: string, ): Promise { const wSim = config.scoringWeightSimilarity; const wImp = config.scoringWeightImportance; @@ -121,31 +125,40 @@ export async function searchVectorsInWorkspace( nextParam += agentClause.paramsAdded; const visibilityClause = buildVisibilityClauseForSearch(callerAgentId, params, nextParam); + let sessionClause = ''; + if (sessionId) { + params.push(sessionId); + sessionClause = `AND episodes.session_id = $${params.length}`; + } const result = await pool.query( - `SELECT *, - 1 - (embedding <=> $1) AS similarity, - 1 - (embedding <=> $1) AS semantic_similarity, - GREATEST(0, LEAST(1, 1 - (embedding <=> $1))) AS relevance, + `SELECT memories.*, episodes.session_id, + 1 - (memories.embedding <=> $1) AS similarity, + 1 - (memories.embedding <=> $1) AS semantic_similarity, + GREATEST(0, LEAST(1, 1 - (memories.embedding <=> $1))) AS relevance, ( - $4 * (1 - (embedding <=> $1)) - + $5 * importance - + $6 * EXP(-EXTRACT(EPOCH FROM ($7::timestamptz - last_accessed_at)) / 2592000.0) - ) * COALESCE(trust_score, 1.0) AS score, + $4 * (1 - (memories.embedding <=> $1)) + + $5 * memories.importance + + $6 * EXP(-EXTRACT(EPOCH FROM ($7::timestamptz - memories.last_accessed_at)) / 2592000.0) + ) * COALESCE(memories.trust_score, 1.0) AS score, ( - $4 * (1 - (embedding <=> $1)) - + CASE WHEN (1 - (embedding <=> $1)) >= $8 THEN ( - $5 * importance - + $6 * EXP(-EXTRACT(EPOCH FROM ($7::timestamptz - last_accessed_at)) / 2592000.0) + $4 * (1 - (memories.embedding <=> $1)) + + CASE WHEN (1 - (memories.embedding <=> $1)) >= $8 THEN ( + $5 * memories.importance + + $6 * EXP(-EXTRACT(EPOCH FROM ($7::timestamptz - memories.last_accessed_at)) / 2592000.0) ) ELSE 0 END - ) * COALESCE(trust_score, 1.0) AS ranking_score + ) * COALESCE(memories.trust_score, 1.0) AS ranking_score FROM memories - WHERE workspace_id = $2 - AND deleted_at IS NULL - AND expired_at IS NULL - AND status = 'active' + LEFT JOIN episodes + ON episodes.id = memories.episode_id + AND episodes.user_id = memories.user_id + WHERE memories.workspace_id = $2 + AND memories.deleted_at IS NULL + AND memories.expired_at IS NULL + AND memories.status = 'active' ${agentClause.sql} ${visibilityClause.sql} + ${sessionClause} ORDER BY ranking_score DESC LIMIT $3`, params, @@ -203,18 +216,18 @@ function buildAgentScopeClause( if (scope === 'others') { if (!callerAgentId) return { sql: '', paramsAdded: 0 }; params.push(callerAgentId); - return { sql: `AND (agent_id IS NULL OR agent_id != $${nextParam})`, paramsAdded: 1 }; + return { sql: `AND (memories.agent_id IS NULL OR memories.agent_id != $${nextParam})`, paramsAdded: 1 }; } if (Array.isArray(scope) && scope.length > 0) { params.push(scope); - return { sql: `AND agent_id = ANY($${nextParam}::uuid[])`, paramsAdded: 1 }; + return { sql: `AND memories.agent_id = ANY($${nextParam}::uuid[])`, paramsAdded: 1 }; } const targetId = scope === 'self' ? callerAgentId : scope; if (!targetId) return { sql: '', paramsAdded: 0 }; params.push(targetId); - return { sql: `AND agent_id = $${nextParam}`, paramsAdded: 1 }; + return { sql: `AND memories.agent_id = $${nextParam}`, paramsAdded: 1 }; } /** @@ -230,11 +243,11 @@ function buildVisibilityClauseForSearch( params.push(callerAgentId); return { sql: `AND ( - visibility = 'workspace' - OR visibility IS NULL - OR (visibility = 'agent_only' AND agent_id = $${nextParam}) - OR (visibility = 'restricted' AND ( - agent_id = $${nextParam} + memories.visibility = 'workspace' + OR memories.visibility IS NULL + OR (memories.visibility = 'agent_only' AND memories.agent_id = $${nextParam}) + OR (memories.visibility = 'restricted' AND ( + memories.agent_id = $${nextParam} OR EXISTS ( SELECT 1 FROM memory_visibility_grants g WHERE g.memory_id = memories.id AND g.grantee_agent_id = $${nextParam} @@ -252,32 +265,36 @@ async function searchVectorsPg( limit: number, sourceSite?: string, referenceTime?: Date, + sessionId?: string, ): Promise { - const { params, siteClause } = buildVectorSearchParams(queryEmbedding, userId, limit, sourceSite, referenceTime); + const { params, siteClause } = buildVectorSearchParams(queryEmbedding, userId, limit, sourceSite, referenceTime, sessionId); const result = await pool.query( - `SELECT *, - 1 - (embedding <=> $1) AS similarity, - 1 - (embedding <=> $1) AS semantic_similarity, - GREATEST(0, LEAST(1, 1 - (embedding <=> $1))) AS relevance, + `SELECT memories.*, episodes.session_id, + 1 - (memories.embedding <=> $1) AS similarity, + 1 - (memories.embedding <=> $1) AS semantic_similarity, + GREATEST(0, LEAST(1, 1 - (memories.embedding <=> $1))) AS relevance, ( - $4 * (1 - (embedding <=> $1)) - + $5 * importance - + $6 * EXP(-EXTRACT(EPOCH FROM ($7::timestamptz - last_accessed_at)) / 2592000.0) - ) * COALESCE(trust_score, 1.0) AS score, + $4 * (1 - (memories.embedding <=> $1)) + + $5 * memories.importance + + $6 * EXP(-EXTRACT(EPOCH FROM ($7::timestamptz - memories.last_accessed_at)) / 2592000.0) + ) * COALESCE(memories.trust_score, 1.0) AS score, ( - $4 * (1 - (embedding <=> $1)) - + CASE WHEN (1 - (embedding <=> $1)) >= $8 THEN ( - $5 * importance - + $6 * EXP(-EXTRACT(EPOCH FROM ($7::timestamptz - last_accessed_at)) / 2592000.0) + $4 * (1 - (memories.embedding <=> $1)) + + CASE WHEN (1 - (memories.embedding <=> $1)) >= $8 THEN ( + $5 * memories.importance + + $6 * EXP(-EXTRACT(EPOCH FROM ($7::timestamptz - memories.last_accessed_at)) / 2592000.0) ) ELSE 0 END - ) * COALESCE(trust_score, 1.0) AS ranking_score + ) * COALESCE(memories.trust_score, 1.0) AS ranking_score FROM memories - WHERE user_id = $2 - AND deleted_at IS NULL - AND expired_at IS NULL - AND status = 'active' - AND workspace_id IS NULL + LEFT JOIN episodes + ON episodes.id = memories.episode_id + AND episodes.user_id = memories.user_id + WHERE memories.user_id = $2 + AND memories.deleted_at IS NULL + AND memories.expired_at IS NULL + AND memories.status = 'active' + AND memories.workspace_id IS NULL ${siteClause} ORDER BY ranking_score DESC LIMIT $3`, @@ -315,27 +332,42 @@ async function searchKeywordPg( queryText: string, limit: number, sourceSite?: string, + sessionId?: string, ): Promise { - const siteFilter = sourceSite ? 'AND source_site = $4' : ''; const params: unknown[] = [userId, queryText, normalizeLimit(limit)]; + const filters: string[] = []; if (sourceSite) params.push(sourceSite); + if (sourceSite) filters.push(`AND memories.source_site = $${params.length}`); + if (sessionId) { + params.push(sessionId); + filters.push(`AND EXISTS ( + SELECT 1 FROM episodes e + WHERE e.id = memories.episode_id + AND e.user_id = memories.user_id + AND e.session_id = $${params.length} + )`); + } + const siteFilter = filters.join('\n '); const result = await pool.query( - `SELECT *, - LEAST(ts_rank(search_vector, plainto_tsquery('english', $2)), 1.0) AS similarity, - LEAST(ts_rank(search_vector, plainto_tsquery('english', $2)), 1.0) AS semantic_similarity, - LEAST(ts_rank(search_vector, plainto_tsquery('english', $2)), 1.0) AS relevance, - ts_rank(search_vector, plainto_tsquery('english', $2)) AS score, - ts_rank(search_vector, plainto_tsquery('english', $2)) AS ranking_score + `SELECT memories.*, episodes.session_id, + LEAST(ts_rank(memories.search_vector, plainto_tsquery('english', $2)), 1.0) AS similarity, + LEAST(ts_rank(memories.search_vector, plainto_tsquery('english', $2)), 1.0) AS semantic_similarity, + LEAST(ts_rank(memories.search_vector, plainto_tsquery('english', $2)), 1.0) AS relevance, + ts_rank(memories.search_vector, plainto_tsquery('english', $2)) AS score, + ts_rank(memories.search_vector, plainto_tsquery('english', $2)) AS ranking_score FROM memories - WHERE user_id = $1 - AND deleted_at IS NULL - AND expired_at IS NULL - AND status = 'active' - AND workspace_id IS NULL + LEFT JOIN episodes + ON episodes.id = memories.episode_id + AND episodes.user_id = memories.user_id + WHERE memories.user_id = $1 + AND memories.deleted_at IS NULL + AND memories.expired_at IS NULL + AND memories.status = 'active' + AND memories.workspace_id IS NULL ${siteFilter} - AND search_vector @@ plainto_tsquery('english', $2) - ORDER BY ts_rank(search_vector, plainto_tsquery('english', $2)) DESC, importance DESC + AND memories.search_vector @@ plainto_tsquery('english', $2) + ORDER BY ts_rank(memories.search_vector, plainto_tsquery('english', $2)) DESC, memories.importance DESC LIMIT $3`, params, ); @@ -350,9 +382,10 @@ async function searchHybridPg( limit: number, sourceSite?: string, referenceTime?: Date, + sessionId?: string, ): Promise { const { params, siteFilter } = buildHybridSearchParams( - queryEmbedding, userId, queryText, limit, 'source_site', sourceSite, referenceTime, + queryEmbedding, userId, queryText, limit, 'memories.source_site', sourceSite, referenceTime, sessionId, 'memories.episode_id', ); const result = await pool.query( @@ -377,7 +410,7 @@ async function searchHybridPg( FROM vector_ranked v FULL OUTER JOIN fts_ranked f ON v.id = f.id ) - SELECT m.*, + SELECT m.*, episodes.session_id, 1 - (m.embedding <=> $1) AS similarity, 1 - (m.embedding <=> $1) AS semantic_similarity, GREATEST(0, LEAST(1, 1 - (m.embedding <=> $1))) AS relevance, @@ -398,6 +431,9 @@ async function searchHybridPg( ) * COALESCE(m.trust_score, 1.0) AS ranking_score FROM fused f JOIN memories m ON m.id = f.id + LEFT JOIN episodes + ON episodes.id = m.episode_id + AND episodes.user_id = m.user_id ORDER BY ranking_score DESC LIMIT $4`, params, @@ -407,9 +443,9 @@ async function searchHybridPg( async function searchVectorsRuvectorMock( pool: pg.Pool, userId: string, queryEmbedding: number[], limit: number, - sourceSite?: string, referenceTime?: Date, + sourceSite?: string, referenceTime?: Date, sessionId?: string, ): Promise { - const memories = await loadActiveMemories(pool, userId, sourceSite); + const memories = await loadActiveMemories(pool, userId, sourceSite, sessionId); return rankAndSortMemories(memories, queryEmbedding, limit, referenceTime); } @@ -422,9 +458,9 @@ async function findDuplicateVectorsRuvectorMock( async function searchVectorsZvecMock( pool: pg.Pool, userId: string, queryEmbedding: number[], limit: number, - sourceSite?: string, referenceTime?: Date, + sourceSite?: string, referenceTime?: Date, sessionId?: string, ): Promise { - const memories = await loadActiveMemories(pool, userId, sourceSite); + const memories = await loadActiveMemories(pool, userId, sourceSite, sessionId); const shortlist = buildApproximateShortlist(memories, queryEmbedding, limit); return rankAndSortMemories(shortlist, queryEmbedding, limit, referenceTime); } @@ -458,16 +494,35 @@ function rankAndSortMemories( .slice(0, normalizeLimit(limit)); } -async function loadActiveMemories(pool: pg.Pool, userId: string, sourceSite?: string): Promise { +async function loadActiveMemories(pool: pg.Pool, userId: string, sourceSite?: string, sessionId?: string): Promise { + const params: unknown[] = [userId]; + const filters: string[] = []; + if (sourceSite) { + params.push(sourceSite); + filters.push(`AND memories.source_site = $${params.length}`); + } + if (sessionId) { + params.push(sessionId); + filters.push(`AND EXISTS ( + SELECT 1 FROM episodes e + WHERE e.id = memories.episode_id + AND e.user_id = memories.user_id + AND e.session_id = $${params.length} + )`); + } const result = await pool.query( - `SELECT * FROM memories - WHERE user_id = $1 - AND deleted_at IS NULL - AND expired_at IS NULL - AND status = 'active' - AND workspace_id IS NULL - ${sourceSite ? 'AND source_site = $2' : ''}`, - sourceSite ? [userId, sourceSite] : [userId], + `SELECT memories.*, episodes.session_id + FROM memories + LEFT JOIN episodes + ON episodes.id = memories.episode_id + AND episodes.user_id = memories.user_id + WHERE memories.user_id = $1 + AND memories.deleted_at IS NULL + AND memories.expired_at IS NULL + AND memories.status = 'active' + AND memories.workspace_id IS NULL + ${filters.join('\n ')}`, + params, ); return result.rows.map(normalizeMemoryRow); } diff --git a/src/db/repository-wipe.ts b/src/db/repository-wipe.ts index 9b182ea..e13902e 100644 --- a/src/db/repository-wipe.ts +++ b/src/db/repository-wipe.ts @@ -6,8 +6,8 @@ * the workspace's 400-non-comment-LOC cap. The wipe path is its own * concern: it cleans managed blobs first (so a failure can mark * surviving rows `raw_storage_failed` + sync the linked artifact) - * and then hard-deletes the memory tables, raw_documents, - * storage_artifacts, and raw_sources in FK-safe order. + * and then hard-deletes memory tables, derived user projections, + * raw_documents, storage_artifacts, and raw_sources in FK-safe order. */ import type pg from 'pg'; @@ -24,6 +24,30 @@ import { markCleanupFailedAndSyncArtifact, } from './raw-doc-artifact-sync.js'; +const USER_SCOPED_WIPE_TABLES_BEFORE_MEMORIES = [ + 'memory_contradictions', + 'memory_conflicts', + 'belief_edges', + 'session_reflections', + 'reflection_jobs', + 'entity_cards', + 'entity_values', + 'entity_attributes', + 'user_profiles', + 'recaps', + 'session_summaries', + 'conv_summaries', + 'lessons', + 'agent_trust', + 'first_mention_events', + 'temporal_linkage_list', + 'entity_relations', + 'memory_atomic_facts', + 'memory_foresight', + 'canonical_memory_objects', + 'observation_dirty', +] as const; + export interface DeleteAllOptions { rawContentStore?: RawContentStore | null; /** @@ -100,18 +124,22 @@ async function markDeleteAllCleanupFailure( } /** - * User-scoped hard-delete. Chunks → documents → storage_artifacts → - * sources, in FK-safe order. `storage_artifacts` is hard-deleted - * AFTER `raw_documents` because the composite FK + * User-scoped hard-delete. Derived memory projections are deleted + * before base memories; documents are deleted before storage_artifacts + * because the composite FK * `raw_documents(storage_artifact_id, user_id) → storage_artifacts` - * points one way. + * points from documents to artifacts. */ async function deleteAllForUser(pool: pg.Pool, userId: string): Promise { - await pool.query('DELETE FROM memory_evidence WHERE claim_version_id IN (SELECT id FROM memory_claim_versions WHERE user_id = $1)', [userId]); + await deleteUserScopedTables(pool, userId, USER_SCOPED_WIPE_TABLES_BEFORE_MEMORIES); + await pool.query('DELETE FROM memory_visibility_grants WHERE memory_id IN (SELECT id FROM memories WHERE user_id = $1)', [userId]); + await pool.query('DELETE FROM memory_entities WHERE memory_id IN (SELECT id FROM memories WHERE user_id = $1) OR entity_id IN (SELECT id FROM entities WHERE user_id = $1)', [userId]); + await pool.query('DELETE FROM memory_evidence WHERE claim_version_id IN (SELECT id FROM memory_claim_versions WHERE user_id = $1) OR memory_id IN (SELECT id FROM memories WHERE user_id = $1)', [userId]); await pool.query('DELETE FROM memory_claim_versions WHERE user_id = $1', [userId]); await pool.query('DELETE FROM memory_claims WHERE user_id = $1', [userId]); - await pool.query('DELETE FROM memory_links WHERE source_id IN (SELECT id FROM memories WHERE user_id = $1)', [userId]); + await pool.query('DELETE FROM memory_links WHERE source_id IN (SELECT id FROM memories WHERE user_id = $1) OR target_id IN (SELECT id FROM memories WHERE user_id = $1)', [userId]); await pool.query('DELETE FROM memories WHERE user_id = $1', [userId]); + await pool.query('DELETE FROM entities WHERE user_id = $1', [userId]); await pool.query('DELETE FROM episodes WHERE user_id = $1', [userId]); await pool.query('DELETE FROM document_chunks WHERE user_id = $1', [userId]); await pool.query('DELETE FROM raw_documents WHERE user_id = $1', [userId]); @@ -121,14 +149,34 @@ async function deleteAllForUser(pool: pg.Pool, userId: string): Promise { /** Global hard-delete. Same FK-safe order; no user filter. */ async function deleteAllGlobal(pool: pg.Pool): Promise { + await deleteGlobalTables(pool, USER_SCOPED_WIPE_TABLES_BEFORE_MEMORIES); + await pool.query('DELETE FROM memory_visibility_grants'); + await pool.query('DELETE FROM memory_entities'); await pool.query('DELETE FROM memory_evidence'); await pool.query('DELETE FROM memory_claim_versions'); await pool.query('DELETE FROM memory_claims'); await pool.query('DELETE FROM memory_links'); await pool.query('DELETE FROM memories'); + await pool.query('DELETE FROM entities'); await pool.query('DELETE FROM episodes'); await pool.query('DELETE FROM document_chunks'); await pool.query('DELETE FROM raw_documents'); await pool.query('DELETE FROM storage_artifacts'); await pool.query('DELETE FROM raw_sources'); } + +async function deleteUserScopedTables( + pool: pg.Pool, + userId: string, + tableNames: readonly string[], +): Promise { + for (const tableName of tableNames) { + await pool.query(`DELETE FROM ${tableName} WHERE user_id = $1`, [userId]); + } +} + +async function deleteGlobalTables(pool: pg.Pool, tableNames: readonly string[]): Promise { + for (const tableName of tableNames) { + await pool.query(`DELETE FROM ${tableName}`); + } +} diff --git a/src/db/stores.ts b/src/db/stores.ts index 909e720..6d27509 100644 --- a/src/db/stores.ts +++ b/src/db/stores.ts @@ -78,7 +78,7 @@ export interface MemoryStore { storeMemory(input: StoreMemoryInput): Promise; getMemory(id: string, userId?: string): Promise; getMemoryIncludingDeleted(id: string, userId?: string): Promise; - listMemories(userId: string, limit?: number, offset?: number, sourceSite?: string, episodeId?: string): Promise; + listMemories(userId: string, limit?: number, offset?: number, sourceSite?: string, episodeId?: string, sessionId?: string): Promise; softDeleteMemory(userId: string, id: string): Promise; updateMemoryContent(userId: string, id: string, content: string, embedding: number[], importance: number, keywords?: string, trustScore?: number): Promise; updateMemoryMetadata(userId: string, id: string, metadata: Record): Promise; @@ -110,7 +110,7 @@ export interface MemoryStore { }): Promise; // Workspace variants getMemoryInWorkspace(id: string, workspaceId: string, callerAgentId?: string): Promise; - listMemoriesInWorkspace(workspaceId: string, limit?: number, offset?: number, callerAgentId?: string): Promise; + listMemoriesInWorkspace(workspaceId: string, limit?: number, offset?: number, callerAgentId?: string, sessionId?: string): Promise; softDeleteMemoryInWorkspace(id: string, workspaceId: string): Promise; } @@ -128,17 +128,17 @@ export interface EpisodeStore { // --------------------------------------------------------------------------- export interface SearchStore { - searchSimilar(userId: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date): Promise; - searchHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date): Promise; - searchKeyword(userId: string, queryText: string, limit: number, sourceSite?: string): Promise; - searchAtomicFactsHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date): Promise; + searchSimilar(userId: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date, sessionId?: string): Promise; + searchHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date, sessionId?: string): Promise; + searchKeyword(userId: string, queryText: string, limit: number, sourceSite?: string, sessionId?: string): Promise; + searchAtomicFactsHybrid(userId: string, queryText: string, queryEmbedding: number[], limit: number, sourceSite?: string, referenceTime?: Date, sessionId?: string): Promise; findNearDuplicates(userId: string, embedding: number[], threshold: number, limit?: number): Promise; findKeywordCandidates(userId: string, keywords: string[], limit?: number, includeExpired?: boolean): Promise; findTopicCandidates(userId: string, queryEmbedding: number[], limit: number): Promise; findTemporalNeighbors(userId: string, anchorTimestamps: Date[], queryEmbedding: number[], windowMinutes: number, excludeIds: Set, limit: number, referenceTime?: Date): Promise; fetchMemoriesByIds(userId: string, ids: string[], queryEmbedding: number[], referenceTime?: Date, includeExpired?: boolean): Promise; // Workspace variants - searchSimilarInWorkspace(workspaceId: string, queryEmbedding: number[], limit: number, agentScope?: AgentScope, callerAgentId?: string, referenceTime?: Date): Promise; + searchSimilarInWorkspace(workspaceId: string, queryEmbedding: number[], limit: number, agentScope?: AgentScope, callerAgentId?: string, referenceTime?: Date, sessionId?: string): Promise; findNearDuplicatesInWorkspace(workspaceId: string, embedding: number[], threshold: number, limit?: number, agentScope?: AgentScope, callerAgentId?: string): Promise; } diff --git a/src/index.ts b/src/index.ts index 4838e6d..8df6c23 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,19 @@ export { MemoryService, type IngestResult, type RetrievalResult } from './servic export { MemoryRepository, type MemoryRow, type SearchResult, type EpisodeRow, type MemoryMetadata } from './db/memory-repository.js'; export { ClaimRepository } from './db/claim-repository.js'; export { pool } from './db/pool.js'; +export { + migrate, + migrationStatus, + MigrationLockTimeout, + MigrationHistoryMismatch, + type MigrateOptions, + type MigrateResult, + type EmbeddingDimensionStatus, + type EmbeddingDimensionStatusValue, + type MigrationHistoryStatus, + type MigrationStatus, + type MigrationStatusOptions, +} from './db/migration-api.js'; export { config, applyRuntimeConfigUpdates, diff --git a/src/routes/__tests__/admin.test.ts b/src/routes/__tests__/admin.test.ts new file mode 100644 index 0000000..f70946f --- /dev/null +++ b/src/routes/__tests__/admin.test.ts @@ -0,0 +1,122 @@ +/** + * @file Route tests for admin-only smoke-scope cleanup. + * + * The router is mounted on a tiny Express app with `requireBearer` so the + * test covers the real authorization middleware, request validation, the + * allow-pattern guard, and the repository calls without touching Postgres. + */ + +import type { AddressInfo } from 'node:net'; +import type { Server } from 'node:http'; +import express from 'express'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { requireBearer } from '../../middleware/require-bearer.js'; +import { createAdminRouter, type AdminMemoryRepository } from '../admin.js'; + +const ADMIN_KEY = 'admin-test-key'; + +interface MountedAdmin { + baseUrl: string; + repo: AdminMemoryRepository; + server: Server; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +async function mountAdmin(repo?: AdminMemoryRepository): Promise { + const app = express(); + const memory = repo ?? { + countMemories: vi.fn(async () => 2), + deleteAll: vi.fn(async () => undefined), + }; + app.use(express.json()); + app.use( + '/v1/admin', + requireBearer(ADMIN_KEY), + createAdminRouter({ + memory, + testScopeAllowPattern: '^(smoke-|docker-|test-).+', + }), + ); + const server = await new Promise((resolve) => { + const s = app.listen(0, () => resolve(s)); + }); + const { port } = server.address() as AddressInfo; + return { baseUrl: `http://127.0.0.1:${port}`, repo: memory, server }; +} + +async function closeServer(server: Server): Promise { + await new Promise((resolve, reject) => { + server.close((err) => (err ? reject(err) : resolve())); + }); +} + +function adminHeaders(key = ADMIN_KEY): Record { + return { authorization: `Bearer ${key}`, 'content-type': 'application/json' }; +} + +async function deleteScope(baseUrl: string, userId?: string, key = ADMIN_KEY): Promise { + const body = userId === undefined ? {} : { user_id: userId }; + return fetch(`${baseUrl}/v1/admin/scope`, { + method: 'DELETE', + headers: adminHeaders(key), + body: JSON.stringify(body), + }); +} + +describe('DELETE /v1/admin/scope', () => { + it('requires the dedicated admin bearer', async () => { + const mounted = await mountAdmin(); + try { + const res = await deleteScope(mounted.baseUrl, 'smoke-123', 'wrong-key'); + expect(res.status).toBe(401); + expect(mounted.repo.deleteAll).not.toHaveBeenCalled(); + } finally { + await closeServer(mounted.server); + } + }); + + it('rejects user ids outside the configured test-scope pattern', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const mounted = await mountAdmin(); + try { + const res = await deleteScope(mounted.baseUrl, 'real-user'); + const body = (await res.json()) as { error: string }; + expect(res.status).toBe(403); + expect(body.error).toMatch(/CORE_TEST_SCOPE_ALLOW_PATTERN/); + expect(mounted.repo.deleteAll).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('"status":"rejected"')); + } finally { + await closeServer(mounted.server); + } + }); + + it('deletes only the requested allowed scope and reports count', async () => { + const info = vi.spyOn(console, 'info').mockImplementation(() => undefined); + const mounted = await mountAdmin(); + try { + const res = await deleteScope(mounted.baseUrl, 'smoke-123'); + const body = (await res.json()) as { deleted: number }; + expect(res.status).toBe(200); + expect(body.deleted).toBe(2); + expect(mounted.repo.countMemories).toHaveBeenCalledWith('smoke-123'); + expect(mounted.repo.deleteAll).toHaveBeenCalledWith('smoke-123'); + expect(info).toHaveBeenCalledWith(expect.stringContaining('"status":"deleted"')); + } finally { + await closeServer(mounted.server); + } + }); + + it('returns validation errors for missing user_id', async () => { + const mounted = await mountAdmin(); + try { + const res = await deleteScope(mounted.baseUrl); + expect(res.status).toBe(400); + expect(mounted.repo.deleteAll).not.toHaveBeenCalled(); + } finally { + await closeServer(mounted.server); + } + }); +}); diff --git a/src/routes/admin.ts b/src/routes/admin.ts new file mode 100644 index 0000000..4ad0b59 --- /dev/null +++ b/src/routes/admin.ts @@ -0,0 +1,66 @@ +/** + * @file Admin-only test-scope cleanup routes. + * + * These routes are intentionally absent unless the composition root mounts + * them with a dedicated admin bearer and an explicit test-scope allow-pattern. + * They exist for disposable smoke/eval infrastructure that needs to clean up + * user-scoped memory data after external-core runs. + */ + +import { Router, type Request, type Response } from 'express'; +import { z } from '../schemas/zod-setup.js'; +import { validateBody } from '../middleware/validate.js'; +import { handleRouteError } from './route-errors.js'; + +export interface AdminMemoryRepository { + countMemories(userId?: string): Promise; + deleteAll(userId?: string): Promise; +} + +export interface AdminRouterDeps { + memory: AdminMemoryRepository; + testScopeAllowPattern: string; +} + +const DeleteScopeBodySchema = z + .object({ user_id: z.string().trim().min(1) }) + .transform(({ user_id }) => ({ userId: user_id })); + +export function createAdminRouter(deps: AdminRouterDeps): Router { + const router = Router(); + const allowPattern = new RegExp(deps.testScopeAllowPattern); + + router.delete('/scope', validateBody(DeleteScopeBodySchema), async (req: Request, res: Response) => { + try { + const { userId } = req.body as { userId: string }; + if (!allowPattern.test(userId)) { + logCleanup('rejected', userId, deps.testScopeAllowPattern, 0); + res.status(403).json({ error: 'scope rejected by CORE_TEST_SCOPE_ALLOW_PATTERN' }); + return; + } + const before = await deps.memory.countMemories(userId); + await deps.memory.deleteAll(userId); + logCleanup('deleted', userId, deps.testScopeAllowPattern, before); + res.json({ deleted: before }); + } catch (err) { + handleRouteError(res, 'DELETE /v1/admin/scope', err); + } + }); + + return router; +} + +function logCleanup(status: string, userId: string, pattern: string, deleted: number): void { + const line = JSON.stringify({ + event: 'admin.scope_cleanup', + status, + user_id: userId, + allow_pattern: pattern, + deleted, + }); + if (status === 'rejected') { + console.warn(line); + return; + } + console.info(line); +} diff --git a/src/routes/memories.ts b/src/routes/memories.ts index b9318e3..68e1c9e 100644 --- a/src/routes/memories.ts +++ b/src/routes/memories.ts @@ -237,15 +237,45 @@ async function runIngest( mode: 'full' | 'quick', ) { if (body.workspace) { - return service.workspaceIngest(body.userId, body.conversation, body.sourceSite, body.sourceUrl, body.workspace, undefined, effectiveConfig); + return service.workspaceIngest({ + userId: body.userId, + conversationText: body.conversation, + sourceSite: body.sourceSite, + sourceUrl: body.sourceUrl, + workspace: body.workspace, + effectiveConfig, + sessionId: body.sessionId, + }); } if (mode === 'full') { - return service.ingest(body.userId, body.conversation, body.sourceSite, body.sourceUrl, undefined, effectiveConfig); + return service.ingest({ + userId: body.userId, + conversationText: body.conversation, + sourceSite: body.sourceSite, + sourceUrl: body.sourceUrl, + effectiveConfig, + sessionId: body.sessionId, + }); } if (body.skipExtraction) { - return service.storeVerbatim(body.userId, body.conversation, body.sourceSite, body.sourceUrl, body.metadata, effectiveConfig); + return service.storeVerbatim({ + userId: body.userId, + content: body.conversation, + sourceSite: body.sourceSite, + sourceUrl: body.sourceUrl, + metadata: body.metadata, + effectiveConfig, + sessionId: body.sessionId, + }); } - return service.quickIngest(body.userId, body.conversation, body.sourceSite, body.sourceUrl, undefined, effectiveConfig); + return service.quickIngest({ + userId: body.userId, + conversationText: body.conversation, + sourceSite: body.sourceSite, + sourceUrl: body.sourceUrl, + effectiveConfig, + sessionId: body.sessionId, + }); } /** @@ -311,6 +341,7 @@ function registerSearchRoute( limit: requestLimit, asOf: body.asOf, namespaceScope: body.namespaceScope, + sessionId: body.sessionId, retrievalOptions, effectiveConfig, }); @@ -341,6 +372,7 @@ function registerFastSearchRoute( sourceSite: body.sourceSite, limit: requestLimit, namespaceScope: body.namespaceScope, + sessionId: body.sessionId, retrievalOptions, effectiveConfig, }); @@ -404,14 +436,23 @@ function registerListRoute(router: Router, service: MemoryService): void { agentId: string | undefined; sourceSite: string | undefined; episodeId: string | undefined; + sessionId: string | undefined; }; const memories = q.workspaceId - ? await service.scopedList( - { kind: 'workspace', userId: q.userId, workspaceId: q.workspaceId, agentId: q.agentId! }, - q.limit, - q.offset, - ) - : await service.list(q.userId, q.limit, q.offset, q.sourceSite, q.episodeId); + ? await service.scopedList({ + scope: { kind: 'workspace', userId: q.userId, workspaceId: q.workspaceId, agentId: q.agentId! }, + limit: q.limit, + offset: q.offset, + sessionId: q.sessionId, + }) + : await service.list({ + userId: q.userId, + limit: q.limit, + offset: q.offset, + sourceSite: q.sourceSite, + episodeId: q.episodeId, + sessionId: q.sessionId, + }); res.json({ memories, count: memories.length }); } catch (err) { handleRouteError(res, 'GET /v1/memories/list', err); @@ -911,6 +952,7 @@ function formatSearchResponse(result: RetrievalResult, scope: MemoryScope) { relevance: memory.relevance ?? memory.similarity, importance: memory.importance, source_site: memory.source_site, + session_id: memory.session_id, created_at: memory.created_at, metadata: memory.metadata, })), diff --git a/src/schemas/__tests__/memories.test.ts b/src/schemas/__tests__/memories.test.ts index 2106905..e36d2f4 100644 --- a/src/schemas/__tests__/memories.test.ts +++ b/src/schemas/__tests__/memories.test.ts @@ -15,6 +15,7 @@ import { IngestBodySchema, SearchBodySchema, ExpandBodySchema, + ListQuerySchema, ResetSourceBodySchema, LessonReportBodySchema, } from '../memories'; @@ -92,12 +93,69 @@ describe('SearchBodySchema — preserved empty-string pass-through', () => { expect(r.namespaceScope).toBe(''); }); + it('preserves session_id as sessionId', () => { + const r = SearchBodySchema.parse({ + user_id: 'u', + query: 'q', + session_id: 'thread-1', + }); + expect(r.sessionId).toBe('thread-1'); + }); + + it('rejects non-string session_id values', () => { + for (const session_id of [42, ['a'], { x: 1 }]) { + const r = SearchBodySchema.safeParse({ user_id: 'u', query: 'q', session_id }); + expect(firstIssueMessage(r)).toMatch(/session_id/); + } + }); + it('required fields still emit exact prior-parser messages', () => { const r = SearchBodySchema.safeParse({ query: 'q' }); expect(firstIssueMessage(r)).toBe('user_id (string) is required'); }); }); +describe('IngestBodySchema — session scope', () => { + it('preserves session_id as sessionId', () => { + const r = IngestBodySchema.parse({ + user_id: 'u', + conversation: 'x', + source_site: 'sdk', + session_id: 'thread-1', + }); + expect(r.sessionId).toBe('thread-1'); + }); + + it('rejects invalid session_id values', () => { + const base = { + user_id: 'u', + conversation: 'x', + source_site: 'sdk', + }; + for (const session_id of [' ', 'abc\n123', 'x'.repeat(257), 42, ['a'], { x: 1 }]) { + const r = IngestBodySchema.safeParse({ ...base, session_id }); + expect(firstIssueMessage(r)).toMatch(/session_id/); + } + }); +}); + +describe('ListQuerySchema — session scope', () => { + it('preserves session_id as sessionId', () => { + const r = ListQuerySchema.parse({ + user_id: 'u', + session_id: 'thread-1', + }); + expect(r.sessionId).toBe('thread-1'); + }); + + it('rejects invalid session_id values', () => { + for (const session_id of [' ', 'abc\n123', 'x'.repeat(257), 42, ['a'], { x: 1 }]) { + const r = ListQuerySchema.safeParse({ user_id: 'u', session_id }); + expect(firstIssueMessage(r)).toMatch(/session_id/); + } + }); +}); + describe('ExpandBodySchema — preserved error messages', () => { it('missing memory_ids → "memory_ids (string[]) is required"', () => { const r = ExpandBodySchema.safeParse({ user_id: 'u' }); diff --git a/src/schemas/__tests__/openapi-security.test.ts b/src/schemas/__tests__/openapi-security.test.ts index 60a034b..04e6c7b 100644 --- a/src/schemas/__tests__/openapi-security.test.ts +++ b/src/schemas/__tests__/openapi-security.test.ts @@ -36,6 +36,10 @@ describe('OpenAPI security — Bearer scheme on every documented route', () => { type: 'http', scheme: 'bearer', })); + expect(schemes?.adminBearerAuth).toEqual(expect.objectContaining({ + type: 'http', + scheme: 'bearer', + })); }); it('declares a document-level `bearerAuth` security requirement', () => { @@ -57,4 +61,9 @@ describe('OpenAPI security — Bearer scheme on every documented route', () => { expect(p.startsWith('/v1/')).toBe(true); } }); + + it('admin cleanup uses the separate admin bearer scheme', () => { + const adminDelete = doc.paths?.['/v1/admin/scope']?.delete; + expect(adminDelete?.security).toEqual([{ adminBearerAuth: [] }]); + }); }); diff --git a/src/schemas/memories.ts b/src/schemas/memories.ts index c020ff8..d7328a4 100644 --- a/src/schemas/memories.ts +++ b/src/schemas/memories.ts @@ -55,9 +55,11 @@ import { RESERVED_METADATA_KEYS } from '../db/repository-types.js'; const MAX_CONVERSATION_LENGTH = 100_000; const MAX_METADATA_SERIALIZED_BYTES = 32 * 1024; const MAX_SEARCH_LIMIT = 100; +const MAX_SESSION_ID_LENGTH = 256; const MAX_TOKEN_BUDGET = 50_000; const MIN_TOKEN_BUDGET = 100; const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const CONTROL_CHAR_REGEX = /[\u0000-\u001f\u007f]/; // --------------------------------------------------------------------------- // Reusable body-level field schemas @@ -74,6 +76,44 @@ const OptionalBooleanField = (description?: string) => .transform(v => (typeof v === 'boolean' ? v : undefined)) .openapi({ type: 'boolean', ...(description ? { description } : {}) }); +const SessionIdField = z + .unknown() + .superRefine((value, ctx) => { + if (value === undefined || value === null) return; + if (typeof value !== 'string') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'session_id must be a string', + }); + return; + } + if (value.length === 0 || value.trim().length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'session_id must be a non-empty string without control characters', + }); + } + if (value.length > MAX_SESSION_ID_LENGTH) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `session_id exceeds ${MAX_SESSION_ID_LENGTH} characters`, + }); + } + if (CONTROL_CHAR_REGEX.test(value)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'session_id must be a non-empty string without control characters', + }); + } + }) + .transform(value => (typeof value === 'string' ? value : undefined)) + .openapi({ + type: 'string', + maxLength: MAX_SESSION_ID_LENGTH, + description: + 'Optional thread/session identifier used to scope ingest, search, and list symmetrically.', + }); + /** * Build a schema that produces `"${label} (string[]) is required"` for * every failure mode that the old array guard threw on. Matches the @@ -238,6 +278,7 @@ export const IngestBodySchema = z ), source_site: requiredStringBody('source_site'), source_url: OptionalBodyString, + session_id: SessionIdField, workspace_id: WorkspaceIdField, agent_id: AgentIdField, visibility: VisibilityField, @@ -292,6 +333,7 @@ export const IngestBodySchema = z conversation: b.conversation, sourceSite: b.source_site, sourceUrl: b.source_url ?? '', + sessionId: b.session_id, workspace: buildWorkspaceContext(b.workspace_id, b.agent_id, b.visibility), skipExtraction: b.skip_extraction === true, configOverride: b.config_override, @@ -321,6 +363,7 @@ export const SearchBodySchema = z token_budget: TokenBudgetSchema, threshold: SearchThresholdSchema, namespace_scope: OptionalBodyString, + session_id: SessionIdField, skip_repair: OptionalBooleanField(), workspace_id: WorkspaceIdField, agent_id: AgentIdField, @@ -338,6 +381,7 @@ export const SearchBodySchema = z tokenBudget: b.token_budget, relevanceThreshold: b.threshold, namespaceScope: b.namespace_scope, + sessionId: b.session_id, skipRepair: b.skip_repair === true, workspace: buildWorkspaceContext(b.workspace_id, b.agent_id, b.visibility), agentScope: b.agent_scope, @@ -501,6 +545,7 @@ export const ListQuerySchema = z agent_id: OptionalUuidQueryField('agent_id'), source_site: OptionalQueryField(), episode_id: OptionalUuidQueryField('episode_id'), + session_id: SessionIdField, }) .transform(q => ({ userId: q.user_id, @@ -510,6 +555,7 @@ export const ListQuerySchema = z agentId: q.agent_id, sourceSite: q.source_site, episodeId: q.episode_id, + sessionId: q.session_id, })) .refine(q => !(q.workspaceId && !q.agentId), { message: 'agent_id is required for workspace queries', diff --git a/src/schemas/openapi.ts b/src/schemas/openapi.ts index f82f7f0..0e08527 100644 --- a/src/schemas/openapi.ts +++ b/src/schemas/openapi.ts @@ -89,6 +89,14 @@ const TAG_CONFIG = 'Configuration'; const TAG_AGENTS = 'Agents'; const TAG_DOCUMENTS = 'Documents'; const TAG_STORAGE = 'Storage'; +const TAG_ADMIN = 'Admin'; + +const AdminDeleteScopeBodySchema = z.object({ + user_id: z.string().min(1), +}); +const AdminDeleteScopeResponseSchema = z.object({ + deleted: z.number().int().min(0), +}); /** Build and populate the OpenAPI registry. */ export function buildRegistry(): OpenAPIRegistry { @@ -107,11 +115,20 @@ export function buildRegistry(): OpenAPIRegistry { 'The key is the deployment-wide secret configured via ' + "`CORE_API_KEY`; the middleware uses constant-time comparison.", }); + registry.registerComponent('securitySchemes', 'adminBearerAuth', { + type: 'http', + scheme: 'bearer', + description: + 'Send `Authorization: Bearer ` on admin-only ' + + 'cleanup requests. This scheme is separate from normal client auth.', + }); registry.register('ErrorBasic', ErrorBasicSchema); registry.register('ErrorConfig400', ErrorConfig400Schema); registry.register('ErrorConfig410', ErrorConfig410Schema); registry.register('ErrorUpstreamProvider', ErrorUpstreamProviderSchema); + registry.register('AdminDeleteScopeBody', AdminDeleteScopeBodySchema); + registry.register('AdminDeleteScopeResponse', AdminDeleteScopeResponseSchema); registerMemoryCoreRoutes(registry); registerMemoryLifecycleRoutes(registry); @@ -121,6 +138,7 @@ export function buildRegistry(): OpenAPIRegistry { registerAgentRoutes(registry); registerDocumentRoutes(registry); registerStorageRoutes(registry); + registerAdminRoutes(registry); return registry; } @@ -133,6 +151,14 @@ const RESPONSE_400 = { description: 'Input validation error', content: { 'application/json': { schema: ErrorBasicSchema } }, }; +const RESPONSE_401 = { + description: 'Missing or invalid bearer token', + content: { 'application/json': { schema: ErrorBasicSchema } }, +}; +const RESPONSE_403 = { + description: 'Request is authenticated but not allowed', + content: { 'application/json': { schema: ErrorBasicSchema } }, +}; const RESPONSE_500 = { description: 'Internal server error', content: { 'application/json': { schema: ErrorBasicSchema } }, @@ -602,6 +628,38 @@ function registerMemoryConfigRoutes(registry: OpenAPIRegistry): void { }); } +// --------------------------------------------------------------------------- +// /v1/admin — operator-only maintenance routes +// --------------------------------------------------------------------------- + +function registerAdminRoutes(registry: OpenAPIRegistry): void { + registry.registerPath({ + method: 'delete', + path: '/v1/admin/scope', + operationId: 'deleteAdminScope', + tags: [TAG_ADMIN], + summary: 'Delete one allowed disposable test scope.', + description: + 'Mounted only when CORE_ADMIN_API_KEY and CORE_TEST_SCOPE_ALLOW_PATTERN ' + + 'are both configured. The server refuses user_id values that do not ' + + 'match the configured test-scope pattern.', + security: [{ adminBearerAuth: [] }], + request: { + body: { + content: { 'application/json': { schema: AdminDeleteScopeBodySchema } }, + required: true, + }, + }, + responses: { + 200: ok('Number of memories deleted for the requested scope.', AdminDeleteScopeResponseSchema), + 400: RESPONSE_400, + 401: RESPONSE_401, + 403: RESPONSE_403, + 500: RESPONSE_500, + }, + }); +} + // --------------------------------------------------------------------------- // /v1/agents // --------------------------------------------------------------------------- diff --git a/src/services/__tests__/budget-constrained-integration.test.ts b/src/services/__tests__/budget-constrained-integration.test.ts index 956467a..4383954 100644 --- a/src/services/__tests__/budget-constrained-integration.test.ts +++ b/src/services/__tests__/budget-constrained-integration.test.ts @@ -67,18 +67,16 @@ interface RunSearchOptions { async function runSearch(memories: SearchResult[], options: RunSearchOptions) { stubs.pipeline.mockResolvedValue({ filtered: memories, trace: makeTrace(memories) }); - return performSearch( - options.deps ?? createDeps(), - TEST_USER, - options.query ?? 'q', - undefined, undefined, undefined, undefined, undefined, - { + return performSearch(options.deps ?? createDeps(), { + userId: TEST_USER, + query: options.query ?? 'q', + retrievalOptions: { retrievalMode: options.retrievalMode ?? 'tiered', tokenBudget: options.tokenBudget, skipRepairLoop: true, skipReranking: true, }, - ); + }); } describe('budget_constrained contract — integration', () => { diff --git a/src/services/__tests__/canonical-memory-lineage.test.ts b/src/services/__tests__/canonical-memory-lineage.test.ts index 4d447af..ef00096 100644 --- a/src/services/__tests__/canonical-memory-lineage.test.ts +++ b/src/services/__tests__/canonical-memory-lineage.test.ts @@ -89,7 +89,13 @@ describe('canonical memory lineage', () => { clarificationNote: null, }); - await ctx.service.ingest(TEST_USER, updateConversation, 'test', 'https://source/update', updateAt); + await ctx.service.ingest({ + userId: TEST_USER, + conversationText: updateConversation, + sourceSite: 'test', + sourceUrl: 'https://source/update', + sessionTimestamp: updateAt, + }); const updatedMemory = await ctx.repo.getMemory(originalMemory!.id, TEST_USER); const updatedVersion = await ctx.claimRepo.getClaimVersionByMemoryId(TEST_USER, originalMemory!.id); @@ -126,7 +132,13 @@ describe('canonical memory lineage', () => { clarificationNote: null, }); - await ctx.service.ingest(TEST_USER, deleteConversation, 'test', 'https://source/delete', deleteAt); + await ctx.service.ingest({ + userId: TEST_USER, + conversationText: deleteConversation, + sourceSite: 'test', + sourceUrl: 'https://source/delete', + sessionTimestamp: deleteAt, + }); const claim = await ctx.claimRepo.getClaim(originalVersion!.claim_id, TEST_USER); const deleteCmoRow = await pool.query( @@ -171,7 +183,13 @@ describe('canonical memory lineage', () => { clarificationNote: null, }); - await ctx.service.ingest(TEST_USER, deleteConversation, 'test', 'https://source/delete-employer', deleteAt); + await ctx.service.ingest({ + userId: TEST_USER, + conversationText: deleteConversation, + sourceSite: 'test', + sourceUrl: 'https://source/delete-employer', + sessionTimestamp: deleteAt, + }); const claim = await ctx.claimRepo.getClaim(originalVersion!.claim_id, TEST_USER); const tombstoneVersion = await ctx.claimRepo.getClaimVersion(claim!.invalidated_by_version_id!, TEST_USER); @@ -221,7 +239,13 @@ describe('canonical memory lineage', () => { /** Ingest a conversation and return its first memory, version, and raw result. */ async function ingestAndCapture(conversation: string, timestamp: Date, sourceUrl = 'https://source/original') { - const result = await ctx.service.ingest(TEST_USER, conversation, 'test', sourceUrl, timestamp); + const result = await ctx.service.ingest({ + userId: TEST_USER, + conversationText: conversation, + sourceSite: 'test', + sourceUrl, + sessionTimestamp: timestamp, + }); const memory = await ctx.repo.getMemory(result.memoryIds[0], TEST_USER); const version = await ctx.claimRepo.getClaimVersionByMemoryId(TEST_USER, result.memoryIds[0]); return { result, memory, version }; diff --git a/src/services/__tests__/current-state-retrieval-regression.test.ts b/src/services/__tests__/current-state-retrieval-regression.test.ts index cf1eeba..e15d3b8 100644 --- a/src/services/__tests__/current-state-retrieval-regression.test.ts +++ b/src/services/__tests__/current-state-retrieval-regression.test.ts @@ -64,8 +64,18 @@ describe('current-state retrieval regression', () => { }); it('keeps stale Mem0 references out of the top current-state result after switching to AtomicMemory', async () => { - await service.ingest(TEST_USER, OLD_CONVERSATION, 'test', '', new Date('2026-02-01T00:00:00.000Z')); - await service.ingest(TEST_USER, NEW_CONVERSATION, 'test', '', new Date('2026-03-01T00:00:00.000Z')); + await service.ingest({ + userId: TEST_USER, + conversationText: OLD_CONVERSATION, + sourceSite: 'test', + sessionTimestamp: new Date('2026-02-01T00:00:00.000Z'), + }); + await service.ingest({ + userId: TEST_USER, + conversationText: NEW_CONVERSATION, + sourceSite: 'test', + sessionTimestamp: new Date('2026-03-01T00:00:00.000Z'), + }); const current = await service.search( TEST_USER, diff --git a/src/services/__tests__/extraction-meta-fact-integration.test.ts b/src/services/__tests__/extraction-meta-fact-integration.test.ts new file mode 100644 index 0000000..4b66fde --- /dev/null +++ b/src/services/__tests__/extraction-meta-fact-integration.test.ts @@ -0,0 +1,190 @@ +/** + * Integration test — extract → normalize → anchor → enrich → filter pipeline. + * + * Unit tests cover the filter primitive in isolation. This test exercises + * the *wiring* in `extractFacts()` so a regression in the post-process + * chain (e.g. someone re-orders the pipeline and runs filterMetaFacts + * before enrichExtractedFacts) is caught here, not in production. + * + * The LLM module is mocked to return a controlled mixed batch of durable + * facts and meta-facts. The full real post-process runs against that + * output; the assertion is that durable facts survive intact and + * meta-facts are dropped + counted in the telemetry surface. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +// Mock the entire llm module before the SUT imports it. Hoisted by vitest +// per `vi.mock` semantics, so this beats the import order in extraction.ts. +// `vi.hoisted` is required so `chatMock` exists when the factory runs. +const { chatMock } = vi.hoisted(() => ({ chatMock: vi.fn() })); +vi.mock('../llm.js', () => ({ + llm: { + chat: chatMock, + }, +})); + +// Imports after mock so extractFacts picks up the mocked llm. +import { extractFacts } from '../extraction.js'; +import { + getMetaFactDropStats, + resetMetaFactDropStats, +} from '../meta-fact-filter.js'; + +const MIXED_LLM_RESPONSE = JSON.stringify({ + memories: [ + { + statement: 'User prefers oat-milk flat whites for morning coffee.', + headline: 'Prefers oat-milk flat whites', + importance: 0.7, + type: 'preference', + keywords: ['coffee', 'oat-milk', 'flat-white'], + entities: [{ name: 'oat-milk flat white', type: 'product' }], + relations: [], + }, + { + statement: "The user asked for the user's name.", + headline: 'User asked about name', + importance: 0.3, + type: 'knowledge', + keywords: [], + entities: [], + relations: [], + }, + { + statement: 'User works as a software engineer at a startup.', + headline: 'Software engineer at a startup', + importance: 0.8, + type: 'project', + keywords: ['software', 'engineer', 'startup'], + entities: [{ name: 'startup', type: 'organization' }], + relations: [], + }, + { + statement: 'The user is me.', + headline: 'User is me', + importance: 0.2, + type: 'knowledge', + keywords: [], + entities: [], + relations: [], + }, + { + statement: 'As of May 14, 2026, golden retriever is a term mentioned in the conversation.', + headline: 'Golden retriever mentioned', + importance: 0.2, + type: 'knowledge', + keywords: ['golden retriever'], + entities: [], + relations: [], + }, + ], +}); + +describe('extractFacts integration — meta-fact filter wiring', () => { + beforeEach(() => { + chatMock.mockReset(); + chatMock.mockResolvedValue(MIXED_LLM_RESPONSE); + resetMetaFactDropStats(); + vi.spyOn(console, 'info').mockImplementation(() => {}); + vi.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + resetMetaFactDropStats(); + }); + + it('drops meta-facts but preserves durable user facts', async () => { + const result = await extractFacts( + 'User: I love oat-milk flat whites in the morning.\nAssistant: Got it.', + ); + + // Three meta-facts in the mock response (rows 1, 3, 4) should be dropped. + // Two durable facts (rows 0, 2) should remain. + const statements = result.map((f) => f.fact); + expect(statements).toContain('User prefers oat-milk flat whites for morning coffee.'); + expect(statements).toContain('User works as a software engineer at a startup.'); + + // No meta-fact survived. + expect(statements).not.toContain("The user asked for the user's name."); + expect(statements).not.toContain('The user is me.'); + expect(statements).not.toContain( + 'As of May 14, 2026, golden retriever is a term mentioned in the conversation.', + ); + }); + + it('updates drop telemetry by source=extract', async () => { + await extractFacts('User: short input.'); + + const stats = getMetaFactDropStats(); + expect(stats.total).toBe(3); + // Pattern 0 (the user asked/requested/said/is asking/is me) fires + // for two of the three meta-facts; pattern 1 (As of ..., X is a + // term mentioned ...) fires for the third. + expect(stats.byPattern[0]).toBe(2); + expect(stats.byPattern[1]).toBe(1); + }); + + it('is a no-op when ATOMICMEMORY_META_FACT_FILTER is disabled at runtime', async () => { + // Stash + flip the env flag before calling. + const prev = process.env.ATOMICMEMORY_META_FACT_FILTER; + process.env.ATOMICMEMORY_META_FACT_FILTER = 'off'; + try { + const result = await extractFacts('User: anything.'); + const statements = result.map((f) => f.fact); + // All five extracted facts should pass through unchanged. + expect(statements).toHaveLength(5); + expect(statements).toContain("The user asked for the user's name."); + // And the drop counter stays at zero. + expect(getMetaFactDropStats().total).toBe(0); + } finally { + if (prev === undefined) delete process.env.ATOMICMEMORY_META_FACT_FILTER; + else process.env.ATOMICMEMORY_META_FACT_FILTER = prev; + } + }); + + it('survives an empty LLM response without throwing', async () => { + chatMock.mockResolvedValue(''); + const result = await extractFacts('User: hi'); + expect(result).toEqual([]); + expect(getMetaFactDropStats().total).toBe(0); + }); + + it('survives a non-JSON LLM response without throwing', async () => { + chatMock.mockResolvedValue('the model rambled instead of returning JSON'); + const result = await extractFacts('User: hi'); + expect(result).toEqual([]); + expect(getMetaFactDropStats().total).toBe(0); + }); + + it('survives an all-meta-fact LLM response by returning an empty list', async () => { + chatMock.mockResolvedValue( + JSON.stringify({ + memories: [ + { + statement: "The user asked for the user's name.", + headline: 'meta', + importance: 0.2, + type: 'knowledge', + keywords: [], + entities: [], + relations: [], + }, + { + statement: 'The user is me.', + headline: 'meta', + importance: 0.2, + type: 'knowledge', + keywords: [], + entities: [], + relations: [], + }, + ], + }), + ); + const result = await extractFacts('User: hi'); + expect(result).toEqual([]); + expect(getMetaFactDropStats().total).toBe(2); + }); +}); diff --git a/src/services/__tests__/memory-search-runtime-config.test.ts b/src/services/__tests__/memory-search-runtime-config.test.ts index a2dc405..bfa8c16 100644 --- a/src/services/__tests__/memory-search-runtime-config.test.ts +++ b/src/services/__tests__/memory-search-runtime-config.test.ts @@ -98,7 +98,10 @@ describe('performSearch runtime config seam', () => { auditLoggingEnabled: false, }; - const result = await performSearch(createDeps(runtimeConfig), 'user-1', 'find alpha'); + const result = await performSearch(createDeps(runtimeConfig), { + userId: 'user-1', + query: 'find alpha', + }); expect(result.memories).toHaveLength(1); expect(mockRunSearchPipelineWithTrace.mock.calls[0]?.[6]?.runtimeConfig).toBe(runtimeConfig); diff --git a/src/services/__tests__/memory-service-config.test.ts b/src/services/__tests__/memory-service-config.test.ts index d119d7d..80d08d3 100644 --- a/src/services/__tests__/memory-service-config.test.ts +++ b/src/services/__tests__/memory-service-config.test.ts @@ -6,17 +6,19 @@ * override is provided. */ -import { describe, expect, it, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; const { mockPerformSearch, mockPerformIngest, mockPerformQuickIngest, + mockPerformStoreVerbatim, mockPerformWorkspaceIngest, } = vi.hoisted(() => ({ mockPerformSearch: vi.fn(), mockPerformIngest: vi.fn(), mockPerformQuickIngest: vi.fn(), + mockPerformStoreVerbatim: vi.fn(), mockPerformWorkspaceIngest: vi.fn(), })); @@ -31,7 +33,7 @@ vi.mock('../../config.js', () => ({ config: moduleConfig })); vi.mock('../memory-ingest.js', () => ({ performIngest: mockPerformIngest, performQuickIngest: mockPerformQuickIngest, - performStoreVerbatim: vi.fn(), + performStoreVerbatim: mockPerformStoreVerbatim, performWorkspaceIngest: mockPerformWorkspaceIngest, })); vi.mock('../memory-search.js', () => ({ @@ -50,6 +52,10 @@ vi.mock('../atomicmem-uri.js', () => ({ const { MemoryService } = await import('../memory-service.js'); describe('MemoryService config seam', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + it('threads an explicit runtime config into delegated search deps', async () => { const runtimeConfig = { lessonsEnabled: false, @@ -77,14 +83,7 @@ describe('MemoryService config seam', () => { expect(mockPerformSearch).toHaveBeenCalledWith( expect.objectContaining({ config: runtimeConfig }), - 'user-1', - 'config seam query', - undefined, - undefined, - undefined, - undefined, - undefined, - undefined, + expect.objectContaining({ userId: 'user-1', query: 'config seam query' }), ); }); @@ -112,7 +111,11 @@ describe('MemoryService config seam', () => { runtimeConfig as any, ); - await service.ingest('user-1', 'text', 'site'); + await service.ingest({ + userId: 'user-1', + conversationText: 'text', + sourceSite: 'site', + }); expect(mockPerformIngest).toHaveBeenCalledWith( expect.objectContaining({ config: runtimeConfig }), @@ -121,6 +124,34 @@ describe('MemoryService config seam', () => { 'site', '', undefined, + undefined, + ); + }); + + it('forwards named ingest input without positional ambiguity', async () => { + const effectiveConfig = { ...moduleConfig, ingestTraceEnabled: true }; + const sessionTimestamp = new Date('2026-05-16T12:00:00.000Z'); + mockPerformIngest.mockResolvedValue({ episodeId: 'ep-1' }); + const service = new MemoryService({} as any, {} as any); + + await service.ingest({ + userId: 'user-1', + conversationText: 'text', + sourceSite: 'site', + sourceUrl: 'https://example.test/thread', + sessionTimestamp, + sessionId: 'thread-1', + effectiveConfig: effectiveConfig as any, + }); + + expect(mockPerformIngest).toHaveBeenCalledWith( + expect.objectContaining({ config: effectiveConfig }), + 'user-1', + 'text', + 'site', + 'https://example.test/thread', + sessionTimestamp, + 'thread-1', ); }); @@ -148,7 +179,11 @@ describe('MemoryService config seam', () => { runtimeConfig as any, ); - await service.quickIngest('user-1', 'text', 'site'); + await service.quickIngest({ + userId: 'user-1', + conversationText: 'text', + sourceSite: 'site', + }); expect(mockPerformQuickIngest).toHaveBeenCalledWith( expect.objectContaining({ config: runtimeConfig }), @@ -157,6 +192,61 @@ describe('MemoryService config seam', () => { 'site', '', undefined, + undefined, + ); + }); + + it('forwards named quick-ingest input without positional ambiguity', async () => { + const effectiveConfig = { ...moduleConfig, entropyGateEnabled: false }; + const sessionTimestamp = new Date('2026-05-16T12:30:00.000Z'); + mockPerformQuickIngest.mockResolvedValue({ episodeId: 'ep-1' }); + const service = new MemoryService({} as any, {} as any); + + await service.quickIngest({ + userId: 'user-1', + conversationText: 'quick text', + sourceSite: 'quick-site', + sourceUrl: 'https://example.test/quick', + sessionTimestamp, + sessionId: 'thread-quick', + effectiveConfig: effectiveConfig as any, + }); + + expect(mockPerformQuickIngest).toHaveBeenCalledWith( + expect.objectContaining({ config: effectiveConfig }), + 'user-1', + 'quick text', + 'quick-site', + 'https://example.test/quick', + sessionTimestamp, + 'thread-quick', + ); + }); + + it('forwards named store-verbatim input without positional ambiguity', async () => { + const effectiveConfig = { ...moduleConfig, ingestTraceEnabled: true }; + const metadata = { source: 'test' }; + mockPerformStoreVerbatim.mockResolvedValue({ episodeId: 'ep-1' }); + const service = new MemoryService({} as any, {} as any); + + await service.storeVerbatim({ + userId: 'user-1', + content: 'verbatim text', + sourceSite: 'verbatim-site', + sourceUrl: 'https://example.test/verbatim', + metadata, + sessionId: 'thread-verbatim', + effectiveConfig: effectiveConfig as any, + }); + + expect(mockPerformStoreVerbatim).toHaveBeenCalledWith( + expect.objectContaining({ config: effectiveConfig }), + 'user-1', + 'verbatim text', + 'verbatim-site', + 'https://example.test/verbatim', + metadata, + 'thread-verbatim', ); }); @@ -189,16 +279,27 @@ describe('MemoryService config seam', () => { runtimeConfig as any, ); - await service.workspaceIngest('user-1', 'text', 'site', '', workspace as any); + const sessionTimestamp = new Date('2026-05-16T13:00:00.000Z'); + + await service.workspaceIngest({ + userId: 'user-1', + conversationText: 'text', + sourceSite: 'site', + sourceUrl: 'https://example.test/workspace', + sessionTimestamp, + sessionId: 'thread-workspace', + workspace: workspace as any, + }); expect(mockPerformWorkspaceIngest).toHaveBeenCalledWith( expect.objectContaining({ config: runtimeConfig }), 'user-1', 'text', 'site', - '', + 'https://example.test/workspace', workspace, - undefined, + sessionTimestamp, + 'thread-workspace', ); }); @@ -216,14 +317,7 @@ describe('MemoryService config seam', () => { expect(mockPerformSearch).toHaveBeenCalledWith( expect.objectContaining({ config: moduleConfig }), - 'user-1', - 'default config query', - undefined, - undefined, - undefined, - undefined, - undefined, - undefined, + expect.objectContaining({ userId: 'user-1', query: 'default config query' }), ); }); }); diff --git a/src/services/__tests__/meta-fact-filter.test.ts b/src/services/__tests__/meta-fact-filter.test.ts new file mode 100644 index 0000000..36fa0bd --- /dev/null +++ b/src/services/__tests__/meta-fact-filter.test.ts @@ -0,0 +1,262 @@ +/** + * Tests for meta-fact-filter. + * + * Covers: + * - isMetaFactStatement against the partner-demo / AlignBench distractor pool + * - metaFactFilterEnabled env-flag resolution + * - filterMetaFacts end-to-end with onDrop telemetry + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + DEFAULT_META_FACT_PATTERNS, + filterMetaFacts, + getMetaFactDropStats, + isMetaFactStatement, + metaFactFilterEnabled, + resetMetaFactDropStats, + type MetaFactCandidate, +} from '../meta-fact-filter.js'; + +describe('isMetaFactStatement', () => { + it.each([ + "The user asked for the user's name.", + 'The user is asking a question.', + 'The user is me.', + 'The user requested information.', + 'The user said something.', + 'As of May 14, 2026, Apollo is a term mentioned in the conversation.', + 'As of January 2026, the user is a term mentioned in the conversation.', + 'A name was mentioned in the conversation.', + 'The conversation involves the user.', + 'The user has started a conversation.', + ])('matches the meta-fact shape: "%s"', (statement) => { + expect(isMetaFactStatement(statement)).toBe(true); + }); + + it.each([ + "User's name is SgtPooki", + 'The user lives in Lisbon.', + "The user's dog is named Apollo.", + 'As of January 2026, the user lives in Lisbon.', + 'The user prefers oat milk in coffee.', + ])('does not match a durable user fact: "%s"', (statement) => { + expect(isMetaFactStatement(statement)).toBe(false); + }); + + it('is case-insensitive', () => { + expect(isMetaFactStatement('THE USER IS ME.')).toBe(true); + expect(isMetaFactStatement('the user asked for the user\'s name.')).toBe(true); + }); + + it.each([null, undefined, 42, {}, [], ''])( + 'returns false on non-string / empty input (%s)', + (input) => { + expect(isMetaFactStatement(input as unknown)).toBe(false); + }, + ); +}); + +describe('metaFactFilterEnabled', () => { + it('defaults to true when env is empty', () => { + expect(metaFactFilterEnabled({})).toBe(true); + }); + + it.each(['off', 'OFF', 'false', '0', 'disabled', ' off '])( + 'disables when ATOMICMEMORY_META_FACT_FILTER=%s', + (raw) => { + expect(metaFactFilterEnabled({ ATOMICMEMORY_META_FACT_FILTER: raw })).toBe(false); + }, + ); + + it.each(['on', 'true', '1', 'enabled', 'yes', ''])( + 'keeps enabled for non-disable values like "%s"', + (raw) => { + expect(metaFactFilterEnabled({ ATOMICMEMORY_META_FACT_FILTER: raw })).toBe(true); + }, + ); +}); + +describe('filterMetaFacts', () => { + const facts: MetaFactCandidate[] = [ + { fact: "User's name is example", }, + { fact: "The user asked for the user's name.", }, + { fact: 'The user is me.', }, + { fact: 'The user lives in some city.', }, + { fact: 'As of May 14, 2026, Apollo is a term mentioned in the conversation.', }, + ]; + + it('is a no-op when explicitly disabled', () => { + const out = filterMetaFacts(facts, { enabled: false }); + expect(out).toEqual(facts); + expect(out).not.toBe(facts); // shallow copy + }); + + it('drops the three meta-facts by default', () => { + const out = filterMetaFacts(facts, { enabled: true }); + expect(out.map((f) => f.fact)).toEqual([ + "User's name is example", + 'The user lives in some city.', + ]); + }); + + it('falls back to .statement when .fact is missing (raw LLM shape)', () => { + const rawShape: MetaFactCandidate[] = [ + { statement: 'A durable user fact.' }, + { statement: 'The user is me.' }, + ]; + const out = filterMetaFacts(rawShape, { enabled: true }); + expect(out).toHaveLength(1); + expect(out[0].statement).toBe('A durable user fact.'); + }); + + it('invokes onDrop once per dropped fact', () => { + const dropped: string[] = []; + filterMetaFacts(facts, { + enabled: true, + onDrop: (text) => dropped.push(text), + }); + expect(dropped).toEqual([ + "The user asked for the user's name.", + 'The user is me.', + 'As of May 14, 2026, Apollo is a term mentioned in the conversation.', + ]); + }); + + it('swallows onDrop exceptions so extraction never breaks', () => { + const out = filterMetaFacts(facts, { + enabled: true, + onDrop: () => { + throw new Error('telemetry blew up'); + }, + }); + expect(out).toHaveLength(2); + }); + + it('honours custom patterns (replaces defaults)', () => { + const out = filterMetaFacts(facts, { + enabled: true, + patterns: [/^User's name/], + }); + // The custom rule drops only the literal "User's name" fact; defaults + // are NOT applied when a custom set is provided. + expect(out.map((f) => f.fact)).toEqual([ + "The user asked for the user's name.", + 'The user is me.', + 'The user lives in some city.', + 'As of May 14, 2026, Apollo is a term mentioned in the conversation.', + ]); + }); + + it('returns input unchanged when pattern set is empty', () => { + const out = filterMetaFacts(facts, { enabled: true, patterns: [] }); + expect(out).toEqual(facts); + }); + + it('handles missing/non-string fact fields gracefully', () => { + const weird: MetaFactCandidate[] = [ + { fact: 'The user lives in some city.' }, + { fact: undefined }, + { fact: null as unknown as string }, + { /* no fact field at all */ }, + ]; + const out = filterMetaFacts(weird, { enabled: true }); + // Real fact + the three text-less entries survive (we only drop on a positive match). + expect(out).toHaveLength(4); + }); + + it('preserves order of kept facts', () => { + const ordered: MetaFactCandidate[] = [ + { fact: 'fact-a' }, + { fact: 'The user is me.' }, + { fact: 'fact-b' }, + { fact: "The user asked for the user's name." }, + { fact: 'fact-c' }, + ]; + const out = filterMetaFacts(ordered, { enabled: true }); + expect(out.map((f) => f.fact)).toEqual(['fact-a', 'fact-b', 'fact-c']); + }); +}); + +describe('DEFAULT_META_FACT_PATTERNS shape', () => { + it('is frozen', () => { + expect(Object.isFrozen(DEFAULT_META_FACT_PATTERNS)).toBe(true); + }); + it('contains the five anchored families', () => { + expect(DEFAULT_META_FACT_PATTERNS).toHaveLength(5); + }); +}); + +describe('drop telemetry (counters + structured log)', () => { + let infoSpy: ReturnType; + + beforeEach(() => { + resetMetaFactDropStats(); + infoSpy = vi.spyOn(console, 'info').mockImplementation(() => {}); + }); + afterEach(() => { + infoSpy.mockRestore(); + resetMetaFactDropStats(); + }); + + it('starts with all counters at zero', () => { + const stats = getMetaFactDropStats(); + expect(stats.total).toBe(0); + expect(stats.byPattern.every((n) => n === 0)).toBe(true); + }); + + it('increments per-pattern + total counters by default', () => { + filterMetaFacts( + [ + { fact: "The user asked for the user's name." }, // pattern 0 + { fact: 'The user is me.' }, // pattern 0 + { fact: 'A name was mentioned.' }, // pattern 2 + { fact: 'Durable fact, untouched.' }, + ], + { enabled: true, source: 'unit' }, + ); + const stats = getMetaFactDropStats(); + expect(stats.total).toBe(3); + expect(stats.byPattern[0]).toBe(2); + expect(stats.byPattern[2]).toBe(1); + }); + + it('emits structured drop lines tagged with source', () => { + filterMetaFacts( + [{ fact: 'The user is me.' }], + { enabled: true, source: 'migration' }, + ); + expect(infoSpy).toHaveBeenCalledTimes(1); + const line = infoSpy.mock.calls[0]![0]; + expect(typeof line).toBe('string'); + expect(line).toMatch(/^\[meta-fact-filter\] dropped pattern=\d+ len=\d+ source=migration$/); + }); + + it('explicit onDrop: null suppresses both counters and structured log', () => { + filterMetaFacts( + [{ fact: 'The user is me.' }], + { enabled: true, onDrop: null, source: 'unit' }, + ); + expect(getMetaFactDropStats().total).toBe(0); + expect(infoSpy).not.toHaveBeenCalled(); + }); + + it('custom onDrop function suppresses default telemetry', () => { + const seen: number[] = []; + filterMetaFacts( + [{ fact: 'The user is me.' }], + { enabled: true, onDrop: (_, i) => seen.push(i), source: 'unit' }, + ); + expect(seen).toEqual([0]); + // Counters NOT bumped — custom hook owns observability now. + expect(getMetaFactDropStats().total).toBe(0); + expect(infoSpy).not.toHaveBeenCalled(); + }); + + it('resetMetaFactDropStats clears all counters', () => { + filterMetaFacts([{ fact: 'The user is me.' }], { enabled: true }); + expect(getMetaFactDropStats().total).toBe(1); + resetMetaFactDropStats(); + expect(getMetaFactDropStats().total).toBe(0); + }); +}); diff --git a/src/services/__tests__/msr-search-integration.test.ts b/src/services/__tests__/msr-search-integration.test.ts index 4dafd25..524a345 100644 --- a/src/services/__tests__/msr-search-integration.test.ts +++ b/src/services/__tests__/msr-search-integration.test.ts @@ -141,7 +141,7 @@ describe('performSearch — MSR aggregator integration', () => { }); mockLlmChat.mockResolvedValue('Discussed forecast and alerts.'); - const result = await performSearch(createDeps(true), 'user-1', MSR_QUERY); + const result = await performSearch(createDeps(true), { userId: 'user-1', query: MSR_QUERY }); expect(result.injectionText).toContain('## CROSS-SESSION SUMMARY'); expect(result.injectionText).toContain('## CONVERSATION 1 SUMMARY'); @@ -165,7 +165,7 @@ describe('performSearch — MSR aggregator integration', () => { questionType: 0, }); - const result = await performSearch(createDeps(false), 'user-1', MSR_QUERY); + const result = await performSearch(createDeps(false), { userId: 'user-1', query: MSR_QUERY }); expect(result.injectionText).not.toContain('## CROSS-SESSION SUMMARY'); expect(mockLlmChat).not.toHaveBeenCalled(); @@ -185,11 +185,10 @@ describe('performSearch — MSR aggregator integration', () => { questionType: 0, }); - const result = await performSearch( - createDeps(true), - 'user-1', - "What's the latest version of my dashboard?", - ); + const result = await performSearch(createDeps(true), { + userId: 'user-1', + query: "What's the latest version of my dashboard?", + }); expect(result.injectionText).not.toContain('## CROSS-SESSION SUMMARY'); expect(mockLlmChat).not.toHaveBeenCalled(); diff --git a/src/services/__tests__/retrieval-relevance-regression.test.ts b/src/services/__tests__/retrieval-relevance-regression.test.ts index bd7fc38..d548b99 100644 --- a/src/services/__tests__/retrieval-relevance-regression.test.ts +++ b/src/services/__tests__/retrieval-relevance-regression.test.ts @@ -65,17 +65,12 @@ describe('retrieval relevance regression', () => { const trace = createTrace(fixture.all.map((memory) => memory.id)); mockRunSearchPipelineWithTrace.mockResolvedValue({ filtered: fixture.all, trace, queryEmbedding: [1, 0, 0] }); - const result = await performSearch( - createDeps(0.5), - TEST_USER, - 'What is my favorite color?', - undefined, - 5, - undefined, - undefined, - undefined, - { skipRepairLoop: true, skipReranking: true }, - ); + const result = await performSearch(createDeps(0.5), { + userId: TEST_USER, + query: 'What is my favorite color?', + limit: 5, + retrievalOptions: { skipRepairLoop: true, skipReranking: true }, + }); const ids = result.memories.map((memory) => memory.id); expect(ids).toEqual([fixture.answer.id]); @@ -102,17 +97,12 @@ describe('retrieval relevance regression', () => { queryEmbedding: [1, 0, 0], }); - const result = await performSearch( - createDeps(0.1), - TEST_USER, - 'What is my favorite color?', - undefined, - 5, - undefined, - undefined, - undefined, - { relevanceThreshold: 0.5, skipRepairLoop: true, skipReranking: true }, - ); + const result = await performSearch(createDeps(0.1), { + userId: TEST_USER, + query: 'What is my favorite color?', + limit: 5, + retrievalOptions: { relevanceThreshold: 0.5, skipRepairLoop: true, skipReranking: true }, + }); expect(result.memories.map((memory) => memory.id)).toEqual([fixture.answer.id]); }); @@ -122,7 +112,10 @@ describe('retrieval relevance regression', () => { const trace = createTrace(fixture.all.map((memory) => memory.id)); mockRunSearchPipelineWithTrace.mockResolvedValue({ filtered: fixture.all, trace, queryEmbedding: [1, 0, 0] }); - await performSearch(createDeps(0.5), TEST_USER, 'What is my favorite color?'); + await performSearch(createDeps(0.5), { + userId: TEST_USER, + query: 'What is my favorite color?', + }); expect(trace.stage).toHaveBeenCalledWith( 'relevance-filter', @@ -167,7 +160,10 @@ describe('retrieval relevance regression', () => { const trace = createTrace([driverBlog.id, twitterishLocal.id]); mockRunSearchPipelineWithTrace.mockResolvedValue({ filtered: [driverBlog, twitterishLocal], trace }); - await performSearch(createDeps(0.5), TEST_USER, 'What is my favorite color?'); + await performSearch(createDeps(0.5), { + userId: TEST_USER, + query: 'What is my favorite color?', + }); expect(trace.stage).toHaveBeenCalledWith( 'relevance-filter', @@ -242,7 +238,7 @@ async function expectRecallPreservedForQuery(query: string) { queryEmbedding: [1, 0, 0], }); - const result = await performSearch(createDeps(0.5), TEST_USER, query); + const result = await performSearch(createDeps(0.5), { userId: TEST_USER, query }); expect(result.memories.map((memory) => memory.id)).toEqual(fixture.all.map((memory) => memory.id)); } diff --git a/src/services/__tests__/scoped-dispatch.test.ts b/src/services/__tests__/scoped-dispatch.test.ts index abbfed4..92030dc 100644 --- a/src/services/__tests__/scoped-dispatch.test.ts +++ b/src/services/__tests__/scoped-dispatch.test.ts @@ -16,6 +16,10 @@ const { mockExpandMemories, mockExpandInWorkspace } = vi.hoisted(() => ({ mockExpandMemories: vi.fn(), mockExpandInWorkspace: vi.fn(), })); +const { mockListMemories, mockListInWorkspace } = vi.hoisted(() => ({ + mockListMemories: vi.fn(), + mockListInWorkspace: vi.fn(), +})); vi.mock('../memory-search.js', () => ({ performSearch: mockPerformSearch, @@ -31,8 +35,8 @@ vi.mock('../memory-ingest.js', () => ({ vi.mock('../memory-crud.js', () => ({ expandMemories: mockExpandMemories, expandMemoriesInWorkspace: mockExpandInWorkspace, - listMemories: vi.fn(), - listMemoriesInWorkspace: vi.fn(), + listMemories: mockListMemories, + listMemoriesInWorkspace: mockListInWorkspace, getMemory: vi.fn(), getMemoryInWorkspace: vi.fn(), deleteMemory: vi.fn(), @@ -78,13 +82,16 @@ describe('scopedSearch', () => { it('dispatches user scope to performSearch', async () => { await service.scopedSearch({ kind: 'user', userId: 'u1' }, 'query', { sourceSite: 'test', limit: 5 }); - expect(mockPerformSearch).toHaveBeenCalledWith(expect.anything(), 'u1', 'query', 'test', 5, undefined, undefined, undefined, undefined); + expect(mockPerformSearch).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ userId: 'u1', query: 'query', sourceSite: 'test', limit: 5 }), + ); expect(mockPerformWorkspaceSearch).not.toHaveBeenCalled(); }); it('dispatches user scope with fast option to performFastSearch', async () => { await service.scopedSearch({ kind: 'user', userId: 'u1' }, 'query', { fast: true, sourceSite: 'test', limit: 10 }); - expect(mockPerformFastSearch).toHaveBeenCalledWith(expect.anything(), 'u1', 'query', 'test', 10, undefined, undefined); + expect(mockPerformFastSearch).toHaveBeenCalledWith(expect.anything(), 'u1', 'query', 'test', 10, undefined, undefined, undefined); expect(mockPerformSearch).not.toHaveBeenCalled(); }); @@ -100,6 +107,70 @@ describe('scopedSearch', () => { ); expect(mockPerformSearch).not.toHaveBeenCalled(); }); + + it('forwards workspace sessionId to performWorkspaceSearch', async () => { + await service.scopedSearch( + { kind: 'workspace', userId: 'u1', workspaceId: 'ws1', agentId: 'a1' }, + 'query', + { sessionId: 'thread-1' }, + ); + expect(mockPerformWorkspaceSearch).toHaveBeenCalledWith( + expect.anything(), + 'u1', + 'query', + { workspaceId: 'ws1', agentId: 'a1' }, + expect.objectContaining({ sessionId: 'thread-1' }), + ); + }); +}); + +describe('scopedList', () => { + let service: InstanceType; + + beforeEach(() => { + vi.clearAllMocks(); + service = new MemoryService({} as any, {} as any); + mockListMemories.mockResolvedValue([]); + mockListInWorkspace.mockResolvedValue([]); + }); + + it('forwards user list options to listMemories', async () => { + await service.list({ + userId: 'u1', + limit: 7, + offset: 2, + sourceSite: 'site-1', + episodeId: 'episode-1', + sessionId: 'thread-user', + }); + + expect(mockListMemories).toHaveBeenCalledWith( + expect.anything(), + 'u1', + 7, + 2, + 'site-1', + 'episode-1', + 'thread-user', + ); + }); + + it('forwards workspace sessionId to listMemoriesInWorkspace', async () => { + await service.scopedList({ + scope: { kind: 'workspace', userId: 'u1', workspaceId: 'ws1', agentId: 'a1' }, + limit: 20, + offset: 0, + sessionId: 'thread-1', + }); + expect(mockListInWorkspace).toHaveBeenCalledWith( + expect.anything(), + 'ws1', + 20, + 0, + 'a1', + 'thread-1', + ); + }); }); describe('scopedExpand', () => { diff --git a/src/services/__tests__/temporal-mutation-regression.test.ts b/src/services/__tests__/temporal-mutation-regression.test.ts index 7448c76..31cab6c 100644 --- a/src/services/__tests__/temporal-mutation-regression.test.ts +++ b/src/services/__tests__/temporal-mutation-regression.test.ts @@ -50,7 +50,12 @@ describe('temporal mutation regression', () => { registerConversation(oldConversation, oldFact, backendBase); registerConversation(newConversation, newFact, offsetVector(backendBase, 17, 0.01)); - const oldResult = await ctx.service.ingest(TEST_USER, oldConversation, 'test', '', oldAt); + const oldResult = await ctx.service.ingest({ + userId: TEST_USER, + conversationText: oldConversation, + sourceSite: 'test', + sessionTimestamp: oldAt, + }); decisionPlans.set(newFact, { action: 'SUPERSEDE', targetMemoryId: oldResult.memoryIds[0], @@ -59,7 +64,12 @@ describe('temporal mutation regression', () => { clarificationNote: null, }); - const newResult = await ctx.service.ingest(TEST_USER, newConversation, 'test', '', newAt); + const newResult = await ctx.service.ingest({ + userId: TEST_USER, + conversationText: newConversation, + sourceSite: 'test', + sessionTimestamp: newAt, + }); const queryEmbedding = offsetVector(backendBase, 31, 0.002); const currentResults = await ctx.repo.searchSimilar(TEST_USER, queryEmbedding, 5, 'test'); const oldVersion = await ctx.claimRepo.getClaimVersionByMemoryId(TEST_USER, oldResult.memoryIds[0]); @@ -101,7 +111,12 @@ describe('temporal mutation regression', () => { registerConversation(oldConversation, oldFact, salaryBase); registerConversation(newConversation, updatedFact, offsetVector(salaryBase, 23, 0.01)); - const original = await ctx.service.ingest(TEST_USER, oldConversation, 'test', '', oldAt); + const original = await ctx.service.ingest({ + userId: TEST_USER, + conversationText: oldConversation, + sourceSite: 'test', + sessionTimestamp: oldAt, + }); decisionPlans.set(updatedFact, { action: 'UPDATE', targetMemoryId: original.memoryIds[0], @@ -110,7 +125,12 @@ describe('temporal mutation regression', () => { clarificationNote: null, }); - await ctx.service.ingest(TEST_USER, newConversation, 'test', '', updateAt); + await ctx.service.ingest({ + userId: TEST_USER, + conversationText: newConversation, + sourceSite: 'test', + sessionTimestamp: updateAt, + }); const queryEmbedding = offsetVector(salaryBase, 29, 0.002); const currentResults = await ctx.repo.searchSimilar(TEST_USER, queryEmbedding, 5, 'test'); @@ -153,7 +173,12 @@ describe('temporal mutation regression', () => { registerConversation(originalConversation, originalFact, birthdayBase); registerConversation(clarifyConversation, conflictingFact, offsetVector(birthdayBase, 43, 0.008)); - const original = await ctx.service.ingest(TEST_USER, originalConversation, 'test', '', originalAt); + const original = await ctx.service.ingest({ + userId: TEST_USER, + conversationText: originalConversation, + sourceSite: 'test', + sessionTimestamp: originalAt, + }); const originalVersion = await ctx.claimRepo.getClaimVersionByMemoryId(TEST_USER, original.memoryIds[0]); decisionPlans.set(conflictingFact, { action: 'CLARIFY', @@ -163,7 +188,12 @@ describe('temporal mutation regression', () => { clarificationNote: 'Conflicting birthday requires confirmation.', }); - await ctx.service.ingest(TEST_USER, clarifyConversation, 'test', '', clarifyAt); + await ctx.service.ingest({ + userId: TEST_USER, + conversationText: clarifyConversation, + sourceSite: 'test', + sessionTimestamp: clarifyAt, + }); const clarificationRows = await pool.query( `SELECT content, status, created_at, observed_at, metadata diff --git a/src/services/__tests__/tll-augmentation-integration.test.ts b/src/services/__tests__/tll-augmentation-integration.test.ts index ea04a79..100fb84 100644 --- a/src/services/__tests__/tll-augmentation-integration.test.ts +++ b/src/services/__tests__/tll-augmentation-integration.test.ts @@ -76,7 +76,10 @@ describe('TLL augmentation through performSearch', () => { trace: createTrace([SEED_ID]), }); - const result = await performSearch(createDeps({ chainIds: [CHAIN_ID] }, 0.65), TEST_USER, ORDERING_QUERY); + const result = await performSearch(createDeps({ chainIds: [CHAIN_ID] }, 0.65), { + userId: TEST_USER, + query: ORDERING_QUERY, + }); const ids = result.memories.map((m) => m.id); expect(ids).toContain(SEED_ID); @@ -103,7 +106,7 @@ describe('TLL augmentation through performSearch', () => { }); const deps = createDeps({ chainIds: [CHAIN_ID_A, CHAIN_ID_B, CHAIN_ID_C] }, 0.65); - await performSearch(deps, TEST_USER, ORDERING_QUERY); + await performSearch(deps, { userId: TEST_USER, query: ORDERING_QUERY }); const hydrateCall = deps.stores.pool.query.mock.calls.find( ([sql]: [string]) => /JOIN\s+memories/i.test(sql), @@ -122,8 +125,7 @@ describe('TLL augmentation through performSearch', () => { const result = await performSearch( createDeps({ chainIds: [WORKSPACE_LEAK_ID, CHAIN_ID], leakWorkspaceId: WORKSPACE_ID }, 0.65), - TEST_USER, - ORDERING_QUERY, + { userId: TEST_USER, query: ORDERING_QUERY }, ); const ids = result.memories.map((m) => m.id); @@ -146,12 +148,11 @@ describe('TLL augmentation through performSearch', () => { trace: createTrace([SEED_ID]), }); - const result = await performSearch( - createDeps({ chainIds: [CHAIN_ID] }, 0.65), - TEST_USER, ORDERING_QUERY, - undefined, undefined, undefined, undefined, undefined, - { relevanceThreshold: 0.99 }, - ); + const result = await performSearch(createDeps({ chainIds: [CHAIN_ID] }, 0.65), { + userId: TEST_USER, + query: ORDERING_QUERY, + retrievalOptions: { relevanceThreshold: 0.99 }, + }); const chainRow = result.memories.find((m) => m.id === CHAIN_ID); expect(chainRow?.retrieval_signal).toBe('tll-chain'); @@ -165,7 +166,10 @@ describe('TLL augmentation through performSearch', () => { trace: createTrace([SEED_ID]), }); - const result = await performSearch(createDeps({ chainIds: [CHAIN_ID] }, 0.65), TEST_USER, 'what editor do I use'); + const result = await performSearch(createDeps({ chainIds: [CHAIN_ID] }, 0.65), { + userId: TEST_USER, + query: 'what editor do I use', + }); expect(result.memories.map((m) => m.id)).toEqual([SEED_ID]); }); diff --git a/src/services/extraction.ts b/src/services/extraction.ts index f3de7d2..6bbb71f 100644 --- a/src/services/extraction.ts +++ b/src/services/extraction.ts @@ -17,6 +17,7 @@ import { buildExtractionUserMessage, type ExtractionOptions, } from './observation-date-extraction.js'; +import { filterMetaFacts } from './meta-fact-filter.js'; const EXTRACTION_MAX_TOKENS = 4096; const AUDN_MAX_TOKENS = 2048; @@ -214,8 +215,16 @@ RULES: RIGHT: "User wants PostgreSQL for the production backend, replacing the earlier MongoDB choice." - Skip pleasantries, filler, acknowledgments, and meta-conversation. - Skip generic assistant chatter (acknowledgments, "sure!", "got it", "as an AI"). +- **Extract durable facts about the user or the world. Do NOT extract observations about the conversation itself.** A meta-observation describes the chat session (who is speaking, what was just asked, that a term appeared) rather than recording any durable state. If a turn contains only a question or a meta-observation and no new durable fact, emit nothing for that turn. When in doubt, ask: "would this fact still be useful months later in a completely different conversation?" — if no, drop it. - DO extract specific factual content from assistant responses: named entities, recommendations with proper nouns, schedules, data tables, creative writing with specific details. Prefix these with "Assistant mentioned:" or "Assistant recommended:". -- SHORT INPUTS: Even a single sentence like "My email is bob@example.com" contains an extractable fact. Do NOT skip short inputs — if the user stated something factual, extract it regardless of length. +- SHORT INPUTS: Length is NOT a reason to skip a fact. A single user sentence that names a person, place, profession, possession, preference, allergy, hobby, or any other durable attribute of the user IS extractable. Examples: + - "I'm Alex." → fact: user's name is Alex. + - "My email is bob@example.com" → fact: user's email is bob@example.com. + - "I live in Lisbon." → fact: user lives in Lisbon. + - "I play piano." → fact: user plays piano. + - "I have a golden retriever named Apollo." → fact: user has a golden retriever named Apollo. + - "I'm vegetarian." → fact: user is vegetarian. + Do NOT classify short user statements as "pleasantries" or "filler" when they contain a named entity or durable attribute. - CONTACT INFO: Email addresses, phone numbers, home addresses, and similar personal details are always extractable facts with importance >= 0.5. - Rate importance 0.0-1.0: 0.0-0.3 = trivial (greeting style, minor preferences) @@ -321,7 +330,17 @@ export async function extractFacts( const normalized: ExtractedFact[] = rawFacts.map((m) => normalizeRawFact(m)); const anchoredFacts = applyObservationDateAnchors(normalized, conversationText, options); const baseFacts = enrichExtractedFacts(normalizeExtractedFacts(anchoredFacts)); - return mergeSupplementalFacts(baseFacts, conversationText); + const merged = mergeSupplementalFacts(baseFacts, conversationText); + // Drop extraction-style meta-facts that describe the conversation + // itself rather than recording a durable user fact. These poison + // the embedding pool downstream. The filter is on by default; + // operators can disable for incident response via + // ATOMICMEMORY_META_FACT_FILTER=off. See + // src/services/meta-fact-filter.ts for rationale + AlignBench v0 + // (am-sdk-internal:benchmarks/alignbench/RESULTS.md) for evidence. + // Drops are logged structured ("[meta-fact-filter] dropped …") and + // counted via getMetaFactDropStats() for operator monitoring. + return filterMetaFacts(merged, { source: 'extract' }); }); } diff --git a/src/services/memory-crud.ts b/src/services/memory-crud.ts index af496d6..bfc9188 100644 --- a/src/services/memory-crud.ts +++ b/src/services/memory-crud.ts @@ -27,12 +27,12 @@ export interface ClaimSlotBackfillResult { updated: number; } -export async function listMemories(deps: MemoryServiceDeps, userId: string, limit: number = 20, offset: number = 0, sourceSite?: string, episodeId?: string) { - return deps.stores.memory.listMemories(userId, limit, offset, sourceSite, episodeId); +export async function listMemories(deps: MemoryServiceDeps, userId: string, limit: number = 20, offset: number = 0, sourceSite?: string, episodeId?: string, sessionId?: string) { + return deps.stores.memory.listMemories(userId, limit, offset, sourceSite, episodeId, sessionId); } -export async function listMemoriesInWorkspace(deps: MemoryServiceDeps, workspaceId: string, limit: number = 20, offset: number = 0, callerAgentId: string) { - return deps.stores.memory.listMemoriesInWorkspace(workspaceId, limit, offset, callerAgentId); +export async function listMemoriesInWorkspace(deps: MemoryServiceDeps, workspaceId: string, limit: number = 20, offset: number = 0, callerAgentId: string, sessionId?: string) { + return deps.stores.memory.listMemoriesInWorkspace(workspaceId, limit, offset, callerAgentId, sessionId); } export async function getMemory(deps: MemoryServiceDeps, id: string, userId: string) { diff --git a/src/services/memory-ingest.ts b/src/services/memory-ingest.ts index 194987e..00c349e 100644 --- a/src/services/memory-ingest.ts +++ b/src/services/memory-ingest.ts @@ -108,10 +108,11 @@ export async function performIngest( sourceSite: string, sourceUrl: string = '', sessionTimestamp?: Date, + sessionId?: string, ): Promise { const ingestStart = performance.now(); const logicalSessionTimestamp = resolveSessionDate(sessionTimestamp, conversationText); - const episodeId = await timed('ingest.store-episode', () => deps.stores.episode.storeEpisode({ userId, content: conversationText, sourceSite, sourceUrl })); + const episodeId = await timed('ingest.store-episode', () => deps.stores.episode.storeEpisode({ userId, content: conversationText, sourceSite, sourceUrl, sessionId })); const facts = await timed('ingest.extract', () => consensusExtractFacts(conversationText, deps.config)); const traceCollector = new IngestTraceCollector(deps.config.ingestTraceEnabled); const acc = createIngestAccumulator(); @@ -196,10 +197,11 @@ export async function performQuickIngest( sourceSite: string, sourceUrl: string = '', sessionTimestamp?: Date, + sessionId?: string, ): Promise { const ingestStart = performance.now(); const logicalSessionTimestamp = resolveSessionDate(sessionTimestamp, conversationText); - const episodeId = await deps.stores.episode.storeEpisode({ userId, content: conversationText, sourceSite, sourceUrl }); + const episodeId = await deps.stores.episode.storeEpisode({ userId, content: conversationText, sourceSite, sourceUrl, sessionId }); const facts = timed('quick-ingest.extract', () => Promise.resolve(quickExtractFacts(conversationText))); const extractedFacts = await facts; const traceCollector = new IngestTraceCollector(deps.config.ingestTraceEnabled); @@ -245,8 +247,9 @@ export async function performStoreVerbatim( sourceSite: string, sourceUrl: string = '', metadata?: MemoryMetadata, + sessionId?: string, ): Promise { - const episodeId = await deps.stores.episode.storeEpisode({ userId, content, sourceSite, sourceUrl }); + const episodeId = await deps.stores.episode.storeEpisode({ userId, content, sourceSite, sourceUrl, sessionId }); const embedding = await embedText(content); const writeSecurity = assessWriteSecurity(content, sourceSite, deps.config); const trustScore = writeSecurity.allowed ? writeSecurity.trust.score : 0.5; @@ -313,12 +316,14 @@ export async function performWorkspaceIngest( sourceUrl: string = '', workspace: WorkspaceContext, sessionTimestamp?: Date, + sessionId?: string, ): Promise { const ingestStart = performance.now(); const logicalSessionTimestamp = resolveSessionDate(sessionTimestamp, conversationText); const episodeId = await timed('ws-ingest.store-episode', () => deps.stores.episode.storeEpisode({ userId, content: conversationText, sourceSite, sourceUrl, + sessionId, workspaceId: workspace.workspaceId, agentId: workspace.agentId, }), ); diff --git a/src/services/memory-search.ts b/src/services/memory-search.ts index 383a95b..14d2b16 100644 --- a/src/services/memory-search.ts +++ b/src/services/memory-search.ts @@ -79,6 +79,18 @@ interface PackagedSearchOutput { assemblySummary: ReturnType['assemblySummary']; } +export interface PerformSearchInput { + userId: string; + query: string; + sourceSite?: string; + limit?: number; + asOf?: string; + referenceTime?: Date; + namespaceScope?: string; + retrievalOptions?: RetrievalOptions; + sessionId?: string; +} + /** * Fetch a contradiction counterpart memory and shape it as a SearchResult. * Includes expired rows because contradiction counterparts may have been @@ -201,6 +213,7 @@ async function executeSearchStep( sourceSite: string | undefined, referenceTime: Date | undefined, namespaceScope: string | undefined, + sessionId: string | undefined, retrievalOptions: RetrievalOptions | undefined, asOf: string | undefined, trace: TraceCollector, @@ -223,6 +236,7 @@ async function executeSearchStep( }; const pipelineResult = await runSearchPipelineWithTrace(pipelineStores, userId, query, effectiveLimit, sourceSite, referenceTime, { namespaceScope, + sessionId, retrievalMode: retrievalOptions?.retrievalMode, searchStrategy: retrievalOptions?.searchStrategy, skipRepairLoop: retrievalOptions?.skipRepairLoop, @@ -545,15 +559,19 @@ function buildRetrievalResult( /** Full search with lesson check, URI resolution, pipeline, post-processing, and packaging. */ export async function performSearch( deps: MemoryServiceDeps, - userId: string, - query: string, - sourceSite?: string, - limit?: number, - asOf?: string, - referenceTime?: Date, - namespaceScope?: string, - retrievalOptions?: RetrievalOptions, + input: PerformSearchInput, ): Promise { + const { + userId, + query, + sourceSite, + limit, + asOf, + referenceTime, + namespaceScope, + retrievalOptions, + sessionId, + } = input; const lessonCheck = await checkSearchLessons(deps, userId, query); if (lessonCheck && !lessonCheck.safe) { return { @@ -573,7 +591,7 @@ export async function performSearch( const uriResult = await tryUriResolution(deps, query, userId, retrievalOptions, trace); if (uriResult) return uriResult; - const { memories: rawMemories, activeTrace, queryEmbedding, chainResult, reflections, questionType } = await executeSearchStep(deps, userId, query, effectiveLimit, sourceSite, referenceTime, namespaceScope, retrievalOptions, asOf, trace); + const { memories: rawMemories, activeTrace, queryEmbedding, chainResult, reflections, questionType } = await executeSearchStep(deps, userId, query, effectiveLimit, sourceSite, referenceTime, namespaceScope, sessionId, retrievalOptions, asOf, trace); const filteredMemories = await postProcessResults( deps, rawMemories, activeTrace, userId, query, asOf, sourceSite, retrievalOptions, ); @@ -620,16 +638,25 @@ export async function performFastSearch( sourceSite?: string, limit?: number, namespaceScope?: string, + sessionId?: string, retrievalOptions?: RetrievalOptions, ): Promise { const label = classifyQueryDetailed(query).label; const escalate = label === 'multi-hop' || label === 'aggregation' || label === 'complex'; // Fast search owns these latency toggles based on query class; caller options // still flow through for packaging, threshold, and strategy controls. - return performSearch(deps, userId, query, sourceSite, limit, undefined, undefined, namespaceScope, { - ...retrievalOptions, - skipRepairLoop: !escalate, - skipReranking: !escalate, + return performSearch(deps, { + userId, + query, + sourceSite, + limit, + namespaceScope, + retrievalOptions: { + ...retrievalOptions, + skipRepairLoop: !escalate, + skipReranking: !escalate, + }, + sessionId, }); } @@ -666,7 +693,11 @@ export async function performRescueSearch( const keywords = extractKeywordsFromQuery(opts.query); const augmentedQuery = keywords ? `${opts.query} ${keywords}` : opts.query; const rescueK = deps.config.abstentionRescueRetrieveK; - const rescueResult = await performSearch(deps, opts.userId, augmentedQuery, undefined, rescueK); + const rescueResult = await performSearch(deps, { + userId: opts.userId, + query: augmentedQuery, + limit: rescueK, + }); const { injectionText: rescueInjection } = rescueResult; const mergedInjection = applyConfidencePrefix( @@ -705,6 +736,7 @@ export async function performWorkspaceSearch( limit?: number; referenceTime?: Date; retrievalOptions?: RetrievalOptions; + sessionId?: string; } = {}, ): Promise { const { limit: effectiveLimit } = resolveSearchLimitDetailed(query, options.limit, deps.config); @@ -712,13 +744,14 @@ export async function performWorkspaceSearch( const memories = await deps.stores.search.searchSimilarInWorkspace( workspace.workspaceId, queryEmbedding, effectiveLimit, - options.agentScope ?? 'all', workspace.agentId, options.referenceTime, + options.agentScope ?? 'all', workspace.agentId, options.referenceTime, options.sessionId, ); const trace = new TraceCollector(query, userId); trace.stage('workspace-search', memories, { workspaceId: workspace.workspaceId, agentId: workspace.agentId, agentScope: options.agentScope ?? 'all', + sessionId: options.sessionId, }); const { filtered: staleFilteredMemories, removedCompositeIds } = diff --git a/src/services/memory-service-types.ts b/src/services/memory-service-types.ts index c867422..847ca28 100644 --- a/src/services/memory-service-types.ts +++ b/src/services/memory-service-types.ts @@ -274,6 +274,7 @@ export interface ScopedSearchOptions { asOf?: string; referenceTime?: Date; namespaceScope?: string; + sessionId?: string; retrievalOptions?: RetrievalOptions; /** When true, skips the LLM repair loop (used by /search/fast). */ fast?: boolean; diff --git a/src/services/memory-service.ts b/src/services/memory-service.ts index b5f7547..fe41a5a 100644 --- a/src/services/memory-service.ts +++ b/src/services/memory-service.ts @@ -31,6 +31,46 @@ export type { TierAssignment }; export type { crud as CrudModule }; export type ClaimSlotBackfillResult = crud.ClaimSlotBackfillResult; +interface IngestInput { + userId: string; + conversationText: string; + sourceSite: string; + sourceUrl?: string; + sessionTimestamp?: Date; + effectiveConfig?: MemoryServiceDeps['config']; + sessionId?: string; +} + +interface StoreVerbatimInput { + userId: string; + content: string; + sourceSite: string; + sourceUrl?: string; + metadata?: MemoryMetadata; + effectiveConfig?: MemoryServiceDeps['config']; + sessionId?: string; +} + +interface WorkspaceIngestInput extends IngestInput { + workspace: WorkspaceContext; +} + +interface ListOptions { + limit?: number; + offset?: number; + sourceSite?: string; + episodeId?: string; + sessionId?: string; +} + +interface ScopedListInput extends ListOptions { + scope: MemoryScope; +} + +interface ListInput extends ListOptions { + userId: string; +} + /** Bag of optional constructor inputs forwarded to {@link buildMemoryServiceDeps}. */ interface MemoryServiceConstructorBag { repo: MemoryRepository; @@ -142,12 +182,14 @@ export class MemoryService { // --- Ingest --- - async ingest(userId: string, conversationText: string, sourceSite: string, sourceUrl: string = '', sessionTimestamp?: Date, effectiveConfig?: MemoryServiceDeps['config']): Promise { - return performIngest(this.depsFor(effectiveConfig), userId, conversationText, sourceSite, sourceUrl, sessionTimestamp); + async ingest(input: IngestInput): Promise { + const { userId, conversationText, sourceSite, sourceUrl = '', sessionTimestamp, effectiveConfig, sessionId } = input; + return performIngest(this.depsFor(effectiveConfig), userId, conversationText, sourceSite, sourceUrl, sessionTimestamp, sessionId); } - async quickIngest(userId: string, conversationText: string, sourceSite: string, sourceUrl: string = '', sessionTimestamp?: Date, effectiveConfig?: MemoryServiceDeps['config']): Promise { - return performQuickIngest(this.depsFor(effectiveConfig), userId, conversationText, sourceSite, sourceUrl, sessionTimestamp); + async quickIngest(input: IngestInput): Promise { + const { userId, conversationText, sourceSite, sourceUrl = '', sessionTimestamp, effectiveConfig, sessionId } = input; + return performQuickIngest(this.depsFor(effectiveConfig), userId, conversationText, sourceSite, sourceUrl, sessionTimestamp, sessionId); } /** @@ -155,12 +197,14 @@ export class MemoryService { * Used for user-created contexts (text/file uploads) where * the content should remain as one canonical memory record. */ - async storeVerbatim(userId: string, content: string, sourceSite: string, sourceUrl: string = '', metadata?: MemoryMetadata, effectiveConfig?: MemoryServiceDeps['config']): Promise { - return performStoreVerbatim(this.depsFor(effectiveConfig), userId, content, sourceSite, sourceUrl, metadata); + async storeVerbatim(input: StoreVerbatimInput): Promise { + const { userId, content, sourceSite, sourceUrl = '', metadata, effectiveConfig, sessionId } = input; + return performStoreVerbatim(this.depsFor(effectiveConfig), userId, content, sourceSite, sourceUrl, metadata, sessionId); } - async workspaceIngest(userId: string, conversationText: string, sourceSite: string, sourceUrl: string = '', workspace: WorkspaceContext, sessionTimestamp?: Date, effectiveConfig?: MemoryServiceDeps['config']): Promise { - return performWorkspaceIngest(this.depsFor(effectiveConfig), userId, conversationText, sourceSite, sourceUrl, workspace, sessionTimestamp); + async workspaceIngest(input: WorkspaceIngestInput): Promise { + const { userId, conversationText, sourceSite, sourceUrl = '', workspace, sessionTimestamp, effectiveConfig, sessionId } = input; + return performWorkspaceIngest(this.depsFor(effectiveConfig), userId, conversationText, sourceSite, sourceUrl, workspace, sessionTimestamp, sessionId); } // --- Search (scope-dispatching) --- @@ -175,6 +219,7 @@ export class MemoryService { limit: options.limit, referenceTime: options.referenceTime, retrievalOptions: options.retrievalOptions, + sessionId: options.sessionId, }); } if (options.fast) { @@ -185,10 +230,21 @@ export class MemoryService { options.sourceSite, options.limit, options.namespaceScope, + options.sessionId, options.retrievalOptions, ); } - return performSearch(deps, scope.userId, query, options.sourceSite, options.limit, options.asOf, options.referenceTime, options.namespaceScope, options.retrievalOptions); + return performSearch(deps, { + userId: scope.userId, + query, + sourceSite: options.sourceSite, + limit: options.limit, + asOf: options.asOf, + referenceTime: options.referenceTime, + namespaceScope: options.namespaceScope, + retrievalOptions: options.retrievalOptions, + sessionId: options.sessionId, + }); } /** Scope-dispatching expand with agent visibility enforcement for workspace operations. */ @@ -211,31 +267,40 @@ export class MemoryService { } /** Scope-dispatching list with agent visibility enforcement for workspace operations. */ - async scopedList(scope: MemoryScope, limit: number = 20, offset: number = 0, sourceSite?: string, episodeId?: string) { - if (scope.kind === 'workspace') return crud.listMemoriesInWorkspace(this.deps, scope.workspaceId, limit, offset, scope.agentId); - return crud.listMemories(this.deps, scope.userId, limit, offset, sourceSite, episodeId); + async scopedList(input: ScopedListInput) { + const { scope, ...options } = input; + const { limit = 20, offset = 0, sourceSite, episodeId, sessionId } = options; + if (scope.kind === 'workspace') return crud.listMemoriesInWorkspace(this.deps, scope.workspaceId, limit, offset, scope.agentId, sessionId); + return crud.listMemories(this.deps, scope.userId, limit, offset, sourceSite, episodeId, sessionId); } // --- Search (legacy, prefer scopedSearch) --- /** @deprecated Use scopedSearch instead. */ - async search(userId: string, query: string, sourceSite?: string, limit?: number, asOf?: string, referenceTime?: Date, namespaceScope?: string, retrievalOptions?: RetrievalOptions): Promise { - return performSearch(this.deps, userId, query, sourceSite, limit, asOf, referenceTime, namespaceScope, retrievalOptions); + async search(userId: string, query: string, sourceSite?: string, limit?: number, asOf?: string, referenceTime?: Date, namespaceScope?: string, retrievalOptions?: RetrievalOptions, sessionId?: string): Promise { + return performSearch(this.deps, { + userId, query, sourceSite, limit, asOf, referenceTime, + namespaceScope, retrievalOptions, sessionId, + }); } /** @deprecated Use scopedSearch instead. */ - async fastSearch(userId: string, query: string, sourceSite?: string, limit?: number, namespaceScope?: string): Promise { - return performFastSearch(this.deps, userId, query, sourceSite, limit, namespaceScope); + async fastSearch(userId: string, query: string, sourceSite?: string, limit?: number, namespaceScope?: string, sessionId?: string): Promise { + return performFastSearch(this.deps, userId, query, sourceSite, limit, namespaceScope, sessionId); } /** @deprecated Use scopedSearch instead. */ - async workspaceSearch(userId: string, query: string, workspace: WorkspaceContext, options: { agentScope?: AgentScope; limit?: number; referenceTime?: Date; retrievalOptions?: RetrievalOptions } = {}): Promise { + async workspaceSearch(userId: string, query: string, workspace: WorkspaceContext, options: { agentScope?: AgentScope; limit?: number; referenceTime?: Date; retrievalOptions?: RetrievalOptions; sessionId?: string } = {}): Promise { return performWorkspaceSearch(this.deps, userId, query, workspace, options); } // --- CRUD --- - async list(userId: string, limit: number = 20, offset: number = 0, sourceSite?: string, episodeId?: string) { return crud.listMemories(this.deps, userId, limit, offset, sourceSite, episodeId); } + async list(input: ListInput) { + const { userId, ...options } = input; + const { limit = 20, offset = 0, sourceSite, episodeId, sessionId } = options; + return crud.listMemories(this.deps, userId, limit, offset, sourceSite, episodeId, sessionId); + } async get(id: string, userId: string) { return crud.getMemory(this.deps, id, userId); } async expand(userId: string, memoryIds: string[]) { return crud.expandMemories(this.deps, userId, memoryIds); } async delete(id: string, userId: string) { return crud.deleteMemory(this.deps, id, userId); } diff --git a/src/services/meta-fact-filter.ts b/src/services/meta-fact-filter.ts new file mode 100644 index 0000000..2900730 --- /dev/null +++ b/src/services/meta-fact-filter.ts @@ -0,0 +1,248 @@ +/** + * @file Meta-fact filter for the extraction pipeline. + * + * Drops extraction-style "meta-facts" — outputs that describe the + * conversation itself ("The user asked for the user's name.", "As of + * , X is a term mentioned in the conversation.") rather than + * recording a durable fact about the user. + * + * Empirically motivated by AlignBench v0 (am-sdk-internal: + * benchmarks/alignbench/RESULTS.md). When meta-facts sit in the recall + * pool alongside durable facts, they outrank real facts at thin cosine + * margins, producing the partner-visible "I don't recall..." failures + * observed in the Filecoin demo (atomicmem.filecoin.cloud) and the 31% + * "no info" refusal rate on LongMemEval-S. + * + * Cleaning at extraction time means meta-facts never enter the + * database, so every downstream search-style query (semantic, BM25, + * package, temporal) is uniformly cleaner. The SDK ships a complementary + * post-retrieval filter (am-sdk-internal: + * src/memory/meta-fact-filter.ts) for deployments running an older + * core release. + * + * This filter is intentionally: + * - pure (deterministic regex, no I/O, no LLM calls); + * - on by default (the patterns describe outputs that are never + * useful durable facts — there is no defensible reason to keep + * them); + * - configurable via an environment flag for emergency disable; + * - logged when it drops, so operators can audit extractor noise. + */ + +/** + * Minimal shape probed by the filter. Real `ExtractedFact` instances carry + * the durable text on `.fact`; we tolerate `.statement` (the raw LLM key, + * before normalization) too so the filter is safe to apply earlier in the + * pipeline as well. + */ +export interface MetaFactCandidate { + fact?: string; + statement?: string; +} + +/** + * Extract the durable-fact text from a candidate. Prefers `.fact` (the + * normalized post-process shape) and falls back to `.statement` (the raw + * LLM key) so this filter is safe to apply pre- or post-normalization. + */ +function readFactText(fact: MetaFactCandidate): string { + if (typeof fact.fact === 'string') return fact.fact; + if (typeof fact.statement === 'string') return fact.statement; + return ''; +} + +/** + * Default regex set targeting the verbatim meta-fact shapes observed in + * the Filecoin partner demo and in AlignBench v0's distractor pool. + * + * 1. "The user asked/requested/said/is asking/is me ..." — meta-facts + * about user actions in the conversation, not about the user. + * 2. "As of , X is a term mentioned in the conversation." — + * vacuous acknowledgements of vocabulary. + * 3. "A name was mentioned." — observation about the chat session. + * 4. "The conversation involves the user." — meta-observation. + * 5. "The user has started a conversation." — meta-observation. + * + * Patterns are case-insensitive and anchored at the start so legitimate + * sentences like "The user lives in Lisbon, where they asked their + * landlord about renewal." are preserved. + */ +export const DEFAULT_META_FACT_PATTERNS: readonly RegExp[] = Object.freeze([ + /^\s*the user (asked|requested|said|is asking|is me)\b/i, + /^\s*as of [^,]+,\s+.+\s+is a term mentioned in the conversation\.?$/i, + /^\s*a name was mentioned\b/i, + /^\s*the conversation involves the user\b/i, + /^\s*the user has started a conversation\b/i, +]); + +/** + * Test whether a text string matches any active meta-fact pattern. Pure; + * defensive against non-string input. Exposed for callers that already + * have the raw text (e.g. tests, ad-hoc audits). + */ +export function isMetaFactStatement( + text: unknown, + patterns: readonly RegExp[] = DEFAULT_META_FACT_PATTERNS, +): boolean { + if (typeof text !== 'string' || text.length === 0) return false; + for (const p of patterns) { + if (p.test(text)) return true; + } + return false; +} + +/** + * Default master-switch resolver. Operators can disable the filter + * entirely with `ATOMICMEMORY_META_FACT_FILTER=off` (used for incident + * response — never recommended for steady state). Defaults to ON. + */ +export function metaFactFilterEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const raw = env.ATOMICMEMORY_META_FACT_FILTER; + if (raw == null) return true; + return !['off', 'false', '0', 'disabled'].includes(raw.trim().toLowerCase()); +} + +/** + * Process-lifetime counters keyed by pattern index. Lets operators + * aggregate filter activity without needing log scraping. Exposed via + * `getMetaFactDropStats()` and reset between tests with + * `resetMetaFactDropStats()`. + */ +const dropCounts: number[] = new Array(DEFAULT_META_FACT_PATTERNS.length).fill(0); +let dropTotal = 0; + +export interface MetaFactDropStats { + total: number; + byPattern: ReadonlyArray; +} + +export function getMetaFactDropStats(): MetaFactDropStats { + return { total: dropTotal, byPattern: [...dropCounts] }; +} + +export function resetMetaFactDropStats(): void { + for (let i = 0; i < dropCounts.length; i++) dropCounts[i] = 0; + dropTotal = 0; +} + +/** + * Emit a structured, grep-friendly drop event. Format: + * + * [meta-fact-filter] dropped pattern=2 len=47 source=extract + * + * Use `source` to disambiguate runtime extraction drops from migration + * runs (cleanup-meta-facts.ts emits its own audit JSONL — this log line + * is for the live pipeline). + */ +function recordDrop( + patternIndex: number, + statementLength: number, + source: string, +): void { + if (patternIndex >= 0 && patternIndex < dropCounts.length) { + dropCounts[patternIndex] += 1; + } + dropTotal += 1; + // Single-line log, no PII (no statement text). Operators correlate by + // conversation_id at the call site, not here. + console.info( + `[meta-fact-filter] dropped pattern=${patternIndex} len=${statementLength} source=${source}`, + ); +} + +export interface FilterMetaFactsOptions { + /** Override the default regex set. */ + patterns?: readonly RegExp[]; + /** Force-enable / disable, bypassing `metaFactFilterEnabled`. */ + enabled?: boolean; + /** + * Telemetry hook fired once per dropped fact. Receives the statement + * and the pattern index that matched. Exceptions are swallowed so + * telemetry can never break extraction. + * + * When `null` is passed, telemetry is fully suppressed (useful for + * tests). When `undefined` (the default), the structured drop logger + * fires automatically via `source` below. + */ + onDrop?: ((statement: string, patternIndex: number) => void) | null; + /** + * Tag identifying the call site for the structured drop log line + * (`extract`, `migration`, `test`, etc.). Ignored when `onDrop` is + * supplied explicitly. Defaults to `'extract'` to match the most + * common call site. + */ + source?: string; +} + +/** + * Return the index of the first pattern that matches `text`, or -1 if + * none match or `text` is empty. Pulled out of `filterMetaFacts` so the + * loop body stays simple. + */ +function findMetaFactPatternIndex( + text: string, + patterns: readonly RegExp[], +): number { + if (text.length === 0) return -1; + for (let i = 0; i < patterns.length; i++) { + if (patterns[i].test(text)) return i; + } + return -1; +} + +/** + * Invoke a drop hook safely, then always update process counters / + * structured log unless the caller explicitly opted out by passing + * `onDrop: null`. The default path (no `onDrop` supplied) records via + * `recordDrop` keyed on the caller's `source` tag. + */ +function safeOnDrop( + onDrop: FilterMetaFactsOptions['onDrop'], + text: string, + patternIndex: number, + source: string, +): void { + // Explicit opt-out: don't touch counters or log. + if (onDrop === null) return; + if (onDrop) { + try { + onDrop(text, patternIndex); + } catch { + // Intentional: telemetry must never break extraction. + } + return; + } + // Default path: structured log + counter update. + recordDrop(patternIndex, text.length, source); +} + +/** + * Drop meta-facts from an extracted-fact array. Returns a new array; + * does not mutate the input. When the filter is disabled (env flag or + * `enabled: false`), returns a shallow copy of the input unchanged. + * + * Generic over `T` so callers can pass `ExtractedFact[]`, raw LLM + * output, or any other shape that exposes `.fact` or `.statement`. + */ +export function filterMetaFacts( + facts: readonly T[], + options: FilterMetaFactsOptions = {}, +): T[] { + const enabled = options.enabled ?? metaFactFilterEnabled(); + if (!enabled) return [...facts]; + const patterns = options.patterns ?? DEFAULT_META_FACT_PATTERNS; + if (patterns.length === 0) return [...facts]; + + const source = options.source ?? 'extract'; + const kept: T[] = []; + for (const fact of facts) { + const text = readFactText(fact); + const matchedIndex = findMetaFactPatternIndex(text, patterns); + if (matchedIndex >= 0) { + safeOnDrop(options.onDrop, text, matchedIndex, source); + continue; + } + kept.push(fact); + } + return kept; +} diff --git a/src/services/search-pipeline.ts b/src/services/search-pipeline.ts index af947a1..9c9e35c 100644 --- a/src/services/search-pipeline.ts +++ b/src/services/search-pipeline.ts @@ -152,6 +152,7 @@ function readNormalizedSimilarity(result: SearchResult | undefined): number { export interface SearchPipelineOptions { namespaceScope?: string; + sessionId?: string; retrievalMode?: RetrievalMode; searchStrategy?: SearchStrategy; /** Skip the LLM repair loop for latency-critical paths (UC1: <200ms). */ @@ -219,11 +220,11 @@ export async function runSearchPipelineWithTrace( const searchQuery = augmentation.searchQuery; const initialResults = await timed('search.vector', () => runInitialRetrieval( - stores, userId, searchQuery, queryEmbedding, candidateDepth, sourceSite, referenceTime, options.searchStrategy, policyConfig, + stores, userId, searchQuery, queryEmbedding, candidateDepth, sourceSite, referenceTime, options.searchStrategy, policyConfig, options.sessionId, )); const seededResults = await timed('search.hybrid-fallback', () => maybeApplyAbstractHybridFallback( stores, userId, query, searchQuery, queryEmbedding, candidateDepth, sourceSite, referenceTime, - options.retrievalMode, options.searchStrategy, initialResults, trace, policyConfig, + options.retrievalMode, options.searchStrategy, initialResults, trace, policyConfig, options.sessionId, )); console.log(`[search] Query: "${query}", Results: ${seededResults.length}`); @@ -274,6 +275,7 @@ export async function runSearchPipelineWithTrace( policyConfig, options.searchStrategy, temporalExpansion.temporalAnchorFingerprints, + options.sessionId, )); const iterated = await timed('search.iterative-retrieval', async () => { @@ -338,7 +340,8 @@ export async function runSearchPipelineWithTrace( if (namespaceScope) { trace.event('namespace-filtering', { scope: namespaceScope }); } - const filtered = applyNamespaceScopeFilter(selected, namespaceScope, trace); + const namespaceFiltered = applyNamespaceScopeFilter(selected, namespaceScope, trace); + const filtered = await applySessionScopeFilter(stores.pool, namespaceFiltered, userId, options.sessionId ?? null, trace); const chainResult = detectEventChains({ candidates: filtered.map((m) => ({ @@ -388,6 +391,37 @@ function applyNamespaceScopeFilter( return filtered; } +async function applySessionScopeFilter( + pool: pg.Pool, + selected: SearchResult[], + userId: string, + sessionId: string | null, + trace: TraceCollector, +): Promise { + if (!sessionId || selected.length === 0) return selected; + // Search channels such as temporal/TLL augmentation can append rows after + // initial retrieval. Keep this final DB-backed gate so thread-scoped reads + // cannot leak broader-scope memories through an auxiliary channel. + const ids = selected.map((result) => result.id); + const result = await pool.query( + `SELECT m.id + FROM memories m + JOIN episodes e ON e.id = m.episode_id + WHERE m.id = ANY($1::uuid[]) + AND m.user_id = $2 + AND e.user_id = m.user_id + AND e.session_id = $3`, + [ids, userId, sessionId], + ); + const keptIds = new Set(result.rows.map((row) => row.id as string)); + const filtered = selected.filter((memory) => keptIds.has(memory.id)); + trace.stage('session-filter', filtered, { + sessionId, + removedIds: ids.filter((id) => !keptIds.has(id)), + }); + return filtered; +} + async function runInitialRetrieval( stores: SearchPipelineStores, userId: string, @@ -398,6 +432,7 @@ async function runInitialRetrieval( referenceTime?: Date, searchStrategy: SearchStrategy = 'memory', policyConfig: SearchPipelineRuntimeConfig = config, + sessionId?: string, ): Promise { if (searchStrategy === 'fact-hybrid') { return stores.search.searchAtomicFactsHybrid( @@ -407,6 +442,7 @@ async function runInitialRetrieval( candidateDepth, sourceSite, referenceTime, + sessionId, ); } return runMemoryRrfRetrieval( @@ -419,6 +455,7 @@ async function runInitialRetrieval( referenceTime, policyConfig.hybridSearchEnabled, policyConfig, + sessionId, ); } @@ -436,6 +473,7 @@ async function maybeApplyAbstractHybridFallback( initialResults: SearchResult[], trace: TraceCollector, policyConfig: SearchPipelineRuntimeConfig = config, + sessionId?: string, ): Promise { if (searchStrategy === 'fact-hybrid') return initialResults; if (policyConfig.hybridSearchEnabled || policyConfig.entityGraphEnabled) return initialResults; @@ -452,6 +490,7 @@ async function maybeApplyAbstractHybridFallback( referenceTime, true, policyConfig, + sessionId, ); trace.stage('abstract-hybrid-fallback', fallbackResults, { candidateDepth }); return fallbackResults; @@ -473,6 +512,7 @@ async function applyRepairLoop( policyConfig: SearchPipelineRuntimeConfig, searchStrategy: SearchStrategy = 'memory', protectedIds: string[] = [], + sessionId?: string, ): Promise<{ memories: SearchResult[]; queryText: string }> { if (!shouldRunRepairLoop(query, initialResults, policyConfig)) { return { memories: initialResults, queryText: query }; @@ -486,7 +526,7 @@ async function applyRepairLoop( const rewrittenEmbedding = await embedText(rewrittenQuery, 'query'); const repairedResults = searchStrategy === 'fact-hybrid' - ? await stores.search.searchAtomicFactsHybrid(userId, rewrittenQuery, rewrittenEmbedding, candidateDepth, sourceSite, referenceTime) + ? await stores.search.searchAtomicFactsHybrid(userId, rewrittenQuery, rewrittenEmbedding, candidateDepth, sourceSite, referenceTime, sessionId) : await runMemoryRrfRetrieval( stores, userId, @@ -497,6 +537,7 @@ async function applyRepairLoop( referenceTime, policyConfig.hybridSearchEnabled, policyConfig, + sessionId, ); const decision = shouldAcceptRepair(initialResults, repairedResults, policyConfig); @@ -1099,6 +1140,7 @@ async function runMemoryRrfRetrieval( referenceTime: Date | undefined, includeKeywordChannel: boolean, policyConfig: SearchPipelineRuntimeConfig = config, + sessionId?: string, ): Promise { const semanticResults = await stores.search.searchSimilar( userId, @@ -1106,6 +1148,7 @@ async function runMemoryRrfRetrieval( limit, sourceSite, referenceTime, + sessionId, ); const channels = await assembleRrfChannels({ stores, @@ -1117,6 +1160,7 @@ async function runMemoryRrfRetrieval( semanticResults, includeKeywordChannel, policyConfig, + sessionId, }); const fused = channels.length === 1 @@ -1190,6 +1234,7 @@ interface AssembleChannelsArgs { queryEmbedding: number[]; limit: number; sourceSite: string | undefined; + sessionId?: string; semanticResults: SearchResult[]; includeKeywordChannel: boolean; policyConfig: SearchPipelineRuntimeConfig; @@ -1208,7 +1253,7 @@ interface RrfChannel { * count grows. */ async function assembleRrfChannels(args: AssembleChannelsArgs): Promise { - const { stores, userId, queryText, queryEmbedding, limit, sourceSite, + const { stores, userId, queryText, queryEmbedding, limit, sourceSite, sessionId, semanticResults, includeKeywordChannel, policyConfig } = args; const channels: RrfChannel[] = [ { name: 'semantic', weight: SEMANTIC_RRF_WEIGHT, results: semanticResults }, @@ -1224,7 +1269,7 @@ async function assembleRrfChannels(args: AssembleChannelsArgs): Promise 0) { channels.push({ name: 'keyword', weight: keywordRrfWeight(), results: keywordResults }); }