diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f6018d..ef7fea3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,8 +100,17 @@ jobs: - name: Migration tests run: python scripts/run_acceptance.py --only verify_migrations - - name: CLI tests - run: python scripts/run_acceptance.py --only verify_cli + - name: Content-store tests + run: python scripts/run_acceptance.py --only verify_content_store + + - name: Asset model tests + run: python scripts/run_acceptance.py --only verify_asset_model + + - name: CLI tests (base + asset commands) + run: python scripts/run_acceptance.py --only verify_cli --only verify_asset_cli + + - name: Asset adapter tests + run: python scripts/run_acceptance.py --only verify_asset_adapters - name: MCP smoke test run: | @@ -112,12 +121,17 @@ jobs: from openmind import mcp_server from openmind.runtime import get_runtime - expected = {"search", "route", "dispatch", "get_glossary", - "find_similar_cases", "save_case", "get_doc", - "propose_fix", "apply_fix"} + # The nine core tools are a STABLE external contract; Phase 2 adds + # read-only Asset tools ALONGSIDE them, never in place of one. + core = {"search", "route", "dispatch", "get_glossary", + "find_similar_cases", "save_case", "get_doc", + "propose_fix", "apply_fix"} + asset = {"list_assets", "get_asset", "get_asset_revisions", "get_evidence"} server = mcp_server.create_mcp_server(get_runtime()) names = {t.name for t in asyncio.run(server.list_tools())} - assert names == expected, f"MCP tool drift: {names ^ expected}" + assert core <= names, f"core MCP tool missing: {core - names}" + assert asset <= names, f"asset MCP tool missing: {asset - names}" + assert names == core | asset, f"unexpected MCP tools: {names ^ (core | asset)}" assert server.name == "open-mind", server.name print("MCP smoke OK:", len(names), "tools") PY @@ -213,6 +227,57 @@ jobs: - name: Artifact export contract run: python tests/verify_artifacts.py + - name: Migration to v0003 + content-store byte round-trip + shell: bash + run: | + python - <<'PY' + import os, tempfile + os.environ["OPENMIND_DATA_DIR"] = tempfile.mkdtemp() + os.environ["OPENMIND_MACHINE_DIR"] = tempfile.mkdtemp() + from openmind import db, content_store as cs + db.init_db() + assert db.migration_status()["version"] == 3, db.migration_status() + # content-addressed blob byte round-trip + reuse + SHA-256 identity + import hashlib + data = b"phase2 content \x00\x01\x02 bytes" + h = cs.put("p_smoke0000ci", data) + assert h == hashlib.sha256(data).hexdigest() + assert cs.put("p_smoke0000ci", data) == h # reuse + assert cs.get("p_smoke0000ci", h) == data # round-trip + print("migration v0003 + content-store round-trip OK:", h[:12]) + PY + + - name: CLI asset help + fixture ingest + asset list + evidence read + shell: bash + run: | + export OPENMIND_DATA_DIR="$(mktemp -d)" + export OPENMIND_MACHINE_DIR="$(mktemp -d)" + python -m openmind.cli asset --help > /dev/null + python -m openmind.cli asset list --help > /dev/null + WS=$(python -m openmind.cli init --name smoke --path ./fixtures/sample-repo \ + --ingest --wait --json | python -c "import json,sys; print(json.load(sys.stdin)['workspace_id'])") + # Small fixture ingest -> asset list -> resolve a revision/segment/evidence + # id chain -> read historical evidence back from the immutable snapshot. + OM_WS="$WS" python - <<'PY' + import json, os, subprocess, sys + ws = os.environ["OM_WS"] + def cli(*args): + return json.loads(subprocess.check_output( + [sys.executable, "-m", "openmind.cli", *args])) + data = cli("asset", "list", "--workspace", ws, "--json") + assert data["ok"] and data["total"] >= 1, data + aid = data["assets"][0]["id"] + show = cli("asset", "show", "--workspace", ws, "--asset", aid, "--json") + rid = show["asset"]["current_revision"]["id"] + segs = cli("asset", "segments", "--workspace", ws, "--revision", rid, "--json") + eid = segs["segments"][0]["evidence_id"] + ev = cli("asset", "evidence", "--workspace", ws, "--evidence", eid, "--json") + assert ev["snapshot"]["status"] == "available", ev + assert not ev["locator"]["file"].startswith("/"), ev + print("asset CLI smoke OK:", data["total"], "assets, evidence snapshot", + ev["snapshot"]["status"]) + PY + - name: Skill bridge startup shell: bash run: | diff --git a/README.md b/README.md index 3b53b7d..26f7fd7 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ Point Open Mind at a local repository and it builds persisted artifacts: | Artifact | Implemented behavior | |---|---| | **Source-traceable knowledge index** | Stores repo-relative paths, content hashes, source locations, file metadata, and search chunks. | +| **Canonical Asset model** (v2 Phase 2) | Every indexed file is an **Asset**; every observed version an immutable **Revision**; each revision divided into deterministic **Segments**, each with source-locatable **Evidence**. Historical content is snapshotted in an immutable SHA-256 content store, so evidence for an old revision stays readable after the file changes. Unchanged re-ingestion creates no revision and never re-embeds; removed files are marked removed without erasing history. | | **Verbatim glossary** | Extracts terms/acronyms from definition tables, definition lines, acronym expansions, and code comments; definitions are copied verbatim and carry `source_file`, `line_number`, and `content_hash`. The interactive learn path scans code/config sources; the `.openmind` export walk additionally scans README/docs/GLOSSARY files (primary definition sources). | | **Structure and graph map** | Builds a module tree, per-file definition index, import/dependency graph, entry points, and a name-based call/usage graph. Ambiguous call edges are flagged instead of guessed. | | **Exact-token + hybrid search** | Bare identifiers use token-boundary matching; natural-language queries use vector + lexical retrieval. | @@ -98,6 +99,52 @@ that such agents can call. --- +## Canonical Asset Model (v2 Phase 2) + +OpenMind v2 introduces a canonical content-identity model beneath the retrieval +index. The vector store becomes a *projection*; the durable source of truth is: + +```text +Workspace the project (id p_*; the REST API still says /projects) +└── Asset one logical engineering object — a source/config file + └── Revision an immutable observation of that file's exact bytes + ├── Segment a stable structural unit (Java type/method/constructor, + │ or a deterministic line-range) — verbatim or derived + └── Evidence a source-locatable citation, recoverable from the snapshot +``` + +- **Workspace vs Asset vs Revision.** A *Workspace* is the existing project. An + *Asset* is one file, identified by its normalized workspace-relative path (no + absolute path ever enters the portable database). A *Revision* is one observed + version of that file's bytes; the first observation is sequence 1, a change + creates the next sequence and supersedes the previous, and a revert (A → B → A) + is a new revision that reuses the old content blob. +- **Content snapshots.** Each revision's exact bytes are stored in an immutable, + content-addressed blob store (`data//objects/…`, keyed by SHA-256). + The database stores only the hash. This is why Evidence for a historical + revision stays readable after the source file changes on disk — and why vector + chunks are a *projection*, not the canonical store. +- **Unchanged ingestion reuses revisions.** Re-ingesting an unchanged file + creates no new revision and does not re-embed it. A Phase 1 workspace backfills + Assets on its first Phase 2 ingest *without* re-embedding unchanged files + (their existing Chroma chunks are reused). +- **Removed files preserve history.** Deleting a source file marks its Asset + `removed` and drops its live retrieval chunks, but keeps every revision, + segment, evidence and content blob. Only workspace *terminate* / *delete* wipe + Asset history. +- **Inspect it** through the CLI (`openmind asset list|show|revisions|segments| + evidence|add`), additive read-only REST endpoints under `/projects/{id}/…`, + and read-only MCP tools (`list_assets`, `get_asset`, `get_asset_revisions`, + `get_evidence`) usable straight from Claude Code. + +Deliberately **not** in this phase (still later v2 work): PDF/DOCX/XLSX parsing, +requirement and business-rule extraction, Claim/Relation tables, Knowledge-Graph +edges and requirement-to-code traceability. Nothing here claims document +knowledge or traceability exists yet. Full design: +[docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md). + +--- + ## Built For AI Agent Workflows Open Mind is designed as infrastructure for practical AI agents and tool @@ -286,6 +333,11 @@ python -m openmind.cli ingest --workspace --wait # what does it know? python -m openmind.cli status --workspace +# inspect the canonical Asset model: files -> revisions -> segments -> evidence +python -m openmind.cli asset list --workspace --type source-code --json +python -m openmind.cli asset revisions --workspace --asset --json +python -m openmind.cli asset evidence --workspace --evidence --json + # expose the knowledge layer to an editor or agent over MCP python -m openmind.cli mcp serve @@ -559,8 +611,10 @@ openmind/ domain/ typed application errors + the types crossing service calls ports/ the three narrow boundaries services depend on services/ use-case orchestration shared by CLI, MCP and FastAPI - (workspace, ingest, job, export, health + the container) - migrations/ versioned, checksummed SQLite schema migrations + (workspace, ingest, job, asset, export, health + the container) + content_store.py immutable SHA-256 content-addressed blob store (revision snapshots) + segmentation.py deterministic Segment + Evidence drafts (shares boundaries with rag) + migrations/ versioned, checksummed SQLite schema migrations (v0003 = Asset model) walker.py selection-aware walk, .gitignore handling, hashing detect.py manifest/language detection and stack cues langspec.py declarative language registry @@ -710,15 +764,19 @@ against the neutral fixture repos in `fixtures/`. The following are not claimed as complete in the current build: -**v2 enterprise knowledge layer** — none of this is implemented. Phase 1 -(shipped) is the tool-first runtime foundation described in -[docs/v2/phase-1-core-foundation.md](docs/v2/phase-1-core-foundation.md); it -deliberately creates extension points for the items below rather than building -them: - -- enterprise Asset, Revision, Claim and Relation models; -- the engineering Knowledge Graph; -- PDF, DOCX and XLSX parsing; COBOL and JCL support; +**v2 enterprise knowledge layer.** Two foundation phases have shipped: +Phase 1 is the tool-first runtime +([docs/v2/phase-1-core-foundation.md](docs/v2/phase-1-core-foundation.md)), and +Phase 2 is the canonical **Asset / Revision / Segment / Evidence** model plus the +immutable content store +([docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md)). The following +later-phase items are **not** implemented; the foundation creates extension +points for them rather than building them: + +- Claim and Relation models; +- the engineering Knowledge Graph and requirement-to-code traceability; +- PDF, DOCX and XLSX parsing; OCR; COBOL and JCL support; requirement and + business-rule extraction; - cloud model providers (OpenAI, Anthropic, Bedrock, Azure, Vertex) — the runtime remains local-first with no cloud credentials; - requirement-to-code traceability, conflict detection and branch overlays; diff --git a/docs/cli.md b/docs/cli.md index 33430c5..e028a6c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -175,6 +175,57 @@ running FastAPI server. A count of `unknown` (`null` in JSON) means that store could not be read — it is deliberately distinct from `0`, which means "read successfully, and empty". +### `asset` + +Inspect the canonical **Asset model** (OpenMind v2 Phase 2): every indexed +file is an **Asset**, every observed version of it an immutable **Revision**, +each revision divided into **Segments**, each with source-locatable **Evidence**. +See [docs/v2/phase-2-asset-model.md](v2/phase-2-asset-model.md) for the model. + +All subcommands take `--workspace` and support `--json` (exactly one object on +stdout). Lists are bounded and report a total; content is never printed unbounded. + +```bash +# list assets (bounded; filter by type/state) +python -m openmind.cli asset list --workspace p_... --type source-code \ + --state active --limit 100 --json + +# one asset + its current-revision summary +python -m openmind.cli asset show --workspace p_... --asset a_... --json + +# an asset's revision history (newest first) +python -m openmind.cli asset revisions --workspace p_... --asset a_... --json + +# a revision's segments (each carries an evidence_id for the next step) +python -m openmind.cli asset segments --workspace p_... --revision r_... \ + --limit 100 --json + +# one evidence citation: locator, snapshot + current-source status, +# and bounded content recovered from the immutable snapshot +python -m openmind.cli asset evidence --workspace p_... --evidence e_... \ + --max-chars 4000 --json + +# ingest a single existing file that lives under a registered source root +python -m openmind.cli asset add --workspace p_... --path ./src/File.java \ + --wait --json +``` + +`asset add` accepts **one existing file** under an already-registered source +root; a directory is rejected (exit `2`) with a pointer to `openmind add`. An +unsupported format is registered as an `unsupported` Asset — recorded honestly, +never falsely reported as parsed — and not ingested. Without `--wait` it returns +the job id. + +`asset evidence` reports both a **snapshot** status (`available` / `corrupt` / +`missing`, recovered from the immutable content blob) and a **current source** +status (`matches` / `changed` / `missing`), so historical evidence stays readable +after the source file changes. Cross-workspace access to any asset/revision/ +segment/evidence id is a typed not-found (exit `1`), never a leak. + +`status` additionally reports asset counts (`assets_total` / `assets_active` / +`assets_removed`, `revisions`, `segments`, `evidence`) — additive; every prior +`status` key is unchanged. + ### `export` ```bash @@ -200,9 +251,12 @@ python -m openmind.cli mcp serve ``` Runs the MCP stdio server — the *same* implementation as -`python -m openmind.mcp_server`, not a second copy of the tools. The nine tools -(`search`, `route`, `dispatch`, `get_glossary`, `find_similar_cases`, -`save_case`, `get_doc`, `propose_fix`, `apply_fix`) are unchanged. +`python -m openmind.mcp_server`, not a second copy of the tools. The nine core +tools (`search`, `route`, `dispatch`, `get_glossary`, `find_similar_cases`, +`save_case`, `get_doc`, `propose_fix`, `apply_fix`) are unchanged. Phase 2 adds +four **read-only** Asset tools alongside them (`list_assets`, `get_asset`, +`get_asset_revisions`, `get_evidence`); merely serving MCP never starts the +ingestion worker. stdout is the MCP transport, so all CLI chatter goes to stderr on this command. diff --git a/docs/database-migrations.md b/docs/database-migrations.md index e596054..d985ba1 100644 --- a/docs/database-migrations.md +++ b/docs/database-migrations.md @@ -75,18 +75,33 @@ runner switches the connection to explicit-transaction mode | --- | --- | --- | | 1 | `baseline` | the schema as it stood before migrations existed: `projects`, `jobs`, `model_config`, `file_index`, `kv`, `ask_history` + its index | | 2 | `paths_sidecar` | moves legacy in-DB project paths into the machine-local sidecar and blanks `projects.paths_json` | +| 3 | `asset_model` | the canonical Asset model: `assets`, `asset_revisions`, `segments`, `evidence` + their indexes. Additive — creates only new tables (all `IF NOT EXISTS`), touches no existing row | + +`v0003` adds the OpenMind v2 canonical content-identity model. `assets` +references `projects(id)` and the whole subtree cascades on `ON DELETE CASCADE`, +so removing a project drops its assets, revisions, segments and evidence in one +statement (with the connection's `PRAGMA foreign_keys=ON`). `(asset_id, +content_hash)` is deliberately **not** unique so an A → B → A revert is +representable. Content bytes live in an immutable content-addressed blob store +(`data//objects/…`), never in the database — the DB stores only the +SHA-256 hash. See [docs/v2/phase-2-asset-model.md](v2/phase-2-asset-model.md). ### Upgrading an existing database Nothing to do — open OpenMind and it migrates itself. Concretely: ```text -empty database -> v0001 creates every table -> ledger records 1, 2 -legacy database -> v0001 statements are all no-ops -> ledger records 1, 2 - (data untouched) -current database -> nothing to apply -> no writes +empty database -> v0001..v0003 create every table -> ledger records 1, 2, 3 +legacy database -> v0001 statements are all no-ops, -> ledger records 1, 2, 3 + v0002/v0003 apply additively (existing data untouched) +current database -> nothing to apply -> no writes ``` +A Phase 1 database (already at v0002) upgrades to v0003 with no data loss: +`v0003` only *creates* the new Asset tables. The Asset rows for existing files +are then backfilled on the next ingestion — without re-embedding unchanged +files, reusing their existing Chroma chunks. + A legacy database is **baselined, not recreated**. `v0001` is written entirely with `CREATE TABLE IF NOT EXISTS`, so against a database that already has those tables every statement is a no-op and the runner simply records version 1. diff --git a/docs/v2/phase-2-asset-model.md b/docs/v2/phase-2-asset-model.md new file mode 100644 index 0000000..3b18b7b --- /dev/null +++ b/docs/v2/phase-2-asset-model.md @@ -0,0 +1,374 @@ +# OpenMind v2 — Phase 2: Workspace Asset, Revision, Segment and Evidence Foundation + +Status: implemented on branch `feat/v2-phase2-asset-model`. +Runtime version introduced by this phase: **`1.2.0-dev`**. +Artifact export contract: **unchanged** — `.openmind` schema stays `1.1.0`. + +This phase builds the canonical engineering-asset data foundation that every +later OpenMind v2 capability depends on. It is a **data-model foundation**, not +a feature release, and it deliberately implements **none** of the document +parsing, requirement/business-rule extraction, Claim/Relation tables or +Knowledge-Graph work (see [§13 Deferred work](#13-deferred-phase-3-work)). + +It builds directly on the Phase 1 boundaries — `OpenMindRuntime`, the +`ServiceContainer`, the strict SQLite migration runner, the typed domain +errors, and the CLI / FastAPI / MCP adapters — and does not rewrite them. + +--- + +## 1. Baseline: what already worked + +Phase 2 started from a clean working tree at `20c1026` (branch `main`), with the +whole core acceptance suite green. Recorded before any change was made +(`python scripts/run_acceptance.py --json`, 24/24 core scripts): + +| Script | Result | Script | Result | +| --- | --- | --- | --- | +| verify_migrations | 49 passed | verify_fixes | 7/7 | +| verify_structure | 14 passed | verify_fixes2 | 7/7 | +| verify_glossary | 20 passed | verify_resources | 15 passed | +| verify_router | 23 passed | verify_ask | 15/15 | +| verify_diagrams | 7 passed | verify_ask2 | 29/29 | +| verify_grounding | 8 passed | verify_async_delete | 6/6 | +| verify_artifacts | 31 passed | verify_delete_race | 26/26 | +| verify_runtime | 31 passed | verify_source_link | 25 passed | +| verify_services | 97 passed | verify_modelserver | 10/10 | +| verify_cli | 112 passed | verify_templates | 21/21 | +| verify_adapters | 85 passed | verify_facets | 22/22 | +| verify | 17/17 | verify_guide | 19/19 | + +Result: `ok=true, 24 passed, 0 failed, 0 skipped`. Schema head before this phase: +**version 2** (`0001_baseline`, `0002_paths_sidecar`). + +--- + +## 2. Current storage model (before this phase) + +``` +Project (projects row, state, meta) +├── paths → machine-local sidecar (~/.openmind/paths.json) +├── file_index → per-file SHA-1 hash + chunk-id list (incremental key) +├── map/*.json → glossary, structure, facets (deterministic artifacts) +└── vector chunks → Chroma collection (RAG projection) +``` + +The per-file **SHA-1 content hash** (`walker.hash_text`) is the incremental key: +an unchanged file is neither re-scanned nor re-embedded. There is **no record of +prior versions**: once a source file changes, its previous content is gone from +OpenMind's stores — the only surviving copy is the live file on disk. + +## 3. Target canonical model (this phase) + +``` +Workspace (= the existing project row; id p_*) +└── Asset one logical engineering object (a source/config file) + └── AssetRevision an immutable observation of that Asset's bytes + ├── Segment a stable structural unit inside one revision + └── Evidence a source-locatable citation for a segment +``` + +The Asset model becomes the **durable source of engineering-content identity**. +Chroma remains a *retrieval projection* built from the same source bytes; it is +not the canonical knowledge store. Historical content lives in an **immutable +content-addressed blob store** so Evidence for an old revision is still readable +after the source file changes. + +"Workspace" is internal vocabulary. The stored entity is still a project, the +REST API still says `/projects`, and `workspace_id` **is** the existing `p_*` +project id. Nothing about the project row's shape changes. + +--- + +## 4. Identity rules + +* **Asset identity** = `(workspace_id, logical_key)`, enforced by a UNIQUE + constraint. For a source file, `logical_key` = the **normalized + workspace-relative path** (e.g. `src/services/order-service.ts`, + `config/application.yaml`). It is produced by `machine.to_rel(workspace_id, + abspath)`, exactly the same relativization the RAG index and glossary already + use. **No absolute path ever enters the portable database** (zero-origin-traces + constraint); the machine-local root stays in the sidecar. +* `source_path` is the same workspace-relative key; `source_kind` = `"file"`. +* **Asset type** is deterministic and extension/path based (§10), never a model. +* Two different files never merge into one Asset; one file never splits. + +## 5. Revision rules + +An `AssetRevision` is an immutable observation of an Asset's contents. + +1. First observed content creates **sequence 1**. +2. Re-ingesting **unchanged** content creates **no new revision**. +3. Changed content creates the **next sequence**. +4. The new revision records `supersedes_revision_id` = the previous current + revision; the previous revision's `status` becomes `superseded`. +5. Reverting to historically-seen content (A → B → A) **still creates a new + revision** (sequence 3). The blob is reused; the transition is recorded. +6. A revision is **immutable** after creation (no UPDATE ever touches its row + except the one-time `status → superseded` flip in rule 4). +7. `Asset.current_revision_id` points at the active revision. +8. Removing a source file sets `Asset.state = removed`; it does **not** delete + any revision, segment, evidence or blob. +9. A reappearing logical key **reactivates the same Asset**; a new revision is + created only when the observed content differs from the last current revision. + +**Change signal.** The revision layer reuses the ingestion's existing per-file +change decision (the SHA-1 `file_index` comparison) to decide *when to look*, and +then compares the file's **canonical SHA-256** against the Asset's current +revision `content_hash` to decide *whether a new revision is required*. This +keeps revision creation in lockstep with RAG re-embedding without hashing the +whole corpus twice, while still letting a reappeared-but-identical file reactivate +its Asset without minting a redundant revision. + +### 5.1 Revision status + +Vocabulary: `unknown, draft, reviewed, approved, effective, superseded, +withdrawn, archived`. Code revisions in Phase 2 are created `unknown` — approval +authority is **not** inferred. The only automatic transition is +`unknown → superseded` when a newer current revision is created. + +### 5.2 Source commit + +The Git commit SHA is recorded when it can be determined **locally**, with no +network access and no failure outside a repo. A per-ingestion cache maps a repo +root → its resolved `HEAD` SHA (read once from `.git/HEAD` + refs, never by +shelling out per file). A dirty working tree is recorded in revision metadata +(`git_dirty: true`), never by inventing a SHA. When no Git data is available, +`source_commit` is `""`. + +## 6. Content snapshot strategy + +`openmind/content_store.py` — a small immutable, content-addressed blob store: + +``` +data//objects// +``` + +* `put(workspace_id, data: bytes) -> blob_hash` — atomic (`tmp` + `os.replace`), + reuses an existing matching blob, returns the SHA-256 hex. +* `get(workspace_id, blob_hash) -> bytes` — re-hashes on read; a mismatch raises + `ContentCorruption` (never returns silently-wrong bytes). +* `exists`, `verify` — presence and integrity checks. + +Invariants: blob identity is SHA-256 of the exact bytes; the database stores only +the blob hash (never an absolute blob path); old revision blobs are retained; +binary bytes round-trip even when Phase 2 cannot parse them; blobs live inside +`data//`, so workspace delete removes them with the data dir and +workspace terminate removes them explicitly. Snapshots are **never** stored in +Chroma. The store is testable with no Chroma and no FastAPI. + +## 7. Segment strategy + +A `Segment` is a stable structural unit inside one revision. Phase 2 introduces +`openmind/segmentation.py`, a **deterministic** segmenter that shares its +boundary primitives with the RAG chunker (the tree-sitter helpers in +`javaparse.py` and the `CHUNK_MAX_LINES` / `CHUNK_OVERLAP_LINES` line-range split +used by `rag.chunk_file`). Producing canonical Segments and preserving the exact +current RAG chunk projection are decoupled: the segmenter emits Segment records; +`rag.chunk_file` is **left byte-for-byte unchanged**, so search behaviour, +chunk ids and chunk metadata cannot regress. A test asserts the two agree on line +ranges, which is what keeps them from drifting. + +* **Java** (tree-sitter available and parse succeeds): one `type` segment per + class/interface/record/enum (content mode `derived` — the signature summary, + matching the class-summary retrieval chunk), one `method` / `constructor` + segment per member (content mode `verbatim`). Real source line ranges. +* **Generic source / config / parse-failure**: deterministic bounded + `file` line-range segments (`_split_lines(text, 1, CHUNK_MAX_LINES, + CHUNK_OVERLAP_LINES)`), 1:1 with the current generic RAG chunks. + +**Segment identity.** `segment_key` is stable within one revision: +`type:org.example.Client`, `method:org.example.Client#send(Request)`, +`file-range:000001`. Ambiguous/duplicate symbols get a deterministic +`@` position suffix rather than a bare array index. + +**Content mode.** `verbatim` for real source slices; `derived` for the generated +Java class summary — a generated summary is never misrepresented as verbatim +source. `segment.content_hash` is SHA-256 of the represented content. + +## 8. Evidence rules + +Every Segment gets exactly one Evidence row citing its source range: + +```json +{ "kind": "source-range", + "file": "src/services/order-service.ts", + "startLine": 10, "endLine": 42, + "symbol": "OrderService.create(Request)" } +``` + +* `file` is workspace-relative; line numbers are 1-based. +* `evidence.content_hash` = SHA-256 of the **verbatim** source slice at + `[startLine, endLine]`, so it is recomputable from the immutable revision blob. + For a `derived` segment the evidence still cites the **verbatim** class range. +* `excerpt` is a bounded, verbatim preview. +* No Evidence cites a file outside the workspace source roots. +* Evidence retrieval reports source state honestly, distinguishing: + **current source matches / source changed / source missing / snapshot available / + snapshot corrupt** — recovered from the immutable blob, never a running model. + +## 9. Incremental-ingestion transaction boundary + +Integrated into the existing hash-keyed ingest (`jobs._run_ingest` step 4), not a +rewrite. Per changed file, ordered for crash-recoverability: + +1. Build Segment + Evidence drafts in memory (deterministic; no I/O). +2. Write the immutable content **blob** (idempotent, content-addressed). +3. Build + upsert the **vector projection** (existing `rag.embed_and_upsert`, + batched across files, stable ids → idempotent). +4. In **one** SQLite transaction, commit `AssetRevision` + `Segments` + + `Evidence`, flip the superseded status, set `current_revision_id`, and write + the compatibility `file_index` row. + +If the DB commit fails after the vector upsert, the job fails honestly; because +the blob and vector upsert are idempotent and `file_index` still holds the old +state, the next ingestion safely repeats them. **A revision is never made current +before its segments and evidence are committed**, so a partial revision can never +be observed. + +### 9.1 Unchanged legacy backfill (mandatory) + +A Phase 1 workspace has `file_index` rows, Chroma chunks and maps but no Asset +rows. On the first Phase 2 ingestion, **unchanged** files must receive +Asset/Revision/Segment/Evidence records **without being re-embedded** and while +**reusing** their existing Chroma data. The ingest loads an in-memory *asset +index* once (like `file_index`); a file that the `file_index` reports unchanged +but that has no in-sync Asset is backfilled through the blob + transactional +commit path **with the embedding step skipped entirely**. A test asserts the +embedding function is never called during an unchanged-legacy backfill. + +### 9.2 Removed files + +During prune (`jobs._run_ingest` step 5), a file no longer present: +delete its active vector chunks, delete its `file_index` row (existing +behaviour), and **mark its Asset `removed`** — preserving every revision, segment, +evidence and blob. Normal source deletion never erases Asset history. + +### 9.3 Reappearing files + +A removed logical key that reappears reactivates the same Asset (`state = active`) +and creates a new revision only when its content differs from the last current +revision; previous history is preserved. + +### 9.4 Progress counters (additive) + +`assets_created, assets_reused, assets_reactivated, assets_removed, +revisions_created, revisions_reused, segments_created, evidence_created, +content_blobs_created, content_blobs_reused`. No existing progress key is removed +or renamed. + +## 10. SQLite schema (migration `v0003_asset_model`) + +A new immutable migration; `v0001` / `v0002` are untouched. Tables: + +* **`assets`** — `id, workspace_id, logical_key, asset_type, title, source_kind, + source_path, media_type, state, current_revision_id, metadata_json, + created_at, updated_at`; `UNIQUE(workspace_id, logical_key)`; + `FOREIGN KEY(workspace_id) REFERENCES projects(id) ON DELETE CASCADE`. +* **`asset_revisions`** — `id, asset_id, sequence, content_hash, content_size, + content_blob_hash, status, version_label, source_commit, + supersedes_revision_id, metadata_json, created_at`; + `UNIQUE(asset_id, sequence)`; FK → `assets(id) ON DELETE CASCADE`. + `(asset_id, content_hash)` is deliberately **not** unique — A → B → A must be + representable. +* **`segments`** — `id, revision_id, segment_key, segment_type, ordinal, + start_line, end_line, symbol, content_hash, content_mode, metadata_json`; + `UNIQUE(revision_id, segment_key)`; FK → `asset_revisions(id) ON DELETE CASCADE`. +* **`evidence`** — `id, revision_id, segment_id, locator_json, excerpt, + content_hash, created_at`; FK → `asset_revisions(id)` and `segments(id)`, + both `ON DELETE CASCADE`. + +Indexes: workspace asset listing `(workspace_id, state)`; logical-key lookup +`(workspace_id, logical_key)` (covered by UNIQUE); current-revision + history +`(asset_id, sequence)`; segment symbol `(symbol)`; segment type +`(revision_id, segment_type)`; evidence-by-revision `(revision_id)` and +evidence-by-segment `(segment_id)`. No speculative Claim/Relation tables. + +**Deletion.** With `PRAGMA foreign_keys=ON` (already set on the shared +connection), `db.delete_project`'s `DELETE FROM projects` cascades through +`assets → asset_revisions → segments → evidence`. `db.delete_project` also adds +explicit `DELETE FROM assets WHERE workspace_id=?` for clarity and for any future +caller that disables the pragma. Blobs are removed with the data dir by +`jobs._cleanup_deleted`. + +## 11. Persistence, service and adapters + +* **`openmind/ports/asset_repository.py`** — a narrow `Protocol` (the test seam), + structurally satisfied by `openmind.db`. +* **`openmind/db.py`** — a focused, grouped "Asset model" section: `upsert_asset, + get_asset, find_asset_by_logical_key, list_assets, count_assets, list_asset_index, + create_revision, get_revision, list_revisions, set_current_revision, + commit_revision (the single transactional writer), replace_segments_and_evidence_for_revision, + list_segments, get_segment, get_evidence, mark_asset_removed, reactivate_asset, + clear_workspace_assets`. Every revision+segments+evidence+current-pointer write + is one transaction. +* **`openmind/services/asset_service.py`** — `AssetService`, exposed as + `runtime.assets` and `ServiceContainer.assets`. Methods: `list_assets, get_asset, + get_asset_by_logical_key, list_revisions, get_revision, list_segments, + get_segment, get_evidence, stats, sync_file`. **Every read verifies the entity + belongs to the supplied workspace** — an Asset id from workspace A is never + readable through workspace B. Typed errors `AssetNotFound, RevisionNotFound, + SegmentNotFound, EvidenceNotFound, ContentCorruption` map to HTTP/CLI through the + existing inheritance-driven mapping (`http_status` / `exit_code`). +* **CLI** — an additive `asset` command group: `list, show, revisions, segments, + evidence, add`; plus additive Asset counts on `status`. Same one-JSON-object + stdout contract and exit codes. +* **REST** — additive read-only routes under the existing `/projects/{id}` tree + (`/assets`, `/assets/stats`, `/assets/{id}`, `/assets/{id}/revisions`, + `/revisions/{id}`, `/revisions/{id}/segments`, `/evidence/{id}`), plus an + optional `POST /projects/{id}/assets/sync` delegating to `AssetService.sync_file`. + Existing routes and the `/projects` naming are unchanged; cross-workspace access + returns 404; lists are bounded. +* **MCP** — additive read-only tools `list_assets, get_asset, get_asset_revisions, + get_evidence`. The existing nine tools are byte-for-byte unchanged; `mcp serve` + still does not start the ingestion worker. + +## 12. Compatibility requirements + +* **REST**: all existing routes/response shapes operational; `/projects` not + renamed; new endpoints additive. +* **MCP**: `search, route, dispatch, get_glossary, find_similar_cases, save_case, + get_doc, propose_fix, apply_fix` retain names, arguments and response shapes. +* **Artifact export**: command and `.openmind` schema `1.1.0` unchanged; the Asset + model is **not** exported yet. +* **Skill bridge**: JSON-lines protocol unchanged and independent of the app DB. +* **Existing databases**: migrate to head with no loss of projects, paths, jobs, + file index, Ask history, cases, template metadata, maps or Chroma collections. +* **Glossary / structure / RAG / UI**: unchanged; the UI still loads existing + workspaces (no UI rewrite). + +## 12.1 Testing strategy + +New acceptance scripts, each registered in `scripts/run_acceptance.py` (an +unregistered `verify_*.py` fails the runner) and gated in CI: + +* **`verify_content_store.py`** — atomic write, blob reuse, changed→new blob, + SHA-256 identity, corruption detection, no absolute path stored, binary + round-trip, workspace cleanup. +* **`verify_asset_model.py`** — migration (tables/indexes, empty→head, legacy + upgrade with no data loss, checksum immutability, no-op re-run, FK cascade, + failed-migration rollback); first ingest (Assets/Revisions/Segments/Evidence, + valid 1-based ranges, verbatim excerpt, workspace-relative paths); idempotency + (0 new revisions, stable counts, blob reuse, **no re-embed**); change (one new + revision, prior still queryable, historical evidence intact, supersedes chain); + revert A→B→A (three revisions, blob reuse); removal (Asset removed, history + kept); reappearance (same Asset reactivated); legacy backfill (**no embed**). +* **`verify_asset_cli.py`** — every `asset` command supports `--json`, one JSON + object on stdout, cross-workspace access fails honestly, typed not-found, + bounded lists, bounded evidence output. +* **`verify_asset_adapters.py`** — all old REST routes + MCP tools still present; + new endpoints/tools work; cross-workspace reads 404; evidence locators are + source-traceable. + +Lifecycle regressions (`verify_async_delete`, `verify_delete_race`) must stay +green; delete stays responsive and the startup janitor race fix is preserved. + +## 13. Deferred Phase 3+ work + +Intentionally **not** implemented here: PDF/DOCX/XLSX parsing, OCR pipelines, +COBOL/JCL parsers, requirement/business-rule extraction, cloud LLM providers, +induced Project Lens, Claim/Relation tables, Knowledge-Graph edges, +requirement-to-code traceability, conflict detection, branch/PR overlays, +webhooks, Bundle 2.0, Titan Mind / Agent Skill Forge integration, new workflow +Skills, Neo4j, per-project Chroma-directory migration, a typed worker pool, and a +new UI. Document knowledge and requirement traceability **do not exist yet**; +nothing in this phase should be read as claiming they do. diff --git a/openmind/cli.py b/openmind/cli.py index 7f86817..3034433 100644 --- a/openmind/cli.py +++ b/openmind/cli.py @@ -305,6 +305,9 @@ def cmd_status(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, An status = runtime.workspaces.status(args.workspace) jobs = runtime.jobs.list([args.workspace]) active = [j for j in jobs if j.get("status") in ("queued", "running")] + # Additive Asset-model counts (assets_total/active/removed, revisions, + # segments, evidence). Backward compatible: every prior key is unchanged. + assets = runtime.assets.stats(args.workspace) payload = _ok({ "workspace_id": args.workspace, @@ -313,6 +316,9 @@ def cmd_status(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, An "paths": status["paths"], "template": status["template"], "counts": status["counts"], + "assets": {k: assets[k] for k in ( + "assets_total", "assets_active", "assets_removed", + "revisions", "segments", "evidence")}, "source_root": runtime.workspaces.source_root(args.workspace), "active_jobs": active, "recent_jobs": jobs[:args.jobs], @@ -345,6 +351,13 @@ def human(p: Dict[str, Any]) -> None: value = p["counts"].get(key) # None means "could not read that store", NOT zero. print(f" {label:16} {'unknown' if value is None else value}") + a = p["assets"] + print(" assets:") + print(f" {'total':16} {a['assets_total']} " + f"({a['assets_active']} active, {a['assets_removed']} removed)") + print(f" {'revisions':16} {a['revisions']}") + print(f" {'segments':16} {a['segments']}") + print(f" {'evidence':16} {a['evidence']}") if p["active_jobs"]: print(" active jobs:") for job in p["active_jobs"]: @@ -362,6 +375,164 @@ def human(p: Dict[str, Any]) -> None: return EXIT_OK, payload +# --------------------------------------------------------------------------- +# Asset model inspection (OpenMind v2 Phase 2) +# --------------------------------------------------------------------------- +def cmd_asset_list(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """List a workspace's assets (bounded), with the total count.""" + from .runtime import get_runtime + + result = get_runtime().assets.list_assets( + args.workspace, asset_type=args.type, state=args.state, + limit=args.limit, offset=args.offset) + payload = _ok(result) + + def human(p: Dict[str, Any]) -> None: + print(f"{p['count']} of {p['total']} asset(s) " + f"(limit {p['limit']}, offset {p['offset']}):") + for a in p["assets"]: + rev = a.get("current_revision_id") or "-" + print(f" {a['id']} {a['state']:11} {a['asset_type']:16} " + f"{a['logical_key']} (rev {rev})") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_asset_show(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """Show one asset's metadata and its current-revision summary.""" + from .runtime import get_runtime + + asset = get_runtime().assets.get_asset(args.workspace, args.asset) + payload = _ok({"asset": asset}) + + def human(p: Dict[str, Any]) -> None: + a = p["asset"] + print(f"{a['logical_key']} ({a['id']})") + print(f" type: {a['asset_type']}") + print(f" state: {a['state']}") + print(f" created: {a['created_at']}") + cur = a.get("current_revision") + if cur: + print(f" current revision: {cur['id']} (seq {cur['sequence']}, " + f"{cur['segment_count']} segment(s), status {cur['status']})") + print(f" content_hash: {cur['content_hash']}") + if cur.get("source_commit"): + print(f" source_commit: {cur['source_commit']}") + else: + print(" current revision: none") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_asset_revisions(args: argparse.Namespace, + out: Output) -> Tuple[int, Dict[str, Any]]: + """List an asset's revision history, newest first.""" + from .runtime import get_runtime + + result = get_runtime().assets.list_revisions( + args.workspace, args.asset, limit=args.limit) + payload = _ok(result) + + def human(p: Dict[str, Any]) -> None: + print(f"{p['count']} revision(s) for asset {p['asset_id']}:") + for r in p["revisions"]: + print(f" seq {r['sequence']:>4} {r['status']:11} " + f"{r['content_hash'][:12]} {r['created_at']}" + + (f" commit {r['source_commit'][:10]}" if r["source_commit"] else "")) + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_asset_segments(args: argparse.Namespace, + out: Output) -> Tuple[int, Dict[str, Any]]: + """List a revision's segments (bounded). Never prints full source content.""" + from .runtime import get_runtime + + result = get_runtime().assets.list_segments( + args.workspace, args.revision, limit=args.limit, offset=args.offset) + payload = _ok(result) + + def human(p: Dict[str, Any]) -> None: + print(f"{p['count']} of {p['total']} segment(s) for revision " + f"{p['revision_id']}:") + for s in p["segments"]: + span = (f"{s['start_line']}-{s['end_line']}" + if s.get("start_line") else "-") + print(f" {s['ordinal']:>4} {s['segment_type']:11} {s['content_mode']:8} " + f"L{span:<11} {s['segment_key']}") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_asset_evidence(args: argparse.Namespace, + out: Output) -> Tuple[int, Dict[str, Any]]: + """Show one evidence citation: its locator, snapshot/current-source status + and bounded recovered content.""" + from .runtime import get_runtime + + result = get_runtime().assets.get_evidence( + args.workspace, args.evidence, max_chars=args.max_chars) + payload = _ok(result) + + def human(p: Dict[str, Any]) -> None: + loc = p["locator"] + print(f"evidence {p['id']} (revision {p['revision_id']})") + print(f" locator: {loc.get('file')}:{loc.get('startLine')}-{loc.get('endLine')}" + f" {loc.get('symbol') or ''}") + print(f" snapshot: {p['snapshot']['status']}") + print(f" current source: {p['current_source']['status']}") + rev = p.get("revision") or {} + if rev: + print(f" revision: seq {rev['sequence']} ({rev['status']})") + print(f" content ({len(p['content'])} chars" + f"{', truncated' if p['truncated'] else ''}):") + for line in p["content"].splitlines(): + print(f" | {line}") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_asset_add(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: + """Ingest a single existing file that lives under a registered source root.""" + from .runtime import get_runtime + + runtime = get_runtime() + if args.wait: + out.say("Starting the job worker...") + try: + result = runtime.assets.sync_file(args.workspace, args.path, + wait=args.wait, timeout=args.timeout) + except OperationTimeout as exc: + out.warn(str(exc)) + payload = {"ok": False, "error": exc.as_dict()} + payload.update(exc.details) + return EXIT_TIMEOUT, payload + + payload = _ok(result) + if not result.get("supported"): + out.say(f"Registered unsupported file {result['logical_key']} " + f"(state: {result['state']}); not parsed.") + elif result.get("waited") and not result.get("completed"): + payload["ok"] = False + payload["error"] = {"code": "job_failed", + "message": f"asset add did not complete " + f"(status: {result.get('status')})"} + out.warn(payload["error"]["message"]) + out.emit(payload, lambda p: print(p.get("job_id", ""))) + return EXIT_JOB_FAILURE, payload + else: + out.say(f"Synced {result['logical_key']} " + f"(job {result.get('job_id')}).") + + out.emit(payload, lambda p: print(p.get("asset_id") or p.get("job_id") or "")) + return EXIT_OK, payload + + def cmd_export(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: """Generate the deterministic ``.openmind`` artifact directory. @@ -554,6 +725,65 @@ def build_parser() -> argparse.ArgumentParser: mcp_serve.set_defaults(func=cmd_mcp_serve) mcp.set_defaults(func=None, _parser=mcp) + asset = sub.add_parser("asset", parents=[common], + help="inspect the canonical Asset model") + asset_sub = asset.add_subparsers(dest="asset_command", metavar="") + + a_list = asset_sub.add_parser("list", parents=[common], + help="list a workspace's assets (bounded)") + a_list.add_argument("--workspace", required=True, help="workspace id") + a_list.add_argument("--type", dest="type", help="filter by asset type") + a_list.add_argument("--state", help="filter by state (active/removed/unsupported)") + a_list.add_argument("--limit", type=int, default=100, metavar="N", + help="max assets to return (default: 100)") + a_list.add_argument("--offset", type=int, default=0, metavar="N", + help="skip the first N (default: 0)") + a_list.set_defaults(func=cmd_asset_list) + + a_show = asset_sub.add_parser("show", parents=[common], + help="show one asset + its current revision") + a_show.add_argument("--workspace", required=True, help="workspace id") + a_show.add_argument("--asset", required=True, help="asset id") + a_show.set_defaults(func=cmd_asset_show) + + a_revs = asset_sub.add_parser("revisions", parents=[common], + help="list an asset's revision history") + a_revs.add_argument("--workspace", required=True, help="workspace id") + a_revs.add_argument("--asset", required=True, help="asset id") + a_revs.add_argument("--limit", type=int, default=50, metavar="N", + help="max revisions to return (default: 50)") + a_revs.set_defaults(func=cmd_asset_revisions) + + a_segs = asset_sub.add_parser("segments", parents=[common], + help="list a revision's segments (bounded)") + a_segs.add_argument("--workspace", required=True, help="workspace id") + a_segs.add_argument("--revision", required=True, help="revision id") + a_segs.add_argument("--limit", type=int, default=100, metavar="N", + help="max segments to return (default: 100)") + a_segs.add_argument("--offset", type=int, default=0, metavar="N", + help="skip the first N (default: 0)") + a_segs.set_defaults(func=cmd_asset_segments) + + a_ev = asset_sub.add_parser("evidence", parents=[common], + help="show one evidence citation (bounded content)") + a_ev.add_argument("--workspace", required=True, help="workspace id") + a_ev.add_argument("--evidence", required=True, help="evidence id") + a_ev.add_argument("--max-chars", dest="max_chars", type=int, default=4000, + metavar="N", help="max recovered content chars (default: 4000)") + a_ev.set_defaults(func=cmd_asset_evidence) + + a_add = asset_sub.add_parser("add", parents=[common], + help="ingest a single existing file") + a_add.add_argument("--workspace", required=True, help="workspace id") + a_add.add_argument("--path", required=True, help="a single existing file") + a_add.add_argument("--wait", action="store_true", + help="wait for the ingest to finish") + a_add.add_argument("--timeout", type=float, default=3600.0, metavar="SECONDS", + help="bound for --wait (default: 3600)") + a_add.set_defaults(func=cmd_asset_add) + + asset.set_defaults(func=None, _parser=asset) + serve = sub.add_parser("serve", parents=[common], help="run the FastAPI web application") serve.add_argument("--host", default="127.0.0.1", diff --git a/openmind/content_store.py b/openmind/content_store.py new file mode 100644 index 0000000..6ec4df6 --- /dev/null +++ b/openmind/content_store.py @@ -0,0 +1,161 @@ +"""Immutable, content-addressed blob store for canonical Asset content. + +WHY THIS EXISTS +--------------- +An :class:`~openmind.domain.types.Evidence` citation must stay readable after +the source file it came from has changed on disk. The live file is not a +reliable source of historical content, and Chroma is a lossy retrieval +projection (headers, truncation, derived summaries) — neither can reconstruct +the exact bytes of a prior revision. So every observed revision's exact bytes +are snapshotted here, keyed by their SHA-256, and the database stores only the +hash. + +LAYOUT +------ +``data//objects//`` + +The blob lives INSIDE the workspace data directory, so deleting the workspace +(``shutil.rmtree(config.project_dir(id))``) reclaims its blobs with everything +else, and terminating the workspace can drop just ``objects/`` via +:func:`clear_workspace`. + +INVARIANTS +---------- +1. Blob identity is the SHA-256 of the exact bytes. +2. Writes are atomic: a temp file in the same directory + ``os.replace``. +3. An existing matching blob is reused (writing it again is a no-op). +4. A hash mismatch on read raises :class:`ContentCorruption` — never returns + silently-wrong bytes. +5. The database stores only the blob hash, never an absolute blob path. +6. Old revision blobs are retained; nothing here ever prunes. +7. Binary content round-trips even when Phase 2 cannot parse it. + +This module depends only on :mod:`config` and the domain error, so it is +testable with no Chroma, no FastAPI and no database. +""" +from __future__ import annotations + +import hashlib +import os +import shutil +import uuid +from pathlib import Path + +from . import config +from .domain.errors import ContentCorruption + +#: Read blobs in bounded chunks so verifying a large file never loads it whole +#: just to hash it. +_READ_CHUNK = 1 << 20 # 1 MiB + + +def hash_bytes(data: bytes) -> str: + """The canonical blob identity: SHA-256 hex of the exact bytes.""" + return hashlib.sha256(data).hexdigest() + + +def objects_dir(workspace_id: str) -> Path: + """The workspace's blob root: ``data//objects``.""" + return config.project_dir(workspace_id) / "objects" + + +def _blob_path(workspace_id: str, blob_hash: str) -> Path: + # A two-hex fan-out keeps any single directory from holding every blob. + prefix = blob_hash[:2] if len(blob_hash) >= 2 else "00" + return objects_dir(workspace_id) / prefix / blob_hash + + +def put(workspace_id: str, data: bytes) -> str: + """Store *data* and return its SHA-256 hex. Idempotent: an identical blob is + reused, so re-ingesting the same content writes nothing. + + Returns the blob hash the database should persist. The write is atomic — a + reader never sees a half-written blob. + """ + if not isinstance(data, (bytes, bytearray)): + raise TypeError("content_store.put expects bytes") + blob_hash = hash_bytes(bytes(data)) + dest = _blob_path(workspace_id, blob_hash) + if dest.exists(): + return blob_hash # invariant 3: reuse + dest.parent.mkdir(parents=True, exist_ok=True) + # Temp file in the SAME directory so os.replace is a same-filesystem atomic + # rename. The name is unique per CALL (pid + a random token), so two writers + # of the same not-yet-present blob never interleave writes into one temp + # file; each fully writes its own temp and atomically replaces the single + # content-addressed path with identical bytes — last writer simply wins. + tmp = dest.parent / f".{blob_hash}.{os.getpid()}.{uuid.uuid4().hex[:8]}.tmp" + try: + with open(tmp, "wb") as fh: + fh.write(data) + fh.flush() + os.fsync(fh.fileno()) + os.replace(tmp, dest) # invariant 2: atomic + finally: + if tmp.exists(): + try: + tmp.unlink() + except OSError: + pass + return blob_hash + + +def exists(workspace_id: str, blob_hash: str) -> bool: + """Whether a blob with this hash is present (does not verify integrity).""" + return bool(blob_hash) and _blob_path(workspace_id, blob_hash).is_file() + + +def get(workspace_id: str, blob_hash: str) -> bytes: + """Return the exact bytes for *blob_hash*. + + Raises :class:`ContentCorruption` if the blob is missing or if its bytes no + longer hash to *blob_hash* (invariant 4). + """ + path = _blob_path(workspace_id, blob_hash) + if not path.is_file(): + raise ContentCorruption( + f"content blob missing: {blob_hash}", blob_hash=blob_hash, + details={"workspace_id": workspace_id}) + data = path.read_bytes() + actual = hash_bytes(data) + if actual != blob_hash: + raise ContentCorruption( + f"content blob {blob_hash} is corrupt (bytes hash to {actual})", + blob_hash=blob_hash, + details={"workspace_id": workspace_id, "computed": actual}) + return data + + +def verify(workspace_id: str, blob_hash: str) -> bool: + """True iff the blob exists and its bytes still hash to *blob_hash*. + + Never raises — a missing or corrupt blob returns ``False`` — so callers that + only want a health check do not have to catch :class:`ContentCorruption`. + """ + path = _blob_path(workspace_id, blob_hash) + if not path.is_file(): + return False + h = hashlib.sha256() + try: + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(_READ_CHUNK), b""): + h.update(chunk) + except OSError: + return False + return h.hexdigest() == blob_hash + + +def clear_workspace(workspace_id: str) -> None: + """Remove every blob for a workspace (its whole ``objects/`` tree). + + Used by workspace TERMINATE, which wipes learned data but keeps the + workspace. Workspace DELETE removes the entire data directory, which + includes ``objects/``, so it does not need this. + """ + root = objects_dir(workspace_id) + if root.exists(): + shutil.rmtree(root, ignore_errors=True) + + +__all__ = ["hash_bytes", "objects_dir", "put", "get", "exists", "verify", + "clear_workspace"] diff --git a/openmind/db.py b/openmind/db.py index 6415045..b98d4ba 100644 --- a/openmind/db.py +++ b/openmind/db.py @@ -135,10 +135,15 @@ def list_projects(state: Optional[str] = None) -> List[Dict[str, Any]]: def delete_project(project_id: str) -> None: - """Remove the project entity + its jobs + file index rows + Ask history. - (Callers also drop the vector collections and the data dir — see - jobs.delete_project.)""" + """Remove the project entity + its jobs + file index rows + Ask history + + Asset model rows. (Callers also drop the vector collections and the data dir — + see jobs.delete_project.)""" with _lock: + # Assets cascade to revisions/segments/evidence via the v0003 FK + # (ON DELETE CASCADE, foreign_keys=ON). This explicit DELETE runs first + # so the cleanup is correct even if a future caller has the pragma off, + # and so the intent is visible beside the other per-project deletes. + _c().execute("DELETE FROM assets WHERE workspace_id=?", (project_id,)) _c().execute("DELETE FROM projects WHERE id=?", (project_id,)) _c().execute("DELETE FROM jobs WHERE project_id=?", (project_id,)) _c().execute("DELETE FROM file_index WHERE project_id=?", (project_id,)) @@ -572,3 +577,512 @@ def kv_set(key: str, value: str) -> None: (key, value), ) _c().commit() + + +# --------------------------------------------------------------------------- +# Canonical Asset model (OpenMind v2 Phase 2) +# +# Assets/revisions/segments/evidence are the durable content-identity model; +# Chroma stays a retrieval projection. Every write goes through the same single +# WAL connection and _lock as the rest of this module. Reads are workspace-scoped +# at the SQL level (a JOIN back to assets.workspace_id), so an id from one +# workspace can never be read through another. See migrations/versions/ +# v0003_asset_model.py for the schema. +# --------------------------------------------------------------------------- +def _asset_row(row: sqlite3.Row) -> Dict[str, Any]: + return { + "id": row["id"], + "workspace_id": row["workspace_id"], + "logical_key": row["logical_key"], + "asset_type": row["asset_type"], + "title": row["title"], + "source_kind": row["source_kind"], + "source_path": row["source_path"], + "media_type": row["media_type"], + "state": row["state"], + "current_revision_id": row["current_revision_id"], + "metadata": json.loads(row["metadata_json"]), + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +def _revision_row(row: sqlite3.Row) -> Dict[str, Any]: + return { + "id": row["id"], + "asset_id": row["asset_id"], + "sequence": row["sequence"], + "content_hash": row["content_hash"], + "content_size": row["content_size"], + "content_blob_hash": row["content_blob_hash"], + "status": row["status"], + "version_label": row["version_label"], + "source_commit": row["source_commit"], + "supersedes_revision_id": row["supersedes_revision_id"], + "metadata": json.loads(row["metadata_json"]), + "created_at": row["created_at"], + } + + +def _segment_row(row: sqlite3.Row) -> Dict[str, Any]: + return { + "id": row["id"], + "revision_id": row["revision_id"], + "segment_key": row["segment_key"], + "segment_type": row["segment_type"], + "ordinal": row["ordinal"], + "start_line": row["start_line"], + "end_line": row["end_line"], + "symbol": row["symbol"], + "content_hash": row["content_hash"], + "content_mode": row["content_mode"], + "metadata": json.loads(row["metadata_json"]), + } + + +def _evidence_row(row: sqlite3.Row) -> Dict[str, Any]: + return { + "id": row["id"], + "revision_id": row["revision_id"], + "segment_id": row["segment_id"], + "locator": json.loads(row["locator_json"]), + "excerpt": row["excerpt"], + "content_hash": row["content_hash"], + "created_at": row["created_at"], + } + + +# -- asset reads ------------------------------------------------------------ +def get_asset(workspace_id: str, asset_id: str) -> Optional[Dict[str, Any]]: + with _lock: + row = _c().execute( + "SELECT * FROM assets WHERE id=? AND workspace_id=?", + (asset_id, workspace_id), + ).fetchone() + return _asset_row(row) if row else None + + +def find_asset_by_logical_key(workspace_id: str, + logical_key: str) -> Optional[Dict[str, Any]]: + with _lock: + row = _c().execute( + "SELECT * FROM assets WHERE workspace_id=? AND logical_key=?", + (workspace_id, logical_key), + ).fetchone() + return _asset_row(row) if row else None + + +def list_assets(workspace_id: str, asset_type: Optional[str] = None, + state: Optional[str] = None, limit: int = 100, + offset: int = 0) -> List[Dict[str, Any]]: + q = "SELECT * FROM assets WHERE workspace_id=?" + args: List[Any] = [workspace_id] + if asset_type: + q += " AND asset_type=?" + args.append(asset_type) + if state: + q += " AND state=?" + args.append(state) + # Deterministic ordering so a bounded page is stable across calls. + q += " ORDER BY logical_key LIMIT ? OFFSET ?" + args.extend([max(0, int(limit)), max(0, int(offset))]) + with _lock: + rows = _c().execute(q, args).fetchall() + return [_asset_row(r) for r in rows] + + +def count_assets(workspace_id: str, asset_type: Optional[str] = None, + state: Optional[str] = None) -> int: + q = "SELECT COUNT(*) FROM assets WHERE workspace_id=?" + args: List[Any] = [workspace_id] + if asset_type: + q += " AND asset_type=?" + args.append(asset_type) + if state: + q += " AND state=?" + args.append(state) + with _lock: + row = _c().execute(q, args).fetchone() + return int(row[0]) if row else 0 + + +def list_asset_index(workspace_id: str) -> Dict[str, Dict[str, Any]]: + """logical_key -> {asset_id, state, current_revision_id, content_hash} for the + whole workspace, in one query. The ingestion loop loads this once (like + :func:`get_file_index`) to decide, per file, whether an Asset already exists + and whether its current revision already matches — without a query per file + and without re-hashing unchanged content.""" + with _lock: + rows = _c().execute( + "SELECT a.logical_key AS lk, a.id AS aid, a.state AS state, " + "a.current_revision_id AS crid, r.content_hash AS chash " + "FROM assets a LEFT JOIN asset_revisions r " + "ON a.current_revision_id = r.id WHERE a.workspace_id=?", + (workspace_id,), + ).fetchall() + return { + r["lk"]: {"asset_id": r["aid"], "state": r["state"], + "current_revision_id": r["crid"], "content_hash": r["chash"]} + for r in rows + } + + +# -- revision reads --------------------------------------------------------- +def get_revision(workspace_id: str, revision_id: str) -> Optional[Dict[str, Any]]: + with _lock: + row = _c().execute( + "SELECT r.* FROM asset_revisions r JOIN assets a ON r.asset_id=a.id " + "WHERE r.id=? AND a.workspace_id=?", + (revision_id, workspace_id), + ).fetchone() + return _revision_row(row) if row else None + + +def list_revisions(workspace_id: str, asset_id: str, + limit: int = 50) -> List[Dict[str, Any]]: + with _lock: + rows = _c().execute( + "SELECT r.* FROM asset_revisions r JOIN assets a ON r.asset_id=a.id " + "WHERE a.id=? AND a.workspace_id=? ORDER BY r.sequence DESC LIMIT ?", + (asset_id, workspace_id, max(0, int(limit))), + ).fetchall() + return [_revision_row(r) for r in rows] + + +# -- segment / evidence reads (all workspace-scoped via JOIN) --------------- +def list_segments(workspace_id: str, revision_id: str, limit: int = 200, + offset: int = 0) -> List[Dict[str, Any]]: + with _lock: + rows = _c().execute( + "SELECT s.* FROM segments s " + "JOIN asset_revisions r ON s.revision_id=r.id " + "JOIN assets a ON r.asset_id=a.id " + "WHERE s.revision_id=? AND a.workspace_id=? " + "ORDER BY s.ordinal LIMIT ? OFFSET ?", + (revision_id, workspace_id, max(0, int(limit)), max(0, int(offset))), + ).fetchall() + return [_segment_row(r) for r in rows] + + +def count_segments(workspace_id: str, revision_id: str) -> int: + with _lock: + row = _c().execute( + "SELECT COUNT(*) FROM segments s " + "JOIN asset_revisions r ON s.revision_id=r.id " + "JOIN assets a ON r.asset_id=a.id " + "WHERE s.revision_id=? AND a.workspace_id=?", + (revision_id, workspace_id), + ).fetchone() + return int(row[0]) if row else 0 + + +def get_segment(workspace_id: str, segment_id: str) -> Optional[Dict[str, Any]]: + with _lock: + row = _c().execute( + "SELECT s.* FROM segments s " + "JOIN asset_revisions r ON s.revision_id=r.id " + "JOIN assets a ON r.asset_id=a.id " + "WHERE s.id=? AND a.workspace_id=?", + (segment_id, workspace_id), + ).fetchone() + return _segment_row(row) if row else None + + +def get_evidence(workspace_id: str, evidence_id: str) -> Optional[Dict[str, Any]]: + with _lock: + row = _c().execute( + "SELECT e.* FROM evidence e " + "JOIN asset_revisions r ON e.revision_id=r.id " + "JOIN assets a ON r.asset_id=a.id " + "WHERE e.id=? AND a.workspace_id=?", + (evidence_id, workspace_id), + ).fetchone() + return _evidence_row(row) if row else None + + +def evidence_ids_for_revision(workspace_id: str, + revision_id: str) -> Dict[str, str]: + """segment_id -> evidence_id for one revision, in a single scoped query, so a + segment listing can surface each segment's evidence id (the discovery path + for ``asset evidence``) without a query per segment.""" + with _lock: + rows = _c().execute( + "SELECT e.segment_id AS sid, e.id AS eid FROM evidence e " + "JOIN asset_revisions r ON e.revision_id=r.id " + "JOIN assets a ON r.asset_id=a.id " + "WHERE e.revision_id=? AND a.workspace_id=? AND e.segment_id IS NOT NULL", + (revision_id, workspace_id), + ).fetchall() + return {r["sid"]: r["eid"] for r in rows} + + +def get_evidence_for_segment(workspace_id: str, + segment_id: str) -> Optional[Dict[str, Any]]: + with _lock: + row = _c().execute( + "SELECT e.* FROM evidence e " + "JOIN asset_revisions r ON e.revision_id=r.id " + "JOIN assets a ON r.asset_id=a.id " + "WHERE e.segment_id=? AND a.workspace_id=? " + "ORDER BY e.created_at LIMIT 1", + (segment_id, workspace_id), + ).fetchone() + return _evidence_row(row) if row else None + + +def asset_stats(workspace_id: str) -> Dict[str, int]: + """Aggregate counts for the workspace: assets (total/active/removed), + revisions, segments, evidence. One small query per count, all scoped.""" + with _lock: + c = _c() + total = c.execute("SELECT COUNT(*) FROM assets WHERE workspace_id=?", + (workspace_id,)).fetchone()[0] + active = c.execute( + "SELECT COUNT(*) FROM assets WHERE workspace_id=? AND state='active'", + (workspace_id,)).fetchone()[0] + removed = c.execute( + "SELECT COUNT(*) FROM assets WHERE workspace_id=? AND state='removed'", + (workspace_id,)).fetchone()[0] + revisions = c.execute( + "SELECT COUNT(*) FROM asset_revisions r JOIN assets a " + "ON r.asset_id=a.id WHERE a.workspace_id=?", (workspace_id,)).fetchone()[0] + segments = c.execute( + "SELECT COUNT(*) FROM segments s JOIN asset_revisions r " + "ON s.revision_id=r.id JOIN assets a ON r.asset_id=a.id " + "WHERE a.workspace_id=?", (workspace_id,)).fetchone()[0] + evidence = c.execute( + "SELECT COUNT(*) FROM evidence e JOIN asset_revisions r " + "ON e.revision_id=r.id JOIN assets a ON r.asset_id=a.id " + "WHERE a.workspace_id=?", (workspace_id,)).fetchone()[0] + return { + "assets_total": int(total), "assets_active": int(active), + "assets_removed": int(removed), "revisions": int(revisions), + "segments": int(segments), "evidence": int(evidence), + } + + +# -- writes ----------------------------------------------------------------- +def upsert_asset(workspace_id: str, logical_key: str, *, asset_type: str, + title: str, source_path: str, state: str = "active", + media_type: str = "", source_kind: str = "file", + metadata: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Create or update an Asset row WITHOUT a revision — used to register an + unsupported file honestly (``state='unsupported'``), so it is recorded but + never falsely reported as parsed. Never touches revisions/segments/evidence.""" + ts = now() + with _lock: + row = _c().execute( + "SELECT id FROM assets WHERE workspace_id=? AND logical_key=?", + (workspace_id, logical_key)).fetchone() + if row: + _c().execute( + "UPDATE assets SET asset_type=?, title=?, source_path=?, state=?, " + "media_type=?, source_kind=?, updated_at=? WHERE id=?", + (asset_type, title, source_path, state, media_type, source_kind, + ts, row["id"])) + asset_id = row["id"] + else: + asset_id = new_id("a_") + _c().execute( + "INSERT INTO assets (id,workspace_id,logical_key,asset_type,title," + "source_kind,source_path,media_type,state,current_revision_id," + "metadata_json,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (asset_id, workspace_id, logical_key, asset_type, title, source_kind, + source_path, media_type, state, None, json.dumps(metadata or {}), + ts, ts)) + _c().commit() + return get_asset(workspace_id, asset_id) # type: ignore[return-value] + + +def mark_asset_removed(workspace_id: str, logical_key: str) -> Optional[str]: + """Set an Asset's state to ``removed`` (source file gone/excluded), keeping + ALL of its revisions, segments, evidence and content blobs. Returns the asset + id if a state change happened, else None (already removed or absent).""" + with _lock: + row = _c().execute( + "SELECT id, state FROM assets WHERE workspace_id=? AND logical_key=?", + (workspace_id, logical_key), + ).fetchone() + if not row or row["state"] == "removed": + return None + _c().execute( + "UPDATE assets SET state='removed', updated_at=? " + "WHERE id=?", (now(), row["id"]), + ) + _c().commit() + return row["id"] + + +def clear_workspace_assets(workspace_id: str) -> None: + """Delete every Asset (and, by cascade, its revisions/segments/evidence) for + a workspace. Used by TERMINATE, which wipes learned data. Content blobs are + removed separately by the caller (content_store.clear_workspace).""" + with _lock: + _c().execute("DELETE FROM assets WHERE workspace_id=?", (workspace_id,)) + _c().commit() + + +def commit_revision( + workspace_id: str, + logical_key: str, + *, + asset_type: str, + title: str, + source_path: str, + content_hash: str, + content_size: int, + content_blob_hash: str, + segments: List[Dict[str, Any]], + media_type: str = "", + source_kind: str = "file", + source_commit: str = "", + revision_status: str = "unknown", + version_label: str = "", + revision_metadata: Optional[Dict[str, Any]] = None, + asset_metadata: Optional[Dict[str, Any]] = None, + asset_state: str = "active", +) -> Dict[str, Any]: + """The single transactional writer for the Asset model. + + Creates or reuses the Asset for ``(workspace_id, logical_key)`` and, when the + observed ``content_hash`` differs from the Asset's current revision, mints the + next immutable revision together with all of its segments and evidence, + flips the previous current revision to ``superseded``, and repoints + ``current_revision_id`` — ALL in one transaction. A revision is therefore + never made current before its segments and evidence are committed. + + When the content already matches the current revision this creates NO new + revision (idempotent / revert-safe at the identity layer); it still + reactivates a ``removed`` asset that has reappeared. + + ``segments`` is a list of drafts; each may carry an ``evidence`` dict + ``{locator, excerpt, content_hash}``. Returns a summary: + ``{asset_id, revision, revision_created, asset_created, reactivated, + segments_created, evidence_created}``. + """ + ts = now() + rev_meta = json.dumps(revision_metadata or {}) + with _lock: + c = _c() + try: + existing = c.execute( + "SELECT * FROM assets WHERE workspace_id=? AND logical_key=?", + (workspace_id, logical_key), + ).fetchone() + + reactivated = False + asset_created = False + + if existing is not None: + asset_id = existing["id"] + cur_rev_id = existing["current_revision_id"] + cur_hash = None + if cur_rev_id: + cr = c.execute( + "SELECT content_hash FROM asset_revisions WHERE id=?", + (cur_rev_id,)).fetchone() + cur_hash = cr["content_hash"] if cr else None + # Unchanged content that already IS the current revision: no new + # revision. Reactivate a reappeared (removed) asset. + if cur_rev_id and cur_hash == content_hash: + if existing["state"] != asset_state: + reactivated = existing["state"] == "removed" + c.execute( + "UPDATE assets SET state=?, updated_at=? WHERE id=?", + (asset_state, ts, asset_id)) + c.commit() + else: + c.commit() + rev = c.execute( + "SELECT * FROM asset_revisions WHERE id=?", + (cur_rev_id,)).fetchone() + return { + "asset_id": asset_id, + "revision": _revision_row(rev) if rev else None, + "revision_created": False, "asset_created": False, + "reactivated": reactivated, + "segments_created": 0, "evidence_created": 0, + } + # Content differs -> a new revision follows. Reactivating flag is + # true when the asset was removed before this new observation. + reactivated = existing["state"] == "removed" + supersedes = cur_rev_id + else: + asset_id = new_id("a_") + asset_created = True + supersedes = None + c.execute( + "INSERT INTO assets (id,workspace_id,logical_key,asset_type," + "title,source_kind,source_path,media_type,state," + "current_revision_id,metadata_json,created_at,updated_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (asset_id, workspace_id, logical_key, asset_type, title, + source_kind, source_path, media_type, asset_state, None, + json.dumps(asset_metadata or {}), ts, ts)) + + # next dense sequence for this asset + seq_row = c.execute( + "SELECT COALESCE(MAX(sequence),0) FROM asset_revisions WHERE asset_id=?", + (asset_id,)).fetchone() + sequence = int(seq_row[0]) + 1 + + revision_id = new_id("r_") + c.execute( + "INSERT INTO asset_revisions (id,asset_id,sequence,content_hash," + "content_size,content_blob_hash,status,version_label,source_commit," + "supersedes_revision_id,metadata_json,created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (revision_id, asset_id, sequence, content_hash, int(content_size), + content_blob_hash, revision_status, version_label, source_commit, + supersedes, rev_meta, ts)) + + seg_count = 0 + ev_count = 0 + for seg in segments: + seg_id = new_id("s_") + c.execute( + "INSERT INTO segments (id,revision_id,segment_key,segment_type," + "ordinal,start_line,end_line,symbol,content_hash,content_mode," + "metadata_json) VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (seg_id, revision_id, seg["segment_key"], seg["segment_type"], + int(seg["ordinal"]), seg.get("start_line"), seg.get("end_line"), + seg.get("symbol", ""), seg.get("content_hash", ""), + seg.get("content_mode", "verbatim"), + json.dumps(seg.get("metadata") or {}))) + seg_count += 1 + ev = seg.get("evidence") + if ev: + c.execute( + "INSERT INTO evidence (id,revision_id,segment_id,locator_json," + "excerpt,content_hash,created_at) VALUES (?,?,?,?,?,?,?)", + (new_id("e_"), revision_id, seg_id, + json.dumps(ev.get("locator") or {}), ev.get("excerpt", ""), + ev.get("content_hash", ""), ts)) + ev_count += 1 + + if supersedes: + c.execute( + "UPDATE asset_revisions SET status='superseded' WHERE id=?", + (supersedes,)) + + # repoint current + (re)activate + refresh derived asset fields last, + # so current_revision_id never names a half-written revision. + c.execute( + "UPDATE assets SET current_revision_id=?, state=?, asset_type=?, " + "title=?, source_path=?, media_type=?, updated_at=? WHERE id=?", + (revision_id, asset_state, asset_type, title, source_path, + media_type, ts, asset_id)) + c.commit() + except Exception: + _c().rollback() + raise + + rev = c.execute("SELECT * FROM asset_revisions WHERE id=?", + (revision_id,)).fetchone() + return { + "asset_id": asset_id, + "revision": _revision_row(rev) if rev else None, + "revision_created": True, "asset_created": asset_created, + "reactivated": reactivated, + "segments_created": seg_count, "evidence_created": ev_count, + } diff --git a/openmind/domain/__init__.py b/openmind/domain/__init__.py index e16a4bf..6837bed 100644 --- a/openmind/domain/__init__.py +++ b/openmind/domain/__init__.py @@ -1,17 +1,24 @@ """Domain layer: application errors and the types that cross service boundaries. Imports nothing from FastAPI, the CLI, or the persistence layer. """ -from .errors import (DependencyUnavailable, InvalidRequest, JobFailed, - JobNotFound, NotFound, OpenMindError, OperationTimeout, - WorkspaceNotFound) +from .errors import (AssetNotFound, ContentCorruption, DependencyUnavailable, + EvidenceNotFound, InvalidRequest, JobFailed, JobNotFound, + NotFound, OpenMindError, OperationTimeout, RevisionNotFound, + SegmentNotFound, WorkspaceNotFound) from .types import (ACTIVE_JOB_STATUSES, SETTLED_JOB_STATUSES, - TERMINAL_JOB_STATUSES, HealthCheck, HealthReport, - JobWaitResult, STATUS_ERROR, STATUS_OK, STATUS_WARN) + TERMINAL_JOB_STATUSES, Asset, AssetRevision, AssetState, + AssetType, ContentMode, Evidence, HealthCheck, HealthReport, + JobWaitResult, RevisionStatus, Segment, SegmentType, + SourceLocator, STATUS_ERROR, STATUS_OK, STATUS_WARN) __all__ = [ - "DependencyUnavailable", "InvalidRequest", "JobFailed", "JobNotFound", - "NotFound", "OpenMindError", "OperationTimeout", "WorkspaceNotFound", + "AssetNotFound", "ContentCorruption", "DependencyUnavailable", + "EvidenceNotFound", "InvalidRequest", "JobFailed", "JobNotFound", + "NotFound", "OpenMindError", "OperationTimeout", "RevisionNotFound", + "SegmentNotFound", "WorkspaceNotFound", "ACTIVE_JOB_STATUSES", "SETTLED_JOB_STATUSES", "TERMINAL_JOB_STATUSES", "HealthCheck", "HealthReport", "JobWaitResult", "STATUS_ERROR", "STATUS_OK", "STATUS_WARN", + "Asset", "AssetRevision", "AssetState", "AssetType", "ContentMode", + "Evidence", "RevisionStatus", "Segment", "SegmentType", "SourceLocator", ] diff --git a/openmind/domain/errors.py b/openmind/domain/errors.py index 0fcda90..bfa5fcd 100644 --- a/openmind/domain/errors.py +++ b/openmind/domain/errors.py @@ -8,10 +8,15 @@ --------------------- ---- -------- WorkspaceNotFound 404 1 JobNotFound 404 1 + AssetNotFound 404 1 + RevisionNotFound 404 1 + SegmentNotFound 404 1 + EvidenceNotFound 404 1 InvalidRequest 400 2 DependencyUnavailable 503 3 JobFailed (n/a) 4 OperationTimeout (n/a) 5 + ContentCorruption 500 4 Every error carries a machine-readable ``code`` and an optional ``details`` dict, so ``--json`` output stays parseable and callers never have to parse @@ -67,6 +72,72 @@ def __init__(self, job_id: str) -> None: self.job_id = job_id +class AssetNotFound(NotFound): + code = "asset_not_found" + + def __init__(self, asset_id: str, *, workspace_id: str = "") -> None: + details: Dict[str, Any] = {"asset_id": asset_id} + if workspace_id: + details["workspace_id"] = workspace_id + super().__init__(f"asset not found: {asset_id!r}", details=details) + self.asset_id = asset_id + self.workspace_id = workspace_id + + +class RevisionNotFound(NotFound): + code = "revision_not_found" + + def __init__(self, revision_id: str, *, workspace_id: str = "") -> None: + details: Dict[str, Any] = {"revision_id": revision_id} + if workspace_id: + details["workspace_id"] = workspace_id + super().__init__(f"revision not found: {revision_id!r}", details=details) + self.revision_id = revision_id + self.workspace_id = workspace_id + + +class SegmentNotFound(NotFound): + code = "segment_not_found" + + def __init__(self, segment_id: str, *, workspace_id: str = "") -> None: + details: Dict[str, Any] = {"segment_id": segment_id} + if workspace_id: + details["workspace_id"] = workspace_id + super().__init__(f"segment not found: {segment_id!r}", details=details) + self.segment_id = segment_id + self.workspace_id = workspace_id + + +class EvidenceNotFound(NotFound): + code = "evidence_not_found" + + def __init__(self, evidence_id: str, *, workspace_id: str = "") -> None: + details: Dict[str, Any] = {"evidence_id": evidence_id} + if workspace_id: + details["workspace_id"] = workspace_id + super().__init__(f"evidence not found: {evidence_id!r}", details=details) + self.evidence_id = evidence_id + self.workspace_id = workspace_id + + +class ContentCorruption(OpenMindError): + """A stored content blob failed its SHA-256 integrity check on read, or an + evidence range no longer resolves against the immutable snapshot. The bytes + are never returned silently-wrong; the failure is explicit.""" + + code = "content_corruption" + exit_code = 4 + http_status = 500 + + def __init__(self, message: str, *, blob_hash: str = "", + details: Optional[Dict[str, Any]] = None) -> None: + merged = dict(details or {}) + if blob_hash: + merged.setdefault("blob_hash", blob_hash) + super().__init__(message, details=merged) + self.blob_hash = blob_hash + + class InvalidRequest(OpenMindError): """Caller-supplied arguments or configuration are unusable.""" @@ -131,5 +202,7 @@ def __init__(self, message: str, *, timeout: Optional[float] = None, __all__ = [ "OpenMindError", "NotFound", "WorkspaceNotFound", "JobNotFound", + "AssetNotFound", "RevisionNotFound", "SegmentNotFound", "EvidenceNotFound", + "ContentCorruption", "InvalidRequest", "DependencyUnavailable", "JobFailed", "OperationTimeout", ] diff --git a/openmind/domain/types.py b/openmind/domain/types.py index 0e9ac28..3cb6f2a 100644 --- a/openmind/domain/types.py +++ b/openmind/domain/types.py @@ -18,8 +18,9 @@ """ from __future__ import annotations +import os from dataclasses import dataclass, field -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional # Job statuses, exactly as jobs.py writes them. JOB_STATUS_QUEUED = "queued" @@ -140,10 +141,326 @@ def as_dict(self) -> Dict[str, Any]: } +# --------------------------------------------------------------------------- +# Canonical Asset model vocabulary (OpenMind v2 Phase 2) +# +# Each vocabulary is a small class of string constants plus a ``VALUES`` +# frozenset, so a value can be validated at a write boundary without an enum's +# instance/serialization ceremony. The vocabularies are forward-compatible: +# ``coerce`` maps an unknown value to a safe default rather than raising, so a +# database written by a newer OpenMind that added a member still reads back. +# --------------------------------------------------------------------------- +class AssetState: + """Lifecycle state of an Asset.""" + ACTIVE = "active" + REMOVED = "removed" + UNSUPPORTED = "unsupported" + VALUES = frozenset({ACTIVE, REMOVED, UNSUPPORTED}) + + @classmethod + def coerce(cls, value: Any) -> str: + v = str(value or "").strip().lower() + return v if v in cls.VALUES else cls.ACTIVE + + +class AssetType: + """Deterministic, extension/path-based classification. Never a model.""" + SOURCE_CODE = "source-code" + CONFIGURATION = "configuration" + DATABASE_SCHEMA = "database-schema" + DOCUMENTATION_TEXT = "documentation-text" + TEST_SOURCE = "test-source" + BUILD_DEFINITION = "build-definition" + UNKNOWN = "unknown" + VALUES = frozenset({SOURCE_CODE, CONFIGURATION, DATABASE_SCHEMA, + DOCUMENTATION_TEXT, TEST_SOURCE, BUILD_DEFINITION, UNKNOWN}) + + #: Exact (lower-cased) filenames that are build definitions regardless of + #: extension (pom.xml is .xml but a build file; package.json is .json). + _BUILD_FILENAMES = frozenset({ + "pom.xml", "build.gradle", "build.gradle.kts", "settings.gradle", + "settings.gradle.kts", "build.xml", "package.json", "package-lock.json", + "makefile", "cmakelists.txt", "dockerfile", "requirements.txt", + "setup.py", "setup.cfg", "pyproject.toml", "go.mod", "go.sum", + "cargo.toml", "cargo.lock", "gemfile", "build.sbt", + }) + _BUILD_EXTS = frozenset({".gradle"}) + _SCHEMA_EXTS = frozenset({".sql", ".ddl"}) + _CODE_EXTS = frozenset({ + ".java", ".kt", ".kts", ".groovy", ".scala", ".ts", ".tsx", ".js", + ".jsx", ".mjs", ".cjs", ".vue", ".svelte", ".go", ".py", ".rb", ".cs", + ".php", ".rs", ".html", ".htm", ".css", ".scss", ".less", + }) + _CONFIG_EXTS = frozenset({ + ".properties", ".yml", ".yaml", ".xml", ".json", ".toml", ".ini", + ".conf", ".cfg", ".env", ".config", + }) + _DOC_EXTS = frozenset({".md", ".markdown", ".rst", ".adoc", ".txt"}) + + @classmethod + def coerce(cls, value: Any) -> str: + v = str(value or "").strip().lower() + return v if v in cls.VALUES else cls.UNKNOWN + + @classmethod + def _is_test(cls, logical_key: str, base: str) -> bool: + parts = {p.lower() for p in logical_key.replace("\\", "/").split("/")} + if parts & {"test", "tests", "__tests__", "testing"}: + return True + b = base.lower() + return ( + b.endswith(("test.java", "tests.java", "it.java", "itcase.java", + ".test.ts", ".test.tsx", ".test.js", ".test.jsx", + ".spec.ts", ".spec.tsx", ".spec.js", ".spec.jsx", + "_test.py", "_test.go", "test.cs", "tests.cs", + "_test.rb", "_spec.rb")) + or b.startswith("test_") and b.endswith(".py")) + + @classmethod + def classify(cls, logical_key: str) -> str: + """Map a workspace-relative path to a deterministic asset type.""" + key = (logical_key or "").replace("\\", "/") + base = key.rsplit("/", 1)[-1] + ext = os.path.splitext(base)[1].lower() + if base.lower() in cls._BUILD_FILENAMES or ext in cls._BUILD_EXTS: + return cls.BUILD_DEFINITION + if ext in cls._SCHEMA_EXTS: + return cls.DATABASE_SCHEMA + if ext in cls._CODE_EXTS: + return cls.TEST_SOURCE if cls._is_test(key, base) else cls.SOURCE_CODE + if ext in cls._CONFIG_EXTS: + return cls.CONFIGURATION + if ext in cls._DOC_EXTS: + return cls.DOCUMENTATION_TEXT + return cls.UNKNOWN + + +class RevisionStatus: + """Lifecycle status of a revision. Phase 2 code revisions are ``unknown``; + approval authority is never inferred.""" + UNKNOWN = "unknown" + DRAFT = "draft" + REVIEWED = "reviewed" + APPROVED = "approved" + EFFECTIVE = "effective" + SUPERSEDED = "superseded" + WITHDRAWN = "withdrawn" + ARCHIVED = "archived" + VALUES = frozenset({UNKNOWN, DRAFT, REVIEWED, APPROVED, EFFECTIVE, + SUPERSEDED, WITHDRAWN, ARCHIVED}) + + @classmethod + def coerce(cls, value: Any) -> str: + v = str(value or "").strip().lower() + return v if v in cls.VALUES else cls.UNKNOWN + + +class SegmentType: + """Structural unit kind inside one revision.""" + TYPE = "type" + METHOD = "method" + CONSTRUCTOR = "constructor" + FILE = "file" + VALUES = frozenset({TYPE, METHOD, CONSTRUCTOR, FILE}) + + @classmethod + def coerce(cls, value: Any) -> str: + v = str(value or "").strip().lower() + return v if v in cls.VALUES else cls.FILE + + +class ContentMode: + """Whether a segment's represented content is verbatim source or derived.""" + VERBATIM = "verbatim" + DERIVED = "derived" + VALUES = frozenset({VERBATIM, DERIVED}) + + @classmethod + def coerce(cls, value: Any) -> str: + v = str(value or "").strip().lower() + return v if v in cls.VALUES else cls.VERBATIM + + +@dataclass +class SourceLocator: + """Where a piece of Evidence lives in the workspace source. ``file`` is + always workspace-relative; line numbers are 1-based.""" + file: str + start_line: int + end_line: int + symbol: str = "" + kind: str = "source-range" + + def as_dict(self) -> Dict[str, Any]: + return {"kind": self.kind, "file": self.file, + "startLine": self.start_line, "endLine": self.end_line, + "symbol": self.symbol} + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "SourceLocator": + return cls(file=str(data.get("file", "")), + start_line=int(data.get("startLine", 0) or 0), + end_line=int(data.get("endLine", 0) or 0), + symbol=str(data.get("symbol", "") or ""), + kind=str(data.get("kind", "source-range") or "source-range")) + + +@dataclass +class Asset: + """One logical engineering object (in Phase 2, a source/config file).""" + id: str + workspace_id: str + logical_key: str + asset_type: str + title: str + source_kind: str = "file" + source_path: str = "" + media_type: str = "" + state: str = AssetState.ACTIVE + current_revision_id: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + created_at: str = "" + updated_at: str = "" + + def as_dict(self) -> Dict[str, Any]: + return { + "id": self.id, "workspace_id": self.workspace_id, + "logical_key": self.logical_key, "asset_type": self.asset_type, + "title": self.title, "source_kind": self.source_kind, + "source_path": self.source_path, "media_type": self.media_type, + "state": self.state, "current_revision_id": self.current_revision_id, + "metadata": dict(self.metadata), + "created_at": self.created_at, "updated_at": self.updated_at, + } + + @classmethod + def from_row(cls, row: Dict[str, Any]) -> "Asset": + return cls( + id=row["id"], workspace_id=row["workspace_id"], + logical_key=row["logical_key"], asset_type=row["asset_type"], + title=row["title"], source_kind=row.get("source_kind", "file"), + source_path=row.get("source_path", ""), + media_type=row.get("media_type", ""), + state=row.get("state", AssetState.ACTIVE), + current_revision_id=row.get("current_revision_id"), + metadata=dict(row.get("metadata") or {}), + created_at=row.get("created_at", ""), + updated_at=row.get("updated_at", "")) + + +@dataclass +class AssetRevision: + """An immutable observation of an Asset's contents.""" + id: str + asset_id: str + sequence: int + content_hash: str + content_size: int + content_blob_hash: str + status: str = RevisionStatus.UNKNOWN + version_label: str = "" + source_commit: str = "" + supersedes_revision_id: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + created_at: str = "" + + def as_dict(self) -> Dict[str, Any]: + return { + "id": self.id, "asset_id": self.asset_id, "sequence": self.sequence, + "content_hash": self.content_hash, "content_size": self.content_size, + "content_blob_hash": self.content_blob_hash, "status": self.status, + "version_label": self.version_label, "source_commit": self.source_commit, + "supersedes_revision_id": self.supersedes_revision_id, + "metadata": dict(self.metadata), "created_at": self.created_at, + } + + @classmethod + def from_row(cls, row: Dict[str, Any]) -> "AssetRevision": + return cls( + id=row["id"], asset_id=row["asset_id"], sequence=int(row["sequence"]), + content_hash=row["content_hash"], content_size=int(row["content_size"]), + content_blob_hash=row["content_blob_hash"], + status=row.get("status", RevisionStatus.UNKNOWN), + version_label=row.get("version_label", ""), + source_commit=row.get("source_commit", ""), + supersedes_revision_id=row.get("supersedes_revision_id"), + metadata=dict(row.get("metadata") or {}), + created_at=row.get("created_at", "")) + + +@dataclass +class Segment: + """A stable structural unit inside one revision.""" + id: str + revision_id: str + segment_key: str + segment_type: str + ordinal: int + start_line: Optional[int] = None + end_line: Optional[int] = None + symbol: str = "" + content_hash: str = "" + content_mode: str = ContentMode.VERBATIM + metadata: Dict[str, Any] = field(default_factory=dict) + + def as_dict(self) -> Dict[str, Any]: + return { + "id": self.id, "revision_id": self.revision_id, + "segment_key": self.segment_key, "segment_type": self.segment_type, + "ordinal": self.ordinal, "start_line": self.start_line, + "end_line": self.end_line, "symbol": self.symbol, + "content_hash": self.content_hash, "content_mode": self.content_mode, + "metadata": dict(self.metadata), + } + + @classmethod + def from_row(cls, row: Dict[str, Any]) -> "Segment": + return cls( + id=row["id"], revision_id=row["revision_id"], + segment_key=row["segment_key"], segment_type=row["segment_type"], + ordinal=int(row["ordinal"]), start_line=row.get("start_line"), + end_line=row.get("end_line"), symbol=row.get("symbol", ""), + content_hash=row.get("content_hash", ""), + content_mode=row.get("content_mode", ContentMode.VERBATIM), + metadata=dict(row.get("metadata") or {})) + + +@dataclass +class Evidence: + """A source-locatable citation for a segment, recoverable from the + immutable revision blob.""" + id: str + revision_id: str + segment_id: Optional[str] + locator: Dict[str, Any] + excerpt: str = "" + content_hash: str = "" + created_at: str = "" + + def as_dict(self) -> Dict[str, Any]: + return { + "id": self.id, "revision_id": self.revision_id, + "segment_id": self.segment_id, "locator": dict(self.locator), + "excerpt": self.excerpt, "content_hash": self.content_hash, + "created_at": self.created_at, + } + + @classmethod + def from_row(cls, row: Dict[str, Any]) -> "Evidence": + return cls( + id=row["id"], revision_id=row["revision_id"], + segment_id=row.get("segment_id"), + locator=dict(row.get("locator") or {}), + excerpt=row.get("excerpt", ""), content_hash=row.get("content_hash", ""), + created_at=row.get("created_at", "")) + + __all__ = [ "JOB_STATUS_QUEUED", "JOB_STATUS_RUNNING", "JOB_STATUS_PAUSED", "JOB_STATUS_INTERRUPTED", "JOB_STATUS_DONE", "JOB_STATUS_FAILED", "TERMINAL_JOB_STATUSES", "SETTLED_JOB_STATUSES", "ACTIVE_JOB_STATUSES", "STATUS_OK", "STATUS_WARN", "STATUS_ERROR", "HealthCheck", "HealthReport", "JobWaitResult", + "AssetState", "AssetType", "RevisionStatus", "SegmentType", "ContentMode", + "SourceLocator", "Asset", "AssetRevision", "Segment", "Evidence", ] diff --git a/openmind/jobs.py b/openmind/jobs.py index 863886e..40c7c7d 100644 --- a/openmind/jobs.py +++ b/openmind/jobs.py @@ -18,9 +18,11 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from . import (ask as askmod, cases, config, conversation, db, docs as docsmod, - embeddings, glossary, llm_client, machine, mapio, rag, resources, - structure, vectorstore, walker, wikienrich) +from . import (ask as askmod, cases, config, content_store, conversation, db, + docs as docsmod, embeddings, glossary, llm_client, machine, mapio, + rag, resources, segmentation, structure, vectorstore, walker, + wikienrich) +from .domain.types import AssetType from .model_server import server as model_server _wake = threading.Event() @@ -433,6 +435,12 @@ def terminate_project(project_id: str, clear_cases: bool = False) -> Dict[str, A # ready to re-learn in one click — clearing the repo paths here would make # "Start / Re-learn" fail with "add a path first" (use Delete to drop a project). db.clear_file_index(project_id) + # 3b) drop the canonical Asset model (assets cascade to revisions/segments/ + # evidence) and its immutable content blobs. Terminate is a full wipe of + # LEARNED data back to init, so Asset history goes too — unlike an ordinary + # source-file removal, which only marks an Asset removed and keeps its history. + db.clear_workspace_assets(project_id) + content_store.clear_workspace(project_id) # cases: keep but flag stale, unless clearing if clear_cases: cases.clear_cases(project_id) @@ -532,10 +540,126 @@ def _project_path_specs(project_id: str, only_path: Optional[str]) -> List[Dict[ return specs +# --------------------------------------------------------------------------- +# Asset model: local Git provenance + the per-file commit into the canonical +# Asset/Revision/Segment/Evidence store (see openmind/segmentation.py and +# db.commit_revision). All local, deterministic, model-free. +# --------------------------------------------------------------------------- +def _find_git_root(start_dir: str) -> str: + """Nearest ancestor of *start_dir* that contains a ``.git`` entry, or ''.""" + cur = Path(start_dir) + for _ in range(64): # bounded: never climb forever + if (cur / ".git").exists(): + return str(cur) + if cur.parent == cur: + break + cur = cur.parent + return "" + + +def _read_git_head(git_root: str) -> str: + """Resolve a repo's current HEAD commit SHA from the filesystem only — no + network, no ``git`` subprocess. Handles a ``.git`` directory, a ``.git`` + file (worktree/submodule), a symbolic ``ref:`` HEAD, a detached SHA, and the + ``packed-refs`` fallback. Any failure returns '' rather than raising.""" + try: + dot_git = Path(git_root) / ".git" + if dot_git.is_file(): # worktree/submodule: 'gitdir: ' + content = dot_git.read_text(encoding="utf-8", errors="ignore").strip() + if content.startswith("gitdir:"): + gd = content.split(":", 1)[1].strip() + dot_git = Path(gd) if os.path.isabs(gd) else (Path(git_root) / gd) + head = (dot_git / "HEAD").read_text(encoding="utf-8", errors="ignore").strip() + if head.startswith("ref:"): + ref = head.split(":", 1)[1].strip() + ref_file = dot_git / ref + if ref_file.is_file(): + return ref_file.read_text(encoding="utf-8", errors="ignore").strip() + packed = dot_git / "packed-refs" + if packed.is_file(): + for line in packed.read_text(encoding="utf-8", + errors="ignore").splitlines(): + line = line.strip() + if not line or line.startswith(("#", "^")): + continue + parts = line.split(" ", 1) + if len(parts) == 2 and parts[1].strip() == ref: + return parts[0].strip() + return "" + return head if head else "" # detached HEAD (a raw SHA) + except Exception: + return "" + + +def _git_commit_for(file_abs: str, dir_cache: Dict[str, str], + sha_cache: Dict[str, str]) -> str: + """The HEAD SHA of the repo containing *file_abs*, cached per repository for + one ingestion. Records only the real HEAD — a dirty working tree is never + turned into an invented SHA (dirtiness is not asserted here).""" + d = os.path.dirname(file_abs) + root = dir_cache.get(d) + if root is None: + root = _find_git_root(d) + dir_cache[d] = root + if not root: + return "" + sha = sha_cache.get(root) + if sha is None: + sha = _read_git_head(root) + sha_cache[root] = sha + return sha + + +def _commit_file_asset(project_id: str, rel_key: str, text: str, + source_commit: str, ac: Dict[str, int]) -> Dict[str, Any]: + """Snapshot one file's content + segments into the canonical Asset store and + tally the additive progress counters in *ac*. Writes NO vector data — the RAG + projection is owned by the ingest's embed batch (or reused unchanged for a + legacy backfill), so this never re-embeds.""" + data = text.encode("utf-8", "replace") + blob_hash = content_store.hash_bytes(data) + if content_store.exists(project_id, blob_hash): + ac["content_blobs_reused"] += 1 + else: + ac["content_blobs_created"] += 1 + content_store.put(project_id, data) # immutable snapshot + segments = segmentation.segment_source(rel_key, text) + title = rel_key.replace("\\", "/").rsplit("/", 1)[-1] + res = db.commit_revision( + project_id, rel_key, asset_type=AssetType.classify(rel_key), title=title, + source_path=rel_key, content_hash=blob_hash, content_size=len(data), + content_blob_hash=blob_hash, segments=segments, source_commit=source_commit, + revision_metadata=({"git_head": source_commit} if source_commit else {})) + if res["asset_created"]: + ac["assets_created"] += 1 + elif res["reactivated"]: + ac["assets_reactivated"] += 1 + else: + ac["assets_reused"] += 1 + if res["revision_created"]: + ac["revisions_created"] += 1 + ac["segments_created"] += res["segments_created"] + ac["evidence_created"] += res["evidence_created"] + else: + ac["revisions_reused"] += 1 + return res + + +_ASSET_COUNTER_KEYS = ( + "assets_created", "assets_reused", "assets_reactivated", "assets_removed", + "revisions_created", "revisions_reused", "segments_created", + "evidence_created", "content_blobs_created", "content_blobs_reused", +) + + def _run_ingest(job_id: str) -> None: job = db.get_job(job_id) project_id = job["project_id"] db.set_project_state(project_id, "learning") + # A FILTERED ingest (a single file or subtree, e.g. `asset add`) must not + # rebuild the whole-project artifacts (glossary/structure/templates/facets) + # from a partial file set, and must not prune files outside the filter. + filtered = bool(job.get("path")) specs = _project_path_specs(project_id, job.get("path")) if not specs: _log(job_id, "[ingest] no paths to ingest.") @@ -547,15 +671,29 @@ def _run_ingest(job_id: str) -> None: db.update_job(job_id, step="scanning") current_files: List[str] = [] file_meta: Dict[str, Dict[str, str]] = {} + project_root_abs = machine.project_root(project_id) for spec in specs: root = walker.norm(spec["path"]) - for f in walker.iter_files(root, spec.get("exclude", [])): + # A single-file spec (a filtered `asset add`) is not a directory tree, so + # os.walk would yield nothing; feed the one file through directly, still + # honouring the indexable-extension / size / binary gates. Repo/service + # attribution anchors to the project root, not the lone file. + if os.path.isfile(root): + ext = os.path.splitext(root)[1].lower() + walk_root = project_root_abs or os.path.dirname(root) + files_iter = ([root] if (ext in config.INDEX_EXTENSIONS + and ext not in config.BINARY_EXTENSIONS) + else []) + else: + walk_root = root + files_iter = walker.iter_files(root, spec.get("exclude", [])) + for f in files_iter: if f in file_meta: # overlapping path selections (e.g. a root and its subfolder): # index each file ONCE — duplicates in one embed batch would # collide on identical chunk ids. First selection wins. continue - repo = walker.find_repo_root(f, root) + repo = walker.find_repo_root(f, walk_root) svc = walker.service_name(repo) current_files.append(f) file_meta[f] = {"repo": repo, "service": svc} @@ -591,97 +729,109 @@ def rel_of(f: str) -> str: _checkpoint(job_id) resources.guard_or_abort(lambda m: _log(job_id, m)) # keep RAM headroom - # ---- template auto-selection (recorded for later consumers, not yet applied) ---- - # Deterministic stack detection scores the available template profiles; the - # winner (if any) is RECORDED in project meta as template_auto together with - # its score. A user override (meta["template"]) always wins at resolve time - # (openmind.templates.resolve_for_project). Guarded end to end: template - # selection must never be able to break a learn. - try: - from . import detect as detectmod, templates as templatesmod - detection = detectmod.detect( - [(rel_of(f), cache_text[f]) for f in current_files]) - sel = templatesmod.auto_select(detection, current_set) - new_auto = (sel or {}).get("name") - p = db.get_project(project_id) or {} - meta = dict(p.get("meta") or {}) - if (meta.get("template_auto") != new_auto - or meta.get("template_auto_score") != (sel or {}).get("score")): - meta["template_auto"] = new_auto - meta["template_auto_score"] = (sel or {}).get("score") - db.update_project_meta(project_id, meta) - if new_auto: - _log(job_id, f"[templates] auto-selected profile '{new_auto}' " - f"(score {sel['score']}); a project override wins if set.") - except Exception as exc: - _log(job_id, f"[templates] auto-selection skipped: {exc}") - - # ---- template facts (roles + facet captures for the RESOLVED profile) ---- - # Consumes the selection above (override > auto) and persists the facts map - # like the other learn artifacts: deterministic, file:line evidence. A learn - # that resolves NO template removes any stale facts file so readers never - # serve facts from a profile that is no longer selected. Guarded the same - # way: a facet error logs and skips, never fails the learn. - try: - from . import facets as facetsmod, templates as templatesmod - tpl = templatesmod.resolve_for_project(db.get_project(project_id)) - if tpl and (tpl.roles or tpl.facets): - fdoc = facetsmod.build_facts( - [(rel_of(f), cache_text[f], cache_hash[f]) for f in current_files], tpl) - mapio.save_facts(project_id, fdoc) - _log(job_id, f"[templates] '{tpl.name}' facts: " - f"{fdoc['stats']['files_classified']} file(s) classified, " - f"{fdoc['stats']['fact_count']} fact(s) captured.") - else: - mapio.delete_facts(project_id) - except Exception as exc: - _log(job_id, f"[templates] facet build skipped: {exc}") - - # ---- step 3: deterministic glossary (term/acronym -> VERBATIM definition) ---- - # The primary build-time artifact: term definitions are extracted ONCE here - # (pure pattern-matching, no LLM, verbatim from authoritative sources) and - # thereafter QUERIED (never re-parsed). Incremental via the same per-file - # content hash — an unchanged definition source is carried over, not re-scanned. - db.update_job(job_id, step="glossary") - try: - gfiles = [(rel_of(f), cache_text[f], cache_hash[f]) for f in current_files] - # cancel-aware: a Terminate landing mid-build stops the pass promptly - # rather than scanning the whole corpus first. Skip persisting a partial - # artifact in that case — the wipe removes map/* anyway. - gdoc = glossary.build_glossary( - gfiles, prior=mapio.load_glossary(project_id), - cancel=lambda: _control(job_id) == "terminate") - if _control(job_id) != "terminate": - mapio.save_glossary(project_id, gdoc) - _log(job_id, f"[glossary] {gdoc['stats']['term_count']} term(s) from " - f"{gdoc['stats']['source_count']} source(s) " - f"({gdoc['stats']['sources_reused']} reused).") - except Exception as exc: - _log(job_id, f"[glossary] WARNING: build failed ({exc}) — the glossary " - f"was NOT updated for this run; prior artifact (if any) kept.") - _checkpoint(job_id) # raises if a pause/terminate arrived during the build - - # ---- step 3b: deterministic STRUCTURE map (modules / defs / dependency / - # call / entry-point-flow graphs) — the basis of the Graphs view. Same - # incremental, hash-keyed, never-fabricated contract as the glossary. ---- - db.update_job(job_id, step="structure") - try: - sfiles = [(rel_of(f), cache_text[f], cache_hash[f]) for f in current_files] - # paths are already relative -> store an empty root (no absolute trace) - sdoc = structure.build_structure(sfiles, root="", - prior=mapio.load_structure(project_id)) - if _control(job_id) != "terminate": - mapio.save_structure(project_id, sdoc) - st = sdoc.get("stats", {}) - _log(job_id, f"[structure] {st.get('definition_count', 0)} defs · " - f"{st.get('module_count', 0)} modules · " - f"{st.get('call_edges', 0)} call edges · " - f"{st.get('entry_point_count', 0)} entry points " - f"({st.get('sources_reused', 0)} reused).") - except Exception as exc: - _log(job_id, f"[structure] WARNING: build failed ({exc}) — the structure " - f"map was NOT updated for this run; prior artifact (if any) kept.") - _checkpoint(job_id) + # The glossary, structure map, template selection and facet captures are + # WHOLE-PROJECT artifacts, reconciled against the full walked file set. A + # filtered ingest (a single file / subtree — e.g. `asset add`) sees only a + # slice of that set, so rebuilding them here would drop every term, symbol + # and fact that lives outside the slice. Skip them for a filtered run; a full + # ingest refreshes them. The Asset model + RAG projection below still update + # for exactly the filtered file(s). + if not filtered: + # ---- template auto-selection (recorded for later consumers, not yet applied) ---- + # Deterministic stack detection scores the available template profiles; the + # winner (if any) is RECORDED in project meta as template_auto together with + # its score. A user override (meta["template"]) always wins at resolve time + # (openmind.templates.resolve_for_project). Guarded end to end: template + # selection must never be able to break a learn. + try: + from . import detect as detectmod, templates as templatesmod + detection = detectmod.detect( + [(rel_of(f), cache_text[f]) for f in current_files]) + sel = templatesmod.auto_select(detection, current_set) + new_auto = (sel or {}).get("name") + p = db.get_project(project_id) or {} + meta = dict(p.get("meta") or {}) + if (meta.get("template_auto") != new_auto + or meta.get("template_auto_score") != (sel or {}).get("score")): + meta["template_auto"] = new_auto + meta["template_auto_score"] = (sel or {}).get("score") + db.update_project_meta(project_id, meta) + if new_auto: + _log(job_id, f"[templates] auto-selected profile '{new_auto}' " + f"(score {sel['score']}); a project override wins if set.") + except Exception as exc: + _log(job_id, f"[templates] auto-selection skipped: {exc}") + + # ---- template facts (roles + facet captures for the RESOLVED profile) ---- + # Consumes the selection above (override > auto) and persists the facts map + # like the other learn artifacts: deterministic, file:line evidence. A learn + # that resolves NO template removes any stale facts file so readers never + # serve facts from a profile that is no longer selected. Guarded the same + # way: a facet error logs and skips, never fails the learn. + try: + from . import facets as facetsmod, templates as templatesmod + tpl = templatesmod.resolve_for_project(db.get_project(project_id)) + if tpl and (tpl.roles or tpl.facets): + fdoc = facetsmod.build_facts( + [(rel_of(f), cache_text[f], cache_hash[f]) for f in current_files], tpl) + mapio.save_facts(project_id, fdoc) + _log(job_id, f"[templates] '{tpl.name}' facts: " + f"{fdoc['stats']['files_classified']} file(s) classified, " + f"{fdoc['stats']['fact_count']} fact(s) captured.") + else: + mapio.delete_facts(project_id) + except Exception as exc: + _log(job_id, f"[templates] facet build skipped: {exc}") + + # ---- step 3: deterministic glossary (term/acronym -> VERBATIM definition) ---- + # The primary build-time artifact: term definitions are extracted ONCE here + # (pure pattern-matching, no LLM, verbatim from authoritative sources) and + # thereafter QUERIED (never re-parsed). Incremental via the same per-file + # content hash — an unchanged definition source is carried over, not re-scanned. + db.update_job(job_id, step="glossary") + try: + gfiles = [(rel_of(f), cache_text[f], cache_hash[f]) for f in current_files] + # cancel-aware: a Terminate landing mid-build stops the pass promptly + # rather than scanning the whole corpus first. Skip persisting a partial + # artifact in that case — the wipe removes map/* anyway. + gdoc = glossary.build_glossary( + gfiles, prior=mapio.load_glossary(project_id), + cancel=lambda: _control(job_id) == "terminate") + if _control(job_id) != "terminate": + mapio.save_glossary(project_id, gdoc) + _log(job_id, f"[glossary] {gdoc['stats']['term_count']} term(s) from " + f"{gdoc['stats']['source_count']} source(s) " + f"({gdoc['stats']['sources_reused']} reused).") + except Exception as exc: + _log(job_id, f"[glossary] WARNING: build failed ({exc}) — the glossary " + f"was NOT updated for this run; prior artifact (if any) kept.") + _checkpoint(job_id) # raises if a pause/terminate arrived during the build + + # ---- step 3b: deterministic STRUCTURE map (modules / defs / dependency / + # call / entry-point-flow graphs) — the basis of the Graphs view. Same + # incremental, hash-keyed, never-fabricated contract as the glossary. ---- + db.update_job(job_id, step="structure") + try: + sfiles = [(rel_of(f), cache_text[f], cache_hash[f]) for f in current_files] + # paths are already relative -> store an empty root (no absolute trace) + sdoc = structure.build_structure(sfiles, root="", + prior=mapio.load_structure(project_id)) + if _control(job_id) != "terminate": + mapio.save_structure(project_id, sdoc) + st = sdoc.get("stats", {}) + _log(job_id, f"[structure] {st.get('definition_count', 0)} defs · " + f"{st.get('module_count', 0)} modules · " + f"{st.get('call_edges', 0)} call edges · " + f"{st.get('entry_point_count', 0)} entry points " + f"({st.get('sources_reused', 0)} reused).") + except Exception as exc: + _log(job_id, f"[structure] WARNING: build failed ({exc}) — the structure " + f"map was NOT updated for this run; prior artifact (if any) kept.") + _checkpoint(job_id) + else: + _log(job_id, "[filtered] single-file/subtree ingest: glossary, structure " + "and template artifacts are left unchanged (a full ingest " + "rebuilds them).") # ---- step 4: RAG indexing (incremental, idempotent; Invariant 12) ---- # Embeddings are BATCHED ACROSS files (BATCH_CHUNKS at a time), not one embed @@ -697,6 +847,14 @@ def rel_of(f: str) -> str: # overrides this and is respected as-is. store = vectorstore.get_code_store(project_id) index = db.get_file_index(project_id) + # Canonical Asset model: load the whole workspace's asset index ONCE (like the + # file index) so the per-file loop can decide, without a query per file, whether + # an Asset already exists and whether its current revision already matches — the + # basis of the unchanged-legacy backfill that must NOT re-embed. + asset_index = db.list_asset_index(project_id) + ac: Dict[str, int] = {k: 0 for k in _ASSET_COUNTER_KEYS} + _git_dir_cache: Dict[str, str] = {} # file dir -> git root (per ingestion) + _git_sha_cache: Dict[str, str] = {} # git root -> HEAD sha (per ingestion) # only the changed/new files actually embed (incremental) — so don't stop the # model to free the GPU when an incremental re-ingest has nothing to embed. _needs_embed = any(index.get(rel_of(f), {}).get("file_hash") != cache_hash[f] @@ -725,9 +883,23 @@ def _flush_batch() -> None: nonlocal pending, pending_chunks, chunk_total if not pending: return + # Vector projection first (idempotent, stable ids), then per file the + # compatibility file-index row and the canonical Asset revision. If the DB + # write fails after the upsert, the next ingest safely repeats the stable + # upsert and reconciles the Asset, and no partial revision becomes current + # (commit_revision is a single transaction). rag.embed_and_upsert(store, [c for rec in pending for c in rec["chunks"]]) for rec in pending: ids = [c["id"] for c in rec["chunks"]] + # Commit the canonical Asset revision BEFORE the compatibility + # file_index row. file_index is the "unchanged" gate the skipped + # branch trusts, so it must be written LAST: if the process dies + # between these two writes, file_index still shows the OLD hash, the + # file is re-indexed next run (changed branch), and commit_revision + # simply reuses the already-committed revision. Writing file_index + # first would instead strand a stale Asset that the skip branch would + # trust forever. + _commit_file_asset(project_id, rec["f"], rec["text"], rec["commit"], ac) db.upsert_file_index(project_id, rec["f"], rec["h"], ids, rec["service"], rec["topics"]) chunk_total += len(ids) @@ -746,6 +918,18 @@ def _flush_batch() -> None: topics: List[str] = [] # chunk-header topic tags (architecture-agnostic: none) if prior and prior.get("file_hash") == h: skipped += 1 + # Legacy backfill: this file is unchanged in the RAG index, but a + # Phase 1 workspace has no Asset for it yet. Snapshot it into the + # canonical store WITHOUT re-embedding (its Chroma chunks are reused). + # A file whose Asset is already in sync just tallies reuse. + ai = asset_index.get(r) + if ai and ai.get("content_hash") and ai.get("state") == "active": + ac["assets_reused"] += 1 + ac["revisions_reused"] += 1 + else: + _commit_file_asset( + project_id, r, cache_text[f], + _git_commit_for(f, _git_dir_cache, _git_sha_cache), ac) else: if prior and prior.get("chunk_ids"): rag.delete_file_chunks(store, r, prior["chunk_ids"]) @@ -753,8 +937,14 @@ def _flush_batch() -> None: changed += 1 chunks = rag.chunk_file(project_id, r, cache_text[f], meta["service"], repo_rel, topics, h) + # Carry the file text + resolved Git HEAD so _flush_batch can snapshot + # the blob and commit the Asset revision transactionally, once the + # chunks for this batch are embedded. pending.append({"f": r, "h": h, "service": meta["service"], - "topics": topics, "chunks": chunks}) + "topics": topics, "chunks": chunks, + "text": cache_text[f], + "commit": _git_commit_for(f, _git_dir_cache, + _git_sha_cache)}) pending_chunks += len(chunks) indexed += 1 if pending_chunks >= BATCH_CHUNKS: @@ -764,20 +954,47 @@ def _flush_batch() -> None: if i % 5 == 0 or done == len(current_files): _progress(job_id, files_done=done, files_skipped=skipped, files_indexed=indexed, files_changed=changed, - chunks=chunk_total + pending_chunks) + chunks=chunk_total + pending_chunks, **ac) _checkpoint(job_id) # honor a late pause/terminate before the final embed _flush_batch() # embed + store the final partial batch # ---- step 5: prune removed / now-excluded files ---- + # A FILTERED ingest saw only part of the workspace, so it must prune ONLY + # within the filter root(s) — otherwise a single-file `asset add` would treat + # every other indexed file as "gone". A full ingest prunes the whole index. + if filtered: + scope_roots = [rel_of(walker.norm(s["path"])) for s in specs] + + def _in_scope(fpath: str) -> bool: + for rt in scope_roots: + if not rt or fpath == rt or fpath.startswith(rt.rstrip("/") + "/"): + return True + return False + else: + def _in_scope(_fpath: str) -> bool: + return True + removed = 0 for fpath, rec in list(index.items()): - if fpath not in current_set: + if _in_scope(fpath) and fpath not in current_set: + # Remove the active retrieval projection + the compatibility index + # row, but PRESERVE the Asset's history: mark it removed, keeping every + # revision, segment, evidence and content blob (source deletion is not + # history erasure — that is what terminate/delete are for). rag.delete_file_chunks(store, fpath, rec.get("chunk_ids")) db.delete_file_index_entry(project_id, fpath) + if db.mark_asset_removed(project_id, fpath): + ac["assets_removed"] += 1 removed += 1 - _progress(job_id, files_removed=removed, chunks=store.count()) + _progress(job_id, files_removed=removed, chunks=store.count(), **ac) _log(job_id, f"[index] indexed={indexed} changed={changed} skipped={skipped} " f"removed={removed} chunks={store.count()}") + _log(job_id, f"[assets] +{ac['assets_created']} new " + f"~{ac['assets_reused']} reused ^{ac['assets_reactivated']} reactivated " + f"-{ac['assets_removed']} removed | revisions +{ac['revisions_created']} " + f"={ac['revisions_reused']} reused | segments {ac['segments_created']} " + f"evidence {ac['evidence_created']} | blobs +{ac['content_blobs_created']} " + f"={ac['content_blobs_reused']} reused") # ---- done ---- _checkpoint(job_id) # if terminate raced in after the last file, abort before finalize diff --git a/openmind/main.py b/openmind/main.py index 80c8f2f..8fc1c46 100644 --- a/openmind/main.py +++ b/openmind/main.py @@ -325,6 +325,64 @@ def delete_project(project_id: str) -> Dict[str, Any]: return _svc().workspaces.request_delete(project_id) +# --------------------------------------------------------------------------- +# Canonical Asset model (OpenMind v2 Phase 2) — ADDITIVE, read-only. +# Every route is workspace-scoped through AssetService; a cross-workspace id +# resolves to a typed not-found and is mapped to 404 by the exception handler. +# The `stats` literal path is registered before `{asset_id}` so it is never +# captured as an asset id. +# --------------------------------------------------------------------------- +@app.get("/projects/{project_id}/assets") +def list_assets(project_id: str, type: Optional[str] = None, + state: Optional[str] = None, limit: int = 100, + offset: int = 0) -> Dict[str, Any]: + return _svc().assets.list_assets(project_id, asset_type=type, state=state, + limit=limit, offset=offset) + + +@app.get("/projects/{project_id}/assets/stats") +def asset_stats(project_id: str) -> Dict[str, Any]: + return _svc().assets.stats(project_id) + + +@app.post("/projects/{project_id}/assets/sync") +def sync_asset(project_id: str, req: models.AssetSyncReq) -> Dict[str, Any]: + """Ingest a single existing file under a registered source root (Phase 2). + Delegates to AssetService.sync_file; a directory or a file outside the root + is a 400, an unsupported format is registered as `unsupported` (not parsed).""" + return _svc().assets.sync_file(project_id, req.path, wait=req.wait, + timeout=req.timeout) + + +@app.get("/projects/{project_id}/assets/{asset_id}") +def get_asset(project_id: str, asset_id: str) -> Dict[str, Any]: + return _svc().assets.get_asset(project_id, asset_id) + + +@app.get("/projects/{project_id}/assets/{asset_id}/revisions") +def list_asset_revisions(project_id: str, asset_id: str, + limit: int = 50) -> Dict[str, Any]: + return _svc().assets.list_revisions(project_id, asset_id, limit=limit) + + +@app.get("/projects/{project_id}/revisions/{revision_id}") +def get_revision(project_id: str, revision_id: str) -> Dict[str, Any]: + return _svc().assets.get_revision(project_id, revision_id) + + +@app.get("/projects/{project_id}/revisions/{revision_id}/segments") +def list_revision_segments(project_id: str, revision_id: str, limit: int = 200, + offset: int = 0) -> Dict[str, Any]: + return _svc().assets.list_segments(project_id, revision_id, limit=limit, + offset=offset) + + +@app.get("/projects/{project_id}/evidence/{evidence_id}") +def get_evidence(project_id: str, evidence_id: str, + max_chars: int = 4000) -> Dict[str, Any]: + return _svc().assets.get_evidence(project_id, evidence_id, max_chars=max_chars) + + @app.get("/scope/{scope_id}") def describe_scope(scope_id: str) -> Dict[str, Any]: return scope.describe(scope_id) diff --git a/openmind/mcp_server.py b/openmind/mcp_server.py index 4cffe6a..9b5654c 100644 --- a/openmind/mcp_server.py +++ b/openmind/mcp_server.py @@ -166,7 +166,9 @@ def apply_fix(file_path: str, find: str, replace: str, test_cmd: str, #: The published tool set, in registration order. The names are the MCP tool -#: names clients call. +#: names clients call. STABLE external contract — do not rename or change a +#: returned key. New capabilities are ADDED via ASSET_TOOLS below, never by +#: changing one of these nine. TOOLS: List[Callable[..., Any]] = [ search, route, dispatch, get_glossary, find_similar_cases, save_case, get_doc, propose_fix, apply_fix, @@ -175,6 +177,91 @@ def apply_fix(file_path: str, find: str, replace: str, test_cmd: str, TOOL_NAMES = tuple(fn.__name__ for fn in TOOLS) +# --------------------------------------------------------------------------- +# Canonical Asset model tools (OpenMind v2 Phase 2) — ADDITIVE, read-only. +# +# These route through the shared AssetService (unlike the deterministic query +# tools above) because they need its workspace-scoping and snapshot-based +# evidence recovery. ``scope`` resolves to a workspace; an entity that does not +# belong to it is an honest not-found. Results are bounded, full source is never +# returned without an explicit bounded parameter, and merely listing the tools +# never starts the ingestion worker. +# --------------------------------------------------------------------------- +def _asset_workspaces(scope: str) -> List[str]: + return _pids(scope) + + +def list_assets(scope: str, asset_type: Optional[str] = None, + state: str = "active", limit: int = 50) -> Dict[str, Any]: + """List a workspace's canonical Assets (bounded). ``scope`` = the workspace + (project) id; ``state`` defaults to ``active`` (also: removed / unsupported / + None for all). Returns {assets, total, count} — file/config objects with their + logical key, type, state and current revision id.""" + from .runtime import get_runtime + pid = _asset_workspaces(scope)[0] + result = get_runtime().assets.list_assets( + pid, asset_type=asset_type, state=(state or None), limit=limit) + return {"workspace_id": pid, "assets": result["assets"], + "total": result["total"], "count": result["count"]} + + +def get_asset(scope: str, asset_id: str) -> Dict[str, Any]: + """One Asset plus its current-revision summary. Searches every workspace the + scope resolves to and returns the first owner; an id in no in-scope workspace + is an honest not-found.""" + from .runtime import get_runtime + from .domain.errors import AssetNotFound + assets = get_runtime().assets + last: Optional[Exception] = None + for pid in _asset_workspaces(scope): + try: + return assets.get_asset(pid, asset_id) + except AssetNotFound as exc: + last = exc + raise last or AssetNotFound(asset_id) + + +def get_asset_revisions(scope: str, asset_id: str, + limit: int = 20) -> Dict[str, Any]: + """An Asset's revision history (bounded, newest first).""" + from .runtime import get_runtime + from .domain.errors import AssetNotFound + assets = get_runtime().assets + last: Optional[Exception] = None + for pid in _asset_workspaces(scope): + try: + return assets.list_revisions(pid, asset_id, limit=limit) + except AssetNotFound as exc: + last = exc + raise last or AssetNotFound(asset_id) + + +def get_evidence(scope: str, evidence_id: str, + max_chars: int = 4000) -> Dict[str, Any]: + """One Evidence citation with a WORKSPACE-RELATIVE source locator, bounded + content recovered from the immutable snapshot, and an honest report of whether + it came from the current source, the historical snapshot, or both.""" + from .runtime import get_runtime + from .domain.errors import EvidenceNotFound + assets = get_runtime().assets + last: Optional[Exception] = None + for pid in _asset_workspaces(scope): + try: + return assets.get_evidence(pid, evidence_id, max_chars=max_chars) + except EvidenceNotFound as exc: + last = exc + raise last or EvidenceNotFound(evidence_id) + + +#: Additive read-only Asset tools. Registered ALONGSIDE the nine core tools, +#: never in place of one. +ASSET_TOOLS: List[Callable[..., Any]] = [ + list_assets, get_asset, get_asset_revisions, get_evidence, +] + +ASSET_TOOL_NAMES = tuple(fn.__name__ for fn in ASSET_TOOLS) + + # --------------------------------------------------------------------------- # Construction # --------------------------------------------------------------------------- @@ -196,6 +283,8 @@ def create_mcp_server(runtime: Optional[Any] = None) -> FastMCP: server = FastMCP(SERVER_NAME) for fn in TOOLS: server.tool()(fn) + for fn in ASSET_TOOLS: # additive read-only Asset tools (Phase 2) + server.tool()(fn) return server diff --git a/openmind/migrations/versions/v0003_asset_model.py b/openmind/migrations/versions/v0003_asset_model.py new file mode 100644 index 0000000..23b29c9 --- /dev/null +++ b/openmind/migrations/versions/v0003_asset_model.py @@ -0,0 +1,120 @@ +"""The canonical Asset model: assets, revisions, segments, evidence. + +OpenMind v2 Phase 2. Additive and non-destructive: it only CREATEs new tables +and indexes (all ``IF NOT EXISTS``), and touches no existing row. A legacy or +Phase 1 database migrates to this head with every project, path, job, file-index +row, Ask exchange, case, template selection, map and Chroma collection intact. + +The canonical content identity model: + + assets one logical engineering object (a source/config file) + asset_revisions an immutable observation of that asset's bytes + segments a stable structural unit inside one revision + evidence a source-locatable citation for a segment + +``assets.workspace_id`` references ``projects(id)`` and the whole subtree +cascades on ``ON DELETE CASCADE``, so removing a project (with the connection's +``PRAGMA foreign_keys=ON``) drops its assets, revisions, segments and evidence +in one statement. + +Two identity choices worth stating explicitly: + +* ``UNIQUE(asset_id, sequence)`` — revision sequence is dense and unique per + asset. +* ``(asset_id, content_hash)`` is DELIBERATELY NOT UNIQUE — a source may move + A -> B and back to A, and that revert must be representable as a new revision + that happens to reuse the old content blob. + +FROZEN once applied: the runner checksums this file's ``STATEMENTS``; express any +later schema change as a NEW migration, never an edit here. +""" +from __future__ import annotations + +STATEMENTS = ( + # -- assets -------------------------------------------------------------- + """ + CREATE TABLE IF NOT EXISTS assets ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + logical_key TEXT NOT NULL, + asset_type TEXT NOT NULL, + title TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT 'file', + source_path TEXT NOT NULL, + media_type TEXT NOT NULL DEFAULT '', + state TEXT NOT NULL DEFAULT 'active', + current_revision_id TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE(workspace_id, logical_key), + FOREIGN KEY(workspace_id) REFERENCES projects(id) ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_assets_ws_state ON assets(workspace_id, state)", + "CREATE INDEX IF NOT EXISTS idx_assets_ws_type ON assets(workspace_id, asset_type)", + + # -- asset_revisions ----------------------------------------------------- + """ + CREATE TABLE IF NOT EXISTS asset_revisions ( + id TEXT PRIMARY KEY, + asset_id TEXT NOT NULL, + sequence INTEGER NOT NULL, + content_hash TEXT NOT NULL, + content_size INTEGER NOT NULL, + content_blob_hash TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'unknown', + version_label TEXT NOT NULL DEFAULT '', + source_commit TEXT NOT NULL DEFAULT '', + supersedes_revision_id TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL, + UNIQUE(asset_id, sequence), + FOREIGN KEY(asset_id) REFERENCES assets(id) ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_revisions_asset_seq " + "ON asset_revisions(asset_id, sequence)", + "CREATE INDEX IF NOT EXISTS idx_revisions_blob " + "ON asset_revisions(content_blob_hash)", + + # -- segments ------------------------------------------------------------ + """ + CREATE TABLE IF NOT EXISTS segments ( + id TEXT PRIMARY KEY, + revision_id TEXT NOT NULL, + segment_key TEXT NOT NULL, + segment_type TEXT NOT NULL, + ordinal INTEGER NOT NULL, + start_line INTEGER, + end_line INTEGER, + symbol TEXT NOT NULL DEFAULT '', + content_hash TEXT NOT NULL, + content_mode TEXT NOT NULL DEFAULT 'verbatim', + metadata_json TEXT NOT NULL DEFAULT '{}', + UNIQUE(revision_id, segment_key), + FOREIGN KEY(revision_id) REFERENCES asset_revisions(id) ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_segments_revision ON segments(revision_id)", + "CREATE INDEX IF NOT EXISTS idx_segments_symbol ON segments(symbol)", + "CREATE INDEX IF NOT EXISTS idx_segments_rev_type " + "ON segments(revision_id, segment_type)", + + # -- evidence ------------------------------------------------------------ + """ + CREATE TABLE IF NOT EXISTS evidence ( + id TEXT PRIMARY KEY, + revision_id TEXT NOT NULL, + segment_id TEXT, + locator_json TEXT NOT NULL, + excerpt TEXT NOT NULL DEFAULT '', + content_hash TEXT NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY(revision_id) REFERENCES asset_revisions(id) ON DELETE CASCADE, + FOREIGN KEY(segment_id) REFERENCES segments(id) ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_evidence_revision ON evidence(revision_id)", + "CREATE INDEX IF NOT EXISTS idx_evidence_segment ON evidence(segment_id)", +) diff --git a/openmind/models.py b/openmind/models.py index 8c6a896..fe32c47 100644 --- a/openmind/models.py +++ b/openmind/models.py @@ -95,6 +95,14 @@ class TerminateReq(BaseModel): clear_cases: bool = False +class AssetSyncReq(BaseModel): + """Sync a single existing file into the canonical Asset model (Phase 2). + ``path`` must resolve under a registered workspace source root.""" + path: str + wait: bool = False + timeout: float = 3600.0 + + class RegenDocReq(BaseModel): scope: str page: Optional[str] = None diff --git a/openmind/ports/__init__.py b/openmind/ports/__init__.py index d2446d2..fd7e64d 100644 --- a/openmind/ports/__init__.py +++ b/openmind/ports/__init__.py @@ -1,10 +1,12 @@ """Ports: the narrow boundaries the application services depend on. -Only three exist, and each is justified by a real test seam that is exercised -by ``tests/verify_services.py``: +Each is justified by a real test seam exercised by ``tests/verify_services.py`` +or ``tests/verify_asset_model.py``: * :class:`~openmind.ports.workspace_repository.WorkspaceRepository` — satisfied by :mod:`openmind.db`; +* :class:`~openmind.ports.asset_repository.AssetRepository` — satisfied by + :mod:`openmind.db`; * :class:`~openmind.ports.job_repository.JobReader` / :class:`JobEngine` — satisfied by :mod:`openmind.db` and :mod:`openmind.jobs`; * :class:`~openmind.ports.runtime_ports.Clock` — real vs. fake, so bounded @@ -14,9 +16,10 @@ and the deterministic extractors. Those have one implementation and an existing env-var test seam; a Protocol over them would be indirection without coverage. """ +from .asset_repository import AssetRepository from .job_repository import JobEngine, JobReader from .runtime_ports import Clock, FakeClock, SystemClock from .workspace_repository import WorkspaceRepository -__all__ = ["JobEngine", "JobReader", "Clock", "FakeClock", "SystemClock", - "WorkspaceRepository"] +__all__ = ["AssetRepository", "JobEngine", "JobReader", "Clock", "FakeClock", + "SystemClock", "WorkspaceRepository"] diff --git a/openmind/ports/asset_repository.py b/openmind/ports/asset_repository.py new file mode 100644 index 0000000..1c5b98d --- /dev/null +++ b/openmind/ports/asset_repository.py @@ -0,0 +1,63 @@ +"""The persistence surface :class:`~openmind.services.asset_service.AssetService` +depends on. + +Like :mod:`openmind.ports.workspace_repository`, this is a +:class:`typing.Protocol` structurally satisfied by :mod:`openmind.db` — nothing +is registered, subclassed or adapted. The port exists so service tests can inject +a fake repository and exercise the workspace-scoping, not-found and ordering +paths without a real SQLite file, a real content store, or a real ingest. + +It is intentionally narrow: only the reads the Asset use cases need plus the one +transactional writer (:meth:`commit_revision`). Everything else in the Asset +section of :mod:`openmind.db` (the ingestion-facing helpers, cascade cleanup) is +reached directly by the modules that own those concerns. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Protocol, runtime_checkable + + +@runtime_checkable +class AssetRepository(Protocol): + """Structural contract satisfied by :mod:`openmind.db`.""" + + def get_asset(self, workspace_id: str, + asset_id: str) -> Optional[Dict[str, Any]]: ... + + def find_asset_by_logical_key(self, workspace_id: str, + logical_key: str) -> Optional[Dict[str, Any]]: ... + + def list_assets(self, workspace_id: str, asset_type: Optional[str] = None, + state: Optional[str] = None, limit: int = 100, + offset: int = 0) -> List[Dict[str, Any]]: ... + + def count_assets(self, workspace_id: str, asset_type: Optional[str] = None, + state: Optional[str] = None) -> int: ... + + def get_revision(self, workspace_id: str, + revision_id: str) -> Optional[Dict[str, Any]]: ... + + def list_revisions(self, workspace_id: str, asset_id: str, + limit: int = 50) -> List[Dict[str, Any]]: ... + + def list_segments(self, workspace_id: str, revision_id: str, limit: int = 200, + offset: int = 0) -> List[Dict[str, Any]]: ... + + def count_segments(self, workspace_id: str, revision_id: str) -> int: ... + + def get_segment(self, workspace_id: str, + segment_id: str) -> Optional[Dict[str, Any]]: ... + + def get_evidence(self, workspace_id: str, + evidence_id: str) -> Optional[Dict[str, Any]]: ... + + def get_evidence_for_segment(self, workspace_id: str, + segment_id: str) -> Optional[Dict[str, Any]]: ... + + def evidence_ids_for_revision(self, workspace_id: str, + revision_id: str) -> Dict[str, str]: ... + + def asset_stats(self, workspace_id: str) -> Dict[str, int]: ... + + +__all__ = ["AssetRepository"] diff --git a/openmind/runtime.py b/openmind/runtime.py index c607c71..5c3946c 100644 --- a/openmind/runtime.py +++ b/openmind/runtime.py @@ -103,6 +103,10 @@ def jobs(self): def ingest(self): return self.services.ingest + @property + def assets(self): + return self.services.assets + @property def export(self): return self.services.export diff --git a/openmind/segmentation.py b/openmind/segmentation.py new file mode 100644 index 0000000..702d198 --- /dev/null +++ b/openmind/segmentation.py @@ -0,0 +1,219 @@ +"""Deterministic source segmentation for the canonical Asset model. + +A Segment is a stable structural unit inside one immutable revision, and every +Segment carries source-locatable Evidence. This module turns one revision's +decoded text into Segment + Evidence drafts ready for +:func:`openmind.db.commit_revision`. + +SHARED BOUNDARIES, NOT A REWRITE +-------------------------------- +The RAG chunker (:func:`openmind.rag.chunk_file`) and this segmenter derive from +the *same* deterministic boundary primitives — the tree-sitter type/method +facts in :mod:`openmind.javaparse` and the ``CHUNK_MAX_LINES`` / +``CHUNK_OVERLAP_LINES`` line-range split — so their line ranges agree. But +``rag.chunk_file`` is deliberately left byte-for-byte unchanged: the RAG chunk +ids, the embedded header, the chunk metadata and the search response are a stable +contract, and forcing a one-to-one Segment↔chunk mapping would risk regressing +retrieval for no gain. Preserving current retrieval behaviour matters more than +an artificial mapping (see docs/v2/phase-2-asset-model.md §7). A determinism test +asserts the two agree on line ranges so they cannot drift. + +CONTENT MODE +------------ +* Java methods/constructors and generic line-range segments are ``verbatim`` — + their content is exactly the source lines they cover. +* The Java class-summary segment is ``derived`` — its content is the generated + signature summary (the same one the RAG class-summary chunk embeds), never + misrepresented as verbatim source. Its Evidence still cites the verbatim class + source range. + +RECOVERABILITY +-------------- +Every segment's line range and every evidence ``content_hash`` is computed over +the exact line slice ``[startLine, endLine]`` of the revision text, so the +content is recomputable from the immutable content blob after the source file +changes — with no running model involved. +""" +from __future__ import annotations + +import hashlib +from typing import Any, Dict, List + +from . import config +from . import javaparse as jp + +#: The stored evidence excerpt is a bounded verbatim preview; the authoritative, +#: full content is recovered from the immutable blob at read time. Bounding the +#: stored copy keeps the database small and honours "excerpt is bounded". +EXCERPT_STORE_MAX = 1200 + + +def hash_text_utf8(text: str) -> str: + """SHA-256 of ``text`` encoded UTF-8 — the same encoding the content blob is + stored in, so a segment/evidence hash is recomputable from the blob.""" + return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest() + + +def slice_lines(text: str, start_line: int, end_line: int) -> str: + """The verbatim 1-based inclusive line slice ``[start_line, end_line]`` of + *text*. Tolerant of out-of-range bounds (clamped), so a stale locator never + raises here — the caller decides how to report a mismatch.""" + if start_line <= 0 or end_line <= 0: + return "" + lines = text.split("\n") + lo = max(0, start_line - 1) + hi = min(len(lines), end_line) + if lo >= hi: + return "" + return "\n".join(lines[lo:hi]) + + +def _split_line_ranges(text: str, start_line: int, max_lines: int, + overlap: int): + """Deterministic bounded line-range split. Byte-for-byte the same boundaries + as :func:`openmind.rag._split_lines` (kept in lockstep by a determinism + test), so a generic file's segments and its RAG chunks cover identical + ranges.""" + lines = text.split("\n") + if len(lines) <= max_lines: + yield start_line, start_line + len(lines) - 1 + return + i = 0 + while i < len(lines): + seg = lines[i:i + max_lines] + yield start_line + i, start_line + i + len(seg) - 1 + if i + max_lines >= len(lines): + break + i += max_lines - overlap + + +def _evidence(logical_key: str, text: str, start_line: int, end_line: int, + symbol: str) -> Dict[str, Any]: + """Build an Evidence draft citing the verbatim source range. ``content_hash`` + validates the FULL slice; ``excerpt`` is a bounded preview of it.""" + verbatim = slice_lines(text, start_line, end_line) + excerpt = verbatim[:EXCERPT_STORE_MAX] + return { + "locator": {"kind": "source-range", "file": logical_key, + "startLine": start_line, "endLine": end_line, "symbol": symbol}, + "excerpt": excerpt, + "content_hash": hash_text_utf8(verbatim), + } + + +def _java_segments(logical_key: str, text: str) -> List[Dict[str, Any]]: + """Java segments via tree-sitter, or raise so the caller falls back to the + generic line-range strategy.""" + tree = jp.parse(text) + root = tree.root_node + src = bytes(text, "utf8") + package = jp.get_package(root, src) + drafts: List[Dict[str, Any]] = [] + for t in jp.iter_types(root, src): + cls = t["name"] + fqcn = f"{package}.{cls}" if package else cls + node = t["node"] + s_line, e_line = t["start_line"], t["end_line"] + # class-summary segment (DERIVED: the signature digest, not raw source) — + # the same shape the RAG class-summary chunk embeds. + sig_lines = [f"class {cls} ({t['kind']})"] + for f in jp.iter_fields(node, src): + sig_lines.append(f" field {f['type']} {f['name']}") + methods = list(jp.iter_methods(node, src)) + for m in methods: + sig_lines.append(f" method {m['signature']}") + summary = "\n".join(sig_lines) + drafts.append({ + "segment_key": f"type:{fqcn}", + "segment_type": "type", + "symbol": fqcn, + "start_line": s_line, "end_line": e_line, + "content_hash": hash_text_utf8(summary), + "content_mode": "derived", + "metadata": {"kind": t["kind"], "package": package, "class": cls}, + "evidence": _evidence(logical_key, text, s_line, e_line, fqcn), + }) + # one verbatim segment per method / constructor + for m in methods: + is_ctor = getattr(m["node"], "type", "") == "constructor_declaration" + seg_type = "constructor" if is_ctor else "method" + signature = " ".join((m["signature"] or "").split()) + member = f"{fqcn}#{signature}" + ms, me = m["start_line"], m["end_line"] + verbatim = slice_lines(text, ms, me) + drafts.append({ + "segment_key": f"{seg_type}:{member}", + "segment_type": seg_type, + "symbol": member, + "start_line": ms, "end_line": me, + "content_hash": hash_text_utf8(verbatim), + "content_mode": "verbatim", + "metadata": {"class": cls, "package": package, + "signature": signature}, + "evidence": _evidence(logical_key, text, ms, me, member), + }) + return drafts + + +def _generic_segments(logical_key: str, text: str) -> List[Dict[str, Any]]: + """Deterministic bounded line-range segments for config / non-Java / parse- + failure files — 1:1 with the generic RAG chunks.""" + basename = logical_key.replace("\\", "/").rsplit("/", 1)[-1] + drafts: List[Dict[str, Any]] = [] + for i, (sl, el) in enumerate( + _split_line_ranges(text, 1, config.CHUNK_MAX_LINES, + config.CHUNK_OVERLAP_LINES)): + verbatim = slice_lines(text, sl, el) + drafts.append({ + "segment_key": f"file-range:{i + 1:06d}", + "segment_type": "file", + "symbol": basename, + "start_line": sl, "end_line": el, + "content_hash": hash_text_utf8(verbatim), + "content_mode": "verbatim", + "metadata": {}, + "evidence": _evidence(logical_key, text, sl, el, basename), + }) + return drafts + + +def _disambiguate(drafts: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Guarantee ``segment_key`` is unique within a revision (the DB enforces + ``UNIQUE(revision_id, segment_key)``). Ambiguous/duplicate symbols get a + deterministic ``@`` position suffix rather than a bare index; an + exact collision at the same line (should not occur) falls back to ordinal.""" + seen: Dict[str, int] = {} + for i, d in enumerate(drafts): + d["ordinal"] = i + key = d["segment_key"] + if key in seen: + candidate = f"{key}@{d.get('start_line') or 0}" + if candidate in seen: + candidate = f"{candidate}#{i}" + d["segment_key"] = candidate + seen[d["segment_key"]] = i + return drafts + + +def segment_source(logical_key: str, text: str) -> List[Dict[str, Any]]: + """Decompose one revision's text into ordered Segment drafts (each carrying an + Evidence draft), ready for :func:`openmind.db.commit_revision`. + + Deterministic and model-free. Java files use tree-sitter when available and + fall back to the generic line-range strategy on any parse failure, exactly as + the RAG chunker does. Always returns at least one segment (an empty file + yields a single ``file-range:000001``). + """ + drafts: List[Dict[str, Any]] = [] + if logical_key.lower().endswith(".java") and jp.available(): + try: + drafts = _java_segments(logical_key, text) + except Exception: + drafts = [] + if not drafts: + drafts = _generic_segments(logical_key, text) + return _disambiguate(drafts) + + +__all__ = ["segment_source", "slice_lines", "hash_text_utf8", + "EXCERPT_STORE_MAX"] diff --git a/openmind/services/__init__.py b/openmind/services/__init__.py index 11878af..b5b4f52 100644 --- a/openmind/services/__init__.py +++ b/openmind/services/__init__.py @@ -25,10 +25,11 @@ """ from typing import Any -__all__ = ["ExportService", "HealthService", "IngestService", "JobService", - "ServiceContainer", "WorkspaceService"] +__all__ = ["AssetService", "ExportService", "HealthService", "IngestService", + "JobService", "ServiceContainer", "WorkspaceService"] _EXPORTS = { + "AssetService": ".asset_service", "ExportService": ".export_service", "HealthService": ".health_service", "IngestService": ".ingest_service", diff --git a/openmind/services/asset_service.py b/openmind/services/asset_service.py new file mode 100644 index 0000000..4d89fa3 --- /dev/null +++ b/openmind/services/asset_service.py @@ -0,0 +1,352 @@ +"""Asset model use cases — read the canonical Asset/Revision/Segment/Evidence +store, and drive a single-file sync into it. + +Exposed as ``runtime.assets`` and ``ServiceContainer.assets``, and shared by the +CLI, the FastAPI adapter and the MCP server, so the same queries are reachable +without constructing an HTTP request. + +WORKSPACE SCOPING IS ENFORCED ON EVERY READ +------------------------------------------- +Every method takes a ``workspace_id`` and validates it first (raising +:class:`WorkspaceNotFound`), then reads through the workspace-scoped repository +queries — so an Asset/Revision/Segment/Evidence id from workspace A can never be +read through workspace B (it simply resolves to nothing and raises the typed +not-found). The service raises typed domain errors, never ``HTTPException``, so +every adapter maps them its own way. + +EVIDENCE IS RECOVERED FROM THE IMMUTABLE SNAPSHOT, NOT A MODEL +------------------------------------------------------------- +:meth:`get_evidence` reconstructs the cited content from the immutable content +blob and reports, honestly and separately, whether the snapshot is available or +corrupt and whether the live source still matches, has changed, or is missing. +No running model is involved. +""" +from __future__ import annotations + +import os +from typing import Any, Dict, List, Optional + +from .. import config, content_store, db as db_module, machine, segmentation, walker +from ..domain.errors import (AssetNotFound, ContentCorruption, EvidenceNotFound, + InvalidRequest, RevisionNotFound, SegmentNotFound) +from ..domain.types import (Asset, AssetRevision, AssetState, AssetType, + Evidence, Segment) + + +class AssetService: + """Read/query use cases over the canonical Asset model.""" + + #: Hard upper bounds so no adapter can request an unbounded page. + MAX_ASSET_LIMIT = 500 + MAX_SEGMENT_LIMIT = 1000 + MAX_REVISION_LIMIT = 500 + MAX_EVIDENCE_CHARS = 100_000 + + def __init__(self, workspaces: Any, ingest: Any, repo: Any = None, + content: Any = None) -> None: + self._workspaces = workspaces + self._ingest = ingest + self._repo: Any = repo if repo is not None else db_module + self._content: Any = content if content is not None else content_store + + # -- helpers ------------------------------------------------------------ + def _require_workspace(self, workspace_id: str) -> Dict[str, Any]: + # Raises WorkspaceNotFound for an unknown or deleting workspace, so every + # asset read fails the same honest way a bad workspace id does elsewhere. + return self._workspaces.get(workspace_id) + + @staticmethod + def _bound(limit: int, hard: int) -> int: + try: + limit = int(limit) + except (TypeError, ValueError): + limit = hard + return max(1, min(limit, hard)) + + # -- asset reads -------------------------------------------------------- + def list_assets(self, workspace_id: str, asset_type: Optional[str] = None, + state: Optional[str] = None, limit: int = 100, + offset: int = 0) -> Dict[str, Any]: + """A bounded page of the workspace's assets plus the total count. + + ``asset_type`` / ``state`` are validated against the closed vocabularies; + an unknown value is a caller error, not a silent empty page. + """ + self._require_workspace(workspace_id) + if asset_type is not None and asset_type not in AssetType.VALUES: + raise InvalidRequest( + f"unknown asset_type: {asset_type!r}", + details={"asset_type": asset_type, + "allowed": sorted(AssetType.VALUES)}) + if state is not None and state not in AssetState.VALUES: + raise InvalidRequest( + f"unknown state: {state!r}", + details={"state": state, "allowed": sorted(AssetState.VALUES)}) + limit = self._bound(limit, self.MAX_ASSET_LIMIT) + offset = max(0, int(offset)) + rows = self._repo.list_assets(workspace_id, asset_type=asset_type, + state=state, limit=limit, offset=offset) + total = self._repo.count_assets(workspace_id, asset_type=asset_type, + state=state) + return { + "workspace_id": workspace_id, + "assets": [Asset.from_row(r).as_dict() for r in rows], + "total": total, "limit": limit, "offset": offset, + "count": len(rows), + } + + def get_asset(self, workspace_id: str, asset_id: str) -> Dict[str, Any]: + """An asset plus a summary of its current revision.""" + self._require_workspace(workspace_id) + row = self._repo.get_asset(workspace_id, asset_id) + if not row: + raise AssetNotFound(asset_id, workspace_id=workspace_id) + return self._asset_with_current(workspace_id, row) + + def get_asset_by_logical_key(self, workspace_id: str, + logical_key: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + row = self._repo.find_asset_by_logical_key(workspace_id, logical_key) + if not row: + raise AssetNotFound(logical_key, workspace_id=workspace_id) + return self._asset_with_current(workspace_id, row) + + def _asset_with_current(self, workspace_id: str, + row: Dict[str, Any]) -> Dict[str, Any]: + out = Asset.from_row(row).as_dict() + current = None + if row.get("current_revision_id"): + rev = self._repo.get_revision(workspace_id, row["current_revision_id"]) + if rev: + current = AssetRevision.from_row(rev).as_dict() + current["segment_count"] = self._repo.count_segments( + workspace_id, rev["id"]) + out["current_revision"] = current + return out + + # -- revision reads ----------------------------------------------------- + def list_revisions(self, workspace_id: str, asset_id: str, + limit: int = 50) -> Dict[str, Any]: + self._require_workspace(workspace_id) + if not self._repo.get_asset(workspace_id, asset_id): + raise AssetNotFound(asset_id, workspace_id=workspace_id) + limit = self._bound(limit, self.MAX_REVISION_LIMIT) + rows = self._repo.list_revisions(workspace_id, asset_id, limit=limit) + return { + "workspace_id": workspace_id, "asset_id": asset_id, + "revisions": [AssetRevision.from_row(r).as_dict() for r in rows], + "count": len(rows), "limit": limit, + } + + def get_revision(self, workspace_id: str, + revision_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + row = self._repo.get_revision(workspace_id, revision_id) + if not row: + raise RevisionNotFound(revision_id, workspace_id=workspace_id) + out = AssetRevision.from_row(row).as_dict() + out["segment_count"] = self._repo.count_segments(workspace_id, revision_id) + return out + + # -- segment reads ------------------------------------------------------ + def list_segments(self, workspace_id: str, revision_id: str, + limit: int = 200, offset: int = 0) -> Dict[str, Any]: + self._require_workspace(workspace_id) + if not self._repo.get_revision(workspace_id, revision_id): + raise RevisionNotFound(revision_id, workspace_id=workspace_id) + limit = self._bound(limit, self.MAX_SEGMENT_LIMIT) + offset = max(0, int(offset)) + rows = self._repo.list_segments(workspace_id, revision_id, limit=limit, + offset=offset) + total = self._repo.count_segments(workspace_id, revision_id) + # Attach each segment's evidence id (one scoped query) so callers can + # follow a segment straight to `get_evidence`. + ev_by_seg = self._repo.evidence_ids_for_revision(workspace_id, revision_id) + segments = [] + for r in rows: + d = Segment.from_row(r).as_dict() + d["evidence_id"] = ev_by_seg.get(r["id"]) + segments.append(d) + return { + "workspace_id": workspace_id, "revision_id": revision_id, + "segments": segments, + "total": total, "count": len(rows), "limit": limit, "offset": offset, + } + + def get_segment(self, workspace_id: str, segment_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + row = self._repo.get_segment(workspace_id, segment_id) + if not row: + raise SegmentNotFound(segment_id, workspace_id=workspace_id) + out = Segment.from_row(row).as_dict() + ev = self._repo.get_evidence_for_segment(workspace_id, segment_id) + out["evidence_id"] = ev["id"] if ev else None + return out + + # -- evidence reads ----------------------------------------------------- + def get_evidence(self, workspace_id: str, evidence_id: str, + max_chars: int = 4000) -> Dict[str, Any]: + """Evidence with its content recovered from the IMMUTABLE snapshot, plus + an honest report of snapshot integrity and whether the live source still + matches. Never depends on a running model.""" + self._require_workspace(workspace_id) + ev = self._repo.get_evidence(workspace_id, evidence_id) + if not ev: + raise EvidenceNotFound(evidence_id, workspace_id=workspace_id) + max_chars = max(1, min(int(max_chars), self.MAX_EVIDENCE_CHARS)) + + rev = self._repo.get_revision(workspace_id, ev["revision_id"]) + locator = ev.get("locator") or {} + start = int(locator.get("startLine", 0) or 0) + end = int(locator.get("endLine", 0) or 0) + expected_hash = ev.get("content_hash", "") + + snapshot_status = "missing" + content = "" + blob_hash = (rev or {}).get("content_blob_hash", "") + if rev and blob_hash: + try: + blob = self._content.get(workspace_id, blob_hash) + snap_text = blob.decode("utf-8", "replace") + recovered = segmentation.slice_lines(snap_text, start, end) + if segmentation.hash_text_utf8(recovered) == expected_hash: + snapshot_status = "available" + content = recovered + else: + # blob intact but the cited range no longer hashes as recorded + snapshot_status = "corrupt" + except ContentCorruption: + snapshot_status = "corrupt" + + current_status = self._current_source_status( + workspace_id, locator.get("file", ""), start, end, expected_hash) + + truncated = len(content) > max_chars + return { + "id": ev["id"], + "revision_id": ev["revision_id"], + "segment_id": ev.get("segment_id"), + "locator": locator, + "content_hash": expected_hash, + "excerpt": ev.get("excerpt", ""), + "content": content[:max_chars], + "truncated": truncated, + "snapshot": {"status": snapshot_status}, + "current_source": {"status": current_status}, + "revision": None if not rev else { + "id": rev["id"], "sequence": rev["sequence"], + "status": rev["status"], "content_hash": rev["content_hash"], + "content_blob_hash": blob_hash}, + } + + def _current_source_status(self, workspace_id: str, rel_file: str, + start: int, end: int, expected_hash: str) -> str: + """Whether the live on-disk source still matches the cited range: + ``matches`` / ``changed`` / ``missing``. Never reads outside the + workspace source roots.""" + if not rel_file: + return "missing" + abspath = machine.from_rel(workspace_id, rel_file) + # Defense in depth: never read a path that resolves outside the + # workspace's source root, even if a locator were somehow crafted with a + # traversal component. Locators are only ever produced from workspace- + # relative keys of ingested files, so this should always hold — but the + # guarantee ("no evidence cites a file outside the source roots") is + # enforced here, not merely assumed. + root = machine.project_root(workspace_id) + if root: + root_n = os.path.normcase(os.path.normpath(root)) + abs_n = os.path.normcase(os.path.normpath(abspath)) + if abs_n != root_n and not abs_n.startswith(root_n + os.sep): + return "missing" + if not abspath or not os.path.isfile(abspath): + return "missing" + try: + text = walker.read_text(abspath) + except Exception: + return "missing" + recovered = segmentation.slice_lines(text, start, end) + return "matches" if segmentation.hash_text_utf8(recovered) == expected_hash \ + else "changed" + + # -- stats -------------------------------------------------------------- + def stats(self, workspace_id: str) -> Dict[str, Any]: + """Aggregate Asset-model counts for the workspace.""" + self._require_workspace(workspace_id) + out: Dict[str, Any] = {"workspace_id": workspace_id} + out.update(self._repo.asset_stats(workspace_id)) + return out + + # -- single-file sync --------------------------------------------------- + def sync_file(self, workspace_id: str, path: str, wait: bool = False, + timeout: float = 3600.0) -> Dict[str, Any]: + """Ingest ONE existing file that lives under a registered source root. + + A directory is rejected (use ``openmind add`` to register a source path). + An unsupported format is registered as an ``unsupported`` Asset — recorded + honestly, never falsely reported as parsed — and NOT ingested. A supported + file triggers a filtered ingest of just that file (which updates its Asset + and RAG projection without rebuilding the whole-project artifacts). + """ + self._require_workspace(workspace_id) + raw = (path or "").strip() + if not raw: + raise InvalidRequest("path must not be empty", details={"field": "path"}) + abspath = walker.norm(os.path.abspath(os.path.expanduser(raw))) + if os.path.isdir(abspath): + raise InvalidRequest( + "asset add takes a single file; use 'openmind add' to register a " + "directory as a source path", + details={"path": abspath}) + if not os.path.isfile(abspath): + raise InvalidRequest(f"file not found: {abspath}", + details={"path": abspath}) + + root = machine.project_root(workspace_id) + logical_key = machine.relativize(abspath, root) + if not root or logical_key == abspath or machine._is_absolute(logical_key): + raise InvalidRequest( + "file is not under a registered workspace source root; add its " + "directory with 'openmind add' first", + details={"path": abspath, "source_root": root}) + + ext = os.path.splitext(abspath)[1].lower() + supported = (ext in config.INDEX_EXTENSIONS + and ext not in config.BINARY_EXTENSIONS + and self._within_size(abspath)) + title = logical_key.rsplit("/", 1)[-1] + if not supported: + asset = self._repo.upsert_asset( + workspace_id, logical_key, asset_type=AssetType.classify(logical_key), + title=title, source_path=logical_key, state=AssetState.UNSUPPORTED) + return { + "workspace_id": workspace_id, "logical_key": logical_key, + "supported": False, "state": AssetState.UNSUPPORTED, + "asset_id": asset["id"], "asset": asset, "job_id": None, + } + + result = self._ingest.start(workspace_id, path=abspath, wait=wait, + timeout=timeout) + asset = self._repo.find_asset_by_logical_key(workspace_id, logical_key) + out = { + "workspace_id": workspace_id, "logical_key": logical_key, + "supported": True, "job_id": result.get("job_id"), + "waited": result.get("waited", False), + "asset_id": asset["id"] if asset else None, + } + for k in ("status", "completed", "waited_seconds"): + if k in result: + out[k] = result[k] + if asset: + out["asset"] = self._asset_with_current(workspace_id, asset) + return out + + @staticmethod + def _within_size(abspath: str) -> bool: + try: + return os.path.getsize(abspath) <= config.MAX_FILE_BYTES + except OSError: + return False + + +__all__ = ["AssetService"] diff --git a/openmind/services/service_container.py b/openmind/services/service_container.py index d91027d..028ad52 100644 --- a/openmind/services/service_container.py +++ b/openmind/services/service_container.py @@ -15,6 +15,7 @@ from typing import Callable, Optional from ..ports.runtime_ports import Clock +from .asset_service import AssetService from .export_service import ExportService from .health_service import HealthService from .ingest_service import IngestService @@ -32,6 +33,7 @@ def __init__(self, ensure_worker: Optional[Callable[[], None]] = None, self._workspaces: Optional[WorkspaceService] = None self._jobs: Optional[JobService] = None self._ingest: Optional[IngestService] = None + self._assets: Optional[AssetService] = None self._export: Optional[ExportService] = None self._health: Optional[HealthService] = None @@ -54,6 +56,15 @@ def ingest(self) -> IngestService: ensure_worker=self._ensure_worker) return self._ingest + @property + def assets(self) -> AssetService: + # Read-only Asset queries plus single-file sync. It reuses the ingest + # service for sync_file; constructing it never starts a worker (only a + # waited sync does), so the read-only MCP asset tools stay side-effect free. + if self._assets is None: + self._assets = AssetService(self.workspaces, self.ingest) + return self._assets + @property def export(self) -> ExportService: if self._export is None: diff --git a/openmind/version.py b/openmind/version.py index 30a055e..caee786 100644 --- a/openmind/version.py +++ b/openmind/version.py @@ -5,19 +5,22 @@ ``generator.version`` — reads it from here, so the value can never drift between surfaces. -This is a PRE-v2 development version on purpose. OpenMind v2's enterprise -knowledge features (Asset/Revision/Claim tables, the engineering Knowledge -Graph, cloud providers, traceability) are NOT implemented; labelling the -current build ``2.0.0`` would be a false claim. ``1.1.0-dev`` is the honest -reading: the shipped artifact contract is schema 1.1.x, and this is -development work on top of it. +This is a PRE-v2 development version on purpose. OpenMind v2's later enterprise +knowledge features (Claim/Relation tables, the engineering Knowledge Graph, +cloud providers, requirement traceability, document parsing) are NOT +implemented; labelling the current build ``2.0.0`` would be a false claim. +``1.2.0-dev`` is the honest reading: Phase 2 added the canonical +Asset/Revision/Segment/Evidence foundation and its immutable content store on +top of the Phase 1 tool-first runtime, while the shipped artifact contract is +still schema 1.1.x. The artifact ``schemaVersion`` is deliberately NOT derived from this constant — it is a separate, frozen integration contract owned by -:mod:`openmind.artifacts`. +:mod:`openmind.artifacts` and remains ``1.1.0`` (the Asset model is not exported +yet). """ from __future__ import annotations -RUNTIME_VERSION = "1.1.0-dev" +RUNTIME_VERSION = "1.2.0-dev" __all__ = ["RUNTIME_VERSION"] diff --git a/scripts/run_acceptance.py b/scripts/run_acceptance.py index 337fe6c..de4af6c 100644 --- a/scripts/run_acceptance.py +++ b/scripts/run_acceptance.py @@ -58,6 +58,8 @@ def path(self) -> Path: SUITE: List[Script] = [ # -- deterministic, dependency-light ------------------------------------ Script("verify_migrations", CORE, "schema migrations: empty, legacy, checksum, rollback"), + Script("verify_content_store", CORE, + "immutable content-addressed blob store: atomic, reuse, corruption"), Script("verify_structure", CORE, "detection + structure extraction, incremental"), Script("verify_glossary", CORE, "deterministic glossary extraction and lookup"), Script("verify_router", CORE, "capability routing and graceful degradation"), @@ -69,6 +71,14 @@ def path(self) -> Path: Script("verify_cli", CORE, "CLI contract: JSON, exit codes, commands"), Script("verify_adapters", CORE, "FastAPI route + MCP + skill-bridge compatibility"), + # -- canonical Asset model (v2 Phase 2) --------------------------------- + Script("verify_asset_model", CORE, + "Asset/Revision/Segment/Evidence: ingest, idempotency, change, " + "revert, removal, reappearance, legacy backfill, lifecycle"), + Script("verify_asset_cli", CORE, "CLI asset commands: JSON, scoping, bounds"), + Script("verify_asset_adapters", CORE, + "additive REST endpoints + read-only MCP asset tools"), + # -- template / docs pipeline ------------------------------------------- Script("verify_templates", CORE, "template profile selection and validation"), Script("verify_facets", CORE, "template facets, roles and projections"), diff --git a/tests/verify_adapters.py b/tests/verify_adapters.py index 52c64df..2ced86b 100644 --- a/tests/verify_adapters.py +++ b/tests/verify_adapters.py @@ -262,12 +262,22 @@ def check(desc, cond, extra=""): check("the server keeps the published name", server.name == "open-mind") registered = sorted(t.name for t in asyncio.run(server.list_tools())) -check("all nine tools are registered on the server", - registered == sorted(REQUIRED_TOOLS), str(registered)) +# The nine core tools are a STABLE contract: they must remain registered, +# unchanged. Phase 2 adds read-only Asset tools ALONGSIDE them. +check("all nine core tools are still registered on the server", + set(REQUIRED_TOOLS) <= set(registered), str(registered)) + +ASSET_TOOLS = ("list_assets", "get_asset", "get_asset_revisions", "get_evidence") +check("the four read-only Asset tools are registered additively", + set(ASSET_TOOLS) <= set(registered), str(registered)) +check("the core tool set is unchanged (no core tool renamed/removed)", + set(mcp_server.TOOL_NAMES) == set(REQUIRED_TOOLS), str(mcp_server.TOOL_NAMES)) +check("the Asset tools are the only additions", + set(registered) == set(REQUIRED_TOOLS) | set(ASSET_TOOLS), str(registered)) descriptions = {t.name: (t.description or "") for t in asyncio.run(server.list_tools())} check("every tool still carries its docstring description", - all(descriptions[n].strip() for n in REQUIRED_TOOLS)) + all(descriptions[n].strip() for n in REQUIRED_TOOLS + ASSET_TOOLS)) second = mcp_server.create_mcp_server(get_runtime()) check("create_mcp_server can be called repeatedly (independent instances)", diff --git a/tests/verify_asset_adapters.py b/tests/verify_asset_adapters.py new file mode 100644 index 0000000..cb19972 --- /dev/null +++ b/tests/verify_asset_adapters.py @@ -0,0 +1,151 @@ +"""Asset model adapters — additive REST endpoints and read-only MCP tools, +with every existing route/tool still present and cross-workspace reads refused. +""" +import asyncio +import os +import sys +import tempfile + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 — forces an isolated data dir (never the live one) + +from fastapi.testclient import TestClient # noqa: E402 + +from openmind import main, mcp_server # noqa: E402 +from openmind.runtime import get_runtime # noqa: E402 + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +FIXTURE = os.path.join(REPO, "fixtures", "sample-repo") + +_results = [] + + +def check(desc, cond, extra=""): + _results.append((desc, bool(cond))) + print(("PASS" if cond else "FAIL") + " - " + desc + + (f" [{extra}]" if extra and not cond else "")) + + +# --------------------------------------------------------------------------- +# Set-up: a real ingested workspace + an empty second workspace. +# --------------------------------------------------------------------------- +runtime = get_runtime() +runtime.ensure_worker() +WS = runtime.workspaces.create("adapters-assets", path=FIXTURE)["id"] +WS2 = runtime.workspaces.create("adapters-other")["id"] +runtime.ingest.start(WS, wait=True, timeout=180) + +client = TestClient(main.app) + +# --------------------------------------------------------------------------- +# Existing REST routes still operational (a representative sample). +# --------------------------------------------------------------------------- +for path in ("/projects", f"/projects/{WS}", "/api/health", "/templates"): + r = client.get(path) + check(f"existing REST route still works: GET {path}", r.status_code == 200) +check("GET /projects is still shaped {projects:[...]} (not renamed)", + "projects" in client.get("/projects").json()) + +# --------------------------------------------------------------------------- +# Existing MCP tools still present, unchanged; new asset tools added. +# --------------------------------------------------------------------------- +server = mcp_server.create_mcp_server(runtime) +tool_names = {t.name for t in asyncio.run(server.list_tools())} +CORE = {"search", "route", "dispatch", "get_glossary", "find_similar_cases", + "save_case", "get_doc", "propose_fix", "apply_fix"} +ASSET = {"list_assets", "get_asset", "get_asset_revisions", "get_evidence"} +check("all nine core MCP tools remain", CORE <= tool_names) +check("the four read-only asset MCP tools are added", ASSET <= tool_names) +check("only the asset tools were added", tool_names == CORE | ASSET) +check("the MCP server keeps its name", server.name == "open-mind") + +# --------------------------------------------------------------------------- +# New REST asset endpoints. +# --------------------------------------------------------------------------- +r = client.get(f"/projects/{WS}/assets") +check("GET /assets exits 200", r.status_code == 200) +listing = r.json() +check("GET /assets returns a bounded page with a total", + "assets" in listing and "total" in listing and listing["total"] >= 1) + +r = client.get(f"/projects/{WS}/assets/stats") +check("GET /assets/stats exits 200 (literal path not shadowed by {asset_id})", + r.status_code == 200 and "assets_total" in r.json()) + +ASSET_ID = listing["assets"][0]["id"] +r = client.get(f"/projects/{WS}/assets/{ASSET_ID}") +check("GET /assets/{id} exits 200 with a current revision", + r.status_code == 200 and r.json().get("current_revision")) +REV_ID = r.json()["current_revision"]["id"] + +r = client.get(f"/projects/{WS}/assets/{ASSET_ID}/revisions") +check("GET /assets/{id}/revisions exits 200", r.status_code == 200 and r.json()["count"] >= 1) + +r = client.get(f"/projects/{WS}/revisions/{REV_ID}") +check("GET /revisions/{id} exits 200", r.status_code == 200) + +r = client.get(f"/projects/{WS}/revisions/{REV_ID}/segments") +segs = r.json() +check("GET /revisions/{id}/segments exits 200 and is bounded", + r.status_code == 200 and "limit" in segs and segs["total"] >= 1) +EV_ID = segs["segments"][0]["evidence_id"] + +r = client.get(f"/projects/{WS}/evidence/{EV_ID}", params={"max_chars": 40}) +ev = r.json() +check("GET /evidence/{id} exits 200", r.status_code == 200) +check("evidence content is bounded by max_chars", len(ev["content"]) <= 40) +check("evidence reports snapshot + current-source state", + "snapshot" in ev and "current_source" in ev) +check("evidence locator is source-traceable (workspace-relative)", + not str(ev["locator"].get("file", "")).startswith("/") + and ev["locator"].get("startLine", 0) >= 1) + +# --------------------------------------------------------------------------- +# Cross-workspace reads are refused (404), not leaked. +# --------------------------------------------------------------------------- +check("cross-workspace GET /assets/{id} -> 404", + client.get(f"/projects/{WS2}/assets/{ASSET_ID}").status_code == 404) +check("cross-workspace GET /revisions/{id} -> 404", + client.get(f"/projects/{WS2}/revisions/{REV_ID}").status_code == 404) +check("cross-workspace GET /evidence/{id} -> 404", + client.get(f"/projects/{WS2}/evidence/{EV_ID}").status_code == 404) +check("unknown workspace GET /assets -> 404", + client.get("/projects/p_nope00000000/assets").status_code == 404) + +# --------------------------------------------------------------------------- +# New MCP asset tools work and are workspace-scoped. +# --------------------------------------------------------------------------- +mla = mcp_server.list_assets(WS, limit=5) +check("MCP list_assets returns a bounded set", mla["total"] >= 1 and mla["count"] <= 5) +mid = mla["assets"][0]["id"] +ma = mcp_server.get_asset(WS, mid) +check("MCP get_asset returns the asset", ma["id"] == mid) +mrevs = mcp_server.get_asset_revisions(WS, mid) +check("MCP get_asset_revisions returns history", mrevs["count"] >= 1) +mev = mcp_server.get_evidence(WS, EV_ID) +check("MCP get_evidence returns a workspace-relative locator", + not str(mev["locator"].get("file", "")).startswith("/")) +check("MCP get_evidence states snapshot provenance", + "snapshot" in mev and "current_source" in mev) + +# cross-scope MCP reads are refused +from openmind.domain.errors import AssetNotFound, EvidenceNotFound # noqa: E402 +_scoped = False +try: + mcp_server.get_asset(WS2, mid) +except AssetNotFound: + _scoped = True +check("MCP get_asset refuses a cross-workspace id", _scoped) +_scoped_ev = False +try: + mcp_server.get_evidence(WS2, EV_ID) +except EvidenceNotFound: + _scoped_ev = True +check("MCP get_evidence refuses a cross-workspace id", _scoped_ev) + +# --------------------------------------------------------------------------- +bad = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") +sys.exit(1 if bad else 0) diff --git a/tests/verify_asset_cli.py b/tests/verify_asset_cli.py new file mode 100644 index 0000000..eb10425 --- /dev/null +++ b/tests/verify_asset_cli.py @@ -0,0 +1,208 @@ +"""CLI ``asset`` command group — JSON contract, exit codes, bounded output, +workspace scoping and honest not-found. + +Each case runs the CLI as a REAL subprocess: what matters is the exit code the +shell sees and that stdout carries exactly one JSON object with every diagnostic +on stderr. +""" +import json +import os +import subprocess +import sys +import tempfile + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 — forces an isolated data dir (never the live one) + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +FIXTURE = os.path.join(REPO, "fixtures", "sample-repo") + +_results = [] + + +def check(desc, cond, extra=""): + _results.append((desc, bool(cond))) + print(("PASS" if cond else "FAIL") + " - " + desc + + (f" [{extra}]" if extra and not cond else "")) + + +ENV = dict(os.environ) +ENV.update({ + "OPENMIND_DATA_DIR": tempfile.mkdtemp(prefix="om_acli_"), + "OPENMIND_MACHINE_DIR": tempfile.mkdtemp(prefix="om_acli_machine_"), + "OPENMIND_EMBED_OFFLINE": "1", + "OPENMIND_EMBED_DEVICE": "cpu", + "OPENMIND_INGEST_FREE_GPU": "0", + "OPENMIND_ENRICH_EGRESS": "0", + "OPENMIND_SOURCELINK_EGRESS": "0", + "PYTHONIOENCODING": "utf-8", +}) + + +def run(*args, timeout=900): + proc = subprocess.run([sys.executable, "-m", "openmind.cli", *args], + cwd=REPO, env=ENV, capture_output=True, text=True, + errors="replace", timeout=timeout) + return proc.returncode, proc.stdout, proc.stderr + + +def as_json(stdout): + try: + return json.loads(stdout) + except (ValueError, TypeError): + return None + + +# --------------------------------------------------------------------------- +# Set-up: a real workspace, ingested with the offline embedder. +# --------------------------------------------------------------------------- +code, out, err = run("init", "--name", "cli-assets", "--path", FIXTURE, + "--ingest", "--wait", "--json") +init = as_json(out) +check("init --ingest --wait exits 0", code == 0, str(code)) +WS = init["workspace_id"] if init else "" +check("workspace was created", bool(WS)) + +# --------------------------------------------------------------------------- +# asset list +# --------------------------------------------------------------------------- +code, out, err = run("asset", "list", "--workspace", WS, "--json") +data = as_json(out) +check("asset list --json exits 0", code == 0) +check("asset list --json prints exactly one JSON object", data is not None) +check("asset list carries the total and a bounded page", + data and "total" in data and "count" in data and "limit" in data) +check("asset list found the ingested assets", data and data["total"] >= 1) +check("asset list stdout is pure JSON (diagnostics on stderr)", + "\x1b[" not in out) # no ANSI escapes ever + +# list limit is enforced +code, out, err = run("asset", "list", "--workspace", WS, "--limit", "2", "--json") +data = as_json(out) +check("asset list --limit bounds the page", data and data["count"] <= 2) + +# type + state filters +code, out, err = run("asset", "list", "--workspace", WS, "--type", "source-code", + "--state", "active", "--json") +data = as_json(out) +check("asset list --type/--state filters exit 0", code == 0 and data is not None) + +# an unknown type is a usage error (exit 2), not a silent empty page +code, out, err = run("asset", "list", "--workspace", WS, "--type", "bogus", "--json") +data = as_json(out) +check("asset list rejects an unknown --type", code == 2 and data and data["ok"] is False) + +# --------------------------------------------------------------------------- +# asset show / revisions / segments / evidence +# --------------------------------------------------------------------------- +code, out, err = run("asset", "list", "--workspace", WS, "--limit", "1", "--json") +first = as_json(out)["assets"][0] +AID = first["id"] + +code, out, err = run("asset", "show", "--workspace", WS, "--asset", AID, "--json") +show = as_json(out) +check("asset show --json exits 0", code == 0 and show is not None) +check("asset show returns the current revision summary", + show and show["asset"].get("current_revision")) +RID = show["asset"]["current_revision"]["id"] + +code, out, err = run("asset", "revisions", "--workspace", WS, "--asset", AID, "--json") +revs = as_json(out) +check("asset revisions --json exits 0", code == 0 and revs is not None) +check("asset revisions lists at least one revision", revs and revs["count"] >= 1) + +code, out, err = run("asset", "segments", "--workspace", WS, "--revision", RID, "--json") +segs = as_json(out) +check("asset segments --json exits 0", code == 0 and segs is not None) +check("asset segments is bounded (limit present)", segs and "limit" in segs) +check("asset segments surfaces an evidence id for discovery", + segs and segs["segments"] and segs["segments"][0].get("evidence_id")) +EID = segs["segments"][0]["evidence_id"] + +code, out, err = run("asset", "evidence", "--workspace", WS, "--evidence", EID, + "--max-chars", "50", "--json") +ev = as_json(out) +check("asset evidence --json exits 0", code == 0 and ev is not None) +check("asset evidence reports snapshot + current-source status", + ev and "snapshot" in ev and "current_source" in ev) +check("asset evidence content is bounded by --max-chars", + ev and len(ev["content"]) <= 50) +check("asset evidence carries a workspace-relative locator", + ev and not str(ev["locator"].get("file", "")).startswith("/")) + +# --------------------------------------------------------------------------- +# Workspace scoping + honest not-found (typed errors, right exit codes) +# --------------------------------------------------------------------------- +code2, out2, err2 = run("init", "--name", "other", "--json") +WS2 = as_json(out2)["workspace_id"] + +code, out, err = run("asset", "show", "--workspace", WS2, "--asset", AID, "--json") +data = as_json(out) +check("cross-workspace asset show fails with exit 1", code == 1) +check("cross-workspace asset show reports a typed not-found", + data and data["ok"] is False and data["error"]["code"] == "asset_not_found") + +code, out, err = run("asset", "evidence", "--workspace", WS2, "--evidence", EID, "--json") +data = as_json(out) +check("cross-workspace evidence fails with a typed error", + code == 1 and data and data["error"]["code"] == "evidence_not_found") + +code, out, err = run("asset", "show", "--workspace", "p_nonexistent0", "--asset", + AID, "--json") +data = as_json(out) +check("unknown workspace fails honestly (exit 1, typed not-found)", + code == 1 and data and data["error"]["code"] == "workspace_not_found") + +# a missing revision/evidence id is a typed not-found, not a crash +code, out, err = run("asset", "segments", "--workspace", WS, "--revision", + "r_doesnotexist", "--json") +data = as_json(out) +check("missing revision -> typed revision_not_found (exit 1)", + code == 1 and data and data["error"]["code"] == "revision_not_found") + +# --------------------------------------------------------------------------- +# asset add — single file rules +# --------------------------------------------------------------------------- +# a directory is rejected with guidance to use `openmind add` +code, out, err = run("asset", "add", "--workspace", WS, "--path", FIXTURE, "--json") +data = as_json(out) +check("asset add rejects a directory (exit 2)", code == 2) +check("asset add directory error is a typed invalid_request", + data and data["ok"] is False and data["error"]["code"] == "invalid_request") + +# a supported single file syncs — it must live UNDER the registered source +# root (the fixture), so create it there and remove it afterwards. +newfile = os.path.join(FIXTURE, "src", "_cli_added_tmp.ts") +with open(newfile, "w", encoding="utf-8") as fh: + fh.write("export const Added = 1;\n") +try: + code, out, err = run("asset", "add", "--workspace", WS, "--path", newfile, + "--wait", "--json") + data = as_json(out) + check("asset add of a supported file exits 0", code == 0, str(code) + " " + (err or "")[-200:]) + check("asset add reports the new asset", data and data.get("asset_id")) + check("asset add did not corrupt the glossary (filtered ingest)", + data is not None) +finally: + if os.path.exists(newfile): + os.remove(newfile) + +# --------------------------------------------------------------------------- +# status includes additive asset counts +# --------------------------------------------------------------------------- +code, out, err = run("status", "--workspace", WS, "--json") +st = as_json(out) +check("status --json exits 0", code == 0 and st is not None) +check("status carries additive asset counts", + st and "assets" in st and "assets_total" in st["assets"] + and "revisions" in st["assets"] and "evidence" in st["assets"]) +check("status still carries every prior key (backward compatible)", + st and all(k in st for k in ("counts", "paths", "template", "state", + "schema_version", "version"))) + +# --------------------------------------------------------------------------- +bad = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") +sys.exit(1 if bad else 0) diff --git a/tests/verify_asset_model.py b/tests/verify_asset_model.py new file mode 100644 index 0000000..cea7303 --- /dev/null +++ b/tests/verify_asset_model.py @@ -0,0 +1,335 @@ +"""Canonical Asset model, end to end through the real ingestion pipeline: +first ingest, idempotency (no re-embed), change, revert, removal, reappearance, +legacy backfill (no embed), FK cascade, and lifecycle (terminate/delete). + +Runs a real ingest in an isolated data dir with the offline hashing embedder. +The embedding function is wrapped with a call counter so the "unchanged content +is never re-embedded" invariant is asserted directly, not assumed. +""" +import os +import sys +import tempfile +import time + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 — forces an isolated data dir (never the live one) + +from openmind import (config, content_store as cs, db, embeddings, # noqa: E402 + javaparse, jobs, segmentation) +from openmind.runtime import get_runtime # noqa: E402 + +_results = [] + + +def check(desc, cond): + _results.append((desc, bool(cond))) + print(("PASS" if cond else "FAIL") + " - " + desc) + + +# --------------------------------------------------------------------------- +# Embedding call counter — proves "no re-embed" invariants directly. +# --------------------------------------------------------------------------- +_embed_orig = embeddings.embed +_embed_calls = {"n": 0} + + +def _embed_spy(docs, *a, **k): + _embed_calls["n"] += 1 + return _embed_orig(docs, *a, **k) + + +embeddings.embed = _embed_spy + + +def _reset_embed(): + _embed_calls["n"] = 0 + + +# --------------------------------------------------------------------------- +# A tiny fixture repo built on disk (deterministic content we control). +# --------------------------------------------------------------------------- +REPO = tempfile.mkdtemp(prefix="om_assets_repo_") + + +def _write(rel, text): + path = os.path.join(REPO, rel.replace("/", os.sep)) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + return path + + +JAVA = ( + "package org.example;\n" + "public class Client {\n" + " private int id;\n" + " public Client(int id) { this.id = id; }\n" + " public void send(Request r) {\n" + " transport.write(r);\n" + " }\n" + "}\n" +) +_write("src/main/java/org/example/Client.java", JAVA) +_write("src/app.py", "def main():\n return 42\n") +_write("config/application.yaml", "server:\n port: 8080\n") + +runtime = get_runtime() +runtime.ensure_worker() +ws = runtime.workspaces.create("assets-demo", path=REPO)["id"] +other = runtime.workspaces.create("other-ws")["id"] + + +def _ingest(): + res = runtime.ingest.start(ws, wait=True, timeout=180) + return res["job"]["progress"], res + + +# =========================================================================== +# 1. First ingest — every indexed file becomes an Asset with a full subtree +# =========================================================================== +_reset_embed() +p1, _ = _ingest() +files = db.get_file_index(ws) +assets = db.list_assets(ws, limit=100) +check("first ingest: every indexed file became an Asset", + len(assets) == len(files) and len(assets) == 3) +check("first ingest: assets_created counter matches", p1.get("assets_created") == 3) +check("first ingest: a revision was created per asset", p1.get("revisions_created") == 3) +check("first ingest: segments were created", p1.get("segments_created", 0) >= 3) +check("first ingest: evidence was created", p1.get("evidence_created", 0) >= 3) +check("first ingest: content blobs were created", p1.get("content_blobs_created") == 3) +check("first ingest: the embedder ran at least once", _embed_calls["n"] >= 1) + +# every active asset has a current revision, every revision >=1 segment, every +# segment has evidence with a valid 1-based range and a verbatim excerpt. +for a in assets: + check(f"asset {a['logical_key']}: has a current revision", + bool(a["current_revision_id"])) + check(f"asset {a['logical_key']}: source_path is workspace-relative", + not a["source_path"].startswith("/") and ":" not in a["source_path"][:3]) + segs = db.list_segments(ws, a["current_revision_id"]) + check(f"asset {a['logical_key']}: current revision has >=1 segment", len(segs) >= 1) + for s in segs: + ev = db.get_evidence_for_segment(ws, s["id"]) + loc = (ev or {}).get("locator") or {} + ok_range = (ev is not None and loc.get("startLine", 0) >= 1 + and loc.get("endLine", 0) >= loc.get("startLine", 0)) + check(f"segment {s['segment_key']}: has evidence with a valid 1-based range", + ok_range) + check(f"segment {s['segment_key']}: evidence locator file is relative", + not str(loc.get("file", "")).startswith("/")) + +# evidence excerpt is verbatim source recoverable from the blob +a_py = db.find_asset_by_logical_key(ws, "src/app.py") +rev_py = db.get_revision(ws, a_py["current_revision_id"]) +seg_py = db.list_segments(ws, rev_py["id"])[0] +ev_py = db.get_evidence_for_segment(ws, seg_py["id"]) +blob = cs.get(ws, rev_py["content_blob_hash"]).decode("utf-8") +recovered = segmentation.slice_lines(blob, ev_py["locator"]["startLine"], + ev_py["locator"]["endLine"]) +check("evidence content_hash is recoverable from the immutable blob", + segmentation.hash_text_utf8(recovered) == ev_py["content_hash"]) +check("evidence excerpt is a verbatim prefix of the recovered source", + recovered.startswith(ev_py["excerpt"]) or ev_py["excerpt"] in recovered) + +# Java segmentation (only when tree-sitter is available) +a_java = db.find_asset_by_logical_key(ws, "src/main/java/org/example/Client.java") +check("the Java file is classified as source-code", a_java["asset_type"] == "source-code") +if javaparse.available(): + jsegs = db.list_segments(ws, a_java["current_revision_id"]) + types = {s["segment_type"] for s in jsegs} + check("java: a 'type' segment exists (derived class summary)", "type" in types) + check("java: a 'method' segment exists", "method" in types) + check("java: a 'constructor' segment exists", "constructor" in types) + derived = [s for s in jsegs if s["content_mode"] == "derived"] + check("java: the class-summary segment is marked derived (not verbatim)", + len(derived) >= 1) +else: + check("java: tree-sitter unavailable — generic segments used (still valid)", + len(db.list_segments(ws, a_java["current_revision_id"])) >= 1) + +# consistency invariant (guards the commit-ordering fix): every file_index entry +# must have an in-sync active Asset whose current revision hashes to the file's +# actual content. file_index is written AFTER the Asset revision commits, so a +# file_index row can never point at a stale/absent Asset. +import hashlib # noqa: E402 + + +def _sha256_file(rel): + with open(os.path.join(REPO, rel.replace("/", os.sep)), "rb") as fh: + # blob is stored as the utf-8 re-encoding of the decoded text + from openmind import walker as _w + return hashlib.sha256(_w.read_text( + os.path.join(REPO, rel.replace("/", os.sep))).encode("utf-8", "replace")).hexdigest() + + +idx = db.list_asset_index(ws) +_consistent = True +for rel in db.get_file_index(ws): + ai = idx.get(rel) + if not (ai and ai["state"] == "active" and ai["content_hash"] == _sha256_file(rel)): + _consistent = False +check("every file_index entry has an in-sync active Asset (no stranded rows)", + _consistent) + +# =========================================================================== +# 2. Idempotency — unchanged re-ingest creates nothing and never re-embeds +# =========================================================================== +seg_count_before = db.asset_stats(ws)["segments"] +_reset_embed() +p2, _ = _ingest() +check("re-ingest: zero new revisions", p2.get("revisions_created") == 0) +check("re-ingest: every asset reused", p2.get("assets_reused") == 3) +check("re-ingest: revisions reused", p2.get("revisions_reused") == 3) +check("re-ingest: content blobs reused, none created", + p2.get("content_blobs_created", 0) == 0) +check("re-ingest: segment count is stable", + db.asset_stats(ws)["segments"] == seg_count_before) +check("re-ingest: the embedder was NOT called (nothing changed)", + _embed_calls["n"] == 0) + +# =========================================================================== +# 3. Change one file — exactly one new revision; others untouched; history kept +# =========================================================================== +time.sleep(1.0) # ensure a distinct mtime; content hash is what actually decides +_write("src/app.py", "def main():\n return 43 # changed\n") +first_rev_id = a_py["current_revision_id"] +first_blob = rev_py["content_blob_hash"] +_reset_embed() +p3, _ = _ingest() +check("change: exactly one new revision", p3.get("revisions_created") == 1) +a_py2 = db.find_asset_by_logical_key(ws, "src/app.py") +revs_py = db.list_revisions(ws, a_py2["id"]) +check("change: the changed asset now has 2 revisions", len(revs_py) == 2) +check("change: current revision advanced", a_py2["current_revision_id"] != first_rev_id) +old = db.get_revision(ws, first_rev_id) +check("change: the previous revision is still queryable", old is not None) +check("change: the previous revision is marked superseded", old["status"] == "superseded") +check("change: the new revision supersedes the previous current", + db.get_revision(ws, a_py2["current_revision_id"])["supersedes_revision_id"] == first_rev_id) +# unrelated assets keep their current revision +a_yaml = db.find_asset_by_logical_key(ws, "config/application.yaml") +check("change: an unrelated asset kept its current revision", + len(db.list_revisions(ws, a_yaml["id"])) == 1) +# historical evidence still returns the OLD content from the snapshot +old_seg = db.list_segments(ws, first_rev_id)[0] +old_ev = db.get_evidence_for_segment(ws, old_seg["id"]) +old_blob_text = cs.get(ws, first_blob).decode("utf-8") +check("change: the old revision's blob is retained", + "return 42" in old_blob_text and "changed" not in old_blob_text) +check("change: historical evidence still validates against the old blob", + segmentation.hash_text_utf8( + segmentation.slice_lines(old_blob_text, old_ev["locator"]["startLine"], + old_ev["locator"]["endLine"])) == old_ev["content_hash"]) + +# =========================================================================== +# 4. Revert A -> B -> A — three revisions, final reuses the original blob +# =========================================================================== +_write("src/app.py", "def main():\n return 42\n") # back to the ORIGINAL A +_reset_embed() +p4, _ = _ingest() +a_py3 = db.find_asset_by_logical_key(ws, "src/app.py") +revs3 = db.list_revisions(ws, a_py3["id"]) +check("revert: a third revision was created (revert is a new observation)", + len(revs3) == 3) +check("revert: revision sequences are dense 1..3", + sorted(r["sequence"] for r in revs3) == [1, 2, 3]) +check("revert: the final revision reuses the original content blob", + db.get_revision(ws, a_py3["current_revision_id"])["content_blob_hash"] == first_blob) +check("revert: no new blob was created (the original was reused)", + p4.get("content_blobs_created", 0) == 0 and p4.get("content_blobs_reused", 0) >= 1) + +# =========================================================================== +# 5. Removal — source file gone => Asset removed, history preserved +# =========================================================================== +os.remove(os.path.join(REPO, "config", "application.yaml")) +p5, _ = _ingest() +a_yaml2 = db.find_asset_by_logical_key(ws, "config/application.yaml") +check("removal: assets_removed counter incremented", p5.get("assets_removed") == 1) +check("removal: the asset state is 'removed'", a_yaml2["state"] == "removed") +check("removal: the file_index row is gone", + "config/application.yaml" not in db.get_file_index(ws)) +check("removal: the removed asset's revision history is preserved", + len(db.list_revisions(ws, a_yaml2["id"])) >= 1) +check("removal: an unrelated active asset is untouched", + db.find_asset_by_logical_key(ws, "src/app.py")["state"] == "active") + +# =========================================================================== +# 6. Reappearance — the same logical key comes back => same Asset reactivated +# =========================================================================== +removed_asset_id = a_yaml2["id"] +_write("config/application.yaml", "server:\n port: 8080\n") +p6, _ = _ingest() +a_yaml3 = db.find_asset_by_logical_key(ws, "config/application.yaml") +check("reappearance: the SAME asset id is reused", a_yaml3["id"] == removed_asset_id) +check("reappearance: the asset is active again", a_yaml3["state"] == "active") +check("reappearance: revision history remained intact", + len(db.list_revisions(ws, a_yaml3["id"])) >= 1) + +# =========================================================================== +# 7. Legacy backfill — file_index + Chroma exist, but no Asset rows. +# Re-ingest must create Assets WITHOUT re-embedding. +# =========================================================================== +db.clear_workspace_assets(ws) +cs.clear_workspace(ws) +check("backfill setup: asset rows wiped, file_index kept", + db.count_assets(ws) == 0 and len(db.get_file_index(ws)) == 3) +_reset_embed() +p7, _ = _ingest() +check("backfill: assets recreated for every unchanged file", + p7.get("assets_created") == 3) +check("backfill: revisions recreated", p7.get("revisions_created") == 3) +check("backfill: the embedder was NOT called (Chroma data reused)", + _embed_calls["n"] == 0) +check("backfill: asset stats restored", db.asset_stats(ws)["assets_total"] == 3) + +# =========================================================================== +# 8. FK cascade + cross-workspace isolation +# =========================================================================== +some_asset = db.list_assets(ws, limit=1)[0] +check("scoping: an asset id is not readable through another workspace", + db.get_asset(other, some_asset["id"]) is None) +# a raw project-row delete cascades through the whole Asset subtree +tmp = db.create_project("cascade-victim")["id"] +tmp_root = tempfile.mkdtemp() +_write_root = tmp_root +with open(os.path.join(tmp_root, "x.py"), "w") as fh: + fh.write("y = 1\n") +db.commit_revision(tmp, "x.py", asset_type="source-code", title="x.py", + source_path="x.py", content_hash="h" * 64, content_size=6, + content_blob_hash="h" * 64, + segments=[{"segment_key": "file-range:000001", "segment_type": "file", + "ordinal": 0, "start_line": 1, "end_line": 1, "symbol": "x.py", + "content_hash": "h" * 64, "content_mode": "verbatim", + "metadata": {}, "evidence": {"locator": {}, "excerpt": "y=1", + "content_hash": "h" * 64}}]) +check("cascade: victim workspace has asset data before delete", + db.count_assets(tmp) == 1) +db.delete_project(tmp) +conn = db._c() +check("cascade: assets deleted with the project", db.count_assets(tmp) == 0) +check("cascade: revisions/segments/evidence for the victim are gone (FK cascade)", + conn.execute("SELECT COUNT(*) FROM asset_revisions r JOIN " + "(SELECT 1) WHERE r.asset_id NOT IN (SELECT id FROM assets)" + ).fetchone()[0] == 0) + +# =========================================================================== +# 9. Lifecycle — terminate clears Asset data + blobs, keeps the workspace +# =========================================================================== +assert db.count_assets(ws) == 3 +jobs.terminate_project(ws) +check("terminate: Asset rows cleared", db.count_assets(ws) == 0) +check("terminate: content blobs cleared", + not cs.objects_dir(ws).exists() + or not any(p.is_file() for p in cs.objects_dir(ws).rglob("*"))) +check("terminate: the workspace itself survives", db.get_project(ws) is not None) +check("terminate: the registered source path survives", + len(db.get_project_paths(ws)) == 1) +check("terminate: the workspace is back to 'init'", db.get_project(ws)["state"] == "init") + +# --------------------------------------------------------------------------- +bad = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") +sys.exit(1 if bad else 0) diff --git a/tests/verify_content_store.py b/tests/verify_content_store.py new file mode 100644 index 0000000..371e1c1 --- /dev/null +++ b/tests/verify_content_store.py @@ -0,0 +1,126 @@ +"""Immutable content-addressed blob store — atomic write, blob reuse, +SHA-256 identity, corruption detection, binary round-trip, workspace cleanup. + +Pure stdlib + the content store: no embeddings, no vector store, no FastAPI, no +database. Proves the snapshot store is testable in isolation (invariant 9). +""" +import hashlib +import os +import sys +import tempfile + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 — forces an isolated data dir (never the live one) + +from openmind import config, content_store as cs # noqa: E402 +from openmind.domain.errors import ContentCorruption # noqa: E402 + +_results = [] + + +def check(desc, cond): + _results.append((desc, bool(cond))) + print(("PASS" if cond else "FAIL") + " - " + desc) + + +WS = "p_cstore00test" + +# --------------------------------------------------------------------------- +# 1. Write + SHA-256 identity +# --------------------------------------------------------------------------- +data = b"class Client { void send(Request r) {} }" +h = cs.put(WS, data) +check("put returns the SHA-256 hex of the exact bytes", + h == hashlib.sha256(data).hexdigest()) +check("the returned hash is 64 hex chars", len(h) == 64 and all(c in "0123456789abcdef" for c in h)) +check("hash_bytes agrees with put", cs.hash_bytes(data) == h) + +# --------------------------------------------------------------------------- +# 2. Round-trip + presence +# --------------------------------------------------------------------------- +check("get returns the exact bytes", cs.get(WS, h) == data) +check("exists is True for a stored blob", cs.exists(WS, h) is True) +check("verify is True for an intact blob", cs.verify(WS, h) is True) +check("exists is False for an unknown hash", cs.exists(WS, "0" * 64) is False) + +# --------------------------------------------------------------------------- +# 3. Blob reuse: identical content writes nothing new +# --------------------------------------------------------------------------- +blob_path = cs.objects_dir(WS) / h[:2] / h +mtime_before = blob_path.stat().st_mtime_ns +h2 = cs.put(WS, data) +check("re-putting identical content returns the same hash", h2 == h) +check("re-putting identical content does not rewrite the blob", + blob_path.stat().st_mtime_ns == mtime_before) +n_objects_1 = sum(1 for p in cs.objects_dir(WS).rglob("*") if p.is_file()) +cs.put(WS, data) +n_objects_2 = sum(1 for p in cs.objects_dir(WS).rglob("*") if p.is_file()) +check("re-putting identical content adds no new object file", n_objects_1 == n_objects_2) + +# --------------------------------------------------------------------------- +# 4. Changed content creates a new blob +# --------------------------------------------------------------------------- +data_b = b"class Client { void send(Request r) { log(); } }" +hb = cs.put(WS, data_b) +check("changed content produces a different hash", hb != h) +check("both blobs coexist (old one is retained)", + cs.exists(WS, h) and cs.exists(WS, hb)) +check("the new blob round-trips independently", cs.get(WS, hb) == data_b) + +# --------------------------------------------------------------------------- +# 5. Atomicity: no temp files linger after a write +# --------------------------------------------------------------------------- +leftovers = [p.name for p in cs.objects_dir(WS).rglob("*") + if p.is_file() and p.name.endswith(".tmp")] +check("no temp files remain after atomic writes", leftovers == []) + +# --------------------------------------------------------------------------- +# 6. Binary bytes round-trip (Phase 2 may store what it cannot parse) +# --------------------------------------------------------------------------- +binary = bytes(range(256)) * 4 +hbin = cs.put(WS, binary) +check("binary content is stored by its SHA-256", + hbin == hashlib.sha256(binary).hexdigest()) +check("binary content round-trips byte-for-byte", cs.get(WS, hbin) == binary) + +# --------------------------------------------------------------------------- +# 7. Corruption is detected explicitly, never returned silently +# --------------------------------------------------------------------------- +tampered_path = cs.objects_dir(WS) / hb[:2] / hb +tampered_path.write_bytes(b"TAMPERED") +check("verify is False for a corrupt blob", cs.verify(WS, hb) is False) +raised = False +try: + cs.get(WS, hb) +except ContentCorruption: + raised = True +check("get raises ContentCorruption on a hash mismatch", raised) + +missing_raised = False +try: + cs.get(WS, "a" * 64) +except ContentCorruption: + missing_raised = True +check("get raises ContentCorruption for a missing blob", missing_raised) + +# --------------------------------------------------------------------------- +# 8. No absolute path leaks into the identity (the DB stores only the hash) +# --------------------------------------------------------------------------- +check("the blob identity is a bare hash, not a path", + "/" not in h and "\\" not in h and os.sep not in h) +check("blobs live under the workspace data dir (removed with it on delete)", + str(cs.objects_dir(WS)).startswith(str(config.project_dir(WS)))) + +# --------------------------------------------------------------------------- +# 9. Workspace cleanup removes every blob +# --------------------------------------------------------------------------- +cs.clear_workspace(WS) +check("clear_workspace removes the objects tree", not cs.objects_dir(WS).exists()) +check("a cleared blob no longer exists", cs.exists(WS, h) is False) + +# --------------------------------------------------------------------------- +bad = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") +sys.exit(1 if bad else 0) diff --git a/tests/verify_migrations.py b/tests/verify_migrations.py index 5a81e37..0629222 100644 --- a/tests/verify_migrations.py +++ b/tests/verify_migrations.py @@ -57,14 +57,23 @@ def apply(conn): result = migrations.migrate(conn) tables = _tables(conn) -check("empty db: runner reports the head version", result.version == 2) -check("empty db: both migrations are applied", - result.applied == ["0001_baseline", "0002_paths_sidecar"]) +check("empty db: runner reports the head version", result.version == 3) +check("empty db: every migration is applied in order", + result.applied == ["0001_baseline", "0002_paths_sidecar", "0003_asset_model"]) check("empty db: nothing was reported as already applied", result.already_applied == []) check("empty db: not flagged as a legacy baseline", result.baselined_legacy is False) check("empty db: schema_migrations ledger exists", "schema_migrations" in tables) for t in ("projects", "jobs", "model_config", "file_index", "kv", "ask_history"): check(f"empty db: table '{t}' created", t in tables) +# v0003 canonical Asset model tables + indexes +for t in ("assets", "asset_revisions", "segments", "evidence"): + check(f"empty db: v0003 table '{t}' created", t in tables) +_idx = {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='index'").fetchall()} +for i in ("idx_assets_ws_state", "idx_assets_ws_type", "idx_revisions_asset_seq", + "idx_revisions_blob", "idx_segments_revision", "idx_segments_symbol", + "idx_segments_rev_type", "idx_evidence_revision", "idx_evidence_segment"): + check(f"empty db: v0003 index '{i}' created", i in _idx) check("empty db: ask_history index created", conn.execute("SELECT 1 FROM sqlite_master WHERE type='index' AND " "name='idx_ask_scope'").fetchone() is not None) @@ -77,8 +86,9 @@ def apply(conn): # --------------------------------------------------------------------------- again = migrations.migrate(conn) check("repeat run: applies nothing", again.applied == []) -check("repeat run: reports both as already applied", - again.already_applied == ["0001_baseline", "0002_paths_sidecar"]) +check("repeat run: reports every migration as already applied", + again.already_applied == ["0001_baseline", "0002_paths_sidecar", + "0003_asset_model"]) check("repeat run: version is unchanged", again.version == result.version) # --------------------------------------------------------------------------- @@ -128,9 +138,13 @@ def apply(conn): legacy_result = migrations.migrate(legacy) check("legacy db: reported as a legacy baseline", legacy_result.baselined_legacy is True) -check("legacy db: brought to head version", legacy_result.version == 2) +check("legacy db: brought to head version", legacy_result.version == 3) check("legacy db: baseline recorded, not skipped", "0001_baseline" in legacy_result.applied) +check("legacy db: v0003 applied on top of the baseline", + "0003_asset_model" in legacy_result.applied) +check("legacy db: v0003 asset tables created on the legacy database", + {"assets", "asset_revisions", "segments", "evidence"} <= _tables(legacy)) check("legacy db: project row survived", legacy.execute("SELECT COUNT(*) FROM projects").fetchone()[0] == 1) check("legacy db: project meta survived intact",