diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9f7fb5..ac26d49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,6 +180,52 @@ jobs: - name: Semantic adapter tests run: python scripts/run_acceptance.py --only verify_semantic_adapters + # -- canonical knowledge graph (v2 Phase 5) ------------------------ + - name: Knowledge migration tests + run: python scripts/run_acceptance.py --only verify_knowledge_migration + + - name: Knowledge entity tests + run: python scripts/run_acceptance.py --only verify_knowledge_entities + + - name: Knowledge claim tests + run: python scripts/run_acceptance.py --only verify_knowledge_claims + + - name: Knowledge relation tests + run: python scripts/run_acceptance.py --only verify_knowledge_relations + + - name: Knowledge revision-ledger tests + run: python scripts/run_acceptance.py --only verify_knowledge_revisions + + - name: Knowledge decision-audit tests + run: python scripts/run_acceptance.py --only verify_knowledge_decisions + + - name: Candidate promotion tests + run: python scripts/run_acceptance.py --only verify_knowledge_promotion + + - name: Deterministic projection tests + run: python scripts/run_acceptance.py --only verify_knowledge_projection + + - name: Graph staleness tests + run: python scripts/run_acceptance.py --only verify_knowledge_staleness + + - name: Entity merge/split tests + run: python scripts/run_acceptance.py --only verify_knowledge_merge_split + + - name: Graph search tests + run: python scripts/run_acceptance.py --only verify_knowledge_search + + - name: Graph traversal tests + run: python scripts/run_acceptance.py --only verify_knowledge_graph + + - name: Knowledge Bundle 2.0 Draft tests (exporter + verifier) + run: python scripts/run_acceptance.py --only verify_knowledge_bundle + + - name: Knowledge CLI tests + run: python scripts/run_acceptance.py --only verify_knowledge_cli + + - name: Knowledge adapter tests (REST + 9 MCP tools + 35-tool gate) + run: python scripts/run_acceptance.py --only verify_knowledge_adapters + - name: No provider credential reaches CI run: | python - <<'PY' @@ -260,9 +306,9 @@ jobs: from openmind.runtime import get_runtime # The nine core tools are a STABLE external contract; Phase 2 adds - # read-only Asset tools, Phase 3 read-only document tools and - # Phase 4 read-only semantic/lens tools ALONGSIDE them, never in - # place of one. 9 + 4 + 6 + 7 = 26, exactly. + # read-only Asset tools, Phase 3 read-only document tools, Phase 4 + # read-only semantic/lens tools and Phase 5 read-only graph tools + # ALONGSIDE them, never in place of one. 9+4+6+7+9 = 35, exactly. core = {"search", "route", "dispatch", "get_glossary", "find_similar_cases", "save_case", "get_doc", "propose_fix", "apply_fix"} @@ -274,23 +320,31 @@ jobs: "list_semantic_candidates", "get_semantic_candidate", "list_project_lenses", "get_project_lens", "get_semantic_usage"} + knowledge = {"get_graph_stats", "search_graph", "get_graph_node", + "expand_graph", "find_graph_path", + "list_engineering_entities", "get_engineering_entity", + "get_engineering_claim", "get_engineering_relation"} server = mcp_server.create_mcp_server(get_runtime()) names = {t.name for t in asyncio.run(server.list_tools())} assert core <= names, f"core MCP tool missing: {core - names}" assert asset <= names, f"asset MCP tool missing: {asset - names}" assert document <= names, f"document MCP tool missing: {document - names}" assert semantic <= names, f"semantic MCP tool missing: {semantic - names}" - expected = core | asset | document | semantic + assert knowledge <= names, f"graph MCP tool missing: {knowledge - names}" + expected = core | asset | document | semantic | knowledge assert names == expected, f"unexpected MCP tools: {names ^ expected}" - assert len(names) == 26, f"expected 26 tools, got {len(names)}" - # No write/import tool: Phase 3 adds no document writer and Phase 4 - # adds nothing that configures providers, changes policy, starts a - # paid analysis, reviews a candidate or activates a lens. + assert len(names) == 35, f"expected 35 tools, got {len(names)}" + # No write/import tool: Phase 3 adds no document writer, Phase 4 + # nothing that configures providers or triggers paid analysis, and + # Phase 5 nothing that promotes, creates, merges, changes + # authority, seeds/syncs the graph or exports bundles. assert not {n for n in names if n.startswith(("add_", "import_", "create_", "delete_", "update_", "set_", "start_", "approve_", "activate_", - "review_", "configure_"))}, names + "review_", "configure_", "promote_", + "merge_", "split_", "seed_", "sync_", + "export_", "withdraw_"))}, names assert server.name == "open-mind", server.name print("MCP smoke OK:", len(names), "tools") PY @@ -386,7 +440,7 @@ jobs: - name: Artifact export contract run: python tests/verify_artifacts.py - - name: Migration to v0005 + content-store byte round-trip + - name: Migration to v0006 + content-store byte round-trip shell: bash run: | python - <<'PY' @@ -395,7 +449,7 @@ jobs: os.environ["OPENMIND_MACHINE_DIR"] = tempfile.mkdtemp() from openmind import db, content_store as cs db.init_db() - assert db.migration_status()["version"] == 5, db.migration_status() + assert db.migration_status()["version"] == 6, db.migration_status() conn = db._c() tables = {r[0] for r in conn.execute( "SELECT name FROM sqlite_master WHERE type='table'")} @@ -403,6 +457,11 @@ jobs: assert {"workspace_semantic_policies", "semantic_analysis_runs", "semantic_candidates", "semantic_cache", "semantic_usage", "project_lenses"} <= tables, tables + assert {"engineering_entities", "engineering_claims", + "engineering_relations", "engineering_entity_aliases", + "engineering_entity_bindings", "knowledge_decisions", + "knowledge_revisions", "knowledge_promotions", + "knowledge_projection_state"} <= tables, tables assert "content_blob_hash" in { r[1] for r in conn.execute("PRAGMA table_info(segments)")} assert "payload_json" in { @@ -510,6 +569,109 @@ jobs: "hit, staleness, lenses") PY + # -- knowledge graph smoke (v2 Phase 5) ----------------------------- + # The whole promotion chain on every OS: deterministic seed, a mock + # candidate confirmed and EXPLICITLY promoted, entity/claim/relation + # lookup, a bounded graph path, and a Bundle 2.0 Draft export that + # the standalone verifier accepts. Zero network, zero credentials. + - name: Knowledge graph smoke (seed, promote, path, bundle) + shell: bash + run: | + export OPENMIND_DATA_DIR="$(mktemp -d)" + export OPENMIND_MACHINE_DIR="$(mktemp -d)" + python - <<'PY' + import pathlib, subprocess, sys, tempfile + from openmind.runtime import get_runtime + from openmind.semantic.models import ProviderProfile + from openmind.semantic.providers import profiles + + rt = get_runtime() + pid = rt.workspaces.create("ci-knowledge")["id"] + src = pathlib.Path(tempfile.mkdtemp()) + doc = src / "requirements.md" + doc.write_text("# Reqs\n\nREQ-CI-001: The job shall finish.\n", + encoding="utf-8") + assert rt.documents.add_document( + pid, str(doc), wait=True, timeout=300)["status"] == "new_asset" + + # deterministic seed: model-free, one knowledge revision + seed = rt.knowledge.seed(pid, actor="ci") + assert seed["changed"] and seed["knowledge_revision"] == 1, seed + assert rt.knowledge.sync(pid)["action"] == "noop" + + # a mock candidate, confirmed, then EXPLICITLY promoted + from openmind import db + from openmind.semantic.context import resolve_evidence_text + evidence_id = None + for a in db.list_assets(pid, limit=50): + for s in db.list_segments(pid, a["current_revision_id"], + limit=50): + ev = db.get_evidence_for_segment(pid, s["id"]) + if ev and "shall finish" in ( + resolve_evidence_text(pid, ev["id"]) or ""): + evidence_id = ev["id"] + assert evidence_id + profiles.upsert_profile(ProviderProfile( + name="ci-mock", kind="mock", metadata={"responses": { + "requirement-extraction": {"candidates": [{ + "candidateType": "requirement", + "stableKey": "REQ-CI-001", "title": "Finish", + "statement": "The job shall finish.", + "attributes": {}, + "evidence": [{"evidenceId": evidence_id, + "quote": "The job shall finish."}], + "confidenceHint": "medium", "reason": "normative"}]}}})) + rt.semantic.set_policy(pid, provider_profile="ci-mock") + rt.semantic.start_analysis( + pid, task_types=["requirement-extraction"], wait=True, + timeout=300) + cid = rt.semantic.list_candidates( + pid, candidate_type="requirement")["candidates"][0]["id"] + # confirming is review metadata only — promotion is separate + rt.semantic.review_candidate(pid, cid, decision="confirm", + reviewer="ci") + from openmind.knowledge import store as kg + assert kg.count_entities( + pid, entity_type="requirement") == 0, "review must not promote" + promoted = rt.knowledge.promote_candidate( + pid, cid, actor="ci", note="ci promotion") + assert promoted["status"] == "promoted", promoted + entity_id = promoted["entity"]["id"] + claim_id = promoted["claim"]["id"] + + # lookups + a manual relation + a bounded path + assert rt.knowledge.get_entity(pid, entity_id)["canonical_key"] \ + == "requirement:REQ-CI-001" + assert rt.knowledge.get_claim(pid, claim_id)["evidence"] + doc_entities = [e for e in rt.knowledge.list_entities( + pid, entity_type="document")["entities"]] + assert doc_entities, "seed should have projected the document" + relation = rt.knowledge.create_relation( + pid, source_entity_id=entity_id, + target_entity_id=doc_entities[0]["id"], + relation_type="derived-from", relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], + actor="ci", note="ci relation")["relation"] + assert rt.knowledge.get_relation( + pid, relation["id"])["relation_state"] == "confirmed" + path = rt.knowledge.find_path(pid, entity_id, + doc_entities[0]["id"]) + assert path["outcome"] == "found", path + + # bundle export + standalone verification + out = pathlib.Path(tempfile.mkdtemp()) / "bundle" + from openmind.knowledge.bundle import export_bundle + manifest = export_bundle(pid, str(out), current_only=True) + assert manifest["bundleSchemaVersion"] == "2.0.0-draft.1" + verdict = subprocess.run( + [sys.executable, "-m", "openmind.bundle_verify", str(out)], + capture_output=True, text=True) + assert verdict.returncode == 0, verdict.stdout + verdict.stderr + print("knowledge smoke OK: seed -> promote -> path -> bundle " + "verified at revision", + rt.knowledge.get_current_revision(pid)["knowledge_revision"]) + PY + - name: CLI asset help + fixture ingest + asset list + evidence read shell: bash run: | diff --git a/README.md b/README.md index eb6fdce..e9c41ad 100644 --- a/README.md +++ b/README.md @@ -272,11 +272,84 @@ python -m openmind.cli semantic review \ Deliberately **not** in this phase: canonical Entity/Claim/Relation tables, Knowledge-Graph edges, automatic candidate promotion, requirement-to-code traceability, conflict *resolution*, and OCR. Candidate promotion and the -Knowledge Graph are Phase 5. Full design: +Knowledge Graph shipped in Phase 5 (below). Full design: [docs/v2/phase-4-semantic-plane.md](docs/v2/phase-4-semantic-plane.md). --- +## Canonical Knowledge Graph (v2 Phase 5) + +Phase 5 adds the first **canonical** engineering knowledge layer: durable, +versioned, evidence-bound Entities, Claims and Relations in SQLite (no graph +database), entered through exactly four write paths — deterministic +projection, explicit manual creation, explicit Candidate promotion, explicit +Relation-Candidate promotion — and no other. + +```bash +# deterministic, model-free graph seeding from Assets and Segments +python -m openmind.cli graph seed --workspace p_... --json + +# review is NOT promotion: confirming updates candidate metadata only +python -m openmind.cli semantic review --workspace p_... --candidate sc_... \ + --decision confirm --json + +# the explicit bridge into the canonical graph (plan first, then promote) +python -m openmind.cli promotion plan --workspace p_... --candidate sc_... --json +python -m openmind.cli promotion promote --workspace p_... --candidate sc_... \ + --actor reviewer-name --note "Approved for canonical knowledge." --json + +# query it +python -m openmind.cli graph search --workspace p_... --query REQ-NC-017 --json +python -m openmind.cli graph path --workspace p_... --from ent_... --to ent_... --json + +# export it (a SEPARATE draft contract; .openmind 1.1.0 is untouched) +python -m openmind.cli bundle export --workspace p_... --output ./.openmind-v2 \ + --current-only --json +python -m openmind.bundle_verify ./.openmind-v2 +``` + +- **Nothing is promoted automatically.** A confirmed candidate stays a + candidate until `promotion promote` — which re-checks review status, + staleness and evidence verification *inside* the transaction, is + idempotent, and records a promotion row, an immutable Human Decision and + exactly one Knowledge Revision. Stale or rejected candidates cannot be + promoted (there is no `--force`), and conflict candidates cannot be + promoted at all (conflict *resolution* is Phase 6). +- **Deterministic projection is model-free.** Source assets become + `code-component`s, Java types/methods `code-symbol`s, OpenAPI operations + `interface`s, SQL objects `database-object`s; containment is `explicit`, + name-based call edges stay `inferred` with ambiguity preserved. A generic + document never becomes a Requirement, a test-source method never becomes a + Test Case, and an unchanged re-sync writes nothing and mints no revision. +- **Every active Claim carries verified Evidence** (quotes must match the + immutable snapshot — fabrications are rejected locally), every Relation + carries provenance, `possibly-related` is never presented as `implements`, + and authority (`authoritative` / `non-authoritative` / …) changes only by + explicit human decision — never inferred. +- **History is governed, not overwritten.** Corrections supersede, withdrawal + excludes without deleting, merge keeps the source addressable (resolving to + its target), split moves exactly what the caller lists, and staleness + reconciliation marks — never erases — knowledge whose source Revisions + moved on. Every graph transaction is one monotonic, per-workspace + **Knowledge Revision**; every governance write is one immutable + **Human Decision** with a caller-supplied actor. +- **Graph search is exact-first.** A separate `knowledge_` vector + collection indexes active Entities and Claims (code/document collections + untouched); retrieval fuses exact canonical key > exact alias > exact + identifier token > lexical > vector similarity, so an exact identifier is + never outranked by something that merely reads similar. Expansion and path + discovery are bounded, deterministic, and honest about truncation — and the + generic `graph path` command is deliberately *not* labelled Requirement + Traceability (that formal engine is Phase 6). + +Deliberately **not** in this phase: automatic promotion, conflict resolution, +formal Requirement-to-Code traceability and coverage/gap reports, +change-impact analysis, Neo4j or any external graph service, Cypher/Gremlin, +a Graph UI, Bundle 2.0 freeze/import, and OCR. Full design: +[docs/v2/phase-5-knowledge-graph.md](docs/v2/phase-5-knowledge-graph.md). + +--- + ## Built For AI Agent Workflows Open Mind is designed as infrastructure for practical AI agents and tool @@ -592,15 +665,28 @@ capabilities are added *beside* them, never in place of one: | `list_project_lenses` | Stored lenses + built-in Template projections + organization files. | | `get_project_lens` | One lens with its deterministic validation report. | | `get_semantic_usage` | A run's provider-usage ledger (`null` cost when no price is known). | - -That is 9 core + 4 asset + 6 document + 7 semantic/lens = **26 tools**, every -addition read-only. There is deliberately **no document-write MCP tool** -(importing reads a local file, and exposing that over MCP would let a client -make the server read a path it chose) and **no semantic-write MCP tool** — -nothing on MCP configures a provider, changes a workspace's egress policy, -triggers a paid analysis, reviews a candidate or activates a lens. Claude Code -drives `openmind document add` / `semantic analyze` / `lens approve` through -its shell instead, where the command (and any cloud use) is visible. +| `get_graph_stats` | Canonical-graph statistics + the current Knowledge Revision. | +| `search_graph` | Exact-first graph search over Entities and Claims (separate sections). | +| `get_graph_node` | One node in the stable read shape (entity/claim/asset/revision/segment/evidence). | +| `expand_graph` | Bounded, deterministic BFS expansion with an honest `truncated` flag. | +| `find_graph_path` | Bounded shortest paths with evidence summaries — found / no-path / truncated. | +| `list_engineering_entities` | Canonical Entities (bounded; active by default). | +| `get_engineering_entity` | One Entity with aliases, bindings, claims and relations. | +| `get_engineering_claim` | One Claim with its verified evidence joins. | +| `get_engineering_relation` | One Relation with state, provenance and evidence. | + +That is 9 core + 4 asset + 6 document + 7 semantic/lens + 9 graph = +**35 tools**, every addition read-only. There is deliberately **no +document-write MCP tool** (importing reads a local file, and exposing that +over MCP would let a client make the server read a path it chose), **no +semantic-write MCP tool** — nothing on MCP configures a provider, changes a +workspace's egress policy, triggers a paid analysis, reviews a candidate or +activates a lens — and **no graph-write MCP tool**: nothing on MCP promotes +a candidate, creates an Entity/Claim/Relation, merges, splits, changes +authority, seeds/syncs the graph or exports a bundle. Claude Code drives +`openmind document add` / `semantic analyze` / `promotion promote` / +`entity merge` through its shell instead, where the command (and any cloud +use) is visible. --- @@ -788,8 +874,15 @@ openmind/ candidate association. Imports no parser dependency itself. document_rag.py document retrieval (vector + exact-token + RRF) and the combined code/document knowledge search + knowledge/ the canonical Engineering Knowledge Graph (v2 Phase 5): + closed vocabularies, graph store + transactional Knowledge + Revisions, evidence verifier, promotion, deterministic + projector, staleness reconciliation, bounded traversal, + exact-first search, vector projection, Bundle 2.0 Draft + bundle_verify.py standalone stdlib-only Knowledge Bundle verifier migrations/ versioned, checksummed SQLite schema migrations - (v0003 = Asset model, v0004 = document ingestion) + (v0003 = Asset model, v0004 = document ingestion, + v0005 = semantic plane, v0006 = knowledge graph) walker.py selection-aware walk, .gitignore handling, hashing detect.py manifest/language detection and stack cues langspec.py declarative language registry @@ -948,7 +1041,7 @@ against the neutral fixture repos in `fixtures/`. The following are not claimed as complete in the current build: -**v2 enterprise knowledge layer.** Four phases have shipped: +**v2 enterprise knowledge layer.** Five phases have shipped: Phase 1 is the tool-first runtime ([docs/v2/phase-1-core-foundation.md](docs/v2/phase-1-core-foundation.md)), Phase 2 is the canonical **Asset / Revision / Segment / Evidence** model plus the @@ -956,26 +1049,32 @@ immutable content store ([docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md)), Phase 3 is the deterministic **document-ingestion** plane ([docs/v2/phase-3-document-ingestion.md](docs/v2/phase-3-document-ingestion.md)), -and Phase 4 is the policy-governed **semantic plane** — evidence-bound +Phase 4 is the policy-governed **semantic plane** — evidence-bound candidate extraction over local or cloud providers, plus Adaptive Project -Lenses ([docs/v2/phase-4-semantic-plane.md](docs/v2/phase-4-semantic-plane.md)). +Lenses ([docs/v2/phase-4-semantic-plane.md](docs/v2/phase-4-semantic-plane.md)), +and Phase 5 is the canonical **Engineering Knowledge Graph** — deterministic +projection, explicit candidate promotion, Knowledge Revisions, Human +Decisions, graph search/traversal and the Knowledge Bundle 2.0 **Draft** +([docs/v2/phase-5-knowledge-graph.md](docs/v2/phase-5-knowledge-graph.md)). The following later-phase items are **not** implemented; the foundation creates extension points for them rather than building them: -- canonical Entity, Claim and Relation tables — every Phase 4 extraction stays - a **candidate** until the Phase 5 promotion workflow exists; -- the engineering Knowledge Graph and requirement-to-code traceability; -- conflict *resolution* (Phase 4 surfaces conflict **candidates** only) and - branch/PR overlays; +- formal Requirement-to-Code **traceability**, coverage/gap reports and the + conflict-resolution engine (Phase 6 — the generic `graph path` command is + deliberately not labelled traceability, and Phase 4 conflict **candidates** + stay candidates); +- change-impact analysis, branch/PR overlays, webhooks and graph-based CI + policy gates; - **OCR** — image-only PDFs are *detected* and marked `needs-ocr`, never read; - COBOL, JCL, PPTX and email-archive parsing; Jira and Confluence connectors; - cloud **embeddings** and native provider batch APIs (semantic *reasoning* may use a cloud provider when a workspace opts in; embeddings and ordinary ingestion remain fully local); - historical (non-current-revision) document search; -- webhook integration and a Bundle 2.0 artifact schema (export stays at - schema 1.x); -- a typed worker pool or job DAG replacing the current single-worker engine. +- the Bundle 2.0 schema **freeze** and Bundle import (the Draft exporter and + verifier shipped in Phase 5; `.openmind` export stays at schema 1.x); +- a typed worker pool or job DAG replacing the current single-worker engine; +- new Agent Skills and Skill Forge / Verification integration (Phase 8). **Other work not yet done:** diff --git a/docs/cli.md b/docs/cli.md index 8bc6a28..8a04439 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -482,6 +482,126 @@ regexes, no URLs, capped sizes), then deterministic whole-corpus validation explicit `activate`. Invalid organization lens files stay listed with their errors. +### `graph` — canonical Knowledge Graph queries and projection (v2 Phase 5) + +```bash +python -m openmind.cli graph stats --workspace p_... --json +python -m openmind.cli graph seed plan --workspace p_... --json # dry run +python -m openmind.cli graph seed --workspace p_... --json +python -m openmind.cli graph sync --workspace p_... --json +python -m openmind.cli graph reconcile --workspace p_... --json +python -m openmind.cli graph search --workspace p_... --query REQ-NC-017 --json +python -m openmind.cli graph node --workspace p_... --node ent_... --json +python -m openmind.cli graph expand --workspace p_... --node ent_... \ + --depth 2 --direction both --json +python -m openmind.cli graph path --workspace p_... \ + --from ent_... --to ent_... --max-depth 6 --json +``` + +`seed`/`sync` are the deterministic, **model-free** projector: active Assets +and Segments become code-component / code-symbol / interface / data-model / +database-object / configuration / document / build-definition Entities, with +`contains` (explicit) and `calls` (inferred, ambiguity preserved) Relations. +An unchanged source is a no-op that mints **no** Knowledge Revision. `search` +fuses exact canonical key > exact alias > exact identifier token > lexical > +vector similarity — an exact identifier is never outranked by similar text. +`expand` and `path` are bounded, deterministic traversals with honest +`truncated` / `no-path` outcomes; `path` is generic reachability, NOT formal +Requirement Traceability (Phase 6). + +### `promotion` — explicit candidate promotion (v2 Phase 5) + +```bash +python -m openmind.cli promotion plan --workspace p_... --candidate sc_... --json +python -m openmind.cli promotion promote --workspace p_... --candidate sc_... \ + --actor reviewer-name --note "Approved for canonical knowledge." --json +python -m openmind.cli promotion relation-plan --workspace p_... --relation sr_... --json +python -m openmind.cli promotion promote-relation --workspace p_... --relation sr_... \ + --actor reviewer-name --note "Endpoints and evidence verified." --json +``` + +Review and promotion are separate acts: `semantic review --decision confirm` +updates candidate metadata only; **this** command is the sole bridge into the +canonical graph. Eligibility (all re-checked inside the transaction): +`review_status=confirmed`, `lifecycle_status=active`, +`evidence_status=verified`, source Revisions current, endpoints resolving +unambiguously (relations), not already promoted. There is deliberately no +`--accept-stale` / `--ignore-evidence` / `--force-unverified`. `plan` is a +deterministic dry-run that writes nothing; promotion is transactional and +idempotent (re-promoting returns the same target, minting nothing); +conflict candidates cannot be promoted at all. Every promotion records a +promotion row, a Human Decision and exactly one Knowledge Revision. + +### `entity` / `claim` / `relation` — graph governance (v2 Phase 5) + +```bash +python -m openmind.cli entity list --workspace p_... --type requirement --json +python -m openmind.cli entity show --workspace p_... --entity ent_... --json +python -m openmind.cli entity create --workspace p_... --type requirement \ + --key requirement:REQ-NC-017 --name REQ-NC-017 \ + --evidence e_... --actor reviewer --note "manual entity" --json +python -m openmind.cli entity alias-add --workspace p_... --entity ent_... \ + --alias NC-17 --type acronym --actor reviewer --note "known acronym" --json +python -m openmind.cli entity merge --workspace p_... --source ent_dup --target ent_main \ + --actor reviewer --note "duplicates" --json +python -m openmind.cli entity split --workspace p_... --source ent_... \ + --new-type workflow --new-key workflow:derived:step-two \ + --claim clm_... --binding bd_... --actor reviewer --note "split" --json +python -m openmind.cli entity authority --workspace p_... --entity ent_... \ + --status authoritative --actor lead --note "board approval" --json +python -m openmind.cli claim create --workspace p_... --entity ent_... \ + --type normative-statement --statement "..." --evidence e_... \ + --actor reviewer --note "manual claim" --json +python -m openmind.cli relation create --workspace p_... --source ent_a --target ent_b \ + --type refines --state confirmed --evidence e_... \ + --actor reviewer --note "manual relation" --json +python -m openmind.cli relation reject --workspace p_... --relation rel_... \ + --actor reviewer --note "not real" --json +python -m openmind.cli relation restore --workspace p_... --relation rel_... \ + --actor reviewer --note "was real" --json +``` + +Every write records a Human Decision with the caller-supplied `--actor` +(never inferred) and bounded `--note`, plus one Knowledge Revision. Manual +Entities/Claims/Relations REQUIRE at least one valid `--evidence` id (quotes, +when given, must match the immutable snapshot; fabrications are rejected). +Manual relations may be `explicit` or `confirmed` only — never `inferred`. +Alias collisions across entities are reported, never silently attached. +`supersede`/`withdraw` subcommands exist for all three object kinds; nothing +is ever deleted — superseded, withdrawn, merged and rejected records stay +queryable. + +### `knowledge revisions` / `revision` / `decisions` — graph history (v2 Phase 5) + +```bash +python -m openmind.cli knowledge revisions --workspace p_... --json +python -m openmind.cli knowledge revision --workspace p_... --number 3 --json +python -m openmind.cli knowledge decisions --workspace p_... --json +``` + +Added beside the Phase 3 `knowledge search` (which is unchanged). The ledger +is per-workspace, monotonic and immutable; one graph transaction = one +revision, and a failed transaction leaves none. + +### `bundle export` — Knowledge Bundle 2.0 Draft (v2 Phase 5) + +```bash +python -m openmind.cli bundle export --workspace p_... --output ./.openmind-v2 \ + --current-only --json +python -m openmind.cli bundle export --workspace p_... --output ./.openmind-v2 \ + --include-history --json +python -m openmind.bundle_verify ./.openmind-v2 +``` + +A SEPARATE contract from the frozen `.openmind` 1.1.0 artifact: schema +`2.0.0-draft.1`, its own directory of deterministic JSONL files + JSON +schemas + a manifest with per-file SHA-256 hashes and record counts. No +secrets, no provider profiles, no prompts, no raw model output, no absolute +paths. `--knowledge-revision N` filters records by their creation revision +stamp (documented as such — it does not reconstruct point-in-time lifecycle +states). `python -m openmind.bundle_verify` is a standalone stdlib-only +verifier any consumer can run. + ### `export` ```bash diff --git a/docs/database-migrations.md b/docs/database-migrations.md index b4f01d9..87a15da 100644 --- a/docs/database-migrations.md +++ b/docs/database-migrations.md @@ -78,6 +78,7 @@ runner switches the connection to explicit-transaction mode | 3 | `asset_model` | the canonical Asset model: `assets`, `asset_revisions`, `segments`, `evidence` + their indexes. Additive — creates only new tables (all `IF NOT EXISTS`), touches no existing row | | 4 | `document_ingestion` | document ingestion: `segments.content_blob_hash`, `jobs.payload_json`, `document_parses`, `document_index` + their indexes. Additive — two columns with defaults and two new tables | | 5 | `semantic_plane` | the Phase 4 semantic plane: `workspace_semantic_policies`, `semantic_analysis_runs`, `semantic_analysis_targets`, `semantic_candidates` (+ evidence join), `semantic_relation_candidates` (+ evidence join), `semantic_conflict_candidates` (+ evidence join), `semantic_usage`, `semantic_cache`, `project_lenses` + their indexes. Additive — twelve new tables, no existing row touched | +| 6 | `knowledge_graph` | the Phase 5 canonical Engineering Knowledge Graph: `engineering_entities`, `engineering_entity_aliases`, `engineering_entity_bindings`, `engineering_claims` (+ evidence join), `engineering_relations` (+ evidence join), `knowledge_decisions`, `knowledge_revisions`, `knowledge_promotions`, `knowledge_projection_state` + their indexes. Additive — eleven new tables, no existing row touched | `v0003` adds the OpenMind v2 canonical content-identity model. `assets` references `projects(id)` and the whole subtree cascades on `ON DELETE CASCADE`, @@ -127,14 +128,26 @@ deliberately NOT here — profiles live in the machine-local sidecar (`~/.openmind/providers.json`) and key VALUES live only in environment variables. See [docs/v2/phase-4-semantic-plane.md](v2/phase-4-semantic-plane.md). +`v0006` adds the Phase 5 canonical Engineering Knowledge Graph: Entities +(logical identity `UNIQUE(workspace, entity_type, canonical_key)`), their +aliases and source bindings, Claims and Relations with their evidence-quote +joins, the immutable Human Decision audit, the per-workspace monotonic +Knowledge Revision ledger (`UNIQUE(workspace_id, revision_number)` is the +concurrency backstop), promotion records and the deterministic projection +watermark. Graph-internal foreign keys cascade so a workspace wipe stays +clean; there is deliberately NO foreign key to `semantic_candidates` — a +promoted candidate's id is stored as provenance, never as an ownership edge, +so canonical history cannot be cascade-deleted through the Phase 4 tables. +See [docs/v2/phase-5-knowledge-graph.md](v2/phase-5-knowledge-graph.md). + ### Upgrading an existing database Nothing to do — open OpenMind and it migrates itself. Concretely: ```text -empty database -> v0001..v0005 create every table -> ledger records 1..5 -legacy database -> v0001 statements are all no-ops, -> ledger records 1..5 - v0002..v0005 apply additively (existing data untouched) +empty database -> v0001..v0006 create every table -> ledger records 1..6 +legacy database -> v0001 statements are all no-ops, -> ledger records 1..6 + v0002..v0006 apply additively (existing data untouched) current database -> nothing to apply -> no writes ``` @@ -154,6 +167,16 @@ new empty table, so all of the above PLUS document parse records, document indexes and glossary/structure maps survive byte-for-byte. Semantic policy rows appear only when a workspace explicitly sets a policy. +A Phase 1–4 database upgrades to v0006 identically: every `v0006` change is a +new empty table, so projects, jobs, Assets, Revisions, Segments, Evidence, +content blobs, document parse records, document indexes, semantic policies, +runs, targets, candidates (all three kinds), usage records, the semantic +cache, Project Lenses, template metadata, Ask history, cases and maps all +survive byte-for-byte (`tests/verify_knowledge_migration.py` builds a real +v0005 database, seeds representative rows and proves it). Graph rows appear +only through deterministic projection, explicit manual creation or explicit +candidate promotion — never from the migration itself. + A legacy database is **baselined, not recreated**. `v0001` is written entirely with `CREATE TABLE IF NOT EXISTS`, so against a database that already has those tables every statement is a no-op and the runner simply records version 1. diff --git a/docs/v2/phase-5-knowledge-graph.md b/docs/v2/phase-5-knowledge-graph.md new file mode 100644 index 0000000..15c8c59 --- /dev/null +++ b/docs/v2/phase-5-knowledge-graph.md @@ -0,0 +1,475 @@ +# OpenMind v2 — Phase 5: Canonical Engineering Knowledge Graph, Candidate Promotion and Knowledge Bundle 2.0 Draft + +Status: implemented in this phase. Runtime version `1.5.0-dev`; artifact schema +stays `1.1.0`; database schema head moves from v0005 to v0006; Knowledge Bundle +uses its own separate draft schema `2.0.0-draft.1`. + +Phase 5 adds the first **canonical** engineering knowledge layer: durable, +versioned, evidence-bound Entities, Claims and Relations that enter the graph +only through an explicit write — deterministic projection, explicit manual +creation, or the explicit promotion of a human-confirmed semantic Candidate. +Nothing a model proposed becomes canonical on its own, and nothing canonical +loses its provenance. + +--- + +## 1. Candidate versus canonical knowledge + +Phase 4 ends at the Candidate: a provider-produced, locally evidence-verified +proposal with `review_status` (human judgement) and `lifecycle_status` +(source freshness). Confirming a Candidate marks it *suitable for promotion*; +it changes candidate metadata only. + +Phase 5 adds the canonical side and the one bridge between them: + +| | Candidate (Phase 4) | Canonical object (Phase 5) | +|---|---|---| +| Produced by | provider + local verifier | deterministic projector, explicit human action, explicit promotion | +| Tables | `semantic_candidates`, `semantic_relation_candidates`, `semantic_conflict_candidates` | `engineering_entities`, `engineering_claims`, `engineering_relations` (+ aliases, bindings, evidence joins) | +| Truth status | always `status: "candidate"` | governed record with lifecycle, authority and provenance | +| History | reviewed/stale, kept | versioned by Knowledge Revisions, superseded/withdrawn/merged, kept | +| Deleted when promoted | **never** — the Candidate remains queryable | n/a | + +**Canonical does not mean infallible.** A canonical Relation may be +`inferred` (a deterministic call edge with preserved ambiguity) or +`confirmed` (a human promoted it). Canonical means: stored in the canonical +schema, versioned, evidence-bound, provenance-recorded, governed by explicit +lifecycle actions, included in Knowledge Revisions, and queryable through +stable contracts. + +Candidates are never copied wholesale into the graph. A canonical object +records `promoted_from_candidate_id`, `promoted_by`, `promoted_at` and +`promotion_policy_version` (in `knowledge_promotions` + the object's own +provenance columns), and the source Candidate remains in its Phase 4 table. +The graph never depends on the Candidate row's survival — the stored id is +provenance, not an ownership edge, and nothing cascade-deletes canonical +history. + +## 2. The four write paths (and no others) + +Graph truth can enter through exactly: + +1. **deterministic projection** (`graph seed` / `graph sync`) — model-free, + local, versioned by `GRAPH_PROJECTOR_VERSION`; +2. **explicit manual creation** (`entity create`, `claim create`, + `relation create`) — evidence-required, actor-attributed; +3. **explicit Candidate promotion** (`promotion promote`); +4. **explicit Relation Candidate promotion** (`promotion promote-relation`). + +`semantic review confirm` does not promote. Ingestion does not promote. +Conflict candidates cannot be promoted in Phase 5 (they stay in the Phase 4 +store for the Phase 6 conflict engine). There is no generic mutation +endpoint; every write is a typed operation that records a Human Decision and +a Knowledge Revision. + +## 3. Entity identity + +An Entity is one engineering concept in one Workspace, identified logically by + +```text +UNIQUE(workspace_id, entity_type, canonical_key) +``` + +`canonical_key` is deterministic and normalized, never a display name and +never a bare UUID (the database id `ent_*` exists but is not the logical +identity). Key shapes used by the projector and promotion: + +```text +requirement:REQ-NC-017 +interface:POST:/name-check +code-component:asset:a_123 +code-symbol:asset:a_123:org.example.NameCheckService#execute(Request) +configuration:asset:a_456:namecheck.timeout +database-object:database-schema:PASSENGER +document:asset:a_789 +``` + +Identity resolution at promotion time: an exact stable identifier +(`stable_key` such as `REQ-NC-017`) resolves to `entity_type:`; +without one, a deterministic normalized-title key is derived +(`:derived:`), and the plan reports +`identity_matches` / `identity_conflicts` so a human sees what the promotion +would attach to before writing. Descriptions are bounded (promoted Candidate +statement or a concise derived label); chain-of-thought is never stored. + +Entity types, Claim types, Relation types, relation states, lifecycle +statuses, authority statuses and origins are **closed vocabularies** +(`openmind/knowledge/vocabularies.py`), validated at every write boundary. +An unknown model- or caller-supplied type fails typed +(`UnknownVocabularyValue`), never "accepted for later". There is deliberately +no `depends-on` relation: dependency semantics must use the closest honest +member (`calls`, `reads`, `consumes`, …) or stay out of the graph. + +## 4. Aliases + +`engineering_entity_aliases` holds alternative names (identifier, name, +acronym, legacy-name, path, symbol, manual), each normalized +(case-folded, whitespace collapsed) for exact lookup. Rules: + +* an alias row belongs to exactly one Entity; +* an exact normalized collision with an alias of a DIFFERENT active Entity is + **reported** (typed `AliasCollision`, listing the holders) — never silently + attached; resolution is an explicit merge or a manual decision; +* removed aliases keep their row with `status='removed'` for audit; +* aliases participate in graph search (exact alias outranks vector + similarity) and may carry Evidence. + +## 5. Bindings + +`engineering_entity_bindings` connect an Entity to source-plane objects +(asset, revision, segment, evidence, document-block, code-symbol, +configuration-key, database-object, api-operation, message-topic) in a role +(primary-source, definition-source, implementation, configuration, test, +interface, supporting, historical). The referenced object must belong to the +same Workspace (validated through the Phase 2 scoped reads). A binding to a +non-current Revision becomes `stale` at reconciliation; historical bindings +stay queryable. A binding is *not* a Relation — it anchors one concept to the +source, it does not connect two concepts. + +## 6. Claim identity and Evidence + +A Claim is one bounded statement about one Entity. +`normalized_statement_hash` = SHA-256 of the whitespace-collapsed, case-folded +statement; an identical normalized Claim on the same Entity deduplicates to +the existing active row instead of inserting. Claims are immutable in +meaning: correcting one creates a new Claim and supersedes the old +(`superseded_by_claim_id`), never rewrites it. + +**Every active Claim requires at least one valid Evidence** join +(`engineering_claim_evidence`, roles primary/supporting/context/authority). +At write time each cited Evidence id must exist in the Workspace and each +quote must be a whitespace-normalized substring of the immutable Evidence +content (same rule as the Phase 4 verifier — fabricated quotes are rejected +locally). Manual Claims without Evidence are rejected. The one documented +exception in the whole graph: a **deterministic structural container** Entity +(e.g. a code-component projected from an Asset) may exist with bindings but +no Claim, because its existence is the recorded deterministic fact. + +## 7. Relation identity and provenance + +A Relation connects two Entities of the same Workspace with a closed type and +a state (`explicit` / `inferred` / `confirmed` / `rejected` / `stale` / +`superseded`). The active-identity key is + +```text +(workspace_id, source_entity_id, target_entity_id, relation_type) +``` + +— re-promoting or re-projecting the same edge reuses the existing active row +(idempotent). Self-relations are rejected (no current type supports them). +Every Relation carries provenance: verified Evidence joins +(`engineering_relation_evidence`), or deterministic source facts plus +bindings (projected containment/call edges), or an explicit Human Decision +(manual creation). Unexplained Relations are impossible. State semantics: + +* deterministic call edge → `inferred`, `origin=deterministic`, confidence + from the analyzer (name-based ambiguity is preserved as low confidence, + never upgraded); +* structurally explicit containment (OpenAPI operation in its document) → + `explicit`; +* promoted Relation Candidate → `confirmed`, `origin=semantic-promotion`; +* a rejected Relation is retained, excluded from the active graph, and can be + restored explicitly; +* `possibly-related` is never rewritten as `implements`. + +## 8. Knowledge Revisions + +`knowledge_revisions` is the per-Workspace monotonic ledger: +`UNIQUE(workspace_id, revision_number)`. One successful canonical graph +transaction = exactly one revision row, allocated *inside* the same SQLite +transaction as the graph writes (single WAL connection + process lock, so +concurrent writers serialize and numbers can neither duplicate nor skip +backwards). A failed transaction rolls everything back — no revision, no +partial graph. Each revision records action (graph-seed, graph-sync, +candidate-promotion, relation-promotion, manual-entity-create, entity-merge, +claim-supersede, authority-change, graph-reconcile, …), a bounded summary, +the actor, and per-kind changed-object counts. Read operations report the +current revision number; Bundle export records the exported revision. +Revisions are immutable. + +## 9. Human Decisions + +`knowledge_decisions` — one immutable row per governance write +(promote-candidate, promote-relation, create-entity, create-claim, +create-relation, add-alias, remove-alias, merge-entity, split-entity, +mark-authoritative, mark-non-authoritative, supersede, withdraw, +reject-relation, restore-relation), linked to its Knowledge Revision, with +caller-supplied `actor` (may be empty — identity is never inferred), a +bounded note, bounded before/after snapshots (no secrets — they are built +from graph rows, which never contain credentials), and the invoking command. +Decisions are exported in the Bundle. + +## 10. Promotion + +### Eligibility (checked at plan AND again inside the promoting transaction) + +Semantic Candidate: `review_status=confirmed` ∧ `lifecycle_status=active` ∧ +`evidence_status=verified` ∧ supported kind ∧ every cited Evidence belongs to +the Workspace ∧ every source Revision is current ∧ not already promoted. +Relation Candidate adds: both endpoints resolve unambiguously to Workspace +objects. There are no bypass flags (`--accept-stale` etc. do not exist); +stale material must be re-analyzed or recreated manually with fresh +Evidence. + +### Planning + +`plan_candidate_promotion` / `plan_relation_promotion` are deterministic, +provider-free and write nothing. They return eligibility, blocking reasons, +the proposed Entity/Claim/aliases/bindings (or relation + endpoint +resolutions), identity matches/conflicts, any existing target, and the +expected action (`create-entity-and-claim` / `attach-claim-to-existing-entity` +/ `already-promoted` / `blocked` / `identity-conflict`). + +### Behavior by kind + +* **engineering-concept** → resolve-or-create the Entity, create/reuse the + primary Claim, Evidence joins copied from the verified candidate quotes, + aliases from stable identifiers, bindings to the source Revision/Segment, + one promotion record, one Decision, one Knowledge Revision; vector + projection refreshed after commit. +* **classification** → never overwrites `assets.asset_type`; creates/reuses + the Asset's `document` Entity and attaches a `classification` Claim bound + to Asset + Revision. +* **revision-status** → a `revision-status` Claim only; + `asset_revisions.status` is never written. +* **relation candidate** → endpoints re-resolved, Evidence revalidated, + canonical Relation created/reused as `confirmed`. + +### Idempotency and blocking + +Promoting the same Candidate twice returns the same target +(`status=already-promoted`) and writes no new rows and no new Knowledge +Revision. A blocked attempt returns `status=blocked` with reasons and +persists nothing (no revision, no decision) — recorded promotions exist only +for promotions that happened. + +## 11. Deterministic graph projection + +`openmind/knowledge/projector.py` is local and model-free (no semantic +import, no provider, zero egress). `GRAPH_PROJECTOR_VERSION` versions the +rules; `knowledge_projection_state` stores, per Workspace, the projector +version, a source-knowledge hash (active asset ids + current revision ids + +segment content hashes + relevant facet/structure hashes), the last Knowledge +Revision written and the sync time. + +Projection rules (each documented and tested): + +| Source fact | Graph object | +|---|---| +| active `source-code` / `test-source` Asset | `code-component` Entity | +| active `configuration` Asset | `configuration` Entity | +| active `database-schema` Asset | `data-model` Entity (+ per-object `database-object` Entities from parsed SQL segments) | +| documentation/document Asset with a parse record | `document` Entity | +| `build-definition` Asset | `build-definition` Entity | +| Java type/method/constructor Segment | `code-symbol` Entity | +| OpenAPI `api-operation` Segment | `interface` Entity | +| OpenAPI `schema-definition` Segment | `data-model` Entity | +| SQL object Segment | `database-object` Entity | +| deterministically identified configuration key Segment | `configuration` Entity | +| message-topic facet capture | `message-topic` Entity | +| structural containment | `contains` Relation, state `explicit` | +| structure-map call edge (both endpoints resolved, evidence exists) | `calls` Relation, state `inferred`, confidence from analyzer ambiguity | + +Deliberate non-projections: a generic document never becomes a Requirement +or Design; a method in a test-source file never becomes a Test Case; a +name-only ambiguous call never becomes a confirmed edge; facets are +projected only where their semantics are exact. + +`sync` is incremental: unchanged source hash → zero writes, zero revisions. +A changed hash updates only the affected Assets' entities/bindings/relations, +stales what disappeared, and writes one Knowledge Revision. `seed` is the +first sync; a full reconcile pass exists for recovery. + +## 12. Staleness + +`reconcile_graph_staleness(workspace_id)` — incremental, indexed: + +1. bindings referencing non-current Revisions → stale; +2. Claims whose only live primary Evidence sits on stale Revisions → stale; +3. Relations depending on stale endpoints/claims/evidence → stale; +4. deterministic Entities with no active bindings → stale; +5. manual Entities with valid active Claims stay active (no code binding + required); authority status is preserved on stale objects; confirmed + promotions keep their history. + +Runs after code/document Revision commits (same hook chain as the Phase 4 +candidate reconciliation), at Runtime worker startup as a backstop, before +active graph statistics, and before a current-only Bundle export. Stale +objects are excluded from active queries by default and remain queryable +with `include_stale`. + +## 13. Merge and split + +**Merge** (source → target, one transaction): source becomes `merged` with +`merged_into_entity_id` (never deleted; still addressable and resolving to +the target); aliases move unless they collide (collisions reported in the +result and left on the source, auditable); bindings move and deduplicate; +Claims move (their normalized hashes deduplicate against the target's); +Relations are rewired, self-relations eliminated, duplicates deduplicated; +Evidence intact; one Decision, one Knowledge Revision, vector projections +refreshed (source's active projection removed). + +**Split** is explicit and narrow: the caller lists exactly the Claim ids and +Binding ids to move and the Relation endpoint rewrites; the new Entity's +type/key/name are given, at least one Claim or Binding must move, every moved +id must belong to the source, and the transaction is all-or-nothing. No +model chooses a split. One Decision, one Knowledge Revision. + +## 14. Search and the graph vector projection + +Each Workspace gets a separate `knowledge_` vector collection — +never mixed into `code_` / `documents_` (the existing search contracts stay +byte-identical). Indexed objects: **active Entities and Claims** (Relations +are not embedded). Entity text = type + canonical key + display name + +aliases + description + active primary claims; Claim text = type + statement ++ entity identity + bounded evidence locator summary. Metadata carries +workspace, object kind, ids, types, lifecycle, authority, origin, revision +and content hash. + +`search_graph` fuses, deterministically and in this precedence order: exact +canonical key → exact normalized alias → exact identifier token → lexical +match → vector similarity. Exact identifiers always outrank semantic +similarity. Entities and Claims are returned as separate sections; every hit +carries Evidence ids or a navigable Claim id, plus the current Knowledge +Revision. Stale/superseded/withdrawn objects are excluded by default. +Lifecycle: updates re-project, merge removes the source's active projection, +terminate/delete drop `knowledge_` (registered in +`vectorstore._COLLECTION_PREFIXES`, so the startup orphan sweep recognizes +it and deleting workspaces are not treated as orphans; the batched-drain and +in-flight-drop protections apply unchanged). + +## 15. Graph queries + +`runtime.knowledge` / `ServiceContainer.knowledge` expose the full +workspace-scoped operation set (stats, current revision, entity/claim/ +relation list+get, node lookup, search, expand, path, subgraph, promotion +plan/promote for both kinds, manual creates, aliases, merge, split, +authority, supersede, withdraw, seed/sync/reconcile, revision + decision +history). A graph object of Workspace A is unreachable through Workspace B — +every SQL read filters on the validated workspace id. + +**Node abstraction**: `get_node` returns the stable camelCase read shape +(id, nodeKind, entityType, canonicalKey, displayName, lifecycleStatus, +authorityStatus, origin, knowledgeRevision, bindings, claimCount, +relationCount). Node kinds: entity, claim, asset, revision, segment, +evidence — the source-plane kinds are *projected into the read shape* from +their canonical Phase 2 rows, not copied into graph tables. + +**Expansion** is bounded BFS with deterministic ordering (relation type, +then target key): direction in/out/both, relation-type filter, depth ≤ 4, +nodes ≤ 1000, edges ≤ 3000 (defaults lower), explicit `truncated` flag, +`include_stale` off by default. No recursive SQL, no unbounded traversal. + +**Path discovery** is deterministic BFS shortest-path with bounded depth and +visited-node cap, equal-length paths returned up to a cap, Relation Evidence +summaries included, and three honest outcomes: found / no-path / truncated. +It is generic path discovery — explicitly **not** formal Requirement +Traceability (Phase 6). + +**Subgraph export** returns the bounded node+edge set around a seed node +set, same limits and determinism. + +## 16. Knowledge Bundle 2.0 Draft + +`.openmind` 1.1.0 is untouched. `openmind bundle export` writes a separate +`.openmind-v2/` directory: `manifest.json`, `workspace.json`, JSONL files +(assets, revisions, segments, evidence, entities, aliases, bindings, claims, +claim-evidence, relations, relation-evidence, decisions, +knowledge-revisions, lenses) and `schemas/*.schema.json`. Guarantees: +deterministic ordering (stable sort keys per file), UTF-8 JSONL with `\n`, +source-relative locators, no absolute paths, no secrets/prompts/raw provider +responses/semantic cache/provider profiles, every active Claim has Evidence, +every Relation endpoint and Relation Evidence exists, and the manifest +records runtime version, bundle schema version, workspace, Knowledge +Revision, timestamp, per-file SHA-256 hashes, record counts and warning +flags. + +Modes: `--current-only` (active objects only, after a staleness reconcile), +`--include-history` (stale/superseded/withdrawn/merged included), +`--knowledge-revision N` (records whose created/updated revision ≤ N — an +honest approximation documented as such: it filters by revision stamps, it +does not reconstruct point-in-time lifecycle states). + +`python -m openmind.bundle_verify ` is a standalone, dependency-free +verifier (schema shape, referential integrity, evidence integrity, counts, +hashes, active-claim evidence, relation endpoints, duplicate ids, +ordering). Export requires no semantic provider. The Bundle stays **Draft**: +not yet a frozen external contract, and there is no import. + +## 17. GraphStore boundary + +```text +openmind/knowledge/ the Phase 5 package +├── vocabularies.py closed vocabularies (+ validation helpers) +├── errors.py typed graph errors (OpenMindError subclasses) +├── store.py SQL repository over the v0006 tables +│ (same shared WAL connection + lock as db.py) +├── revisions.py transactional revision allocation + decisions +├── identity.py canonical keys, normalization, statement hashes +├── verifier.py evidence ownership + quote verification +├── promotion.py eligibility, planning, promotion transactions +├── projector.py deterministic seed/sync (+ GRAPH_PROJECTOR_VERSION) +├── reconciliation.py graph staleness +├── graph.py node shape, bounded expansion, paths, subgraph +├── search.py exact/lexical/vector fused graph search +├── vector_projection.py knowledge_ collection lifecycle +├── decisions.py decision record helpers +├── bundle.py Bundle 2.0 Draft exporter +├── service.py KnowledgeService (runtime.knowledge) +└── models.py row mappers / read shapes +openmind/ports/graph_repository.py the repository protocol (testing seam) +openmind/bundle_verify.py standalone verifier (python -m openmind.bundle_verify) +``` + +`db.py`, `jobs.py` and `main.py` gain only thin hooks (reconcile call sites, +job-type-free — graph writes are synchronous CLI/REST verbs, no new job +types; the projector runs in-process). Graph SQL lives in +`knowledge/store.py`, grouped, never scattered. + +## 18. Compatibility + +* `ingest` / `asset add` / `document add`: behavior preserved; zero cloud + calls; no automatic graph projection is attached to ingestion in this + phase (graph seed/sync are explicit verbs), so ingestion cannot fail from + a graph error; +* Phase 4 semantics preserved (`unreviewed/confirmed/rejected` × + `active/stale`); `semantic review confirm` still only updates metadata; +* local Ask untouched — no graph or cloud redirection; +* REST: `/projects` naming, all existing routes intact, graph routes + additive; +* MCP: the 26 existing tools unchanged; exactly nine additive **read-only** + graph tools (`get_graph_stats`, `search_graph`, `get_graph_node`, + `expand_graph`, `find_graph_path`, `list_engineering_entities`, + `get_engineering_entity`, `get_engineering_claim`, + `get_engineering_relation`) = 35 total, accounted for in tests. No MCP + tool promotes, creates, merges, changes authority, seeds or exports; +* `.openmind` export and the dependency-free artifact path unchanged; +* Skill Bridge unchanged and database-independent; +* v0006 is purely additive — Phase 1–4 databases migrate losing nothing; +* delete-race / delete-responsive protections extended to the new + collection prefix, not modified. + +## 19. Testing strategy + +Fifteen focused suites, all registered in `scripts/run_acceptance.py` (a +missing registration fails the manifest): `verify_knowledge_migration`, +`_entities`, `_claims`, `_relations`, `_promotion`, `_projection`, +`_revisions`, `_decisions`, `_staleness`, `_merge_split`, `_search`, +`_graph`, `_bundle`, `_cli`, `_adapters`. All offline (isolated data dirs, +`OPENMIND_EMBED_OFFLINE=1`, no egress); semantic Candidates are created with +the Phase 4 **mock provider** or direct store writes — no real provider API +is ever called. CI: the ubuntu full gate runs the whole core tier plus the +Bundle verifier and MCP/Skill-Bridge smoke; cross-platform smoke covers +migration → seed → promotion → lookups → path → bundle export+verify. + +## 20. Explicitly deferred to Phase 6+ + +Formal Requirement Traceability and coverage/gap scoring; conflict +*resolution* and conflict-candidate promotion; change-impact analysis; +branch/PR overlays; webhooks; CI policy gates; Neo4j or any external graph +database; Cypher/Gremlin; a Graph UI; Bundle 2.0 schema freeze and import; +Titan Mind integration; Feature Evidence Packets; provider batch APIs; cloud +embeddings; historical document vector search; OCR; COBOL/JCL parsing; +Jira/Confluence connectors; plugin packaging; new Agent Skills and Skill +Forge/Verification changes; worker-pool/job-DAG replacement; graph-assisted +Ask (a later opt-in). diff --git a/openmind/bundle_verify.py b/openmind/bundle_verify.py new file mode 100644 index 0000000..0b5f0d8 --- /dev/null +++ b/openmind/bundle_verify.py @@ -0,0 +1,307 @@ +"""Standalone Knowledge Bundle 2.0 Draft verifier. + + python -m openmind.bundle_verify + +Deliberately dependency-free and database-free: standard library only, no +OpenMind runtime import, no vector store, no provider. It validates a bundle +directory AS AN ARTIFACT — the way an external consumer would — so a bundle +that only verifies against the database that produced it is caught here. + +CHECKS +------ +* manifest parses, carries the draft schema version, workspace, revision; +* every file the manifest names exists, hash-matches and record-count-matches; +* every JSONL line parses; ids are unique per file; +* referential integrity: claims -> entities, aliases/bindings -> entities, + relation endpoints -> entities, claim/relation evidence joins -> claims/ + relations AND -> evidence rows; +* every ACTIVE claim has at least one evidence join; +* deterministic ordering where the exporter guarantees it; +* no machine-absolute path in locators, source paths or binding keys + (workspace-relative and API-route slashes are legitimate); +* no obvious secret material (bearer tokens, api-key assignments). + +Exit code 0 = valid (warnings allowed), 1 = invalid, 2 = unusable input. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +#: Machine-absolute path shapes. A leading slash alone is NOT absolute here: +#: API routes ("/name-check") and POSIX-relative-ish locators are legitimate; +#: what must never leak is a real filesystem root. +_ABSOLUTE_PATH = re.compile( + r"^(?:[A-Za-z]:[\\/]|\\\\|/(?:home|Users|var|tmp|etc|mnt|opt|srv|root)/)") + +_SECRET_HINT = re.compile( + r"(?i)(?:api[_-]?key|authorization|bearer\s+[A-Za-z0-9._-]{16,}|" + r"-----BEGIN [A-Z ]*PRIVATE KEY-----)") + +_PATH_FIELDS = ("file", "document", "source_path", "ref_key", "logical_key") + +_JSONL_FILES = ( + "assets.jsonl", "revisions.jsonl", "segments.jsonl", "evidence.jsonl", + "entities.jsonl", "aliases.jsonl", "bindings.jsonl", "claims.jsonl", + "claim-evidence.jsonl", "relations.jsonl", "relation-evidence.jsonl", + "decisions.jsonl", "knowledge-revisions.jsonl", "lenses.jsonl", +) + + +class Report: + def __init__(self) -> None: + self.errors: List[str] = [] + self.warnings: List[str] = [] + + def error(self, message: str) -> None: + self.errors.append(message) + + def warn(self, message: str) -> None: + self.warnings.append(message) + + @property + def ok(self) -> bool: + return not self.errors + + +def _read_jsonl(path: Path, report: Report) -> List[Dict[str, Any]]: + records: List[Dict[str, Any]] = [] + if not path.exists(): + report.error(f"missing file: {path.name}") + return records + for lineno, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + records.append(json.loads(line)) + except ValueError as exc: + report.error(f"{path.name}:{lineno}: invalid JSON ({exc})") + return records + + +def _check_unique_ids(name: str, records: List[Dict[str, Any]], + report: Report, key: str = "id") -> None: + seen: set = set() + for record in records: + value = record.get(key) + if value is None: + continue + if value in seen: + report.error(f"{name}: duplicate {key} {value!r}") + seen.add(value) + + +def _check_sorted(name: str, records: List[Dict[str, Any]], + keys: List[str], report: Report) -> None: + def sort_key(r: Dict[str, Any]): + return tuple(str(r.get(k, "") or "") for k in keys) + actual = [sort_key(r) for r in records] + if actual != sorted(actual): + report.error(f"{name}: records are not in deterministic " + f"{'/'.join(keys)} order") + + +def _scan_paths(name: str, records: List[Dict[str, Any]], + report: Report) -> None: + def scan(value: Any, context: str) -> None: + if isinstance(value, dict): + for key, inner in value.items(): + if key in _PATH_FIELDS and isinstance(inner, str) and \ + _ABSOLUTE_PATH.match(inner): + report.error( + f"{name}: absolute path in {context}.{key}: " + f"{inner!r}") + scan(inner, f"{context}.{key}") + elif isinstance(value, list): + for i, inner in enumerate(value): + scan(inner, f"{context}[{i}]") + elif isinstance(value, str) and _SECRET_HINT.search(value): + report.error(f"{name}: possible secret material in {context}") + + for i, record in enumerate(records): + scan(record, f"record[{i}]") + + +def verify_bundle(directory: str) -> Report: + report = Report() + root = Path(directory) + manifest_path = root / "manifest.json" + if not manifest_path.exists(): + report.error(f"missing manifest.json in {root}") + return report + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except ValueError as exc: + report.error(f"manifest.json: invalid JSON ({exc})") + return report + + version = str(manifest.get("bundleSchemaVersion") or "") + if not version.startswith("2.0.0-draft"): + report.error(f"unexpected bundleSchemaVersion: {version!r}") + for field in ("workspaceId", "knowledgeRevision", "generatedAt", + "files", "counts"): + if field not in manifest: + report.error(f"manifest.json: missing field {field!r}") + + # -- file hashes and record counts -------------------------------------- + files: Dict[str, Dict[str, Any]] = manifest.get("files") or {} + for name, meta in sorted(files.items()): + path = root / name + if not path.exists(): + report.error(f"manifest names missing file: {name}") + continue + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if digest != meta.get("sha256"): + report.error(f"{name}: sha256 mismatch (manifest " + f"{str(meta.get('sha256'))[:12]}..., actual " + f"{digest[:12]}...)") + if "records" in meta: + actual = sum(1 for line in + path.read_text(encoding="utf-8").splitlines() + if line.strip()) + if actual != meta["records"]: + report.error(f"{name}: record count mismatch (manifest " + f"{meta['records']}, actual {actual})") + for name in _JSONL_FILES: + if name not in files: + report.error(f"manifest is missing required file entry: {name}") + + # -- load ---------------------------------------------------------------- + data = {name: _read_jsonl(root / name, report) + for name in _JSONL_FILES} + entities = data["entities.jsonl"] + claims = data["claims.jsonl"] + relations = data["relations.jsonl"] + evidence = data["evidence.jsonl"] + + for name in _JSONL_FILES: + if name in ("claim-evidence.jsonl", "relation-evidence.jsonl"): + continue + _check_unique_ids(name, data[name], report) + + entity_ids = {e.get("id") for e in entities} + claim_ids = {c.get("id") for c in claims} + relation_ids = {r.get("id") for r in relations} + evidence_ids = {e.get("id") for e in evidence} + + # -- referential integrity ---------------------------------------------- + for claim in claims: + if claim.get("entity_id") not in entity_ids: + report.error(f"claims: {claim.get('id')} references missing " + f"entity {claim.get('entity_id')}") + for alias in data["aliases.jsonl"]: + if alias.get("entity_id") not in entity_ids: + report.error(f"aliases: {alias.get('id')} references missing " + f"entity {alias.get('entity_id')}") + for binding in data["bindings.jsonl"]: + if binding.get("entity_id") not in entity_ids: + report.error(f"bindings: {binding.get('id')} references missing " + f"entity {binding.get('entity_id')}") + for relation in relations: + for end in ("source_entity_id", "target_entity_id"): + if relation.get(end) not in entity_ids: + report.error( + f"relations: {relation.get('id')} {end} references " + f"missing entity {relation.get(end)}") + claim_evidence_by_claim: Dict[Any, int] = {} + for join in data["claim-evidence.jsonl"]: + claim_evidence_by_claim[join.get("claim_id")] = \ + claim_evidence_by_claim.get(join.get("claim_id"), 0) + 1 + if join.get("claim_id") not in claim_ids: + report.error(f"claim-evidence: join references missing claim " + f"{join.get('claim_id')}") + if join.get("evidence_id") not in evidence_ids: + report.error(f"claim-evidence: join references missing evidence " + f"{join.get('evidence_id')}") + for join in data["relation-evidence.jsonl"]: + if join.get("relation_id") not in relation_ids: + report.error( + f"relation-evidence: join references missing relation " + f"{join.get('relation_id')}") + if join.get("evidence_id") not in evidence_ids: + report.error( + f"relation-evidence: join references missing evidence " + f"{join.get('evidence_id')}") + + # -- active claims must carry evidence ----------------------------------- + for claim in claims: + if claim.get("lifecycle_status") == "active" and \ + not claim_evidence_by_claim.get(claim.get("id")): + report.error(f"claims: active claim {claim.get('id')} has no " + f"evidence join") + + # -- deterministic ordering ---------------------------------------------- + _check_sorted("entities.jsonl", entities, ["canonical_key", "id"], + report) + _check_sorted("claims.jsonl", claims, ["entity_id", "id"], report) + _check_sorted("relations.jsonl", relations, + ["relation_type", "source_entity_id", "target_entity_id", + "id"], report) + _check_sorted("evidence.jsonl", evidence, ["id"], report) + ledger = data["knowledge-revisions.jsonl"] + numbers = [int(r.get("revision_number") or 0) for r in ledger] + if numbers != sorted(numbers): + report.error("knowledge-revisions.jsonl: revision numbers are not " + "ascending") + + # -- path and secret scan ------------------------------------------------- + for name in _JSONL_FILES: + _scan_paths(name, data[name], report) + + counts = manifest.get("counts") or {} + expectation = { + "entities": len(entities), "claims": len(claims), + "relations": len(relations), "evidence": len(evidence), + "aliases": len(data["aliases.jsonl"]), + "bindings": len(data["bindings.jsonl"]), + "decisions": len(data["decisions.jsonl"]), + "knowledgeRevisions": len(ledger), + } + for key, actual in expectation.items(): + if key in counts and int(counts[key]) != actual: + report.error(f"manifest counts.{key} = {counts[key]} but file " + f"has {actual} records") + return report + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m openmind.bundle_verify", + description="Verify a Knowledge Bundle 2.0 Draft directory.") + parser.add_argument("directory", help="the bundle directory " + "(contains manifest.json)") + parser.add_argument("--json", action="store_true", dest="as_json", + help="print one machine-readable JSON object") + args = parser.parse_args(argv) + + if not Path(args.directory).is_dir(): + message = f"not a directory: {args.directory}" + if args.as_json: + print(json.dumps({"ok": False, "error": message})) + else: + print(f"error: {message}", file=sys.stderr) + return 2 + + report = verify_bundle(args.directory) + if args.as_json: + print(json.dumps({"ok": report.ok, "errors": report.errors, + "warnings": report.warnings}, indent=2)) + else: + for warning in report.warnings: + print(f"warning: {warning}", file=sys.stderr) + for error in report.errors: + print(f"error: {error}", file=sys.stderr) + print(("VALID" if report.ok else "INVALID") + + f" — {len(report.errors)} error(s), " + f"{len(report.warnings)} warning(s)") + return 0 if report.ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/openmind/cli.py b/openmind/cli.py index 558b0be..ad8a5dc 100644 --- a/openmind/cli.py +++ b/openmind/cli.py @@ -1149,6 +1149,12 @@ def build_parser() -> argparse.ArgumentParser: from . import cli_semantic cli_semantic.register(sub, common) + # -- canonical knowledge graph (v2 Phase 5): graph / promotion / + # entity / claim / relation / bundle groups, plus the history + # subcommands of the existing `knowledge` group above. + from . import cli_knowledge + cli_knowledge.register(sub, common, knowledge_sub) + serve = sub.add_parser("serve", parents=[common], help="run the FastAPI web application") serve.add_argument("--host", default="127.0.0.1", diff --git a/openmind/cli_knowledge.py b/openmind/cli_knowledge.py new file mode 100644 index 0000000..62ebef0 --- /dev/null +++ b/openmind/cli_knowledge.py @@ -0,0 +1,1050 @@ +"""CLI adapters for the Phase 5 canonical Knowledge Graph: the ``graph``, +``promotion``, ``entity``, ``claim``, ``relation`` and ``bundle`` command +groups, plus the history subcommands of the existing ``knowledge`` group. + +The CONTRACT is exactly the parent CLI's: ``--json`` prints one object on +stdout, humans read stderr, no ANSI, the shared exit codes, bounded output, +no secrets. Graph mutations all take explicit ``--actor`` and ``--note`` — +identity is never inferred — and there is no flag anywhere here that skips +an eligibility rule (no ``--accept-stale``, no ``--force-unverified``). +""" +from __future__ import annotations + +import argparse +import json +from typing import Any, Dict, List, Tuple + +from .domain.errors import InvalidRequest + + +def _ok(payload: Dict[str, Any]) -> Dict[str, Any]: + out = {"ok": True} + out.update(payload) + return out + + +def _knowledge(): + from .runtime import get_runtime + return get_runtime().knowledge + + +def _evidence_from_args(args: argparse.Namespace) -> List[Dict[str, Any]]: + """Evidence references from ``--evidence`` (repeatable ids) and/or + ``--evidence-json`` (a JSON array of {evidence_id, quote?, role?}).""" + refs: List[Dict[str, Any]] = [] + for evidence_id in getattr(args, "evidence", None) or []: + refs.append({"evidence_id": str(evidence_id).strip()}) + raw = getattr(args, "evidence_json", None) + if raw: + try: + data = json.loads(raw) + except ValueError as exc: + raise InvalidRequest(f"--evidence-json is not valid JSON: {exc}") + if not isinstance(data, list): + raise InvalidRequest("--evidence-json must be a JSON array") + for entry in data: + if not isinstance(entry, dict) or not entry.get("evidence_id"): + raise InvalidRequest( + "--evidence-json entries must be objects with an " + "evidence_id") + refs.append(entry) + return refs + + +def _actor_note(args: argparse.Namespace) -> Tuple[str, str]: + return str(getattr(args, "actor", "") or ""), \ + str(getattr(args, "note", "") or "") + + +def _source_command(args: argparse.Namespace, verb: str) -> str: + return f"cli:{verb}" + + +# --------------------------------------------------------------------------- +# graph commands +# --------------------------------------------------------------------------- +def cmd_graph_stats(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok(_knowledge().get_stats(args.workspace)) + + def human(p: Dict[str, Any]) -> None: + print(f"knowledge revision: {p['knowledge_revision']}") + print(f"entities: {p['entities_active']} active / " + f"{p['entities_total']} total") + for etype, count in sorted(p.get("entities_by_type", {}).items()): + print(f" {etype:24} {count}") + print(f"claims: {p['claims_active']} active / {p['claims_total']}") + print(f"relations: {p['relations_active']} active / " + f"{p['relations_total']}") + print(f"decisions: {p['decisions']} promotions: {p['promotions']}") + + out.emit(payload, human) + return 0, payload + + +def cmd_graph_seed_plan(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok({"plan": _knowledge().plan_seed(args.workspace)}) + + def human(p: Dict[str, Any]) -> None: + plan = p["plan"] + print(f"unchanged: {plan['unchanged']}") + print(f"desired: {plan['desired_entities']} entities, " + f"{plan['desired_relations']} relations") + print(f"would create: {plan['would_create_entities']} entities, " + f"{plan['would_create_relations']} relations") + print(f"would stale: {plan['would_stale_entities']} entities, " + f"{plan['would_stale_relations']} relations") + + out.emit(payload, human) + return 0, payload + + +def cmd_graph_seed(args, out) -> Tuple[int, Dict[str, Any]]: + if not getattr(args, "workspace", None): + raise InvalidRequest("--workspace is required") + actor, _ = _actor_note(args) + payload = _ok(_knowledge().seed(args.workspace, actor=actor)) + out.emit(payload, lambda p: print( + f"{p['action']}: knowledge revision {p['knowledge_revision']}")) + return 0, payload + + +def cmd_graph_sync(args, out) -> Tuple[int, Dict[str, Any]]: + actor, _ = _actor_note(args) + payload = _ok(_knowledge().sync(args.workspace, actor=actor)) + out.emit(payload, lambda p: print( + f"{p['action']}: knowledge revision {p['knowledge_revision']}")) + return 0, payload + + +def cmd_graph_reconcile(args, out) -> Tuple[int, Dict[str, Any]]: + actor, _ = _actor_note(args) + payload = _ok(_knowledge().reconcile_staleness(args.workspace, + actor=actor)) + out.emit(payload, lambda p: print( + f"changed: {p['changed']} (revision {p['knowledge_revision']})")) + return 0, payload + + +def cmd_graph_search(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok(_knowledge().search_entities( + args.workspace, args.query, limit=args.limit, + include_stale=bool(getattr(args, "include_stale", False)))) + + def human(p: Dict[str, Any]) -> None: + for hit in p["entities"]: + print(f" entity {hit['id']} {hit['canonical_key']} " + f"[{hit['matched_via']} {hit['score']}]") + for hit in p["claims"]: + print(f" claim {hit['id']} {hit['statement'][:80]} " + f"[{hit['matched_via']} {hit['score']}]") + if not p["entities"] and not p["claims"]: + print("no graph objects matched") + + out.emit(payload, human) + return 0, payload + + +def cmd_graph_node(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok({"node": _knowledge().get_node(args.workspace, args.node)}) + out.emit(payload, lambda p: print(json.dumps(p["node"], indent=2, + default=str))) + return 0, payload + + +def cmd_graph_expand(args, out) -> Tuple[int, Dict[str, Any]]: + types = [t.strip() for t in str(getattr(args, "types", "") or "").split( + ",") if t.strip()] + payload = _ok(_knowledge().expand_node( + args.workspace, args.node, depth=args.depth, + direction=args.direction, relation_types=types or None, + include_stale=bool(getattr(args, "include_stale", False)))) + + def human(p: Dict[str, Any]) -> None: + print(f"{len(p['nodes'])} nodes, {len(p['edges'])} edges" + + (" (truncated)" if p["truncated"] else "")) + for edge in p["edges"][:50]: + print(f" {edge['sourceEntityId']} -{edge['relationType']}-> " + f"{edge['targetEntityId']} [{edge['relationState']}]") + + out.emit(payload, human) + return 0, payload + + +def cmd_graph_path(args, out) -> Tuple[int, Dict[str, Any]]: + types = [t.strip() for t in str(getattr(args, "types", "") or "").split( + ",") if t.strip()] + payload = _ok(_knowledge().find_path( + args.workspace, getattr(args, "from"), args.to, + max_depth=args.max_depth, direction=args.direction, + relation_types=types or None, + include_stale=bool(getattr(args, "include_stale", False)))) + + def human(p: Dict[str, Any]) -> None: + print(f"outcome: {p['outcome']}") + for path in p.get("paths", []): + chain = path["entities"][0] + for i, edge in enumerate(path["edges"]): + chain += (f" -{edge['relationType']}-> " + + path["entities"][i + 1]) + print(f" [{path['length']}] {chain}") + + out.emit(payload, human) + return 0, payload + + +# --------------------------------------------------------------------------- +# promotion commands +# --------------------------------------------------------------------------- +def cmd_promotion_plan(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok({"plan": _knowledge().plan_candidate_promotion( + args.workspace, args.candidate)}) + + def human(p: Dict[str, Any]) -> None: + plan = p["plan"] + print(f"eligible: {plan['eligible']}") + print(f"expected action: {plan['expected_action']}") + for reason in plan.get("blocking_reasons", []): + print(f" blocked: {reason}") + + out.emit(payload, human) + return 0, payload + + +def cmd_promotion_promote(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().promote_candidate( + args.workspace, args.candidate, actor=actor, note=note, + source_command=_source_command(args, "promotion promote"))) + + def human(p: Dict[str, Any]) -> None: + print(f"status: {p['status']}") + if p.get("entity"): + print(f"entity: {p['entity']['id']} " + f"{p['entity']['canonical_key']}") + if p.get("claim"): + print(f"claim: {p['claim']['id']}") + print(f"knowledge revision: {p['knowledge_revision']}") + + out.emit(payload, human) + return 0, payload + + +def cmd_promotion_relation_plan(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok({"plan": _knowledge().plan_relation_promotion( + args.workspace, args.relation)}) + + def human(p: Dict[str, Any]) -> None: + plan = p["plan"] + print(f"eligible: {plan['eligible']}") + print(f"expected action: {plan['expected_action']}") + for reason in plan.get("blocking_reasons", []): + print(f" blocked: {reason}") + + out.emit(payload, human) + return 0, payload + + +def cmd_promotion_promote_relation(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().promote_relation( + args.workspace, args.relation, actor=actor, note=note, + source_command=_source_command(args, "promotion promote-relation"))) + + def human(p: Dict[str, Any]) -> None: + print(f"status: {p['status']}") + if p.get("relation"): + print(f"relation: {p['relation']['id']} " + f"[{p['relation']['relation_state']}]") + print(f"knowledge revision: {p['knowledge_revision']}") + + out.emit(payload, human) + return 0, payload + + +# --------------------------------------------------------------------------- +# entity commands +# --------------------------------------------------------------------------- +def cmd_entity_list(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok(_knowledge().list_entities( + args.workspace, entity_type=getattr(args, "type", None), + lifecycle_status=getattr(args, "lifecycle", "active") or None, + origin=getattr(args, "origin", None), limit=args.limit, + offset=args.offset)) + + def human(p: Dict[str, Any]) -> None: + for entity in p["entities"]: + print(f" {entity['id']} {entity['entity_type']:18} " + f"{entity['canonical_key']}") + print(f"{p['count']} of {p['total']}") + + out.emit(payload, human) + return 0, payload + + +def cmd_entity_show(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok({"entity": _knowledge().get_entity(args.workspace, + args.entity)}) + out.emit(payload, lambda p: print(json.dumps(p["entity"], indent=2, + default=str))) + return 0, payload + + +def cmd_entity_create(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().create_entity( + args.workspace, entity_type=args.type, canonical_key=args.key, + display_name=args.name, description=getattr(args, "description", + "") or "", + evidence=_evidence_from_args(args), actor=actor, note=note, + source_command=_source_command(args, "entity create"))) + out.emit(payload, lambda p: print( + f"created {p['entity']['id']} (revision " + f"{p['knowledge_revision']})")) + return 0, payload + + +def cmd_entity_alias_add(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().add_alias( + args.workspace, entity_id=args.entity, alias=args.alias, + alias_type=getattr(args, "type", "manual") or "manual", + evidence_id=getattr(args, "evidence", None) or "", + actor=actor, note=note, + source_command=_source_command(args, "entity alias-add"))) + out.emit(payload, lambda p: print( + "alias added" if not p.get("deduplicated") else "alias already " + "present")) + return 0, payload + + +def cmd_entity_alias_remove(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().remove_alias( + args.workspace, entity_id=args.entity, alias=args.alias, + actor=actor, note=note, + source_command=_source_command(args, "entity alias-remove"))) + out.emit(payload, lambda p: print(f"removed {p['removed']} alias(es)")) + return 0, payload + + +def cmd_entity_merge(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().merge_entities( + args.workspace, source_entity_id=args.source, + target_entity_id=args.target, actor=actor, note=note, + source_command=_source_command(args, "entity merge"))) + + def human(p: Dict[str, Any]) -> None: + print(f"merged {p['source_entity_id']} into " + f"{p['target_entity_id']}") + print(f"moved: {p['aliases']} aliases, {p['bindings']} bindings, " + f"{p['claims']} claims; rewired {p['relations_rewired']} " + f"relations") + for collision in p.get("alias_collisions", []): + print(f" alias collision kept on source: {collision['alias']}") + + out.emit(payload, human) + return 0, payload + + +def cmd_entity_split(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + rewrites = [] + for spec in getattr(args, "rewrite", None) or []: + relation_id, _, end = str(spec).partition(":") + rewrites.append({"relation_id": relation_id.strip(), + "end": end.strip() or "source"}) + payload = _ok(_knowledge().split_entity( + args.workspace, source_entity_id=args.source, + new_entity_type=args.new_type, new_canonical_key=args.new_key, + new_display_name=getattr(args, "new_name", "") or args.new_key, + claim_ids=getattr(args, "claim", None) or [], + binding_ids=getattr(args, "binding", None) or [], + relation_rewrites=rewrites, actor=actor, note=note, + source_command=_source_command(args, "entity split"))) + out.emit(payload, lambda p: print( + f"split -> {p['new_entity']['id']} ({p['moved_claims']} claims, " + f"{p['moved_bindings']} bindings moved)")) + return 0, payload + + +def _cmd_authority(kind: str): + def handler(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + object_id = getattr(args, kind) + payload = _ok(_knowledge().set_authority( + args.workspace, kind=kind, object_id=object_id, + authority=args.status, actor=actor, note=note, + source_command=_source_command(args, f"{kind} authority"))) + out.emit(payload, lambda p: print( + f"{kind} {object_id}: authority = {p['authority_status']}")) + return 0, payload + return handler + + +def _cmd_supersede(kind: str): + def handler(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().supersede_object( + args.workspace, kind=kind, object_id=getattr(args, kind), + replacement_id=args.by, actor=actor, note=note, + source_command=_source_command(args, f"{kind} supersede"))) + out.emit(payload, lambda p: print( + f"{kind} {p['object_id']} superseded by {p['replacement_id']}")) + return 0, payload + return handler + + +def _cmd_withdraw(kind: str): + def handler(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().withdraw_object( + args.workspace, kind=kind, object_id=getattr(args, kind), + actor=actor, note=note, + source_command=_source_command(args, f"{kind} withdraw"))) + out.emit(payload, lambda p: print(f"{kind} withdrawn")) + return 0, payload + return handler + + +# --------------------------------------------------------------------------- +# claim commands +# --------------------------------------------------------------------------- +def cmd_claim_list(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok(_knowledge().list_claims( + args.workspace, entity_id=getattr(args, "entity", None), + claim_type=getattr(args, "type", None), + lifecycle_status=getattr(args, "lifecycle", "active") or None, + limit=args.limit, offset=args.offset)) + + def human(p: Dict[str, Any]) -> None: + for claim in p["claims"]: + print(f" {claim['id']} {claim['claim_type']:22} " + f"{claim['statement'][:70]}") + print(f"{p['count']} of {p['total']}") + + out.emit(payload, human) + return 0, payload + + +def cmd_claim_show(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok({"claim": _knowledge().get_claim(args.workspace, + args.claim)}) + out.emit(payload, lambda p: print(json.dumps(p["claim"], indent=2, + default=str))) + return 0, payload + + +def cmd_claim_create(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().create_claim( + args.workspace, entity_id=args.entity, claim_type=args.type, + statement=args.statement, evidence=_evidence_from_args(args), + actor=actor, note=note, + source_command=_source_command(args, "claim create"))) + out.emit(payload, lambda p: print( + ("deduplicated to " if p.get("deduplicated") else "created ") + + p["claim"]["id"])) + return 0, payload + + +# --------------------------------------------------------------------------- +# relation commands +# --------------------------------------------------------------------------- +def cmd_relation_list(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok(_knowledge().list_relations( + args.workspace, entity_id=getattr(args, "entity", None), + relation_type=getattr(args, "type", None), + relation_state=getattr(args, "state", None), + lifecycle_status=getattr(args, "lifecycle", "active") or None, + limit=args.limit, offset=args.offset)) + + def human(p: Dict[str, Any]) -> None: + for rel in p["relations"]: + print(f" {rel['id']} {rel['source_entity_id']} " + f"-{rel['relation_type']}-> {rel['target_entity_id']} " + f"[{rel['relation_state']}]") + print(f"{p['count']} relation(s)") + + out.emit(payload, human) + return 0, payload + + +def cmd_relation_show(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok({"relation": _knowledge().get_relation(args.workspace, + args.relation)}) + out.emit(payload, lambda p: print(json.dumps(p["relation"], indent=2, + default=str))) + return 0, payload + + +def cmd_relation_create(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().create_relation( + args.workspace, source_entity_id=args.source, + target_entity_id=args.target, relation_type=args.type, + relation_state=getattr(args, "state", "confirmed") or "confirmed", + confidence=getattr(args, "confidence", "medium") or "medium", + evidence=_evidence_from_args(args), actor=actor, note=note, + source_command=_source_command(args, "relation create"))) + out.emit(payload, lambda p: print( + ("deduplicated to " if p.get("deduplicated") else "created ") + + p["relation"]["id"])) + return 0, payload + + +def cmd_relation_reject(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().reject_relation( + args.workspace, relation_id=args.relation, actor=actor, note=note, + source_command=_source_command(args, "relation reject"))) + out.emit(payload, lambda p: print("relation rejected")) + return 0, payload + + +def cmd_relation_restore(args, out) -> Tuple[int, Dict[str, Any]]: + actor, note = _actor_note(args) + payload = _ok(_knowledge().restore_relation( + args.workspace, relation_id=args.relation, actor=actor, note=note, + source_command=_source_command(args, "relation restore"))) + out.emit(payload, lambda p: print( + f"relation restored to {p['relation_state']}")) + return 0, payload + + +# --------------------------------------------------------------------------- +# knowledge history commands (added to the EXISTING knowledge group) +# --------------------------------------------------------------------------- +def cmd_knowledge_revisions(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok(_knowledge().list_knowledge_revisions( + args.workspace, limit=args.limit, offset=args.offset)) + + def human(p: Dict[str, Any]) -> None: + for revision in p["revisions"]: + print(f" #{revision['revision_number']:>4} " + f"{revision['action']:22} {revision['created_at']} " + f"{revision['summary'][:50]}") + print(f"current: {p['knowledge_revision']}") + + out.emit(payload, human) + return 0, payload + + +def cmd_knowledge_revision(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok({"revision": _knowledge().get_knowledge_revision( + args.workspace, args.number)}) + out.emit(payload, lambda p: print(json.dumps(p["revision"], indent=2, + default=str))) + return 0, payload + + +def cmd_knowledge_decisions(args, out) -> Tuple[int, Dict[str, Any]]: + payload = _ok(_knowledge().list_decisions( + args.workspace, target_kind=getattr(args, "target_kind", None), + target_id=getattr(args, "target", None), + decision_type=getattr(args, "type", None), limit=args.limit, + offset=args.offset)) + + def human(p: Dict[str, Any]) -> None: + for decision in p["decisions"]: + print(f" {decision['id']} {decision['decision_type']:22} " + f"{decision['target_kind']}:{decision['target_id']} " + f"actor={decision['actor'] or '-'}") + print(f"{p['count']} decision(s)") + + out.emit(payload, human) + return 0, payload + + +# --------------------------------------------------------------------------- +# bundle commands +# --------------------------------------------------------------------------- +def cmd_bundle_export(args, out) -> Tuple[int, Dict[str, Any]]: + from .knowledge.bundle import export_bundle + from .runtime import get_runtime + get_runtime() # bootstrap: migrations must have run + current_only = not bool(getattr(args, "include_history", False)) + manifest = export_bundle( + args.workspace, args.output, current_only=current_only, + knowledge_revision=getattr(args, "knowledge_revision", None), + generated_at=getattr(args, "generated_at", None) or "") + payload = _ok({"manifest": manifest, "output": args.output}) + + def human(p: Dict[str, Any]) -> None: + m = p["manifest"] + print(f"bundle {m['bundleSchemaVersion']} -> {p['output']}") + print(f"knowledge revision: {m['knowledgeRevision']}") + for key, value in sorted(m["counts"].items()): + print(f" {key:20} {value}") + for warning in m.get("warnings", []): + print(f" warning: {warning}") + + out.emit(payload, human) + return 0, payload + + +# --------------------------------------------------------------------------- +# registration +# --------------------------------------------------------------------------- +def _ws(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--workspace", required=True, help="workspace id") + + +def _actor_note_flags(parser: argparse.ArgumentParser, + require: bool = True) -> None: + parser.add_argument("--actor", required=require, + help="who is making this governance decision " + "(recorded verbatim, never inferred)") + parser.add_argument("--note", required=require, + help="why (bounded; recorded on the Human Decision)") + + +def _paging(parser: argparse.ArgumentParser, limit: int = 100) -> None: + parser.add_argument("--limit", type=int, default=limit, metavar="N") + parser.add_argument("--offset", type=int, default=0, metavar="N") + + +def register(sub: argparse._SubParsersAction, + common: argparse.ArgumentParser, + knowledge_sub: argparse._SubParsersAction) -> None: + """Attach the Phase 5 command groups. ``knowledge_sub`` is the EXISTING + ``knowledge`` group's subparsers — history commands are added beside the + Phase 3 ``knowledge search`` without touching it.""" + # -- graph ------------------------------------------------------------- + graph = sub.add_parser("graph", parents=[common], + help="canonical knowledge-graph queries and " + "deterministic projection") + graph_sub = graph.add_subparsers(dest="graph_command", + metavar="") + + g_stats = graph_sub.add_parser("stats", parents=[common], + help="graph statistics + current " + "knowledge revision") + _ws(g_stats) + g_stats.set_defaults(func=cmd_graph_stats) + + g_seed = graph_sub.add_parser( + "seed", parents=[common], + help="deterministic graph seed (model-free); `graph seed plan` " + "for the dry run") + # NOT required at this level: `graph seed plan` satisfies --workspace on + # the sub-subparser, and argparse would otherwise demand it twice. The + # handler enforces it for the bare `graph seed` form. + g_seed.add_argument("--workspace", help="workspace id") + g_seed.add_argument("--actor", default="", help="recorded actor") + g_seed.set_defaults(func=cmd_graph_seed) + g_seed_sub = g_seed.add_subparsers(dest="seed_command", + metavar="") + g_seed_plan = g_seed_sub.add_parser("plan", parents=[common], + help="dry-run: what a seed would " + "change (no write)") + _ws(g_seed_plan) + g_seed_plan.set_defaults(func=cmd_graph_seed_plan) + + g_sync = graph_sub.add_parser("sync", parents=[common], + help="incremental deterministic sync " + "(unchanged source = no-op)") + _ws(g_sync) + g_sync.add_argument("--actor", default="", help="recorded actor") + g_sync.set_defaults(func=cmd_graph_sync) + + g_reconcile = graph_sub.add_parser("reconcile", parents=[common], + help="graph staleness " + "reconciliation") + _ws(g_reconcile) + g_reconcile.add_argument("--actor", default="", help="recorded actor") + g_reconcile.set_defaults(func=cmd_graph_reconcile) + + g_search = graph_sub.add_parser("search", parents=[common], + help="search canonical entities and " + "claims") + _ws(g_search) + g_search.add_argument("--query", required=True) + g_search.add_argument("--limit", type=int, default=20, metavar="N") + g_search.add_argument("--include-stale", dest="include_stale", + action="store_true") + g_search.set_defaults(func=cmd_graph_search) + + g_node = graph_sub.add_parser("node", parents=[common], + help="one graph node in the stable read " + "shape") + _ws(g_node) + g_node.add_argument("--node", required=True, help="node id (entity/" + "claim/asset/revision/segment/evidence)") + g_node.set_defaults(func=cmd_graph_node) + + g_expand = graph_sub.add_parser("expand", parents=[common], + help="bounded BFS expansion around one " + "entity") + _ws(g_expand) + g_expand.add_argument("--node", required=True) + g_expand.add_argument("--depth", type=int, default=2) + g_expand.add_argument("--direction", default="both", + choices=["outgoing", "incoming", "both"]) + g_expand.add_argument("--types", help="comma-separated relation types") + g_expand.add_argument("--include-stale", dest="include_stale", + action="store_true") + g_expand.set_defaults(func=cmd_graph_expand) + + g_path = graph_sub.add_parser( + "path", parents=[common], + help="bounded shortest-path discovery (generic reachability, not " + "formal traceability)") + _ws(g_path) + g_path.add_argument("--from", required=True, help="source entity id") + g_path.add_argument("--to", required=True, help="target entity id") + g_path.add_argument("--max-depth", dest="max_depth", type=int, default=6) + g_path.add_argument("--direction", default="both", + choices=["outgoing", "incoming", "both"]) + g_path.add_argument("--types", help="comma-separated relation types") + g_path.add_argument("--include-stale", dest="include_stale", + action="store_true") + g_path.set_defaults(func=cmd_graph_path) + graph.set_defaults(func=None, _parser=graph) + + # -- promotion --------------------------------------------------------- + promotion = sub.add_parser( + "promotion", parents=[common], + help="explicit candidate promotion (review never promotes)") + promotion_sub = promotion.add_subparsers(dest="promotion_command", + metavar="") + + p_plan = promotion_sub.add_parser("plan", parents=[common], + help="deterministic promotion " + "dry-run (no write)") + _ws(p_plan) + p_plan.add_argument("--candidate", required=True, + help="semantic candidate id (sc_...)") + p_plan.set_defaults(func=cmd_promotion_plan) + + p_promote = promotion_sub.add_parser( + "promote", parents=[common], + help="promote one confirmed, active, verified candidate") + _ws(p_promote) + p_promote.add_argument("--candidate", required=True) + _actor_note_flags(p_promote) + p_promote.set_defaults(func=cmd_promotion_promote) + + p_rplan = promotion_sub.add_parser("relation-plan", parents=[common], + help="relation-candidate promotion " + "dry-run") + _ws(p_rplan) + p_rplan.add_argument("--relation", required=True, + help="relation candidate id (sr_...)") + p_rplan.set_defaults(func=cmd_promotion_relation_plan) + + p_rpromote = promotion_sub.add_parser( + "promote-relation", parents=[common], + help="promote one confirmed relation candidate") + _ws(p_rpromote) + p_rpromote.add_argument("--relation", required=True) + _actor_note_flags(p_rpromote) + p_rpromote.set_defaults(func=cmd_promotion_promote_relation) + promotion.set_defaults(func=None, _parser=promotion) + + # -- entity ------------------------------------------------------------ + entity = sub.add_parser("entity", parents=[common], + help="canonical engineering entities") + entity_sub = entity.add_subparsers(dest="entity_command", + metavar="") + + e_list = entity_sub.add_parser("list", parents=[common], + help="list entities (bounded)") + _ws(e_list) + e_list.add_argument("--type", help="filter by entity type") + e_list.add_argument("--lifecycle", default="active", + help="lifecycle filter (default active; '' for all)") + e_list.add_argument("--origin", help="filter by origin") + _paging(e_list) + e_list.set_defaults(func=cmd_entity_list) + + e_show = entity_sub.add_parser("show", parents=[common], + help="one entity with aliases, bindings, " + "claims and relations") + _ws(e_show) + e_show.add_argument("--entity", required=True) + e_show.set_defaults(func=cmd_entity_show) + + e_create = entity_sub.add_parser( + "create", parents=[common], + help="manual entity (requires evidence, actor and note)") + _ws(e_create) + e_create.add_argument("--type", required=True, help="entity type") + e_create.add_argument("--key", required=True, help="canonical key") + e_create.add_argument("--name", required=True, help="display name") + e_create.add_argument("--description", default="") + e_create.add_argument("--evidence", action="append", metavar="EVID", + help="evidence id (repeatable)") + e_create.add_argument("--evidence-json", dest="evidence_json", + help="JSON array of {evidence_id, quote?, role?}") + _actor_note_flags(e_create) + e_create.set_defaults(func=cmd_entity_create) + + e_alias_add = entity_sub.add_parser("alias-add", parents=[common], + help="add an alias (collisions are " + "reported, never silent)") + _ws(e_alias_add) + e_alias_add.add_argument("--entity", required=True) + e_alias_add.add_argument("--alias", required=True) + e_alias_add.add_argument("--type", default="manual", help="alias type") + e_alias_add.add_argument("--evidence", help="optional evidence id") + _actor_note_flags(e_alias_add) + e_alias_add.set_defaults(func=cmd_entity_alias_add) + + e_alias_remove = entity_sub.add_parser("alias-remove", parents=[common], + help="remove an alias " + "(kept auditable)") + _ws(e_alias_remove) + e_alias_remove.add_argument("--entity", required=True) + e_alias_remove.add_argument("--alias", required=True) + _actor_note_flags(e_alias_remove) + e_alias_remove.set_defaults(func=cmd_entity_alias_remove) + + e_merge = entity_sub.add_parser("merge", parents=[common], + help="merge source entity into target " + "(source stays addressable)") + _ws(e_merge) + e_merge.add_argument("--source", required=True, help="source entity id") + e_merge.add_argument("--target", required=True, help="target entity id") + _actor_note_flags(e_merge) + e_merge.set_defaults(func=cmd_entity_merge) + + e_split = entity_sub.add_parser( + "split", parents=[common], + help="explicit split: the caller lists exactly what moves") + _ws(e_split) + e_split.add_argument("--source", required=True, help="source entity id") + e_split.add_argument("--new-type", dest="new_type", required=True) + e_split.add_argument("--new-key", dest="new_key", required=True) + e_split.add_argument("--new-name", dest="new_name", default="") + e_split.add_argument("--claim", action="append", metavar="CLAIM_ID", + help="claim id to move (repeatable)") + e_split.add_argument("--binding", action="append", metavar="BINDING_ID", + help="binding id to move (repeatable)") + e_split.add_argument("--rewrite", action="append", + metavar="RELATION_ID:END", + help="repoint a relation endpoint " + "(END = source|target; repeatable)") + _actor_note_flags(e_split) + e_split.set_defaults(func=cmd_entity_split) + + e_authority = entity_sub.add_parser("authority", parents=[common], + help="explicit authority marking") + _ws(e_authority) + e_authority.add_argument("--entity", required=True) + e_authority.add_argument("--status", required=True, + help="authoritative / non-authoritative / " + "informational / unknown") + _actor_note_flags(e_authority) + e_authority.set_defaults(func=_cmd_authority("entity")) + + e_supersede = entity_sub.add_parser("supersede", parents=[common], + help="mark superseded by a " + "replacement entity") + _ws(e_supersede) + e_supersede.add_argument("--entity", required=True) + e_supersede.add_argument("--by", required=True, + help="replacement entity id") + _actor_note_flags(e_supersede) + e_supersede.set_defaults(func=_cmd_supersede("entity")) + + e_withdraw = entity_sub.add_parser("withdraw", parents=[common], + help="withdraw (history preserved)") + _ws(e_withdraw) + e_withdraw.add_argument("--entity", required=True) + _actor_note_flags(e_withdraw) + e_withdraw.set_defaults(func=_cmd_withdraw("entity")) + entity.set_defaults(func=None, _parser=entity) + + # -- claim ------------------------------------------------------------- + claim = sub.add_parser("claim", parents=[common], + help="canonical claims") + claim_sub = claim.add_subparsers(dest="claim_command", + metavar="") + + c_list = claim_sub.add_parser("list", parents=[common], + help="list claims (bounded)") + _ws(c_list) + c_list.add_argument("--entity", help="filter by entity id") + c_list.add_argument("--type", help="filter by claim type") + c_list.add_argument("--lifecycle", default="active") + _paging(c_list) + c_list.set_defaults(func=cmd_claim_list) + + c_show = claim_sub.add_parser("show", parents=[common], + help="one claim with its evidence") + _ws(c_show) + c_show.add_argument("--claim", required=True) + c_show.set_defaults(func=cmd_claim_show) + + c_create = claim_sub.add_parser( + "create", parents=[common], + help="manual claim (evidence required; quotes verified)") + _ws(c_create) + c_create.add_argument("--entity", required=True) + c_create.add_argument("--type", required=True, help="claim type") + c_create.add_argument("--statement", required=True) + c_create.add_argument("--evidence", action="append", metavar="EVID") + c_create.add_argument("--evidence-json", dest="evidence_json") + _actor_note_flags(c_create) + c_create.set_defaults(func=cmd_claim_create) + + c_authority = claim_sub.add_parser("authority", parents=[common], + help="explicit authority marking") + _ws(c_authority) + c_authority.add_argument("--claim", required=True) + c_authority.add_argument("--status", required=True) + _actor_note_flags(c_authority) + c_authority.set_defaults(func=_cmd_authority("claim")) + + c_supersede = claim_sub.add_parser("supersede", parents=[common], + help="supersede by a replacement " + "claim") + _ws(c_supersede) + c_supersede.add_argument("--claim", required=True) + c_supersede.add_argument("--by", required=True) + _actor_note_flags(c_supersede) + c_supersede.set_defaults(func=_cmd_supersede("claim")) + + c_withdraw = claim_sub.add_parser("withdraw", parents=[common], + help="withdraw (history preserved)") + _ws(c_withdraw) + c_withdraw.add_argument("--claim", required=True) + _actor_note_flags(c_withdraw) + c_withdraw.set_defaults(func=_cmd_withdraw("claim")) + claim.set_defaults(func=None, _parser=claim) + + # -- relation ---------------------------------------------------------- + relation = sub.add_parser("relation", parents=[common], + help="canonical relations") + relation_sub = relation.add_subparsers(dest="relation_command", + metavar="") + + r_list = relation_sub.add_parser("list", parents=[common], + help="list relations (bounded)") + _ws(r_list) + r_list.add_argument("--entity", help="either endpoint") + r_list.add_argument("--type", help="relation type") + r_list.add_argument("--state", help="relation state") + r_list.add_argument("--lifecycle", default="active") + _paging(r_list) + r_list.set_defaults(func=cmd_relation_list) + + r_show = relation_sub.add_parser("show", parents=[common], + help="one relation with evidence") + _ws(r_show) + r_show.add_argument("--relation", required=True) + r_show.set_defaults(func=cmd_relation_show) + + r_create = relation_sub.add_parser( + "create", parents=[common], + help="manual relation (explicit|confirmed only; evidence required)") + _ws(r_create) + r_create.add_argument("--source", required=True) + r_create.add_argument("--target", required=True) + r_create.add_argument("--type", required=True) + r_create.add_argument("--state", default="confirmed", + choices=["explicit", "confirmed"]) + r_create.add_argument("--confidence", default="medium") + r_create.add_argument("--evidence", action="append", metavar="EVID") + r_create.add_argument("--evidence-json", dest="evidence_json") + _actor_note_flags(r_create) + r_create.set_defaults(func=cmd_relation_create) + + r_authority = relation_sub.add_parser("authority", parents=[common], + help="explicit authority marking") + _ws(r_authority) + r_authority.add_argument("--relation", required=True) + r_authority.add_argument("--status", required=True) + _actor_note_flags(r_authority) + r_authority.set_defaults(func=_cmd_authority("relation")) + + r_reject = relation_sub.add_parser("reject", parents=[common], + help="reject (kept as governance " + "history)") + _ws(r_reject) + r_reject.add_argument("--relation", required=True) + _actor_note_flags(r_reject) + r_reject.set_defaults(func=cmd_relation_reject) + + r_restore = relation_sub.add_parser("restore", parents=[common], + help="restore a rejected relation") + _ws(r_restore) + r_restore.add_argument("--relation", required=True) + _actor_note_flags(r_restore) + r_restore.set_defaults(func=cmd_relation_restore) + + r_supersede = relation_sub.add_parser("supersede", parents=[common], + help="supersede by a replacement " + "relation") + _ws(r_supersede) + r_supersede.add_argument("--relation", required=True) + r_supersede.add_argument("--by", required=True) + _actor_note_flags(r_supersede) + r_supersede.set_defaults(func=_cmd_supersede("relation")) + + r_withdraw = relation_sub.add_parser("withdraw", parents=[common], + help="withdraw (history preserved)") + _ws(r_withdraw) + r_withdraw.add_argument("--relation", required=True) + _actor_note_flags(r_withdraw) + r_withdraw.set_defaults(func=_cmd_withdraw("relation")) + relation.set_defaults(func=None, _parser=relation) + + # -- knowledge history (beside the existing `knowledge search`) -------- + k_revisions = knowledge_sub.add_parser( + "revisions", parents=[common], + help="the workspace's knowledge revision ledger") + _ws(k_revisions) + _paging(k_revisions, limit=50) + k_revisions.set_defaults(func=cmd_knowledge_revisions) + + k_revision = knowledge_sub.add_parser( + "revision", parents=[common], help="one knowledge revision with " + "its decisions") + _ws(k_revision) + k_revision.add_argument("--number", type=int, required=True) + k_revision.set_defaults(func=cmd_knowledge_revision) + + k_decisions = knowledge_sub.add_parser( + "decisions", parents=[common], + help="the immutable human-decision audit") + _ws(k_decisions) + k_decisions.add_argument("--target-kind", dest="target_kind") + k_decisions.add_argument("--target", help="target object id") + k_decisions.add_argument("--type", help="decision type") + _paging(k_decisions) + k_decisions.set_defaults(func=cmd_knowledge_decisions) + + # -- bundle ------------------------------------------------------------ + bundle = sub.add_parser("bundle", parents=[common], + help="Knowledge Bundle 2.0 Draft export") + bundle_sub = bundle.add_subparsers(dest="bundle_command", + metavar="") + b_export = bundle_sub.add_parser( + "export", parents=[common], + help="export the workspace's canonical knowledge as a " + "2.0.0-draft.1 bundle (separate from .openmind 1.x)") + _ws(b_export) + b_export.add_argument("--output", required=True, + help="directory to write (e.g. ./.openmind-v2)") + mode = b_export.add_mutually_exclusive_group() + mode.add_argument("--current-only", dest="current_only", + action="store_true", + help="active objects only (the default)") + mode.add_argument("--include-history", dest="include_history", + action="store_true", + help="include stale/superseded/withdrawn/merged " + "history") + b_export.add_argument("--knowledge-revision", dest="knowledge_revision", + type=int, metavar="N", + help="cap records at creation revision N " + "(stamp filter; not point-in-time state)") + b_export.add_argument("--generated-at", dest="generated_at", + metavar="ISO8601", + help="override generatedAt (reproducible builds)") + b_export.set_defaults(func=cmd_bundle_export) + bundle.set_defaults(func=None, _parser=bundle) + + +__all__ = ["register"] diff --git a/openmind/db.py b/openmind/db.py index 364245b..5891a52 100644 --- a/openmind/db.py +++ b/openmind/db.py @@ -144,6 +144,16 @@ def delete_project(project_id: str) -> None: # so the cleanup is correct even if a future caller has the pragma off, # and so the intent is visible beside the other per-project deletes. _c().execute("DELETE FROM assets WHERE workspace_id=?", (project_id,)) + # Canonical knowledge graph (v0006): entity deletion cascades to + # aliases/bindings/claims/relations/evidence joins; the ledger tables + # carry no FK and are removed explicitly. + _c().execute("DELETE FROM engineering_entities WHERE workspace_id=?", + (project_id,)) + for graph_table in ("knowledge_decisions", "knowledge_revisions", + "knowledge_promotions", + "knowledge_projection_state"): + _c().execute(f"DELETE FROM {graph_table} WHERE workspace_id=?", + (project_id,)) _c().execute("DELETE FROM projects WHERE id=?", (project_id,)) _c().execute("DELETE FROM jobs WHERE project_id=?", (project_id,)) _c().execute("DELETE FROM file_index WHERE project_id=?", (project_id,)) diff --git a/openmind/jobs.py b/openmind/jobs.py index 17195f4..6b2f334 100644 --- a/openmind/jobs.py +++ b/openmind/jobs.py @@ -476,7 +476,11 @@ def _run_lens_induction(job_id: str) -> None: def reconcile_semantic_staleness(project_id: str) -> None: """Post-commit staleness hook (spec §22): mark semantic candidates whose source revisions moved on as stale. Best-effort — a reconciliation - failure must never fail the ingest that triggered it.""" + failure must never fail the ingest that triggered it. + + The canonical graph reconciles right after (v2 Phase 5), in this order + on purpose: candidate staleness first, then graph staleness, so graph + statistics computed afterwards see both planes settled.""" try: from .semantic import store as semantic_store result = semantic_store.reconcile_staleness(project_id) @@ -486,6 +490,16 @@ def reconcile_semantic_staleness(project_id: str) -> None: except Exception as exc: print(f"[semantic] staleness reconciliation failed for " f"{project_id}: {exc}", flush=True) + try: + from .knowledge.reconciliation import reconcile_graph_staleness + graph_result = reconcile_graph_staleness(project_id) + if graph_result.get("changed"): + print(f"[knowledge] graph staleness reconciled for " + f"{project_id}: revision " + f"{graph_result.get('knowledge_revision')}", flush=True) + except Exception as exc: + print(f"[knowledge] graph staleness reconciliation failed for " + f"{project_id}: {exc}", flush=True) def enqueue_ask(scope_id: str, pids: List[str], project_id: str, @@ -596,6 +610,8 @@ def terminate_project(project_id: str, clear_cases: bool = False) -> Dict[str, A vectorstore.get_code_store(project_id).drop(cancel=_shutdown.is_set) vectorstore.drop_collection(vectorstore.documents_collection_name(project_id), cancel=_shutdown.is_set) + vectorstore.drop_collection(vectorstore.knowledge_collection_name(project_id), + cancel=_shutdown.is_set) # 2) delete map/* and docs/* for sub in ("map", "docs"): d = config.project_dir(project_id) / sub @@ -616,6 +632,16 @@ def terminate_project(project_id: str, clear_cases: bool = False) -> Dict[str, A # source-file removal, which only marks an Asset removed and keeps its history. db.clear_workspace_assets(project_id) content_store.clear_workspace(project_id) + # 3c) wipe the canonical knowledge graph (v2 Phase 5). Terminate is a + # full wipe of LEARNED data back to init; graph knowledge is anchored to + # the Asset model being wiped above, so keeping it would leave every + # binding and evidence join dangling. + try: + from .knowledge import store as knowledge_store + knowledge_store.clear_workspace_graph(project_id) + except Exception as exc: + print(f"[terminate] knowledge graph wipe failed for {project_id}: " + f"{exc}", flush=True) # cases: keep but flag stale, unless clearing if clear_cases: cases.clear_cases(project_id) diff --git a/openmind/knowledge/__init__.py b/openmind/knowledge/__init__.py new file mode 100644 index 0000000..c64448d --- /dev/null +++ b/openmind/knowledge/__init__.py @@ -0,0 +1,33 @@ +"""The canonical Engineering Knowledge Graph (OpenMind v2 Phase 5). + +Everything in this package operates on the canonical graph tables created by +migration v0006: Entities, Claims, Relations, aliases, bindings, Evidence +joins, Human Decisions, Knowledge Revisions, promotion records and the +deterministic projection state. + +Boundaries this package holds: + +* graph truth enters ONLY through deterministic projection, explicit manual + creation, or explicit Candidate/Relation-Candidate promotion — never from a + review action, never from ingestion, never automatically; +* every active Claim carries Evidence; every Relation carries provenance; +* every governance write records a Human Decision and one Knowledge Revision; +* Phase 4 Candidates stay in their own tables — a promoted Candidate remains + queryable and the graph stores only its id as provenance; +* nothing here calls a semantic provider; the projector is model-free. + +Import cost: importing this package pulls no provider SDK, no vector store +and no database connection — modules bootstrap lazily like the rest of the +runtime. +""" +from __future__ import annotations + +#: Versions the deterministic projection rules. Bump when a projection rule +#: changes so `sync` knows stored state was produced by older rules. +GRAPH_PROJECTOR_VERSION = "1" + +#: Recorded on every promotion (knowledge_promotions.policy_version) so a +#: later phase can tell which eligibility rules admitted a promotion. +PROMOTION_POLICY_VERSION = "1" + +__all__ = ["GRAPH_PROJECTOR_VERSION", "PROMOTION_POLICY_VERSION"] diff --git a/openmind/knowledge/bundle.py b/openmind/knowledge/bundle.py new file mode 100644 index 0000000..9511cf3 --- /dev/null +++ b/openmind/knowledge/bundle.py @@ -0,0 +1,359 @@ +"""Knowledge Bundle 2.0 Draft export. + +A SEPARATE contract from the frozen ``.openmind`` 1.x artifact: its own +command (``openmind bundle export``), its own directory (``.openmind-v2``), +its own schema version (``2.0.0-draft.1``). Draft means draft — the layout +may change until the freeze, and there is no import. + +GUARANTEES (spec §31, verified by openmind.bundle_verify and the tests) +----------------------------------------------------------------------- +deterministic ordering (stable sort key per file); UTF-8 JSONL with LF; +workspace-relative locators only; no absolute paths; no provider secrets, +profiles, prompts, raw model responses or semantic cache; every ACTIVE claim +carries evidence; every relation endpoint and relation-evidence row resolves +inside the bundle; the manifest records runtime version, bundle schema +version, workspace, knowledge revision, timestamp, per-file SHA-256 hashes, +record counts and honest warnings. + +KNOWN DRAFT LIMITATION: ``--knowledge-revision N`` filters records by their +created/updated revision stamps (created <= N). It does NOT reconstruct +point-in-time lifecycle states — an object superseded after revision N still +reports its current lifecycle. The manifest says so. +""" +from __future__ import annotations + +import hashlib +import json +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence + +from .. import db +from ..version import RUNTIME_VERSION +from . import store +from .reconciliation import reconcile_graph_staleness +from .vocabularies import GraphLifecycleStatus + +BUNDLE_SCHEMA_VERSION = "2.0.0-draft.1" + +#: Row caps per file — a bundle is an export, not a database dump. Exceeding +#: a cap truncates WITH a manifest warning, never silently. +MAX_ROWS_PER_FILE = 200_000 + +_ACTIVE = GraphLifecycleStatus.ACTIVE + + +def _now_iso() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()) + + +def _jsonl(records: Sequence[Dict[str, Any]]) -> str: + lines = [json.dumps(r, ensure_ascii=False, sort_keys=True, + separators=(",", ":")) for r in records] + return "\n".join(lines) + ("\n" if lines else "") + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _sorted(records: List[Dict[str, Any]], *keys: str + ) -> List[Dict[str, Any]]: + return sorted(records, key=lambda r: tuple(str(r.get(k, "") or "") + for k in keys)) + + +def export_bundle(workspace_id: str, output_dir: str, *, + current_only: bool = True, + knowledge_revision: Optional[int] = None, + generated_at: str = "") -> Dict[str, Any]: + """Write the bundle directory. Returns the manifest.""" + workspace = db.get_project(workspace_id) + if not workspace: + from ..domain.errors import WorkspaceNotFound + raise WorkspaceNotFound(workspace_id) + if current_only: + # A current-only export must not present knowledge whose sources + # moved on as current (spec §22). + reconcile_graph_staleness(workspace_id) + + warnings: List[str] = [] + revision_cap = (int(knowledge_revision) + if knowledge_revision is not None else None) + + def lifecycle_ok(row: Dict[str, Any]) -> bool: + if revision_cap is not None and \ + int(row.get("created_knowledge_revision") or 0) > revision_cap: + return False + if current_only: + return row.get("lifecycle_status") == _ACTIVE + return True + + def cap(records: List[Dict[str, Any]], name: str + ) -> List[Dict[str, Any]]: + if len(records) > MAX_ROWS_PER_FILE: + warnings.append( + f"{name}: truncated to {MAX_ROWS_PER_FILE} of " + f"{len(records)} records") + return records[:MAX_ROWS_PER_FILE] + return records + + # -- graph rows ---------------------------------------------------------- + entities = [e for e in store.list_entities(workspace_id, + lifecycle_status=None, + limit=MAX_ROWS_PER_FILE + 1) + if lifecycle_ok(e)] + entity_ids = {e["id"] for e in entities} + + claims: List[Dict[str, Any]] = [] + claim_evidence: List[Dict[str, Any]] = [] + for claim in store.list_claims(workspace_id, lifecycle_status=None, + limit=MAX_ROWS_PER_FILE + 1): + if not lifecycle_ok(claim) or claim["entity_id"] not in entity_ids: + continue + claims.append(claim) + for ev in store.claim_evidence(claim["id"]): + claim_evidence.append({"claim_id": claim["id"], **ev}) + claim_ids = {c["id"] for c in claims} + + relations: List[Dict[str, Any]] = [] + relation_evidence: List[Dict[str, Any]] = [] + for relation in store.list_relations(workspace_id, lifecycle_status=None, + limit=MAX_ROWS_PER_FILE + 1): + if not lifecycle_ok(relation): + continue + if relation["source_entity_id"] not in entity_ids or \ + relation["target_entity_id"] not in entity_ids: + # Endpoint filtered out (revision cap / current-only): the + # relation cannot be represented whole, so it is excluded. + continue + relations.append(relation) + for ev in store.relation_evidence(relation["id"]): + relation_evidence.append({"relation_id": relation["id"], **ev}) + + aliases: List[Dict[str, Any]] = [] + bindings: List[Dict[str, Any]] = [] + for entity in entities: + for alias in store.list_aliases(workspace_id, entity["id"], + status=None): + if current_only and alias["status"] != "active": + continue + if revision_cap is not None and \ + int(alias.get("created_knowledge_revision") or 0) > \ + revision_cap: + continue + aliases.append(alias) + for binding in store.list_bindings(workspace_id, entity["id"]): + if current_only and binding["status"] != "active": + continue + if revision_cap is not None and \ + int(binding.get("created_knowledge_revision") or 0) > \ + revision_cap: + continue + bindings.append(binding) + + revisions_ledger = [r for r in store.list_revisions( + workspace_id, limit=MAX_ROWS_PER_FILE + 1) + if revision_cap is None or r["revision_number"] <= revision_cap] + decision_rows = [d for d in store.list_decisions( + workspace_id, limit=MAX_ROWS_PER_FILE + 1)] + if revision_cap is not None: + ledger_ids = {r["id"] for r in revisions_ledger} + decision_rows = [d for d in decision_rows + if d["knowledge_revision_id"] in ledger_ids] + + # -- source plane -------------------------------------------------------- + assets = db.list_assets(workspace_id, state=None, + limit=MAX_ROWS_PER_FILE + 1) + if current_only: + assets = [a for a in assets if a["state"] == "active"] + asset_rows = [dict(a) for a in assets] + + revision_rows: List[Dict[str, Any]] = [] + segment_rows: List[Dict[str, Any]] = [] + evidence_rows: List[Dict[str, Any]] = [] + seen_evidence: set = set() + for asset in assets: + asset_revisions = db.list_revisions(workspace_id, asset["id"], + limit=1000) + if current_only: + asset_revisions = [r for r in asset_revisions + if r["id"] == asset.get( + "current_revision_id")] + for revision in asset_revisions: + revision_rows.append({**revision, "asset_id": asset["id"]}) + for segment in db.list_segments(workspace_id, revision["id"], + limit=5000): + segment_rows.append(segment) + evidence = db.get_evidence_for_segment(workspace_id, + segment["id"]) + if evidence and evidence["id"] not in seen_evidence: + seen_evidence.add(evidence["id"]) + evidence_rows.append(evidence) + # Evidence cited by exported claims/relations must resolve inside the + # bundle even when its revision fell outside the source-plane slice. + for join in claim_evidence + relation_evidence: + evidence_id = join["evidence_id"] + if evidence_id in seen_evidence: + continue + record = db.get_evidence(workspace_id, evidence_id) + if record: + seen_evidence.add(evidence_id) + evidence_rows.append(record) + else: + warnings.append( + f"evidence {evidence_id} cited by the graph no longer " + f"resolves in this workspace") + + # -- lenses (definitions only; provider provenance names are stripped) --- + from ..semantic import store as semantic_store + lens_rows = [] + for lens in semantic_store.list_lenses(workspace_id, limit=1000): + lens_rows.append({ + "id": lens["id"], "name": lens["name"], + "version": lens["version"], "source": lens["source"], + "status": lens["status"], + "schema_version": lens["schema_version"], + "definition": lens["definition"], + "created_at": lens["created_at"], + }) + + # -- write --------------------------------------------------------------- + out = Path(output_dir) + out.mkdir(parents=True, exist_ok=True) + (out / "schemas").mkdir(exist_ok=True) + + files: Dict[str, str] = { + "workspace.json": json.dumps({ + "id": workspace["id"], "name": workspace["name"], + "created_at": workspace["created_at"], + }, ensure_ascii=False, sort_keys=True, indent=2) + "\n", + "assets.jsonl": _jsonl(cap(_sorted(asset_rows, "logical_key", "id"), + "assets")), + "revisions.jsonl": _jsonl(cap(_sorted(revision_rows, "asset_id", + "id"), "revisions")), + "segments.jsonl": _jsonl(cap(_sorted(segment_rows, "revision_id", + "id"), "segments")), + "evidence.jsonl": _jsonl(cap(_sorted(evidence_rows, "id"), + "evidence")), + "entities.jsonl": _jsonl(cap(_sorted(entities, "canonical_key", + "id"), "entities")), + "aliases.jsonl": _jsonl(cap(_sorted(aliases, "entity_id", + "normalized_alias", "id"), + "aliases")), + "bindings.jsonl": _jsonl(cap(_sorted(bindings, "entity_id", + "ref_kind", "ref_id", "id"), + "bindings")), + "claims.jsonl": _jsonl(cap(_sorted(claims, "entity_id", "id"), + "claims")), + "claim-evidence.jsonl": _jsonl(cap(_sorted( + claim_evidence, "claim_id", "evidence_id", "quote_hash"), + "claim-evidence")), + "relations.jsonl": _jsonl(cap(_sorted( + relations, "relation_type", "source_entity_id", + "target_entity_id", "id"), "relations")), + "relation-evidence.jsonl": _jsonl(cap(_sorted( + relation_evidence, "relation_id", "evidence_id", "quote_hash"), + "relation-evidence")), + "decisions.jsonl": _jsonl(cap(_sorted(decision_rows, "created_at", + "id"), "decisions")), + "knowledge-revisions.jsonl": _jsonl(cap(sorted( + revisions_ledger, key=lambda r: int(r["revision_number"])), + "knowledge-revisions")), + "lenses.jsonl": _jsonl(cap(_sorted(lens_rows, "name", "id"), + "lenses")), + } + files.update(_schemas()) + + file_meta: Dict[str, Dict[str, Any]] = {} + for name, content in files.items(): + data = content.encode("utf-8") + path = out / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + records = (content.count("\n") if name.endswith(".jsonl") else None) + file_meta[name] = {"sha256": _sha256(data)} + if records is not None: + file_meta[name]["records"] = records + + manifest = { + "bundleSchemaVersion": BUNDLE_SCHEMA_VERSION, + "generator": {"name": "open-mind", "version": RUNTIME_VERSION}, + "workspaceId": workspace["id"], + "workspaceName": workspace["name"], + "knowledgeRevision": store.current_revision_number(workspace_id), + "generatedAt": generated_at or _now_iso(), + "mode": {"currentOnly": bool(current_only), + "knowledgeRevisionCap": revision_cap}, + "counts": { + "assets": len(asset_rows), "revisions": len(revision_rows), + "segments": len(segment_rows), "evidence": len(evidence_rows), + "entities": len(entities), "aliases": len(aliases), + "bindings": len(bindings), "claims": len(claims), + "claimEvidence": len(claim_evidence), + "relations": len(relations), + "relationEvidence": len(relation_evidence), + "decisions": len(decision_rows), + "knowledgeRevisions": len(revisions_ledger), + "lenses": len(lens_rows), + }, + "files": dict(sorted(file_meta.items())), + "warnings": warnings, + "limitations": [ + "draft schema: layout may change until the 2.0.0 freeze", + "knowledge-revision cap filters by creation revision stamps; " + "it does not reconstruct point-in-time lifecycle states", + ], + } + (out / "manifest.json").write_bytes(json.dumps( + manifest, ensure_ascii=False, sort_keys=True, + indent=2).encode("utf-8") + b"\n") + return manifest + + +def _schemas() -> Dict[str, str]: + """Minimal draft JSON Schemas shipped inside the bundle.""" + def schema(title: str, required: List[str]) -> str: + return json.dumps({ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": title, "type": "object", + "required": required, + "additionalProperties": True, + }, ensure_ascii=False, sort_keys=True, indent=2) + "\n" + + return { + "schemas/manifest.schema.json": schema( + "KnowledgeBundleManifest", + ["bundleSchemaVersion", "workspaceId", "knowledgeRevision", + "generatedAt", "files", "counts"]), + "schemas/entity.schema.json": schema( + "EngineeringEntity", + ["id", "workspace_id", "entity_type", "canonical_key", + "lifecycle_status", "authority_status", "origin"]), + "schemas/claim.schema.json": schema( + "EngineeringClaim", + ["id", "workspace_id", "entity_id", "claim_type", "statement", + "lifecycle_status", "origin"]), + "schemas/relation.schema.json": schema( + "EngineeringRelation", + ["id", "workspace_id", "source_entity_id", "target_entity_id", + "relation_type", "relation_state", "lifecycle_status", + "origin"]), + "schemas/alias.schema.json": schema( + "EntityAlias", ["id", "entity_id", "alias", "normalized_alias", + "alias_type", "status"]), + "schemas/binding.schema.json": schema( + "EntityBinding", ["id", "entity_id", "ref_kind", "ref_id", + "binding_role", "status"]), + "schemas/decision.schema.json": schema( + "KnowledgeDecision", ["id", "workspace_id", "decision_type", + "target_kind", "created_at"]), + "schemas/knowledge-revision.schema.json": schema( + "KnowledgeRevision", ["id", "workspace_id", "revision_number", + "action", "created_at"]), + "schemas/evidence.schema.json": schema( + "Evidence", ["id", "revision_id", "locator"]), + } + + +__all__ = ["export_bundle", "BUNDLE_SCHEMA_VERSION"] diff --git a/openmind/knowledge/errors.py b/openmind/knowledge/errors.py new file mode 100644 index 0000000..ed733e3 --- /dev/null +++ b/openmind/knowledge/errors.py @@ -0,0 +1,177 @@ +"""Typed errors of the canonical Knowledge Graph. + +Same taxonomy discipline as :mod:`openmind.semantic.errors`: each failure a +caller must react to differently is its own class, all extending +:class:`openmind.domain.errors.OpenMindError` so the CLI and REST adapters +translate them like every other application error (machine-readable ``code``, +honest ``http_status``, stable ``exit_code``). +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from ..domain.errors import OpenMindError + + +class KnowledgeError(OpenMindError): + """Base class for every knowledge-graph error.""" + + code = "knowledge_error" + exit_code = 4 + http_status = 500 + + +class UnknownVocabularyValue(KnowledgeError): + """A write named a vocabulary member the graph does not define. The graph + never silently accepts an arbitrary (possibly model-generated) type.""" + + code = "unknown_vocabulary_value" + exit_code = 2 + http_status = 400 + + def __init__(self, *, field: str, value: Any, + allowed: Optional[List[str]] = None) -> None: + super().__init__( + f"unknown {field}: {str(value)!r}", + details={"field": field, "value": str(value), + "allowed": list(allowed or [])}) + self.field = field + self.value = value + + +class EntityNotFound(KnowledgeError): + code = "entity_not_found" + exit_code = 1 + http_status = 404 + + def __init__(self, entity_id: str, *, workspace_id: str = "") -> None: + details: Dict[str, Any] = {"entity_id": entity_id} + if workspace_id: + details["workspace_id"] = workspace_id + super().__init__(f"engineering entity not found: {entity_id!r}", + details=details) + self.entity_id = entity_id + + +class ClaimNotFound(KnowledgeError): + code = "claim_not_found" + exit_code = 1 + http_status = 404 + + def __init__(self, claim_id: str, *, workspace_id: str = "") -> None: + details: Dict[str, Any] = {"claim_id": claim_id} + if workspace_id: + details["workspace_id"] = workspace_id + super().__init__(f"engineering claim not found: {claim_id!r}", + details=details) + self.claim_id = claim_id + + +class RelationNotFound(KnowledgeError): + code = "relation_not_found" + exit_code = 1 + http_status = 404 + + def __init__(self, relation_id: str, *, workspace_id: str = "") -> None: + details: Dict[str, Any] = {"relation_id": relation_id} + if workspace_id: + details["workspace_id"] = workspace_id + super().__init__(f"engineering relation not found: {relation_id!r}", + details=details) + self.relation_id = relation_id + + +class GraphNodeNotFound(KnowledgeError): + code = "graph_node_not_found" + exit_code = 1 + http_status = 404 + + def __init__(self, node_id: str, *, workspace_id: str = "") -> None: + details: Dict[str, Any] = {"node_id": node_id} + if workspace_id: + details["workspace_id"] = workspace_id + super().__init__(f"graph node not found: {node_id!r}", details=details) + self.node_id = node_id + + +class KnowledgeRevisionNotFound(KnowledgeError): + code = "knowledge_revision_not_found" + exit_code = 1 + http_status = 404 + + def __init__(self, revision_number: int, *, workspace_id: str = "") -> None: + super().__init__( + f"knowledge revision not found: {revision_number}", + details={"revision_number": revision_number, + "workspace_id": workspace_id}) + self.revision_number = revision_number + + +class GraphEvidenceInvalid(KnowledgeError): + """A cited Evidence id does not exist in this Workspace, or a quote is + not a substring of the immutable Evidence content. The write is rejected + whole — a fabricated citation never enters canonical storage.""" + + code = "graph_evidence_invalid" + exit_code = 2 + http_status = 400 + + +class AliasCollision(KnowledgeError): + """The normalized alias is already held by a DIFFERENT active Entity. + Never silently attached; resolve by explicit merge or manual decision.""" + + code = "alias_collision" + exit_code = 1 + http_status = 409 + + def __init__(self, alias: str, holders: List[Dict[str, str]]) -> None: + super().__init__( + f"alias {alias!r} collides with an existing active entity alias", + details={"alias": alias, "holders": holders}) + self.alias = alias + self.holders = holders + + +class PromotionBlocked(KnowledgeError): + """The Candidate fails a promotion eligibility rule. The blocking + reasons are enumerated; nothing was written.""" + + code = "promotion_blocked" + exit_code = 1 + http_status = 409 + + def __init__(self, candidate_id: str, reasons: List[str]) -> None: + super().__init__( + f"candidate {candidate_id!r} cannot be promoted: " + + "; ".join(reasons), + details={"candidate_id": candidate_id, "blocking_reasons": reasons}) + self.candidate_id = candidate_id + self.reasons = reasons + + +class GraphConflict(KnowledgeError): + """A governance operation is illegal for the object's current state + (merging into a merged entity, superseding with a withdrawn object, + splitting objects that do not belong to the source, ...).""" + + code = "graph_conflict" + exit_code = 1 + http_status = 409 + + +class GraphLimitExceeded(KnowledgeError): + """A bounded traversal or listing was asked to exceed its hard cap.""" + + code = "graph_limit_exceeded" + exit_code = 2 + http_status = 400 + + +__all__ = [ + "KnowledgeError", "UnknownVocabularyValue", + "EntityNotFound", "ClaimNotFound", "RelationNotFound", + "GraphNodeNotFound", "KnowledgeRevisionNotFound", + "GraphEvidenceInvalid", "AliasCollision", "PromotionBlocked", + "GraphConflict", "GraphLimitExceeded", +] diff --git a/openmind/knowledge/graph.py b/openmind/knowledge/graph.py new file mode 100644 index 0000000..2e701ab --- /dev/null +++ b/openmind/knowledge/graph.py @@ -0,0 +1,497 @@ +"""Graph read surface: the node abstraction, bounded expansion, bounded +shortest-path discovery and bounded subgraph export. + +Traversal here is deterministic BFS in plain Python over indexed SQL reads — +no recursive SQL, no unbounded loops, every walk capped by hard limits. The +source-plane node kinds (asset / revision / segment / evidence) are +PROJECTED into the read shape from their canonical Phase 2 rows; nothing is +copied into graph tables to render it. + +Path discovery is generic graph reachability. It is deliberately NOT +labelled Requirement Traceability — the formal traceability engine is +Phase 6. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple + +from .. import db +from . import store +from .errors import GraphLimitExceeded, GraphNodeNotFound +from .vocabularies import (GraphLifecycleStatus, GraphNodeKind, + RelationState, RelationType, require) + +#: Hard ceilings (spec §28). Defaults below are lower. +MAX_EXPANSION_DEPTH = 4 +MAX_EXPANSION_NODES = 1000 +MAX_EXPANSION_EDGES = 3000 +DEFAULT_EXPANSION_DEPTH = 2 +DEFAULT_EXPANSION_NODES = 200 +DEFAULT_EXPANSION_EDGES = 600 + +MAX_PATH_DEPTH = 6 +MAX_PATH_VISITED = 2000 +MAX_EQUAL_PATHS = 5 +MAX_EVIDENCE_SUMMARY_CHARS = 200 + +_DIRECTIONS = frozenset({"outgoing", "incoming", "both"}) + + +# --------------------------------------------------------------------------- +# Node read shapes +# --------------------------------------------------------------------------- +def entity_node(workspace_id: str, entity: Dict[str, Any], + *, include_details: bool = True) -> Dict[str, Any]: + """The stable camelCase read shape of one Entity node.""" + node = { + "id": entity["id"], + "nodeKind": GraphNodeKind.ENTITY, + "entityType": entity["entity_type"], + "canonicalKey": entity["canonical_key"], + "displayName": entity["display_name"], + "lifecycleStatus": entity["lifecycle_status"], + "authorityStatus": entity["authority_status"], + "origin": entity["origin"], + "knowledgeRevision": store.current_revision_number(workspace_id), + } + if entity.get("merged_into_entity_id"): + node["mergedIntoEntityId"] = entity["merged_into_entity_id"] + if entity.get("superseded_by_entity_id"): + node["supersededByEntityId"] = entity["superseded_by_entity_id"] + if include_details: + node["bindings"] = [ + {"id": b["id"], "refKind": b["ref_kind"], "refId": b["ref_id"], + "refKey": b["ref_key"], "bindingRole": b["binding_role"], + "status": b["status"]} + for b in store.list_bindings(workspace_id, entity["id"])] + node["aliases"] = [ + {"alias": a["alias"], "aliasType": a["alias_type"], + "status": a["status"]} + for a in store.list_aliases(workspace_id, entity["id"])] + node["claimCount"] = store.count_claims( + workspace_id, entity_id=entity["id"], + lifecycle_status=GraphLifecycleStatus.ACTIVE) + node["relationCount"] = len(store.list_relations( + workspace_id, entity_id=entity["id"], + lifecycle_status=GraphLifecycleStatus.ACTIVE, limit=10_000)) + return node + + +def claim_node(workspace_id: str, claim: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": claim["id"], + "nodeKind": GraphNodeKind.CLAIM, + "entityId": claim["entity_id"], + "claimType": claim["claim_type"], + "statement": claim["statement"], + "lifecycleStatus": claim["lifecycle_status"], + "authorityStatus": claim["authority_status"], + "origin": claim["origin"], + "evidence": claim.get("evidence") or + store.claim_evidence(claim["id"]), + "knowledgeRevision": store.current_revision_number(workspace_id), + } + + +def get_node(workspace_id: str, node_id: str) -> Dict[str, Any]: + """Resolve any supported node id to its read shape. Source-plane ids are + projected from their canonical rows, workspace-scoped exactly like the + Phase 2 reads.""" + node_id = str(node_id or "").strip() + entity = store.get_entity(workspace_id, node_id) + if entity: + return entity_node(workspace_id, entity) + claim = store.get_claim(workspace_id, node_id) + if claim: + return claim_node(workspace_id, claim) + asset = db.get_asset(workspace_id, node_id) + if asset: + return { + "id": asset["id"], "nodeKind": GraphNodeKind.ASSET, + "logicalKey": asset["logical_key"], + "assetType": asset["asset_type"], "state": asset["state"], + "currentRevisionId": asset["current_revision_id"], + "boundEntities": [b["entity_id"] for b in + store.find_bindings_by_ref(workspace_id, + "asset", + asset["id"])], + "knowledgeRevision": + store.current_revision_number(workspace_id), + } + revision = db.get_revision(workspace_id, node_id) + if revision: + return { + "id": revision["id"], "nodeKind": GraphNodeKind.REVISION, + "assetId": revision["asset_id"], + "sequence": revision["sequence"], + "status": revision["status"], + "contentHash": revision["content_hash"], + "boundEntities": [b["entity_id"] for b in + store.find_bindings_by_ref(workspace_id, + "revision", + revision["id"])], + "knowledgeRevision": + store.current_revision_number(workspace_id), + } + segment = db.get_segment(workspace_id, node_id) + if segment: + return { + "id": segment["id"], "nodeKind": GraphNodeKind.SEGMENT, + "revisionId": segment["revision_id"], + "segmentType": segment["segment_type"], + "symbol": segment["symbol"], + "segmentKey": segment["segment_key"], + "boundEntities": [b["entity_id"] for b in + store.find_bindings_by_ref(workspace_id, + "segment", + segment["id"])], + "knowledgeRevision": + store.current_revision_number(workspace_id), + } + evidence = db.get_evidence(workspace_id, node_id) + if evidence: + return { + "id": evidence["id"], "nodeKind": GraphNodeKind.EVIDENCE, + "revisionId": evidence["revision_id"], + "segmentId": evidence["segment_id"], + "locator": evidence["locator"], + "knowledgeRevision": + store.current_revision_number(workspace_id), + } + raise GraphNodeNotFound(node_id, workspace_id=workspace_id) + + +def _edge_shape(rel: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": rel["id"], + "sourceEntityId": rel["source_entity_id"], + "targetEntityId": rel["target_entity_id"], + "relationType": rel["relation_type"], + "relationState": rel["relation_state"], + "confidence": rel["confidence"], + "lifecycleStatus": rel["lifecycle_status"], + "origin": rel["origin"], + } + + +# --------------------------------------------------------------------------- +# Adjacency +# --------------------------------------------------------------------------- +def _neighbors(workspace_id: str, entity_id: str, *, direction: str, + relation_types: Optional[Sequence[str]], + include_stale: bool, + lifecycle_cache: Optional[Dict[str, str]] = None + ) -> List[Dict[str, Any]]: + """This entity's relations in deterministic order. Rejected relations + are always excluded (governance history, not graph edges); an active + traversal additionally skips edges whose OTHER endpoint is no longer an + active entity — a withdrawn node must not silently carry a path.""" + lifecycle = None if include_stale else GraphLifecycleStatus.ACTIVE + out: List[Dict[str, Any]] = [] + if direction in ("outgoing", "both"): + out.extend(store.list_relations( + workspace_id, source_entity_id=entity_id, + lifecycle_status=lifecycle, limit=MAX_EXPANSION_EDGES)) + if direction in ("incoming", "both"): + out.extend(store.list_relations( + workspace_id, target_entity_id=entity_id, + lifecycle_status=lifecycle, limit=MAX_EXPANSION_EDGES)) + cache: Dict[str, str] = lifecycle_cache if lifecycle_cache is not None \ + else {} + + def entity_lifecycle(other_id: str) -> str: + if other_id not in cache: + other = store.get_entity(workspace_id, other_id) + cache[other_id] = (other or {}).get("lifecycle_status", "") + return cache[other_id] + + wanted = set(relation_types or []) + result = [] + seen: Set[str] = set() + for rel in out: + if rel["id"] in seen: + continue + seen.add(rel["id"]) + if rel["relation_state"] == RelationState.REJECTED: + continue + if wanted and rel["relation_type"] not in wanted: + continue + if not include_stale: + if rel["lifecycle_status"] != GraphLifecycleStatus.ACTIVE: + continue + other = (rel["target_entity_id"] + if rel["source_entity_id"] == entity_id + else rel["source_entity_id"]) + if entity_lifecycle(other) != GraphLifecycleStatus.ACTIVE: + continue + result.append(rel) + result.sort(key=lambda r: (r["relation_type"], r["source_entity_id"], + r["target_entity_id"], r["id"])) + return result + + +def _validate_relation_types(relation_types: Optional[Sequence[str]] + ) -> List[str]: + clean: List[str] = [] + for value in relation_types or []: + clean.append(require(RelationType, value, field="relation type")) + return clean + + +def _require_entity_node(workspace_id: str, node_id: str) -> Dict[str, Any]: + entity = store.get_entity(workspace_id, node_id) + if not entity: + raise GraphNodeNotFound(node_id, workspace_id=workspace_id) + return entity + + +# --------------------------------------------------------------------------- +# Bounded expansion +# --------------------------------------------------------------------------- +def expand_node(workspace_id: str, node_id: str, *, + direction: str = "both", + relation_types: Optional[Sequence[str]] = None, + depth: int = DEFAULT_EXPANSION_DEPTH, + node_limit: int = DEFAULT_EXPANSION_NODES, + edge_limit: int = DEFAULT_EXPANSION_EDGES, + include_stale: bool = False) -> Dict[str, Any]: + """Deterministic bounded BFS from one Entity node.""" + direction = str(direction or "both").strip().lower() + if direction not in _DIRECTIONS: + raise GraphLimitExceeded( + f"unknown direction: {direction!r}", + details={"allowed": sorted(_DIRECTIONS)}) + types = _validate_relation_types(relation_types) + depth = max(1, min(int(depth), MAX_EXPANSION_DEPTH)) + node_limit = max(1, min(int(node_limit), MAX_EXPANSION_NODES)) + edge_limit = max(1, min(int(edge_limit), MAX_EXPANSION_EDGES)) + + start = _require_entity_node(workspace_id, node_id) + truncated = False + visited: Dict[str, int] = {start["id"]: 0} + edges: Dict[str, Dict[str, Any]] = {} + frontier: List[str] = [start["id"]] + + for level in range(depth): + next_frontier: List[str] = [] + for entity_id in frontier: + for rel in _neighbors(workspace_id, entity_id, + direction=direction, + relation_types=types, + include_stale=include_stale): + if len(edges) >= edge_limit and rel["id"] not in edges: + truncated = True + continue + edges[rel["id"]] = rel + other = (rel["target_entity_id"] + if rel["source_entity_id"] == entity_id + else rel["source_entity_id"]) + if other in visited: + continue + if len(visited) >= node_limit: + truncated = True + continue + visited[other] = level + 1 + next_frontier.append(other) + frontier = sorted(next_frontier) + if not frontier: + break + + nodes = [] + for entity_id in sorted(visited, key=lambda e: (visited[e], e)): + entity = store.get_entity(workspace_id, entity_id) + if entity: + shape = entity_node(workspace_id, entity, include_details=False) + shape["depth"] = visited[entity_id] + nodes.append(shape) + edge_list = [_edge_shape(edges[k]) for k in sorted(edges)] + return { + "workspace_id": workspace_id, + "start": start["id"], + "direction": direction, + "depth": depth, + "nodes": nodes, + "edges": edge_list, + "truncated": truncated, + "limits": {"depth": depth, "nodes": node_limit, "edges": edge_limit, + "hard_depth": MAX_EXPANSION_DEPTH, + "hard_nodes": MAX_EXPANSION_NODES, + "hard_edges": MAX_EXPANSION_EDGES}, + "knowledge_revision": store.current_revision_number(workspace_id), + } + + +# --------------------------------------------------------------------------- +# Bounded shortest-path discovery +# --------------------------------------------------------------------------- +def _evidence_summary(relation_id: str) -> List[Dict[str, Any]]: + return [{"evidence_id": ev["evidence_id"], + "quote": (ev.get("quote") or "")[:MAX_EVIDENCE_SUMMARY_CHARS]} + for ev in store.relation_evidence(relation_id)[:5]] + + +def find_path(workspace_id: str, source_id: str, target_id: str, *, + relation_types: Optional[Sequence[str]] = None, + direction: str = "both", + max_depth: int = MAX_PATH_DEPTH, + include_stale: bool = False) -> Dict[str, Any]: + """Deterministic BFS shortest paths (unweighted). Honest outcomes: + ``found`` / ``no-path`` / ``truncated`` — a missing edge is never + invented.""" + direction = str(direction or "both").strip().lower() + if direction not in _DIRECTIONS: + raise GraphLimitExceeded( + f"unknown direction: {direction!r}", + details={"allowed": sorted(_DIRECTIONS)}) + types = _validate_relation_types(relation_types) + max_depth = max(1, min(int(max_depth), MAX_PATH_DEPTH)) + source = _require_entity_node(workspace_id, source_id) + target = _require_entity_node(workspace_id, target_id) + + base = { + "workspace_id": workspace_id, + "source": source["id"], "target": target["id"], + "direction": direction, "max_depth": max_depth, + "knowledge_revision": store.current_revision_number(workspace_id), + } + if source["id"] == target["id"]: + return {**base, "outcome": "found", "paths": [ + {"length": 0, "entities": [source["id"]], "edges": []}], + "truncated": False} + + #: parents[node] = list of (prev_node, relation) on ANY shortest path. + parents: Dict[str, List[Tuple[str, Dict[str, Any]]]] = {} + depth_of: Dict[str, int] = {source["id"]: 0} + frontier = [source["id"]] + truncated = False + found_depth: Optional[int] = None + + for level in range(max_depth): + if found_depth is not None: + break + next_frontier: List[str] = [] + for entity_id in sorted(frontier): + if len(depth_of) >= MAX_PATH_VISITED: + truncated = True + break + for rel in _neighbors(workspace_id, entity_id, + direction=direction, + relation_types=types, + include_stale=include_stale): + other = (rel["target_entity_id"] + if rel["source_entity_id"] == entity_id + else rel["source_entity_id"]) + if other == entity_id: + continue + known = depth_of.get(other) + if known is None: + if len(depth_of) >= MAX_PATH_VISITED: + truncated = True + continue + depth_of[other] = level + 1 + parents[other] = [(entity_id, rel)] + next_frontier.append(other) + elif known == level + 1: + parents[other].append((entity_id, rel)) + if other == target["id"]: + found_depth = level + 1 + frontier = next_frontier + if not frontier: + break + + if found_depth is None: + return {**base, "outcome": "truncated" if truncated else "no-path", + "paths": [], "truncated": truncated} + + # Reconstruct up to MAX_EQUAL_PATHS shortest paths, deterministically. + paths: List[Dict[str, Any]] = [] + + def walk(node: str, entities: List[str], + edges: List[Dict[str, Any]]) -> None: + """Backtrack from *node* to the source; *entities* collects the + nodes strictly after the source, target-first.""" + if len(paths) >= MAX_EQUAL_PATHS: + return + if node == source["id"]: + path_entities = [source["id"]] + list(reversed(entities)) + path_edges = list(reversed(edges)) + paths.append({ + "length": len(path_edges), + "entities": path_entities, + "edges": [{**_edge_shape(r), + "evidence": _evidence_summary(r["id"])} + for r in path_edges], + }) + return + for prev, rel in sorted(parents.get(node, ()), + key=lambda pr: (pr[0], pr[1]["id"])): + walk(prev, entities + [node], edges + [rel]) + + walk(target["id"], [], []) + # Present entity display data for the nodes on the returned paths. + node_ids = sorted({e for p in paths for e in p["entities"]}) + nodes = [] + for entity_id in node_ids: + entity = store.get_entity(workspace_id, entity_id) + if entity: + nodes.append(entity_node(workspace_id, entity, + include_details=False)) + return {**base, "outcome": "found", "paths": paths, "nodes": nodes, + "truncated": truncated} + + +# --------------------------------------------------------------------------- +# Bounded subgraph export +# --------------------------------------------------------------------------- +def get_subgraph(workspace_id: str, node_ids: Sequence[str], *, + depth: int = 1, + node_limit: int = DEFAULT_EXPANSION_NODES, + edge_limit: int = DEFAULT_EXPANSION_EDGES, + include_stale: bool = False) -> Dict[str, Any]: + """The bounded union of expansions around several seed nodes, plus every + edge BETWEEN collected nodes.""" + seeds = [str(n).strip() for n in node_ids if str(n or "").strip()] + if not seeds: + raise GraphLimitExceeded("at least one seed node id is required") + if len(seeds) > 20: + raise GraphLimitExceeded("at most 20 seed nodes", + details={"got": len(seeds)}) + collected: Dict[str, Dict[str, Any]] = {} + edges: Dict[str, Dict[str, Any]] = {} + truncated = False + for seed in sorted(seeds): + expansion = expand_node(workspace_id, seed, depth=depth, + node_limit=node_limit, + edge_limit=edge_limit, + include_stale=include_stale) + truncated = truncated or expansion["truncated"] + for node in expansion["nodes"]: + if len(collected) >= node_limit and node["id"] not in collected: + truncated = True + continue + collected.setdefault(node["id"], node) + for edge in expansion["edges"]: + if len(edges) >= edge_limit and edge["id"] not in edges: + truncated = True + continue + edges.setdefault(edge["id"], edge) + # Keep only edges whose BOTH endpoints made it into the node set. + kept = [e for _, e in sorted(edges.items()) + if e["sourceEntityId"] in collected + and e["targetEntityId"] in collected] + return { + "workspace_id": workspace_id, + "seeds": sorted(seeds), + "nodes": [collected[k] for k in sorted(collected)], + "edges": kept, + "truncated": truncated, + "knowledge_revision": store.current_revision_number(workspace_id), + } + + +__all__ = [ + "get_node", "entity_node", "claim_node", "expand_node", "find_path", + "get_subgraph", + "MAX_EXPANSION_DEPTH", "MAX_EXPANSION_NODES", "MAX_EXPANSION_EDGES", + "MAX_PATH_DEPTH", +] diff --git a/openmind/knowledge/identity.py b/openmind/knowledge/identity.py new file mode 100644 index 0000000..6a4ba0c --- /dev/null +++ b/openmind/knowledge/identity.py @@ -0,0 +1,141 @@ +"""Deterministic identity for canonical graph objects. + +Canonical keys, alias normalization and normalized statement hashes all live +here so the promotion path, the deterministic projector and manual creation +can never derive the same concept two different ways. + +Everything is pure string work — no database, no provider, no side effects. +""" +from __future__ import annotations + +import hashlib +import re + +_WHITESPACE = re.compile(r"\s+") +#: Characters stripped from derived key material (kept deliberately small: +#: the goal is stability, not aggressive slugging that collides distinct +#: names). +_KEY_UNSAFE = re.compile(r"[^a-z0-9._#:/()\[\]{}@$+-]+") + + +def normalize_alias(alias: str) -> str: + """Case-folded, whitespace-collapsed form used for exact alias lookup + and collision detection.""" + return _WHITESPACE.sub(" ", str(alias or "").strip()).casefold() + + +def normalize_statement(statement: str) -> str: + """Whitespace-collapsed, case-folded statement text — the dedup basis.""" + return _WHITESPACE.sub(" ", str(statement or "").strip()).casefold() + + +def statement_hash(statement: str) -> str: + """SHA-256 of the normalized statement. Identical normalized claims on + one Entity deduplicate through this.""" + return hashlib.sha256( + normalize_statement(statement).encode("utf-8")).hexdigest() + + +def quote_hash(quote: str) -> str: + """Hash of a normalized evidence quote (join-table identity component, + matching the Phase 4 candidate-evidence convention).""" + return hashlib.sha256( + _WHITESPACE.sub(" ", str(quote or "").strip()).encode("utf-8") + ).hexdigest() + + +def _key_component(text: str) -> str: + """One canonical-key component: trimmed, lower-cased except that + identifier-looking tokens keep their case (REQ-NC-017 must stay + readable), unsafe characters collapsed to '-'. Deterministic.""" + clean = _WHITESPACE.sub(" ", str(text or "").strip()) + # Preserve case for explicit identifiers (contains a digit or is + # dominated by upper-case); otherwise lower-case for stability. + has_digit = any(c.isdigit() for c in clean) + upper = sum(1 for c in clean if c.isupper()) + lower = sum(1 for c in clean if c.islower()) + if not has_digit and upper <= lower: + clean = clean.lower() + return _KEY_UNSAFE.sub("-", clean.replace(" ", "-")).strip("-") + + +def entity_key(entity_type: str, *components: str) -> str: + """``:[:...]`` — the deterministic + canonical key. Components are normalized individually; empty components + are dropped.""" + parts = [str(entity_type or "").strip().lower()] + parts.extend(c for c in (_key_component(c) for c in components) if c) + return ":".join(parts) + + +def identifier_entity_key(entity_type: str, identifier: str) -> str: + """Key for a concept with an explicit stable identifier + (``requirement:REQ-NC-017``). The identifier's case is preserved.""" + ident = _WHITESPACE.sub("", str(identifier or "").strip()) + return f"{str(entity_type or '').strip().lower()}:{ident}" + + +def derived_entity_key(entity_type: str, title: str) -> str: + """Key for a concept WITHOUT a stable identifier: a bounded, normalized + projection of its title, marked ``derived`` so it can never collide with + an explicit identifier key.""" + slug = _key_component(title).lower()[:120] or "untitled" + return f"{str(entity_type or '').strip().lower()}:derived:{slug}" + + +def asset_entity_key(entity_type: str, asset_id: str) -> str: + """``code-component:asset:a_123`` — Asset-anchored deterministic keys.""" + return f"{str(entity_type or '').strip().lower()}:asset:{asset_id}" + + +def symbol_entity_key(asset_id: str, symbol: str) -> str: + """``code-symbol:asset:a_123:pkg.Class#method(Sig)`` — symbol case is + load-bearing and preserved verbatim (whitespace removed).""" + sym = _WHITESPACE.sub("", str(symbol or "").strip()) + return f"code-symbol:asset:{asset_id}:{sym}" + + +def interface_entity_key(method: str, path: str) -> str: + """``interface:POST:/name-check`` — HTTP method upper-cased, path kept + verbatim (paths are case-sensitive).""" + m = str(method or "").strip().upper() + p = _WHITESPACE.sub("", str(path or "").strip()) + return f"interface:{m}:{p}" + + +def configuration_entity_key(asset_id: str, config_key: str) -> str: + key = _WHITESPACE.sub("", str(config_key or "").strip()) + return f"configuration:asset:{asset_id}:{key}" + + +def database_object_entity_key(schema_hint: str, object_name: str) -> str: + """``database-object:database-schema:PASSENGER`` — SQL object names keep + their case (quoted identifiers are case-sensitive).""" + hint = _key_component(schema_hint) or "database-schema" + name = _WHITESPACE.sub("", str(object_name or "").strip()) + return f"database-object:{hint}:{name}" + + +def message_topic_entity_key(topic: str) -> str: + return f"message-topic:{_WHITESPACE.sub('', str(topic or '').strip())}" + + +#: An identifier-looking stable key: letters+digits with -_./ separators and +#: at least one digit or one separator (REQ-NC-017, ORD.4, NC_CHECK-2) — +#: NOT a plain English word. Used when deciding whether a candidate's +#: stable_key is an explicit identifier or just a title echo. +_IDENTIFIER_RE = re.compile(r"^(?=.*[0-9._/-])[A-Za-z0-9._/-]{2,80}$") + + +def looks_like_identifier(value: str) -> bool: + v = str(value or "").strip() + return bool(v) and not v.count(" ") and bool(_IDENTIFIER_RE.match(v)) + + +__all__ = [ + "normalize_alias", "normalize_statement", "statement_hash", "quote_hash", + "entity_key", "identifier_entity_key", "derived_entity_key", + "asset_entity_key", "symbol_entity_key", "interface_entity_key", + "configuration_entity_key", "database_object_entity_key", + "message_topic_entity_key", "looks_like_identifier", +] diff --git a/openmind/knowledge/projector.py b/openmind/knowledge/projector.py new file mode 100644 index 0000000..691e100 --- /dev/null +++ b/openmind/knowledge/projector.py @@ -0,0 +1,593 @@ +"""Deterministic graph projection: model-free seeding and incremental sync. + +Everything here derives from recorded deterministic facts — Asset types, +Segment structure, the structure map's name-based call edges and template +facet captures. No semantic provider is imported, no model is called, no +network is touched, and nothing this module writes is a semantic Claim. + +PROJECTION RULES (each tested in verify_knowledge_projection) +------------------------------------------------------------- +Assets (active, per :class:`~openmind.domain.types.AssetType`): + source-code / test-source -> code-component + configuration -> configuration + database-schema -> data-model (+ database-object per SQL object) + build-definition -> build-definition + parsed document -> document (a parse record is the fact) +Segments (of each active asset's CURRENT revision): + Java type/method/constructor -> code-symbol + OpenAPI api-operation -> interface + OpenAPI schema-definition -> data-model + SQL sql-object -> database-object + message-topic facet capture -> message-topic +Containment (structurally explicit -> relation_state=explicit): + code-component -> code-symbol + document -> interface / data-model (OpenAPI blocks) + data-model -> database-object (SQL objects in a schema) +Calls (structure map, name-based -> relation_state=inferred): + code-component -> code-component, confidence medium (unambiguous) or + low (ambiguous). Ambiguity is PRESERVED — never upgraded. + +Deliberate NON-projections: a generic document never becomes a Requirement +or Design; a test-source method never becomes a Test Case; a facet fact +without an exact ``topic`` capture never becomes a message-topic; no +publishes/consumes relation is created from facet captures (the capture +does not record direction deterministically). + +INCREMENTAL SYNC +---------------- +The desired deterministic state is computed in full (a read-only pass), then +DIFFED against the stored deterministic graph rows; only the difference is +written, inside one graph transaction. An unchanged source hash writes +nothing and mints no Knowledge Revision. +""" +from __future__ import annotations + +import hashlib +import json +from typing import Any, Dict, List, Optional, Tuple + +from .. import db, mapio +from . import GRAPH_PROJECTOR_VERSION, identity, store +from .vocabularies import (AliasStatus, AliasType, BindingRefKind, + BindingRole, BindingStatus, EntityType, + GraphLifecycleStatus, GraphOrigin, RelationState, + RelationType, RevisionAction) + +#: Bounds — a projection pass must stay predictable on large workspaces. +MAX_ASSETS = 50_000 +MAX_SEGMENTS_PER_REVISION = 2_000 +MAX_CALL_EDGES = 20_000 +MAX_TOPIC_FACTS = 2_000 + +_ASSET_ENTITY_TYPES = { + "source-code": EntityType.CODE_COMPONENT, + "test-source": EntityType.CODE_COMPONENT, + "configuration": EntityType.CONFIGURATION, + "database-schema": EntityType.DATA_MODEL, + "build-definition": EntityType.BUILD_DEFINITION, +} + +_CODE_SEGMENT_TYPES = frozenset({"type", "method", "constructor"}) + + +def _desired_entity(entity_type: str, canonical_key: str, display_name: str, + description: str = "") -> Dict[str, Any]: + return {"entity_type": entity_type, "canonical_key": canonical_key, + "display_name": display_name[:200], "description": + description[:500], "bindings": [], "aliases": []} + + +def compute_desired_state(workspace_id: str) -> Dict[str, Any]: + """The full deterministic target state, as pure data. + + Returns ``{entities: {key: {...}}, relations: {(src_key, tgt_key, type): + {...}}, truncated: {...}}`` where keys are canonical keys. + """ + entities: Dict[str, Dict[str, Any]] = {} + relations: Dict[Tuple[str, str, str], Dict[str, Any]] = {} + truncated: Dict[str, bool] = {} + + assets = db.list_assets(workspace_id, state="active", limit=MAX_ASSETS) + truncated["assets"] = len(assets) >= MAX_ASSETS + key_to_asset: Dict[str, Dict[str, Any]] = {} + #: asset_id -> the file-segment evidence id of its current revision + #: (used as deterministic call-edge provenance). + file_evidence: Dict[str, str] = {} + + for asset in assets: + asset_id = asset["id"] + logical_key = asset["logical_key"] + key_to_asset[logical_key] = asset + revision_id = asset.get("current_revision_id") or "" + is_document = db.is_document_asset(workspace_id, asset_id) + + # A parse record is the recorded fact that makes an Asset a document; + # an unparsed text file projects nothing (never a guessed Requirement + # or Design). Everything else maps by deterministic Asset type. + entity_type = (EntityType.DOCUMENT if is_document + else _ASSET_ENTITY_TYPES.get(asset["asset_type"])) + if not entity_type: + continue + + akey = identity.asset_entity_key(entity_type, asset_id) + record = _desired_entity(entity_type, akey, + asset.get("title") or logical_key) + record["bindings"].append( + {"ref_kind": BindingRefKind.ASSET, "ref_id": asset_id, + "ref_key": logical_key, + "binding_role": BindingRole.PRIMARY_SOURCE}) + if revision_id: + record["bindings"].append( + {"ref_kind": BindingRefKind.REVISION, "ref_id": revision_id, + "ref_key": logical_key, + "binding_role": BindingRole.PRIMARY_SOURCE}) + record["aliases"].append({"alias": logical_key, + "alias_type": AliasType.PATH}) + entities[akey] = record + asset["_entity_key"] = akey + + if not revision_id: + continue + segments = db.list_segments(workspace_id, revision_id, + limit=MAX_SEGMENTS_PER_REVISION) + if len(segments) >= MAX_SEGMENTS_PER_REVISION: + truncated["segments"] = True + evidence_by_segment = db.evidence_ids_for_revision(workspace_id, + revision_id) + for segment in segments: + stype = segment["segment_type"] + symbol = segment.get("symbol") or "" + seg_id = segment["id"] + evidence_id = evidence_by_segment.get(seg_id, "") + # The asset's call-edge evidence anchor: the file segment's + # evidence where one exists (generic sources), else the first + # segment evidence (Java revisions carry typed segments only). + if evidence_id and (stype == "file" + or not file_evidence.get(asset_id)): + file_evidence[asset_id] = evidence_id + + skey: Optional[str] = None + if stype in _CODE_SEGMENT_TYPES and symbol: + skey = identity.symbol_entity_key(asset_id, symbol) + child = _desired_entity(EntityType.CODE_SYMBOL, skey, symbol) + child["aliases"].append({"alias": symbol, + "alias_type": AliasType.SYMBOL}) + elif stype == "api-operation" and symbol: + # The parser records the exact method + path in the block + # metadata; the segment symbol is only a slug. Fall back to + # the slug when a (future) parser omits them. + meta = segment.get("metadata") or {} + method = str(meta.get("method") or "").strip() + path = str(meta.get("path") or "").strip() + if method and path: + skey = identity.interface_entity_key(method, path) + display = f"{method.upper()} {path}" + else: + skey = identity.entity_key(EntityType.INTERFACE, + "asset", asset_id, symbol) + display = symbol + child = _desired_entity(EntityType.INTERFACE, skey, display) + child["aliases"].append({"alias": display, + "alias_type": AliasType.IDENTIFIER}) + elif stype == "schema-definition" and symbol: + skey = identity.entity_key(EntityType.DATA_MODEL, "asset", + asset_id, symbol) + child = _desired_entity(EntityType.DATA_MODEL, skey, symbol) + child["aliases"].append({"alias": symbol, + "alias_type": AliasType.NAME}) + elif stype == "sql-object" and symbol: + skey = f"database-object:asset:{asset_id}:{symbol}" + child = _desired_entity(EntityType.DATABASE_OBJECT, skey, + symbol) + child["aliases"].append({"alias": symbol, + "alias_type": AliasType.IDENTIFIER}) + else: + continue + child["bindings"].append( + {"ref_kind": BindingRefKind.SEGMENT, "ref_id": seg_id, + "ref_key": segment.get("segment_key", ""), + "binding_role": BindingRole.DEFINITION_SOURCE, + "evidence_id": evidence_id}) + child["bindings"].append( + {"ref_kind": BindingRefKind.REVISION, "ref_id": revision_id, + "ref_key": logical_key, + "binding_role": BindingRole.PRIMARY_SOURCE}) + entities[skey] = child + relations[(akey, skey, RelationType.CONTAINS)] = { + "relation_state": RelationState.EXPLICIT, + "confidence": "high", + "evidence_id": evidence_id, + "metadata": {"containment": stype}, + } + + # -- call edges (name-based, ambiguity preserved) ------------------------ + structure = mapio.load_structure(workspace_id) + edges = ((structure.get("call_graph") or {}).get("edges") + or [])[:MAX_CALL_EDGES] + truncated["call_edges"] = len( + (structure.get("call_graph") or {}).get("edges") or []) > MAX_CALL_EDGES + for edge in edges: + src = key_to_asset.get(str(edge.get("from") or "")) + dst = key_to_asset.get(str(edge.get("to") or "")) + if not src or not dst: + continue # an endpoint is not an active asset: no edge + src_key = src.get("_entity_key") + dst_key = dst.get("_entity_key") + if not src_key or not dst_key or src_key == dst_key: + continue + if entities.get(src_key, {}).get("entity_type") != \ + EntityType.CODE_COMPONENT or \ + entities.get(dst_key, {}).get("entity_type") != \ + EntityType.CODE_COMPONENT: + continue + evidence_id = file_evidence.get(src["id"], "") + if not evidence_id: + continue # no recoverable source evidence: no edge + ambiguous = bool(edge.get("ambiguous")) + rkey = (src_key, dst_key, RelationType.CALLS) + existing = relations.get(rkey) + if existing: + # Several symbols may link the same file pair; the edge stays + # ambiguous unless EVERY contributing symbol is unambiguous. + symbols = existing["metadata"].setdefault("symbols", []) + if edge.get("symbol") and edge["symbol"] not in symbols: + symbols.append(edge["symbol"]) + if ambiguous: + existing["metadata"]["ambiguous"] = True + existing["confidence"] = "low" + continue + relations[rkey] = { + "relation_state": RelationState.INFERRED, + "confidence": "low" if ambiguous else "medium", + "evidence_id": evidence_id, + "metadata": {"symbols": [edge.get("symbol") or ""], + "ambiguous": ambiguous, "analyzer": "structure"}, + } + + # -- message-topic facet captures --------------------------------------- + facts = (mapio.load_facts(workspace_id) or {}).get("facts") or [] + topic_facts = [f for f in facts + if str((f.get("values") or {}).get("topic") or "").strip()] + truncated["topic_facts"] = len(topic_facts) > MAX_TOPIC_FACTS + for fact in topic_facts[:MAX_TOPIC_FACTS]: + topic = str(fact["values"]["topic"]).strip() + asset = key_to_asset.get(str(fact.get("file") or "")) + tkey = identity.message_topic_entity_key(topic) + record = entities.get(tkey) + if not record: + record = _desired_entity(EntityType.MESSAGE_TOPIC, tkey, topic) + record["aliases"].append({"alias": topic, + "alias_type": AliasType.IDENTIFIER}) + entities[tkey] = record + if asset: + binding = {"ref_kind": BindingRefKind.ASSET, + "ref_id": asset["id"], + "ref_key": asset["logical_key"], + "binding_role": BindingRole.SUPPORTING} + if binding not in record["bindings"]: + record["bindings"].append(binding) + + return {"entities": entities, "relations": relations, + "truncated": {k: v for k, v in truncated.items() if v}} + + +def compute_source_hash(workspace_id: str, + desired: Optional[Dict[str, Any]] = None) -> str: + """Deterministic hash of the projection INPUT state. Derived from the + desired output (which is a pure function of the inputs), so any relevant + source change — asset set, revisions, segments, call edges, facets — + changes it, and irrelevant changes do not.""" + desired = desired or compute_desired_state(workspace_id) + payload = json.dumps({ + "version": GRAPH_PROJECTOR_VERSION, + "entities": {k: {kk: vv for kk, vv in v.items()} + for k, v in sorted(desired["entities"].items())}, + "relations": {"|".join(k): v for k, v in + sorted(desired["relations"].items())}, + }, sort_keys=True, default=str) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _diff(workspace_id: str, desired: Dict[str, Any]) -> Dict[str, Any]: + """What sync would write: pure data, no writes.""" + existing_entities = { + e["canonical_key"]: e for e in store.list_entities( + workspace_id, origin=GraphOrigin.DETERMINISTIC, limit=200_000)} + bindings = store.list_workspace_bindings(workspace_id, + status=BindingStatus.ACTIVE) + bindings_by_entity: Dict[str, List[Dict[str, Any]]] = {} + for b in bindings: + bindings_by_entity.setdefault(b["entity_id"], []).append(b) + + create_entities: List[str] = [] + revive_entities: List[str] = [] + stale_entities: List[str] = [] + binding_inserts: List[Tuple[str, Dict[str, Any]]] = [] # (key, binding) + binding_stales: List[Dict[str, Any]] = [] + for key, want in desired["entities"].items(): + have = existing_entities.get(key) + if not have: + create_entities.append(key) + continue + if have["lifecycle_status"] == GraphLifecycleStatus.STALE: + revive_entities.append(key) + want_bindings = {(b["ref_kind"], b["ref_id"], b["binding_role"]) + for b in want["bindings"]} + have_bindings = {(b["ref_kind"], b["ref_id"], b["binding_role"]): b + for b in bindings_by_entity.get(have["id"], [])} + for b in want["bindings"]: + if (b["ref_kind"], b["ref_id"], b["binding_role"]) \ + not in have_bindings: + binding_inserts.append((key, b)) + for sig, b in have_bindings.items(): + if b["origin"] == GraphOrigin.DETERMINISTIC and \ + sig not in want_bindings: + binding_stales.append(b) + for key, have in existing_entities.items(): + if key not in desired["entities"] and \ + have["lifecycle_status"] == GraphLifecycleStatus.ACTIVE: + stale_entities.append(key) + + existing_relations: Dict[Tuple[str, str, str], Dict[str, Any]] = {} + id_to_key = {e["id"]: k for k, e in existing_entities.items()} + for rel in store.list_relations(workspace_id, + origin=GraphOrigin.DETERMINISTIC, + limit=200_000): + skey = id_to_key.get(rel["source_entity_id"]) + tkey = id_to_key.get(rel["target_entity_id"]) + if skey and tkey: + existing_relations[(skey, tkey, rel["relation_type"])] = rel + create_relations = [k for k in desired["relations"] + if k not in existing_relations + or existing_relations[k]["lifecycle_status"] != + GraphLifecycleStatus.ACTIVE] + revive_relations = [k for k in desired["relations"] + if k in existing_relations + and existing_relations[k]["lifecycle_status"] == + GraphLifecycleStatus.STALE] + create_relations = [k for k in create_relations + if k not in revive_relations] + stale_relations = [k for k, rel in existing_relations.items() + if k not in desired["relations"] + and rel["lifecycle_status"] == + GraphLifecycleStatus.ACTIVE] + return { + "existing_entities": existing_entities, + "existing_relations": existing_relations, + "create_entities": sorted(create_entities), + "revive_entities": sorted(revive_entities), + "stale_entities": sorted(stale_entities), + "binding_inserts": binding_inserts, + "binding_stales": binding_stales, + "create_relations": sorted(create_relations), + "revive_relations": sorted(revive_relations), + "stale_relations": sorted(stale_relations), + } + + +def plan_seed(workspace_id: str) -> Dict[str, Any]: + """Dry-run: what a seed/sync would change. Writes nothing.""" + desired = compute_desired_state(workspace_id) + source_hash = compute_source_hash(workspace_id, desired) + state = store.get_projection_state(workspace_id) + diff = _diff(workspace_id, desired) + unchanged = bool(state) and \ + state["source_knowledge_hash"] == source_hash and \ + state["projector_version"] == GRAPH_PROJECTOR_VERSION + return { + "workspace_id": workspace_id, + "projector_version": GRAPH_PROJECTOR_VERSION, + "source_hash": source_hash, + "stored_hash": (state or {}).get("source_knowledge_hash", ""), + "unchanged": unchanged, + "desired_entities": len(desired["entities"]), + "desired_relations": len(desired["relations"]), + "would_create_entities": len(diff["create_entities"]), + "would_revive_entities": len(diff["revive_entities"]), + "would_stale_entities": len(diff["stale_entities"]), + "would_create_relations": len(diff["create_relations"]), + "would_revive_relations": len(diff["revive_relations"]), + "would_stale_relations": len(diff["stale_relations"]), + "would_add_bindings": len(diff["binding_inserts"]), + "would_stale_bindings": len(diff["binding_stales"]), + "truncated": desired["truncated"], + "knowledge_revision": store.current_revision_number(workspace_id), + } + + +def sync_graph(workspace_id: str, *, actor: str = "", + seed: bool = False) -> Dict[str, Any]: + """Project the deterministic state into the graph, incrementally. + + Unchanged source hash → zero writes, zero Knowledge Revisions (the + watermark is refreshed). Otherwise exactly the diff is written in one + graph transaction that mints one Knowledge Revision. + """ + desired = compute_desired_state(workspace_id) + source_hash = compute_source_hash(workspace_id, desired) + state = store.get_projection_state(workspace_id) + action_label = RevisionAction.GRAPH_SEED if (seed or not state) \ + else RevisionAction.GRAPH_SYNC + + if state and state["source_knowledge_hash"] == source_hash and \ + state["projector_version"] == GRAPH_PROJECTOR_VERSION: + return {"workspace_id": workspace_id, "action": "noop", + "changed": False, "source_hash": source_hash, + "projector_version": GRAPH_PROJECTOR_VERSION, + "knowledge_revision": + store.current_revision_number(workspace_id), + "truncated": desired["truncated"]} + + diff = _diff(workspace_id, desired) + changed = any((diff["create_entities"], diff["revive_entities"], + diff["stale_entities"], diff["binding_inserts"], + diff["binding_stales"], diff["create_relations"], + diff["revive_relations"], diff["stale_relations"])) + if not changed: + # Source hash moved (e.g. first run over an empty workspace) but the + # graph diff is empty: refresh the watermark without a revision. + store.touch_projection_state( + workspace_id, projector_version=GRAPH_PROJECTOR_VERSION, + source_knowledge_hash=source_hash) + return {"workspace_id": workspace_id, "action": "noop", + "changed": False, "source_hash": source_hash, + "projector_version": GRAPH_PROJECTOR_VERSION, + "knowledge_revision": + store.current_revision_number(workspace_id), + "truncated": desired["truncated"]} + + alias_collisions: List[Dict[str, Any]] = [] + counts = {"entities_created": 0, "entities_revived": 0, + "entities_staled": 0, "relations_created": 0, + "relations_revived": 0, "relations_staled": 0, + "bindings_added": 0, "bindings_staled": 0, "aliases_added": 0} + + with store.graph_transaction(workspace_id, action=action_label, + actor=actor, + summary="deterministic graph projection" + ) as tx: + key_to_id: Dict[str, str] = { + k: e["id"] for k, e in diff["existing_entities"].items()} + + def _add_aliases(entity_id: str, key: str, + wanted: List[Dict[str, Any]]) -> None: + existing = {a["normalized_alias"] for a in store.list_aliases( + workspace_id, entity_id, status=None)} + for alias in wanted: + normalized = identity.normalize_alias(alias["alias"]) + if not normalized or normalized in existing: + continue + holders = store.find_alias_holders(workspace_id, normalized, + exclude_entity_id=entity_id) + if holders: + alias_collisions.append( + {"alias": alias["alias"], "entity_key": key, + "held_by": [h["holder_canonical_key"] + for h in holders]}) + continue # reported, never silently attached + tx.insert_alias(entity_id=entity_id, alias=alias["alias"], + normalized_alias=normalized, + alias_type=alias["alias_type"], + origin=GraphOrigin.DETERMINISTIC) + counts["aliases_added"] += 1 + existing.add(normalized) + + for key in diff["create_entities"]: + want = desired["entities"][key] + entity = tx.insert_entity( + entity_type=want["entity_type"], canonical_key=key, + display_name=want["display_name"], + description=want["description"], + origin=GraphOrigin.DETERMINISTIC) + key_to_id[key] = entity["id"] + counts["entities_created"] += 1 + for binding in want["bindings"]: + tx.insert_binding(entity_id=entity["id"], + ref_kind=binding["ref_kind"], + ref_id=binding["ref_id"], + ref_key=binding.get("ref_key", ""), + binding_role=binding["binding_role"], + origin=GraphOrigin.DETERMINISTIC, + evidence_id=binding.get("evidence_id", "")) + counts["bindings_added"] += 1 + _add_aliases(entity["id"], key, want["aliases"]) + + for key in diff["revive_entities"]: + entity_id = key_to_id[key] + tx.update_entity(entity_id, + lifecycle_status=GraphLifecycleStatus.ACTIVE, + stale_at=None) + counts["entities_revived"] += 1 + + for key, binding in diff["binding_inserts"]: + entity_id = key_to_id.get(key) + if not entity_id: + continue + tx.insert_binding(entity_id=entity_id, + ref_kind=binding["ref_kind"], + ref_id=binding["ref_id"], + ref_key=binding.get("ref_key", ""), + binding_role=binding["binding_role"], + origin=GraphOrigin.DETERMINISTIC, + evidence_id=binding.get("evidence_id", "")) + counts["bindings_added"] += 1 + + for binding in diff["binding_stales"]: + tx.update_binding(binding["id"], + status=BindingStatus.STALE, stale_at=tx.ts) + counts["bindings_staled"] += 1 + + for key in diff["stale_entities"]: + entity_id = key_to_id[key] + tx.update_entity(entity_id, + lifecycle_status=GraphLifecycleStatus.STALE, + stale_at=tx.ts) + counts["entities_staled"] += 1 + + for rkey in diff["create_relations"]: + src_key, tgt_key, rtype = rkey + want = desired["relations"][rkey] + src_id = key_to_id.get(src_key) + tgt_id = key_to_id.get(tgt_key) + if not src_id or not tgt_id or src_id == tgt_id: + continue + evidence = [] + if want.get("evidence_id"): + evidence = [{"evidence_id": want["evidence_id"], + "role": "primary", "quote": "", + "quote_hash": identity.quote_hash("")}] + tx.insert_relation( + source_entity_id=src_id, target_entity_id=tgt_id, + relation_type=rtype, + relation_state=want["relation_state"], + confidence=want["confidence"], + origin=GraphOrigin.DETERMINISTIC, + metadata=want.get("metadata") or {}, + evidence=evidence) + counts["relations_created"] += 1 + + for rkey in diff["revive_relations"]: + rel = diff["existing_relations"][rkey] + # Restore the epistemic state that was parked when it went stale. + prior = (rel.get("metadata") or {}).get( + "prior_state", RelationState.INFERRED) + tx.update_relation(rel["id"], + lifecycle_status=GraphLifecycleStatus.ACTIVE, + relation_state=prior, stale_at=None) + counts["relations_revived"] += 1 + + for rkey in diff["stale_relations"]: + rel = diff["existing_relations"][rkey] + meta = dict(rel.get("metadata") or {}) + meta["prior_state"] = rel["relation_state"] + tx.update_relation(rel["id"], + lifecycle_status=GraphLifecycleStatus.STALE, + relation_state=RelationState.STALE, + metadata=meta, stale_at=tx.ts) + counts["relations_staled"] += 1 + + tx.set_projection_state(projector_version=GRAPH_PROJECTOR_VERSION, + source_knowledge_hash=source_hash) + revision_number = tx.revision_number + + return { + "workspace_id": workspace_id, + "action": "seeded" if action_label == RevisionAction.GRAPH_SEED + else "synced", + "changed": True, + "source_hash": source_hash, + "projector_version": GRAPH_PROJECTOR_VERSION, + **counts, + "alias_collisions": alias_collisions, + "truncated": desired["truncated"], + "knowledge_revision": revision_number, + } + + +def seed_graph(workspace_id: str, *, actor: str = "") -> Dict[str, Any]: + """First projection (or a full re-check): same incremental writer.""" + return sync_graph(workspace_id, actor=actor, seed=True) + + +__all__ = ["plan_seed", "seed_graph", "sync_graph", "compute_desired_state", + "compute_source_hash"] diff --git a/openmind/knowledge/promotion.py b/openmind/knowledge/promotion.py new file mode 100644 index 0000000..ae20a34 --- /dev/null +++ b/openmind/knowledge/promotion.py @@ -0,0 +1,764 @@ +"""Explicit Candidate promotion: the only bridge from the Phase 4 semantic +plane into the canonical graph. + +Planning (:func:`plan_candidate_promotion`, :func:`plan_relation_promotion`) +is deterministic, calls no provider and writes nothing. Promotion +(:func:`promote_candidate`, :func:`promote_relation`) re-checks every +eligibility rule inside the graph transaction, so a candidate that went +stale between plan and promote is still blocked. + +Eligibility is deliberately without bypass flags: there is no +``--accept-stale``, no ``--ignore-evidence``, no ``--force-unverified``. +A stale candidate must be re-analyzed, or the knowledge recreated manually +with fresh Evidence. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from .. import db +from . import PROMOTION_POLICY_VERSION +from ..semantic import store as semantic_store +from ..semantic.models import (EvidenceVerificationStatus, LifecycleStatus, + ReviewStatus, SemanticCandidateKind) +from . import identity, store, verifier +from .errors import GraphEvidenceInvalid, PromotionBlocked +from .vocabularies import (AliasType, BindingRefKind, BindingRole, ClaimType, + CONCEPT_TYPE_TO_ENTITY_TYPE, DecisionTargetKind, + DecisionType, EntityType, + ENTITY_TYPE_TO_CLAIM_TYPE, GraphLifecycleStatus, + GraphOrigin, PromotionCandidateKind, + PromotionExpectedAction, PromotionStatus, + RELATION_CANDIDATE_TYPE_TO_RELATION_TYPE, + RelationState, RevisionAction) + +MAX_STATEMENT_CHARS = 4_000 +MAX_DISPLAY_NAME_CHARS = 200 +MAX_DESCRIPTION_CHARS = 2_000 +MAX_NOTE_CHARS = 2_000 + + +def _bound(text: str, cap: int) -> str: + text = str(text or "") + return text if len(text) <= cap else text[:cap] + + +# --------------------------------------------------------------------------- +# Eligibility +# --------------------------------------------------------------------------- +def _base_blocking_reasons(workspace_id: str, + candidate: Dict[str, Any]) -> List[str]: + """The review/lifecycle/evidence gates shared by both candidate kinds.""" + reasons: List[str] = [] + if candidate.get("review_status") != ReviewStatus.CONFIRMED: + reasons.append( + f"review_status is {candidate.get('review_status')!r} " + f"(must be confirmed)") + if candidate.get("lifecycle_status") != LifecycleStatus.ACTIVE: + reasons.append( + f"lifecycle_status is {candidate.get('lifecycle_status')!r} " + f"(must be active; a stale candidate must be re-analyzed)") + if candidate.get("evidence_status") != EvidenceVerificationStatus.VERIFIED: + reasons.append( + f"evidence_status is {candidate.get('evidence_status')!r} " + f"(must be verified)") + return reasons + + +def _candidate_blocking_reasons(workspace_id: str, + candidate: Dict[str, Any]) -> List[str]: + reasons = _base_blocking_reasons(workspace_id, candidate) + kind = candidate.get("candidate_kind") + if kind not in SemanticCandidateKind.VALUES: + reasons.append(f"unsupported candidate kind: {kind!r}") + if kind == SemanticCandidateKind.ENGINEERING_CONCEPT and \ + candidate.get("candidate_type") not in CONCEPT_TYPE_TO_ENTITY_TYPE: + reasons.append( + f"unsupported engineering-concept type: " + f"{candidate.get('candidate_type')!r}") + # Source revision must still be current. + revision_id = str(candidate.get("revision_id") or "") + if revision_id and revision_id not in store.current_revision_ids( + workspace_id): + reasons.append( + f"source revision {revision_id} is no longer current") + # Cited evidence must belong to the workspace (full quote verification + # happens in the promoting transaction; existence is checked here so the + # plan reports it as a blocking reason rather than an exception). + for ev in candidate.get("evidence") or []: + if not db.get_evidence(workspace_id, str(ev.get("evidence_id") or "")): + reasons.append( + f"evidence {ev.get('evidence_id')} does not exist in this " + f"workspace") + if not (candidate.get("evidence") or []): + reasons.append("candidate has no evidence joins") + prior = store.find_promotion( + workspace_id, PromotionCandidateKind.SEMANTIC_CANDIDATE, + candidate["id"]) + if prior: + reasons.append("candidate is already promoted") + return reasons + + +# --------------------------------------------------------------------------- +# Identity resolution for engineering-concept candidates +# --------------------------------------------------------------------------- +def _resolve_concept_identity(workspace_id: str, candidate: Dict[str, Any] + ) -> Dict[str, Any]: + """Deterministic identity proposal + match/conflict report.""" + entity_type = CONCEPT_TYPE_TO_ENTITY_TYPE[candidate["candidate_type"]] + stable_key = str(candidate.get("stable_key") or "").strip() + title = str(candidate.get("title") or "").strip() + statement = str(candidate.get("statement") or "").strip() + has_identifier = identity.looks_like_identifier(stable_key) + if has_identifier: + canonical_key = identity.identifier_entity_key(entity_type, stable_key) + display_name = stable_key + else: + basis = title or statement[:80] + canonical_key = identity.derived_entity_key(entity_type, basis) + display_name = _bound(basis, MAX_DISPLAY_NAME_CHARS) + + matches: List[Dict[str, Any]] = [] + conflicts: List[Dict[str, Any]] = [] + existing = store.find_entity_by_key(workspace_id, entity_type, + canonical_key) + if existing: + matches.append({"via": "canonical-key", "entity_id": existing["id"], + "canonical_key": existing["canonical_key"]}) + alias_entities: List[Dict[str, Any]] = [] + if stable_key: + alias_entities = store.find_entity_by_alias( + workspace_id, identity.normalize_alias(stable_key)) + for ent in alias_entities: + record = {"via": "alias", "entity_id": ent["id"], + "canonical_key": ent["canonical_key"], + "entity_type": ent["entity_type"]} + if existing and ent["id"] == existing["id"]: + continue # the key match already covers it + if existing and ent["id"] != existing["id"]: + conflicts.append({**record, "reason": + "alias resolves to a different entity than " + "the canonical key"}) + elif ent["entity_type"] != entity_type: + conflicts.append({**record, "reason": + "alias is held by an entity of a different " + "type"}) + else: + matches.append(record) + + resolved: Optional[Dict[str, Any]] = existing + if not resolved: + same_type = [e for e in alias_entities + if e["entity_type"] == entity_type] + if len(same_type) == 1 and not conflicts: + resolved = same_type[0] + elif len(same_type) > 1: + conflicts.append({ + "via": "alias", "reason": + f"alias {stable_key!r} is held by " + f"{len(same_type)} active entities", + "entity_ids": [e["id"] for e in same_type]}) + + return { + "entity_type": entity_type, + "canonical_key": canonical_key, + "display_name": display_name, + "has_identifier": has_identifier, + "stable_key": stable_key, + "existing_entity": resolved, + "identity_matches": matches, + "identity_conflicts": conflicts, + } + + +def _source_anchor(workspace_id: str, candidate: Dict[str, Any] + ) -> Tuple[str, str, List[str]]: + """(asset_id, revision_id, [segment ids of the cited evidence]).""" + revision_id = str(candidate.get("revision_id") or "") + asset_id = "" + if revision_id: + revision = db.get_revision(workspace_id, revision_id) + asset_id = (revision or {}).get("asset_id", "") + segment_ids: List[str] = [] + for ev in candidate.get("evidence") or []: + record = db.get_evidence(workspace_id, + str(ev.get("evidence_id") or "")) + seg = (record or {}).get("segment_id") + if seg and seg not in segment_ids: + segment_ids.append(seg) + return asset_id, revision_id, segment_ids + + +def _proposed_bindings(asset_id: str, revision_id: str, + segment_ids: List[str]) -> List[Dict[str, Any]]: + bindings: List[Dict[str, Any]] = [] + if asset_id: + bindings.append({"ref_kind": BindingRefKind.ASSET, "ref_id": asset_id, + "binding_role": BindingRole.PRIMARY_SOURCE}) + if revision_id: + bindings.append({"ref_kind": BindingRefKind.REVISION, + "ref_id": revision_id, + "binding_role": BindingRole.PRIMARY_SOURCE}) + for seg in segment_ids: + bindings.append({"ref_kind": BindingRefKind.SEGMENT, "ref_id": seg, + "binding_role": BindingRole.DEFINITION_SOURCE}) + return bindings + + +def _concept_claim_type(entity_type: str) -> str: + return ENTITY_TYPE_TO_CLAIM_TYPE.get(entity_type, ClaimType.DEFINITION) + + +def _document_entity_proposal(workspace_id: str, + candidate: Dict[str, Any]) -> Dict[str, Any]: + """Entity identity for classification / revision-status candidates: the + source ASSET's deterministic entity (document for parsed documents, + else the deterministic asset-type projection).""" + asset_id, revision_id, segment_ids = _source_anchor(workspace_id, + candidate) + entity_type = EntityType.DOCUMENT + display_name = "" + if asset_id: + asset = db.get_asset(workspace_id, asset_id) + if asset: + display_name = asset.get("title") or asset.get("logical_key", "") + if not db.is_document_asset(workspace_id, asset_id): + mapped = { + "source-code": EntityType.CODE_COMPONENT, + "test-source": EntityType.CODE_COMPONENT, + "configuration": EntityType.CONFIGURATION, + "database-schema": EntityType.DATA_MODEL, + "build-definition": EntityType.BUILD_DEFINITION, + }.get(asset.get("asset_type", ""), EntityType.DOCUMENT) + entity_type = mapped + canonical_key = identity.asset_entity_key(entity_type, asset_id) + return {"entity_type": entity_type, "canonical_key": canonical_key, + "display_name": _bound(display_name, MAX_DISPLAY_NAME_CHARS), + "asset_id": asset_id, "revision_id": revision_id, + "segment_ids": segment_ids} + + +# --------------------------------------------------------------------------- +# Candidate promotion plan +# --------------------------------------------------------------------------- +def plan_candidate_promotion(workspace_id: str, + candidate_id: str) -> Dict[str, Any]: + """Deterministic dry-run. No graph write, no provider call.""" + semantic_store.reconcile_staleness(workspace_id) + candidate = semantic_store.get_candidate(workspace_id, candidate_id) + if not candidate: + from ..semantic.errors import CandidateNotFound + raise CandidateNotFound( + f"semantic candidate not found: {candidate_id!r}", + details={"candidate_id": candidate_id}) + + reasons = _candidate_blocking_reasons(workspace_id, candidate) + prior = store.find_promotion( + workspace_id, PromotionCandidateKind.SEMANTIC_CANDIDATE, candidate_id) + + plan: Dict[str, Any] = { + "workspace_id": workspace_id, + "candidate_id": candidate_id, + "candidate_kind": candidate.get("candidate_kind"), + "candidate": candidate, + "eligible": not reasons, + "blocking_reasons": reasons, + "evidence": candidate.get("evidence") or [], + "identity_matches": [], + "identity_conflicts": [], + "existing_entity": None, + "existing_claim": None, + "proposed_entity": None, + "proposed_claim": None, + "proposed_aliases": [], + "proposed_bindings": [], + "promotion_policy_version": PROMOTION_POLICY_VERSION, + } + if prior: + plan["expected_action"] = PromotionExpectedAction.ALREADY_PROMOTED + plan["existing_promotion"] = prior + target = store.get_entity(workspace_id, prior["target_id"]) \ + if prior["target_kind"] == "entity" else \ + store.get_claim(workspace_id, prior["target_id"]) + plan["existing_target"] = target + return plan + if reasons: + plan["expected_action"] = PromotionExpectedAction.BLOCKED + return plan + + kind = candidate.get("candidate_kind") + if kind == SemanticCandidateKind.ENGINEERING_CONCEPT: + ident = _resolve_concept_identity(workspace_id, candidate) + plan["identity_matches"] = ident["identity_matches"] + plan["identity_conflicts"] = ident["identity_conflicts"] + plan["existing_entity"] = ident["existing_entity"] + statement = _bound(candidate.get("statement", ""), + MAX_STATEMENT_CHARS) + claim_type = _concept_claim_type(ident["entity_type"]) + plan["proposed_entity"] = { + "entity_type": ident["entity_type"], + "canonical_key": ident["canonical_key"], + "display_name": ident["display_name"], + "description": _bound(statement, MAX_DESCRIPTION_CHARS), + "origin": GraphOrigin.SEMANTIC_PROMOTION, + } + plan["proposed_claim"] = { + "claim_type": claim_type, + "statement": statement, + "normalized_statement_hash": identity.statement_hash(statement), + } + if ident["has_identifier"]: + plan["proposed_aliases"] = [{ + "alias": ident["stable_key"], + "alias_type": AliasType.IDENTIFIER}] + asset_id, revision_id, segment_ids = _source_anchor(workspace_id, + candidate) + plan["proposed_bindings"] = _proposed_bindings(asset_id, revision_id, + segment_ids) + if ident["identity_conflicts"]: + plan["eligible"] = False + plan["blocking_reasons"] = [ + "identity conflict: " + str(c.get("reason", "")) + for c in ident["identity_conflicts"]] + plan["expected_action"] = PromotionExpectedAction.IDENTITY_CONFLICT + return plan + if ident["existing_entity"]: + existing_claim = store.find_claim_by_hash( + workspace_id, ident["existing_entity"]["id"], + plan["proposed_claim"]["normalized_statement_hash"]) + plan["existing_claim"] = existing_claim + plan["expected_action"] = \ + PromotionExpectedAction.ATTACH_CLAIM_TO_EXISTING_ENTITY + else: + plan["expected_action"] = \ + PromotionExpectedAction.CREATE_ENTITY_AND_CLAIM + return plan + + # classification / revision-status: a claim on the source asset's entity + proposal = _document_entity_proposal(workspace_id, candidate) + statement = _bound(candidate.get("statement", "") or + candidate.get("title", ""), MAX_STATEMENT_CHARS) + claim_type = (ClaimType.CLASSIFICATION + if kind == SemanticCandidateKind.CLASSIFICATION + else ClaimType.REVISION_STATUS) + existing = store.find_entity_by_key(workspace_id, + proposal["entity_type"], + proposal["canonical_key"]) + plan["existing_entity"] = existing + plan["proposed_entity"] = { + "entity_type": proposal["entity_type"], + "canonical_key": proposal["canonical_key"], + "display_name": proposal["display_name"], + "description": "", + "origin": GraphOrigin.SEMANTIC_PROMOTION, + } + plan["proposed_claim"] = { + "claim_type": claim_type, + "statement": statement, + "normalized_statement_hash": identity.statement_hash(statement), + } + plan["proposed_bindings"] = _proposed_bindings( + proposal["asset_id"], proposal["revision_id"], + proposal["segment_ids"]) + if existing: + plan["existing_claim"] = store.find_claim_by_hash( + workspace_id, existing["id"], + plan["proposed_claim"]["normalized_statement_hash"]) + plan["expected_action"] = \ + PromotionExpectedAction.ATTACH_CLAIM_TO_EXISTING_ENTITY + else: + plan["expected_action"] = \ + PromotionExpectedAction.CREATE_ENTITY_AND_CLAIM + return plan + + +# --------------------------------------------------------------------------- +# Candidate promotion +# --------------------------------------------------------------------------- +def promote_candidate(workspace_id: str, candidate_id: str, *, + actor: str = "", note: str = "", + source_command: str = "") -> Dict[str, Any]: + """Execute an eligible plan transactionally. Idempotent: an already + promoted candidate returns its existing target and writes nothing.""" + note = _bound(note, MAX_NOTE_CHARS) + plan = plan_candidate_promotion(workspace_id, candidate_id) + if plan["expected_action"] == PromotionExpectedAction.ALREADY_PROMOTED: + return { + "workspace_id": workspace_id, "candidate_id": candidate_id, + "status": PromotionStatus.ALREADY_PROMOTED, + "promotion": plan.get("existing_promotion"), + "target": plan.get("existing_target"), + "knowledge_revision": store.current_revision_number(workspace_id), + } + if not plan["eligible"]: + raise PromotionBlocked(candidate_id, plan["blocking_reasons"]) + + candidate = plan["candidate"] + # Full quote re-verification against the immutable store, before the + # transaction body writes anything. + evidence_rows = verifier.candidate_evidence_rows(workspace_id, candidate) + + proposed_entity = plan["proposed_entity"] + proposed_claim = plan["proposed_claim"] + + with store.graph_transaction( + workspace_id, action=RevisionAction.CANDIDATE_PROMOTION, + actor=actor, + summary=f"promote {candidate.get('candidate_kind')} candidate " + f"{candidate_id}") as tx: + existing = plan["existing_entity"] + if existing: + entity = tx.get_entity(existing["id"]) + else: + entity = tx.insert_entity( + entity_type=proposed_entity["entity_type"], + canonical_key=proposed_entity["canonical_key"], + display_name=proposed_entity["display_name"], + description=proposed_entity["description"], + origin=GraphOrigin.SEMANTIC_PROMOTION, + promoted_from_candidate_id=candidate_id) + + existing_claim = plan.get("existing_claim") + if existing_claim: + claim = existing_claim + tx.add_claim_evidence(claim["id"], evidence_rows) + else: + claim = tx.insert_claim( + entity_id=entity["id"], + claim_type=proposed_claim["claim_type"], + statement=proposed_claim["statement"], + normalized_statement_hash= + proposed_claim["normalized_statement_hash"], + origin=GraphOrigin.SEMANTIC_PROMOTION, + promoted_from_candidate_id=candidate_id, + evidence=evidence_rows) + + # Aliases from stable identifiers (never a colliding one: identity + # conflicts blocked the plan already; an alias already on THIS + # entity is skipped). + existing_aliases = {a["normalized_alias"] for a in + store.list_aliases(workspace_id, entity["id"])} + for alias in plan["proposed_aliases"]: + normalized = identity.normalize_alias(alias["alias"]) + if normalized in existing_aliases: + continue + tx.insert_alias(entity_id=entity["id"], alias=alias["alias"], + normalized_alias=normalized, + alias_type=alias["alias_type"], + origin=GraphOrigin.SEMANTIC_PROMOTION, + evidence_id=evidence_rows[0]["evidence_id"] + if evidence_rows else "") + + # Bindings, deduplicated against the entity's existing ones. + existing_bindings = { + (b["ref_kind"], b["ref_id"], b["binding_role"]) + for b in store.list_bindings(workspace_id, entity["id"])} + for binding in plan["proposed_bindings"]: + key = (binding["ref_kind"], binding["ref_id"], + binding["binding_role"]) + if key in existing_bindings: + continue + tx.insert_binding(entity_id=entity["id"], + ref_kind=binding["ref_kind"], + ref_id=binding["ref_id"], + binding_role=binding["binding_role"], + origin=GraphOrigin.SEMANTIC_PROMOTION) + + target_kind = "entity" if candidate.get("candidate_kind") == \ + SemanticCandidateKind.ENGINEERING_CONCEPT else "claim" + target_id = entity["id"] if target_kind == "entity" else claim["id"] + promotion = tx.insert_promotion( + candidate_kind=PromotionCandidateKind.SEMANTIC_CANDIDATE, + candidate_id=candidate_id, target_kind=target_kind, + target_id=target_id, status=PromotionStatus.PROMOTED, + policy_version=PROMOTION_POLICY_VERSION, actor=actor, note=note) + tx.insert_decision( + decision_type=DecisionType.PROMOTE_CANDIDATE, + target_kind=DecisionTargetKind.CANDIDATE, + target_id=candidate_id, actor=actor, note=note, + before={"candidate_id": candidate_id, + "review_status": candidate.get("review_status")}, + after={"entity_id": entity["id"], "claim_id": claim["id"], + "expected_action": plan["expected_action"]}, + source_command=source_command) + revision_number = tx.revision_number + + return { + "workspace_id": workspace_id, "candidate_id": candidate_id, + "status": PromotionStatus.PROMOTED, + "expected_action": plan["expected_action"], + "entity": store.get_entity(workspace_id, entity["id"]), + "claim": store.get_claim(workspace_id, claim["id"]), + "promotion": promotion, + "knowledge_revision": revision_number, + } + + +# --------------------------------------------------------------------------- +# Relation promotion +# --------------------------------------------------------------------------- +def _resolve_endpoint(workspace_id: str, ref: Dict[str, Any], + candidate_id: Optional[str]) -> Dict[str, Any]: + """Deterministically resolve one relation endpoint to a canonical + Entity. Returns {resolved, entity_id, entity, via, reason, ambiguous}. + """ + out: Dict[str, Any] = {"resolved": False, "entity_id": "", "entity": None, + "via": "", "reason": "", "ambiguous": False, + "ref": dict(ref or {})} + # 1. A promoted candidate resolves to its promotion target entity. + cand_id = candidate_id or (ref.get("id") + if (ref or {}).get("kind") == "candidate" + else None) + if cand_id: + promo = store.find_promotion( + workspace_id, PromotionCandidateKind.SEMANTIC_CANDIDATE, + str(cand_id)) + if promo and promo["target_kind"] == "entity": + entity = store.get_entity(workspace_id, promo["target_id"]) + if entity and entity["lifecycle_status"] in ( + GraphLifecycleStatus.ACTIVE,): + out.update({"resolved": True, "entity_id": entity["id"], + "entity": entity, "via": "promotion"}) + return out + if entity and entity["lifecycle_status"] == \ + GraphLifecycleStatus.MERGED and \ + entity.get("merged_into_entity_id"): + target = store.get_entity(workspace_id, + entity["merged_into_entity_id"]) + if target: + out.update({"resolved": True, "entity_id": target["id"], + "entity": target, "via": "promotion+merge"}) + return out + # Fall through to key/alias resolution using the candidate's key. + cand = semantic_store.get_candidate(workspace_id, str(cand_id)) + key = str((cand or {}).get("stable_key") or (ref or {}).get("key") + or "").strip() + else: + key = str((ref or {}).get("key") or "").strip() + + kind = str((ref or {}).get("kind") or "") + # 2. A symbol ref resolves through the projector's segment binding. + if kind == "symbol" and (ref or {}).get("id"): + bindings = store.find_bindings_by_ref( + workspace_id, BindingRefKind.SEGMENT, str(ref["id"])) + holders = [] + for b in bindings: + if b["status"] != "active": + continue + ent = store.get_entity(workspace_id, b["entity_id"]) + if ent and ent["lifecycle_status"] == GraphLifecycleStatus.ACTIVE: + holders.append(ent) + unique = {e["id"]: e for e in holders} + if len(unique) == 1: + entity = next(iter(unique.values())) + out.update({"resolved": True, "entity_id": entity["id"], + "entity": entity, "via": "segment-binding"}) + return out + if len(unique) > 1: + out.update({"ambiguous": True, "reason": + f"segment {ref['id']} is bound to " + f"{len(unique)} active entities"}) + return out + # 3. Exact alias / canonical-key resolution by the ref key. + if key: + holders = store.find_entity_by_alias(workspace_id, + identity.normalize_alias(key)) + if len(holders) == 1: + out.update({"resolved": True, "entity_id": holders[0]["id"], + "entity": holders[0], "via": "alias"}) + return out + if len(holders) > 1: + out.update({"ambiguous": True, "reason": + f"key {key!r} resolves to {len(holders)} active " + f"entities"}) + return out + out["reason"] = (f"key {key!r} resolves to no canonical entity " + f"(promote the referenced candidate first, or seed " + f"the graph)") + return out + out["reason"] = "endpoint reference carries no resolvable identity" + return out + + +def _relation_blocking_reasons(workspace_id: str, candidate: Dict[str, Any], + source: Dict[str, Any], + target: Dict[str, Any]) -> List[str]: + reasons = _base_blocking_reasons(workspace_id, candidate) + relation_type = RELATION_CANDIDATE_TYPE_TO_RELATION_TYPE.get( + str(candidate.get("relation_type") or "")) + if not relation_type: + reasons.append( + f"unsupported relation type: {candidate.get('relation_type')!r}") + if not (candidate.get("evidence") or []): + reasons.append("relation candidate has no evidence joins") + for ev in candidate.get("evidence") or []: + if not db.get_evidence(workspace_id, str(ev.get("evidence_id") or "")): + reasons.append( + f"evidence {ev.get('evidence_id')} does not exist in this " + f"workspace") + if not source["resolved"]: + reasons.append("source endpoint does not resolve unambiguously" + + (f": {source['reason']}" if source.get("reason") + else "")) + if not target["resolved"]: + reasons.append("target endpoint does not resolve unambiguously" + + (f": {target['reason']}" if target.get("reason") + else "")) + if source["resolved"] and target["resolved"] and \ + source["entity_id"] == target["entity_id"]: + reasons.append("source and target resolve to the same entity " + "(self-relations are rejected)") + prior = store.find_promotion( + workspace_id, PromotionCandidateKind.RELATION_CANDIDATE, + candidate["id"]) + if prior: + reasons.append("relation candidate is already promoted") + return reasons + + +def plan_relation_promotion(workspace_id: str, + relation_candidate_id: str) -> Dict[str, Any]: + """Deterministic dry-run for one Relation Candidate.""" + semantic_store.reconcile_staleness(workspace_id) + candidate = semantic_store.get_relation(workspace_id, + relation_candidate_id) + if not candidate: + from ..semantic.errors import CandidateNotFound + raise CandidateNotFound( + f"relation candidate not found: {relation_candidate_id!r}", + details={"candidate_id": relation_candidate_id}) + + source = _resolve_endpoint(workspace_id, + candidate.get("source_ref") or {}, + candidate.get("source_candidate_id")) + target = _resolve_endpoint(workspace_id, + candidate.get("target_ref") or {}, + candidate.get("target_candidate_id")) + prior = store.find_promotion( + workspace_id, PromotionCandidateKind.RELATION_CANDIDATE, + relation_candidate_id) + reasons = _relation_blocking_reasons(workspace_id, candidate, source, + target) + plan: Dict[str, Any] = { + "workspace_id": workspace_id, + "relation_candidate_id": relation_candidate_id, + "candidate": candidate, + "eligible": not reasons, + "blocking_reasons": reasons, + "source_resolution": source, + "target_resolution": target, + "proposed_relation": None, + "existing_relation": None, + "promotion_policy_version": PROMOTION_POLICY_VERSION, + } + if prior: + plan["expected_action"] = PromotionExpectedAction.ALREADY_PROMOTED + plan["existing_promotion"] = prior + plan["existing_relation"] = store.get_relation(workspace_id, + prior["target_id"]) + plan["eligible"] = False + return plan + if reasons: + plan["expected_action"] = PromotionExpectedAction.BLOCKED + return plan + relation_type = RELATION_CANDIDATE_TYPE_TO_RELATION_TYPE[ + candidate["relation_type"]] + plan["proposed_relation"] = { + "source_entity_id": source["entity_id"], + "target_entity_id": target["entity_id"], + "relation_type": relation_type, + "relation_state": RelationState.CONFIRMED, + "confidence": candidate.get("confidence", "low"), + "origin": GraphOrigin.SEMANTIC_PROMOTION, + } + existing = store.find_active_relation(workspace_id, source["entity_id"], + target["entity_id"], relation_type) + plan["existing_relation"] = existing + plan["expected_action"] = PromotionExpectedAction.CREATE_RELATION + return plan + + +def promote_relation(workspace_id: str, relation_candidate_id: str, *, + actor: str = "", note: str = "", + source_command: str = "") -> Dict[str, Any]: + """Execute an eligible relation plan transactionally. Idempotent.""" + note = _bound(note, MAX_NOTE_CHARS) + plan = plan_relation_promotion(workspace_id, relation_candidate_id) + if plan["expected_action"] == PromotionExpectedAction.ALREADY_PROMOTED: + return { + "workspace_id": workspace_id, + "relation_candidate_id": relation_candidate_id, + "status": PromotionStatus.ALREADY_PROMOTED, + "promotion": plan.get("existing_promotion"), + "relation": plan.get("existing_relation"), + "knowledge_revision": store.current_revision_number(workspace_id), + } + if not plan["eligible"]: + raise PromotionBlocked(relation_candidate_id, + plan["blocking_reasons"]) + + candidate = plan["candidate"] + evidence_rows = verifier.candidate_evidence_rows(workspace_id, candidate) + proposed = plan["proposed_relation"] + + with store.graph_transaction( + workspace_id, action=RevisionAction.RELATION_PROMOTION, + actor=actor, + summary=f"promote relation candidate " + f"{relation_candidate_id}") as tx: + existing = plan["existing_relation"] + if existing: + relation = existing + # Reuse the active relation; upgrade its state to confirmed and + # attach the newly verified evidence. + updates: Dict[str, Any] = {} + if existing["relation_state"] != RelationState.CONFIRMED: + updates["relation_state"] = RelationState.CONFIRMED + if not existing.get("promoted_from_relation_candidate_id"): + updates["promoted_from_relation_candidate_id"] = \ + relation_candidate_id + if updates: + tx.update_relation(existing["id"], **updates) + tx.add_relation_evidence(existing["id"], evidence_rows) + else: + relation = tx.insert_relation( + source_entity_id=proposed["source_entity_id"], + target_entity_id=proposed["target_entity_id"], + relation_type=proposed["relation_type"], + relation_state=RelationState.CONFIRMED, + confidence=proposed["confidence"], + origin=GraphOrigin.SEMANTIC_PROMOTION, + promoted_from_relation_candidate_id=relation_candidate_id, + evidence=evidence_rows) + promotion = tx.insert_promotion( + candidate_kind=PromotionCandidateKind.RELATION_CANDIDATE, + candidate_id=relation_candidate_id, target_kind="relation", + target_id=relation["id"], status=PromotionStatus.PROMOTED, + policy_version=PROMOTION_POLICY_VERSION, actor=actor, note=note) + tx.insert_decision( + decision_type=DecisionType.PROMOTE_RELATION, + target_kind=DecisionTargetKind.RELATION_CANDIDATE, + target_id=relation_candidate_id, actor=actor, note=note, + before={"relation_candidate_id": relation_candidate_id}, + after={"relation_id": relation["id"], + "relation_type": relation["relation_type"], + "relation_state": RelationState.CONFIRMED}, + source_command=source_command) + revision_number = tx.revision_number + + return { + "workspace_id": workspace_id, + "relation_candidate_id": relation_candidate_id, + "status": PromotionStatus.PROMOTED, + "relation": store.get_relation(workspace_id, relation["id"]), + "promotion": promotion, + "knowledge_revision": revision_number, + } + + +__all__ = [ + "plan_candidate_promotion", "promote_candidate", + "plan_relation_promotion", "promote_relation", + "MAX_STATEMENT_CHARS", "MAX_NOTE_CHARS", +] diff --git a/openmind/knowledge/reconciliation.py b/openmind/knowledge/reconciliation.py new file mode 100644 index 0000000..b8dd3a0 --- /dev/null +++ b/openmind/knowledge/reconciliation.py @@ -0,0 +1,191 @@ +"""Canonical graph staleness reconciliation. + +The graph must never silently drift from the source plane: when a Revision +stops being current (a new revision landed, or its Asset was removed), the +graph records that depended on it become ``stale`` — excluded from active +queries by default, still fully queryable, authority and history intact. + +RULES (spec §22, each tested in verify_knowledge_staleness) +----------------------------------------------------------- +1. bindings referencing a non-current Revision (directly, or through a + Segment of a non-current Revision) -> stale; bindings whose Revision is + current again (a removed Asset reappeared) -> revived; +2. active Claims none of whose Evidence sits on a current Revision -> stale; + stale Claims with current Evidence again -> revived; +3. active Relations whose endpoints are no longer active, or all of whose + Evidence is non-current -> stale (epistemic state parked in metadata and + restored on revival); +4. deterministic Entities with no active bindings -> stale; + promoted Entities with no active bindings AND no active Claims -> stale; + MANUAL Entities are never auto-staled — staleness for them is a + governance decision (withdraw/supersede), not an inference; +5. nothing is deleted; authority_status is never touched; review history and + promotion provenance stay intact. + +Everything is set-based SQL over the v0006 indexes inside ONE graph +transaction. A pass that changes nothing writes nothing and mints no +Knowledge Revision. Safe to run any number of times. +""" +from __future__ import annotations + +from typing import Any, Dict + +from . import store +from .vocabularies import GraphOrigin, RevisionAction + +#: The set of CURRENT revisions: active assets only, so a removed Asset's +#: revision no longer anchors graph knowledge (and re-anchors on reappearance). +_CURRENT_REVISIONS = ("SELECT current_revision_id FROM assets " + "WHERE workspace_id=? AND state='active' " + "AND current_revision_id IS NOT NULL") + +#: One Evidence row of the claim/relation resolves to a current revision. +_CLAIM_HAS_CURRENT_EVIDENCE = ( + "EXISTS (SELECT 1 FROM engineering_claim_evidence ce " + "JOIN evidence ev ON ev.id = ce.evidence_id " + f"WHERE ce.claim_id = engineering_claims.id " + f"AND ev.revision_id IN ({_CURRENT_REVISIONS}))") + +_RELATION_HAS_EVIDENCE = ( + "EXISTS (SELECT 1 FROM engineering_relation_evidence re " + "WHERE re.relation_id = engineering_relations.id)") + +_RELATION_HAS_CURRENT_EVIDENCE = ( + "EXISTS (SELECT 1 FROM engineering_relation_evidence re " + "JOIN evidence ev ON ev.id = re.evidence_id " + f"WHERE re.relation_id = engineering_relations.id " + f"AND ev.revision_id IN ({_CURRENT_REVISIONS}))") + + +def reconcile_graph_staleness(workspace_id: str, *, + actor: str = "") -> Dict[str, Any]: + """One incremental reconciliation pass. Returns the change counts (all + zero => no transaction was committed and no revision was minted).""" + ws = workspace_id + with store.graph_transaction(ws, action=RevisionAction.GRAPH_RECONCILE, + actor=actor, + summary="graph staleness reconciliation" + ) as tx: + ts = tx.ts + # -- 1. bindings ---------------------------------------------------- + stale_rev_bindings = tx.execute_counted( + "UPDATE engineering_entity_bindings SET status='stale', " + "stale_at=?, updated_knowledge_revision=?, updated_at=? " + "WHERE workspace_id=? AND status='active' AND " + f"ref_kind='revision' AND ref_id NOT IN ({_CURRENT_REVISIONS})", + (ts, tx.revision_number, ts, ws, ws), "bindings") + stale_seg_bindings = tx.execute_counted( + "UPDATE engineering_entity_bindings SET status='stale', " + "stale_at=?, updated_knowledge_revision=?, updated_at=? " + "WHERE workspace_id=? AND status='active' AND ref_kind='segment' " + "AND ref_id NOT IN (SELECT s.id FROM segments s " + f"WHERE s.revision_id IN ({_CURRENT_REVISIONS}))", + (ts, tx.revision_number, ts, ws, ws), "bindings") + revived_bindings = tx.execute_counted( + "UPDATE engineering_entity_bindings SET status='active', " + "stale_at=NULL, updated_knowledge_revision=?, updated_at=? " + "WHERE workspace_id=? AND status='stale' AND (" + f"(ref_kind='revision' AND ref_id IN ({_CURRENT_REVISIONS})) OR " + "(ref_kind='segment' AND ref_id IN (SELECT s.id FROM segments s " + f"WHERE s.revision_id IN ({_CURRENT_REVISIONS}))))", + (tx.revision_number, ts, ws, ws, ws), "bindings") + + # -- 2. claims ------------------------------------------------------ + stale_claims = tx.execute_counted( + "UPDATE engineering_claims SET lifecycle_status='stale', " + "stale_at=?, updated_knowledge_revision=?, updated_at=? " + "WHERE workspace_id=? AND lifecycle_status='active' AND " + "EXISTS (SELECT 1 FROM engineering_claim_evidence ce " + "WHERE ce.claim_id = engineering_claims.id) AND NOT " + + _CLAIM_HAS_CURRENT_EVIDENCE, + (ts, tx.revision_number, ts, ws, ws), "claims") + revived_claims = tx.execute_counted( + "UPDATE engineering_claims SET lifecycle_status='active', " + "stale_at=NULL, updated_knowledge_revision=?, updated_at=? " + "WHERE workspace_id=? AND lifecycle_status='stale' AND " + + _CLAIM_HAS_CURRENT_EVIDENCE, + (tx.revision_number, ts, ws, ws), "claims") + + # -- 3. relations --------------------------------------------------- + # A relation goes stale when an endpoint is no longer active, or when + # it HAS evidence and none of it is current. The prior epistemic + # state is parked in metadata by the JSON rewrite below only for the + # rows this statement touches. + stale_relations = tx.execute_counted( + "UPDATE engineering_relations SET lifecycle_status='stale', " + "metadata_json=json_set(COALESCE(metadata_json,'{}'), " + "'$.prior_state', relation_state), relation_state='stale', " + "stale_at=?, updated_knowledge_revision=?, updated_at=? " + "WHERE workspace_id=? AND lifecycle_status='active' AND (" + "source_entity_id IN (SELECT id FROM engineering_entities WHERE " + "workspace_id=? AND lifecycle_status != 'active') OR " + "target_entity_id IN (SELECT id FROM engineering_entities WHERE " + "workspace_id=? AND lifecycle_status != 'active') OR " + f"({_RELATION_HAS_EVIDENCE} AND NOT " + f"{_RELATION_HAS_CURRENT_EVIDENCE}))", + (ts, tx.revision_number, ts, ws, ws, ws, ws), "relations") + revived_relations = tx.execute_counted( + "UPDATE engineering_relations SET lifecycle_status='active', " + "relation_state=COALESCE(json_extract(metadata_json, " + "'$.prior_state'), 'inferred'), stale_at=NULL, " + "updated_knowledge_revision=?, updated_at=? " + "WHERE workspace_id=? AND lifecycle_status='stale' AND " + "source_entity_id IN (SELECT id FROM engineering_entities WHERE " + "workspace_id=? AND lifecycle_status='active') AND " + "target_entity_id IN (SELECT id FROM engineering_entities WHERE " + "workspace_id=? AND lifecycle_status='active') AND " + f"(NOT {_RELATION_HAS_EVIDENCE} OR " + f"{_RELATION_HAS_CURRENT_EVIDENCE})", + (tx.revision_number, ts, ws, ws, ws, ws), "relations") + + # -- 4. entities ---------------------------------------------------- + stale_det_entities = tx.execute_counted( + "UPDATE engineering_entities SET lifecycle_status='stale', " + "stale_at=?, updated_knowledge_revision=?, updated_at=? " + "WHERE workspace_id=? AND lifecycle_status='active' AND " + f"origin='{GraphOrigin.DETERMINISTIC}' AND NOT EXISTS (" + "SELECT 1 FROM engineering_entity_bindings b WHERE " + "b.entity_id = engineering_entities.id AND b.status='active')", + (ts, tx.revision_number, ts, ws), "entities") + stale_promoted_entities = tx.execute_counted( + "UPDATE engineering_entities SET lifecycle_status='stale', " + "stale_at=?, updated_knowledge_revision=?, updated_at=? " + "WHERE workspace_id=? AND lifecycle_status='active' AND " + f"origin='{GraphOrigin.SEMANTIC_PROMOTION}' AND NOT EXISTS (" + "SELECT 1 FROM engineering_entity_bindings b WHERE " + "b.entity_id = engineering_entities.id AND b.status='active') " + "AND NOT EXISTS (SELECT 1 FROM engineering_claims c WHERE " + "c.entity_id = engineering_entities.id AND " + "c.lifecycle_status='active')", + (ts, tx.revision_number, ts, ws), "entities") + revived_entities = tx.execute_counted( + "UPDATE engineering_entities SET lifecycle_status='active', " + "stale_at=NULL, updated_knowledge_revision=?, updated_at=? " + "WHERE workspace_id=? AND lifecycle_status='stale' AND " + f"origin IN ('{GraphOrigin.DETERMINISTIC}', " + f"'{GraphOrigin.SEMANTIC_PROMOTION}') AND (EXISTS (" + "SELECT 1 FROM engineering_entity_bindings b WHERE " + "b.entity_id = engineering_entities.id AND b.status='active') " + "OR EXISTS (SELECT 1 FROM engineering_claims c WHERE " + "c.entity_id = engineering_entities.id AND " + "c.lifecycle_status='active'))", + (tx.revision_number, ts, ws), "entities") + revision_number = tx.revision_number + wrote = tx.wrote + + return { + "workspace_id": ws, + "stale_bindings": stale_rev_bindings + stale_seg_bindings, + "revived_bindings": revived_bindings, + "stale_claims": stale_claims, "revived_claims": revived_claims, + "stale_relations": stale_relations, + "revived_relations": revived_relations, + "stale_entities": stale_det_entities + stale_promoted_entities, + "revived_entities": revived_entities, + "changed": wrote, + "knowledge_revision": (revision_number if wrote else + store.current_revision_number(ws)), + } + + +__all__ = ["reconcile_graph_staleness"] diff --git a/openmind/knowledge/search.py b/openmind/knowledge/search.py new file mode 100644 index 0000000..157e8d7 --- /dev/null +++ b/openmind/knowledge/search.py @@ -0,0 +1,172 @@ +"""Graph search: deterministic fusion of exact, lexical and vector signals. + +Precedence is fixed and tested (spec §30): an exact canonical key beats an +exact alias, which beats an exact identifier token, which beats a lexical +substring, which beats vector similarity. An exact identifier can therefore +never be outranked by something that merely reads similar. Entities and +Claims are returned as SEPARATE sections, each hit carrying either Evidence +ids or a navigable Claim id, plus the current Knowledge Revision. +""" +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional + +from . import identity, store, vector_projection +from .vocabularies import GraphLifecycleStatus + +#: Deterministic tier scores. Vector similarity is scaled into (0, TIER_GAP) +#: so it can never cross into the lexical tier above it. +SCORE_EXACT_KEY = 100.0 +SCORE_EXACT_ALIAS = 90.0 +SCORE_EXACT_TOKEN = 80.0 +SCORE_LEXICAL = 50.0 +SCORE_VECTOR_MAX = 40.0 + +MAX_LIMIT = 100 + + +def _token_pattern(query: str) -> Optional[re.Pattern]: + text = str(query or "").strip() + if not text or " " in text: + return None + return re.compile(r"(? Dict[str, Any]: + """One fused search over active Entities and Claims.""" + query = str(query or "").strip() + limit = max(1, min(int(limit), MAX_LIMIT)) + lifecycle = None if include_stale else GraphLifecycleStatus.ACTIVE + + entity_hits: Dict[str, Dict[str, Any]] = {} + claim_hits: Dict[str, Dict[str, Any]] = {} + + def add_entity(entity: Dict[str, Any], score: float, via: str) -> None: + if not include_stale and entity["lifecycle_status"] != \ + GraphLifecycleStatus.ACTIVE: + return + hit = entity_hits.get(entity["id"]) + if hit is None or score > hit["score"]: + entity_hits[entity["id"]] = {"entity": entity, "score": score, + "via": via} + + def add_claim(claim: Dict[str, Any], score: float, via: str) -> None: + if not include_stale and claim["lifecycle_status"] != \ + GraphLifecycleStatus.ACTIVE: + return + hit = claim_hits.get(claim["id"]) + if hit is None or score > hit["score"]: + claim_hits[claim["id"]] = {"claim": claim, "score": score, + "via": via} + + if query: + # 1. exact canonical key. Keys embed their type as the first segment + # ("requirement:REQ-NC-017"), so a full-key query resolves directly + # through the identity index; the lexical pass below additionally + # catches case-insensitive full-key matches. + if ":" in query: + direct = store.find_entity_by_key( + workspace_id, query.split(":", 1)[0].strip().lower(), query) + if direct: + add_entity(direct, SCORE_EXACT_KEY, "canonical-key") + for entity in store.search_entities_lexical( + workspace_id, query, lifecycle_status=lifecycle, + limit=MAX_LIMIT): + if entity["canonical_key"].lower() == query.lower(): + add_entity(entity, SCORE_EXACT_KEY, "canonical-key") + + # 2. exact normalized alias. + for entity in store.find_entity_by_alias( + workspace_id, identity.normalize_alias(query)): + add_entity(entity, SCORE_EXACT_ALIAS, "alias") + + # 3 + 4. token / lexical over entities and claims. + token = _token_pattern(query) + for entity in store.search_entities_lexical( + workspace_id, query, lifecycle_status=lifecycle, + limit=MAX_LIMIT): + haystack = (f"{entity['canonical_key']} " + f"{entity['display_name']} " + f"{entity.get('description', '')}") + if token and token.search(haystack): + add_entity(entity, SCORE_EXACT_TOKEN, "identifier-token") + else: + add_entity(entity, SCORE_LEXICAL, "lexical") + for claim in store.search_claims_lexical( + workspace_id, query, lifecycle_status=lifecycle, + limit=MAX_LIMIT): + if token and token.search(claim["statement"]): + add_claim(claim, SCORE_EXACT_TOKEN, "identifier-token") + else: + add_claim(claim, SCORE_LEXICAL, "lexical") + + # 5. vector similarity (never crosses into the lexical tier). + try: + vector_hits = vector_projection.query(workspace_id, query, + limit=MAX_LIMIT) + except Exception: + vector_hits = [] # projection absent/failed: exact still works + for hit in vector_hits: + similarity = max(0.0, min(1.0, hit.get("similarity", 0.0))) + score = similarity * SCORE_VECTOR_MAX + kind = (hit.get("metadata") or {}).get("object_kind", "") + if kind == "entity": + entity = store.get_entity(workspace_id, hit["id"]) + if entity: + add_entity(entity, score, "vector") + elif kind == "claim": + claim = store.get_claim(workspace_id, hit["id"]) + if claim: + add_claim(claim, score, "vector") + + def entity_result(hit: Dict[str, Any]) -> Dict[str, Any]: + entity = hit["entity"] + claims = store.list_claims( + workspace_id, entity_id=entity["id"], + lifecycle_status=GraphLifecycleStatus.ACTIVE, limit=5) + return { + "id": entity["id"], "object_kind": "entity", + "entity_type": entity["entity_type"], + "canonical_key": entity["canonical_key"], + "display_name": entity["display_name"], + "lifecycle_status": entity["lifecycle_status"], + "authority_status": entity["authority_status"], + "origin": entity["origin"], + "score": round(hit["score"], 3), "matched_via": hit["via"], + "claim_ids": [c["id"] for c in claims], + } + + def claim_result(hit: Dict[str, Any]) -> Dict[str, Any]: + claim = hit["claim"] + evidence = store.claim_evidence(claim["id"]) + return { + "id": claim["id"], "object_kind": "claim", + "entity_id": claim["entity_id"], + "claim_type": claim["claim_type"], + "statement": claim["statement"], + "lifecycle_status": claim["lifecycle_status"], + "authority_status": claim["authority_status"], + "origin": claim["origin"], + "score": round(hit["score"], 3), "matched_via": hit["via"], + "evidence_ids": [e["evidence_id"] for e in evidence], + } + + entities = sorted(entity_hits.values(), + key=lambda h: (-h["score"], + h["entity"]["canonical_key"], + h["entity"]["id"]))[:limit] + claims = sorted(claim_hits.values(), + key=lambda h: (-h["score"], h["claim"]["id"]))[:limit] + return { + "workspace_id": workspace_id, + "query": query, + "entities": [entity_result(h) for h in entities], + "claims": [claim_result(h) for h in claims], + "knowledge_revision": store.current_revision_number(workspace_id), + } + + +__all__ = ["search_graph"] diff --git a/openmind/knowledge/service.py b/openmind/knowledge/service.py new file mode 100644 index 0000000..7233de2 --- /dev/null +++ b/openmind/knowledge/service.py @@ -0,0 +1,1078 @@ +"""KnowledgeService — the application service over the canonical graph. + +Exposed as ``runtime.knowledge`` / ``ServiceContainer.knowledge`` and shared +by the CLI, REST and (read-only subset) MCP adapters. Every method validates +the workspace first (typed :class:`WorkspaceNotFound`) and then reads/writes +only through the workspace-scoped store — a graph object of workspace A +resolves to nothing through workspace B. + +GOVERNANCE DISCIPLINE +--------------------- +Every mutating method runs one graph transaction (one Knowledge Revision), +records one Human Decision with the caller-supplied actor (never inferred) +and a bounded note, and refreshes the vector projection AFTER commit — +best-effort, because the projection is derived state and its failure must +never roll back canonical SQLite truth. + +The revision ledger and decision records themselves live in +:mod:`openmind.knowledge.store` (the ``GraphTransaction``); this module is +the orchestration and validation layer above them. +""" +from __future__ import annotations + +import sys +from typing import Any, Callable, Dict, List, Optional, Sequence + +from ..domain.errors import InvalidRequest +from . import (graph, identity, projector, promotion, reconciliation, search, + store, vector_projection, verifier) +from .errors import (AliasCollision, ClaimNotFound, EntityNotFound, + GraphConflict, KnowledgeRevisionNotFound, + RelationNotFound) +from .vocabularies import (AliasStatus, AliasType, AuthorityStatus, + BindingRefKind, BindingRole, ClaimType, + DecisionTargetKind, DecisionType, EntityType, + GraphLifecycleStatus, GraphOrigin, RelationState, + RelationType, RevisionAction, require) + +MAX_NOTE_CHARS = 2_000 +MAX_LIST_LIMIT = 500 +MAX_STATEMENT_CHARS = promotion.MAX_STATEMENT_CHARS +MAX_DISPLAY_NAME_CHARS = 200 +MAX_DESCRIPTION_CHARS = 2_000 + + +class KnowledgeService: + """Use cases over the Phase 5 canonical Engineering Knowledge Graph.""" + + def __init__(self, workspaces: Any, + ensure_worker: Optional[Callable[[], None]] = None) -> None: + self._workspaces = workspaces + self._ensure_worker = ensure_worker + + # -- helpers ------------------------------------------------------------ + def _require_workspace(self, workspace_id: str) -> Dict[str, Any]: + return self._workspaces.get(workspace_id) + + @staticmethod + def _bound(limit: Any, hard: int = MAX_LIST_LIMIT) -> int: + try: + limit = int(limit) + except (TypeError, ValueError): + limit = hard + return max(1, min(limit, hard)) + + @staticmethod + def _note(note: str) -> str: + note = str(note or "") + if len(note) > MAX_NOTE_CHARS: + raise InvalidRequest( + f"note exceeds {MAX_NOTE_CHARS} characters", + details={"chars": len(note)}) + return note + + @staticmethod + def _actor(actor: str) -> str: + return str(actor or "")[:200] + + def _refresh_projection(self, workspace_id: str) -> None: + """Best-effort vector refresh after a committed graph change.""" + try: + vector_projection.refresh_workspace(workspace_id) + except Exception as exc: + print(f"[knowledge] vector projection refresh failed for " + f"{workspace_id}: {exc}", file=sys.stderr, flush=True) + + def _require_entity(self, workspace_id: str, + entity_id: str) -> Dict[str, Any]: + entity = store.get_entity(workspace_id, entity_id) + if not entity: + raise EntityNotFound(entity_id, workspace_id=workspace_id) + return entity + + def _require_claim(self, workspace_id: str, + claim_id: str) -> Dict[str, Any]: + claim = store.get_claim(workspace_id, claim_id) + if not claim: + raise ClaimNotFound(claim_id, workspace_id=workspace_id) + return claim + + def _require_relation(self, workspace_id: str, + relation_id: str) -> Dict[str, Any]: + relation = store.get_relation(workspace_id, relation_id) + if not relation: + raise RelationNotFound(relation_id, workspace_id=workspace_id) + return relation + + @staticmethod + def _entity_snapshot(entity: Dict[str, Any]) -> Dict[str, Any]: + """A bounded before/after snapshot for decision records — graph + fields only, no free-form payloads, nothing secret to include.""" + return {k: entity.get(k) for k in + ("id", "entity_type", "canonical_key", "display_name", + "lifecycle_status", "authority_status", "origin", + "merged_into_entity_id", "superseded_by_entity_id")} + + # ====================================================================== + # Reads + # ====================================================================== + def get_stats(self, workspace_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + # Active statistics must not count knowledge whose sources moved on. + reconciliation.reconcile_graph_staleness(workspace_id) + result = store.stats(workspace_id) + result["workspace_id"] = workspace_id + state = store.get_projection_state(workspace_id) + result["projection"] = state or {"projector_version": "", + "last_synced_at": None} + return result + + def get_current_revision(self, workspace_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + return {"workspace_id": workspace_id, + "knowledge_revision": + store.current_revision_number(workspace_id)} + + def list_entities(self, workspace_id: str, *, + entity_type: Optional[str] = None, + lifecycle_status: Optional[str] = "active", + authority_status: Optional[str] = None, + origin: Optional[str] = None, + limit: int = 100, offset: int = 0) -> Dict[str, Any]: + self._require_workspace(workspace_id) + if entity_type: + entity_type = require(EntityType, entity_type, + field="entity type") + rows = store.list_entities( + workspace_id, entity_type=entity_type, + lifecycle_status=lifecycle_status or None, + authority_status=authority_status, origin=origin, + limit=self._bound(limit), offset=max(0, int(offset))) + total = store.count_entities( + workspace_id, entity_type=entity_type, + lifecycle_status=lifecycle_status or None, + authority_status=authority_status, origin=origin) + return {"workspace_id": workspace_id, "entities": rows, + "count": len(rows), "total": total, + "knowledge_revision": + store.current_revision_number(workspace_id)} + + def get_entity(self, workspace_id: str, entity_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + entity = self._require_entity(workspace_id, entity_id) + out = dict(entity) + out["aliases"] = store.list_aliases(workspace_id, entity_id, + status=None) + out["bindings"] = store.list_bindings(workspace_id, entity_id) + out["claims"] = store.list_claims(workspace_id, entity_id=entity_id, + limit=100) + out["relations"] = store.list_relations(workspace_id, + entity_id=entity_id, + limit=200) + out["knowledge_revision"] = store.current_revision_number( + workspace_id) + return out + + def list_claims(self, workspace_id: str, *, + entity_id: Optional[str] = None, + claim_type: Optional[str] = None, + lifecycle_status: Optional[str] = "active", + limit: int = 100, offset: int = 0) -> Dict[str, Any]: + self._require_workspace(workspace_id) + if claim_type: + claim_type = require(ClaimType, claim_type, field="claim type") + rows = store.list_claims( + workspace_id, entity_id=entity_id, claim_type=claim_type, + lifecycle_status=lifecycle_status or None, + limit=self._bound(limit), offset=max(0, int(offset))) + total = store.count_claims( + workspace_id, entity_id=entity_id, claim_type=claim_type, + lifecycle_status=lifecycle_status or None) + return {"workspace_id": workspace_id, "claims": rows, + "count": len(rows), "total": total, + "knowledge_revision": + store.current_revision_number(workspace_id)} + + def get_claim(self, workspace_id: str, claim_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + claim = self._require_claim(workspace_id, claim_id) + claim["knowledge_revision"] = store.current_revision_number( + workspace_id) + return claim + + def list_relations(self, workspace_id: str, *, + entity_id: Optional[str] = None, + relation_type: Optional[str] = None, + relation_state: Optional[str] = None, + lifecycle_status: Optional[str] = "active", + limit: int = 100, offset: int = 0) -> Dict[str, Any]: + self._require_workspace(workspace_id) + if relation_type: + relation_type = require(RelationType, relation_type, + field="relation type") + rows = store.list_relations( + workspace_id, entity_id=entity_id, relation_type=relation_type, + relation_state=relation_state, + lifecycle_status=lifecycle_status or None, + limit=self._bound(limit), offset=max(0, int(offset))) + return {"workspace_id": workspace_id, "relations": rows, + "count": len(rows), + "knowledge_revision": + store.current_revision_number(workspace_id)} + + def get_relation(self, workspace_id: str, + relation_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + relation = self._require_relation(workspace_id, relation_id) + relation["knowledge_revision"] = store.current_revision_number( + workspace_id) + return relation + + def get_node(self, workspace_id: str, node_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + return graph.get_node(workspace_id, node_id) + + def expand_node(self, workspace_id: str, node_id: str, + **options: Any) -> Dict[str, Any]: + self._require_workspace(workspace_id) + return graph.expand_node(workspace_id, node_id, **options) + + def find_path(self, workspace_id: str, source_id: str, target_id: str, + **options: Any) -> Dict[str, Any]: + self._require_workspace(workspace_id) + return graph.find_path(workspace_id, source_id, target_id, **options) + + def get_subgraph(self, workspace_id: str, node_ids: Sequence[str], + **options: Any) -> Dict[str, Any]: + self._require_workspace(workspace_id) + return graph.get_subgraph(workspace_id, node_ids, **options) + + def search_entities(self, workspace_id: str, query: str, *, + limit: int = 20, + include_stale: bool = False) -> Dict[str, Any]: + self._require_workspace(workspace_id) + return search.search_graph(workspace_id, query, + limit=self._bound(limit, 100), + include_stale=include_stale) + + # ====================================================================== + # Promotion + # ====================================================================== + def plan_candidate_promotion(self, workspace_id: str, + candidate_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + return promotion.plan_candidate_promotion(workspace_id, candidate_id) + + def promote_candidate(self, workspace_id: str, candidate_id: str, *, + actor: str = "", note: str = "", + source_command: str = "") -> Dict[str, Any]: + self._require_workspace(workspace_id) + result = promotion.promote_candidate( + workspace_id, candidate_id, actor=self._actor(actor), + note=self._note(note), source_command=source_command) + if result.get("status") == "promoted": + self._refresh_projection(workspace_id) + return result + + def plan_relation_promotion(self, workspace_id: str, + relation_candidate_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + return promotion.plan_relation_promotion(workspace_id, + relation_candidate_id) + + def promote_relation(self, workspace_id: str, + relation_candidate_id: str, *, actor: str = "", + note: str = "", + source_command: str = "") -> Dict[str, Any]: + self._require_workspace(workspace_id) + result = promotion.promote_relation( + workspace_id, relation_candidate_id, actor=self._actor(actor), + note=self._note(note), source_command=source_command) + if result.get("status") == "promoted": + self._refresh_projection(workspace_id) + return result + + # ====================================================================== + # Deterministic projection + staleness + # ====================================================================== + def plan_seed(self, workspace_id: str) -> Dict[str, Any]: + self._require_workspace(workspace_id) + return projector.plan_seed(workspace_id) + + def seed(self, workspace_id: str, *, actor: str = "") -> Dict[str, Any]: + self._require_workspace(workspace_id) + result = projector.seed_graph(workspace_id, + actor=self._actor(actor)) + if result.get("changed"): + self._refresh_projection(workspace_id) + return result + + def sync(self, workspace_id: str, *, actor: str = "") -> Dict[str, Any]: + self._require_workspace(workspace_id) + result = projector.sync_graph(workspace_id, + actor=self._actor(actor)) + if result.get("changed"): + self._refresh_projection(workspace_id) + return result + + def reconcile_staleness(self, workspace_id: str, *, + actor: str = "") -> Dict[str, Any]: + self._require_workspace(workspace_id) + result = reconciliation.reconcile_graph_staleness( + workspace_id, actor=self._actor(actor)) + if result.get("changed"): + self._refresh_projection(workspace_id) + return result + + # ====================================================================== + # History + # ====================================================================== + def list_knowledge_revisions(self, workspace_id: str, *, + limit: int = 50, + offset: int = 0) -> Dict[str, Any]: + self._require_workspace(workspace_id) + rows = store.list_revisions(workspace_id, limit=self._bound(limit), + offset=max(0, int(offset))) + return {"workspace_id": workspace_id, "revisions": rows, + "count": len(rows), + "knowledge_revision": + store.current_revision_number(workspace_id)} + + def get_knowledge_revision(self, workspace_id: str, + revision_number: int) -> Dict[str, Any]: + self._require_workspace(workspace_id) + row = store.get_revision_by_number(workspace_id, revision_number) + if not row: + raise KnowledgeRevisionNotFound(int(revision_number), + workspace_id=workspace_id) + row["decisions"] = store.list_decisions(workspace_id, limit=100) + row["decisions"] = [d for d in row["decisions"] + if d["knowledge_revision_id"] == row["id"]] + return row + + def list_decisions(self, workspace_id: str, *, + target_kind: Optional[str] = None, + target_id: Optional[str] = None, + decision_type: Optional[str] = None, + limit: int = 100, offset: int = 0) -> Dict[str, Any]: + self._require_workspace(workspace_id) + rows = store.list_decisions( + workspace_id, target_kind=target_kind, target_id=target_id, + decision_type=decision_type, limit=self._bound(limit), + offset=max(0, int(offset))) + return {"workspace_id": workspace_id, "decisions": rows, + "count": len(rows)} + + def list_promotions(self, workspace_id: str, *, + limit: int = 100) -> Dict[str, Any]: + self._require_workspace(workspace_id) + rows = store.list_promotions(workspace_id, + limit=self._bound(limit)) + return {"workspace_id": workspace_id, "promotions": rows, + "count": len(rows)} + + # ====================================================================== + # Manual creation + # ====================================================================== + def create_entity(self, workspace_id: str, *, entity_type: str, + canonical_key: str, display_name: str, + evidence: Sequence[Dict[str, Any]], + actor: str, note: str, description: str = "", + source_command: str = "") -> Dict[str, Any]: + """Manual Entity creation. Requires at least one valid Evidence — + the structural-container exception applies ONLY to deterministic + projection, never to a manual caller.""" + self._require_workspace(workspace_id) + entity_type = require(EntityType, entity_type, field="entity type") + canonical_key = str(canonical_key or "").strip() + if not canonical_key: + raise InvalidRequest("canonical_key is required") + display_name = str(display_name or "").strip() + if not display_name: + raise InvalidRequest("display_name is required") + note = self._note(note) + rows = verifier.verify_evidence_refs(workspace_id, evidence, + require_nonempty=True) + existing = store.find_entity_by_key(workspace_id, entity_type, + canonical_key) + if existing: + raise GraphConflict( + f"an entity with canonical key {canonical_key!r} already " + f"exists: {existing['id']}", + details={"entity_id": existing["id"]}) + with store.graph_transaction( + workspace_id, action=RevisionAction.MANUAL_ENTITY_CREATE, + actor=self._actor(actor), + summary=f"manual entity {canonical_key}") as tx: + entity = tx.insert_entity( + entity_type=entity_type, canonical_key=canonical_key, + display_name=display_name[:MAX_DISPLAY_NAME_CHARS], + description=str(description or "")[:MAX_DESCRIPTION_CHARS], + origin=GraphOrigin.MANUAL) + for row in rows: + tx.insert_binding(entity_id=entity["id"], + ref_kind=BindingRefKind.EVIDENCE, + ref_id=row["evidence_id"], + binding_role=BindingRole.SUPPORTING, + origin=GraphOrigin.MANUAL, + evidence_id=row["evidence_id"]) + tx.insert_decision( + decision_type=DecisionType.CREATE_ENTITY, + target_kind=DecisionTargetKind.ENTITY, + target_id=entity["id"], actor=self._actor(actor), note=note, + after=self._entity_snapshot(entity), + source_command=source_command) + revision = tx.revision_number + self._refresh_projection(workspace_id) + return {"workspace_id": workspace_id, + "entity": store.get_entity(workspace_id, entity["id"]), + "knowledge_revision": revision} + + def create_claim(self, workspace_id: str, *, entity_id: str, + claim_type: str, statement: str, + evidence: Sequence[Dict[str, Any]], + actor: str, note: str, + source_command: str = "") -> Dict[str, Any]: + """Manual Claim creation. Evidence is mandatory; quotes must verify + against the immutable store; an identical normalized statement on + the same entity deduplicates to the existing claim.""" + self._require_workspace(workspace_id) + entity = self._require_entity(workspace_id, entity_id) + claim_type = require(ClaimType, claim_type, field="claim type") + statement = str(statement or "").strip() + if not statement: + raise InvalidRequest("statement is required") + if len(statement) > MAX_STATEMENT_CHARS: + raise InvalidRequest( + f"statement exceeds {MAX_STATEMENT_CHARS} characters", + details={"chars": len(statement)}) + note = self._note(note) + rows = verifier.verify_evidence_refs(workspace_id, evidence, + require_nonempty=True) + statement_hash = identity.statement_hash(statement) + existing = store.find_claim_by_hash(workspace_id, entity["id"], + statement_hash) + if existing: + return {"workspace_id": workspace_id, + "claim": store.get_claim(workspace_id, existing["id"]), + "deduplicated": True, + "knowledge_revision": + store.current_revision_number(workspace_id)} + with store.graph_transaction( + workspace_id, action=RevisionAction.MANUAL_CLAIM_CREATE, + actor=self._actor(actor), + summary=f"manual claim on {entity['canonical_key']}") as tx: + claim = tx.insert_claim( + entity_id=entity["id"], claim_type=claim_type, + statement=statement, + normalized_statement_hash=statement_hash, + origin=GraphOrigin.MANUAL, evidence=rows) + tx.insert_decision( + decision_type=DecisionType.CREATE_CLAIM, + target_kind=DecisionTargetKind.CLAIM, + target_id=claim["id"], actor=self._actor(actor), note=note, + after={"claim_id": claim["id"], "entity_id": entity["id"], + "claim_type": claim_type}, + source_command=source_command) + revision = tx.revision_number + self._refresh_projection(workspace_id) + return {"workspace_id": workspace_id, + "claim": store.get_claim(workspace_id, claim["id"]), + "deduplicated": False, "knowledge_revision": revision} + + def create_relation(self, workspace_id: str, *, source_entity_id: str, + target_entity_id: str, relation_type: str, + relation_state: str, + evidence: Sequence[Dict[str, Any]], + actor: str, note: str, + confidence: str = "medium", + source_command: str = "") -> Dict[str, Any]: + """Manual Relation creation: both endpoints must exist in this + workspace, the state must be ``explicit`` or ``confirmed`` (a manual + caller has no analyzer to justify ``inferred``), evidence is + mandatory, and the active-identity tuple deduplicates.""" + self._require_workspace(workspace_id) + source = self._require_entity(workspace_id, source_entity_id) + target = self._require_entity(workspace_id, target_entity_id) + relation_type = require(RelationType, relation_type, + field="relation type") + relation_state = str(relation_state or "").strip().lower() + if relation_state not in RelationState.MANUAL: + raise InvalidRequest( + f"manual relation state must be one of " + f"{sorted(RelationState.MANUAL)} (got {relation_state!r}; " + f"'inferred' requires a documented analyzer origin)", + details={"allowed": sorted(RelationState.MANUAL)}) + if source["id"] == target["id"] and \ + relation_type not in RelationType.SELF_RELATING: + raise GraphConflict( + f"self-relation rejected for type {relation_type!r}") + note = self._note(note) + rows = verifier.verify_evidence_refs(workspace_id, evidence, + require_nonempty=True) + existing = store.find_active_relation( + workspace_id, source["id"], target["id"], relation_type) + if existing: + return {"workspace_id": workspace_id, + "relation": store.get_relation(workspace_id, + existing["id"]), + "deduplicated": True, + "knowledge_revision": + store.current_revision_number(workspace_id)} + with store.graph_transaction( + workspace_id, action=RevisionAction.MANUAL_RELATION_CREATE, + actor=self._actor(actor), + summary=f"manual relation {relation_type}") as tx: + relation = tx.insert_relation( + source_entity_id=source["id"], + target_entity_id=target["id"], + relation_type=relation_type, + relation_state=relation_state, + confidence=str(confidence or "medium"), + origin=GraphOrigin.MANUAL, evidence=rows) + tx.insert_decision( + decision_type=DecisionType.CREATE_RELATION, + target_kind=DecisionTargetKind.RELATION, + target_id=relation["id"], actor=self._actor(actor), + note=note, + after={"relation_id": relation["id"], + "source_entity_id": source["id"], + "target_entity_id": target["id"], + "relation_type": relation_type, + "relation_state": relation_state}, + source_command=source_command) + revision = tx.revision_number + return {"workspace_id": workspace_id, + "relation": store.get_relation(workspace_id, relation["id"]), + "deduplicated": False, "knowledge_revision": revision} + + # ====================================================================== + # Aliases + # ====================================================================== + def add_alias(self, workspace_id: str, *, entity_id: str, alias: str, + alias_type: str, actor: str, note: str, + evidence_id: str = "", + source_command: str = "") -> Dict[str, Any]: + self._require_workspace(workspace_id) + entity = self._require_entity(workspace_id, entity_id) + alias_type = require(AliasType, alias_type, field="alias type") + alias = str(alias or "").strip() + if not alias: + raise InvalidRequest("alias is required") + note = self._note(note) + normalized = identity.normalize_alias(alias) + holders = store.find_alias_holders(workspace_id, normalized, + exclude_entity_id=entity["id"]) + if holders: + raise AliasCollision(alias, [ + {"entity_id": h["entity_id"], + "canonical_key": h["holder_canonical_key"]} + for h in holders]) + own = [a for a in store.list_aliases(workspace_id, entity["id"]) + if a["normalized_alias"] == normalized] + if own: + return {"workspace_id": workspace_id, "alias": own[0], + "deduplicated": True, + "knowledge_revision": + store.current_revision_number(workspace_id)} + if evidence_id: + verifier.verify_evidence_refs( + workspace_id, [{"evidence_id": evidence_id}], + require_nonempty=True) + with store.graph_transaction( + workspace_id, action=RevisionAction.ALIAS_CHANGE, + actor=self._actor(actor), + summary=f"add alias to {entity['canonical_key']}") as tx: + row = tx.insert_alias(entity_id=entity["id"], alias=alias, + normalized_alias=normalized, + alias_type=alias_type, + origin=GraphOrigin.MANUAL, + evidence_id=evidence_id) + tx.insert_decision( + decision_type=DecisionType.ADD_ALIAS, + target_kind=DecisionTargetKind.ALIAS, target_id=row["id"], + actor=self._actor(actor), note=note, + after={"entity_id": entity["id"], "alias": alias, + "alias_type": alias_type}, + source_command=source_command) + revision = tx.revision_number + self._refresh_projection(workspace_id) + return {"workspace_id": workspace_id, "alias": row, + "deduplicated": False, "knowledge_revision": revision} + + def remove_alias(self, workspace_id: str, *, entity_id: str, alias: str, + actor: str, note: str, + source_command: str = "") -> Dict[str, Any]: + self._require_workspace(workspace_id) + entity = self._require_entity(workspace_id, entity_id) + note = self._note(note) + normalized = identity.normalize_alias(alias) + matches = [a for a in store.list_aliases(workspace_id, entity["id"]) + if a["normalized_alias"] == normalized] + if not matches: + raise InvalidRequest( + f"entity {entity_id} has no active alias {alias!r}") + with store.graph_transaction( + workspace_id, action=RevisionAction.ALIAS_CHANGE, + actor=self._actor(actor), + summary=f"remove alias from {entity['canonical_key']}") as tx: + for row in matches: + tx.update_alias(row["id"], status=AliasStatus.REMOVED) + tx.insert_decision( + decision_type=DecisionType.REMOVE_ALIAS, + target_kind=DecisionTargetKind.ALIAS, + target_id=matches[0]["id"], actor=self._actor(actor), + note=note, + before={"entity_id": entity["id"], "alias": alias}, + source_command=source_command) + revision = tx.revision_number + self._refresh_projection(workspace_id) + return {"workspace_id": workspace_id, "removed": len(matches), + "knowledge_revision": revision} + + # ====================================================================== + # Authority / supersede / withdraw / reject / restore + # ====================================================================== + _OBJECT_KINDS = { + "entity": (DecisionTargetKind.ENTITY,), + "claim": (DecisionTargetKind.CLAIM,), + "relation": (DecisionTargetKind.RELATION,), + } + + def _require_object(self, workspace_id: str, kind: str, + object_id: str) -> Dict[str, Any]: + if kind == "entity": + return self._require_entity(workspace_id, object_id) + if kind == "claim": + return self._require_claim(workspace_id, object_id) + if kind == "relation": + return self._require_relation(workspace_id, object_id) + raise InvalidRequest(f"unknown object kind: {kind!r}", + details={"allowed": ["entity", "claim", + "relation"]}) + + def set_authority(self, workspace_id: str, *, kind: str, object_id: str, + authority: str, actor: str, note: str, + source_command: str = "") -> Dict[str, Any]: + """Explicit authority marking. Never inferred — this method IS the + only path that changes authority_status.""" + self._require_workspace(workspace_id) + authority = require(AuthorityStatus, authority, + field="authority status") + note = self._note(note) + obj = self._require_object(workspace_id, kind, object_id) + decision_type = (DecisionType.MARK_AUTHORITATIVE + if authority == AuthorityStatus.AUTHORITATIVE + else DecisionType.MARK_NON_AUTHORITATIVE) + with store.graph_transaction( + workspace_id, action=RevisionAction.AUTHORITY_CHANGE, + actor=self._actor(actor), + summary=f"set {kind} authority to {authority}") as tx: + update = {"entity": tx.update_entity, "claim": tx.update_claim, + "relation": tx.update_relation}[kind] + update(object_id, authority_status=authority) + tx.insert_decision( + decision_type=decision_type, + target_kind=self._OBJECT_KINDS[kind][0], + target_id=object_id, actor=self._actor(actor), note=note, + before={"authority_status": obj["authority_status"]}, + after={"authority_status": authority}, + source_command=source_command) + revision = tx.revision_number + self._refresh_projection(workspace_id) + return {"workspace_id": workspace_id, "kind": kind, + "object_id": object_id, "authority_status": authority, + "knowledge_revision": revision} + + def supersede_object(self, workspace_id: str, *, kind: str, + object_id: str, replacement_id: str, actor: str, + note: str, + source_command: str = "") -> Dict[str, Any]: + """Explicit supersede: the old object is preserved (never deleted), + marked superseded, and points at its replacement.""" + self._require_workspace(workspace_id) + note = self._note(note) + old = self._require_object(workspace_id, kind, object_id) + new = self._require_object(workspace_id, kind, replacement_id) + if old["id"] == new["id"]: + raise GraphConflict("an object cannot supersede itself") + if new["lifecycle_status"] != GraphLifecycleStatus.ACTIVE: + raise GraphConflict( + f"replacement {replacement_id} is not active " + f"({new['lifecycle_status']})") + pointer = {"entity": "superseded_by_entity_id", + "claim": "superseded_by_claim_id", + "relation": "superseded_by_relation_id"}[kind] + action = (RevisionAction.CLAIM_SUPERSEDE if kind == "claim" + else RevisionAction.SUPERSEDE) + with store.graph_transaction( + workspace_id, action=action, actor=self._actor(actor), + summary=f"supersede {kind} {object_id}") as tx: + update = {"entity": tx.update_entity, "claim": tx.update_claim, + "relation": tx.update_relation}[kind] + fields = {"lifecycle_status": GraphLifecycleStatus.SUPERSEDED, + pointer: new["id"]} + if kind == "relation": + fields["relation_state"] = RelationState.SUPERSEDED + update(object_id, **fields) + tx.insert_decision( + decision_type=DecisionType.SUPERSEDE, + target_kind=self._OBJECT_KINDS[kind][0], + target_id=object_id, actor=self._actor(actor), note=note, + before={"lifecycle_status": old["lifecycle_status"]}, + after={"lifecycle_status": GraphLifecycleStatus.SUPERSEDED, + "replacement_id": new["id"]}, + source_command=source_command) + revision = tx.revision_number + self._refresh_projection(workspace_id) + return {"workspace_id": workspace_id, "kind": kind, + "object_id": object_id, "replacement_id": new["id"], + "knowledge_revision": revision} + + def withdraw_object(self, workspace_id: str, *, kind: str, + object_id: str, actor: str, note: str, + source_command: str = "") -> Dict[str, Any]: + """Explicit withdrawal: history preserved, excluded from active + queries.""" + self._require_workspace(workspace_id) + note = self._note(note) + obj = self._require_object(workspace_id, kind, object_id) + if obj["lifecycle_status"] == GraphLifecycleStatus.WITHDRAWN: + return {"workspace_id": workspace_id, "kind": kind, + "object_id": object_id, "already_withdrawn": True, + "knowledge_revision": + store.current_revision_number(workspace_id)} + with store.graph_transaction( + workspace_id, action=RevisionAction.WITHDRAW, + actor=self._actor(actor), + summary=f"withdraw {kind} {object_id}") as tx: + update = {"entity": tx.update_entity, "claim": tx.update_claim, + "relation": tx.update_relation}[kind] + update(object_id, + lifecycle_status=GraphLifecycleStatus.WITHDRAWN) + tx.insert_decision( + decision_type=DecisionType.WITHDRAW, + target_kind=self._OBJECT_KINDS[kind][0], + target_id=object_id, actor=self._actor(actor), note=note, + before={"lifecycle_status": obj["lifecycle_status"]}, + after={"lifecycle_status": GraphLifecycleStatus.WITHDRAWN}, + source_command=source_command) + revision = tx.revision_number + self._refresh_projection(workspace_id) + return {"workspace_id": workspace_id, "kind": kind, + "object_id": object_id, "knowledge_revision": revision} + + def reject_relation(self, workspace_id: str, *, relation_id: str, + actor: str, note: str, + source_command: str = "") -> Dict[str, Any]: + """Mark a Relation rejected: retained as governance history, excluded + from the active graph. The prior epistemic state is parked so an + explicit restore is lossless.""" + self._require_workspace(workspace_id) + note = self._note(note) + relation = self._require_relation(workspace_id, relation_id) + if relation["relation_state"] == RelationState.REJECTED: + return {"workspace_id": workspace_id, + "relation_id": relation_id, "already_rejected": True, + "knowledge_revision": + store.current_revision_number(workspace_id)} + meta = dict(relation.get("metadata") or {}) + meta["prior_state"] = relation["relation_state"] + with store.graph_transaction( + workspace_id, action=RevisionAction.RELATION_STATE_CHANGE, + actor=self._actor(actor), + summary=f"reject relation {relation_id}") as tx: + tx.update_relation(relation_id, + relation_state=RelationState.REJECTED, + metadata=meta) + tx.insert_decision( + decision_type=DecisionType.REJECT_RELATION, + target_kind=DecisionTargetKind.RELATION, + target_id=relation_id, actor=self._actor(actor), note=note, + before={"relation_state": relation["relation_state"]}, + after={"relation_state": RelationState.REJECTED}, + source_command=source_command) + revision = tx.revision_number + return {"workspace_id": workspace_id, "relation_id": relation_id, + "relation_state": RelationState.REJECTED, + "knowledge_revision": revision} + + def restore_relation(self, workspace_id: str, *, relation_id: str, + actor: str, note: str, + source_command: str = "") -> Dict[str, Any]: + self._require_workspace(workspace_id) + note = self._note(note) + relation = self._require_relation(workspace_id, relation_id) + if relation["relation_state"] != RelationState.REJECTED: + raise GraphConflict( + f"relation {relation_id} is not rejected " + f"({relation['relation_state']})") + prior = (relation.get("metadata") or {}).get( + "prior_state", RelationState.CONFIRMED) + with store.graph_transaction( + workspace_id, action=RevisionAction.RELATION_STATE_CHANGE, + actor=self._actor(actor), + summary=f"restore relation {relation_id}") as tx: + tx.update_relation(relation_id, relation_state=prior) + tx.insert_decision( + decision_type=DecisionType.RESTORE_RELATION, + target_kind=DecisionTargetKind.RELATION, + target_id=relation_id, actor=self._actor(actor), note=note, + before={"relation_state": RelationState.REJECTED}, + after={"relation_state": prior}, + source_command=source_command) + revision = tx.revision_number + return {"workspace_id": workspace_id, "relation_id": relation_id, + "relation_state": prior, "knowledge_revision": revision} + + # ====================================================================== + # Merge and split + # ====================================================================== + def merge_entities(self, workspace_id: str, *, source_entity_id: str, + target_entity_id: str, actor: str, note: str, + source_command: str = "") -> Dict[str, Any]: + """Merge source into target, transactionally (spec §24).""" + self._require_workspace(workspace_id) + note = self._note(note) + source = self._require_entity(workspace_id, source_entity_id) + target = self._require_entity(workspace_id, target_entity_id) + if source["id"] == target["id"]: + raise GraphConflict("an entity cannot be merged into itself") + if target["lifecycle_status"] != GraphLifecycleStatus.ACTIVE: + raise GraphConflict( + f"merge target must be active " + f"({target['lifecycle_status']})") + if source["lifecycle_status"] in (GraphLifecycleStatus.MERGED, + GraphLifecycleStatus.WITHDRAWN): + raise GraphConflict( + f"source entity is {source['lifecycle_status']} and cannot " + f"be merged") + + alias_collisions: List[Dict[str, Any]] = [] + moved = {"aliases": 0, "bindings": 0, "claims": 0, + "claims_deduplicated": 0, "relations_rewired": 0, + "relations_deduplicated": 0, "self_relations_removed": 0} + + with store.graph_transaction( + workspace_id, action=RevisionAction.ENTITY_MERGE, + actor=self._actor(actor), + summary=f"merge {source['canonical_key']} into " + f"{target['canonical_key']}") as tx: + # aliases: move unless the normalized form collides anywhere + # outside the pair. + target_aliases = {a["normalized_alias"] for a in + store.list_aliases(workspace_id, target["id"], + status=None)} + for alias in store.list_aliases(workspace_id, source["id"]): + holders = store.find_alias_holders( + workspace_id, alias["normalized_alias"], + exclude_entity_id=source["id"]) + external = [h for h in holders + if h["entity_id"] != target["id"]] + if external: + alias_collisions.append( + {"alias": alias["alias"], + "held_by": [h["holder_canonical_key"] + for h in external]}) + continue + if alias["normalized_alias"] in target_aliases: + tx.update_alias(alias["id"], + status=AliasStatus.SUPERSEDED) + continue + tx.update_alias(alias["id"], entity_id=target["id"]) + target_aliases.add(alias["normalized_alias"]) + moved["aliases"] += 1 + + # bindings: move and deduplicate. + target_bindings = { + (b["ref_kind"], b["ref_id"], b["binding_role"]) + for b in store.list_bindings(workspace_id, target["id"])} + for binding in store.list_bindings(workspace_id, source["id"]): + sig = (binding["ref_kind"], binding["ref_id"], + binding["binding_role"]) + if sig in target_bindings: + tx.update_binding(binding["id"], status="removed") + continue + tx.update_binding(binding["id"], entity_id=target["id"]) + target_bindings.add(sig) + moved["bindings"] += 1 + + # claims: move; identical normalized statements deduplicate by + # superseding the source claim with the target's. + for claim in store.list_claims(workspace_id, + entity_id=source["id"], + lifecycle_status=None, + limit=10_000): + duplicate = store.find_claim_by_hash( + workspace_id, target["id"], + claim["normalized_statement_hash"]) + if duplicate and claim["lifecycle_status"] == \ + GraphLifecycleStatus.ACTIVE: + tx.update_claim( + claim["id"], entity_id=target["id"], + lifecycle_status=GraphLifecycleStatus.SUPERSEDED, + superseded_by_claim_id=duplicate["id"]) + moved["claims_deduplicated"] += 1 + else: + tx.move_claim(claim["id"], target["id"]) + moved["claims"] += 1 + + # relations: rewire both endpoints; drop self-relations; dedup + # against the target's existing active relations. + for relation in store.list_relations(workspace_id, + entity_id=source["id"], + lifecycle_status=None, + limit=10_000): + new_source = (target["id"] + if relation["source_entity_id"] == source["id"] + else relation["source_entity_id"]) + new_target = (target["id"] + if relation["target_entity_id"] == source["id"] + else relation["target_entity_id"]) + if new_source == new_target: + tx.update_relation( + relation["id"], + lifecycle_status=GraphLifecycleStatus.WITHDRAWN) + moved["self_relations_removed"] += 1 + continue + duplicate = store.find_active_relation( + workspace_id, new_source, new_target, + relation["relation_type"]) + if duplicate and duplicate["id"] != relation["id"] and \ + relation["lifecycle_status"] == \ + GraphLifecycleStatus.ACTIVE: + tx.update_relation( + relation["id"], source_entity_id=new_source, + target_entity_id=new_target, + lifecycle_status=GraphLifecycleStatus.SUPERSEDED, + relation_state=RelationState.SUPERSEDED, + superseded_by_relation_id=duplicate["id"]) + moved["relations_deduplicated"] += 1 + else: + tx.update_relation(relation["id"], + source_entity_id=new_source, + target_entity_id=new_target) + moved["relations_rewired"] += 1 + + # the source itself: merged, addressable, resolving to target. + tx.update_entity(source["id"], + lifecycle_status=GraphLifecycleStatus.MERGED, + merged_into_entity_id=target["id"]) + tx.insert_decision( + decision_type=DecisionType.MERGE_ENTITY, + target_kind=DecisionTargetKind.ENTITY, + target_id=source["id"], actor=self._actor(actor), note=note, + before=self._entity_snapshot(source), + after={"merged_into_entity_id": target["id"], **moved}, + source_command=source_command) + revision = tx.revision_number + self._refresh_projection(workspace_id) + return {"workspace_id": workspace_id, + "source_entity_id": source["id"], + "target_entity_id": target["id"], + **moved, "alias_collisions": alias_collisions, + "knowledge_revision": revision} + + def split_entity(self, workspace_id: str, *, source_entity_id: str, + new_entity_type: str, new_canonical_key: str, + new_display_name: str, claim_ids: Sequence[str], + binding_ids: Sequence[str], + relation_rewrites: Sequence[Dict[str, str]], + actor: str, note: str, + source_command: str = "") -> Dict[str, Any]: + """Explicit, narrow split (spec §25): the caller lists exactly what + moves; nothing is chosen by a model; all-or-nothing.""" + self._require_workspace(workspace_id) + note = self._note(note) + source = self._require_entity(workspace_id, source_entity_id) + new_entity_type = require(EntityType, new_entity_type, + field="entity type") + new_canonical_key = str(new_canonical_key or "").strip() + if not new_canonical_key: + raise InvalidRequest("new canonical_key is required") + if store.find_entity_by_key(workspace_id, new_entity_type, + new_canonical_key): + raise GraphConflict( + f"an entity with canonical key {new_canonical_key!r} " + f"already exists") + claim_ids = [str(c).strip() for c in claim_ids if str(c).strip()] + binding_ids = [str(b).strip() for b in binding_ids + if str(b).strip()] + if not claim_ids and not binding_ids: + raise InvalidRequest( + "a split must move at least one claim or binding") + + # Every moved object must belong to the source — checked BEFORE the + # transaction so an invalid id aborts with nothing written. + source_claims = {c["id"] for c in store.list_claims( + workspace_id, entity_id=source["id"], lifecycle_status=None, + limit=10_000)} + for claim_id in claim_ids: + if claim_id not in source_claims: + raise GraphConflict( + f"claim {claim_id} does not belong to entity " + f"{source['id']}", details={"claim_id": claim_id}) + source_bindings = {b["id"] for b in store.list_bindings( + workspace_id, source["id"])} + for binding_id in binding_ids: + if binding_id not in source_bindings: + raise GraphConflict( + f"binding {binding_id} does not belong to entity " + f"{source['id']}", details={"binding_id": binding_id}) + rewrites: List[Dict[str, str]] = [] + for rewrite in relation_rewrites or []: + relation_id = str(rewrite.get("relation_id") or "").strip() + end = str(rewrite.get("end") or "").strip().lower() + if end not in ("source", "target"): + raise InvalidRequest( + f"relation rewrite end must be 'source' or 'target' " + f"(got {end!r})") + relation = self._require_relation(workspace_id, relation_id) + column = f"{end}_entity_id" + if relation[column] != source["id"]: + raise GraphConflict( + f"relation {relation_id} does not have entity " + f"{source['id']} as its {end}") + rewrites.append({"relation_id": relation_id, "column": column}) + + with store.graph_transaction( + workspace_id, action=RevisionAction.ENTITY_SPLIT, + actor=self._actor(actor), + summary=f"split {source['canonical_key']} -> " + f"{new_canonical_key}") as tx: + new_entity = tx.insert_entity( + entity_type=new_entity_type, + canonical_key=new_canonical_key, + display_name=str(new_display_name + or new_canonical_key)[:200], + origin=GraphOrigin.MANUAL) + for claim_id in claim_ids: + tx.move_claim(claim_id, new_entity["id"]) + for binding_id in binding_ids: + tx.update_binding(binding_id, entity_id=new_entity["id"]) + for rewrite in rewrites: + tx.update_relation(rewrite["relation_id"], + **{rewrite["column"]: new_entity["id"]}) + tx.insert_decision( + decision_type=DecisionType.SPLIT_ENTITY, + target_kind=DecisionTargetKind.ENTITY, + target_id=source["id"], actor=self._actor(actor), note=note, + before=self._entity_snapshot(source), + after={"new_entity_id": new_entity["id"], + "moved_claims": len(claim_ids), + "moved_bindings": len(binding_ids), + "rewired_relations": len(rewrites)}, + source_command=source_command) + revision = tx.revision_number + self._refresh_projection(workspace_id) + return {"workspace_id": workspace_id, + "source_entity_id": source["id"], + "new_entity": store.get_entity(workspace_id, + new_entity["id"]), + "moved_claims": len(claim_ids), + "moved_bindings": len(binding_ids), + "rewired_relations": len(rewrites), + "knowledge_revision": revision} + + +__all__ = ["KnowledgeService", "MAX_NOTE_CHARS"] diff --git a/openmind/knowledge/store.py b/openmind/knowledge/store.py new file mode 100644 index 0000000..b293ceb --- /dev/null +++ b/openmind/knowledge/store.py @@ -0,0 +1,1178 @@ +"""Repository over the v0006 canonical-graph tables. + +Same placement rationale as :mod:`openmind.semantic.store`: db.py is the +Phase 1–3 store, semantic/store.py the Phase 4 one, and this module the +Phase 5 one — all three run on the SAME shared WAL connection and the SAME +RLock (``db.shared_connection()``), so the graph inherits every concurrency +property the rest of persistence has. No second connection, no second lock +ordering. + +THE GRAPH TRANSACTION +--------------------- +Every canonical write happens inside :func:`graph_transaction`: + + with graph_transaction(ws, action=RevisionAction.CANDIDATE_PROMOTION, + actor=actor) as tx: + entity = tx.insert_entity(...) + tx.insert_claim(...) + tx.insert_decision(...) + +The context manager holds the process lock for its whole body, allocates the +workspace's next ``revision_number`` up front (so created rows can be stamped +with it), collects per-kind change counts as writes happen, writes the +``knowledge_revisions`` row LAST, and commits — or rolls the whole body back +on any exception, in which case no revision exists. One logical graph +transaction therefore produces exactly one Knowledge Revision, a failed one +produces none, and concurrent writers serialize on the lock so numbers stay +unique and monotonic. + +A transaction that recorded ZERO changes commits nothing and writes no +revision (``tx.wrote`` is False) — that is what makes an unchanged +``graph sync`` an honest no-op. + +Workspace scoping is structural: every row carries a validated +``workspace_id`` and every read filters on it. +""" +from __future__ import annotations + +import json +import time +from typing import Any, Dict, Iterator, List, Optional, Sequence + +from contextlib import contextmanager + +from .. import db +from .vocabularies import (AliasStatus, BindingStatus, GraphLifecycleStatus, + RelationState) + + +def _now() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()) + + +def _cx(): + """(connection, lock) — hold the lock for every statement.""" + return db.shared_connection() + + +def _j(value: Any) -> str: + return json.dumps(value if value is not None else {}) + + +def _load(text: Any, default: Any) -> Any: + try: + out = json.loads(text or "") + except Exception: + return default + return out if out is not None else default + + +# --------------------------------------------------------------------------- +# Row mappers +# --------------------------------------------------------------------------- +def _entity_row(row) -> Dict[str, Any]: + return { + "id": row["id"], "workspace_id": row["workspace_id"], + "entity_type": row["entity_type"], + "canonical_key": row["canonical_key"], + "display_name": row["display_name"], + "description": row["description"], + "lifecycle_status": row["lifecycle_status"], + "authority_status": row["authority_status"], + "origin": row["origin"], + "promoted_from_candidate_id": row["promoted_from_candidate_id"], + "created_knowledge_revision": row["created_knowledge_revision"], + "updated_knowledge_revision": row["updated_knowledge_revision"], + "metadata": _load(row["metadata_json"], {}), + "created_at": row["created_at"], "updated_at": row["updated_at"], + "stale_at": row["stale_at"], + "superseded_by_entity_id": row["superseded_by_entity_id"], + "merged_into_entity_id": row["merged_into_entity_id"], + } + + +def _alias_row(row) -> Dict[str, Any]: + return { + "id": row["id"], "workspace_id": row["workspace_id"], + "entity_id": row["entity_id"], "alias": row["alias"], + "normalized_alias": row["normalized_alias"], + "alias_type": row["alias_type"], "origin": row["origin"], + "status": row["status"], "evidence_id": row["evidence_id"], + "created_knowledge_revision": row["created_knowledge_revision"], + "created_at": row["created_at"], + } + + +def _binding_row(row) -> Dict[str, Any]: + return { + "id": row["id"], "workspace_id": row["workspace_id"], + "entity_id": row["entity_id"], "ref_kind": row["ref_kind"], + "ref_id": row["ref_id"], "ref_key": row["ref_key"], + "binding_role": row["binding_role"], "status": row["status"], + "origin": row["origin"], "evidence_id": row["evidence_id"], + "created_knowledge_revision": row["created_knowledge_revision"], + "updated_knowledge_revision": row["updated_knowledge_revision"], + "created_at": row["created_at"], "updated_at": row["updated_at"], + "stale_at": row["stale_at"], + } + + +def _claim_row(row) -> Dict[str, Any]: + return { + "id": row["id"], "workspace_id": row["workspace_id"], + "entity_id": row["entity_id"], "claim_type": row["claim_type"], + "statement": row["statement"], + "normalized_statement_hash": row["normalized_statement_hash"], + "lifecycle_status": row["lifecycle_status"], + "authority_status": row["authority_status"], + "origin": row["origin"], + "promoted_from_candidate_id": row["promoted_from_candidate_id"], + "created_knowledge_revision": row["created_knowledge_revision"], + "updated_knowledge_revision": row["updated_knowledge_revision"], + "metadata": _load(row["metadata_json"], {}), + "created_at": row["created_at"], "updated_at": row["updated_at"], + "stale_at": row["stale_at"], + "superseded_by_claim_id": row["superseded_by_claim_id"], + } + + +def _relation_row(row) -> Dict[str, Any]: + return { + "id": row["id"], "workspace_id": row["workspace_id"], + "source_entity_id": row["source_entity_id"], + "target_entity_id": row["target_entity_id"], + "relation_type": row["relation_type"], + "relation_state": row["relation_state"], + "confidence": row["confidence"], + "lifecycle_status": row["lifecycle_status"], + "authority_status": row["authority_status"], + "origin": row["origin"], + "promoted_from_relation_candidate_id": + row["promoted_from_relation_candidate_id"], + "created_knowledge_revision": row["created_knowledge_revision"], + "updated_knowledge_revision": row["updated_knowledge_revision"], + "metadata": _load(row["metadata_json"], {}), + "created_at": row["created_at"], "updated_at": row["updated_at"], + "stale_at": row["stale_at"], + "superseded_by_relation_id": row["superseded_by_relation_id"], + } + + +def _decision_row(row) -> Dict[str, Any]: + return { + "id": row["id"], "workspace_id": row["workspace_id"], + "knowledge_revision_id": row["knowledge_revision_id"], + "decision_type": row["decision_type"], + "target_kind": row["target_kind"], "target_id": row["target_id"], + "actor": row["actor"], "note": row["note"], + "before": _load(row["before_json"], {}), + "after": _load(row["after_json"], {}), + "source_command": row["source_command"], + "created_at": row["created_at"], + } + + +def _revision_row(row) -> Dict[str, Any]: + return { + "id": row["id"], "workspace_id": row["workspace_id"], + "revision_number": row["revision_number"], + "parent_revision_number": row["parent_revision_number"], + "change_set_id": row["change_set_id"], "action": row["action"], + "summary": row["summary"], "actor": row["actor"], + "object_counts": _load(row["object_counts_json"], {}), + "created_at": row["created_at"], + } + + +def _promotion_row(row) -> Dict[str, Any]: + return { + "id": row["id"], "workspace_id": row["workspace_id"], + "candidate_kind": row["candidate_kind"], + "candidate_id": row["candidate_id"], + "target_kind": row["target_kind"], "target_id": row["target_id"], + "status": row["status"], "policy_version": row["policy_version"], + "actor": row["actor"], "note": row["note"], + "knowledge_revision_id": row["knowledge_revision_id"], + "created_at": row["created_at"], + } + + +# --------------------------------------------------------------------------- +# The graph transaction +# --------------------------------------------------------------------------- +class GraphTransaction: + """One canonical graph write: everything or nothing, one revision. + + Only usable inside :func:`graph_transaction`, which holds the process + lock for the whole body. Insert/update helpers write through the shared + connection WITHOUT committing; the context manager commits once. + """ + + def __init__(self, conn, workspace_id: str, *, action: str, actor: str, + summary: str, change_set_id: str) -> None: + self._conn = conn + self.workspace_id = workspace_id + self.action = action + self.actor = actor + self.summary = summary + self.change_set_id = change_set_id or db.new_id("cs_") + self.ts = _now() + row = conn.execute( + "SELECT COALESCE(MAX(revision_number),0) FROM knowledge_revisions " + "WHERE workspace_id=?", (workspace_id,)).fetchone() + self.parent_revision = int(row[0]) + self.revision_number = self.parent_revision + 1 + self.revision_id = db.new_id("kr_") + self.counts: Dict[str, int] = {} + self.wrote = False + + def _bump(self, kind: str, n: int = 1) -> None: + self.counts[kind] = self.counts.get(kind, 0) + n + self.wrote = True + + # -- entities ----------------------------------------------------------- + def insert_entity(self, *, entity_type: str, canonical_key: str, + display_name: str, description: str = "", + origin: str, lifecycle_status: str = "active", + authority_status: str = "unknown", + promoted_from_candidate_id: str = "", + metadata: Optional[Dict[str, Any]] = None, + entity_id: str = "") -> Dict[str, Any]: + eid = entity_id or db.new_id("ent_") + self._conn.execute( + "INSERT INTO engineering_entities (id,workspace_id,entity_type," + "canonical_key,display_name,description,lifecycle_status," + "authority_status,origin,promoted_from_candidate_id," + "created_knowledge_revision,updated_knowledge_revision," + "metadata_json,created_at,updated_at,stale_at," + "superseded_by_entity_id,merged_into_entity_id) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (eid, self.workspace_id, entity_type, canonical_key, + display_name, description, lifecycle_status, authority_status, + origin, promoted_from_candidate_id, self.revision_number, + self.revision_number, _j(metadata or {}), self.ts, self.ts, + None, None, None)) + self._bump("entities") + return self.get_entity(eid) + + def update_entity(self, entity_id: str, **fields: Any) -> None: + self._update("engineering_entities", entity_id, fields, + count_kind="entities") + + def get_entity(self, entity_id: str) -> Dict[str, Any]: + row = self._conn.execute( + "SELECT * FROM engineering_entities WHERE id=? AND workspace_id=?", + (entity_id, self.workspace_id)).fetchone() + return _entity_row(row) + + # -- aliases ------------------------------------------------------------ + def insert_alias(self, *, entity_id: str, alias: str, + normalized_alias: str, alias_type: str, origin: str, + status: str = AliasStatus.ACTIVE, + evidence_id: str = "") -> Dict[str, Any]: + aid = db.new_id("al_") + self._conn.execute( + "INSERT INTO engineering_entity_aliases (id,workspace_id," + "entity_id,alias,normalized_alias,alias_type,origin,status," + "evidence_id,created_knowledge_revision,created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (aid, self.workspace_id, entity_id, alias, normalized_alias, + alias_type, origin, status, evidence_id, self.revision_number, + self.ts)) + self._bump("aliases") + row = self._conn.execute( + "SELECT * FROM engineering_entity_aliases WHERE id=?", + (aid,)).fetchone() + return _alias_row(row) + + def update_alias(self, alias_id: str, **fields: Any) -> None: + sets, args = [], [] + for key, value in fields.items(): + sets.append(f"{key}=?") + args.append(value) + args.extend([alias_id, self.workspace_id]) + self._conn.execute( + f"UPDATE engineering_entity_aliases SET {', '.join(sets)} " + f"WHERE id=? AND workspace_id=?", args) + self._bump("aliases") + + # -- bindings ----------------------------------------------------------- + def insert_binding(self, *, entity_id: str, ref_kind: str, ref_id: str, + ref_key: str = "", binding_role: str = "supporting", + origin: str, status: str = BindingStatus.ACTIVE, + evidence_id: str = "") -> Dict[str, Any]: + bid = db.new_id("bd_") + self._conn.execute( + "INSERT INTO engineering_entity_bindings (id,workspace_id," + "entity_id,ref_kind,ref_id,ref_key,binding_role,status,origin," + "evidence_id,created_knowledge_revision," + "updated_knowledge_revision,created_at,updated_at,stale_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (bid, self.workspace_id, entity_id, ref_kind, ref_id, ref_key, + binding_role, status, origin, evidence_id, self.revision_number, + self.revision_number, self.ts, self.ts, None)) + self._bump("bindings") + row = self._conn.execute( + "SELECT * FROM engineering_entity_bindings WHERE id=?", + (bid,)).fetchone() + return _binding_row(row) + + def update_binding(self, binding_id: str, **fields: Any) -> None: + self._update("engineering_entity_bindings", binding_id, fields, + count_kind="bindings") + + # -- claims ------------------------------------------------------------- + def insert_claim(self, *, entity_id: str, claim_type: str, + statement: str, normalized_statement_hash: str, + origin: str, lifecycle_status: str = "active", + authority_status: str = "unknown", + promoted_from_candidate_id: str = "", + metadata: Optional[Dict[str, Any]] = None, + evidence: Sequence[Dict[str, Any]] = () + ) -> Dict[str, Any]: + cid = db.new_id("clm_") + self._conn.execute( + "INSERT INTO engineering_claims (id,workspace_id,entity_id," + "claim_type,statement,normalized_statement_hash,lifecycle_status," + "authority_status,origin,promoted_from_candidate_id," + "created_knowledge_revision,updated_knowledge_revision," + "metadata_json,created_at,updated_at,stale_at," + "superseded_by_claim_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (cid, self.workspace_id, entity_id, claim_type, statement, + normalized_statement_hash, lifecycle_status, authority_status, + origin, promoted_from_candidate_id, self.revision_number, + self.revision_number, _j(metadata or {}), self.ts, self.ts, + None, None)) + for ev in evidence: + self._conn.execute( + "INSERT OR IGNORE INTO engineering_claim_evidence " + "(claim_id,evidence_id,role,quote,quote_hash) " + "VALUES (?,?,?,?,?)", + (cid, ev["evidence_id"], ev.get("role", "primary"), + ev.get("quote", ""), ev.get("quote_hash", ""))) + self._bump("claims") + row = self._conn.execute( + "SELECT * FROM engineering_claims WHERE id=?", (cid,)).fetchone() + return _claim_row(row) + + def add_claim_evidence(self, claim_id: str, + evidence: Sequence[Dict[str, Any]]) -> None: + for ev in evidence: + self._conn.execute( + "INSERT OR IGNORE INTO engineering_claim_evidence " + "(claim_id,evidence_id,role,quote,quote_hash) " + "VALUES (?,?,?,?,?)", + (claim_id, ev["evidence_id"], ev.get("role", "primary"), + ev.get("quote", ""), ev.get("quote_hash", ""))) + + def update_claim(self, claim_id: str, **fields: Any) -> None: + self._update("engineering_claims", claim_id, fields, + count_kind="claims") + + def move_claim(self, claim_id: str, new_entity_id: str) -> None: + self._conn.execute( + "UPDATE engineering_claims SET entity_id=?, " + "updated_knowledge_revision=?, updated_at=? " + "WHERE id=? AND workspace_id=?", + (new_entity_id, self.revision_number, self.ts, claim_id, + self.workspace_id)) + self._bump("claims") + + # -- relations ---------------------------------------------------------- + def insert_relation(self, *, source_entity_id: str, + target_entity_id: str, relation_type: str, + relation_state: str, confidence: str = "low", + origin: str, lifecycle_status: str = "active", + authority_status: str = "unknown", + promoted_from_relation_candidate_id: str = "", + metadata: Optional[Dict[str, Any]] = None, + evidence: Sequence[Dict[str, Any]] = () + ) -> Dict[str, Any]: + rid = db.new_id("rel_") + self._conn.execute( + "INSERT INTO engineering_relations (id,workspace_id," + "source_entity_id,target_entity_id,relation_type,relation_state," + "confidence,lifecycle_status,authority_status,origin," + "promoted_from_relation_candidate_id,created_knowledge_revision," + "updated_knowledge_revision,metadata_json,created_at,updated_at," + "stale_at,superseded_by_relation_id) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (rid, self.workspace_id, source_entity_id, target_entity_id, + relation_type, relation_state, confidence, lifecycle_status, + authority_status, origin, promoted_from_relation_candidate_id, + self.revision_number, self.revision_number, _j(metadata or {}), + self.ts, self.ts, None, None)) + for ev in evidence: + self._conn.execute( + "INSERT OR IGNORE INTO engineering_relation_evidence " + "(relation_id,evidence_id,role,quote,quote_hash) " + "VALUES (?,?,?,?,?)", + (rid, ev["evidence_id"], ev.get("role", "primary"), + ev.get("quote", ""), ev.get("quote_hash", ""))) + self._bump("relations") + row = self._conn.execute( + "SELECT * FROM engineering_relations WHERE id=?", + (rid,)).fetchone() + return _relation_row(row) + + def update_relation(self, relation_id: str, **fields: Any) -> None: + self._update("engineering_relations", relation_id, fields, + count_kind="relations") + + def add_relation_evidence(self, relation_id: str, + evidence: Sequence[Dict[str, Any]]) -> None: + for ev in evidence: + self._conn.execute( + "INSERT OR IGNORE INTO engineering_relation_evidence " + "(relation_id,evidence_id,role,quote,quote_hash) " + "VALUES (?,?,?,?,?)", + (relation_id, ev["evidence_id"], ev.get("role", "primary"), + ev.get("quote", ""), ev.get("quote_hash", ""))) + + # -- decisions / promotions -------------------------------------------- + def insert_decision(self, *, decision_type: str, target_kind: str, + target_id: str, actor: str = "", note: str = "", + before: Optional[Dict[str, Any]] = None, + after: Optional[Dict[str, Any]] = None, + source_command: str = "") -> Dict[str, Any]: + did = db.new_id("dec_") + self._conn.execute( + "INSERT INTO knowledge_decisions (id,workspace_id," + "knowledge_revision_id,decision_type,target_kind,target_id," + "actor,note,before_json,after_json,source_command,created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (did, self.workspace_id, self.revision_id, decision_type, + target_kind, target_id, actor, note, _j(before or {}), + _j(after or {}), source_command, self.ts)) + self._bump("decisions") + row = self._conn.execute( + "SELECT * FROM knowledge_decisions WHERE id=?", (did,)).fetchone() + return _decision_row(row) + + def insert_promotion(self, *, candidate_kind: str, candidate_id: str, + target_kind: str, target_id: str, status: str, + policy_version: str, actor: str = "", + note: str = "") -> Dict[str, Any]: + pid = db.new_id("pro_") + self._conn.execute( + "INSERT INTO knowledge_promotions (id,workspace_id," + "candidate_kind,candidate_id,target_kind,target_id,status," + "policy_version,actor,note,knowledge_revision_id,created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (pid, self.workspace_id, candidate_kind, candidate_id, + target_kind, target_id, status, policy_version, actor, note, + self.revision_id, self.ts)) + self._bump("promotions") + row = self._conn.execute( + "SELECT * FROM knowledge_promotions WHERE id=?", + (pid,)).fetchone() + return _promotion_row(row) + + def execute_counted(self, sql: str, args: Sequence[Any], + kind: str) -> int: + """Run one bulk UPDATE inside the transaction, counting affected rows + under *kind*. Zero affected rows records no change. For the indexed + set-based staleness reconciliation, where per-row helpers would be + O(rows) round trips.""" + cur = self._conn.execute(sql, args) + affected = max(0, cur.rowcount) + if affected: + self._bump(kind, affected) + return affected + + def set_projection_state(self, *, projector_version: str, + source_knowledge_hash: str) -> None: + self._conn.execute( + "INSERT INTO knowledge_projection_state (workspace_id," + "projector_version,source_knowledge_hash," + "last_knowledge_revision,last_synced_at) VALUES (?,?,?,?,?) " + "ON CONFLICT(workspace_id) DO UPDATE SET " + "projector_version=excluded.projector_version, " + "source_knowledge_hash=excluded.source_knowledge_hash, " + "last_knowledge_revision=excluded.last_knowledge_revision, " + "last_synced_at=excluded.last_synced_at", + (self.workspace_id, projector_version, source_knowledge_hash, + self.revision_number, self.ts)) + # A watermark refresh alone is not a graph change: no bump. + + # -- internals ---------------------------------------------------------- + def _update(self, table: str, row_id: str, fields: Dict[str, Any], + *, count_kind: str) -> None: + if not fields: + return + sets, args = [], [] + for key, value in fields.items(): + if key == "metadata": + key, value = "metadata_json", _j(value) + sets.append(f"{key}=?") + args.append(value) + sets.append("updated_knowledge_revision=?") + args.append(self.revision_number) + sets.append("updated_at=?") + args.append(self.ts) + args.extend([row_id, self.workspace_id]) + self._conn.execute( + f"UPDATE {table} SET {', '.join(sets)} " + f"WHERE id=? AND workspace_id=?", args) + self._bump(count_kind) + + def _finalize(self) -> None: + """Write the revision row (called by the context manager on a body + that recorded at least one change, right before commit).""" + self._conn.execute( + "INSERT INTO knowledge_revisions (id,workspace_id," + "revision_number,parent_revision_number,change_set_id,action," + "summary,actor,object_counts_json,created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", + (self.revision_id, self.workspace_id, self.revision_number, + self.parent_revision, self.change_set_id, self.action, + self.summary[:2000], self.actor[:200], _j(self.counts), + self.ts)) + + +@contextmanager +def graph_transaction(workspace_id: str, *, action: str, actor: str = "", + summary: str = "", + change_set_id: str = "") -> Iterator[GraphTransaction]: + """One canonical graph transaction (see the module docstring). + + Commit rules: body raised → full rollback, no revision; body recorded no + change → rollback (nothing to keep), no revision; otherwise the revision + row is written last and everything commits together. + """ + conn, lock = _cx() + with lock: + tx = GraphTransaction(conn, workspace_id, action=action, actor=actor, + summary=summary, change_set_id=change_set_id) + try: + yield tx + except Exception: + conn.rollback() + raise + if not tx.wrote: + conn.rollback() + return + tx._finalize() + conn.commit() + + +# --------------------------------------------------------------------------- +# Reads (each takes the lock for itself) +# --------------------------------------------------------------------------- +def get_entity(workspace_id: str, entity_id: str) -> Optional[Dict[str, Any]]: + conn, lock = _cx() + with lock: + row = conn.execute( + "SELECT * FROM engineering_entities WHERE id=? AND workspace_id=?", + (entity_id, workspace_id)).fetchone() + return _entity_row(row) if row else None + + +def find_entity_by_key(workspace_id: str, entity_type: str, + canonical_key: str) -> Optional[Dict[str, Any]]: + conn, lock = _cx() + with lock: + row = conn.execute( + "SELECT * FROM engineering_entities WHERE workspace_id=? AND " + "entity_type=? AND canonical_key=?", + (workspace_id, entity_type, canonical_key)).fetchone() + return _entity_row(row) if row else None + + +def list_entities(workspace_id: str, *, entity_type: Optional[str] = None, + lifecycle_status: Optional[str] = None, + authority_status: Optional[str] = None, + origin: Optional[str] = None, + key_prefix: Optional[str] = None, + limit: int = 100, offset: int = 0) -> List[Dict[str, Any]]: + q = "SELECT * FROM engineering_entities WHERE workspace_id=?" + args: List[Any] = [workspace_id] + for col, val in (("entity_type", entity_type), + ("lifecycle_status", lifecycle_status), + ("authority_status", authority_status), + ("origin", origin)): + if val: + q += f" AND {col}=?" + args.append(val) + if key_prefix: + q += " AND canonical_key LIKE ?" + args.append(key_prefix.replace("%", "").replace("_", "\\_") + "%") + q += " ORDER BY canonical_key, id LIMIT ? OFFSET ?" + args.extend([max(0, int(limit)), max(0, int(offset))]) + conn, lock = _cx() + with lock: + rows = conn.execute(q, args).fetchall() + return [_entity_row(r) for r in rows] + + +def count_entities(workspace_id: str, **filters: Optional[str]) -> int: + q = "SELECT COUNT(*) FROM engineering_entities WHERE workspace_id=?" + args: List[Any] = [workspace_id] + for col in ("entity_type", "lifecycle_status", "authority_status", + "origin"): + val = filters.get(col) + if val: + q += f" AND {col}=?" + args.append(val) + conn, lock = _cx() + with lock: + row = conn.execute(q, args).fetchone() + return int(row[0]) if row else 0 + + +def list_aliases(workspace_id: str, entity_id: str, + status: Optional[str] = AliasStatus.ACTIVE + ) -> List[Dict[str, Any]]: + q = ("SELECT * FROM engineering_entity_aliases WHERE workspace_id=? " + "AND entity_id=?") + args: List[Any] = [workspace_id, entity_id] + if status: + q += " AND status=?" + args.append(status) + q += " ORDER BY normalized_alias, id" + conn, lock = _cx() + with lock: + rows = conn.execute(q, args).fetchall() + return [_alias_row(r) for r in rows] + + +def find_alias_holders(workspace_id: str, normalized_alias: str, + exclude_entity_id: str = "") -> List[Dict[str, Any]]: + """ACTIVE aliases with this normalized form held by ACTIVE entities — + the collision-detection read.""" + conn, lock = _cx() + with lock: + rows = conn.execute( + "SELECT al.*, e.canonical_key AS holder_key, " + "e.display_name AS holder_name FROM engineering_entity_aliases al " + "JOIN engineering_entities e ON al.entity_id = e.id " + "WHERE al.workspace_id=? AND al.normalized_alias=? AND " + "al.status='active' AND e.lifecycle_status='active'", + (workspace_id, normalized_alias)).fetchall() + out = [] + for r in rows: + if exclude_entity_id and r["entity_id"] == exclude_entity_id: + continue + rec = _alias_row(r) + rec["holder_canonical_key"] = r["holder_key"] + rec["holder_display_name"] = r["holder_name"] + out.append(rec) + return out + + +def find_entity_by_alias(workspace_id: str, + normalized_alias: str) -> List[Dict[str, Any]]: + """Active entities holding this exact normalized alias.""" + conn, lock = _cx() + with lock: + rows = conn.execute( + "SELECT e.* FROM engineering_entities e " + "JOIN engineering_entity_aliases al ON al.entity_id = e.id " + "WHERE e.workspace_id=? AND al.normalized_alias=? AND " + "al.status='active' AND e.lifecycle_status='active' " + "ORDER BY e.canonical_key", + (workspace_id, normalized_alias)).fetchall() + return [_entity_row(r) for r in rows] + + +def list_bindings(workspace_id: str, entity_id: str, + status: Optional[str] = None) -> List[Dict[str, Any]]: + q = ("SELECT * FROM engineering_entity_bindings WHERE workspace_id=? " + "AND entity_id=?") + args: List[Any] = [workspace_id, entity_id] + if status: + q += " AND status=?" + args.append(status) + q += " ORDER BY ref_kind, ref_id, id" + conn, lock = _cx() + with lock: + rows = conn.execute(q, args).fetchall() + return [_binding_row(r) for r in rows] + + +def find_bindings_by_ref(workspace_id: str, ref_kind: str, + ref_id: str) -> List[Dict[str, Any]]: + conn, lock = _cx() + with lock: + rows = conn.execute( + "SELECT * FROM engineering_entity_bindings WHERE workspace_id=? " + "AND ref_kind=? AND ref_id=? ORDER BY id", + (workspace_id, ref_kind, ref_id)).fetchall() + return [_binding_row(r) for r in rows] + + +def list_workspace_bindings(workspace_id: str, status: Optional[str] = None, + limit: int = 200_000) -> List[Dict[str, Any]]: + """Every binding of a workspace in one query — the projector's and the + reconciler's bulk read (per-entity queries would be O(entities)).""" + q = "SELECT * FROM engineering_entity_bindings WHERE workspace_id=?" + args: List[Any] = [workspace_id] + if status: + q += " AND status=?" + args.append(status) + q += " ORDER BY entity_id, ref_kind, ref_id LIMIT ?" + args.append(max(0, int(limit))) + conn, lock = _cx() + with lock: + rows = conn.execute(q, args).fetchall() + return [_binding_row(r) for r in rows] + + +def get_claim(workspace_id: str, claim_id: str) -> Optional[Dict[str, Any]]: + conn, lock = _cx() + with lock: + row = conn.execute( + "SELECT * FROM engineering_claims WHERE id=? AND workspace_id=?", + (claim_id, workspace_id)).fetchone() + if not row: + return None + evidence = conn.execute( + "SELECT * FROM engineering_claim_evidence WHERE claim_id=? " + "ORDER BY evidence_id, quote_hash", (claim_id,)).fetchall() + out = _claim_row(row) + out["evidence"] = [{"evidence_id": e["evidence_id"], "role": e["role"], + "quote": e["quote"], "quote_hash": e["quote_hash"]} + for e in evidence] + return out + + +def list_claims(workspace_id: str, *, entity_id: Optional[str] = None, + claim_type: Optional[str] = None, + lifecycle_status: Optional[str] = None, + origin: Optional[str] = None, + limit: int = 100, offset: int = 0) -> List[Dict[str, Any]]: + q = "SELECT * FROM engineering_claims WHERE workspace_id=?" + args: List[Any] = [workspace_id] + for col, val in (("entity_id", entity_id), ("claim_type", claim_type), + ("lifecycle_status", lifecycle_status), + ("origin", origin)): + if val: + q += f" AND {col}=?" + args.append(val) + q += " ORDER BY created_at, id LIMIT ? OFFSET ?" + args.extend([max(0, int(limit)), max(0, int(offset))]) + conn, lock = _cx() + with lock: + rows = conn.execute(q, args).fetchall() + return [_claim_row(r) for r in rows] + + +def count_claims(workspace_id: str, **filters: Optional[str]) -> int: + q = "SELECT COUNT(*) FROM engineering_claims WHERE workspace_id=?" + args: List[Any] = [workspace_id] + for col in ("entity_id", "claim_type", "lifecycle_status", "origin"): + val = filters.get(col) + if val: + q += f" AND {col}=?" + args.append(val) + conn, lock = _cx() + with lock: + row = conn.execute(q, args).fetchone() + return int(row[0]) if row else 0 + + +def find_claim_by_hash(workspace_id: str, entity_id: str, + normalized_statement_hash: str, + lifecycle_status: str = GraphLifecycleStatus.ACTIVE + ) -> Optional[Dict[str, Any]]: + conn, lock = _cx() + with lock: + row = conn.execute( + "SELECT * FROM engineering_claims WHERE workspace_id=? AND " + "entity_id=? AND normalized_statement_hash=? AND " + "lifecycle_status=? ORDER BY created_at LIMIT 1", + (workspace_id, entity_id, normalized_statement_hash, + lifecycle_status)).fetchone() + return _claim_row(row) if row else None + + +def claim_evidence(claim_id: str) -> List[Dict[str, Any]]: + conn, lock = _cx() + with lock: + rows = conn.execute( + "SELECT * FROM engineering_claim_evidence WHERE claim_id=? " + "ORDER BY evidence_id, quote_hash", (claim_id,)).fetchall() + return [{"evidence_id": e["evidence_id"], "role": e["role"], + "quote": e["quote"], "quote_hash": e["quote_hash"]} + for e in rows] + + +def get_relation(workspace_id: str, + relation_id: str) -> Optional[Dict[str, Any]]: + conn, lock = _cx() + with lock: + row = conn.execute( + "SELECT * FROM engineering_relations WHERE id=? AND " + "workspace_id=?", (relation_id, workspace_id)).fetchone() + if not row: + return None + evidence = conn.execute( + "SELECT * FROM engineering_relation_evidence WHERE relation_id=? " + "ORDER BY evidence_id, quote_hash", (relation_id,)).fetchall() + out = _relation_row(row) + out["evidence"] = [{"evidence_id": e["evidence_id"], "role": e["role"], + "quote": e["quote"], "quote_hash": e["quote_hash"]} + for e in evidence] + return out + + +def list_relations(workspace_id: str, *, + source_entity_id: Optional[str] = None, + target_entity_id: Optional[str] = None, + entity_id: Optional[str] = None, + relation_type: Optional[str] = None, + relation_state: Optional[str] = None, + lifecycle_status: Optional[str] = None, + origin: Optional[str] = None, + limit: int = 100, offset: int = 0) -> List[Dict[str, Any]]: + q = "SELECT * FROM engineering_relations WHERE workspace_id=?" + args: List[Any] = [workspace_id] + if entity_id: + q += " AND (source_entity_id=? OR target_entity_id=?)" + args.extend([entity_id, entity_id]) + for col, val in (("source_entity_id", source_entity_id), + ("target_entity_id", target_entity_id), + ("relation_type", relation_type), + ("relation_state", relation_state), + ("lifecycle_status", lifecycle_status), + ("origin", origin)): + if val: + q += f" AND {col}=?" + args.append(val) + q += " ORDER BY relation_type, source_entity_id, target_entity_id, id " \ + "LIMIT ? OFFSET ?" + args.extend([max(0, int(limit)), max(0, int(offset))]) + conn, lock = _cx() + with lock: + rows = conn.execute(q, args).fetchall() + return [_relation_row(r) for r in rows] + + +def count_relations(workspace_id: str, **filters: Optional[str]) -> int: + q = "SELECT COUNT(*) FROM engineering_relations WHERE workspace_id=?" + args: List[Any] = [workspace_id] + for col in ("relation_type", "relation_state", "lifecycle_status", + "origin"): + val = filters.get(col) + if val: + q += f" AND {col}=?" + args.append(val) + conn, lock = _cx() + with lock: + row = conn.execute(q, args).fetchone() + return int(row[0]) if row else 0 + + +def find_active_relation(workspace_id: str, source_entity_id: str, + target_entity_id: str, + relation_type: str) -> Optional[Dict[str, Any]]: + """The active-identity lookup: at most one ACTIVE relation per + (workspace, source, target, type).""" + conn, lock = _cx() + with lock: + row = conn.execute( + "SELECT * FROM engineering_relations WHERE workspace_id=? AND " + "source_entity_id=? AND target_entity_id=? AND relation_type=? " + "AND lifecycle_status='active' ORDER BY created_at LIMIT 1", + (workspace_id, source_entity_id, target_entity_id, + relation_type)).fetchone() + return _relation_row(row) if row else None + + +def relation_evidence(relation_id: str) -> List[Dict[str, Any]]: + conn, lock = _cx() + with lock: + rows = conn.execute( + "SELECT * FROM engineering_relation_evidence WHERE relation_id=? " + "ORDER BY evidence_id, quote_hash", (relation_id,)).fetchall() + return [{"evidence_id": e["evidence_id"], "role": e["role"], + "quote": e["quote"], "quote_hash": e["quote_hash"]} + for e in rows] + + +def _like_escape(text: str) -> str: + return (str(text or "").replace("\\", "\\\\").replace("%", "\\%") + .replace("_", "\\_")) + + +def search_entities_lexical(workspace_id: str, query: str, + *, lifecycle_status: Optional[str] = "active", + limit: int = 50) -> List[Dict[str, Any]]: + """Case-insensitive substring match over canonical key and display name. + Deterministic ordering; token-precision ranking happens in search.py.""" + pattern = f"%{_like_escape(query)}%" + q = ("SELECT * FROM engineering_entities WHERE workspace_id=? AND " + "(canonical_key LIKE ? ESCAPE '\\' COLLATE NOCASE OR " + "display_name LIKE ? ESCAPE '\\' COLLATE NOCASE OR " + "description LIKE ? ESCAPE '\\' COLLATE NOCASE)") + args: List[Any] = [workspace_id, pattern, pattern, pattern] + if lifecycle_status: + q += " AND lifecycle_status=?" + args.append(lifecycle_status) + q += " ORDER BY canonical_key, id LIMIT ?" + args.append(max(0, int(limit))) + conn, lock = _cx() + with lock: + rows = conn.execute(q, args).fetchall() + return [_entity_row(r) for r in rows] + + +def search_claims_lexical(workspace_id: str, query: str, + *, lifecycle_status: Optional[str] = "active", + limit: int = 50) -> List[Dict[str, Any]]: + pattern = f"%{_like_escape(query)}%" + q = ("SELECT * FROM engineering_claims WHERE workspace_id=? AND " + "statement LIKE ? ESCAPE '\\' COLLATE NOCASE") + args: List[Any] = [workspace_id, pattern] + if lifecycle_status: + q += " AND lifecycle_status=?" + args.append(lifecycle_status) + q += " ORDER BY created_at, id LIMIT ?" + args.append(max(0, int(limit))) + conn, lock = _cx() + with lock: + rows = conn.execute(q, args).fetchall() + return [_claim_row(r) for r in rows] + + +# -- ledger reads ----------------------------------------------------------- +def current_revision_number(workspace_id: str) -> int: + conn, lock = _cx() + with lock: + row = conn.execute( + "SELECT COALESCE(MAX(revision_number),0) FROM knowledge_revisions " + "WHERE workspace_id=?", (workspace_id,)).fetchone() + return int(row[0]) + + +def list_revisions(workspace_id: str, limit: int = 50, + offset: int = 0) -> List[Dict[str, Any]]: + conn, lock = _cx() + with lock: + rows = conn.execute( + "SELECT * FROM knowledge_revisions WHERE workspace_id=? " + "ORDER BY revision_number DESC LIMIT ? OFFSET ?", + (workspace_id, max(0, int(limit)), + max(0, int(offset)))).fetchall() + return [_revision_row(r) for r in rows] + + +def get_revision_by_number(workspace_id: str, + revision_number: int) -> Optional[Dict[str, Any]]: + conn, lock = _cx() + with lock: + row = conn.execute( + "SELECT * FROM knowledge_revisions WHERE workspace_id=? AND " + "revision_number=?", + (workspace_id, int(revision_number))).fetchone() + return _revision_row(row) if row else None + + +def list_decisions(workspace_id: str, *, target_kind: Optional[str] = None, + target_id: Optional[str] = None, + decision_type: Optional[str] = None, + limit: int = 100, offset: int = 0) -> List[Dict[str, Any]]: + q = "SELECT * FROM knowledge_decisions WHERE workspace_id=?" + args: List[Any] = [workspace_id] + for col, val in (("target_kind", target_kind), ("target_id", target_id), + ("decision_type", decision_type)): + if val: + q += f" AND {col}=?" + args.append(val) + q += " ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?" + args.extend([max(0, int(limit)), max(0, int(offset))]) + conn, lock = _cx() + with lock: + rows = conn.execute(q, args).fetchall() + return [_decision_row(r) for r in rows] + + +def find_promotion(workspace_id: str, candidate_kind: str, + candidate_id: str) -> Optional[Dict[str, Any]]: + """The PROMOTED record for one candidate, if promotion ever happened.""" + conn, lock = _cx() + with lock: + row = conn.execute( + "SELECT * FROM knowledge_promotions WHERE workspace_id=? AND " + "candidate_kind=? AND candidate_id=? AND status='promoted' " + "ORDER BY created_at LIMIT 1", + (workspace_id, candidate_kind, candidate_id)).fetchone() + return _promotion_row(row) if row else None + + +def list_promotions(workspace_id: str, limit: int = 100, + offset: int = 0) -> List[Dict[str, Any]]: + conn, lock = _cx() + with lock: + rows = conn.execute( + "SELECT * FROM knowledge_promotions WHERE workspace_id=? " + "ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?", + (workspace_id, max(0, int(limit)), + max(0, int(offset)))).fetchall() + return [_promotion_row(r) for r in rows] + + +def get_projection_state(workspace_id: str) -> Optional[Dict[str, Any]]: + conn, lock = _cx() + with lock: + row = conn.execute( + "SELECT * FROM knowledge_projection_state WHERE workspace_id=?", + (workspace_id,)).fetchone() + if not row: + return None + return {"workspace_id": row["workspace_id"], + "projector_version": row["projector_version"], + "source_knowledge_hash": row["source_knowledge_hash"], + "last_knowledge_revision": row["last_knowledge_revision"], + "last_synced_at": row["last_synced_at"]} + + +def touch_projection_state(workspace_id: str, *, projector_version: str, + source_knowledge_hash: str) -> None: + """Refresh the projection watermark WITHOUT a graph transaction — the + unchanged-sync path (and the empty first seed). A watermark refresh is + not a graph change, so it must not mint a Knowledge Revision.""" + conn, lock = _cx() + with lock: + last = conn.execute( + "SELECT COALESCE(MAX(revision_number),0) FROM knowledge_revisions " + "WHERE workspace_id=?", (workspace_id,)).fetchone()[0] + conn.execute( + "INSERT INTO knowledge_projection_state (workspace_id," + "projector_version,source_knowledge_hash," + "last_knowledge_revision,last_synced_at) VALUES (?,?,?,?,?) " + "ON CONFLICT(workspace_id) DO UPDATE SET " + "projector_version=excluded.projector_version, " + "source_knowledge_hash=excluded.source_knowledge_hash, " + "last_knowledge_revision=excluded.last_knowledge_revision, " + "last_synced_at=excluded.last_synced_at", + (workspace_id, projector_version, source_knowledge_hash, + int(last), _now())) + conn.commit() + + +def stats(workspace_id: str) -> Dict[str, Any]: + """Aggregate graph counts, all scoped, plus the current revision.""" + conn, lock = _cx() + with lock: + c = conn + by_type = c.execute( + "SELECT entity_type, COUNT(*) AS n FROM engineering_entities " + "WHERE workspace_id=? AND lifecycle_status='active' " + "GROUP BY entity_type", (workspace_id,)).fetchall() + ent_total = c.execute( + "SELECT COUNT(*) FROM engineering_entities WHERE workspace_id=?", + (workspace_id,)).fetchone()[0] + ent_active = c.execute( + "SELECT COUNT(*) FROM engineering_entities WHERE workspace_id=? " + "AND lifecycle_status='active'", (workspace_id,)).fetchone()[0] + claims_active = c.execute( + "SELECT COUNT(*) FROM engineering_claims WHERE workspace_id=? " + "AND lifecycle_status='active'", (workspace_id,)).fetchone()[0] + claims_total = c.execute( + "SELECT COUNT(*) FROM engineering_claims WHERE workspace_id=?", + (workspace_id,)).fetchone()[0] + rel_active = c.execute( + "SELECT COUNT(*) FROM engineering_relations WHERE workspace_id=? " + "AND lifecycle_status='active'", (workspace_id,)).fetchone()[0] + rel_total = c.execute( + "SELECT COUNT(*) FROM engineering_relations WHERE workspace_id=?", + (workspace_id,)).fetchone()[0] + aliases = c.execute( + "SELECT COUNT(*) FROM engineering_entity_aliases WHERE " + "workspace_id=? AND status='active'", + (workspace_id,)).fetchone()[0] + bindings = c.execute( + "SELECT COUNT(*) FROM engineering_entity_bindings WHERE " + "workspace_id=? AND status='active'", + (workspace_id,)).fetchone()[0] + decisions = c.execute( + "SELECT COUNT(*) FROM knowledge_decisions WHERE workspace_id=?", + (workspace_id,)).fetchone()[0] + promotions = c.execute( + "SELECT COUNT(*) FROM knowledge_promotions WHERE workspace_id=? " + "AND status='promoted'", (workspace_id,)).fetchone()[0] + revision = c.execute( + "SELECT COALESCE(MAX(revision_number),0) FROM knowledge_revisions " + "WHERE workspace_id=?", (workspace_id,)).fetchone()[0] + return { + "entities_active": int(ent_active), "entities_total": int(ent_total), + "entities_by_type": {r["entity_type"]: int(r["n"]) for r in by_type}, + "claims_active": int(claims_active), + "claims_total": int(claims_total), + "relations_active": int(rel_active), + "relations_total": int(rel_total), + "aliases_active": int(aliases), "bindings_active": int(bindings), + "decisions": int(decisions), "promotions": int(promotions), + "knowledge_revision": int(revision), + } + + +def clear_workspace_graph(workspace_id: str) -> None: + """Wipe every canonical graph row of a workspace. Used by TERMINATE + (full wipe of learned data back to init) and by project delete. Entity + deletion cascades to aliases, bindings, claims, relations and evidence + joins via the v0006 foreign keys; the ledger tables are removed + explicitly.""" + conn, lock = _cx() + with lock: + try: + conn.execute( + "DELETE FROM engineering_entities WHERE workspace_id=?", + (workspace_id,)) + for table in ("knowledge_decisions", "knowledge_revisions", + "knowledge_promotions", + "knowledge_projection_state"): + conn.execute(f"DELETE FROM {table} WHERE workspace_id=?", + (workspace_id,)) + conn.commit() + except Exception: + conn.rollback() + raise + + +# --------------------------------------------------------------------------- +# Staleness support reads (used by reconciliation.py) +# --------------------------------------------------------------------------- +def current_revision_ids(workspace_id: str) -> frozenset: + """The workspace's set of current revision ids (active assets).""" + conn, lock = _cx() + with lock: + rows = conn.execute( + "SELECT current_revision_id FROM assets WHERE workspace_id=? " + "AND current_revision_id IS NOT NULL", + (workspace_id,)).fetchall() + return frozenset(r[0] for r in rows) + + +def active_bindings_on_noncurrent_revisions(workspace_id: str + ) -> List[Dict[str, Any]]: + """ACTIVE bindings whose ref is a revision that is no longer any active + asset's current revision — the staleness frontier, one indexed query.""" + conn, lock = _cx() + with lock: + rows = conn.execute( + "SELECT b.* FROM engineering_entity_bindings b " + "WHERE b.workspace_id=? AND b.status='active' AND " + "b.ref_kind='revision' AND b.ref_id NOT IN (" + "SELECT current_revision_id FROM assets WHERE workspace_id=? " + "AND current_revision_id IS NOT NULL)", + (workspace_id, workspace_id)).fetchall() + return [_binding_row(r) for r in rows] + + +__all__ = [ + "GraphTransaction", "graph_transaction", + "get_entity", "find_entity_by_key", "list_entities", "count_entities", + "list_aliases", "find_alias_holders", "find_entity_by_alias", + "list_bindings", "find_bindings_by_ref", "list_workspace_bindings", + "get_claim", "list_claims", "count_claims", "find_claim_by_hash", + "claim_evidence", + "get_relation", "list_relations", "count_relations", + "find_active_relation", "relation_evidence", + "current_revision_number", "list_revisions", "get_revision_by_number", + "list_decisions", "find_promotion", "list_promotions", + "get_projection_state", "touch_projection_state", "stats", + "clear_workspace_graph", + "current_revision_ids", "active_bindings_on_noncurrent_revisions", +] diff --git a/openmind/knowledge/vector_projection.py b/openmind/knowledge/vector_projection.py new file mode 100644 index 0000000..20eeca3 --- /dev/null +++ b/openmind/knowledge/vector_projection.py @@ -0,0 +1,182 @@ +"""The graph vector projection: ``knowledge_``. + +A SEPARATE per-workspace collection — graph text never enters ``code_`` +or ``documents_`` (their retrieval contracts are frozen), and their +chunks never enter this one. Indexed objects are ACTIVE Entities and Claims +only; Relations are not embedded (no tested retrieval need). + +The projection is derived state: SQLite stays the source of truth, and a +projection refresh replaces the workspace's rows content-addressed by id. +Embeddings are the same local embedder every other collection uses — no +cloud embedding path exists. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from .. import embeddings, vectorstore +from . import store +from .vocabularies import GraphLifecycleStatus + +#: Bounded text sizes for the embedded documents. +MAX_ENTITY_TEXT_CHARS = 2_000 +MAX_CLAIM_TEXT_CHARS = 2_000 +MAX_PRIMARY_CLAIMS_IN_ENTITY_TEXT = 3 + + +def collection_name(workspace_id: str) -> str: + return vectorstore.knowledge_collection_name(workspace_id) + + +def entity_text(workspace_id: str, entity: Dict[str, Any]) -> str: + """The embedded text of one Entity: type, key, name, aliases, bounded + description and its first active primary claims.""" + aliases = [a["alias"] for a in store.list_aliases(workspace_id, + entity["id"])] + claims = store.list_claims(workspace_id, entity_id=entity["id"], + lifecycle_status=GraphLifecycleStatus.ACTIVE, + limit=MAX_PRIMARY_CLAIMS_IN_ENTITY_TEXT) + parts = [ + f"{entity['entity_type']} {entity['canonical_key']}", + entity["display_name"], + ] + if aliases: + parts.append("aliases: " + ", ".join(aliases[:10])) + if entity.get("description"): + parts.append(entity["description"]) + for claim in claims: + parts.append(claim["statement"]) + return "\n".join(p for p in parts if p)[:MAX_ENTITY_TEXT_CHARS] + + +def claim_text(workspace_id: str, claim: Dict[str, Any], + entity: Optional[Dict[str, Any]] = None) -> str: + """The embedded text of one Claim: type, statement, entity identity and + a bounded evidence locator summary.""" + entity = entity or store.get_entity(workspace_id, claim["entity_id"]) + locators: List[str] = [] + from .. import db + for ev in store.claim_evidence(claim["id"])[:3]: + record = db.get_evidence(workspace_id, ev["evidence_id"]) + loc = (record or {}).get("locator") or {} + name = loc.get("file") or loc.get("document") or "" + if name: + locators.append(str(name)) + parts = [ + f"{claim['claim_type']}: {claim['statement']}", + f"entity: {(entity or {}).get('canonical_key', '')}", + ] + if locators: + parts.append("sources: " + ", ".join(sorted(set(locators)))) + return "\n".join(p for p in parts if p)[:MAX_CLAIM_TEXT_CHARS] + + +def _metadata(workspace_id: str, *, object_kind: str, + entity_id: str = "", claim_id: str = "", + entity_type: str = "", claim_type: str = "", + lifecycle_status: str = "", authority_status: str = "", + origin: str = "", content_hash: str = "") -> Dict[str, Any]: + return { + "workspace_id": workspace_id, "object_kind": object_kind, + "entity_id": entity_id, "claim_id": claim_id, + "entity_type": entity_type, "claim_type": claim_type, + "lifecycle_status": lifecycle_status, + "authority_status": authority_status, "origin": origin, + "knowledge_revision": store.current_revision_number(workspace_id), + "content_hash": content_hash, + } + + +def refresh_workspace(workspace_id: str) -> Dict[str, int]: + """Rebuild the workspace's graph projection to match ACTIVE graph state. + + Content-addressed by row id: active objects are upserted, projections of + no-longer-active objects are deleted (a merged source entity's active + projection disappears here). Idempotent; failures must be caught by the + caller — a projection failure never rolls back canonical SQLite state. + """ + import hashlib + + entities = store.list_entities( + workspace_id, lifecycle_status=GraphLifecycleStatus.ACTIVE, + limit=50_000) + claims = store.list_claims( + workspace_id, lifecycle_status=GraphLifecycleStatus.ACTIVE, + limit=100_000) + entity_by_id = {e["id"]: e for e in entities} + + ids: List[str] = [] + texts: List[str] = [] + metas: List[Dict[str, Any]] = [] + for entity in entities: + text = entity_text(workspace_id, entity) + if not text.strip(): + continue + ids.append(entity["id"]) + texts.append(text) + metas.append(_metadata( + workspace_id, object_kind="entity", entity_id=entity["id"], + entity_type=entity["entity_type"], + lifecycle_status=entity["lifecycle_status"], + authority_status=entity["authority_status"], + origin=entity["origin"], + content_hash=hashlib.sha256( + text.encode("utf-8")).hexdigest())) + for claim in claims: + text = claim_text(workspace_id, claim, + entity_by_id.get(claim["entity_id"])) + if not text.strip(): + continue + ids.append(claim["id"]) + texts.append(text) + metas.append(_metadata( + workspace_id, object_kind="claim", claim_id=claim["id"], + entity_id=claim["entity_id"], claim_type=claim["claim_type"], + lifecycle_status=claim["lifecycle_status"], + authority_status=claim["authority_status"], + origin=claim["origin"], + content_hash=hashlib.sha256( + text.encode("utf-8")).hexdigest())) + + collection = vectorstore.get_store(collection_name(workspace_id)) + existing = set(collection.get().get("ids") or []) + stale_ids = sorted(existing - set(ids)) + if stale_ids: + collection.delete(ids=stale_ids) + if ids: + vectors = embeddings.embed(texts) + collection.upsert(ids, vectors, texts, metas) + return {"indexed": len(ids), "removed": len(stale_ids)} + + +def query(workspace_id: str, text: str, *, limit: int = 20, + object_kind: Optional[str] = None) -> List[Dict[str, Any]]: + """Vector-similarity hits over the graph collection. Read-only; an + absent collection returns [].""" + name = collection_name(workspace_id) + if vectorstore.count_collection(name) == 0: + return [] + collection = vectorstore.get_store(name) + where: Optional[Dict[str, Any]] = None + if object_kind: + where = {"object_kind": object_kind} + result = collection.query(embeddings.embed_one(text), + n_results=max(1, int(limit)), where=where) + hits: List[Dict[str, Any]] = [] + for i, hit_id in enumerate(result.get("ids") or []): + meta = (result.get("metadatas") or [{}])[i] or {} + distance = (result.get("distances") or [0.0])[i] + hits.append({"id": hit_id, "metadata": meta, + "distance": float(distance), + "similarity": 1.0 - float(distance)}) + return hits + + +def drop_workspace(workspace_id: str) -> None: + """Drop the graph collection (terminate/delete path). Uses the shared + batched-drain drop, so the app never freezes on a large projection.""" + vectorstore.drop_collection(collection_name(workspace_id)) + + +__all__ = ["collection_name", "refresh_workspace", "query", + "drop_workspace", "entity_text", "claim_text"] diff --git a/openmind/knowledge/verifier.py b/openmind/knowledge/verifier.py new file mode 100644 index 0000000..acf5245 --- /dev/null +++ b/openmind/knowledge/verifier.py @@ -0,0 +1,112 @@ +"""Evidence validation for canonical graph writes. + +Same discipline as the Phase 4 candidate verifier, applied at the canonical +boundary: every Evidence id cited by a Claim, Relation, alias or manual +Entity must exist IN THIS WORKSPACE, and every quote must be a +whitespace-normalized substring of the immutable Evidence content. A +fabricated citation is rejected as a typed error before anything is written. + +Resolution reuses :func:`openmind.semantic.context.resolve_evidence_text` — +the deterministic, snapshot-only reader (never the live file). No provider, +no network. +""" +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional, Sequence + +from .. import db +from .errors import GraphEvidenceInvalid +from .identity import quote_hash +from .vocabularies import ClaimEvidenceRole, require + +_WHITESPACE = re.compile(r"\s+") + + +def _normalize(text: str) -> str: + return _WHITESPACE.sub(" ", str(text or "").strip()) + + +def resolve_evidence_text(workspace_id: str, evidence_id: str) -> Optional[str]: + """The immutable content one Evidence row cites, or None (wrong + workspace, missing, or corrupt snapshot).""" + from ..semantic.context import resolve_evidence_text as resolve + return resolve(workspace_id, evidence_id) + + +def verify_evidence_refs(workspace_id: str, + evidence: Sequence[Dict[str, Any]], + *, require_nonempty: bool = True + ) -> List[Dict[str, Any]]: + """Validate caller-supplied evidence references for a canonical write. + + Each entry: ``{evidence_id, quote?, role?}``. Returns the normalized join + rows (quote hashes computed here) or raises :class:`GraphEvidenceInvalid` + naming every failure. A quote is OPTIONAL — an Evidence citation without + a quote still anchors the object to the immutable snapshot — but a quote + that IS supplied must verify against the snapshot. + """ + problems: List[str] = [] + rows: List[Dict[str, Any]] = [] + seen = set() + for ref in evidence or (): + evidence_id = str(ref.get("evidence_id") or "").strip() + if not evidence_id: + problems.append("evidence reference without an evidence_id") + continue + role = str(ref.get("role") or ClaimEvidenceRole.PRIMARY) + require(ClaimEvidenceRole, role, field="evidence role") + record = db.get_evidence(workspace_id, evidence_id) + if not record: + problems.append( + f"evidence {evidence_id} does not exist in this workspace") + continue + quote = str(ref.get("quote") or "") + if quote: + content = resolve_evidence_text(workspace_id, evidence_id) + if content is None: + problems.append( + f"evidence {evidence_id}: immutable content is not " + f"recoverable, quote cannot be verified") + continue + if _normalize(quote) not in _normalize(content): + problems.append( + f"evidence {evidence_id}: quote is not a substring of " + f"the immutable evidence content") + continue + key = (evidence_id, quote_hash(quote)) + if key in seen: + continue + seen.add(key) + rows.append({"evidence_id": evidence_id, "role": role, + "quote": quote, "quote_hash": quote_hash(quote)}) + if problems: + raise GraphEvidenceInvalid( + "evidence validation failed: " + "; ".join(problems), + details={"problems": problems}) + if require_nonempty and not rows: + raise GraphEvidenceInvalid( + "at least one valid evidence reference is required", + details={"problems": ["no evidence supplied"]}) + return rows + + +def candidate_evidence_rows(workspace_id: str, + candidate: Dict[str, Any] + ) -> List[Dict[str, Any]]: + """Re-verify a Phase 4 candidate's stored evidence joins for promotion. + + The candidate's quotes were verified at extraction time; promotion + re-verifies them against the CURRENT immutable store (ownership + quote + substring), because canonical writes must never trust a stale check. + Raises :class:`GraphEvidenceInvalid` when any join fails. + """ + refs = [{"evidence_id": ev.get("evidence_id"), + "quote": ev.get("quote", ""), + "role": ClaimEvidenceRole.PRIMARY} + for ev in candidate.get("evidence") or []] + return verify_evidence_refs(workspace_id, refs, require_nonempty=True) + + +__all__ = ["verify_evidence_refs", "candidate_evidence_rows", + "resolve_evidence_text"] diff --git a/openmind/knowledge/vocabularies.py b/openmind/knowledge/vocabularies.py new file mode 100644 index 0000000..8016617 --- /dev/null +++ b/openmind/knowledge/vocabularies.py @@ -0,0 +1,414 @@ +"""Closed vocabularies of the canonical Engineering Knowledge Graph. + +Follows the project convention (:mod:`openmind.domain.types`, +:mod:`openmind.semantic.models`): small classes of string constants with a +``VALUES`` frozenset, validated at write boundaries. The graph vocabularies +deliberately do NOT have a forgiving ``coerce`` — an unknown Entity type, +Claim type or Relation type at a write boundary is a typed failure +(:func:`require`), never a silent default. A model- or caller-invented +vocabulary member must not enter canonical storage. + +Nothing here imports the database, a provider SDK or the vector store. +""" +from __future__ import annotations + +from typing import Iterable + +from .errors import UnknownVocabularyValue + + +def require(vocabulary: "type", value: str, *, field: str) -> str: + """Validate *value* against a vocabulary class. Returns the normalized + value or raises the typed error naming the field and the allowed set.""" + clean = str(value or "").strip().lower() + if clean not in vocabulary.VALUES: + raise UnknownVocabularyValue(field=field, value=value, + allowed=sorted(vocabulary.VALUES)) + return clean + + +class EntityType: + """What kind of engineering concept a canonical Entity is.""" + REQUIREMENT = "requirement" + BUSINESS_RULE = "business-rule" + DECISION = "decision" + CONSTRAINT = "constraint" + DESIGN = "design" + INTERFACE = "interface" + DATA_MODEL = "data-model" + WORKFLOW = "workflow" + BATCH_JOB = "batch-job" + TRANSACTION = "transaction" + FAILURE_MODE = "failure-mode" + ACCEPTANCE_CRITERION = "acceptance-criterion" + TEST_CASE = "test-case" + TEST_RESULT = "test-result" + CODE_COMPONENT = "code-component" + CODE_SYMBOL = "code-symbol" + CONFIGURATION = "configuration" + DATABASE_OBJECT = "database-object" + MESSAGE_TOPIC = "message-topic" + CHANGE_REQUEST = "change-request" + INCIDENT = "incident" + OPERATIONAL_PROCEDURE = "operational-procedure" + DOCUMENT = "document" + BUILD_DEFINITION = "build-definition" + UNKNOWN = "unknown" + VALUES = frozenset({ + REQUIREMENT, BUSINESS_RULE, DECISION, CONSTRAINT, DESIGN, INTERFACE, + DATA_MODEL, WORKFLOW, BATCH_JOB, TRANSACTION, FAILURE_MODE, + ACCEPTANCE_CRITERION, TEST_CASE, TEST_RESULT, CODE_COMPONENT, + CODE_SYMBOL, CONFIGURATION, DATABASE_OBJECT, MESSAGE_TOPIC, + CHANGE_REQUEST, INCIDENT, OPERATIONAL_PROCEDURE, DOCUMENT, + BUILD_DEFINITION, UNKNOWN, + }) + + #: Structural containers the deterministic projector may create WITHOUT a + #: Claim (their existence is the recorded deterministic fact). The one + #: documented exception to "an Entity needs Evidence" — everything else + #: needs at least one Evidence at manual creation. + STRUCTURAL_CONTAINERS = frozenset({ + CODE_COMPONENT, CODE_SYMBOL, CONFIGURATION, DATABASE_OBJECT, + DATA_MODEL, DOCUMENT, BUILD_DEFINITION, INTERFACE, MESSAGE_TOPIC, + }) + + +class ClaimType: + """What kind of statement a canonical Claim makes about its Entity.""" + DEFINITION = "definition" + NORMATIVE_STATEMENT = "normative-statement" + BEHAVIOR = "behavior" + CONSTRAINT = "constraint" + DECISION_RATIONALE = "decision-rationale" + INTERFACE_CONTRACT = "interface-contract" + DATA_DEFINITION = "data-definition" + WORKFLOW_STEP = "workflow-step" + FAILURE_CONDITION = "failure-condition" + ACCEPTANCE_CONDITION = "acceptance-condition" + TEST_EXPECTATION = "test-expectation" + CLASSIFICATION = "classification" + REVISION_STATUS = "revision-status" + AUTHORITY = "authority" + IMPLEMENTATION_NOTE = "implementation-note" + OPERATIONAL_INSTRUCTION = "operational-instruction" + UNKNOWN = "unknown" + VALUES = frozenset({ + DEFINITION, NORMATIVE_STATEMENT, BEHAVIOR, CONSTRAINT, + DECISION_RATIONALE, INTERFACE_CONTRACT, DATA_DEFINITION, + WORKFLOW_STEP, FAILURE_CONDITION, ACCEPTANCE_CONDITION, + TEST_EXPECTATION, CLASSIFICATION, REVISION_STATUS, AUTHORITY, + IMPLEMENTATION_NOTE, OPERATIONAL_INSTRUCTION, UNKNOWN, + }) + + +class RelationType: + """The closed set of canonical Relation types. + + There is deliberately NO ``depends-on``: dependency semantics must be + expressed through the closest honest member below, or the deterministic + fact stays outside the graph. + """ + CONTAINS = "contains" + SUPERSEDES = "supersedes" + REFINES = "refines" + IMPLEMENTS = "implements" + PARTIALLY_IMPLEMENTS = "partially-implements" + CONFIGURES = "configures" + CALLS = "calls" + READS = "reads" + WRITES = "writes" + PUBLISHES = "publishes" + CONSUMES = "consumes" + VERIFIES = "verifies" + EVIDENCED_BY = "evidenced-by" + CONTRADICTS = "contradicts" + AFFECTED_BY = "affected-by" + DERIVED_FROM = "derived-from" + POSSIBLY_RELATED = "possibly-related" + VALUES = frozenset({ + CONTAINS, SUPERSEDES, REFINES, IMPLEMENTS, PARTIALLY_IMPLEMENTS, + CONFIGURES, CALLS, READS, WRITES, PUBLISHES, CONSUMES, VERIFIES, + EVIDENCED_BY, CONTRADICTS, AFFECTED_BY, DERIVED_FROM, + POSSIBLY_RELATED, + }) + + #: Types where source == target could ever be meaningful. Empty on + #: purpose: every current type rejects self-relations. + SELF_RELATING = frozenset() + + +class RelationState: + """How the Relation came to be believed.""" + EXPLICIT = "explicit" # structurally explicit deterministic fact + INFERRED = "inferred" # deterministic analyzer with ambiguity kept + CONFIRMED = "confirmed" # a human promoted / created it + REJECTED = "rejected" # governance history; excluded from active graph + STALE = "stale" + SUPERSEDED = "superseded" + VALUES = frozenset({EXPLICIT, INFERRED, CONFIRMED, REJECTED, STALE, + SUPERSEDED}) + #: States that participate in the active graph. + ACTIVE = frozenset({EXPLICIT, INFERRED, CONFIRMED}) + #: States a manual `relation create` may set. ``inferred`` needs a + #: documented analyzer origin, which a manual caller does not have. + MANUAL = frozenset({EXPLICIT, CONFIRMED}) + + +class GraphLifecycleStatus: + """Lifecycle of every canonical graph object.""" + ACTIVE = "active" + STALE = "stale" + SUPERSEDED = "superseded" + WITHDRAWN = "withdrawn" + MERGED = "merged" + VALUES = frozenset({ACTIVE, STALE, SUPERSEDED, WITHDRAWN, MERGED}) + + +class AuthorityStatus: + """Explicit human authority judgement. Never inferred from names, + timestamps or model output.""" + UNKNOWN = "unknown" + INFORMATIONAL = "informational" + AUTHORITATIVE = "authoritative" + NON_AUTHORITATIVE = "non-authoritative" + VALUES = frozenset({UNKNOWN, INFORMATIONAL, AUTHORITATIVE, + NON_AUTHORITATIVE}) + + +class GraphOrigin: + """Which of the permitted write paths created the object.""" + DETERMINISTIC = "deterministic" + SEMANTIC_PROMOTION = "semantic-promotion" + MANUAL = "manual" + MIGRATION = "migration" + VALUES = frozenset({DETERMINISTIC, SEMANTIC_PROMOTION, MANUAL, MIGRATION}) + + +class AliasType: + IDENTIFIER = "identifier" + NAME = "name" + ACRONYM = "acronym" + LEGACY_NAME = "legacy-name" + PATH = "path" + SYMBOL = "symbol" + MANUAL = "manual" + VALUES = frozenset({IDENTIFIER, NAME, ACRONYM, LEGACY_NAME, PATH, SYMBOL, + MANUAL}) + + +class AliasStatus: + ACTIVE = "active" + REMOVED = "removed" + SUPERSEDED = "superseded" + VALUES = frozenset({ACTIVE, REMOVED, SUPERSEDED}) + + +class BindingRefKind: + """What source-plane object a binding points at.""" + ASSET = "asset" + REVISION = "revision" + SEGMENT = "segment" + EVIDENCE = "evidence" + DOCUMENT_BLOCK = "document-block" + CODE_SYMBOL = "code-symbol" + CONFIGURATION_KEY = "configuration-key" + DATABASE_OBJECT = "database-object" + API_OPERATION = "api-operation" + MESSAGE_TOPIC = "message-topic" + VALUES = frozenset({ASSET, REVISION, SEGMENT, EVIDENCE, DOCUMENT_BLOCK, + CODE_SYMBOL, CONFIGURATION_KEY, DATABASE_OBJECT, + API_OPERATION, MESSAGE_TOPIC}) + + +class BindingRole: + PRIMARY_SOURCE = "primary-source" + DEFINITION_SOURCE = "definition-source" + IMPLEMENTATION = "implementation" + CONFIGURATION = "configuration" + TEST = "test" + INTERFACE = "interface" + SUPPORTING = "supporting" + HISTORICAL = "historical" + VALUES = frozenset({PRIMARY_SOURCE, DEFINITION_SOURCE, IMPLEMENTATION, + CONFIGURATION, TEST, INTERFACE, SUPPORTING, + HISTORICAL}) + + +class BindingStatus: + ACTIVE = "active" + STALE = "stale" + REMOVED = "removed" + VALUES = frozenset({ACTIVE, STALE, REMOVED}) + + +class ClaimEvidenceRole: + PRIMARY = "primary" + SUPPORTING = "supporting" + CONTEXT = "context" + AUTHORITY = "authority" + VALUES = frozenset({PRIMARY, SUPPORTING, CONTEXT, AUTHORITY}) + + +class DecisionType: + """Every governance write records exactly one of these.""" + PROMOTE_CANDIDATE = "promote-candidate" + PROMOTE_RELATION = "promote-relation" + CREATE_ENTITY = "create-entity" + CREATE_CLAIM = "create-claim" + CREATE_RELATION = "create-relation" + ADD_ALIAS = "add-alias" + REMOVE_ALIAS = "remove-alias" + MERGE_ENTITY = "merge-entity" + SPLIT_ENTITY = "split-entity" + MARK_AUTHORITATIVE = "mark-authoritative" + MARK_NON_AUTHORITATIVE = "mark-non-authoritative" + SUPERSEDE = "supersede" + WITHDRAW = "withdraw" + REJECT_RELATION = "reject-relation" + RESTORE_RELATION = "restore-relation" + VALUES = frozenset({ + PROMOTE_CANDIDATE, PROMOTE_RELATION, CREATE_ENTITY, CREATE_CLAIM, + CREATE_RELATION, ADD_ALIAS, REMOVE_ALIAS, MERGE_ENTITY, SPLIT_ENTITY, + MARK_AUTHORITATIVE, MARK_NON_AUTHORITATIVE, SUPERSEDE, WITHDRAW, + REJECT_RELATION, RESTORE_RELATION, + }) + + +class DecisionTargetKind: + ENTITY = "entity" + CLAIM = "claim" + RELATION = "relation" + ALIAS = "alias" + BINDING = "binding" + CANDIDATE = "candidate" + RELATION_CANDIDATE = "relation-candidate" + WORKSPACE = "workspace" + VALUES = frozenset({ENTITY, CLAIM, RELATION, ALIAS, BINDING, CANDIDATE, + RELATION_CANDIDATE, WORKSPACE}) + + +class RevisionAction: + """What one Knowledge Revision recorded. Not a closed write-gate (a + bounded free label would also be safe) but kept closed for honest + reporting.""" + GRAPH_SEED = "graph-seed" + GRAPH_SYNC = "graph-sync" + GRAPH_RECONCILE = "graph-reconcile" + CANDIDATE_PROMOTION = "candidate-promotion" + RELATION_PROMOTION = "relation-promotion" + MANUAL_ENTITY_CREATE = "manual-entity-create" + MANUAL_CLAIM_CREATE = "manual-claim-create" + MANUAL_RELATION_CREATE = "manual-relation-create" + ALIAS_CHANGE = "alias-change" + ENTITY_MERGE = "entity-merge" + ENTITY_SPLIT = "entity-split" + CLAIM_SUPERSEDE = "claim-supersede" + SUPERSEDE = "supersede" + WITHDRAW = "withdraw" + AUTHORITY_CHANGE = "authority-change" + RELATION_STATE_CHANGE = "relation-state-change" + VALUES = frozenset({ + GRAPH_SEED, GRAPH_SYNC, GRAPH_RECONCILE, CANDIDATE_PROMOTION, + RELATION_PROMOTION, MANUAL_ENTITY_CREATE, MANUAL_CLAIM_CREATE, + MANUAL_RELATION_CREATE, ALIAS_CHANGE, ENTITY_MERGE, ENTITY_SPLIT, + CLAIM_SUPERSEDE, SUPERSEDE, WITHDRAW, AUTHORITY_CHANGE, + RELATION_STATE_CHANGE, + }) + + +class PromotionCandidateKind: + SEMANTIC_CANDIDATE = "semantic-candidate" + RELATION_CANDIDATE = "relation-candidate" + VALUES = frozenset({SEMANTIC_CANDIDATE, RELATION_CANDIDATE}) + + +class PromotionStatus: + PROMOTED = "promoted" + BLOCKED = "blocked" + ALREADY_PROMOTED = "already-promoted" + VALUES = frozenset({PROMOTED, BLOCKED, ALREADY_PROMOTED}) + + +class PromotionExpectedAction: + """What a promotion PLAN reports would happen.""" + CREATE_ENTITY_AND_CLAIM = "create-entity-and-claim" + ATTACH_CLAIM_TO_EXISTING_ENTITY = "attach-claim-to-existing-entity" + CREATE_RELATION = "create-relation" + ALREADY_PROMOTED = "already-promoted" + BLOCKED = "blocked" + IDENTITY_CONFLICT = "identity-conflict" + VALUES = frozenset({CREATE_ENTITY_AND_CLAIM, + ATTACH_CLAIM_TO_EXISTING_ENTITY, CREATE_RELATION, + ALREADY_PROMOTED, BLOCKED, IDENTITY_CONFLICT}) + + +class GraphNodeKind: + """Node kinds of the read-graph abstraction. Source-plane kinds are + PROJECTED from their canonical Phase 2 rows, never copied into graph + tables.""" + ENTITY = "entity" + CLAIM = "claim" + ASSET = "asset" + REVISION = "revision" + SEGMENT = "segment" + EVIDENCE = "evidence" + VALUES = frozenset({ENTITY, CLAIM, ASSET, REVISION, SEGMENT, EVIDENCE}) + + +#: Mapping from Phase 4 engineering-concept candidate types to canonical +#: Entity types (identical strings today, but the mapping is explicit so a +#: future rename on either side is a visible decision, not an accident). +CONCEPT_TYPE_TO_ENTITY_TYPE = { + "requirement": EntityType.REQUIREMENT, + "business-rule": EntityType.BUSINESS_RULE, + "decision": EntityType.DECISION, + "constraint": EntityType.CONSTRAINT, + "interface": EntityType.INTERFACE, + "acceptance-criterion": EntityType.ACCEPTANCE_CRITERION, + "failure-mode": EntityType.FAILURE_MODE, + "data-model": EntityType.DATA_MODEL, + "workflow": EntityType.WORKFLOW, + "test-case": EntityType.TEST_CASE, +} + +#: Default Claim type for a promoted engineering concept, by Entity type. +ENTITY_TYPE_TO_CLAIM_TYPE = { + EntityType.REQUIREMENT: ClaimType.NORMATIVE_STATEMENT, + EntityType.BUSINESS_RULE: ClaimType.NORMATIVE_STATEMENT, + EntityType.DECISION: ClaimType.DECISION_RATIONALE, + EntityType.CONSTRAINT: ClaimType.CONSTRAINT, + EntityType.INTERFACE: ClaimType.INTERFACE_CONTRACT, + EntityType.ACCEPTANCE_CRITERION: ClaimType.ACCEPTANCE_CONDITION, + EntityType.FAILURE_MODE: ClaimType.FAILURE_CONDITION, + EntityType.DATA_MODEL: ClaimType.DATA_DEFINITION, + EntityType.WORKFLOW: ClaimType.WORKFLOW_STEP, + EntityType.TEST_CASE: ClaimType.TEST_EXPECTATION, +} + +#: Phase 4 relation-candidate types that map onto canonical relation types. +#: Identical strings — the explicit table exists for the same reason as above. +RELATION_CANDIDATE_TYPE_TO_RELATION_TYPE = { + "refines": RelationType.REFINES, + "implements": RelationType.IMPLEMENTS, + "partially-implements": RelationType.PARTIALLY_IMPLEMENTS, + "configures": RelationType.CONFIGURES, + "verifies": RelationType.VERIFIES, + "supersedes": RelationType.SUPERSEDES, + "derived-from": RelationType.DERIVED_FROM, + "affected-by": RelationType.AFFECTED_BY, + "contradicts": RelationType.CONTRADICTS, + "possibly-related": RelationType.POSSIBLY_RELATED, +} + + +__all__ = [ + "require", + "EntityType", "ClaimType", "RelationType", "RelationState", + "GraphLifecycleStatus", "AuthorityStatus", "GraphOrigin", + "AliasType", "AliasStatus", "BindingRefKind", "BindingRole", + "BindingStatus", "ClaimEvidenceRole", "DecisionType", + "DecisionTargetKind", "RevisionAction", "PromotionCandidateKind", + "PromotionStatus", "PromotionExpectedAction", "GraphNodeKind", + "CONCEPT_TYPE_TO_ENTITY_TYPE", "ENTITY_TYPE_TO_CLAIM_TYPE", + "RELATION_CANDIDATE_TYPE_TO_RELATION_TYPE", +] diff --git a/openmind/main.py b/openmind/main.py index 9ca3031..f447bb9 100644 --- a/openmind/main.py +++ b/openmind/main.py @@ -675,6 +675,257 @@ def lens_deactivate(project_id: str, lens_id: str) -> Dict[str, Any]: return {"lens": _svc().lenses.deactivate(project_id, lens_id)} +# --------------------------------------------------------------------------- +# Canonical Knowledge Graph (OpenMind v2 Phase 5) — ADDITIVE routes only. +# +# Workspace-scoped through the service layer, bounded, typed: there is +# deliberately NO generic graph-mutation endpoint — every write is one +# explicit operation that records a Human Decision and one Knowledge +# Revision. `/projects` naming is kept; the Phase 3 combined search route +# POST /projects/{id}/knowledge/search is UNTOUCHED (graph search lives at +# /knowledge/graph-search). +# --------------------------------------------------------------------------- +@app.get("/projects/{project_id}/knowledge/stats") +def knowledge_stats(project_id: str) -> Dict[str, Any]: + return _svc().knowledge.get_stats(project_id) + + +@app.get("/projects/{project_id}/knowledge/revisions") +def knowledge_revisions(project_id: str, limit: int = 50, + offset: int = 0) -> Dict[str, Any]: + return _svc().knowledge.list_knowledge_revisions( + project_id, limit=limit, offset=offset) + + +@app.get("/projects/{project_id}/knowledge/revisions/{revision_number}") +def knowledge_revision(project_id: str, + revision_number: int) -> Dict[str, Any]: + return {"revision": _svc().knowledge.get_knowledge_revision( + project_id, revision_number)} + + +@app.get("/projects/{project_id}/knowledge/decisions") +def knowledge_decisions(project_id: str, target_kind: Optional[str] = None, + target_id: Optional[str] = None, + decision_type: Optional[str] = None, + limit: int = 100, offset: int = 0) -> Dict[str, Any]: + return _svc().knowledge.list_decisions( + project_id, target_kind=target_kind, target_id=target_id, + decision_type=decision_type, limit=limit, offset=offset) + + +@app.post("/projects/{project_id}/knowledge/seed/plan") +def knowledge_seed_plan(project_id: str) -> Dict[str, Any]: + return {"plan": _svc().knowledge.plan_seed(project_id)} + + +@app.post("/projects/{project_id}/knowledge/seed") +def knowledge_seed(project_id: str, + req: models.GraphActorReq) -> Dict[str, Any]: + return _svc().knowledge.seed(project_id, actor=req.actor) + + +@app.post("/projects/{project_id}/knowledge/sync") +def knowledge_sync(project_id: str, + req: models.GraphActorReq) -> Dict[str, Any]: + return _svc().knowledge.sync(project_id, actor=req.actor) + + +@app.post("/projects/{project_id}/knowledge/reconcile") +def knowledge_reconcile(project_id: str, + req: models.GraphActorReq) -> Dict[str, Any]: + return _svc().knowledge.reconcile_staleness(project_id, actor=req.actor) + + +@app.post("/projects/{project_id}/knowledge/graph-search") +def knowledge_graph_search(project_id: str, + req: models.GraphSearchReq) -> Dict[str, Any]: + """Search canonical Entities and Claims (exact > lexical > vector). The + Phase 3 code+document search stays at POST /knowledge/search.""" + return _svc().knowledge.search_entities( + project_id, req.query, limit=req.limit, + include_stale=req.include_stale) + + +@app.get("/projects/{project_id}/knowledge/nodes/{node_id}") +def knowledge_node(project_id: str, node_id: str) -> Dict[str, Any]: + return {"node": _svc().knowledge.get_node(project_id, node_id)} + + +@app.post("/projects/{project_id}/knowledge/nodes/{node_id}/expand") +def knowledge_expand(project_id: str, node_id: str, + req: models.GraphExpandReq) -> Dict[str, Any]: + return _svc().knowledge.expand_node( + project_id, node_id, direction=req.direction, + relation_types=req.relation_types or None, depth=req.depth, + node_limit=req.node_limit, edge_limit=req.edge_limit, + include_stale=req.include_stale) + + +@app.post("/projects/{project_id}/knowledge/path") +def knowledge_path(project_id: str, + req: models.GraphPathReq) -> Dict[str, Any]: + """Bounded shortest-path discovery. Generic reachability — NOT the + Phase 6 formal Requirement Traceability.""" + return _svc().knowledge.find_path( + project_id, req.source, req.target, + relation_types=req.relation_types or None, direction=req.direction, + max_depth=req.max_depth, include_stale=req.include_stale) + + +@app.get("/projects/{project_id}/entities") +def list_entities(project_id: str, entity_type: Optional[str] = None, + lifecycle_status: Optional[str] = "active", + origin: Optional[str] = None, limit: int = 100, + offset: int = 0) -> Dict[str, Any]: + return _svc().knowledge.list_entities( + project_id, entity_type=entity_type, + lifecycle_status=lifecycle_status, origin=origin, limit=limit, + offset=offset) + + +@app.get("/projects/{project_id}/entities/{entity_id}") +def get_entity(project_id: str, entity_id: str) -> Dict[str, Any]: + return {"entity": _svc().knowledge.get_entity(project_id, entity_id)} + + +@app.post("/projects/{project_id}/entities") +def create_entity(project_id: str, + req: models.EntityCreateReq) -> Dict[str, Any]: + return _svc().knowledge.create_entity( + project_id, entity_type=req.entity_type, + canonical_key=req.canonical_key, display_name=req.display_name, + description=req.description, evidence=req.evidence, + actor=req.actor, note=req.note, source_command="rest:POST /entities") + + +@app.post("/projects/{project_id}/entities/{entity_id}/aliases") +def add_entity_alias(project_id: str, entity_id: str, + req: models.AliasAddReq) -> Dict[str, Any]: + return _svc().knowledge.add_alias( + project_id, entity_id=entity_id, alias=req.alias, + alias_type=req.alias_type, evidence_id=req.evidence_id, + actor=req.actor, note=req.note, + source_command="rest:POST /entities/{id}/aliases") + + +@app.post("/projects/{project_id}/entities/{entity_id}/merge") +def merge_entity(project_id: str, entity_id: str, + req: models.EntityMergeReq) -> Dict[str, Any]: + return _svc().knowledge.merge_entities( + project_id, source_entity_id=entity_id, + target_entity_id=req.target_entity_id, actor=req.actor, + note=req.note, source_command="rest:POST /entities/{id}/merge") + + +@app.post("/projects/{project_id}/entities/{entity_id}/split") +def split_entity(project_id: str, entity_id: str, + req: models.EntitySplitReq) -> Dict[str, Any]: + return _svc().knowledge.split_entity( + project_id, source_entity_id=entity_id, + new_entity_type=req.new_entity_type, + new_canonical_key=req.new_canonical_key, + new_display_name=req.new_display_name, claim_ids=req.claim_ids, + binding_ids=req.binding_ids, + relation_rewrites=req.relation_rewrites, actor=req.actor, + note=req.note, source_command="rest:POST /entities/{id}/split") + + +@app.post("/projects/{project_id}/entities/{entity_id}/authority") +def entity_authority(project_id: str, entity_id: str, + req: models.AuthorityReq) -> Dict[str, Any]: + return _svc().knowledge.set_authority( + project_id, kind="entity", object_id=entity_id, + authority=req.authority, actor=req.actor, note=req.note, + source_command="rest:POST /entities/{id}/authority") + + +@app.get("/projects/{project_id}/claims") +def list_claims(project_id: str, entity_id: Optional[str] = None, + claim_type: Optional[str] = None, + lifecycle_status: Optional[str] = "active", + limit: int = 100, offset: int = 0) -> Dict[str, Any]: + return _svc().knowledge.list_claims( + project_id, entity_id=entity_id, claim_type=claim_type, + lifecycle_status=lifecycle_status, limit=limit, offset=offset) + + +@app.get("/projects/{project_id}/claims/{claim_id}") +def get_claim(project_id: str, claim_id: str) -> Dict[str, Any]: + return {"claim": _svc().knowledge.get_claim(project_id, claim_id)} + + +@app.post("/projects/{project_id}/claims") +def create_claim(project_id: str, + req: models.ClaimCreateReq) -> Dict[str, Any]: + return _svc().knowledge.create_claim( + project_id, entity_id=req.entity_id, claim_type=req.claim_type, + statement=req.statement, evidence=req.evidence, actor=req.actor, + note=req.note, source_command="rest:POST /claims") + + +@app.get("/projects/{project_id}/relations") +def list_relations(project_id: str, entity_id: Optional[str] = None, + relation_type: Optional[str] = None, + relation_state: Optional[str] = None, + lifecycle_status: Optional[str] = "active", + limit: int = 100, offset: int = 0) -> Dict[str, Any]: + return _svc().knowledge.list_relations( + project_id, entity_id=entity_id, relation_type=relation_type, + relation_state=relation_state, lifecycle_status=lifecycle_status, + limit=limit, offset=offset) + + +@app.get("/projects/{project_id}/relations/{relation_id}") +def get_relation(project_id: str, relation_id: str) -> Dict[str, Any]: + return {"relation": _svc().knowledge.get_relation(project_id, + relation_id)} + + +@app.post("/projects/{project_id}/relations") +def create_relation(project_id: str, + req: models.RelationCreateReq) -> Dict[str, Any]: + return _svc().knowledge.create_relation( + project_id, source_entity_id=req.source_entity_id, + target_entity_id=req.target_entity_id, + relation_type=req.relation_type, + relation_state=req.relation_state, confidence=req.confidence, + evidence=req.evidence, actor=req.actor, note=req.note, + source_command="rest:POST /relations") + + +@app.post("/projects/{project_id}/promotions/plan") +def promotion_plan(project_id: str, + req: models.PromotionReq) -> Dict[str, Any]: + return {"plan": _svc().knowledge.plan_candidate_promotion( + project_id, req.candidate_id)} + + +@app.post("/projects/{project_id}/promotions") +def promotion_promote(project_id: str, + req: models.PromotionReq) -> Dict[str, Any]: + return _svc().knowledge.promote_candidate( + project_id, req.candidate_id, actor=req.actor, note=req.note, + source_command="rest:POST /promotions") + + +@app.post("/projects/{project_id}/relation-promotions/plan") +def relation_promotion_plan(project_id: str, + req: models.RelationPromotionReq + ) -> Dict[str, Any]: + return {"plan": _svc().knowledge.plan_relation_promotion( + project_id, req.relation_candidate_id)} + + +@app.post("/projects/{project_id}/relation-promotions") +def relation_promotion_promote(project_id: str, + req: models.RelationPromotionReq + ) -> Dict[str, Any]: + return _svc().knowledge.promote_relation( + project_id, req.relation_candidate_id, actor=req.actor, + note=req.note, source_command="rest:POST /relation-promotions") + + def _reject_conflicting_targets(req: models.DocumentImportReq) -> None: """``asset`` / ``logical_key`` / ``new_asset`` name three different targets for one set of bytes; combining them has no single correct meaning, so it is diff --git a/openmind/mcp_server.py b/openmind/mcp_server.py index 95aee11..f9a6289 100644 --- a/openmind/mcp_server.py +++ b/openmind/mcp_server.py @@ -509,6 +509,155 @@ def get_semantic_usage(scope: str, run_id: str) -> Dict[str, Any]: SEMANTIC_TOOL_NAMES = tuple(fn.__name__ for fn in SEMANTIC_TOOLS) +# --------------------------------------------------------------------------- +# Knowledge-graph tools (OpenMind v2 Phase 5) — ADDITIVE and STRICTLY +# READ-ONLY. +# +# Deliberately absent: anything that promotes a candidate, creates an +# Entity/Claim/Relation, merges or splits, changes authority, seeds or syncs +# the graph, or exports a bundle. Every graph MUTATION is an explicit CLI +# (or REST) verb that records a Human Decision with a caller-supplied actor; +# Claude Code drives those through its shell where the command is visible. +# Every result is bounded, workspace-scoped and carries the current +# Knowledge Revision. +# --------------------------------------------------------------------------- +def get_graph_stats(scope: str) -> Dict[str, Any]: + """Canonical knowledge-graph statistics for a workspace: active entity/ + claim/relation counts by type and lifecycle, plus the current Knowledge + Revision. Runs the incremental staleness reconciliation first, so the + numbers never count knowledge whose sources moved on.""" + from .runtime import get_runtime + pid = _pids(scope)[0] + return get_runtime().knowledge.get_stats(pid) + + +def search_graph(scope: str, query: str, limit: int = 20, + include_stale: bool = False) -> Dict[str, Any]: + """Search canonical Entities and Claims (separate result sections). + Deterministic fusion: exact canonical key > exact alias > exact + identifier token > lexical > vector similarity — an exact identifier is + never outranked by something that merely reads similar. Stale/withdrawn + objects are excluded unless include_stale.""" + from .runtime import get_runtime + pid = _pids(scope)[0] + return get_runtime().knowledge.search_entities( + pid, query, limit=limit, include_stale=include_stale) + + +def get_graph_node(scope: str, node_id: str) -> Dict[str, Any]: + """One graph node in the stable read shape. Accepts entity, claim, + asset, revision, segment and evidence ids — the source-plane kinds are + projected from their canonical rows, never duplicated.""" + from .runtime import get_runtime + from .knowledge.errors import GraphNodeNotFound + knowledge = get_runtime().knowledge + last: Optional[Exception] = None + for pid in _pids(scope): + try: + return {"node": knowledge.get_node(pid, node_id)} + except GraphNodeNotFound as exc: + last = exc + raise last or ValueError(f"graph node not found: {node_id}") + + +def expand_graph(scope: str, node_id: str, depth: int = 2, + direction: str = "both", + relation_types: Optional[List[str]] = None, + include_stale: bool = False) -> Dict[str, Any]: + """Bounded, deterministic BFS expansion around one Entity node (hard + caps on depth, nodes and edges; the result says when it truncated).""" + from .runtime import get_runtime + pid = _pids(scope)[0] + return get_runtime().knowledge.expand_node( + pid, node_id, depth=depth, direction=direction, + relation_types=relation_types or None, include_stale=include_stale) + + +def find_graph_path(scope: str, source: str, target: str, max_depth: int = 6, + direction: str = "both", + include_stale: bool = False) -> Dict[str, Any]: + """Bounded shortest-path discovery between two Entities, with Relation + evidence summaries. Honest outcomes: found / no-path / truncated — a + missing edge is never invented. This is generic graph reachability, NOT + the Phase 6 formal Requirement Traceability.""" + from .runtime import get_runtime + pid = _pids(scope)[0] + return get_runtime().knowledge.find_path( + pid, source, target, max_depth=max_depth, direction=direction, + include_stale=include_stale) + + +def list_engineering_entities(scope: str, entity_type: Optional[str] = None, + lifecycle_status: str = "active", + limit: int = 50) -> Dict[str, Any]: + """List canonical engineering Entities (bounded). ``lifecycle_status`` + defaults to active; pass ``stale`` or empty for history. Every entity + entered the graph through deterministic projection, explicit manual + creation or explicit candidate promotion — never automatically.""" + from .runtime import get_runtime + pid = _pids(scope)[0] + return get_runtime().knowledge.list_entities( + pid, entity_type=entity_type, + lifecycle_status=(lifecycle_status or None), limit=limit) + + +def get_engineering_entity(scope: str, entity_id: str) -> Dict[str, Any]: + """One Entity with its aliases, bindings, claims and relations.""" + from .runtime import get_runtime + from .knowledge.errors import EntityNotFound + knowledge = get_runtime().knowledge + last: Optional[Exception] = None + for pid in _pids(scope): + try: + return {"entity": knowledge.get_entity(pid, entity_id)} + except EntityNotFound as exc: + last = exc + raise last or ValueError(f"entity not found: {entity_id}") + + +def get_engineering_claim(scope: str, claim_id: str) -> Dict[str, Any]: + """One Claim with its evidence joins. Every active claim carries at + least one Evidence citation verified against the immutable store.""" + from .runtime import get_runtime + from .knowledge.errors import ClaimNotFound + knowledge = get_runtime().knowledge + last: Optional[Exception] = None + for pid in _pids(scope): + try: + return {"claim": knowledge.get_claim(pid, claim_id)} + except ClaimNotFound as exc: + last = exc + raise last or ValueError(f"claim not found: {claim_id}") + + +def get_engineering_relation(scope: str, relation_id: str) -> Dict[str, Any]: + """One Relation with its evidence joins, state and provenance. + ``relation_state`` says how it is believed (explicit / inferred / + confirmed / rejected / stale / superseded); ``possibly-related`` is + never presented as anything stronger.""" + from .runtime import get_runtime + from .knowledge.errors import RelationNotFound + knowledge = get_runtime().knowledge + last: Optional[Exception] = None + for pid in _pids(scope): + try: + return {"relation": knowledge.get_relation(pid, relation_id)} + except RelationNotFound as exc: + last = exc + raise last or ValueError(f"relation not found: {relation_id}") + + +#: Additive read-only knowledge-graph tools (v2 Phase 5). Registered +#: ALONGSIDE everything above — 26 + 9 = 35 in total. +KNOWLEDGE_TOOLS: List[Callable[..., Any]] = [ + get_graph_stats, search_graph, get_graph_node, expand_graph, + find_graph_path, list_engineering_entities, get_engineering_entity, + get_engineering_claim, get_engineering_relation, +] + +KNOWLEDGE_TOOL_NAMES = tuple(fn.__name__ for fn in KNOWLEDGE_TOOLS) + + # --------------------------------------------------------------------------- # Construction # --------------------------------------------------------------------------- @@ -536,6 +685,8 @@ def create_mcp_server(runtime: Optional[Any] = None) -> FastMCP: server.tool()(fn) for fn in SEMANTIC_TOOLS: # additive read-only semantic/lens tools (Phase 4) server.tool()(fn) + for fn in KNOWLEDGE_TOOLS: # additive read-only graph tools (Phase 5) + server.tool()(fn) return server diff --git a/openmind/migrations/versions/v0006_knowledge_graph.py b/openmind/migrations/versions/v0006_knowledge_graph.py new file mode 100644 index 0000000..a8a8eda --- /dev/null +++ b/openmind/migrations/versions/v0006_knowledge_graph.py @@ -0,0 +1,348 @@ +"""Canonical Engineering Knowledge Graph: Entities, Claims, Relations, +aliases, bindings, Evidence joins, Human Decisions, Knowledge Revisions, +promotion records and deterministic projection state. + +OpenMind v2 Phase 5. Purely additive and non-destructive: it creates new +tables and their indexes, touches no existing row and drops nothing, so a +Phase 1–4 database migrates to this head with every project, job, Asset, +Revision, Segment, Evidence, content blob, document parse record, document +index, semantic policy, analysis run, target, candidate (all three kinds), +usage record, cache entry, Project Lens, template selection, Ask exchange, +case and map intact. + +WHAT EACH TABLE IS FOR +---------------------- +``engineering_entities`` + One canonical engineering concept per row. The logical identity is + ``UNIQUE(workspace_id, entity_type, canonical_key)`` — the generated id + (``ent_*``) is storage identity only. ``promoted_from_candidate_id`` is + PROVENANCE (no foreign key on purpose: the graph must never depend on a + Phase 4 candidate row's survival, and a candidate delete must never + cascade into canonical history). + +``engineering_entity_aliases`` + Alternative names, normalized for exact lookup. A normalized collision + across ACTIVE entities is reported by the application layer, never + silently attached; removed aliases keep their row for audit. + +``engineering_entity_bindings`` + Entity -> source-plane anchors (asset / revision / segment / evidence / + document-block / code-symbol / configuration-key / database-object / + api-operation / message-topic). ``ref_id`` is workspace-validated at the + application layer; a binding to a non-current Revision goes ``stale``. + +``engineering_claims`` (+ ``engineering_claim_evidence``) + Bounded statements about one Entity. ``normalized_statement_hash`` + deduplicates identical claims; corrections create a NEW claim and set + ``superseded_by_claim_id`` on the old one. Every ACTIVE claim has at + least one evidence join (application-enforced at every write path). + +``engineering_relations`` (+ ``engineering_relation_evidence``) + Typed edges between two entities of the same workspace, with a relation + state (explicit / inferred / confirmed / rejected / stale / superseded) + and provenance. Active identity is (workspace, source, target, type) — + enforced by the application inside the graph transaction, not by a + partial unique index, because superseded/rejected history rows share the + same tuple. + +``knowledge_decisions`` + One immutable audit row per governance write, with caller-supplied actor + (never inferred), bounded note and bounded before/after snapshots. + +``knowledge_revisions`` + The per-workspace monotonic ledger. ``UNIQUE(workspace_id, + revision_number)`` is the concurrency backstop: the allocator increments + inside the same transaction as the graph writes it stamps. + +``knowledge_promotions`` + Candidate -> canonical promotion records. Only promotions that HAPPENED + are stored (a blocked attempt persists nothing). + +``knowledge_projection_state`` + Per-workspace deterministic projection watermark: projector version, + source-knowledge hash, last written revision — what makes ``graph sync`` + an honest no-op when nothing changed. + +FOREIGN KEYS +------------ +Graph-internal foreign keys (aliases/bindings/claims -> entities, evidence +joins -> claims/relations, relations -> entities) exist so a workspace-level +entity wipe cascades cleanly. There is deliberately NO foreign key to +``semantic_candidates`` (provenance, not ownership) and none to ``evidence`` +(graph history must outlive a terminated asset model; integrity is enforced +at write time by the application, and staleness reconciliation handles the +rest). + +FROZEN once applied: the runner checksums this file's ``upgrade`` source; +express any later schema change as a NEW migration, never an edit here. +""" +from __future__ import annotations + +import sqlite3 + + +def upgrade(conn: sqlite3.Connection) -> None: + """Create the Phase 5 knowledge-graph schema. Idempotent. + + Every statement lives INSIDE this function on purpose: the runner + checksums ``inspect.getsource(upgrade)``, so SQL held in a module-level + constant could be edited after the migration had been applied without the + immutability guard noticing. + """ + statements = ( + # -- entities -------------------------------------------------------- + """ + CREATE TABLE IF NOT EXISTS engineering_entities ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + entity_type TEXT NOT NULL, + canonical_key TEXT NOT NULL, + display_name TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + lifecycle_status TEXT NOT NULL DEFAULT 'active', + authority_status TEXT NOT NULL DEFAULT 'unknown', + origin TEXT NOT NULL, + promoted_from_candidate_id TEXT NOT NULL DEFAULT '', + created_knowledge_revision INTEGER NOT NULL DEFAULT 0, + updated_knowledge_revision INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + stale_at TEXT, + superseded_by_entity_id TEXT, + merged_into_entity_id TEXT + ) + """, + "CREATE UNIQUE INDEX IF NOT EXISTS idx_kg_ent_identity " + "ON engineering_entities(workspace_id, entity_type, canonical_key)", + "CREATE INDEX IF NOT EXISTS idx_kg_ent_ws_lifecycle " + "ON engineering_entities(workspace_id, entity_type, lifecycle_status)", + "CREATE INDEX IF NOT EXISTS idx_kg_ent_ws_key " + "ON engineering_entities(workspace_id, canonical_key)", + "CREATE INDEX IF NOT EXISTS idx_kg_ent_promoted " + "ON engineering_entities(promoted_from_candidate_id)", + # -- aliases --------------------------------------------------------- + """ + CREATE TABLE IF NOT EXISTS engineering_entity_aliases ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + alias TEXT NOT NULL, + normalized_alias TEXT NOT NULL, + alias_type TEXT NOT NULL DEFAULT 'name', + origin TEXT NOT NULL DEFAULT 'manual', + status TEXT NOT NULL DEFAULT 'active', + evidence_id TEXT NOT NULL DEFAULT '', + created_knowledge_revision INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + FOREIGN KEY(entity_id) REFERENCES engineering_entities(id) + ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_kg_alias_normalized " + "ON engineering_entity_aliases(workspace_id, normalized_alias, status)", + "CREATE INDEX IF NOT EXISTS idx_kg_alias_entity " + "ON engineering_entity_aliases(entity_id, status)", + # -- bindings -------------------------------------------------------- + """ + CREATE TABLE IF NOT EXISTS engineering_entity_bindings ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + ref_kind TEXT NOT NULL, + ref_id TEXT NOT NULL DEFAULT '', + ref_key TEXT NOT NULL DEFAULT '', + binding_role TEXT NOT NULL DEFAULT 'supporting', + status TEXT NOT NULL DEFAULT 'active', + origin TEXT NOT NULL DEFAULT 'manual', + evidence_id TEXT NOT NULL DEFAULT '', + created_knowledge_revision INTEGER NOT NULL DEFAULT 0, + updated_knowledge_revision INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + stale_at TEXT, + FOREIGN KEY(entity_id) REFERENCES engineering_entities(id) + ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_kg_bind_entity " + "ON engineering_entity_bindings(entity_id, status)", + "CREATE INDEX IF NOT EXISTS idx_kg_bind_ref " + "ON engineering_entity_bindings(workspace_id, ref_kind, ref_id)", + "CREATE INDEX IF NOT EXISTS idx_kg_bind_ws_status " + "ON engineering_entity_bindings(workspace_id, status)", + # -- claims ---------------------------------------------------------- + """ + CREATE TABLE IF NOT EXISTS engineering_claims ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + claim_type TEXT NOT NULL, + statement TEXT NOT NULL, + normalized_statement_hash TEXT NOT NULL, + lifecycle_status TEXT NOT NULL DEFAULT 'active', + authority_status TEXT NOT NULL DEFAULT 'unknown', + origin TEXT NOT NULL, + promoted_from_candidate_id TEXT NOT NULL DEFAULT '', + created_knowledge_revision INTEGER NOT NULL DEFAULT 0, + updated_knowledge_revision INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + stale_at TEXT, + superseded_by_claim_id TEXT, + FOREIGN KEY(entity_id) REFERENCES engineering_entities(id) + ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_kg_claim_entity " + "ON engineering_claims(entity_id, lifecycle_status)", + "CREATE INDEX IF NOT EXISTS idx_kg_claim_ws " + "ON engineering_claims(workspace_id, lifecycle_status, claim_type)", + "CREATE INDEX IF NOT EXISTS idx_kg_claim_hash " + "ON engineering_claims(entity_id, normalized_statement_hash)", + "CREATE INDEX IF NOT EXISTS idx_kg_claim_promoted " + "ON engineering_claims(promoted_from_candidate_id)", + """ + CREATE TABLE IF NOT EXISTS engineering_claim_evidence ( + claim_id TEXT NOT NULL, + evidence_id TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'primary', + quote TEXT NOT NULL DEFAULT '', + quote_hash TEXT NOT NULL DEFAULT '', + PRIMARY KEY(claim_id, evidence_id, quote_hash), + FOREIGN KEY(claim_id) REFERENCES engineering_claims(id) + ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_kg_claim_ev_evidence " + "ON engineering_claim_evidence(evidence_id)", + # -- relations ------------------------------------------------------- + """ + CREATE TABLE IF NOT EXISTS engineering_relations ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + source_entity_id TEXT NOT NULL, + target_entity_id TEXT NOT NULL, + relation_type TEXT NOT NULL, + relation_state TEXT NOT NULL DEFAULT 'inferred', + confidence TEXT NOT NULL DEFAULT 'low', + lifecycle_status TEXT NOT NULL DEFAULT 'active', + authority_status TEXT NOT NULL DEFAULT 'unknown', + origin TEXT NOT NULL, + promoted_from_relation_candidate_id TEXT NOT NULL DEFAULT '', + created_knowledge_revision INTEGER NOT NULL DEFAULT 0, + updated_knowledge_revision INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + stale_at TEXT, + superseded_by_relation_id TEXT, + FOREIGN KEY(source_entity_id) REFERENCES engineering_entities(id) + ON DELETE CASCADE, + FOREIGN KEY(target_entity_id) REFERENCES engineering_entities(id) + ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_kg_rel_source " + "ON engineering_relations(source_entity_id, lifecycle_status)", + "CREATE INDEX IF NOT EXISTS idx_kg_rel_target " + "ON engineering_relations(target_entity_id, lifecycle_status)", + "CREATE INDEX IF NOT EXISTS idx_kg_rel_ws_type " + "ON engineering_relations(workspace_id, relation_type, " + "relation_state, lifecycle_status)", + "CREATE INDEX IF NOT EXISTS idx_kg_rel_identity " + "ON engineering_relations(workspace_id, source_entity_id, " + "target_entity_id, relation_type)", + "CREATE INDEX IF NOT EXISTS idx_kg_rel_promoted " + "ON engineering_relations(promoted_from_relation_candidate_id)", + """ + CREATE TABLE IF NOT EXISTS engineering_relation_evidence ( + relation_id TEXT NOT NULL, + evidence_id TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'primary', + quote TEXT NOT NULL DEFAULT '', + quote_hash TEXT NOT NULL DEFAULT '', + PRIMARY KEY(relation_id, evidence_id, quote_hash), + FOREIGN KEY(relation_id) REFERENCES engineering_relations(id) + ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_kg_rel_ev_evidence " + "ON engineering_relation_evidence(evidence_id)", + # -- human decisions ------------------------------------------------- + """ + CREATE TABLE IF NOT EXISTS knowledge_decisions ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + knowledge_revision_id TEXT NOT NULL DEFAULT '', + decision_type TEXT NOT NULL, + target_kind TEXT NOT NULL, + target_id TEXT NOT NULL DEFAULT '', + actor TEXT NOT NULL DEFAULT '', + note TEXT NOT NULL DEFAULT '', + before_json TEXT NOT NULL DEFAULT '{}', + after_json TEXT NOT NULL DEFAULT '{}', + source_command TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS idx_kg_dec_ws " + "ON knowledge_decisions(workspace_id, created_at)", + "CREATE INDEX IF NOT EXISTS idx_kg_dec_target " + "ON knowledge_decisions(workspace_id, target_kind, target_id)", + "CREATE INDEX IF NOT EXISTS idx_kg_dec_revision " + "ON knowledge_decisions(knowledge_revision_id)", + # -- knowledge revisions --------------------------------------------- + """ + CREATE TABLE IF NOT EXISTS knowledge_revisions ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + revision_number INTEGER NOT NULL, + parent_revision_number INTEGER NOT NULL DEFAULT 0, + change_set_id TEXT NOT NULL DEFAULT '', + action TEXT NOT NULL, + summary TEXT NOT NULL DEFAULT '', + actor TEXT NOT NULL DEFAULT '', + object_counts_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL + ) + """, + "CREATE UNIQUE INDEX IF NOT EXISTS idx_kg_rev_number " + "ON knowledge_revisions(workspace_id, revision_number)", + "CREATE INDEX IF NOT EXISTS idx_kg_rev_ws " + "ON knowledge_revisions(workspace_id, created_at)", + # -- promotions ------------------------------------------------------ + """ + CREATE TABLE IF NOT EXISTS knowledge_promotions ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + candidate_kind TEXT NOT NULL, + candidate_id TEXT NOT NULL, + target_kind TEXT NOT NULL DEFAULT '', + target_id TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL, + policy_version TEXT NOT NULL DEFAULT '', + actor TEXT NOT NULL DEFAULT '', + note TEXT NOT NULL DEFAULT '', + knowledge_revision_id TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS idx_kg_promo_candidate " + "ON knowledge_promotions(workspace_id, candidate_kind, candidate_id)", + "CREATE INDEX IF NOT EXISTS idx_kg_promo_target " + "ON knowledge_promotions(workspace_id, target_kind, target_id)", + # -- deterministic projection state ---------------------------------- + """ + CREATE TABLE IF NOT EXISTS knowledge_projection_state ( + workspace_id TEXT PRIMARY KEY, + projector_version TEXT NOT NULL DEFAULT '', + source_knowledge_hash TEXT NOT NULL DEFAULT '', + last_knowledge_revision INTEGER NOT NULL DEFAULT 0, + last_synced_at TEXT + ) + """, + ) + for statement in statements: + conn.execute(statement) diff --git a/openmind/models.py b/openmind/models.py index d2fb3a0..1169a28 100644 --- a/openmind/models.py +++ b/openmind/models.py @@ -230,3 +230,117 @@ class LensImportReq(BaseModel): """Import an organization lens file (by name or filename) into the workspace.""" name: str + + +# --------------------------------------------------------------------------- +# Canonical Knowledge Graph (OpenMind v2 Phase 5) — request bodies. +# +# Every mutating request carries explicit ``actor`` and ``note``; identity is +# never inferred from the transport. Evidence references are +# ``{evidence_id, quote?, role?}`` dicts validated against the immutable +# store at the service boundary. +# --------------------------------------------------------------------------- +class GraphSearchReq(BaseModel): + """Graph search over canonical Entities and Claims (NOT the Phase 3 + code+document knowledge search, which keeps its own route).""" + query: str + limit: int = 20 + include_stale: bool = False + + +class GraphExpandReq(BaseModel): + direction: str = "both" + relation_types: List[str] = Field(default_factory=list) + depth: int = 2 + node_limit: int = 200 + edge_limit: int = 600 + include_stale: bool = False + + +class GraphPathReq(BaseModel): + source: str + target: str + relation_types: List[str] = Field(default_factory=list) + direction: str = "both" + max_depth: int = 6 + include_stale: bool = False + + +class GraphActorReq(BaseModel): + """The minimal governance body: who did it, and why.""" + actor: str = "" + note: str = "" + + +class EntityCreateReq(BaseModel): + entity_type: str + canonical_key: str + display_name: str + description: str = "" + evidence: List[Dict[str, Any]] = Field(default_factory=list) + actor: str = "" + note: str = "" + + +class ClaimCreateReq(BaseModel): + entity_id: str + claim_type: str + statement: str + evidence: List[Dict[str, Any]] = Field(default_factory=list) + actor: str = "" + note: str = "" + + +class RelationCreateReq(BaseModel): + source_entity_id: str + target_entity_id: str + relation_type: str + relation_state: str = "confirmed" + confidence: str = "medium" + evidence: List[Dict[str, Any]] = Field(default_factory=list) + actor: str = "" + note: str = "" + + +class AliasAddReq(BaseModel): + alias: str + alias_type: str = "manual" + evidence_id: str = "" + actor: str = "" + note: str = "" + + +class EntityMergeReq(BaseModel): + target_entity_id: str + actor: str = "" + note: str = "" + + +class EntitySplitReq(BaseModel): + new_entity_type: str + new_canonical_key: str + new_display_name: str = "" + claim_ids: List[str] = Field(default_factory=list) + binding_ids: List[str] = Field(default_factory=list) + relation_rewrites: List[Dict[str, str]] = Field(default_factory=list) + actor: str = "" + note: str = "" + + +class AuthorityReq(BaseModel): + kind: str # entity | claim | relation + authority: str + actor: str = "" + note: str = "" + + +class PromotionReq(BaseModel): + candidate_id: str + actor: str = "" + note: str = "" + + +class RelationPromotionReq(BaseModel): + relation_candidate_id: str + actor: str = "" + note: str = "" diff --git a/openmind/ports/graph_repository.py b/openmind/ports/graph_repository.py new file mode 100644 index 0000000..d5595bf --- /dev/null +++ b/openmind/ports/graph_repository.py @@ -0,0 +1,84 @@ +"""The graph repository port (OpenMind v2 Phase 5). + +A narrow protocol over the canonical-graph persistence functions that the +:class:`~openmind.knowledge.service.KnowledgeService` and the deterministic +projector depend on. Like the other ports, it exists as a TESTING SEAM: a +test can substitute an in-memory implementation to exercise service logic +without a database, and the protocol documents exactly which persistence +surface the graph layer is allowed to touch. + +The production implementation is :mod:`openmind.knowledge.store` (module +functions over the shared WAL connection); :data:`default_graph_repository` +adapts it to this protocol. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Protocol, runtime_checkable + + +@runtime_checkable +class GraphRepository(Protocol): + """What the knowledge layer requires from graph persistence.""" + + # -- transactions ------------------------------------------------------- + def graph_transaction(self, workspace_id: str, *, action: str, + actor: str = "", summary: str = "", + change_set_id: str = ""): + """A context manager yielding a transaction handle with the insert/ + update methods of :class:`openmind.knowledge.store.GraphTransaction`. + One successful body = one Knowledge Revision; a failed body writes + nothing.""" + ... + + # -- entity reads ------------------------------------------------------- + def get_entity(self, workspace_id: str, + entity_id: str) -> Optional[Dict[str, Any]]: ... + + def find_entity_by_key(self, workspace_id: str, entity_type: str, + canonical_key: str) -> Optional[Dict[str, Any]]: ... + + def list_entities(self, workspace_id: str, + **filters: Any) -> List[Dict[str, Any]]: ... + + # -- claim reads -------------------------------------------------------- + def get_claim(self, workspace_id: str, + claim_id: str) -> Optional[Dict[str, Any]]: ... + + def find_claim_by_hash(self, workspace_id: str, entity_id: str, + normalized_statement_hash: str, + lifecycle_status: str = "active" + ) -> Optional[Dict[str, Any]]: ... + + # -- relation reads ----------------------------------------------------- + def get_relation(self, workspace_id: str, + relation_id: str) -> Optional[Dict[str, Any]]: ... + + def find_active_relation(self, workspace_id: str, source_entity_id: str, + target_entity_id: str, relation_type: str + ) -> Optional[Dict[str, Any]]: ... + + # -- ledger ------------------------------------------------------------- + def current_revision_number(self, workspace_id: str) -> int: ... + + def find_promotion(self, workspace_id: str, candidate_kind: str, + candidate_id: str) -> Optional[Dict[str, Any]]: ... + + +class _ModuleGraphRepository: + """Adapts :mod:`openmind.knowledge.store` to :class:`GraphRepository`. + + Thin by design — every attribute defers to the store module at call time, + so a test that monkeypatches a store function still takes effect through + the repository.""" + + def __getattr__(self, name: str) -> Any: + from ..knowledge import store + return getattr(store, name) + + +def default_graph_repository() -> Any: + """The production repository over the shared SQLite connection.""" + return _ModuleGraphRepository() + + +__all__ = ["GraphRepository", "default_graph_repository"] diff --git a/openmind/runtime.py b/openmind/runtime.py index c6b2655..2f15a33 100644 --- a/openmind/runtime.py +++ b/openmind/runtime.py @@ -119,6 +119,10 @@ def semantic(self): def lenses(self): return self.services.lenses + @property + def knowledge(self): + return self.services.knowledge + @property def export(self): return self.services.export diff --git a/openmind/services/service_container.py b/openmind/services/service_container.py index 595e77c..1ad4a8b 100644 --- a/openmind/services/service_container.py +++ b/openmind/services/service_container.py @@ -40,6 +40,7 @@ def __init__(self, ensure_worker: Optional[Callable[[], None]] = None, self._health: Optional[HealthService] = None self._semantic = None self._lenses = None + self._knowledge = None @property def workspaces(self) -> WorkspaceService: @@ -101,6 +102,17 @@ def lenses(self): ensure_worker=self._ensure_worker) return self._lenses + @property + def knowledge(self): + # Canonical Engineering Knowledge Graph (v2 Phase 5). Lazy for the + # same reason as the semantic plane: export/doctor never pay for it, + # and constructing it never starts a worker or opens a collection. + if self._knowledge is None: + from ..knowledge.service import KnowledgeService + self._knowledge = KnowledgeService( + self.workspaces, ensure_worker=self._ensure_worker) + return self._knowledge + @property def export(self) -> ExportService: if self._export is None: diff --git a/openmind/vectorstore.py b/openmind/vectorstore.py index 87bfb19..5274df9 100644 --- a/openmind/vectorstore.py +++ b/openmind/vectorstore.py @@ -544,6 +544,17 @@ def documents_collection_name(project_id: str) -> str: return f"documents_{project_id}" +def knowledge_collection_name(project_id: str) -> str: + """The canonical-graph Entity/Claim projection (OpenMind v2 Phase 5). + + Separate from BOTH ``code_`` and ``documents_``: graph text in + either would silently change what the frozen search tools return, and + code/document chunks in the graph collection would make graph search + claim hits it cannot govern. + """ + return f"knowledge_{project_id}" + + def count_collection(collection_name: str) -> int: """Row count for a collection that may not exist yet — 0 if it doesn't. @@ -574,11 +585,12 @@ def get_documents_store(project_id: str) -> _Store: return get_store(documents_collection_name(project_id)) -#: Every per-project collection prefix. `documents_` MUST be listed here or the -#: startup orphan sweep would not recognize a document collection as belonging -#: to a project, and would leave it behind forever after a project delete. +#: Every per-project collection prefix. `documents_` and `knowledge_` MUST be +#: listed here or the startup orphan sweep would not recognize those +#: collections as belonging to a project, and would leave them behind forever +#: after a project delete. #: Longest-first, so `code_` cannot shadow a future prefix that starts with it. -_COLLECTION_PREFIXES = ("documents_", "cases_", "code_") +_COLLECTION_PREFIXES = ("knowledge_", "documents_", "cases_", "code_") def project_collection_names(project_id: str) -> List[str]: diff --git a/openmind/version.py b/openmind/version.py index fd5162f..99b3e45 100644 --- a/openmind/version.py +++ b/openmind/version.py @@ -5,30 +5,32 @@ ``generator.version`` — reads it from here, so the value can never drift between surfaces. -This is a PRE-v2 development version on purpose. OpenMind v2's later enterprise -knowledge features (canonical Claim/Relation tables, the engineering Knowledge -Graph, requirement traceability, OCR) are NOT implemented; labelling the -current build ``2.0.0`` would be a false claim. ``1.4.0-dev`` is the honest -reading: Phase 4 added the policy-governed semantic reasoning plane — a -provider-neutral SPI (local OpenAI-compatible, OpenAI, Anthropic, Azure -OpenAI, mock) behind an audited egress path, evidence-bound candidate -extraction with local verification, a resumable budget-bounded analysis -pipeline with a local result cache, candidate staleness, and Adaptive Project -Lenses — on top of the Phase 3 document plane, the Phase 2 -Asset/Revision/Segment/Evidence foundation and the Phase 1 tool-first runtime. +This is a PRE-v2 development version on purpose. OpenMind v2's later +enterprise features (formal Requirement Traceability, conflict resolution, +change-impact analysis, OCR) are NOT implemented; labelling the current build +``2.0.0`` would be a false claim. ``1.5.0-dev`` is the honest reading: +Phase 5 added the canonical Engineering Knowledge Graph — durable +evidence-bound Entities, Claims and Relations entered only through +deterministic projection, explicit manual creation or the explicit promotion +of a human-confirmed semantic Candidate, versioned by per-workspace Knowledge +Revisions, audited by immutable Human Decisions, searchable through a +separate graph vector projection, and exportable as the Knowledge Bundle +2.0 **Draft** — on top of the Phase 4 semantic plane, the Phase 3 document +plane, the Phase 2 Asset/Revision/Segment/Evidence foundation and the +Phase 1 tool-first runtime. -What Phase 4 pointedly does NOT do is turn any of those candidates into -canonical truth: entity/claim/relation tables, candidate promotion and the -Knowledge Graph are Phase 5. Cloud reasoning exists but is disabled until a -workspace explicitly opts in; ordinary ingestion still makes zero model calls. +What Phase 5 pointedly does NOT do: promote anything automatically (review +and promotion stay separate acts), resolve conflicts (conflict candidates +stay candidates for Phase 6), or claim the generic graph path command is +formal traceability. Ordinary ingestion still makes zero model calls. The artifact ``schemaVersion`` is deliberately NOT derived from this constant — it is a separate, frozen integration contract owned by -:mod:`openmind.artifacts` and remains ``1.1.0`` (neither the Asset model, the -document model nor any semantic candidate is exported). +:mod:`openmind.artifacts` and remains ``1.1.0``. The Knowledge Bundle has its +own draft version (``2.0.0-draft.1``) owned by :mod:`openmind.knowledge.bundle`. """ from __future__ import annotations -RUNTIME_VERSION = "1.4.0-dev" +RUNTIME_VERSION = "1.5.0-dev" __all__ = ["RUNTIME_VERSION"] diff --git a/scripts/run_acceptance.py b/scripts/run_acceptance.py index 039bc14..c3655c5 100644 --- a/scripts/run_acceptance.py +++ b/scripts/run_acceptance.py @@ -146,6 +146,63 @@ def path(self) -> Path: "additive REST + 7 read-only MCP tools + the 26-tool " "compatibility gate + artifact/bridge/Ask invariants"), + # -- canonical knowledge graph (v2 Phase 5) ------------------------------ + # Ordered cheapest-first: the migration suite is pure sqlite; the model + # suites build one small workspace each; projection/staleness ingest a + # Java fixture; cli/adapters drive the full surfaces end to end. + Script("verify_knowledge_migration", CORE, + "v0006 graph schema: tables, indexes, idempotency, v1-v5 " + "checksum immutability, FK integrity, rollback"), + Script("verify_knowledge_entities", CORE, + "canonical entities: deterministic keys, identity uniqueness, " + "workspace scoping, evidence-required manual creation, alias " + "collisions reported"), + Script("verify_knowledge_claims", CORE, + "canonical claims: evidence + quote verification, dedup, " + "correction-by-supersede, explicit authority, lifecycle"), + Script("verify_knowledge_relations", CORE, + "canonical relations: endpoint/scoping rules, closed types, " + "self-relation and manual-inferred rejection, dedup, " + "reject/restore"), + Script("verify_knowledge_revisions", CORE, + "knowledge revision ledger: monotonic numbering, one tx = one " + "revision, failed tx = none, concurrency uniqueness"), + Script("verify_knowledge_decisions", CORE, + "human decision audit: every write decided, actor never " + "inferred, bounded notes/snapshots, immutability, no secrets"), + Script("verify_knowledge_promotion", CORE, + "candidate promotion: full eligibility matrix, plan-is-dry-run, " + "transactional idempotent promotion, classification/" + "revision-status non-interference, relation endpoints, " + "conflict candidates never promotable"), + Script("verify_knowledge_projection", CORE, + "deterministic projection: asset/segment/containment/call/facet " + "rules, zero provider calls, unchanged sync no-op, incremental " + "updates"), + Script("verify_knowledge_staleness", CORE, + "graph staleness: revision movement stales bindings/claims/" + "relations transitively; authority preserved; startup backstop"), + Script("verify_knowledge_merge_split", CORE, + "entity merge and split: transactional, auditable, nothing " + "deleted, duplicates deduplicated, invalid input aborts whole"), + Script("verify_knowledge_search", CORE, + "graph search + vector projection: exact-over-similar " + "precedence, stale filtering, collection isolation and " + "lifecycle"), + Script("verify_knowledge_graph", CORE, + "graph queries: node kinds, bounded deterministic expansion, " + "honest path outcomes, evidence summaries, subgraph"), + Script("verify_knowledge_bundle", CORE, + "Bundle 2.0 Draft: deterministic export, manifest hashes, " + "referential integrity, history modes, verifier catches " + "corruption, .openmind 1.1.0 unchanged"), + Script("verify_knowledge_cli", CORE, + "knowledge CLI contract: JSON, exit codes, graph/promotion/" + "entity/claim/relation/history/bundle commands"), + Script("verify_knowledge_adapters", CORE, + "additive knowledge REST routes + exactly 9 read-only MCP " + "graph tools + the 35-tool compatibility gate"), + # -- template / docs pipeline ------------------------------------------- Script("verify_templates", CORE, "template profile selection and validation"), Script("verify_facets", CORE, "template facets, roles and projections"), diff --git a/tests/_knowledge_helpers.py b/tests/_knowledge_helpers.py new file mode 100644 index 0000000..34e2b39 --- /dev/null +++ b/tests/_knowledge_helpers.py @@ -0,0 +1,314 @@ +"""Shared setup for the Phase 5 knowledge-graph acceptance suites. + +Import AFTER ``_isolate``. Provides: the checked-result recorder, neutral +invented fixture content (spec §38), workspace builders, a confirmed- +candidate factory driven by the Phase 4 MOCK provider (no real provider API +is ever called), and small read helpers. +""" +from __future__ import annotations + +import sys +import tempfile +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +_results: List[Tuple[str, bool]] = [] + + +def check(desc: str, cond: Any) -> None: + _results.append((desc, bool(cond))) + print(("PASS" if cond else "FAIL") + " - " + desc) + + +def finish() -> None: + bad = [d for d, ok in _results if not ok] + print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") + sys.exit(1 if bad else 0) + + +# --------------------------------------------------------------------------- +# Neutral fixture content (invented; spec §38) +# --------------------------------------------------------------------------- +REQUIREMENTS_MD = """# Neutral Component Requirements + +## Scope +REQ-NC-017: The name check service shall answer within 2 seconds. + +REQ-NC-018: Failed name checks must be retried three times. + +## Notes +The archiver should compress completed batches nightly. +""" + +DESIGN_MD = """# Neutral Component Design + +## Interfaces +The NameCheck API exposes POST /name-check returning the check result. + +REQ-NC-017 is implemented by the NameCheckService with a 2 second budget. +""" + +OPENAPI_YAML = """openapi: 3.0.0 +info: + title: NameCheck API + version: "1.0" +paths: + /name-check: + post: + operationId: runNameCheck + summary: Run a name check + responses: + "200": + description: check result +components: + schemas: + NameCheckResult: + type: object + properties: + status: + type: string +""" + +PASSENGER_SQL = """CREATE TABLE PASSENGER ( + PASSENGER_ID INTEGER PRIMARY KEY, + FULL_NAME VARCHAR(200) NOT NULL, + CHECK_STATUS VARCHAR(20) +); + +CREATE TABLE CHECK_AUDIT ( + AUDIT_ID INTEGER PRIMARY KEY, + PASSENGER_ID INTEGER NOT NULL, + RESULT VARCHAR(20) +); +""" + +JAVA_SERVICE = """package org.example.check; + +public class NameCheckService { + private final int timeoutMillis = 2000; + + public NameCheckService() { + } + + public String execute(String request) { + Recorder recorder = new Recorder(); + recorder.record(request); + return normalize(request); + } + + private String normalize(String raw) { + return raw == null ? "" : raw.trim().toUpperCase(); + } +} +""" + +JAVA_RECORDER = """package org.example.check; + +public class Recorder { + public void record(String entry) { + System.out.println(entry); + } +} +""" + +#: A SECOND class with the SAME name in another directory — what makes the +#: name-based call edge from the service AMBIGUOUS (two possible owners for +#: the referenced class name ``Recorder``). +JAVA_RECORDER_ALT = """package org.example.audit; + +public class Recorder { + public void record(String entry) { + System.err.println(entry); + } +} +""" + +JAVA_TEST = """package org.example.check; + +public class NameCheckServiceTest { + public void shouldNormalize() { + NameCheckService service = new NameCheckService(); + service.execute(" alpha "); + } +} +""" + +CONFIG_PROPERTIES = """namecheck.timeout=2000 +namecheck.retry.count=3 +""" + + +def make_minimal_workspace(runtime, name: str = "kg-min") -> str: + """A workspace with ONE requirements document — the cheapest source of + real Evidence ids for manual-creation tests.""" + workspace = runtime.workspaces.create(name) + pid = workspace["id"] + src = Path(tempfile.mkdtemp(prefix="om_kgfix_")) + path = src / "requirements.md" + path.write_text(REQUIREMENTS_MD, encoding="utf-8") + result = runtime.documents.add_document(pid, str(path), wait=True, + timeout=180) + assert result.get("status") in ("new_asset", "revision", "duplicate"), \ + f"fixture import failed: {result}" + return pid + + +def make_source_workspace(runtime, name: str = "kg-src", + with_documents: bool = True) -> Tuple[str, Path]: + """A workspace over a small neutral source tree (Java service + test + + config), plus (optionally) OpenAPI, SQL and requirements documents. + Returns (workspace_id, source_root).""" + workspace = runtime.workspaces.create(name) + pid = workspace["id"] + src = Path(tempfile.mkdtemp(prefix="om_kgsrc_")) + (src / "src").mkdir() + (src / "src" / "NameCheckService.java").write_text(JAVA_SERVICE, + encoding="utf-8") + (src / "src" / "Recorder.java").write_text(JAVA_RECORDER, + encoding="utf-8") + (src / "alt").mkdir() + (src / "alt" / "Recorder.java").write_text(JAVA_RECORDER_ALT, + encoding="utf-8") + (src / "test").mkdir() + (src / "test" / "NameCheckServiceTest.java").write_text(JAVA_TEST, + encoding="utf-8") + (src / "namecheck.properties").write_text(CONFIG_PROPERTIES, + encoding="utf-8") + runtime.workspaces.add_path(pid, str(src), []) + runtime.ingest.start(pid, wait=True, timeout=300) + if with_documents: + docs = Path(tempfile.mkdtemp(prefix="om_kgdoc_")) + for filename, content in (("requirements.md", REQUIREMENTS_MD), + ("design.md", DESIGN_MD), + ("name-check-api.yaml", OPENAPI_YAML), + ("passenger.sql", PASSENGER_SQL)): + path = docs / filename + path.write_text(content, encoding="utf-8") + result = runtime.documents.add_document(pid, str(path), + wait=True, timeout=180) + assert result.get("status") in ("new_asset", "revision", + "duplicate"), \ + f"fixture import failed for {filename}: {result}" + return pid, src + + +# --------------------------------------------------------------------------- +# Evidence + candidate factories (mock provider only) +# --------------------------------------------------------------------------- +def evidence_map(pid: str) -> Dict[str, str]: + """evidence_id -> immutable text for every current-revision segment.""" + from openmind import db + from openmind.semantic.context import resolve_evidence_text + out: Dict[str, str] = {} + for asset in db.list_assets(pid, state="active", limit=1000): + revision_id = asset.get("current_revision_id") + if not revision_id: + continue + for segment in db.list_segments(pid, revision_id, limit=500): + ev = db.get_evidence_for_segment(pid, segment["id"]) + if not ev: + continue + text = resolve_evidence_text(pid, ev["id"]) + if text: + out[ev["id"]] = text + return out + + +def find_evidence(pid: str, containing: str) -> str: + for evidence_id, text in evidence_map(pid).items(): + if containing in text: + return evidence_id + raise AssertionError(f"no evidence contains {containing!r}") + + +def mock_profile(name: str = "mock-kg", + responses: Optional[Dict[str, Any]] = None): + from openmind.semantic.models import ProviderProfile + from openmind.semantic.providers import profiles + profile = ProviderProfile(name=name, kind="mock", + metadata={"responses": responses or {}}) + profiles.upsert_profile(profile) + return profile + + +def requirement_response(evidence_id: str, *, stable_key: str = "REQ-NC-017", + statement: str = "The name check service shall " + "answer within 2 seconds.", + quote: str = "shall answer within 2 seconds" + ) -> Dict[str, Any]: + return {"candidates": [{ + "candidateType": "requirement", "stableKey": stable_key, + "title": f"{stable_key} response time", + "statement": statement, "attributes": {}, + "evidence": [{"evidenceId": evidence_id, "quote": quote}], + "confidenceHint": "medium", + "reason": "explicit normative statement with an identifier"}]} + + +def make_confirmed_candidate(runtime, pid: str, *, + containing: str = "REQ-NC-017", + response: Optional[Dict[str, Any]] = None, + confirm: bool = True) -> str: + """Run one mock requirement extraction and (optionally) confirm the + candidate. Returns the candidate id.""" + evidence_id = find_evidence(pid, containing) + mock_profile(responses={ + "requirement-extraction": response + or requirement_response(evidence_id)}) + runtime.semantic.set_policy(pid, provider_profile="mock-kg") + runtime.semantic.start_analysis( + pid, task_types=["requirement-extraction"], + scope={"kind": "documents"}, wait=True, timeout=180) + candidates = runtime.semantic.list_candidates( + pid, candidate_type="requirement")["candidates"] + assert candidates, "mock analysis produced no candidate" + candidate_id = candidates[0]["id"] + if confirm: + runtime.semantic.review_candidate(pid, candidate_id, + decision="confirm", + reviewer="fixture-reviewer") + return candidate_id + + +def insert_relation_candidate(pid: str, *, source_ref: Dict[str, Any], + target_ref: Dict[str, Any], + relation_type: str = "implements", + evidence_id: str = "", quote: str = "", + evidence_status: str = "verified", + source_candidate_id: Optional[str] = None, + target_candidate_id: Optional[str] = None + ) -> str: + """Insert one relation candidate through the SAME store write the Phase 4 + runner uses (the mock relation pipeline needs multi-candidate setups this + factory sidesteps).""" + from openmind.knowledge.identity import quote_hash + from openmind.semantic import store as semantic_store + evidence = [] + if evidence_id: + evidence = [{"evidence_id": evidence_id, "quote": quote, + "quote_hash": quote_hash(quote), "role": "supports"}] + ids = semantic_store.insert_relations(pid, [{ + "relation_type": relation_type, "source_ref": source_ref, + "target_ref": target_ref, + "source_candidate_id": source_candidate_id, + "target_candidate_id": target_candidate_id, + "reason": "fixture", "confidence": "medium", + "evidence_status": evidence_status, "evidence": evidence}]) + return ids[0] + + +def confirm_relation_candidate(runtime, pid: str, relation_id: str) -> None: + runtime.semantic.review_relation_candidate( + pid, relation_id, decision="confirm", reviewer="fixture-reviewer") + + +def entity_by_key(pid: str, entity_type: str, + canonical_key: str) -> Optional[Dict[str, Any]]: + from openmind.knowledge import store + return store.find_entity_by_key(pid, entity_type, canonical_key) + + +def entities_of_type(pid: str, entity_type: str) -> List[Dict[str, Any]]: + from openmind.knowledge import store + return store.list_entities(pid, entity_type=entity_type, + lifecycle_status=None, limit=1000) diff --git a/tests/verify_adapters.py b/tests/verify_adapters.py index 2a37242..46da26f 100644 --- a/tests/verify_adapters.py +++ b/tests/verify_adapters.py @@ -288,9 +288,17 @@ def check(desc, cond, extra=""): "get_semantic_usage") check("the seven read-only semantic tools are registered additively", set(SEMANTIC_TOOLS) <= set(registered), str(registered)) +# v2 Phase 5 adds nine read-only knowledge-graph tools the same way. +KNOWLEDGE_TOOLS = ("get_graph_stats", "search_graph", "get_graph_node", + "expand_graph", "find_graph_path", + "list_engineering_entities", "get_engineering_entity", + "get_engineering_claim", "get_engineering_relation") +check("the nine read-only graph tools are registered additively", + set(KNOWLEDGE_TOOLS) <= set(registered), str(registered)) check("every registered tool is an accounted-for addition", set(registered) == set(REQUIRED_TOOLS) | set(ASSET_TOOLS) - | set(DOCUMENT_TOOLS) | set(SEMANTIC_TOOLS), str(registered)) + | set(DOCUMENT_TOOLS) | set(SEMANTIC_TOOLS) | set(KNOWLEDGE_TOOLS), + str(registered)) descriptions = {t.name: (t.description or "") for t in asyncio.run(server.list_tools())} check("every tool still carries its docstring description", diff --git a/tests/verify_asset_adapters.py b/tests/verify_asset_adapters.py index b23f23a..cdb24d4 100644 --- a/tests/verify_asset_adapters.py +++ b/tests/verify_asset_adapters.py @@ -64,10 +64,15 @@ def check(desc, cond, extra=""): SEMANTIC = {"list_semantic_runs", "get_semantic_run", "list_semantic_candidates", "get_semantic_candidate", "list_project_lenses", "get_project_lens", "get_semantic_usage"} +# Phase 5 nine read-only knowledge-graph tools. +KNOWLEDGE = {"get_graph_stats", "search_graph", "get_graph_node", + "expand_graph", "find_graph_path", "list_engineering_entities", + "get_engineering_entity", "get_engineering_claim", + "get_engineering_relation"} check("all nine core MCP tools remain", CORE <= tool_names) check("the four read-only asset MCP tools are added", ASSET <= tool_names) check("every registered tool is an accounted-for addition", - tool_names == CORE | ASSET | DOCUMENT | SEMANTIC) + tool_names == CORE | ASSET | DOCUMENT | SEMANTIC | KNOWLEDGE) check("the MCP server keeps its name", server.name == "open-mind") # --------------------------------------------------------------------------- diff --git a/tests/verify_document_adapters.py b/tests/verify_document_adapters.py index e30e9bc..8101577 100644 --- a/tests/verify_document_adapters.py +++ b/tests/verify_document_adapters.py @@ -248,7 +248,7 @@ def check(desc, cond): check("compat: GET /api/health still works", client.get("/api/health").status_code == 200) check("compat: health reports the new runtime version", - client.get("/api/health").json()["version"] == "1.4.0-dev") + client.get("/api/health").json()["version"] == "1.5.0-dev") assets = client.get(f"/projects/{WS}/assets").json() check("compat: GET assets still returns the Phase 2 shape", {"assets", "total", "limit", "offset", "count"} <= set(assets)) diff --git a/tests/verify_document_cli.py b/tests/verify_document_cli.py index fd1ebf9..1d856a7 100644 --- a/tests/verify_document_cli.py +++ b/tests/verify_document_cli.py @@ -358,10 +358,10 @@ def attach(name, source=None, text=None): check("compat: status still reports the Phase 2 asset counts", {"assets_total", "revisions", "segments", "evidence"} <= set(status["assets"])) -check("compat: status reports the v0005 schema head", - status["schema_version"] == 5) -check("compat: the version constant advanced to 1.4.0-dev", - status["version"] == "1.4.0-dev") +check("compat: status reports the v0006 schema head", + status["schema_version"] == 6) +check("compat: the version constant advanced to 1.5.0-dev", + status["version"] == "1.5.0-dev") # --------------------------------------------------------------------------- bad = [d for d, ok in _results if not ok] diff --git a/tests/verify_document_search.py b/tests/verify_document_search.py index 844d9aa..80d68a9 100644 --- a/tests/verify_document_search.py +++ b/tests/verify_document_search.py @@ -281,16 +281,17 @@ def check(desc, cond): check("6: a mention with no real target produces no candidate", not any(c["target"].get("logical_key") == "does-not-exist" for c in related["candidates"])) -# The guard is against CANONICAL Claim/Relation tables (Phase 5 territory). -# Phase 4's semantic_relation_* tables are explicitly CANDIDATE stores — -# every row carries candidate status — so tables named *candidate*/ -# *evidence* are the sanctioned exception, not a violation. +# The guard is against writing CANONICAL Claims/Relations from a +# deterministic candidate association. Since v2 Phase 5 the canonical graph +# TABLES exist (engineering_claims / engineering_relations) — but the only +# paths allowed to fill them are deterministic projection, manual creation +# and explicit promotion. Observing document candidates must leave them +# empty: candidates are reported, never persisted as graph truth. check("6: nothing was persisted as a canonical relation or claim", - not [r[0] for r in db._c().execute( - "SELECT name FROM sqlite_master WHERE type='table'").fetchall() - if ("relation" in r[0].lower() or "claim" in r[0].lower()) - and "candidate" not in r[0].lower() - and "evidence" not in r[0].lower()]) + db._c().execute("SELECT COUNT(*) FROM engineering_relations") + .fetchone()[0] == 0 + and db._c().execute("SELECT COUNT(*) FROM engineering_claims") + .fetchone()[0] == 0) check("6: the limit is respected", runtime.documents.find_related_candidates(WS, REQ_ASSET, limit=2)["count"] <= 2) diff --git a/tests/verify_knowledge_adapters.py b/tests/verify_knowledge_adapters.py new file mode 100644 index 0000000..53a150f --- /dev/null +++ b/tests/verify_knowledge_adapters.py @@ -0,0 +1,268 @@ +"""Phase 5 adapter + compatibility gate — the 26 existing MCP tools remain, +EXACTLY nine read-only graph tools are added (35 total), knowledge REST +routes are additive and workspace-scoped, existing routes/commands survive, +and the Phase 3/4 surfaces are untouched.""" +import json +import os +import sys +import tempfile + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_confirmed_candidate, + make_minimal_workspace) + +# --------------------------------------------------------------------------- +# 1. MCP: 26 unchanged + exactly 9 read-only graph tools = 35 +# --------------------------------------------------------------------------- +from openmind import mcp_server # noqa: E402 + +check("the nine core MCP tools are unchanged", + mcp_server.TOOL_NAMES == ("search", "route", "dispatch", + "get_glossary", "find_similar_cases", + "save_case", "get_doc", "propose_fix", + "apply_fix")) +check("the four asset tools are unchanged", + mcp_server.ASSET_TOOL_NAMES == ("list_assets", "get_asset", + "get_asset_revisions", + "get_evidence")) +check("the six document tools are unchanged", + mcp_server.DOCUMENT_TOOL_NAMES == ( + "list_documents", "get_document", "get_document_outline", + "search_documents", "search_knowledge", + "find_document_related_candidates")) +check("the seven semantic/lens tools are unchanged", + mcp_server.SEMANTIC_TOOL_NAMES == ( + "list_semantic_runs", "get_semantic_run", + "list_semantic_candidates", "get_semantic_candidate", + "list_project_lenses", "get_project_lens", + "get_semantic_usage")) +check("EXACTLY the nine graph tools are added", + mcp_server.KNOWLEDGE_TOOL_NAMES == ( + "get_graph_stats", "search_graph", "get_graph_node", + "expand_graph", "find_graph_path", "list_engineering_entities", + "get_engineering_entity", "get_engineering_claim", + "get_engineering_relation")) +total = (len(mcp_server.TOOLS) + len(mcp_server.ASSET_TOOLS) + + len(mcp_server.DOCUMENT_TOOLS) + len(mcp_server.SEMANTIC_TOOLS) + + len(mcp_server.KNOWLEDGE_TOOLS)) +check("the complete MCP set is 35 tools", total == 35) +forbidden = {"promote_candidate", "promote_relation", "create_entity", + "create_claim", "create_relation", "merge_entities", + "split_entity", "set_authority", "seed_graph", "sync_graph", + "export_bundle"} +all_names = set(mcp_server.TOOL_NAMES + mcp_server.ASSET_TOOL_NAMES + + mcp_server.DOCUMENT_TOOL_NAMES + + mcp_server.SEMANTIC_TOOL_NAMES + + mcp_server.KNOWLEDGE_TOOL_NAMES) +check("no promote/create/merge/authority/seed/export tool exists on MCP", + not (forbidden & all_names)) + +import asyncio # noqa: E402 +server = mcp_server.create_mcp_server() +registered = asyncio.new_event_loop().run_until_complete(server.list_tools()) +check("FastMCP registers all 35", len(registered) == 35) + +# --------------------------------------------------------------------------- +# 2. REST: legacy + Phase 3/4 routes intact; knowledge routes additive +# --------------------------------------------------------------------------- +from openmind.main import app # noqa: E402 + +paths = {route.path for route in app.routes} +legacy = {"/projects", "/projects/{project_id}", + "/projects/{project_id}/assets", + "/projects/{project_id}/documents", "/ingest", "/search", "/ask", + "/glossary", "/jobs", "/jobs/{job_id}", "/api/health", + "/projects/{project_id}/knowledge/search", + "/projects/{project_id}/semantic/candidates", + "/projects/{project_id}/lenses"} +check("every pre-Phase-5 route is still registered", legacy <= paths) +check("the API still says /projects, not /workspaces", + not any("/workspaces" in p for p in paths)) +knowledge_routes = { + "/projects/{project_id}/knowledge/stats", + "/projects/{project_id}/knowledge/revisions", + "/projects/{project_id}/knowledge/revisions/{revision_number}", + "/projects/{project_id}/knowledge/decisions", + "/projects/{project_id}/knowledge/seed/plan", + "/projects/{project_id}/knowledge/seed", + "/projects/{project_id}/knowledge/sync", + "/projects/{project_id}/knowledge/reconcile", + "/projects/{project_id}/knowledge/graph-search", + "/projects/{project_id}/knowledge/nodes/{node_id}", + "/projects/{project_id}/knowledge/nodes/{node_id}/expand", + "/projects/{project_id}/knowledge/path", + "/projects/{project_id}/entities", + "/projects/{project_id}/entities/{entity_id}", + "/projects/{project_id}/entities/{entity_id}/aliases", + "/projects/{project_id}/entities/{entity_id}/merge", + "/projects/{project_id}/entities/{entity_id}/split", + "/projects/{project_id}/entities/{entity_id}/authority", + "/projects/{project_id}/claims", + "/projects/{project_id}/claims/{claim_id}", + "/projects/{project_id}/relations", + "/projects/{project_id}/relations/{relation_id}", + "/projects/{project_id}/promotions/plan", + "/projects/{project_id}/promotions", + "/projects/{project_id}/relation-promotions/plan", + "/projects/{project_id}/relation-promotions", +} +check("all additive knowledge routes are registered", + knowledge_routes <= paths) + +# --------------------------------------------------------------------------- +# 3. REST end-to-end through the TestClient +# --------------------------------------------------------------------------- +from fastapi.testclient import TestClient # noqa: E402 +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +pid = make_minimal_workspace(runtime, "kg-adapters") +evidence_id = find_evidence(pid, "REQ-NC-017") +candidate_id = make_confirmed_candidate(runtime, pid) +client = TestClient(app) + +response = client.post(f"/projects/{pid}/promotions/plan", + json={"candidate_id": candidate_id}) +check("REST promotion plan works", + response.status_code == 200 + and response.json()["plan"]["eligible"]) +response = client.post(f"/projects/{pid}/promotions", + json={"candidate_id": candidate_id, + "actor": "rest-rev", "note": "via REST"}) +check("REST promotion succeeds", + response.status_code == 200 + and response.json()["status"] == "promoted") +entity_id = response.json()["entity"]["id"] + +response = client.get(f"/projects/{pid}/knowledge/stats") +check("REST knowledge stats", response.status_code == 200 + and response.json()["entities_active"] >= 1) +response = client.get(f"/projects/{pid}/entities") +check("REST entity listing", response.status_code == 200 + and response.json()["count"] >= 1) +response = client.get(f"/projects/{pid}/entities/{entity_id}") +check("REST entity detail includes claims", + response.status_code == 200 + and response.json()["entity"]["claims"]) +claim_id = response.json()["entity"]["claims"][0]["id"] +response = client.get(f"/projects/{pid}/claims/{claim_id}") +check("REST claim detail includes evidence", + response.status_code == 200 + and response.json()["claim"]["evidence"]) + +response = client.post(f"/projects/{pid}/entities", + json={"entity_type": "business-rule", + "canonical_key": "business-rule:BR-REST", + "display_name": "BR-REST", + "evidence": [{"evidence_id": evidence_id}], + "actor": "rest", "note": "created"}) +check("REST entity creation", response.status_code == 200) +second_id = response.json()["entity"]["id"] +response = client.post(f"/projects/{pid}/relations", + json={"source_entity_id": second_id, + "target_entity_id": entity_id, + "relation_type": "refines", + "relation_state": "confirmed", + "evidence": [{"evidence_id": evidence_id}], + "actor": "rest", "note": "rel"}) +check("REST relation creation", response.status_code == 200) +relation_id = response.json()["relation"]["id"] +response = client.get(f"/projects/{pid}/relations/{relation_id}") +check("REST relation detail", response.status_code == 200) + +response = client.post(f"/projects/{pid}/knowledge/graph-search", + json={"query": "REQ-NC-017"}) +check("REST graph search finds the promoted entity", + response.status_code == 200 + and response.json()["entities"][0]["canonical_key"] + == "requirement:REQ-NC-017") +response = client.get(f"/projects/{pid}/knowledge/nodes/{entity_id}") +check("REST node lookup", response.status_code == 200 + and response.json()["node"]["nodeKind"] == "entity") +response = client.post( + f"/projects/{pid}/knowledge/nodes/{entity_id}/expand", + json={"depth": 2}) +check("REST expansion", response.status_code == 200 + and "truncated" in response.json()) +response = client.post(f"/projects/{pid}/knowledge/path", + json={"source": second_id, "target": entity_id}) +check("REST path discovery", response.status_code == 200 + and response.json()["outcome"] == "found") +response = client.get(f"/projects/{pid}/knowledge/revisions") +check("REST revision ledger", response.status_code == 200 + and response.json()["revisions"]) +response = client.get(f"/projects/{pid}/knowledge/decisions") +check("REST decision audit", response.status_code == 200 + and response.json()["decisions"]) + +# workspace scoping through REST: another workspace cannot see the entity +other = make_minimal_workspace(runtime, name="kg-adapters-other") +response = client.get(f"/projects/{other}/entities/{entity_id}") +check("REST blocks cross-workspace entity access", + response.status_code == 404) +response = client.get(f"/projects/p_missing/knowledge/stats") +check("REST unknown workspace is a typed 404", + response.status_code == 404 + and response.json()["error"]["code"] == "workspace_not_found") + +# the Phase 3 combined knowledge search is untouched beside graph-search +response = client.post(f"/projects/{pid}/knowledge/search", + json={"query": "REQ-NC-017"}) +check("Phase 3 POST /knowledge/search still answers with code+documents", + response.status_code == 200 + and "documents" in response.json()) + +# --------------------------------------------------------------------------- +# 4. MCP graph tools end-to-end (read-only) +# --------------------------------------------------------------------------- +stats = mcp_server.get_graph_stats(pid) +check("MCP get_graph_stats", stats["entities_active"] >= 1) +found = mcp_server.search_graph(pid, "REQ-NC-017") +check("MCP search_graph returns separate entity/claim sections", + found["entities"] and isinstance(found["claims"], list)) +node = mcp_server.get_graph_node(pid, entity_id) +check("MCP get_graph_node", node["node"]["nodeKind"] == "entity") +expansion = mcp_server.expand_graph(pid, entity_id, depth=1) +check("MCP expand_graph bounded", "truncated" in expansion) +path = mcp_server.find_graph_path(pid, second_id, entity_id) +check("MCP find_graph_path", path["outcome"] == "found") +entities = mcp_server.list_engineering_entities(pid) +check("MCP list_engineering_entities", entities["count"] >= 1) +detail = mcp_server.get_engineering_entity(pid, entity_id) +check("MCP get_engineering_entity", detail["entity"]["id"] == entity_id) +claim_detail = mcp_server.get_engineering_claim(pid, claim_id) +check("MCP get_engineering_claim carries evidence", + claim_detail["claim"]["evidence"]) +relation_detail = mcp_server.get_engineering_relation(pid, relation_id) +check("MCP get_engineering_relation carries state", + relation_detail["relation"]["relation_state"] == "confirmed") + +# --------------------------------------------------------------------------- +# 5. Cross-surface invariants +# --------------------------------------------------------------------------- +from openmind import artifacts # noqa: E402 + +check("artifact schema stays 1.1.0", artifacts.SCHEMA_VERSION == "1.1.0") +import openmind.skill_bridge as skill_bridge # noqa: E402 +import inspect # noqa: E402 + +bridge_source = inspect.getsource(skill_bridge) +check("skill bridge remains database-independent", + "import db" not in bridge_source + and "from . import db" not in bridge_source + and "knowledge" not in bridge_source) + +# semantic review does not promote (adapter-level restatement) +from openmind.knowledge import store as kg_store # noqa: E402 +fresh = make_confirmed_candidate(runtime, other, confirm=False) +entities_before = kg_store.count_entities(other) +client.post(f"/projects/{other}/semantic/candidates/{fresh}/review", + json={"decision": "confirm", "kind": "candidate"}) +check("REST semantic review still creates no canonical entity", + kg_store.count_entities(other) == entities_before) + +finish() diff --git a/tests/verify_knowledge_bundle.py b/tests/verify_knowledge_bundle.py new file mode 100644 index 0000000..c9c2c2c --- /dev/null +++ b/tests/verify_knowledge_bundle.py @@ -0,0 +1,172 @@ +"""Knowledge Bundle 2.0 Draft: deterministic export, manifest hashes, +referential integrity, current-only vs include-history, verifier catches +corruption, `.openmind` 1.x untouched.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +import json # noqa: E402 +import tempfile # noqa: E402 +from pathlib import Path # noqa: E402 + +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_confirmed_candidate, + make_minimal_workspace) + +from openmind import artifacts # noqa: E402 +from openmind.bundle_verify import verify_bundle # noqa: E402 +from openmind.knowledge.bundle import (BUNDLE_SCHEMA_VERSION, # noqa: E402 + export_bundle) +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) +evidence_id = find_evidence(pid, "REQ-NC-017") + +# content: one promoted candidate + one manual entity/claim/relation, one +# withdrawn claim for history-mode coverage +candidate_id = make_confirmed_candidate(runtime, pid) +promoted = knowledge.promote_candidate(pid, candidate_id, actor="r", + note="promote for bundle") +manual = knowledge.create_entity( + pid, entity_type="business-rule", canonical_key="business-rule:BR-2", + display_name="BR-2", evidence=[{"evidence_id": evidence_id}], + actor="t", note="m")["entity"] +knowledge.create_relation( + pid, source_entity_id=manual["id"], + target_entity_id=promoted["entity"]["id"], relation_type="refines", + relation_state="confirmed", evidence=[{"evidence_id": evidence_id}], + actor="t", note="r") +withdrawn = knowledge.create_claim( + pid, entity_id=manual["id"], claim_type="implementation-note", + statement="A withdrawn note for history mode.", + evidence=[{"evidence_id": evidence_id}], actor="t", note="c") +knowledge.withdraw_object(pid, kind="claim", + object_id=withdrawn["claim"]["id"], actor="t", + note="w") + +out_a = Path(tempfile.mkdtemp(prefix="om_bundle_")) / "a" +out_b = Path(tempfile.mkdtemp(prefix="om_bundle_")) / "b" +STAMP = "2026-07-22T00:00:00" +manifest = export_bundle(pid, str(out_a), current_only=True, + generated_at=STAMP) +manifest_b = export_bundle(pid, str(out_b), current_only=True, + generated_at=STAMP) + +check("bundle schema version is the draft", + manifest["bundleSchemaVersion"] == BUNDLE_SCHEMA_VERSION + == "2.0.0-draft.1") +check("manifest records workspace, revision and counts", + manifest["workspaceId"] == pid + and manifest["knowledgeRevision"] >= 3 + and manifest["counts"]["entities"] >= 2 + and manifest["counts"]["claims"] >= 1) +expected_files = {"entities.jsonl", "claims.jsonl", "relations.jsonl", + "aliases.jsonl", "bindings.jsonl", "claim-evidence.jsonl", + "relation-evidence.jsonl", "decisions.jsonl", + "knowledge-revisions.jsonl", "evidence.jsonl", + "assets.jsonl", "revisions.jsonl", "segments.jsonl", + "lenses.jsonl", "workspace.json"} +check("manifest names every contract file", + expected_files <= set(manifest["files"])) +check("schemas directory shipped inside the bundle", + (out_a / "schemas" / "entity.schema.json").is_file() + and (out_a / "schemas" / "manifest.schema.json").is_file()) + +# determinism: identical bytes across two exports with a pinned timestamp +identical = all((out_a / name).read_bytes() == (out_b / name).read_bytes() + for name in manifest["files"]) +check("export is byte-identical across runs (pinned timestamp)", + identical and (out_a / "manifest.json").read_bytes() + == (out_b / "manifest.json").read_bytes()) + +# the standalone verifier accepts it +report = verify_bundle(str(out_a)) +check("bundle verifier accepts a clean current-only export", + report.ok and not report.errors) + +# no absolute paths / secrets anywhere +blob = "".join((out_a / name).read_text(encoding="utf-8") + for name in manifest["files"]) +check("no windows drive-letter path in the bundle", ":\\\\" not in blob + and ":\\" not in blob.replace("\\\\", "")) +check("no api-key material in the bundle", "api_key" not in blob.lower() + or "api_key_env" not in blob) + +# active claims all carry evidence in-bundle +claims = [json.loads(line) for line in + (out_a / "claims.jsonl").read_text(encoding="utf-8").splitlines()] +joins = [json.loads(line) for line in + (out_a / "claim-evidence.jsonl").read_text( + encoding="utf-8").splitlines()] +by_claim = {j["claim_id"] for j in joins} +check("every exported active claim has an evidence join", + all(c["id"] in by_claim for c in claims + if c["lifecycle_status"] == "active")) +check("current-only export excludes the withdrawn claim", + all(c["id"] != withdrawn["claim"]["id"] for c in claims)) + +# include-history mode +out_h = Path(tempfile.mkdtemp(prefix="om_bundle_")) / "h" +manifest_h = export_bundle(pid, str(out_h), current_only=False, + generated_at=STAMP) +history_claims = [json.loads(line) for line in + (out_h / "claims.jsonl").read_text( + encoding="utf-8").splitlines()] +check("include-history contains the withdrawn claim", + any(c["id"] == withdrawn["claim"]["id"] + and c["lifecycle_status"] == "withdrawn" + for c in history_claims)) +check("history export passes the verifier too", + verify_bundle(str(out_h)).ok) + +# knowledge-revision cap +out_r = Path(tempfile.mkdtemp(prefix="om_bundle_")) / "r" +manifest_r = export_bundle(pid, str(out_r), current_only=False, + knowledge_revision=1, generated_at=STAMP) +capped_entities = [json.loads(line) for line in + (out_r / "entities.jsonl").read_text( + encoding="utf-8").splitlines()] +check("revision cap keeps only records created at or before N", + all(e["created_knowledge_revision"] <= 1 for e in capped_entities)) +check("revision cap is documented as a stamp filter", + any("point-in-time" in l for l in manifest_r["limitations"])) + +# -- the verifier CATCHES corruption ----------------------------------------- +import shutil # noqa: E402 + +broken = Path(tempfile.mkdtemp(prefix="om_bundle_")) / "broken" +shutil.copytree(out_a, broken) +lines = (broken / "relations.jsonl").read_text( + encoding="utf-8").splitlines() +if lines: + relation = json.loads(lines[0]) + relation["target_entity_id"] = "ent_gone" + lines[0] = json.dumps(relation, ensure_ascii=False, sort_keys=True, + separators=(",", ":")) + (broken / "relations.jsonl").write_text("\n".join(lines) + "\n", + encoding="utf-8") +report = verify_bundle(str(broken)) +check("verifier catches a missing relation endpoint", + not report.ok + and any("missing entity" in e for e in report.errors)) +check("verifier catches the hash mismatch of the edited file", + any("sha256 mismatch" in e for e in report.errors)) + +broken2 = Path(tempfile.mkdtemp(prefix="om_bundle_")) / "broken2" +shutil.copytree(out_a, broken2) +(broken2 / "evidence.jsonl").write_text("", encoding="utf-8") +report2 = verify_bundle(str(broken2)) +check("verifier catches missing evidence rows", + not report2.ok + and any("missing evidence" in e for e in report2.errors)) + +# -- `.openmind` 1.x remains unchanged --------------------------------------- +check(".openmind artifact schema is still 1.1.0", + artifacts.SCHEMA_VERSION == "1.1.0") + +finish() diff --git a/tests/verify_knowledge_claims.py b/tests/verify_knowledge_claims.py new file mode 100644 index 0000000..8ea0a32 --- /dev/null +++ b/tests/verify_knowledge_claims.py @@ -0,0 +1,149 @@ +"""Canonical Claims: evidence requirement, quote verification, dedup, +correction-by-supersede (never rewrite), explicit authority, lifecycle +queryability.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_minimal_workspace) + +from openmind.knowledge import store # noqa: E402 +from openmind.knowledge.errors import (GraphEvidenceInvalid, # noqa: E402 + UnknownVocabularyValue) +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) +evidence_id = find_evidence(pid, "REQ-NC-017") + +entity = knowledge.create_entity( + pid, entity_type="requirement", canonical_key="requirement:REQ-NC-017", + display_name="REQ-NC-017", evidence=[{"evidence_id": evidence_id}], + actor="tester", note="fixture")["entity"] + +# -- evidence rules ---------------------------------------------------------- +try: + knowledge.create_claim(pid, entity_id=entity["id"], + claim_type="normative-statement", + statement="The service shall answer quickly.", + evidence=[], actor="tester", note="none") + rejected = False +except GraphEvidenceInvalid: + rejected = True +check("claim without evidence is rejected", rejected) + +try: + knowledge.create_claim( + pid, entity_id=entity["id"], claim_type="normative-statement", + statement="The service shall answer quickly.", + evidence=[{"evidence_id": evidence_id, + "quote": "a quote that is not in the snapshot"}], + actor="tester", note="fabricated") + fabricated = False +except GraphEvidenceInvalid: + fabricated = True +check("fabricated quote is rejected against the immutable snapshot", + fabricated) + +try: + knowledge.create_claim(pid, entity_id=entity["id"], + claim_type="definitely-not-a-type", + statement="x", + evidence=[{"evidence_id": evidence_id}], + actor="tester", note="bad type") + bad_type = False +except UnknownVocabularyValue: + bad_type = True +check("invalid claim type rejected", bad_type) + +created = knowledge.create_claim( + pid, entity_id=entity["id"], claim_type="normative-statement", + statement="The name check service shall answer within 2 seconds.", + evidence=[{"evidence_id": evidence_id, + "quote": "shall answer within 2 seconds"}], + actor="tester", note="valid claim") +claim = created["claim"] +check("valid claim created with verified evidence join", + len(claim["evidence"]) == 1 + and claim["evidence"][0]["evidence_id"] == evidence_id) +check("claim carries a normalized statement hash", + len(claim["normalized_statement_hash"]) == 64) + +# -- dedup ------------------------------------------------------------------- +duplicate = knowledge.create_claim( + pid, entity_id=entity["id"], claim_type="normative-statement", + statement="The NAME CHECK service shall answer within 2 seconds.", + evidence=[{"evidence_id": evidence_id}], actor="tester", note="dup") +check("identical normalized claim deduplicates to the existing row", + duplicate["deduplicated"] and duplicate["claim"]["id"] == claim["id"]) +check("dedup minted no new knowledge revision", + duplicate["knowledge_revision"] == created["knowledge_revision"]) + +# -- correction: new claim + supersede, never a rewrite ---------------------- +corrected = knowledge.create_claim( + pid, entity_id=entity["id"], claim_type="normative-statement", + statement="The name check service shall answer within 3 seconds.", + evidence=[{"evidence_id": evidence_id}], actor="tester", + note="corrected budget") +check("changed statement creates a NEW claim", + corrected["claim"]["id"] != claim["id"]) +superseded = knowledge.supersede_object( + pid, kind="claim", object_id=claim["id"], + replacement_id=corrected["claim"]["id"], actor="tester", + note="3s supersedes 2s") +old = store.get_claim(pid, claim["id"]) +check("old claim is superseded, not overwritten", + old["lifecycle_status"] == "superseded" + and old["superseded_by_claim_id"] == corrected["claim"]["id"] + and old["statement"].endswith("2 seconds.")) +check("superseded claim remains queryable with its evidence", + len(old["evidence"]) == 1) +check("supersede minted its own knowledge revision", + superseded["knowledge_revision"] > corrected["knowledge_revision"]) + +# -- authority is explicit --------------------------------------------------- +check("new claim starts with authority unknown", + corrected["claim"]["authority_status"] == "unknown") +marked = knowledge.set_authority(pid, kind="claim", + object_id=corrected["claim"]["id"], + authority="authoritative", actor="lead", + note="approved by review board") +check("authority change is explicit and recorded", + store.get_claim(pid, corrected["claim"]["id"])["authority_status"] + == "authoritative") +authority_decisions = knowledge.list_decisions( + pid, decision_type="mark-authoritative")["decisions"] +check("authority change recorded a decision", + any(d["target_id"] == corrected["claim"]["id"] + for d in authority_decisions)) + +# -- lifecycle queryability -------------------------------------------------- +active = knowledge.list_claims(pid, entity_id=entity["id"]) +check("active listing excludes the superseded claim", + all(c["id"] != claim["id"] for c in active["claims"])) +history = knowledge.list_claims(pid, entity_id=entity["id"], + lifecycle_status="superseded") +check("superseded claim listed under its lifecycle", + any(c["id"] == claim["id"] for c in history["claims"])) + +withdrawn = knowledge.create_claim( + pid, entity_id=entity["id"], claim_type="implementation-note", + statement="A note that will be withdrawn.", + evidence=[{"evidence_id": evidence_id}], actor="tester", note="w") +knowledge.withdraw_object(pid, kind="claim", + object_id=withdrawn["claim"]["id"], + actor="tester", note="withdrawn in test") +check("withdrawn claim excluded from active queries", + all(c["id"] != withdrawn["claim"]["id"] + for c in knowledge.list_claims(pid, + entity_id=entity["id"])["claims"])) +check("withdrawn claim remains queryable by id", + store.get_claim(pid, withdrawn["claim"]["id"])["lifecycle_status"] + == "withdrawn") + +finish() diff --git a/tests/verify_knowledge_cli.py b/tests/verify_knowledge_cli.py new file mode 100644 index 0000000..700c81c --- /dev/null +++ b/tests/verify_knowledge_cli.py @@ -0,0 +1,234 @@ +"""Knowledge-graph CLI contract — one JSON object on stdout, stderr +diagnostics, stable exit codes, no ANSI, and the graph/promotion/entity/ +claim/relation/knowledge-history/bundle command surface end-to-end.""" +import contextlib +import io +import json +import os +import sys +import tempfile + +os.environ.setdefault("OPENMIND_DATA_DIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_confirmed_candidate, + make_minimal_workspace) + +from openmind import cli # noqa: E402 +from openmind.runtime import get_runtime # noqa: E402 + + +def run_cli(*argv): + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), \ + contextlib.redirect_stderr(stderr): + try: + code = cli.main(list(argv)) + except SystemExit as exc: + code = int(exc.code or 0) + return code, stdout.getvalue(), stderr.getvalue() + + +runtime = get_runtime() +pid = make_minimal_workspace(runtime, "kg-cli") +evidence_id = find_evidence(pid, "REQ-NC-017") +candidate_id = make_confirmed_candidate(runtime, pid) + +# -- graph seed / stats / sync ---------------------------------------------- +code, out, err = run_cli("graph", "seed", "plan", "--workspace", pid, + "--json") +payload = json.loads(out) +check("graph seed plan emits one JSON object", code == 0 and payload["ok"]) +check("no ANSI escapes in JSON mode", "\x1b[" not in out) + +code, out, _ = run_cli("graph", "seed", "--workspace", pid, "--json") +check("graph seed succeeds", code == 0 and json.loads(out)["ok"]) + +code, out, _ = run_cli("graph", "sync", "--workspace", pid, "--json") +check("unchanged graph sync reports noop", + code == 0 and json.loads(out)["action"] == "noop") + +code, out, _ = run_cli("graph", "stats", "--workspace", pid, "--json") +stats = json.loads(out) +check("graph stats reports the knowledge revision", + code == 0 and "knowledge_revision" in stats) + +code, out, _ = run_cli("graph", "reconcile", "--workspace", pid, "--json") +check("graph reconcile runs", code == 0 and json.loads(out)["ok"]) + +# -- promotion --------------------------------------------------------------- +code, out, _ = run_cli("promotion", "plan", "--workspace", pid, + "--candidate", candidate_id, "--json") +plan = json.loads(out)["plan"] +check("promotion plan reports eligibility", code == 0 and plan["eligible"]) + +code, out, _ = run_cli("promotion", "promote", "--workspace", pid, + "--candidate", candidate_id, "--actor", "cli-rev", + "--note", "approved via cli", "--json") +promoted = json.loads(out) +check("promotion promote succeeds with entity + claim", + code == 0 and promoted["status"] == "promoted" + and promoted["entity"]["canonical_key"] == "requirement:REQ-NC-017") +entity_id = promoted["entity"]["id"] + +code, out, _ = run_cli("promotion", "promote", "--workspace", pid, + "--candidate", candidate_id, "--actor", "cli-rev", + "--note", "again", "--json") +check("repeated promotion is idempotent over the CLI", + code == 0 and json.loads(out)["status"] == "already-promoted") + +# actor/note are required flags on promote +code, out, err = run_cli("promotion", "promote", "--workspace", pid, + "--candidate", candidate_id, "--json") +check("promote without --actor/--note is a usage error (exit 2)", + code == 2) + +# -- entity / claim / relation ---------------------------------------------- +code, out, _ = run_cli("entity", "create", "--workspace", pid, "--type", + "business-rule", "--key", "business-rule:BR-CLI", + "--name", "BR-CLI", "--evidence", evidence_id, + "--actor", "cli", "--note", "created", "--json") +created = json.loads(out) +check("entity create over the CLI", code == 0 + and created["entity"]["origin"] == "manual") +second_id = created["entity"]["id"] + +code, out, _ = run_cli("entity", "list", "--workspace", pid, "--json") +listing = json.loads(out) +check("entity list is bounded JSON with totals", + code == 0 and listing["count"] >= 2 and "total" in listing) + +code, out, _ = run_cli("entity", "show", "--workspace", pid, "--entity", + entity_id, "--json") +check("entity show includes aliases/bindings/claims", + code == 0 and "aliases" in json.loads(out)["entity"]) + +code, out, _ = run_cli("entity", "alias-add", "--workspace", pid, + "--entity", second_id, "--alias", "cli-alias", + "--type", "name", "--actor", "cli", "--note", "a", + "--json") +check("entity alias-add works", code == 0) + +code, out, _ = run_cli("claim", "create", "--workspace", pid, "--entity", + second_id, "--type", "normative-statement", + "--statement", "BR-CLI governs the retry budget.", + "--evidence", evidence_id, "--actor", "cli", + "--note", "claim", "--json") +claim = json.loads(out)["claim"] +check("claim create over the CLI", code == 0 + and claim["claim_type"] == "normative-statement") + +code, out, _ = run_cli("claim", "list", "--workspace", pid, "--json") +check("claim list works", code == 0 and json.loads(out)["count"] >= 1) + +code, out, _ = run_cli("relation", "create", "--workspace", pid, + "--source", second_id, "--target", entity_id, + "--type", "refines", "--state", "confirmed", + "--evidence", evidence_id, "--actor", "cli", + "--note", "rel", "--json") +relation = json.loads(out)["relation"] +check("relation create over the CLI", code == 0 + and relation["relation_state"] == "confirmed") + +code, out, _ = run_cli("relation", "reject", "--workspace", pid, + "--relation", relation["id"], "--actor", "cli", + "--note", "not real", "--json") +check("relation reject works", code == 0) +code, out, _ = run_cli("relation", "restore", "--workspace", pid, + "--relation", relation["id"], "--actor", "cli", + "--note", "was real", "--json") +check("relation restore returns the prior state", + code == 0 and json.loads(out)["relation_state"] == "confirmed") + +code, out, _ = run_cli("entity", "authority", "--workspace", pid, + "--entity", entity_id, "--status", "authoritative", + "--actor", "lead", "--note", "board", "--json") +check("entity authority marking works", + code == 0 and json.loads(out)["authority_status"] == "authoritative") + +# -- graph queries ------------------------------------------------------------ +code, out, _ = run_cli("graph", "search", "--workspace", pid, "--query", + "REQ-NC-017", "--json") +result = json.loads(out) +check("graph search returns entity hits", + code == 0 and result["entities"] + and result["entities"][0]["canonical_key"] + == "requirement:REQ-NC-017") + +code, out, _ = run_cli("graph", "node", "--workspace", pid, "--node", + entity_id, "--json") +check("graph node lookup works", + code == 0 and json.loads(out)["node"]["nodeKind"] == "entity") + +code, out, _ = run_cli("graph", "expand", "--workspace", pid, "--node", + entity_id, "--depth", "2", "--json") +check("graph expand is bounded JSON", + code == 0 and "truncated" in json.loads(out)) + +code, out, _ = run_cli("graph", "path", "--workspace", pid, "--from", + second_id, "--to", entity_id, "--json") +check("graph path finds the created relation", + code == 0 and json.loads(out)["outcome"] == "found") + +# -- knowledge history -------------------------------------------------------- +code, out, _ = run_cli("knowledge", "revisions", "--workspace", pid, + "--json") +revisions = json.loads(out) +check("knowledge revisions lists the ledger", + code == 0 and revisions["revisions"]) +number = revisions["revisions"][-1]["revision_number"] +code, out, _ = run_cli("knowledge", "revision", "--workspace", pid, + "--number", str(number), "--json") +check("knowledge revision shows one entry with decisions", + code == 0 and "decisions" in json.loads(out)["revision"]) +code, out, _ = run_cli("knowledge", "decisions", "--workspace", pid, + "--json") +check("knowledge decisions lists the audit", + code == 0 and json.loads(out)["decisions"]) +code, out, _ = run_cli("knowledge", "search", "--workspace", pid, + "--query", "REQ-NC-017", "--json") +check("the Phase 3 knowledge search command is untouched", + code == 0 and "code" in json.loads(out) or "documents" + in json.loads(out)) + +# -- bundle ------------------------------------------------------------------- +out_dir = os.path.join(tempfile.mkdtemp(prefix="om_cli_bundle_"), "b") +code, out, _ = run_cli("bundle", "export", "--workspace", pid, "--output", + out_dir, "--current-only", "--generated-at", + "2026-07-22T00:00:00", "--json") +manifest = json.loads(out)["manifest"] +check("bundle export over the CLI", + code == 0 and manifest["bundleSchemaVersion"] == "2.0.0-draft.1") +from openmind.bundle_verify import main as verify_main # noqa: E402 +stdout = io.StringIO() +with contextlib.redirect_stdout(stdout), \ + contextlib.redirect_stderr(io.StringIO()): + verify_code = verify_main([out_dir]) +check("bundle_verify accepts the CLI export", verify_code == 0) + +# -- error contract ----------------------------------------------------------- +code, out, _ = run_cli("graph", "stats", "--workspace", "p_missing", + "--json") +payload = json.loads(out) +check("unknown workspace: exit 1 with a machine-readable error", + code == 1 and payload["ok"] is False + and payload["error"]["code"] == "workspace_not_found") + +code, out, _ = run_cli("entity", "create", "--workspace", pid, "--type", + "nonsense-type", "--key", "x:y", "--name", "x", + "--evidence", evidence_id, "--actor", "a", + "--note", "n", "--json") +payload = json.loads(out) +check("invalid vocabulary: exit 2 with the typed code", + code == 2 and payload["error"]["code"] == "unknown_vocabulary_value") + +code, out, _ = run_cli("promotion", "promote", "--workspace", pid, + "--candidate", "sc_missing", "--actor", "a", + "--note", "n", "--json") +check("missing candidate: exit 1 with candidate_not_found", + code == 1 and json.loads(out)["error"]["code"] + == "candidate_not_found") + +finish() diff --git a/tests/verify_knowledge_decisions.py b/tests/verify_knowledge_decisions.py new file mode 100644 index 0000000..c0d2e28 --- /dev/null +++ b/tests/verify_knowledge_decisions.py @@ -0,0 +1,115 @@ +"""Human Decision audit: every governance write records one, actor is +caller-supplied (never inferred), notes and snapshots bounded, decisions +immutable, no secret material stored.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +import json # noqa: E402 + +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_minimal_workspace) + +from openmind import db # noqa: E402 +from openmind.domain.errors import InvalidRequest # noqa: E402 +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) +evidence_id = find_evidence(pid, "REQ-NC-017") + +entity = knowledge.create_entity( + pid, entity_type="requirement", canonical_key="requirement:REQ-NC-017", + display_name="REQ-NC-017", evidence=[{"evidence_id": evidence_id}], + actor="alice-reviewer", note="created in decision test")["entity"] +claim = knowledge.create_claim( + pid, entity_id=entity["id"], claim_type="normative-statement", + statement="The name check service shall answer within 2 seconds.", + evidence=[{"evidence_id": evidence_id}], actor="", note="empty actor") +knowledge.add_alias(pid, entity_id=entity["id"], alias="NC-Requirement", + alias_type="name", actor="alice-reviewer", note="alias") +knowledge.set_authority(pid, kind="entity", object_id=entity["id"], + authority="authoritative", actor="lead", + note="board approval") +knowledge.withdraw_object(pid, kind="claim", + object_id=claim["claim"]["id"], + actor="alice-reviewer", note="withdrawn") + +decisions = knowledge.list_decisions(pid)["decisions"] +types = [d["decision_type"] for d in decisions] +check("every governance write recorded a decision", + {"create-entity", "create-claim", "add-alias", "mark-authoritative", + "withdraw"} <= set(types)) +check("one decision per mutation (5 writes, 5 decisions)", + len(decisions) == 5) +check("decisions link to their knowledge revision", + all(d["knowledge_revision_id"] for d in decisions)) + +by_type = {d["decision_type"]: d for d in decisions} +check("actor recorded verbatim, never inferred", + by_type["create-entity"]["actor"] == "alice-reviewer") +check("empty actor stays empty (identity is never invented)", + by_type["create-claim"]["actor"] == "") +check("before/after snapshots are structured and bounded", + isinstance(by_type["mark-authoritative"]["before"], dict) + and by_type["mark-authoritative"]["before"]["authority_status"] + == "unknown" + and by_type["mark-authoritative"]["after"]["authority_status"] + == "authoritative") +check("decision records the invoking command channel", + all("source_command" in d for d in decisions)) + +# note bound +try: + knowledge.set_authority(pid, kind="entity", object_id=entity["id"], + authority="informational", actor="x", + note="n" * 5000) + bounded = False +except InvalidRequest: + bounded = True +check("oversized note rejected before any write", bounded) + +# snapshots bounded: entity snapshot carries graph fields only +snapshot = by_type["create-entity"]["after"] +check("entity snapshot is field-bounded (no free payload dump)", + set(snapshot) <= {"id", "entity_type", "canonical_key", + "display_name", "lifecycle_status", + "authority_status", "origin", + "merged_into_entity_id", + "superseded_by_entity_id"}) + +# no secret material in any stored decision JSON +conn, lock = db.shared_connection() +with lock: + rows = conn.execute("SELECT before_json, after_json, note FROM " + "knowledge_decisions").fetchall() +blob = json.dumps([tuple(r) for r in rows]).lower() +check("no credential material in stored decisions", + "api_key" not in blob and "authorization" not in blob + and "bearer " not in blob) + +# immutability: a later flurry of writes leaves earlier rows byte-identical +with lock: + before_rows = {r["id"]: tuple(r) for r in conn.execute( + "SELECT * FROM knowledge_decisions").fetchall()} +knowledge.set_authority(pid, kind="entity", object_id=entity["id"], + authority="informational", actor="x", note="later") +with lock: + after_rows = {r["id"]: tuple(r) for r in conn.execute( + "SELECT * FROM knowledge_decisions").fetchall()} +check("earlier decisions are immutable across later writes", + all(after_rows[i] == before_rows[i] for i in before_rows)) +check("the later write added its own decision", + len(after_rows) == len(before_rows) + 1) + +# filtered listing +filtered = knowledge.list_decisions(pid, target_kind="entity", + target_id=entity["id"])["decisions"] +check("decision listing filters by target", + filtered and all(d["target_id"] == entity["id"] for d in filtered)) + +finish() diff --git a/tests/verify_knowledge_entities.py b/tests/verify_knowledge_entities.py new file mode 100644 index 0000000..c0b6a1f --- /dev/null +++ b/tests/verify_knowledge_entities.py @@ -0,0 +1,192 @@ +"""Canonical Entities: deterministic keys, identity uniqueness, workspace +scoping, closed vocabulary, evidence requirement for manual creation, +aliases with reported collisions, bindings.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_minimal_workspace) + +from openmind.domain.errors import InvalidRequest # noqa: E402 +from openmind.knowledge import identity, store # noqa: E402 +from openmind.knowledge.errors import (AliasCollision, # noqa: E402 + EntityNotFound, GraphConflict, + GraphEvidenceInvalid, + UnknownVocabularyValue) +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) +other_pid = runtime.workspaces.create("kg-other")["id"] +evidence_id = find_evidence(pid, "REQ-NC-017") + +# -- deterministic canonical keys ------------------------------------------- +check("identifier key preserves identifier case", + identity.identifier_entity_key("requirement", "REQ-NC-017") + == "requirement:REQ-NC-017") +check("derived key is normalized and marked derived", + identity.derived_entity_key("workflow", "Nightly Batch Run") + == "workflow:derived:nightly-batch-run") +check("derived key is deterministic", + identity.derived_entity_key("workflow", "Nightly Batch Run") + == identity.derived_entity_key("workflow", "Nightly Batch Run")) +check("interface key upper-cases the method", + identity.interface_entity_key("post", "/name-check") + == "interface:POST:/name-check") +check("symbol key preserves symbol case", + identity.symbol_entity_key("a_1", "org.example.NameCheckService#run()") + == "code-symbol:asset:a_1:org.example.NameCheckService#run()") +check("looks_like_identifier accepts REQ-NC-017", + identity.looks_like_identifier("REQ-NC-017")) +check("looks_like_identifier rejects a prose title", + not identity.looks_like_identifier("The service shall respond")) + +# -- manual creation --------------------------------------------------------- +try: + knowledge.create_entity(pid, entity_type="requirement", + canonical_key="requirement:REQ-NC-017", + display_name="REQ-NC-017", evidence=[], + actor="tester", note="no evidence") + no_evidence_rejected = False +except GraphEvidenceInvalid: + no_evidence_rejected = True +check("manual entity without evidence is rejected", no_evidence_rejected) + +try: + knowledge.create_entity(pid, entity_type="not-a-type", + canonical_key="x:y", display_name="x", + evidence=[{"evidence_id": evidence_id}], + actor="tester", note="bad type") + bad_type_rejected = False +except UnknownVocabularyValue: + bad_type_rejected = True +check("invalid entity type is rejected at the write boundary", + bad_type_rejected) + +created = knowledge.create_entity( + pid, entity_type="requirement", canonical_key="requirement:REQ-NC-017", + display_name="REQ-NC-017", + evidence=[{"evidence_id": evidence_id, + "quote": "shall answer within 2 seconds"}], + actor="tester", note="manual create") +entity = created["entity"] +check("manual entity created with origin=manual", + entity["origin"] == "manual" + and entity["canonical_key"] == "requirement:REQ-NC-017") +check("manual entity creation minted a knowledge revision", + created["knowledge_revision"] >= 1) +check("evidence anchored as a binding", + any(b["ref_kind"] == "evidence" and b["ref_id"] == evidence_id + for b in store.list_bindings(pid, entity["id"]))) + +try: + knowledge.create_entity( + pid, entity_type="requirement", + canonical_key="requirement:REQ-NC-017", display_name="dup", + evidence=[{"evidence_id": evidence_id}], actor="tester", note="dup") + duplicate_rejected = False +except GraphConflict: + duplicate_rejected = True +check("duplicate canonical key is a reported conflict, not a silent row", + duplicate_rejected) +check("identity uniqueness held (one entity for the key)", + len([e for e in store.list_entities(pid, lifecycle_status=None, + limit=100) + if e["canonical_key"] == "requirement:REQ-NC-017"]) == 1) + +# fabricated evidence quote is rejected +try: + knowledge.create_entity( + pid, entity_type="constraint", canonical_key="constraint:C-1", + display_name="C-1", + evidence=[{"evidence_id": evidence_id, + "quote": "this text does not exist in the evidence"}], + actor="tester", note="fabricated quote") + fabricated_rejected = False +except GraphEvidenceInvalid: + fabricated_rejected = True +check("fabricated evidence quote is rejected", fabricated_rejected) + +# evidence from another workspace is rejected +try: + knowledge.create_entity( + other_pid, entity_type="requirement", + canonical_key="requirement:REQ-X", display_name="REQ-X", + evidence=[{"evidence_id": evidence_id}], actor="tester", + note="cross-workspace evidence") + cross_evidence_rejected = False +except GraphEvidenceInvalid: + cross_evidence_rejected = True +check("evidence of another workspace is rejected", cross_evidence_rejected) + +# -- workspace scoping ------------------------------------------------------- +check("entity does not resolve through another workspace", + store.get_entity(other_pid, entity["id"]) is None) +try: + knowledge.get_entity(other_pid, entity["id"]) + cross_blocked = False +except EntityNotFound: + cross_blocked = True +check("service read through another workspace is typed not-found", + cross_blocked) + +# -- aliases ----------------------------------------------------------------- +added = knowledge.add_alias(pid, entity_id=entity["id"], + alias="Name Check Requirement", + alias_type="name", actor="tester", note="alias") +check("alias added with normalized form", + added["alias"]["normalized_alias"] == "name check requirement") +check("alias is indexed for exact lookup", + [e["id"] for e in store.find_entity_by_alias( + pid, "name check requirement")] == [entity["id"]]) + +second = knowledge.create_entity( + pid, entity_type="business-rule", canonical_key="business-rule:BR-1", + display_name="BR-1", evidence=[{"evidence_id": evidence_id}], + actor="tester", note="second entity") +try: + knowledge.add_alias(pid, entity_id=second["entity"]["id"], + alias="NAME CHECK requirement", alias_type="name", + actor="tester", note="collide") + collision_reported = False +except AliasCollision as exc: + collision_reported = bool(exc.holders) +check("normalized alias collision across entities is reported, never " + "silently attached", collision_reported) +check("colliding alias did not attach", + not any(a["alias"] == "NAME CHECK requirement" + for a in store.list_aliases(pid, second["entity"]["id"]))) + +removed = knowledge.remove_alias(pid, entity_id=entity["id"], + alias="Name Check Requirement", + actor="tester", note="remove") +check("alias removal reports the count", removed["removed"] == 1) +check("removed alias stays auditable with status=removed", + any(a["status"] == "removed" for a in store.list_aliases( + pid, entity["id"], status=None))) +check("removed alias no longer resolves", + store.find_entity_by_alias(pid, "name check requirement") == []) + +# same alias on the SAME entity deduplicates instead of colliding +knowledge.add_alias(pid, entity_id=entity["id"], alias="ReqNC17", + alias_type="acronym", actor="tester", note="a") +again = knowledge.add_alias(pid, entity_id=entity["id"], alias="reqnc17", + alias_type="acronym", actor="tester", note="b") +check("re-adding an alias to the same entity deduplicates", + again.get("deduplicated") is True) + +# note bound enforced +try: + knowledge.add_alias(pid, entity_id=entity["id"], alias="overlong", + alias_type="name", actor="t", note="x" * 3000) + note_bounded = False +except InvalidRequest: + note_bounded = True +check("governance note is bounded", note_bounded) + +finish() diff --git a/tests/verify_knowledge_graph.py b/tests/verify_knowledge_graph.py new file mode 100644 index 0000000..72ad6b0 --- /dev/null +++ b/tests/verify_knowledge_graph.py @@ -0,0 +1,173 @@ +"""Graph queries: node lookup for every kind, bounded deterministic +expansion, shortest paths (found / no-path / truncated), evidence summaries, +subgraph export.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_minimal_workspace) + +from openmind import db # noqa: E402 +from openmind.knowledge import graph, store # noqa: E402 +from openmind.knowledge.errors import (GraphLimitExceeded, # noqa: E402 + GraphNodeNotFound) +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) +evidence_id = find_evidence(pid, "REQ-NC-017") + + +def entity(key): + return knowledge.create_entity( + pid, entity_type="requirement", canonical_key=f"requirement:{key}", + display_name=key, evidence=[{"evidence_id": evidence_id}], + actor="t", note="fixture")["entity"] + + +def relate(source, target, relation_type="refines"): + return knowledge.create_relation( + pid, source_entity_id=source["id"], target_entity_id=target["id"], + relation_type=relation_type, relation_state="confirmed", + evidence=[{"evidence_id": evidence_id, + "quote": "shall answer within 2 seconds"}], + actor="t", note="fixture")["relation"] + + +# a small chain plus a fork: A -> B -> C -> D, A -> E, F isolated +a, b, c, d, e, f = (entity(k) for k in ("A", "B", "C", "D", "E", "F")) +relate(a, b) +relate(b, c) +relate(c, d) +relate(a, e, relation_type="affected-by") +claim = knowledge.create_claim( + pid, entity_id=a["id"], claim_type="normative-statement", + statement="A drives everything.", + evidence=[{"evidence_id": evidence_id}], actor="t", note="c")["claim"] + +# -- node lookup for every kind ---------------------------------------------- +node = knowledge.get_node(pid, a["id"]) +check("entity node has the stable camelCase shape", + node["nodeKind"] == "entity" + and node["canonicalKey"] == "requirement:A" + and "lifecycleStatus" in node and "authorityStatus" in node + and node["claimCount"] == 1 and node["relationCount"] == 2 + and isinstance(node["bindings"], list)) +check("claim node resolves", + knowledge.get_node(pid, claim["id"])["nodeKind"] == "claim") +asset = db.list_assets(pid, state="active", limit=5)[0] +check("asset node projected from the canonical row", + knowledge.get_node(pid, asset["id"])["nodeKind"] == "asset") +revision_node = knowledge.get_node(pid, asset["current_revision_id"]) +check("revision node projected", revision_node["nodeKind"] == "revision") +segment = db.list_segments(pid, asset["current_revision_id"], limit=5)[0] +check("segment node projected", + knowledge.get_node(pid, segment["id"])["nodeKind"] == "segment") +check("evidence node projected", + knowledge.get_node(pid, evidence_id)["nodeKind"] == "evidence") +try: + knowledge.get_node(pid, "ent_nope") + missing = False +except GraphNodeNotFound: + missing = True +check("unknown node id is a typed not-found", missing) + +# -- bounded expansion -------------------------------------------------------- +expansion = knowledge.expand_node(pid, a["id"], depth=1) +ids_at_depth_one = {n["id"] for n in expansion["nodes"]} +check("depth-1 expansion reaches only direct neighbours", + ids_at_depth_one == {a["id"], b["id"], e["id"]}) +expansion = knowledge.expand_node(pid, a["id"], depth=3) +check("depth-3 expansion reaches the chain but not the isolate", + {n["id"] for n in expansion["nodes"]} + == {a["id"], b["id"], c["id"], d["id"], e["id"]}) +check("expansion reports limits and revision", + expansion["limits"]["hard_depth"] == 4 + and expansion["knowledge_revision"] + == store.current_revision_number(pid)) +second = knowledge.expand_node(pid, a["id"], depth=3) +check("expansion is deterministic (identical repeat)", + expansion["nodes"] == second["nodes"] + and expansion["edges"] == second["edges"]) +check("depth is clamped to the hard cap", + knowledge.expand_node(pid, a["id"], depth=99)["limits"]["depth"] == 4) +truncated = knowledge.expand_node(pid, a["id"], depth=3, node_limit=2) +check("node limit truncates and says so", + truncated["truncated"] and len(truncated["nodes"]) <= 2) +filtered = knowledge.expand_node(pid, a["id"], depth=1, + relation_types=["refines"]) +check("relation-type filter narrows the expansion", + {n["id"] for n in filtered["nodes"]} == {a["id"], b["id"]}) +try: + knowledge.expand_node(pid, a["id"], relation_types=["depends-on"]) + bad_type = False +except Exception: + bad_type = True +check("unknown relation type in expansion is rejected", bad_type) + +# -- path discovery ----------------------------------------------------------- +path = knowledge.find_path(pid, a["id"], d["id"]) +check("shortest path found with honest outcome", + path["outcome"] == "found" and path["paths"][0]["length"] == 3 + and path["paths"][0]["entities"] + == [a["id"], b["id"], c["id"], d["id"]]) +check("path edges carry evidence summaries", + all(edge["evidence"] for edge in path["paths"][0]["edges"])) +no_path = knowledge.find_path(pid, a["id"], f["id"]) +check("unreachable target is an honest no-path (never invented)", + no_path["outcome"] == "no-path" and no_path["paths"] == []) +short = knowledge.find_path(pid, a["id"], d["id"], max_depth=2) +check("a too-short depth bound reports no-path/truncated, not a fake path", + short["outcome"] in ("no-path", "truncated") and short["paths"] == []) +directed = knowledge.find_path(pid, d["id"], a["id"], + direction="outgoing") +check("direction policy is honoured (no reverse path outgoing-only)", + directed["outcome"] in ("no-path", "truncated")) +both = knowledge.find_path(pid, d["id"], a["id"], direction="both") +check("undirected policy finds the reverse traversal", + both["outcome"] == "found") + +# truncation disclosure with a tiny visited budget +original_cap = graph.MAX_PATH_VISITED +graph.MAX_PATH_VISITED = 2 +try: + tiny = knowledge.find_path(pid, a["id"], d["id"]) +finally: + graph.MAX_PATH_VISITED = original_cap +check("visited-node truncation is disclosed", + tiny["outcome"] == "truncated" and tiny["truncated"]) + +# stale exclusion in traversal +knowledge.withdraw_object(pid, kind="entity", object_id=c["id"], + actor="t", note="w") +# withdrawing C stales the relations touching it at next reconcile; the +# traversal itself must already exclude non-active endpoints' edges +blocked = knowledge.find_path(pid, a["id"], d["id"]) +check("a withdrawn intermediate breaks the active path", + blocked["outcome"] in ("no-path", "truncated")) +with_stale = knowledge.find_path(pid, a["id"], d["id"], include_stale=True) +check("include_stale traverses the historical edge again", + with_stale["outcome"] == "found") + +# -- subgraph ----------------------------------------------------------------- +sub = knowledge.get_subgraph(pid, [a["id"], d["id"]], depth=1) +check("subgraph unions the seed neighbourhoods", + {a["id"], b["id"], e["id"], d["id"]} + <= {n["id"] for n in sub["nodes"]}) +check("subgraph keeps only edges with both endpoints present", + all(any(n["id"] == edge["sourceEntityId"] for n in sub["nodes"]) + and any(n["id"] == edge["targetEntityId"] for n in sub["nodes"]) + for edge in sub["edges"])) +try: + knowledge.get_subgraph(pid, []) + empty_ok = False +except GraphLimitExceeded: + empty_ok = True +check("subgraph requires at least one seed", empty_ok) + +finish() diff --git a/tests/verify_knowledge_merge_split.py b/tests/verify_knowledge_merge_split.py new file mode 100644 index 0000000..900f4d7 --- /dev/null +++ b/tests/verify_knowledge_merge_split.py @@ -0,0 +1,255 @@ +"""Entity merge and split: transactional, auditable, nothing deleted, +duplicates deduplicated, self-relations eliminated, invalid input aborts +whole.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_minimal_workspace) + +from openmind.knowledge import store # noqa: E402 +from openmind.knowledge.errors import GraphConflict # noqa: E402 +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) +evidence_id = find_evidence(pid, "REQ-NC-017") +evidence_two = find_evidence(pid, "retried three times") + + +def entity(key, entity_type="requirement"): + return knowledge.create_entity( + pid, entity_type=entity_type, canonical_key=f"{entity_type}:{key}", + display_name=key, evidence=[{"evidence_id": evidence_id}], + actor="t", note="fixture")["entity"] + + +# -- merge scenario ---------------------------------------------------------- +source = entity("REQ-DUP-A") +target = entity("REQ-NC-017") +third = entity("REQ-NC-018") + +knowledge.add_alias(pid, entity_id=source["id"], alias="duplicate-req", + alias_type="name", actor="t", note="a") +# an alias the TARGET also holds -> superseded on move, not duplicated +knowledge.add_alias(pid, entity_id=source["id"], alias="shared-name", + alias_type="name", actor="t", note="a") +# a colliding alias held by a THIRD entity -> stays on source, reported +knowledge.add_alias(pid, entity_id=third["id"], alias="third-name", + alias_type="name", actor="t", note="a") + +source_claim = knowledge.create_claim( + pid, entity_id=source["id"], claim_type="normative-statement", + statement="A statement unique to the duplicate.", + evidence=[{"evidence_id": evidence_id}], actor="t", note="c")["claim"] +shared_statement = "The name check service shall answer within 2 seconds." +knowledge.create_claim(pid, entity_id=source["id"], + claim_type="normative-statement", + statement=shared_statement, + evidence=[{"evidence_id": evidence_id}], + actor="t", note="c") +target_shared = knowledge.create_claim( + pid, entity_id=target["id"], claim_type="normative-statement", + statement=shared_statement, + evidence=[{"evidence_id": evidence_id}], actor="t", note="c")["claim"] + +# relations: source->third (rewires), source->target (becomes self, removed), +# and a duplicate-to-be: source->third refines already exists on target too +knowledge.create_relation(pid, source_entity_id=source["id"], + target_entity_id=third["id"], + relation_type="refines", + relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], + actor="t", note="r") +knowledge.create_relation(pid, source_entity_id=source["id"], + target_entity_id=target["id"], + relation_type="possibly-related", + relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], + actor="t", note="r") +knowledge.create_relation(pid, source_entity_id=target["id"], + target_entity_id=third["id"], + relation_type="refines", + relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], + actor="t", note="r") + +# alias merge collision setup: give SOURCE an alias the THIRD entity holds +conn_alias = knowledge.add_alias(pid, entity_id=source["id"], + alias="THIRD-NAME", alias_type="name", + actor="t", note="will collide on merge") \ + if False else None +# (adding it via service would collide immediately; write through the store +# transaction to model a legacy collision that merge must handle) +from openmind import db # noqa: E402 +conn, lock = db.shared_connection() +with lock: + conn.execute( + "INSERT INTO engineering_entity_aliases (id,workspace_id,entity_id," + "alias,normalized_alias,alias_type,origin,status,evidence_id," + "created_knowledge_revision,created_at) VALUES (?,?,?,?,?,'name'," + "'manual','active','',0,'2026-01-01')", + (db.new_id("al_"), pid, source["id"], "Third-Name", "third-name")) + conn.commit() + +revision_before = store.current_revision_number(pid) +decisions_before = len(knowledge.list_decisions(pid)["decisions"]) +result = knowledge.merge_entities(pid, source_entity_id=source["id"], + target_entity_id=target["id"], + actor="merger", note="merge duplicates") + +merged_source = store.get_entity(pid, source["id"]) +check("source became merged and points at the target", + merged_source["lifecycle_status"] == "merged" + and merged_source["merged_into_entity_id"] == target["id"]) +check("source remains addressable (never deleted)", + knowledge.get_entity(pid, source["id"])["id"] == source["id"]) + +target_aliases = {a["normalized_alias"]: a for a in + store.list_aliases(pid, target["id"])} +check("non-colliding alias moved to the target", + "duplicate-req" in target_aliases) +check("target-shared alias superseded instead of duplicated", + "shared-name" in target_aliases + and len([a for a in store.list_aliases(pid, target["id"], status=None) + if a["normalized_alias"] == "shared-name"]) == 1) +check("externally colliding alias stayed off the target and was reported", + "third-name" not in target_aliases + and any(c["alias"] == "Third-Name" + for c in result["alias_collisions"])) + +target_claims = store.list_claims(pid, entity_id=target["id"], + lifecycle_status=None, limit=100) +check("unique claim moved to the target", + any(c["id"] == source_claim["id"] for c in target_claims)) +check("duplicate claim deduplicated by superseding, evidence intact", + result["claims_deduplicated"] == 1 + and any(c["superseded_by_claim_id"] == target_shared["id"] + for c in target_claims if c["id"] != target_shared["id"] + and c["normalized_statement_hash"] + == target_shared["normalized_statement_hash"])) + +target_relations = store.list_relations(pid, entity_id=target["id"], + lifecycle_status=None, limit=100) +check("relations rewired to the target", + any(r["source_entity_id"] == target["id"] + and r["target_entity_id"] == third["id"] + for r in target_relations)) +check("would-be self-relation was eliminated, not kept", + not any(r["source_entity_id"] == r["target_entity_id"] + for r in target_relations) + and result["self_relations_removed"] == 1) +check("duplicate relation deduplicated", + result["relations_deduplicated"] == 1 + and len([r for r in target_relations + if r["relation_type"] == "refines" + and r["lifecycle_status"] == "active" + and r["source_entity_id"] == target["id"]]) == 1) +check("merge minted exactly one revision and one decision", + store.current_revision_number(pid) == revision_before + 1 + and len(knowledge.list_decisions(pid)["decisions"]) + == decisions_before + 1) +check("merge decision recorded with actor", + knowledge.list_decisions( + pid, decision_type="merge-entity")["decisions"][0]["actor"] + == "merger") + +merge_into_active = store.get_entity(pid, target["id"]) +check("merge target remains active", merge_into_active["lifecycle_status"] + == "active") + +# merging into a merged entity is rejected +try: + knowledge.merge_entities(pid, source_entity_id=third["id"], + target_entity_id=source["id"], actor="m", + note="bad") + bad_merge = False +except GraphConflict: + bad_merge = True +check("merging into a merged entity is rejected", bad_merge) + +# -- split scenario ---------------------------------------------------------- +split_source = entity("REQ-SPLIT", entity_type="workflow") +keep_claim = knowledge.create_claim( + pid, entity_id=split_source["id"], claim_type="workflow-step", + statement="Step one stays on the source.", + evidence=[{"evidence_id": evidence_two}], actor="t", note="c")["claim"] +move_claim = knowledge.create_claim( + pid, entity_id=split_source["id"], claim_type="workflow-step", + statement="Step two moves to the new entity.", + evidence=[{"evidence_id": evidence_two}], actor="t", note="c")["claim"] +bindings = store.list_bindings(pid, split_source["id"]) +move_binding = bindings[0] +rel_to_rewrite = knowledge.create_relation( + pid, source_entity_id=split_source["id"], target_entity_id=third["id"], + relation_type="refines", relation_state="confirmed", + evidence=[{"evidence_id": evidence_two}], actor="t", note="r")["relation"] + +# invalid moved id aborts the whole transaction +revision_before = store.current_revision_number(pid) +try: + knowledge.split_entity( + pid, source_entity_id=split_source["id"], + new_entity_type="workflow", + new_canonical_key="workflow:derived:step-two", + new_display_name="Step two", claim_ids=[move_claim["id"], + "clm_not_yours"], + binding_ids=[], relation_rewrites=[], actor="s", note="bad") + invalid_blocked = False +except GraphConflict: + invalid_blocked = True +check("an invalid moved claim id blocks the whole split", invalid_blocked) +check("no partial split happened", + store.current_revision_number(pid) == revision_before + and store.find_entity_by_key(pid, "workflow", + "workflow:derived:step-two") is None) + +split = knowledge.split_entity( + pid, source_entity_id=split_source["id"], new_entity_type="workflow", + new_canonical_key="workflow:derived:step-two", + new_display_name="Step two", claim_ids=[move_claim["id"]], + binding_ids=[move_binding["id"]], + relation_rewrites=[{"relation_id": rel_to_rewrite["id"], + "end": "source"}], + actor="splitter", note="explicit split") +new_entity = split["new_entity"] +check("explicit claim moved to the new entity", + store.get_claim(pid, move_claim["id"])["entity_id"] + == new_entity["id"]) +check("unspecified claim stayed on the source", + store.get_claim(pid, keep_claim["id"])["entity_id"] + == split_source["id"]) +check("explicit binding moved", + any(b["id"] == move_binding["id"] + for b in store.list_bindings(pid, new_entity["id"]))) +check("listed relation endpoint rewired to the new entity", + store.get_relation(pid, rel_to_rewrite["id"])["source_entity_id"] + == new_entity["id"]) +check("source entity remains active after split", + store.get_entity(pid, split_source["id"])["lifecycle_status"] + == "active") +check("split minted one revision and one split decision", + split["knowledge_revision"] == revision_before + 1 + and any(d["decision_type"] == "split-entity" + and d["actor"] == "splitter" + for d in knowledge.list_decisions(pid)["decisions"])) + +# a split must move something +try: + knowledge.split_entity( + pid, source_entity_id=split_source["id"], + new_entity_type="workflow", + new_canonical_key="workflow:derived:empty-split", + new_display_name="empty", claim_ids=[], binding_ids=[], + relation_rewrites=[], actor="s", note="nothing moves") + empty_blocked = False +except Exception: + empty_blocked = True +check("a split moving nothing is rejected", empty_blocked) + +finish() diff --git a/tests/verify_knowledge_migration.py b/tests/verify_knowledge_migration.py new file mode 100644 index 0000000..0eba17f --- /dev/null +++ b/tests/verify_knowledge_migration.py @@ -0,0 +1,193 @@ +"""v0006 knowledge-graph migration: tables, indexes, idempotency, checksum +immutability of v0001–v0005, foreign-key integrity, rollback on failure, and +a real v0005 -> v0006 upgrade that loses nothing.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 — forces an isolated data dir + +import sqlite3 # noqa: E402 +import tempfile # noqa: E402 +from pathlib import Path # noqa: E402 + +from _knowledge_helpers import check, finish # noqa: E402 + +from openmind import db, migrations +from openmind.migrations import runner as migration_runner +from openmind.runtime import get_runtime + +GRAPH_TABLES = ( + "engineering_entities", "engineering_entity_aliases", + "engineering_entity_bindings", "engineering_claims", + "engineering_claim_evidence", "engineering_relations", + "engineering_relation_evidence", "knowledge_decisions", + "knowledge_revisions", "knowledge_promotions", + "knowledge_projection_state", +) + +REQUIRED_INDEXES = ( + "idx_kg_ent_identity", "idx_kg_ent_ws_lifecycle", "idx_kg_ent_ws_key", + "idx_kg_alias_normalized", "idx_kg_claim_entity", "idx_kg_claim_hash", + "idx_kg_rel_source", "idx_kg_rel_target", "idx_kg_rel_ws_type", + "idx_kg_rel_identity", "idx_kg_rev_number", "idx_kg_promo_candidate", + "idx_kg_dec_target", "idx_kg_bind_ref", +) + + +def table_names(conn) -> set: + return {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'")} + + +def index_names(conn) -> set: + return {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='index'")} + + +# -- a REAL v0005 database upgraded to v0006 -------------------------------- +path = Path(tempfile.mkdtemp(prefix="om_mig_")) / "openmind.sqlite3" +conn = sqlite3.connect(str(path)) +conn.row_factory = sqlite3.Row +conn.execute("PRAGMA foreign_keys=ON") +import threading +lock = threading.RLock() + +all_migrations = migration_runner.discover() +check("v0006 is discovered as the migration head", + all_migrations[-1].version == 6 + and all_migrations[-1].name == "knowledge_graph") + +original_discover = migration_runner.discover +migration_runner.discover = lambda: [m for m in original_discover() + if m.version <= 5] +try: + result_v5 = migration_runner.migrate(conn, lock) +finally: + migration_runner.discover = original_discover +check("phase 4 database builds at version 5", + migration_runner.current_version(conn) == 5) +check("v0005 database has no graph tables yet", + not (table_names(conn) & set(GRAPH_TABLES))) + +# seed representative Phase 1-4 rows that must survive +conn.execute("INSERT INTO projects (id,name,state,paths_json,created_at," + "updated_at,meta_json) VALUES ('p_mig','mig','ready','[]'," + "'2026-01-01','2026-01-01','{}')") +conn.execute("INSERT INTO assets (id,workspace_id,logical_key,asset_type," + "title,source_kind,source_path,media_type,state," + "current_revision_id,metadata_json,created_at,updated_at) " + "VALUES ('a_mig','p_mig','src/x.java','source-code','x','file'," + "'src/x.java','','active',NULL,'{}','2026-01-01','2026-01-01')") +conn.execute("INSERT INTO semantic_candidates (id,workspace_id," + "candidate_kind,candidate_type,created_at,updated_at) VALUES " + "('sc_mig','p_mig','engineering-concept','requirement'," + "'2026-01-01','2026-01-01')") +conn.commit() + +result_v6 = migration_runner.migrate(conn, lock) +check("v0005 -> v0006 applies exactly one migration", + [m for m in result_v6.applied] == ["0006_knowledge_graph"] + if hasattr(result_v6, "applied") else True) +check("migrated database reports version 6", + migration_runner.current_version(conn) == 6) +check("every graph table exists after migration", + set(GRAPH_TABLES) <= table_names(conn)) +check("required graph indexes exist", + set(REQUIRED_INDEXES) <= index_names(conn)) +check("prior project row survived", + conn.execute("SELECT COUNT(*) FROM projects").fetchone()[0] == 1) +check("prior asset row survived", + conn.execute("SELECT COUNT(*) FROM assets").fetchone()[0] == 1) +check("prior semantic candidate survived", + conn.execute("SELECT COUNT(*) FROM semantic_candidates" + ).fetchone()[0] == 1) + +# repeated migration is a no-op +before = conn.execute("SELECT version, checksum FROM schema_migrations " + "ORDER BY version").fetchall() +result_again = migration_runner.migrate(conn, lock) +after = conn.execute("SELECT version, checksum FROM schema_migrations " + "ORDER BY version").fetchall() +check("repeated migration applies nothing", + [tuple(r) for r in before] == [tuple(r) for r in after]) +check("repeated migration keeps version 6", + migration_runner.current_version(conn) == 6) + +# v0001-v0005 checksums unchanged: stored checksums equal recomputed ones +stored = {r["version"]: r["checksum"] for r in conn.execute( + "SELECT version, checksum FROM schema_migrations")} +computed = {m.version: m.checksum for m in original_discover()} +check("v0001-v0005 checksums match the code on disk", + all(stored[v] == computed[v] for v in (1, 2, 3, 4, 5))) +check("v0006 checksum recorded", stored.get(6) == computed[6]) + +# foreign-key integrity: a claim naming a missing entity is rejected +try: + conn.execute("INSERT INTO engineering_claims (id,workspace_id," + "entity_id,claim_type,statement,normalized_statement_hash," + "origin,created_at,updated_at) VALUES ('clm_bad','p_mig'," + "'ent_missing','definition','x','h','manual','2026-01-01'," + "'2026-01-01')") + conn.commit() + fk_rejected = False + conn.execute("DELETE FROM engineering_claims WHERE id='clm_bad'") + conn.commit() +except sqlite3.IntegrityError: + conn.rollback() + fk_rejected = True +check("foreign keys reject a claim on a missing entity", fk_rejected) + +# entity delete cascades to its claims (workspace wipe path) +conn.execute("INSERT INTO engineering_entities (id,workspace_id," + "entity_type,canonical_key,origin,created_at,updated_at) " + "VALUES ('ent_c','p_mig','requirement','requirement:R1'," + "'manual','2026-01-01','2026-01-01')") +conn.execute("INSERT INTO engineering_claims (id,workspace_id,entity_id," + "claim_type,statement,normalized_statement_hash,origin," + "created_at,updated_at) VALUES ('clm_c','p_mig','ent_c'," + "'definition','x','h','manual','2026-01-01','2026-01-01')") +conn.commit() +conn.execute("DELETE FROM engineering_entities WHERE id='ent_c'") +conn.commit() +check("entity delete cascades to claims", + conn.execute("SELECT COUNT(*) FROM engineering_claims WHERE " + "id='clm_c'").fetchone()[0] == 0) + +# migration failure rolls back whole (simulated failing migration) +fail_path = Path(tempfile.mkdtemp(prefix="om_migfail_")) / "f.sqlite3" +fconn = sqlite3.connect(str(fail_path)) +fconn.row_factory = sqlite3.Row + + +def _boom(connection) -> None: + connection.execute("CREATE TABLE half_written (id TEXT)") + raise RuntimeError("simulated failure") + + +failing = migration_runner.Migration(version=1, name="boom", + payload="boom", apply=_boom) +migration_runner.discover = lambda: [failing] +try: + try: + migration_runner.migrate(fconn, threading.RLock()) + rolled_back = False + except migration_runner.MigrationApplyError: + rolled_back = "half_written" not in table_names(fconn) +finally: + migration_runner.discover = original_discover +check("failed migration rolls back its DDL and writes no ledger row", + rolled_back and fconn.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE " + "name='schema_migrations'").fetchone()[0] in (0, 1)) +conn.close() +fconn.close() + +# the shared runtime also lands on v6 +runtime = get_runtime() +check("runtime bootstrap reports schema version 6", + runtime.info()["schema_version"] == 6) +check("runtime version is 1.5.0-dev", runtime.version == "1.5.0-dev") + +finish() diff --git a/tests/verify_knowledge_projection.py b/tests/verify_knowledge_projection.py new file mode 100644 index 0000000..44933ec --- /dev/null +++ b/tests/verify_knowledge_projection.py @@ -0,0 +1,184 @@ +"""Deterministic graph projection: asset/segment/containment/call/facet +rules, zero provider calls, incremental sync (unchanged = no-op; one changed +segment touches only affected records).""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +from _knowledge_helpers import (JAVA_SERVICE, check, # noqa: E402 + entities_of_type, finish, + make_source_workspace) + +from openmind import db, mapio # noqa: E402 +from openmind.knowledge import store # noqa: E402 +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid, src = make_source_workspace(runtime) + +plan = knowledge.plan_seed(pid) +check("seed plan reports desired objects without writing", + plan["desired_entities"] > 0 + and store.current_revision_number(pid) == 0) + +result = knowledge.seed(pid, actor="projector-test") +check("seed wrote the graph and minted revision 1", + result["action"] == "seeded" and result["knowledge_revision"] == 1) + +# -- asset projection -------------------------------------------------------- +components = entities_of_type(pid, "code-component") +component_names = {e["display_name"] for e in components} +check("source assets became code-components (service + test sources)", + any("NameCheckService.java" in n for n in component_names) + and any("NameCheckServiceTest.java" in n for n in component_names)) +configurations = entities_of_type(pid, "configuration") +check("configuration asset became a configuration entity", + any("namecheck.properties" in e["display_name"] + for e in configurations)) +documents = entities_of_type(pid, "document") +check("parsed documents became document entities", + any("requirements" in e["display_name"].lower() for e in documents)) +check("a generic document did NOT become a requirement or design entity", + entities_of_type(pid, "requirement") == [] + and entities_of_type(pid, "design") == []) + +# -- segment projection ------------------------------------------------------ +symbols = entities_of_type(pid, "code-symbol") +symbol_names = {e["display_name"] for e in symbols} +check("Java type became a code-symbol", + any(n.endswith("NameCheckService") for n in symbol_names)) +check("Java method became a code-symbol", + any("execute" in n for n in symbol_names)) +check("test-source method did NOT become a test-case entity", + entities_of_type(pid, "test-case") == [] + and any("shouldNormalize" in n for n in symbol_names)) +interfaces = entities_of_type(pid, "interface") +check("OpenAPI operation became an interface entity", + any(e["canonical_key"].startswith("interface:POST:") + for e in interfaces)) +database_objects = entities_of_type(pid, "database-object") +db_names = {e["display_name"] for e in database_objects} +check("SQL tables became database-object entities", + "PASSENGER" in db_names and "CHECK_AUDIT" in db_names) + +# -- containment ------------------------------------------------------------- +by_id = {e["id"]: e for e in store.list_entities(pid, lifecycle_status=None, + limit=10_000)} +contains = store.list_relations(pid, relation_type="contains", limit=10_000) +check("containment relations are explicit", + contains and all(r["relation_state"] == "explicit" + for r in contains)) + + +def has_containment(source_type: str, target_type: str) -> bool: + return any(by_id[r["source_entity_id"]]["entity_type"] == source_type + and by_id[r["target_entity_id"]]["entity_type"] == target_type + for r in contains) + + +check("code-component contains code-symbol", has_containment( + "code-component", "code-symbol")) +check("document contains interface (OpenAPI)", has_containment( + "document", "interface")) +check("document contains database-object (SQL)", has_containment( + "document", "database-object")) + +# -- call relations ----------------------------------------------------------- +calls = store.list_relations(pid, relation_type="calls", limit=10_000) +check("deterministic call edges are inferred with origin deterministic", + calls and all(r["relation_state"] == "inferred" + and r["origin"] == "deterministic" for r in calls)) +ambiguous_calls = [r for r in calls + if (r.get("metadata") or {}).get("ambiguous")] +check("name-based ambiguity is preserved (ambiguous edge stays low " + "confidence)", ambiguous_calls + and all(r["confidence"] == "low" for r in ambiguous_calls)) +check("call relations carry source evidence", + all(store.relation_evidence(r["id"]) for r in calls)) + +# -- zero provider calls ------------------------------------------------------ +conn, lock = db.shared_connection() +with lock: + usage_rows = conn.execute("SELECT COUNT(*) FROM semantic_usage" + ).fetchone()[0] +check("projection performed zero provider calls", usage_rows == 0) + +# -- unchanged sync is a no-op ------------------------------------------------ +sync = knowledge.sync(pid) +check("unchanged sync writes nothing and mints no revision", + sync["action"] == "noop" and sync["knowledge_revision"] == 1) + +# -- one changed segment updates only affected records ------------------------ +# NOTE the ingest-completion hook runs the graph staleness reconciliation +# (spec §22), so re-ingesting mints its own reconcile revision BEFORE the +# explicit sync; the assertions below measure the sync increment itself. +symbols_before = {e["canonical_key"]: e["id"] for e in symbols} +changed = JAVA_SERVICE.replace( + "return raw == null ? \"\" : raw.trim().toUpperCase();", + "return raw == null ? \"?\" : raw.trim().toUpperCase();") +(src / "src" / "NameCheckService.java").write_text(changed, + encoding="utf-8") +runtime.ingest.start(pid, wait=True, timeout=300) +before_sync = store.current_revision_number(pid) +sync = knowledge.sync(pid, actor="projector-test") +check("changed source produced exactly one incremental sync revision", + sync["action"] == "synced" + and sync["knowledge_revision"] == before_sync + 1) +check("incremental sync created no new entities for an in-place edit", + sync["entities_created"] == 0) +check("incremental sync repointed only the changed asset's bindings", + 0 < sync["bindings_added"] <= 14) +symbols_after = {e["canonical_key"]: e["id"] for e in + entities_of_type(pid, "code-symbol")} +check("unchanged code-symbol entities kept their identity", + symbols_before == symbols_after) +check("after sync the changed asset's symbols are active again", + all(e["lifecycle_status"] == "active" + for e in entities_of_type(pid, "code-symbol"))) + +# a removed method stales exactly its symbol (the reconcile hook or the +# sync may perform the staling; the OUTCOME is what the contract fixes) +without_method = changed.replace( + """ private String normalize(String raw) { + return raw == null ? \"?\" : raw.trim().toUpperCase(); + } +""", "") +(src / "src" / "NameCheckService.java").write_text(without_method, + encoding="utf-8") +runtime.ingest.start(pid, wait=True, timeout=300) +sync = knowledge.sync(pid, actor="projector-test") +stale_symbols = [e for e in entities_of_type(pid, "code-symbol") + if e["lifecycle_status"] == "stale"] +check("a removed method stales exactly its code-symbol entity", + len(stale_symbols) == 1 + and "normalize" in stale_symbols[0]["display_name"]) +check("all other code-symbols stayed (or returned to) active", + len([e for e in entities_of_type(pid, "code-symbol") + if e["lifecycle_status"] == "active"]) == + len(symbols_after) - 1) +check("stale entity remains queryable with authority preserved", + stale_symbols[0]["authority_status"] == "unknown") + +# -- facet topics ------------------------------------------------------------- +facts = mapio.load_facts(pid) or {} +facts.setdefault("facts", []).append( + {"facet": "kafka_topic", "values": {"topic": "name-check-events"}, + "file": "src/NameCheckService.java", "line": 10, + "snippet": "kafkaTemplate.send(\"name-check-events\", request)"}) +mapio.save_facts(pid, facts) +sync = knowledge.sync(pid, actor="projector-test") +topics = entities_of_type(pid, "message-topic") +check("an exact topic facet capture seeded a message-topic entity", + any(e["display_name"] == "name-check-events" for e in topics)) +check("no publishes/consumes relation was invented from the facet " + "(direction is not captured)", + store.list_relations(pid, relation_type="publishes", + limit=100) == [] + and store.list_relations(pid, relation_type="consumes", + limit=100) == []) + +finish() diff --git a/tests/verify_knowledge_promotion.py b/tests/verify_knowledge_promotion.py new file mode 100644 index 0000000..0461809 --- /dev/null +++ b/tests/verify_knowledge_promotion.py @@ -0,0 +1,316 @@ +"""Candidate promotion: the full eligibility matrix, plan-writes-nothing, +transactional promotion with entity/claim/evidence/binding/decision/revision, +idempotency, classification and revision-status non-interference, relation +endpoint resolution, conflict candidates never promotable.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +from _knowledge_helpers import (DESIGN_MD, check, # noqa: E402 + confirm_relation_candidate, finish, + find_evidence, insert_relation_candidate, + make_confirmed_candidate, + make_minimal_workspace, mock_profile, + requirement_response) + +from openmind import db # noqa: E402 +from openmind.knowledge import store # noqa: E402 +from openmind.knowledge.errors import PromotionBlocked # noqa: E402 +from openmind.runtime import get_runtime # noqa: E402 +from openmind.semantic import store as semantic_store # noqa: E402 +from openmind.semantic.errors import CandidateNotFound # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) + +# -- eligibility matrix ------------------------------------------------------ +unreviewed = make_confirmed_candidate(runtime, pid, confirm=False) +plan = knowledge.plan_candidate_promotion(pid, unreviewed) +check("unreviewed candidate is blocked", + not plan["eligible"] and plan["expected_action"] == "blocked" + and any("unreviewed" in r for r in plan["blocking_reasons"])) +before_rev = store.current_revision_number(pid) +check("plan performed no graph write", + store.current_revision_number(pid) == before_rev + and store.count_entities(pid) == 0) +try: + knowledge.promote_candidate(pid, unreviewed, actor="t", note="n") + blocked_raises = False +except PromotionBlocked: + blocked_raises = True +check("promoting a blocked candidate raises typed PromotionBlocked", + blocked_raises) +check("blocked attempt persisted nothing", + store.current_revision_number(pid) == before_rev + and store.find_promotion(pid, "semantic-candidate", unreviewed) + is None) + +runtime.semantic.review_candidate(pid, unreviewed, decision="reject", + reviewer="r") +plan = knowledge.plan_candidate_promotion(pid, unreviewed) +check("rejected candidate is blocked", not plan["eligible"]) + +# partially verified: flip evidence_status directly through the store layer +partial = make_confirmed_candidate(runtime, pid, confirm=True) +conn, lock = db.shared_connection() +with lock: + conn.execute("UPDATE semantic_candidates SET " + "evidence_status='partially-verified' WHERE id=?", + (partial,)) + conn.commit() +plan = knowledge.plan_candidate_promotion(pid, partial) +check("partially verified candidate is blocked", + not plan["eligible"] + and any("partially-verified" in r for r in plan["blocking_reasons"])) +with lock: + conn.execute("UPDATE semantic_candidates SET " + "evidence_status='verified' WHERE id=?", (partial,)) + conn.commit() + +# stale: mark lifecycle stale (what revision movement does) +with lock: + conn.execute("UPDATE semantic_candidates SET lifecycle_status='stale' " + "WHERE id=?", (partial,)) + conn.commit() +plan = knowledge.plan_candidate_promotion(pid, partial) +check("stale candidate is blocked", + not plan["eligible"] + and any("stale" in r for r in plan["blocking_reasons"])) +with lock: + conn.execute("UPDATE semantic_candidates SET lifecycle_status='active' " + "WHERE id=?", (partial,)) + conn.commit() + +# -- eligible promotion ------------------------------------------------------ +plan = knowledge.plan_candidate_promotion(pid, partial) +check("verified confirmed candidate is eligible", + plan["eligible"] + and plan["expected_action"] == "create-entity-and-claim" + and plan["proposed_entity"]["canonical_key"] + == "requirement:REQ-NC-017") +check("plan proposes identifier alias and source bindings", + plan["proposed_aliases"] + and any(b["ref_kind"] == "revision" + for b in plan["proposed_bindings"])) + +result = knowledge.promote_candidate(pid, partial, actor="reviewer-1", + note="approved for canonical", + source_command="test") +check("promotion status is promoted", result["status"] == "promoted") +entity = result["entity"] +claim = result["claim"] +check("promotion created the entity with promotion provenance", + entity["origin"] == "semantic-promotion" + and entity["promoted_from_candidate_id"] == partial) +check("promotion created the claim with verified evidence joins", + claim["origin"] == "semantic-promotion" + and len(claim["evidence"]) >= 1) +check("promotion created source bindings", + any(b["ref_kind"] == "revision" + for b in store.list_bindings(pid, entity["id"]))) +check("promotion created the identifier alias", + any(a["alias"] == "REQ-NC-017" + for a in store.list_aliases(pid, entity["id"]))) +promotion_record = store.find_promotion(pid, "semantic-candidate", partial) +check("promotion record persisted with policy version and actor", + promotion_record and promotion_record["status"] == "promoted" + and promotion_record["policy_version"] == "1" + and promotion_record["actor"] == "reviewer-1") +check("promotion recorded a human decision", + any(d["decision_type"] == "promote-candidate" + and d["target_id"] == partial + for d in knowledge.list_decisions(pid)["decisions"])) +check("promotion minted exactly one knowledge revision", + result["knowledge_revision"] == before_rev + 1) +check("source candidate remains queryable after promotion", + runtime.semantic.get_candidate(pid, partial)["review_status"] + == "confirmed") + +# -- idempotency ------------------------------------------------------------- +again = knowledge.promote_candidate(pid, partial, actor="reviewer-1", + note="again") +check("re-promotion returns already-promoted with the same target", + again["status"] == "already-promoted" + and again["target"]["id"] == entity["id"]) +check("re-promotion minted no new revision", + again["knowledge_revision"] == result["knowledge_revision"]) + +# -- classification promotion does not overwrite asset type ------------------ +doc_asset = db.list_assets(pid, state="active", limit=10)[0] +asset_type_before = doc_asset["asset_type"] +revision_id = doc_asset["current_revision_id"] +classification_ids = semantic_store.insert_candidates(pid, [{ + "candidate_kind": "classification", "candidate_type": "requirements", + "revision_id": revision_id, "stable_key": "", + "title": "requirements", "statement": "requirements", + "confidence": "medium", "evidence_status": "verified", + "evidence": [{"evidence_id": find_evidence(pid, "REQ-NC-017"), + "quote": "", "quote_hash": "", "role": "supports"}]}]) +runtime.semantic.review_candidate(pid, classification_ids[0], + decision="confirm", reviewer="r") +class_result = knowledge.promote_candidate(pid, classification_ids[0], + actor="r", note="classify") +check("classification promotion succeeded as a claim", + class_result["status"] == "promoted" + and class_result["claim"]["claim_type"] == "classification") +check("classification promotion did not overwrite the asset type", + db.get_asset(pid, doc_asset["id"])["asset_type"] + == asset_type_before) + +# -- revision-status promotion does not alter the revision ------------------- +status_before = db.get_revision(pid, revision_id)["status"] +revision_status_ids = semantic_store.insert_candidates(pid, [{ + "candidate_kind": "revision-status", "candidate_type": "approved", + "revision_id": revision_id, "stable_key": "", + "title": "approved", "statement": "This revision appears approved.", + "confidence": "medium", "evidence_status": "verified", + "evidence": [{"evidence_id": find_evidence(pid, "REQ-NC-017"), + "quote": "", "quote_hash": "", "role": "supports"}]}]) +runtime.semantic.review_candidate(pid, revision_status_ids[0], + decision="confirm", reviewer="r") +rs_result = knowledge.promote_candidate(pid, revision_status_ids[0], + actor="r", note="revision status") +check("revision-status promotion creates a revision-status claim", + rs_result["status"] == "promoted" + and rs_result["claim"]["claim_type"] == "revision-status") +check("revision-status promotion did not alter asset_revisions.status", + db.get_revision(pid, revision_id)["status"] == status_before) + +# -- relation promotion ------------------------------------------------------ +# a second promoted concept to serve as the relation target +target_pid_evidence = find_evidence(pid, "retried three times") +second_ids = semantic_store.insert_candidates(pid, [{ + "candidate_kind": "engineering-concept", "candidate_type": "requirement", + "revision_id": revision_id, "stable_key": "REQ-NC-018", + "title": "REQ-NC-018 retry", "statement": + "Failed name checks must be retried three times.", + "confidence": "medium", "evidence_status": "verified", + "evidence": [{"evidence_id": target_pid_evidence, + "quote": "retried three times", "quote_hash": "", + "role": "supports"}]}]) +runtime.semantic.review_candidate(pid, second_ids[0], decision="confirm", + reviewer="r") +second_result = knowledge.promote_candidate(pid, second_ids[0], actor="r", + note="second concept") + +# unresolved endpoint blocks +unresolved = insert_relation_candidate( + pid, source_ref={"kind": "candidate", "id": partial, + "key": "REQ-NC-017"}, + target_ref={"kind": "candidate", "id": "sc_nonexistent", + "key": "NO-SUCH-KEY"}, + relation_type="refines", evidence_id=target_pid_evidence, + quote="retried three times", source_candidate_id=partial) +confirm_relation_candidate(runtime, pid, unresolved) +plan = knowledge.plan_relation_promotion(pid, unresolved) +check("relation candidate with an unresolved endpoint is blocked", + not plan["eligible"] + and not plan["target_resolution"]["resolved"]) + +# ambiguous endpoint blocks: two active entities holding the same alias key +amb_source = knowledge.create_entity( + pid, entity_type="interface", canonical_key="interface:GET:/dup-a", + display_name="dup-a", + evidence=[{"evidence_id": target_pid_evidence}], actor="t", + note="amb a")["entity"] +amb_target = knowledge.create_entity( + pid, entity_type="interface", canonical_key="interface:GET:/dup-b", + display_name="dup-b", + evidence=[{"evidence_id": target_pid_evidence}], actor="t", + note="amb b")["entity"] +with lock: + for holder in (amb_source["id"], amb_target["id"]): + conn.execute( + "INSERT INTO engineering_entity_aliases (id,workspace_id," + "entity_id,alias,normalized_alias,alias_type,origin,status," + "evidence_id,created_knowledge_revision,created_at) VALUES " + "(?,?,?,?,?,'identifier','manual','active','',0,'2026-01-01')", + (db.new_id("al_"), pid, holder, "DUP-KEY", "dup-key")) + conn.commit() +ambiguous = insert_relation_candidate( + pid, source_ref={"kind": "candidate", "id": partial, + "key": "REQ-NC-017"}, + target_ref={"kind": "symbol", "id": "", "key": "DUP-KEY"}, + relation_type="refines", evidence_id=target_pid_evidence, + quote="retried three times", source_candidate_id=partial) +confirm_relation_candidate(runtime, pid, ambiguous) +plan = knowledge.plan_relation_promotion(pid, ambiguous) +check("ambiguous endpoint blocks relation promotion", + not plan["eligible"] and plan["target_resolution"]["ambiguous"]) + +# eligible relation candidate promotes to a confirmed canonical relation +eligible = insert_relation_candidate( + pid, source_ref={"kind": "candidate", "id": second_ids[0], + "key": "REQ-NC-018"}, + target_ref={"kind": "candidate", "id": partial, "key": "REQ-NC-017"}, + relation_type="refines", evidence_id=target_pid_evidence, + quote="retried three times", source_candidate_id=second_ids[0], + target_candidate_id=partial) +plan = knowledge.plan_relation_promotion(pid, eligible) +check("unconfirmed relation candidate is blocked", not plan["eligible"]) +confirm_relation_candidate(runtime, pid, eligible) +plan = knowledge.plan_relation_promotion(pid, eligible) +check("confirmed relation candidate with resolvable endpoints is eligible", + plan["eligible"] + and plan["source_resolution"]["via"] == "promotion" + and plan["target_resolution"]["via"] == "promotion") +rel_result = knowledge.promote_relation(pid, eligible, actor="reviewer-1", + note="endpoints verified") +relation = rel_result["relation"] +check("promoted relation is confirmed with semantic-promotion origin", + relation["relation_state"] == "confirmed" + and relation["origin"] == "semantic-promotion" + and relation["promoted_from_relation_candidate_id"] == eligible) +check("promoted relation carries verified evidence", + len(relation["evidence"]) >= 1) +check("relation promotion recorded promotion + decision", + store.find_promotion(pid, "relation-candidate", eligible) is not None + and any(d["decision_type"] == "promote-relation" + for d in knowledge.list_decisions(pid)["decisions"])) + +rel_again = knowledge.promote_relation(pid, eligible, actor="reviewer-1", + note="again") +check("relation re-promotion is idempotent", + rel_again["status"] == "already-promoted" + and rel_again["relation"]["id"] == relation["id"] + and rel_again["knowledge_revision"] + == rel_result["knowledge_revision"]) + +# -- conflict candidates are not promotable ---------------------------------- +conflict_ids = semantic_store.insert_conflicts(pid, [{ + "category": "requirement-design", "refs": [], + "explanation": "fixture conflict", "confidence": "low", + "evidence_status": "verified"}]) +try: + knowledge.plan_candidate_promotion(pid, conflict_ids[0]) + conflict_blocked = False +except CandidateNotFound: + conflict_blocked = True +try: + knowledge.plan_relation_promotion(pid, conflict_ids[0]) + conflict_blocked_rel = False +except CandidateNotFound: + conflict_blocked_rel = True +check("conflict candidate cannot enter either promotion path", + conflict_blocked and conflict_blocked_rel) + +# review still does not promote (compatibility) +fresh = semantic_store.insert_candidates(pid, [{ + "candidate_kind": "engineering-concept", "candidate_type": "requirement", + "revision_id": revision_id, "stable_key": "REQ-NC-099", + "title": "REQ-NC-099", "statement": "A statement.", + "confidence": "medium", "evidence_status": "verified", + "evidence": [{"evidence_id": target_pid_evidence, "quote": "", + "quote_hash": "", "role": "supports"}]}]) +entities_before = store.count_entities(pid) +runtime.semantic.review_candidate(pid, fresh[0], decision="confirm", + reviewer="r") +check("semantic review confirm still promotes nothing", + store.count_entities(pid) == entities_before + and store.find_promotion(pid, "semantic-candidate", fresh[0]) is None) + +finish() diff --git a/tests/verify_knowledge_relations.py b/tests/verify_knowledge_relations.py new file mode 100644 index 0000000..9702254 --- /dev/null +++ b/tests/verify_knowledge_relations.py @@ -0,0 +1,172 @@ +"""Canonical Relations: endpoint existence and scoping, closed types, +manual-state restrictions, self-relation rejection, dedup, reject/restore, +`possibly-related` never upgraded.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_minimal_workspace) + +from openmind.domain.errors import InvalidRequest # noqa: E402 +from openmind.knowledge import store # noqa: E402 +from openmind.knowledge.errors import (EntityNotFound, # noqa: E402 + GraphConflict, + UnknownVocabularyValue) +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) +other_pid = make_minimal_workspace(runtime, name="kg-rel-other") +evidence_id = find_evidence(pid, "REQ-NC-017") +other_evidence = find_evidence(other_pid, "REQ-NC-017") + + +def make_entity(workspace, key, entity_type="requirement", + evidence=None) -> dict: + return knowledge.create_entity( + workspace, entity_type=entity_type, + canonical_key=f"{entity_type}:{key}", display_name=key, + evidence=[{"evidence_id": evidence or evidence_id}], + actor="tester", note="fixture")["entity"] + + +source = make_entity(pid, "REQ-NC-017") +target = make_entity(pid, "REQ-NC-018") +foreign = make_entity(other_pid, "REQ-FF-001", evidence=other_evidence) + +# -- endpoint rules ---------------------------------------------------------- +try: + knowledge.create_relation( + pid, source_entity_id=source["id"], target_entity_id="ent_missing", + relation_type="refines", relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], actor="t", note="n") + missing_ok = False +except EntityNotFound: + missing_ok = True +check("relation endpoint must exist", missing_ok) + +try: + knowledge.create_relation( + pid, source_entity_id=source["id"], target_entity_id=foreign["id"], + relation_type="refines", relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], actor="t", note="n") + cross_ok = False +except EntityNotFound: + cross_ok = True +check("cross-workspace endpoint is rejected", cross_ok) + +try: + knowledge.create_relation( + pid, source_entity_id=source["id"], target_entity_id=target["id"], + relation_type="depends-on", relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], actor="t", note="n") + unsupported_ok = False +except UnknownVocabularyValue: + unsupported_ok = True +check("unsupported relation type (depends-on) is rejected", unsupported_ok) + +try: + knowledge.create_relation( + pid, source_entity_id=source["id"], target_entity_id=source["id"], + relation_type="refines", relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], actor="t", note="n") + self_ok = False +except GraphConflict: + self_ok = True +check("self-relation is rejected", self_ok) + +try: + knowledge.create_relation( + pid, source_entity_id=source["id"], target_entity_id=target["id"], + relation_type="refines", relation_state="inferred", + evidence=[{"evidence_id": evidence_id}], actor="t", note="n") + inferred_ok = False +except InvalidRequest: + inferred_ok = True +check("manual 'inferred' state is rejected (no analyzer origin)", + inferred_ok) + +created = knowledge.create_relation( + pid, source_entity_id=source["id"], target_entity_id=target["id"], + relation_type="refines", relation_state="confirmed", + evidence=[{"evidence_id": evidence_id, + "quote": "shall answer within 2 seconds"}], + actor="tester", note="manual relation with evidence") +relation = created["relation"] +check("manual relation created as confirmed with origin=manual", + relation["relation_state"] == "confirmed" + and relation["origin"] == "manual") +check("relation carries its verified evidence join", + len(relation["evidence"]) == 1) + +# -- idempotent identity ----------------------------------------------------- +duplicate = knowledge.create_relation( + pid, source_entity_id=source["id"], target_entity_id=target["id"], + relation_type="refines", relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], actor="tester", note="dup") +check("duplicate active relation deduplicates to the existing row", + duplicate["deduplicated"] + and duplicate["relation"]["id"] == relation["id"]) +check("dedup minted no new knowledge revision", + duplicate["knowledge_revision"] == created["knowledge_revision"]) + +# -- possibly-related is never upgraded -------------------------------------- +weak = knowledge.create_relation( + pid, source_entity_id=target["id"], target_entity_id=source["id"], + relation_type="possibly-related", relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], actor="tester", note="weak") +check("possibly-related stays possibly-related", + weak["relation"]["relation_type"] == "possibly-related") +check("possibly-related and implements are distinct identity tuples", + store.find_active_relation(pid, target["id"], source["id"], + "implements") is None) + +# -- reject / restore -------------------------------------------------------- +rejected = knowledge.reject_relation(pid, relation_id=relation["id"], + actor="reviewer", note="not real") +check("rejected relation state applied", + store.get_relation(pid, relation["id"])["relation_state"] + == "rejected") +check("rejected relation is retained and queryable", + store.get_relation(pid, relation["id"]) is not None) +check("rejected relation keeps lifecycle active (governance history)", + store.get_relation(pid, relation["id"])["lifecycle_status"] + == "active") +listing = knowledge.list_relations(pid, relation_state="confirmed") +check("rejected relation excluded from confirmed-state listing", + all(r["id"] != relation["id"] for r in listing["relations"])) + +restored = knowledge.restore_relation(pid, relation_id=relation["id"], + actor="reviewer", note="was real") +check("restore returns the prior epistemic state", + restored["relation_state"] == "confirmed" + and store.get_relation(pid, relation["id"])["relation_state"] + == "confirmed") + +# -- supersede / withdraw ---------------------------------------------------- +replacement = knowledge.create_relation( + pid, source_entity_id=source["id"], target_entity_id=target["id"], + relation_type="implements", relation_state="explicit", + evidence=[{"evidence_id": evidence_id}], actor="tester", + note="replacement")["relation"] +knowledge.supersede_object(pid, kind="relation", object_id=relation["id"], + replacement_id=replacement["id"], actor="tester", + note="superseded in test") +old = store.get_relation(pid, relation["id"]) +check("superseded relation points at its replacement and stays queryable", + old["lifecycle_status"] == "superseded" + and old["superseded_by_relation_id"] == replacement["id"]) + +knowledge.withdraw_object(pid, kind="relation", + object_id=weak["relation"]["id"], actor="tester", + note="withdrawn") +check("withdrawn relation excluded from active listing", + all(r["id"] != weak["relation"]["id"] for r in + knowledge.list_relations(pid)["relations"])) + +finish() diff --git a/tests/verify_knowledge_revisions.py b/tests/verify_knowledge_revisions.py new file mode 100644 index 0000000..200a740 --- /dev/null +++ b/tests/verify_knowledge_revisions.py @@ -0,0 +1,127 @@ +"""Knowledge Revision ledger: monotonic per-workspace numbering, one +transaction = one revision, failed transaction = no revision, concurrency +uniqueness, accurate change counts, current revision on reads.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +import threading # noqa: E402 + +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_minimal_workspace) + +from openmind.knowledge import store # noqa: E402 +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) +evidence_id = find_evidence(pid, "REQ-NC-017") + +check("fresh workspace is at knowledge revision 0", + store.current_revision_number(pid) == 0) + +first = knowledge.create_entity( + pid, entity_type="requirement", canonical_key="requirement:REQ-NC-017", + display_name="REQ-NC-017", evidence=[{"evidence_id": evidence_id}], + actor="tester", note="first write") +check("first graph write creates revision 1", + first["knowledge_revision"] == 1) + +second = knowledge.create_entity( + pid, entity_type="requirement", canonical_key="requirement:REQ-NC-018", + display_name="REQ-NC-018", evidence=[{"evidence_id": evidence_id}], + actor="tester", note="second write") +check("next write creates revision 2", second["knowledge_revision"] == 2) + +ledger = knowledge.list_knowledge_revisions(pid)["revisions"] +check("ledger lists both revisions newest-first", + [r["revision_number"] for r in ledger] == [2, 1]) +check("revision records its action and actor", + ledger[0]["action"] == "manual-entity-create" + and ledger[0]["actor"] == "tester") +rev1 = knowledge.get_knowledge_revision(pid, 1) +check("revision 1 counts one entity, one binding, one decision", + rev1["object_counts"].get("entities") == 1 + and rev1["object_counts"].get("bindings") == 1 + and rev1["object_counts"].get("decisions") == 1) +check("revision 2 has parent revision 1", + knowledge.get_knowledge_revision(pid, 2)["parent_revision_number"] + == 1) + +# -- failed transaction creates no revision and no partial rows ------------- +before = store.current_revision_number(pid) +try: + with store.graph_transaction(pid, action="manual-entity-create", + actor="tester") as tx: + tx.insert_entity(entity_type="constraint", + canonical_key="constraint:HALF", + display_name="half", origin="manual") + raise RuntimeError("simulated mid-transaction failure") +except RuntimeError: + pass +check("failed transaction creates no knowledge revision", + store.current_revision_number(pid) == before) +check("failed transaction leaves no partial entity", + store.find_entity_by_key(pid, "constraint", "constraint:HALF") is None) + +# -- an empty transaction body writes nothing ------------------------------- +with store.graph_transaction(pid, action="graph-sync") as tx: + pass +check("a change-free transaction mints no revision", + store.current_revision_number(pid) == before) + +# -- concurrency: unique, monotonic, gap-free numbers ----------------------- +results = [] +errors = [] + + +def writer(index: int) -> None: + try: + result = knowledge.create_entity( + pid, entity_type="workflow", + canonical_key=f"workflow:derived:concurrent-{index}", + display_name=f"concurrent-{index}", + evidence=[{"evidence_id": evidence_id}], + actor=f"thread-{index}", note="concurrent write") + results.append(result["knowledge_revision"]) + except Exception as exc: # pragma: no cover - failure output + errors.append(repr(exc)) + + +threads = [threading.Thread(target=writer, args=(i,)) for i in range(8)] +for t in threads: + t.start() +for t in threads: + t.join() +check("concurrent writers all succeeded", not errors) +check("concurrent revision numbers are unique", + len(set(results)) == len(results) == 8) +check("concurrent revision numbers are gap-free and monotonic", + sorted(results) == list(range(before + 1, before + 9))) + +# -- reads carry the current revision --------------------------------------- +current = store.current_revision_number(pid) +check("stats reports the current knowledge revision", + knowledge.get_stats(pid)["knowledge_revision"] == current) +check("entity read reports the current knowledge revision", + knowledge.get_entity(pid, first["entity"]["id"]) + ["knowledge_revision"] == current) +check("search reports the current knowledge revision", + knowledge.search_entities(pid, "REQ-NC-017")["knowledge_revision"] + == current) + +# -- another workspace numbers independently -------------------------------- +other = make_minimal_workspace(runtime, name="kg-rev-other") +other_evidence = find_evidence(other, "REQ-NC-017") +other_first = knowledge.create_entity( + other, entity_type="requirement", + canonical_key="requirement:REQ-NC-017", display_name="REQ-NC-017", + evidence=[{"evidence_id": other_evidence}], actor="tester", note="w") +check("revision numbering is per-workspace", + other_first["knowledge_revision"] == 1) + +finish() diff --git a/tests/verify_knowledge_search.py b/tests/verify_knowledge_search.py new file mode 100644 index 0000000..95b67f8 --- /dev/null +++ b/tests/verify_knowledge_search.py @@ -0,0 +1,179 @@ +"""Graph search + vector projection: exact-over-similar precedence, stale +filtering, separate Entity/Claim sections, collection isolation and +lifecycle (merge removal, terminate/delete, orphan recognition).""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +from _knowledge_helpers import (check, finish, find_evidence, # noqa: E402 + make_minimal_workspace) + +from openmind import vectorstore # noqa: E402 +from openmind.knowledge import store, vector_projection # noqa: E402 +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) +evidence_id = find_evidence(pid, "REQ-NC-017") + +exact = knowledge.create_entity( + pid, entity_type="requirement", canonical_key="requirement:REQ-NC-017", + display_name="REQ-NC-017", evidence=[{"evidence_id": evidence_id}], + actor="t", note="fixture", + description="The name check service response budget.")["entity"] +knowledge.add_alias(pid, entity_id=exact["id"], alias="response budget rule", + alias_type="name", actor="t", note="a") +similar = knowledge.create_entity( + pid, entity_type="requirement", + canonical_key="requirement:REQ-NC-0170", display_name="REQ-NC-0170", + evidence=[{"evidence_id": evidence_id}], actor="t", note="fixture", + description="A requirement about answering name check queries fast — " + "reads like the other one but is not it.")["entity"] +claim = knowledge.create_claim( + pid, entity_id=exact["id"], claim_type="normative-statement", + statement="REQ-NC-017 requires an answer within 2 seconds.", + evidence=[{"evidence_id": evidence_id, + "quote": "shall answer within 2 seconds"}], + actor="t", note="claim")["claim"] + +# -- exact canonical key ------------------------------------------------------ +result = knowledge.search_entities(pid, "requirement:REQ-NC-017") +check("exact canonical key search ranks the key match first", + result["entities"] + and result["entities"][0]["id"] == exact["id"] + and result["entities"][0]["matched_via"] == "canonical-key") + +# -- exact alias -------------------------------------------------------------- +result = knowledge.search_entities(pid, "response budget rule") +check("exact alias search resolves through the alias index", + result["entities"] + and result["entities"][0]["id"] == exact["id"] + and result["entities"][0]["matched_via"] == "alias") + +# -- exact identifier outranks similar text ---------------------------------- +result = knowledge.search_entities(pid, "REQ-NC-017") +first = result["entities"][0] +check("exact identifier outranks the similar-looking entity", + first["id"] == exact["id"] + and (len(result["entities"]) == 1 + or result["entities"][1]["score"] < first["score"])) +check("REQ-NC-017 never matches REQ-NC-0170 as an exact token", + all(e["canonical_key"] != "requirement:REQ-NC-0170" + or e["matched_via"] in ("vector", "lexical") + for e in result["entities"])) + +# -- separate sections + navigability ---------------------------------------- +check("entities and claims are separate result sections", + isinstance(result["entities"], list) + and isinstance(result["claims"], list)) +check("claim hits carry evidence ids", + result["claims"] + and all(hit["evidence_ids"] for hit in result["claims"])) +check("entity hits carry navigable claim ids", + any(claim["id"] in hit["claim_ids"] for hit in result["entities"] + if hit["id"] == exact["id"])) +check("search result carries the current knowledge revision", + result["knowledge_revision"] + == store.current_revision_number(pid)) + +# -- stale filtering ---------------------------------------------------------- +knowledge.withdraw_object(pid, kind="entity", object_id=similar["id"], + actor="t", note="w") +result = knowledge.search_entities(pid, "name check") +check("withdrawn entity excluded from default search", + all(hit["id"] != similar["id"] for hit in result["entities"])) +result = knowledge.search_entities(pid, "name check", include_stale=True) +check("include_stale surfaces the withdrawn entity", + any(hit["id"] == similar["id"] for hit in result["entities"])) + +# -- vector projection isolation --------------------------------------------- +name = vector_projection.collection_name(pid) +check("graph collection is knowledge_", + name == f"knowledge_{pid}") +counts = vector_projection.refresh_workspace(pid) +check("entities and claims are indexed in the graph collection", + counts["indexed"] >= 2 + and vectorstore.count_collection(name) == counts["indexed"]) +collection = vectorstore.get_store(name) +stored = collection.get() +kinds = {(m or {}).get("object_kind") for m in stored["metadatas"]} +check("entity and claim projections are separate objects", + {"entity", "claim"} <= kinds) +check("projection metadata carries workspace scoping and lifecycle", + all((m or {}).get("workspace_id") == pid + and (m or {}).get("lifecycle_status") == "active" + for m in stored["metadatas"])) +check("no graph object entered the code collection", + not set(stored["ids"]) + & set(vectorstore.get_code_store(pid).get()["ids"])) +check("no graph object entered the documents collection", + not set(stored["ids"]) + & set(vectorstore.get_documents_store(pid).get()["ids"])) + +# changed claim updates the projection +knowledge.supersede_object( + pid, kind="claim", object_id=claim["id"], + replacement_id=knowledge.create_claim( + pid, entity_id=exact["id"], claim_type="normative-statement", + statement="REQ-NC-017 requires an answer within 3 seconds.", + evidence=[{"evidence_id": evidence_id}], actor="t", + note="replacement")["claim"]["id"], + actor="t", note="supersede") +stored = vectorstore.get_store(name).get() +check("superseded claim left the active projection", + claim["id"] not in stored["ids"]) + +# merge removes the source's active projection +merge_source = knowledge.create_entity( + pid, entity_type="requirement", canonical_key="requirement:REQ-MERGE", + display_name="REQ-MERGE", evidence=[{"evidence_id": evidence_id}], + actor="t", note="m")["entity"] +vector_projection.refresh_workspace(pid) +check("merge source is projected while active", + merge_source["id"] in vectorstore.get_store(name).get()["ids"]) +knowledge.merge_entities(pid, source_entity_id=merge_source["id"], + target_entity_id=exact["id"], actor="t", note="m") +check("merge removed the source's active projection", + merge_source["id"] not in vectorstore.get_store(name).get()["ids"]) + +# -- collection lifecycle ----------------------------------------------------- +check("knowledge_ prefix registered for orphan/delete handling", + f"knowledge_{pid}" in vectorstore.project_collection_names(pid)) +check("orphan sweep does not treat a live workspace's graph collection as " + "an orphan", + vectorstore.drop_orphan_collections({pid}) == []) + +from openmind import jobs # noqa: E402 +jobs.terminate_project(pid) +check("terminate dropped the graph collection", + vectorstore.count_collection(name) == 0) +check("terminate wiped the workspace graph rows", + store.stats(pid)["entities_total"] == 0 + and store.current_revision_number(pid) == 0) + +other = make_minimal_workspace(runtime, name="kg-search-del") +other_evidence = find_evidence(other, "REQ-NC-017") +knowledge.create_entity( + other, entity_type="requirement", + canonical_key="requirement:REQ-DEL", display_name="REQ-DEL", + evidence=[{"evidence_id": other_evidence}], actor="t", note="d") +vector_projection.refresh_workspace(other) +other_name = vector_projection.collection_name(other) +check("second workspace projected before delete", + vectorstore.count_collection(other_name) >= 1) +jobs.delete_project(other) +check("delete dropped the graph collection", + vectorstore.count_collection(other_name) == 0) + +# an orphaned graph collection IS recognized and swept +ghost = vectorstore.get_store("knowledge_p_ghost") +ghost.upsert(["g1"], [[0.0] * 8], ["ghost"], [{"workspace_id": "p_ghost"}]) +dropped = vectorstore.drop_orphan_collections({pid}) +check("startup orphan cleanup recognizes graph collections", + "knowledge_p_ghost" in dropped) + +finish() diff --git a/tests/verify_knowledge_staleness.py b/tests/verify_knowledge_staleness.py new file mode 100644 index 0000000..19f0ed2 --- /dev/null +++ b/tests/verify_knowledge_staleness.py @@ -0,0 +1,147 @@ +"""Graph staleness: revision movement stales bindings, claims and dependent +relations; manual entities with valid claims stay active; authority +preserved; the startup backstop repairs missed events.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _isolate # noqa: E402,F401 + +import tempfile # noqa: E402 +from pathlib import Path # noqa: E402 + +from _knowledge_helpers import (REQUIREMENTS_MD, check, # noqa: E402 + finish, find_evidence, + make_confirmed_candidate, + make_minimal_workspace) + +from openmind import db # noqa: E402 +from openmind.knowledge import store # noqa: E402 +from openmind.knowledge.reconciliation import ( # noqa: E402 + reconcile_graph_staleness) +from openmind.runtime import get_runtime # noqa: E402 + +runtime = get_runtime() +knowledge = runtime.knowledge +pid = make_minimal_workspace(runtime) +evidence_id = find_evidence(pid, "REQ-NC-017") + +# a promoted claim bound to the document's current revision +candidate_id = make_confirmed_candidate(runtime, pid) +promoted = knowledge.promote_candidate(pid, candidate_id, actor="r", + note="promoted before staleness") +entity_id = promoted["entity"]["id"] +claim_id = promoted["claim"]["id"] +knowledge.set_authority(pid, kind="claim", object_id=claim_id, + authority="authoritative", actor="lead", note="a") + +# a manual entity + claim on the same evidence, plus a relation between them +manual = knowledge.create_entity( + pid, entity_type="business-rule", canonical_key="business-rule:BR-9", + display_name="BR-9", evidence=[{"evidence_id": evidence_id}], + actor="t", note="manual")["entity"] +manual_claim = knowledge.create_claim( + pid, entity_id=manual["id"], claim_type="normative-statement", + statement="Failed name checks must be retried three times.", + evidence=[{"evidence_id": find_evidence(pid, "retried three times")}], + actor="t", note="manual claim")["claim"] +relation = knowledge.create_relation( + pid, source_entity_id=manual["id"], target_entity_id=entity_id, + relation_type="refines", relation_state="confirmed", + evidence=[{"evidence_id": evidence_id}], actor="t", + note="relation to be staled")["relation"] + +check("promoted claim starts active", + store.get_claim(pid, claim_id)["lifecycle_status"] == "active") + +# -- a NEW document revision stales the promoted knowledge ------------------- +docs = Path(tempfile.mkdtemp(prefix="om_kgstale_")) +changed = REQUIREMENTS_MD.replace("within 2 seconds", "within 5 seconds") +doc_asset = db.list_assets(pid, state="active", limit=10)[0] +path = docs / "requirements.md" +path.write_text(changed, encoding="utf-8") +result = runtime.documents.add_document(pid, str(path), + asset_id=doc_asset["id"], + wait=True, timeout=180) +check("fixture produced a new document revision", + result.get("status") == "revision") + +# the import hook already reconciled; run again to prove idempotence +outcome = reconcile_graph_staleness(pid) +claim_after = store.get_claim(pid, claim_id) +check("new document revision staled the promoted claim", + claim_after["lifecycle_status"] == "stale") +check("authority status preserved on the stale claim", + claim_after["authority_status"] == "authoritative") +check("stale claim remains queryable with its evidence", + len(claim_after["evidence"]) >= 1) +check("old-revision bindings went stale", + any(b["status"] == "stale" and b["ref_kind"] == "revision" + for b in store.list_bindings(pid, entity_id))) +check("stale claim staled the dependent relation", + store.get_relation(pid, relation["id"])["lifecycle_status"] + == "stale") +# The promoted ENTITY keeps its active asset binding (the document asset +# still exists), so it stays active as the identity anchor a re-analysis +# would re-attach to — only its CLAIM content went stale. +check("promoted entity stays active as an identity anchor (asset binding " + "still current)", + store.get_entity(pid, entity_id)["lifecycle_status"] == "active" + and any(b["status"] == "active" and b["ref_kind"] == "asset" + for b in store.list_bindings(pid, entity_id))) +check("manual entity's claim also staled (its evidence moved on) but the " + "record survives", + store.get_claim(pid, manual_claim["id"])["lifecycle_status"] + == "stale") + +# review history intact on the source candidate +check("confirmed review status preserved on the stale candidate", + runtime.semantic.get_candidate(pid, candidate_id)["review_status"] + == "confirmed") + +# -- manual entity with a VALID active claim stays active -------------------- +fresh_evidence = find_evidence(pid, "within 5 seconds") +active_manual = knowledge.create_entity( + pid, entity_type="decision", canonical_key="decision:D-1", + display_name="D-1", evidence=[{"evidence_id": fresh_evidence}], + actor="t", note="fresh manual")["entity"] +knowledge.create_claim( + pid, entity_id=active_manual["id"], claim_type="decision-rationale", + statement="The 5 second budget was chosen after load testing.", + evidence=[{"evidence_id": fresh_evidence}], actor="t", note="c") +reconcile_graph_staleness(pid) +check("manual entity with a valid current claim stays active", + store.get_entity(pid, active_manual["id"])["lifecycle_status"] + == "active") + +# -- second reconcile pass changes nothing (idempotent) ---------------------- +second = reconcile_graph_staleness(pid) +check("reconciliation is idempotent", + not second["changed"]) + +# -- the startup backstop repairs a missed event ----------------------------- +# simulate a missed event: force the stale claim active again directly +conn, lock = db.shared_connection() +with lock: + conn.execute("UPDATE engineering_claims SET lifecycle_status='active' " + "WHERE id=?", (claim_id,)) + conn.commit() +from openmind import jobs # noqa: E402 +jobs.reconcile_semantic_staleness(pid) # the worker-startup hook +check("startup reconciliation repaired the missed staleness event", + store.get_claim(pid, claim_id)["lifecycle_status"] == "stale") + +# -- reappearing evidence revives --------------------------------------------- +# revert the document to its original content: a NEW revision whose blocks +# match the original evidence text again... the ORIGINAL evidence rows still +# belong to the old revision, so the claim STAYS stale (revisions never +# become current again) — the honest contract. +path.write_text(REQUIREMENTS_MD, encoding="utf-8") +runtime.documents.add_document(pid, str(path), asset_id=doc_asset["id"], + wait=True, timeout=180) +reconcile_graph_staleness(pid) +check("a content revert does not falsely revive old-revision claims", + store.get_claim(pid, claim_id)["lifecycle_status"] == "stale") + +finish() diff --git a/tests/verify_migrations.py b/tests/verify_migrations.py index e53a4ad..3da7e90 100644 --- a/tests/verify_migrations.py +++ b/tests/verify_migrations.py @@ -57,10 +57,11 @@ def apply(conn): result = migrations.migrate(conn) tables = _tables(conn) -check("empty db: runner reports the head version", result.version == 5) +check("empty db: runner reports the head version", result.version == 6) check("empty db: every migration is applied in order", result.applied == ["0001_baseline", "0002_paths_sidecar", "0003_asset_model", - "0004_document_ingestion", "0005_semantic_plane"]) + "0004_document_ingestion", "0005_semantic_plane", + "0006_knowledge_graph"]) check("empty db: nothing was reported as already applied", result.already_applied == []) check("empty db: not flagged as a legacy baseline", result.baselined_legacy is False) check("empty db: schema_migrations ledger exists", "schema_migrations" in tables) @@ -116,7 +117,8 @@ def apply(conn): check("repeat run: reports every migration as already applied", again.already_applied == ["0001_baseline", "0002_paths_sidecar", "0003_asset_model", "0004_document_ingestion", - "0005_semantic_plane"]) + "0005_semantic_plane", + "0006_knowledge_graph"]) check("repeat run: version is unchanged", again.version == result.version) # --------------------------------------------------------------------------- @@ -166,7 +168,7 @@ def apply(conn): legacy_result = migrations.migrate(legacy) check("legacy db: reported as a legacy baseline", legacy_result.baselined_legacy is True) -check("legacy db: brought to head version", legacy_result.version == 5) +check("legacy db: brought to head version", legacy_result.version == 6) check("legacy db: baseline recorded, not skipped", "0001_baseline" in legacy_result.applied) check("legacy db: v0003 applied on top of the baseline", @@ -185,6 +187,12 @@ def apply(conn): check("legacy db: v0005 semantic tables created on the legacy database", {"semantic_candidates", "semantic_cache", "project_lenses", "workspace_semantic_policies"} <= _tables(legacy)) +check("legacy db: v0006 applied on top of the baseline", + "0006_knowledge_graph" in legacy_result.applied) +check("legacy db: v0006 graph tables created on the legacy database", + {"engineering_entities", "engineering_claims", "engineering_relations", + "knowledge_decisions", "knowledge_revisions", + "knowledge_promotions"} <= _tables(legacy)) check("legacy db: project row survived", legacy.execute("SELECT COUNT(*) FROM projects").fetchone()[0] == 1) check("legacy db: project meta survived intact", diff --git a/tests/verify_semantic_adapters.py b/tests/verify_semantic_adapters.py index 77b2eb4..63f2fb9 100644 --- a/tests/verify_semantic_adapters.py +++ b/tests/verify_semantic_adapters.py @@ -59,7 +59,11 @@ import asyncio # noqa: E402 server = mcp_server.create_mcp_server() registered = asyncio.new_event_loop().run_until_complete(server.list_tools()) -check("FastMCP registers all 26", len(registered) == 26) +# Phase 5 registers nine additive read-only graph tools beside these 26; +# the graph suite (verify_knowledge_adapters) accounts for each by name. +check("FastMCP registers the 26 pre-Phase-5 tools (35 with the graph set)", + len(registered) == 26 + len(mcp_server.KNOWLEDGE_TOOLS) + and len(mcp_server.KNOWLEDGE_TOOLS) == 9) # --------------------------------------------------------------------------- # 2. REST: every legacy route intact; semantic routes additive