From c6fd18d9208a7a48b490591d740fce0ac13fa0b5 Mon Sep 17 00:00:00 2001 From: dale053 Date: Wed, 8 Jul 2026 10:16:08 -0400 Subject: [PATCH] feat(import): add conversation-export import command seed prior agent history into the review queue without bypassing approval. this adds format-specific importers with dry-run, proposal caps, and dedup against approved claims. Co-authored-by: Cursor --- CHANGELOG.md | 8 + src/vouch/cli.py | 78 ++++++++ src/vouch/corpus_import.py | 398 +++++++++++++++++++++++++++++++++++++ tests/test_import.py | 192 ++++++++++++++++++ 4 files changed, 676 insertions(+) create mode 100644 src/vouch/corpus_import.py create mode 100644 tests/test_import.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 523eb817..93bfa4d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ All notable changes to vouch are documented here. Format follows ## [Unreleased] +### Added +- `vouch import ` — conversation-export importers that parse + chat-json, markdown-vault, or memory-export dumps and file pending + proposals via the existing review gate. supports `--dry-run` for counts + without enqueuing and `--max-proposals` to cap a single run. imported + claims are de-duped against approved artifacts via propose-time + similarity (#431). + ## [1.1.0] — 2026-07-03 ### Added diff --git a/src/vouch/cli.py b/src/vouch/cli.py index f8b4a0c8..b49d5c5c 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -24,6 +24,7 @@ from . import __version__, bundle, health, volunteer_context from . import audit as audit_mod from . import capture as capture_mod +from . import corpus_import as corpus_import_mod from . import install_adapter as install_mod from . import lifecycle as life from . import metrics as metrics_mod @@ -2038,6 +2039,83 @@ def export(out_path: str) -> None: ) +@cli.command("import") +@click.argument( + "format", + type=click.Choice(list(corpus_import_mod.IMPORT_FORMATS)), +) +@click.argument("path", type=click.Path(exists=True)) +@click.option( + "--dry-run", + is_flag=True, + help="Report would-propose counts without enqueuing proposals.", +) +@click.option( + "--max-proposals", + type=int, + default=None, + help="Cap proposals filed in one run (claims + pages combined).", +) +@click.option("--json", "as_json", is_flag=True, help="Emit machine-readable summary.") +def import_cmd( + format: str, + path: str, + dry_run: bool, + max_proposals: int | None, + as_json: bool, +) -> None: + """Import conversation exports as review-gated proposals (#431). + + Formats: chat-json, markdown-vault, memory-export. Every item lands in + ``.vouch/proposed/`` — nothing is auto-approved. + """ + store = _load_store() + try: + result = corpus_import_mod.run_import( + store, + format, # type: ignore[arg-type] + Path(path), + dry_run=dry_run, + max_proposals=max_proposals, + actor=_whoami(), + ) + except corpus_import_mod.CorpusImportError as e: + raise click.ClickException(str(e)) from e + + payload = { + "format": result.format, + "path": result.path, + "dry_run": result.dry_run, + "claims_proposed": result.claims_proposed, + "pages_proposed": result.pages_proposed, + "claims_skipped_dedup": result.claims_skipped_dedup, + "pages_skipped_dedup": result.pages_skipped_dedup, + "cap_hit": result.cap_hit, + "proposal_ids": result.proposal_ids, + "warnings": result.warnings, + } + if as_json: + _emit_json(payload) + return + + mode = "dry-run" if dry_run else "import" + click.echo( + f"{mode}: {result.claims_proposed} claim(s), " + f"{result.pages_proposed} page(s) proposed" + ) + if result.claims_skipped_dedup or result.pages_skipped_dedup: + click.echo( + f" skipped dedup: {result.claims_skipped_dedup} claim(s), " + f"{result.pages_skipped_dedup} page(s)" + ) + if result.cap_hit: + click.echo(" cap hit: --max-proposals limit reached") + for pid in result.proposal_ids: + click.echo(f" + {pid}") + for w in result.warnings: + click.echo(f" warning: {w}", err=True) + + @cli.command("export-check") @click.argument("bundle_path", type=click.Path(exists=True, dir_okay=False)) def export_check_cmd(bundle_path: str) -> None: diff --git a/src/vouch/corpus_import.py b/src/vouch/corpus_import.py new file mode 100644 index 00000000..bf7ae3fc --- /dev/null +++ b/src/vouch/corpus_import.py @@ -0,0 +1,398 @@ +"""Conversation-export importers — seed the review queue from prior agent history. + +Each format reader normalizes an export dump into candidate claims/pages and +routes them through ``proposals.propose_*``. Nothing is auto-approved; the +review gate is unchanged (vouchdev/vouch#431). +""" + +from __future__ import annotations + +import json +import logging +import re +from collections.abc import Iterator +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +import yaml + +from .models import ProposalKind, ProposalStatus +from .proposals import ProposeClaimResult, propose_claim, propose_page +from .storage import KBStore + +log = logging.getLogger(__name__) + +ImportFormat = Literal["chat-json", "markdown-vault", "memory-export"] +IMPORT_FORMATS: tuple[ImportFormat, ...] = ( + "chat-json", + "markdown-vault", + "memory-export", +) + +_FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---\n?(.*)$", re.DOTALL) +_CLAIM_MAX_CHARS = 500 +_IMPORT_ACTOR = "vouch-import" + + +class CorpusImportError(RuntimeError): + """Raised when an export path or payload cannot be parsed.""" + + +@dataclass +class ImportCandidate: + """Normalized item ready for propose-time routing.""" + + kind: Literal["claim", "page"] + text: str + title: str | None = None + body: str | None = None + slug_hint: str | None = None + tags: list[str] = field(default_factory=list) + rationale: str | None = None + source_path: Path | None = None + + +@dataclass +class ImportResult: + """Outcome of one ``run_import`` call.""" + + format: ImportFormat + path: str + dry_run: bool + claims_proposed: int = 0 + pages_proposed: int = 0 + claims_skipped_dedup: int = 0 + pages_skipped_dedup: int = 0 + cap_hit: bool = False + proposal_ids: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + +def _slugify(text: str) -> str: + out: list[str] = [] + last_dash = False + for ch in text.lower().strip(): + if ch.isalnum(): + out.append(ch) + last_dash = False + elif not last_dash: + out.append("-") + last_dash = True + slug = "".join(out).strip("-") + return slug[:60] or "untitled" + + +# --- shared markdown extraction (#321 core) --------------------------------- + + +def iter_markdown_files(root: Path) -> Iterator[Path]: + """Yield ``*.md`` files under *root*, sorted for deterministic runs.""" + if root.is_file(): + if root.suffix.lower() == ".md": + yield root + return + for path in sorted(root.rglob("*.md")): + if path.is_file(): + yield path + + +def parse_markdown_file(path: Path) -> tuple[str, str, dict[str, Any]]: + """Return ``(title, body, frontmatter)`` for one markdown file.""" + text = path.read_text(encoding="utf-8").replace("\r\n", "\n") + m = _FRONTMATTER_RE.match(text) + if m: + meta = yaml.safe_load(m.group(1)) or {} + if not isinstance(meta, dict): + meta = {} + body = m.group(2) + title = str(meta.get("title") or path.stem) + return title, body, meta + return path.stem, text, {} + + +def candidates_from_markdown_vault(root: Path) -> list[ImportCandidate]: + out: list[ImportCandidate] = [] + for md_path in iter_markdown_files(root): + title, body, meta = parse_markdown_file(md_path) + slug = meta.get("id") + slug_hint = str(slug) if slug else _slugify(title) + tags_raw = meta.get("tags") + tags = [str(t) for t in tags_raw] if isinstance(tags_raw, list) else [] + out.append(ImportCandidate( + kind="page", + text=title, + title=title, + body=body, + slug_hint=slug_hint, + tags=tags, + rationale=f"imported from {md_path.name}", + source_path=md_path, + )) + return out + + +# --- chat-json -------------------------------------------------------------- + + +def _message_content(raw: Any) -> str: + if raw is None: + return "" + if isinstance(raw, str): + return raw.strip() + if isinstance(raw, list): + parts: list[str] = [] + for item in raw: + if isinstance(item, dict): + text = item.get("text") or item.get("content") + if isinstance(text, str): + parts.append(text) + elif isinstance(item, str): + parts.append(item) + return "\n".join(p for p in parts if p).strip() + return str(raw).strip() + + +def _iter_chat_messages(data: Any) -> Iterator[tuple[str | None, dict[str, Any]]]: + """Yield ``(conversation_title, message)`` from supported chat-json shapes.""" + if isinstance(data, list): + for msg in data: + if isinstance(msg, dict): + yield None, msg + return + if not isinstance(data, dict): + return + if isinstance(data.get("messages"), list): + for msg in data["messages"]: + if isinstance(msg, dict): + yield str(data["title"]) if data.get("title") else None, msg + return + conversations = data.get("conversations") + if isinstance(conversations, list): + for conv in conversations: + if not isinstance(conv, dict): + continue + title = conv.get("title") + messages = conv.get("messages") + if isinstance(messages, list): + for msg in messages: + if isinstance(msg, dict): + yield str(title) if title else None, msg + + +def candidates_from_chat_json(path: Path) -> list[ImportCandidate]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + raise CorpusImportError(f"cannot parse chat-json at {path}: {e}") from e + + out: list[ImportCandidate] = [] + idx = 0 + for conv_title, msg in _iter_chat_messages(data): + role = str(msg.get("role") or msg.get("author") or "").lower() + if role not in {"assistant", "model", "ai"}: + continue + content = _message_content(msg.get("content") or msg.get("text")) + if not content: + continue + idx += 1 + title_base = conv_title or f"message-{idx}" + if len(content) <= _CLAIM_MAX_CHARS: + out.append(ImportCandidate( + kind="claim", + text=content, + rationale=f"imported assistant message from {path.name}", + )) + else: + first_line = content.split("\n", 1)[0].strip("# ").strip() or title_base + out.append(ImportCandidate( + kind="page", + text=first_line[:120], + title=first_line[:120], + body=content, + rationale=f"imported assistant message from {path.name}", + )) + if not out: + raise CorpusImportError(f"no assistant messages found in chat-json: {path}") + return out + + +# --- memory-export ---------------------------------------------------------- + + +def _iter_memory_entries(data: Any) -> Iterator[str]: + if isinstance(data, list): + for item in data: + if isinstance(item, str) and item.strip(): + yield item.strip() + elif isinstance(item, dict): + text = item.get("text") or item.get("content") or item.get("memory") + if isinstance(text, str) and text.strip(): + yield text.strip() + return + if not isinstance(data, dict): + return + for key in ("memories", "entries", "items", "data"): + bucket = data.get(key) + if isinstance(bucket, list): + yield from _iter_memory_entries(bucket) + return + + +def candidates_from_memory_export(path: Path) -> list[ImportCandidate]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + raise CorpusImportError(f"cannot parse memory-export at {path}: {e}") from e + + out = [ + ImportCandidate( + kind="claim", + text=text, + rationale=f"imported memory from {path.name}", + ) + for text in _iter_memory_entries(data) + ] + if not out: + raise CorpusImportError(f"no memory entries found in {path}") + return out + + +def load_candidates(format: ImportFormat, path: Path) -> list[ImportCandidate]: + if format == "markdown-vault": + candidates = candidates_from_markdown_vault(path) + if not candidates: + raise CorpusImportError(f"no markdown files found under {path}") + return candidates + if format == "chat-json": + return candidates_from_chat_json(path) + return candidates_from_memory_export(path) + + +# --- dedup + propose routing ------------------------------------------------ + + +def _claim_is_duplicate(store: KBStore, text: str) -> bool: + try: + from .embeddings.similarity import find_similar_on_propose + + warnings = find_similar_on_propose(store, text) + except ImportError: + return False + return any(w.get("code") == "similar_approved" for w in warnings) + + +def _page_is_duplicate(store: KBStore, slug: str) -> bool: + if (store.kb_dir / "pages" / f"{slug}.md").exists(): + return True + for prop in store.list_proposals(ProposalStatus.PENDING): + if prop.kind == ProposalKind.PAGE and prop.payload.get("id") == slug: + return True + return False + + +def _register_file_source(store: KBStore, file_path: Path, *, root: Path | None = None) -> str: + body = file_path.read_bytes() + locator: str + if root is not None: + try: + rel = file_path.resolve().relative_to(root.resolve()) + locator = f"import:{rel.as_posix()}" + except ValueError: + locator = f"import:{file_path.name}" + else: + locator = f"import:{file_path.resolve()}" + media = "application/json" if file_path.suffix.lower() == ".json" else "text/markdown" + src = store.put_source( + body, + title=file_path.name, + locator=locator, + source_type="file", + media_type=media, + ) + return src.id + + +def run_import( + store: KBStore, + format: ImportFormat, + path: Path, + *, + dry_run: bool = False, + max_proposals: int | None = None, + actor: str = _IMPORT_ACTOR, +) -> ImportResult: + """Parse *path* and enqueue proposals via ``propose_*`` (never approve).""" + resolved = path.resolve() + if not resolved.exists(): + raise CorpusImportError(f"path not found: {path}") + + candidates = load_candidates(format, resolved) + result = ImportResult(format=format, path=str(resolved), dry_run=dry_run) + + json_source_id: str | None = None + if format in {"chat-json", "memory-export"}: + json_source_id = _register_file_source(store, resolved) + + vault_root = resolved if resolved.is_dir() else resolved.parent + proposed = 0 + cap = max_proposals if max_proposals is not None and max_proposals >= 0 else None + + for candidate in candidates: + if cap is not None and proposed >= cap: + result.cap_hit = True + break + + if candidate.kind == "claim": + if _claim_is_duplicate(store, candidate.text): + result.claims_skipped_dedup += 1 + continue + if json_source_id is None: + raise CorpusImportError("internal error: json import missing source id") + claim_result: ProposeClaimResult = propose_claim( + store, + text=candidate.text, + evidence=[json_source_id], + proposed_by=actor, + tags=candidate.tags, + rationale=candidate.rationale, + slug_hint=candidate.slug_hint, + dry_run=dry_run, + ) + result.proposal_ids.append(claim_result.id) + for w in claim_result.warnings: + if w.get("code") == "similar_pending": + result.warnings.append( + f"claim similar to pending {w.get('artifact_id')}" + ) + result.claims_proposed += 1 + proposed += 1 + continue + + title = candidate.title or candidate.text + slug = candidate.slug_hint or _slugify(title) + if _page_is_duplicate(store, slug): + result.pages_skipped_dedup += 1 + continue + + source_ids: list[str] = [] + if candidate.source_path is not None: + source_ids = [_register_file_source(store, candidate.source_path, root=vault_root)] + + page = propose_page( + store, + title=title, + body=candidate.body or "", + page_type="concept", + source_ids=source_ids, + tags=candidate.tags, + rationale=candidate.rationale, + slug_hint=slug, + proposed_by=actor, + dry_run=dry_run, + ) + result.proposal_ids.append(page.id) + result.pages_proposed += 1 + proposed += 1 + + return result diff --git a/tests/test_import.py b/tests/test_import.py new file mode 100644 index 00000000..07835207 --- /dev/null +++ b/tests/test_import.py @@ -0,0 +1,192 @@ +"""Conversation-export importers (vouchdev/vouch#431).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from vouch.cli import cli +from vouch.corpus_import import ( + candidates_from_chat_json, + candidates_from_markdown_vault, + run_import, +) +from vouch.models import Claim, Page, PageStatus, PageType, ProposalStatus +from vouch.storage import KBStore + +_SIMILAR = "Auth uses JWTs in the Authorization header." + + +@pytest.fixture +def store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> KBStore: + s = KBStore.init(tmp_path) + monkeypatch.chdir(s.root) + return s + + +def _install_mock_embedder() -> None: + pytest.importorskip("numpy") + from tests.embeddings._fakes import MockEmbedder + from vouch.embeddings import register + from vouch.embeddings.base import DEFAULT_MODEL_NAME + + register(DEFAULT_MODEL_NAME, lambda: MockEmbedder(dim=8)) + + +def test_markdown_vault_proposes_pages_only(store: KBStore, tmp_path: Path) -> None: + vault = tmp_path / "vault" + vault.mkdir() + (vault / "alpha.md").write_text( + "---\nid: alpha-page\ntitle: Alpha\n---\n\n# Alpha\n\nBody text.\n", + encoding="utf-8", + ) + (vault / "beta.md").write_text("# Beta\n\nNo frontmatter.\n", encoding="utf-8") + + result = run_import(store, "markdown-vault", vault, actor="tester") + + assert result.pages_proposed == 2 + assert result.claims_proposed == 0 + assert len(result.proposal_ids) == 2 + pending = store.list_proposals(ProposalStatus.PENDING) + assert len(pending) == 2 + assert not list(store.list_pages()) + assert not list(store.list_claims()) + + +def test_chat_json_short_messages_become_claims(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "chat.json" + export.write_text( + json.dumps([ + {"role": "user", "content": "What is auth?"}, + {"role": "assistant", "content": "We use JWT bearer tokens."}, + {"role": "assistant", "content": "Refresh tokens rotate hourly."}, + ]), + encoding="utf-8", + ) + + result = run_import(store, "chat-json", export, actor="tester") + + assert result.claims_proposed == 2 + assert result.pages_proposed == 0 + assert len(store.list_proposals(ProposalStatus.PENDING)) == 2 + assert not store.list_claims() + + +def test_dry_run_reports_without_enqueuing(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "mem.json" + export.write_text(json.dumps(["fact one", "fact two"]), encoding="utf-8") + + result = run_import(store, "memory-export", export, dry_run=True, actor="tester") + + assert result.dry_run is True + assert result.claims_proposed == 2 + assert store.list_proposals(ProposalStatus.PENDING) == [] + + +def test_max_proposals_cap(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "mem.json" + export.write_text( + json.dumps(["one", "two", "three", "four"]), + encoding="utf-8", + ) + + result = run_import( + store, "memory-export", export, max_proposals=2, actor="tester", + ) + + assert result.claims_proposed == 2 + assert result.cap_hit is True + assert len(store.list_proposals(ProposalStatus.PENDING)) == 2 + + +def test_claim_dedup_skips_similar_approved(store: KBStore, tmp_path: Path) -> None: + _install_mock_embedder() + src = store.put_source(b"e") + store.put_claim(Claim(id="auth-jwt", text=_SIMILAR, evidence=[src.id])) + + export = tmp_path / "mem.json" + export.write_text(json.dumps([_SIMILAR, "Unrelated new fact."]), encoding="utf-8") + + result = run_import(store, "memory-export", export, actor="tester") + + assert result.claims_proposed == 1 + assert result.claims_skipped_dedup == 1 + pending = store.list_proposals(ProposalStatus.PENDING) + assert len(pending) == 1 + assert pending[0].payload["text"] == "Unrelated new fact." + + +def test_page_dedup_skips_existing_slug(store: KBStore, tmp_path: Path) -> None: + src = store.put_source(b"x", title="x") + store.put_page(Page( + id="alpha-page", + title="Alpha", + body="existing", + type=PageType.CONCEPT, + status=PageStatus.ACTIVE, + sources=[src.id], + )) + + vault = tmp_path / "vault" + vault.mkdir() + (vault / "dup.md").write_text( + "---\nid: alpha-page\ntitle: Alpha copy\n---\n\nNew body.\n", + encoding="utf-8", + ) + (vault / "fresh.md").write_text("---\ntitle: Fresh\n---\n\nNew page.\n", encoding="utf-8") + + result = run_import(store, "markdown-vault", vault, actor="tester") + + assert result.pages_proposed == 1 + assert result.pages_skipped_dedup == 1 + pending = store.list_proposals(ProposalStatus.PENDING) + assert len(pending) == 1 + assert pending[0].payload["id"] == "fresh" + + +def test_candidates_from_chat_json_parses_messages_key(tmp_path: Path) -> None: + path = tmp_path / "c.json" + path.write_text( + json.dumps({ + "title": "Session", + "messages": [ + {"role": "assistant", "content": "Short answer."}, + ], + }), + encoding="utf-8", + ) + cands = candidates_from_chat_json(path) + assert len(cands) == 1 + assert cands[0].kind == "claim" + assert cands[0].text == "Short answer." + + +def test_candidates_from_markdown_vault(tmp_path: Path) -> None: + root = tmp_path / "v" + root.mkdir() + (root / "note.md").write_text("# Title\n\nBody.\n", encoding="utf-8") + cands = candidates_from_markdown_vault(root) + assert len(cands) == 1 + assert cands[0].kind == "page" + assert cands[0].title == "note" + + +def test_cli_import_json_output(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "chat.json" + export.write_text( + json.dumps([{"role": "assistant", "content": "CLI claim."}]), + encoding="utf-8", + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["import", "chat-json", str(export), "--json"], + ) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["claims_proposed"] == 1 + assert payload["pages_proposed"] == 0 + assert len(payload["proposal_ids"]) == 1