From e44af952326f595d3ff3d0366db593793a0f3830 Mon Sep 17 00:00:00 2001 From: Michael Kuehl Date: Thu, 23 Apr 2026 13:26:24 +0200 Subject: [PATCH] feat(verifier): add structured verification checklist and task completion enforcement (fixes #521) --- agent_fox/_templates/profiles/verifier.md | 25 ++ agent_fox/engine/session_lifecycle.py | 1 + agent_fox/session/context.py | 23 ++ agent_fox/spec/verification_checklist.py | 285 +++++++++++++++++ docs/memory.md | 9 + tests/unit/session/test_context_verifier.py | 120 +++++++ .../unit/spec/test_verification_checklist.py | 302 ++++++++++++++++++ 7 files changed, 765 insertions(+) create mode 100644 agent_fox/spec/verification_checklist.py create mode 100644 tests/unit/session/test_context_verifier.py create mode 100644 tests/unit/spec/test_verification_checklist.py diff --git a/agent_fox/_templates/profiles/verifier.md b/agent_fox/_templates/profiles/verifier.md index 156af543..b89f2e3c 100644 --- a/agent_fox/_templates/profiles/verifier.md +++ b/agent_fox/_templates/profiles/verifier.md @@ -24,10 +24,35 @@ Treat this file as executable workflow policy. - Reference specific requirement IDs in your assessment. - Output bare JSON only — no markdown fences, no surrounding prose. +## Verification Checklist + +Your context includes a **Verification Checklist** section with two tables: + +1. **Task Completion Audit** — every subtask checkbox from tasks.md with its + current state. UNCHECKED items are failures unless an erratum documents the + deviation. +2. **Requirement-to-Test Coverage** — maps each requirement ID to test files + that reference it. UNCOVERED requirements are critical findings. + +Use this checklist as your primary verification structure. Walk through every +row and confirm or reject each item. + +### Hard gates + +- If any subtask is **UNCHECKED** and no erratum covers it → **FAIL** verdict + for the corresponding requirement. +- If any requirement is **UNCOVERED** (no test references it) → **FAIL** + verdict for that requirement. +- SKIPPED subtasks (marked `[-]` or `[~]`) are intentional and do not trigger + failure. + ## Focus Areas - **Requirements coverage:** For each requirement in scope, confirm it is implemented and matches the acceptance criteria, including edge cases. + Cross-reference the Requirement-to-Test Coverage table. +- **Task completion:** Verify every subtask checkbox is checked. For unchecked + items, check whether an erratum in `docs/errata/` documents the deviation. - **Test execution:** Run spec tests for the task group first, then the full suite to check for regressions. - **Code quality:** Does the implementation follow the design document's diff --git a/agent_fox/engine/session_lifecycle.py b/agent_fox/engine/session_lifecycle.py index 4a0f770f..91311279 100644 --- a/agent_fox/engine/session_lifecycle.py +++ b/agent_fox/engine/session_lifecycle.py @@ -279,6 +279,7 @@ def _build_prompts( memory_facts=memory_facts, conn=self._knowledge_db.connection, project_root=Path.cwd(), + archetype=self._archetype, ) system_prompt = build_system_prompt( diff --git a/agent_fox/session/context.py b/agent_fox/session/context.py index 76340cdd..ebeacdf9 100644 --- a/agent_fox/session/context.py +++ b/agent_fox/session/context.py @@ -281,6 +281,7 @@ def assemble_context( *, conn: duckdb.DuckDBPyConnection, project_root: Path | None = None, + archetype: str | None = None, ) -> str: """Assemble task-specific context for a coding session. @@ -300,6 +301,9 @@ def assemble_context( .specs/steering.md after spec files and before memory facts (64-REQ-2.1, 64-REQ-2.2). + When archetype is ``"verifier"``, appends a structured verification + checklist (task completion audit + requirement-to-test coverage). + Returns a formatted string with section headers. Logs a warning for any missing spec file but does not raise. @@ -392,6 +396,25 @@ def assemble_context( task_group, ) + # Verification checklist for the verifier archetype + if archetype == "verifier": + try: + from agent_fox.spec.verification_checklist import ( + build_verification_checklist, + render_checklist_markdown, + ) + + tests_dir = project_root / "tests" if project_root is not None else None + checklist = build_verification_checklist(spec_dir, conn, tests_dir=tests_dir) + checklist_md = render_checklist_markdown(checklist) + sections.append(checklist_md) + except Exception: + logger.warning( + "Failed to build verification checklist for %s", + spec_dir.name, + exc_info=True, + ) + # 03-REQ-4.3: Return formatted string with section headers return "\n\n---\n\n".join(sections) diff --git a/agent_fox/spec/verification_checklist.py b/agent_fox/spec/verification_checklist.py new file mode 100644 index 00000000..758c9f63 --- /dev/null +++ b/agent_fox/spec/verification_checklist.py @@ -0,0 +1,285 @@ +"""Verification checklist builder for the verifier archetype. + +Builds a structured checklist from tasks.md checkboxes, requirements.md +acceptance criteria, and errata — injected into the verifier's session +context so it can enforce task completion and requirement coverage. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from agent_fox.spec._patterns import REQ_ID_BARE +from agent_fox.spec.parser import parse_tasks + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class SubtaskAuditEntry: + """Audit entry for a single subtask checkbox.""" + + group_number: int + subtask_id: str + title: str + checked: bool + skipped: bool # [-] or [~] markers + + +@dataclass(frozen=True) +class RequirementMapping: + """Maps a requirement ID to its test coverage status.""" + + requirement_id: str + covered: bool + test_files: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class VerificationChecklist: + """Complete verification checklist for a spec.""" + + spec_name: str + task_audit: list[SubtaskAuditEntry] + requirement_coverage: list[RequirementMapping] + has_errata: bool + + +def build_verification_checklist( + spec_dir: Path, + conn: Any, + *, + tests_dir: Path | None = None, +) -> VerificationChecklist: + """Build a verification checklist from spec files and DB state. + + Args: + spec_dir: Path to the spec directory (e.g. .agent-fox/specs/10_my_spec). + conn: DuckDB connection for errata queries. + tests_dir: Path to the tests directory for requirement-to-test scanning. + + Returns: + A populated VerificationChecklist. + """ + spec_name = spec_dir.name + + task_audit = _audit_task_checkboxes(spec_dir) + has_errata = _check_errata_exist(conn, spec_name) + requirement_coverage = scan_requirement_test_coverage(spec_dir, tests_dir) + + return VerificationChecklist( + spec_name=spec_name, + task_audit=task_audit, + requirement_coverage=requirement_coverage, + has_errata=has_errata, + ) + + +def _audit_task_checkboxes(spec_dir: Path) -> list[SubtaskAuditEntry]: + """Parse tasks.md and audit every subtask checkbox state.""" + tasks_path = spec_dir / "tasks.md" + if not tasks_path.is_file(): + return [] + + try: + groups = parse_tasks(tasks_path) + except Exception: + logger.warning("Failed to parse tasks.md for checklist audit in %s", spec_dir) + return [] + + entries: list[SubtaskAuditEntry] = [] + for group in groups: + for subtask in group.subtasks: + entries.append( + SubtaskAuditEntry( + group_number=group.number, + subtask_id=subtask.id, + title=subtask.title, + checked=subtask.completed, + skipped=_is_subtask_skipped(tasks_path, subtask.id), + ) + ) + return entries + + +_SUBTASK_SKIP_PATTERN = re.compile(r"^\s+- \[([~\-])\] (\d+\.(?:\d+|V))") + + +def _is_subtask_skipped(tasks_path: Path, subtask_id: str) -> bool: + """Check if a subtask is marked with [-] or [~] (intentionally skipped).""" + text = tasks_path.read_text(encoding="utf-8") + for line in text.splitlines(): + m = _SUBTASK_SKIP_PATTERN.match(line) + if m and m.group(2) == subtask_id: + return True + return False + + +def _check_errata_exist(conn: Any, spec_name: str) -> bool: + """Check if any errata exist for this spec in the DB.""" + try: + row = conn.execute( + "SELECT COUNT(*) FROM errata WHERE spec_name = ?", + [spec_name], + ).fetchone() + return row is not None and row[0] > 0 + except Exception: + logger.debug("Could not query errata for %s", spec_name) + return False + + +def scan_requirement_test_coverage( + spec_dir: Path, + tests_dir: Path | None = None, +) -> list[RequirementMapping]: + """Map requirement IDs to test file coverage. + + For each requirement ID found in requirements.md, scans test files + for references (in comments, docstrings, or function names). + + Args: + spec_dir: Path to the spec directory containing requirements.md. + tests_dir: Path to the project's tests directory. If None or + non-existent, all requirements are marked uncovered. + + Returns: + List of RequirementMapping, one per requirement ID. + """ + req_path = spec_dir / "requirements.md" + if not req_path.is_file(): + return [] + + req_text = req_path.read_text(encoding="utf-8") + req_ids = sorted(set(REQ_ID_BARE.findall(req_text))) + if not req_ids: + return [] + + test_content = _load_test_file_contents(tests_dir) + + mappings: list[RequirementMapping] = [] + for req_id in req_ids: + test_files = _find_test_files_for_req(req_id, test_content) + mappings.append( + RequirementMapping( + requirement_id=req_id, + covered=len(test_files) > 0, + test_files=test_files, + ) + ) + return mappings + + +def _load_test_file_contents(tests_dir: Path | None) -> dict[str, str]: + """Load all test file contents into a dict keyed by relative path.""" + if tests_dir is None or not tests_dir.is_dir(): + return {} + contents: dict[str, str] = {} + for test_file in tests_dir.rglob("test_*.py"): + try: + contents[test_file.name] = test_file.read_text(encoding="utf-8") + except OSError: + continue + return contents + + +def _normalize_req_id_for_funcname(req_id: str) -> str: + """Convert '10-REQ-1.1' to 'req_10_1_1' for function name matching.""" + without_prefix = re.sub(r"^(\d+)-REQ-", r"req_\1_", req_id) + return without_prefix.replace(".", "_").replace("-", "_").lower() + + +def _find_test_files_for_req( + req_id: str, + test_content: dict[str, str], +) -> list[str]: + """Find test files that reference a requirement ID.""" + normalized = _normalize_req_id_for_funcname(req_id) + matching: list[str] = [] + for filename, content in test_content.items(): + if req_id in content or normalized in content: + matching.append(filename) + return sorted(matching) + + +def render_checklist_markdown(checklist: VerificationChecklist) -> str: + """Render a verification checklist as markdown for context injection.""" + lines = [ + "## Verification Checklist", + "", + f"Spec: `{checklist.spec_name}`", + "", + ] + + # Task completion audit + lines.append("### Task Completion Audit") + lines.append("") + if checklist.task_audit: + lines.append("| Group | Subtask | Title | Status |") + lines.append("|-------|---------|-------|--------|") + for entry in checklist.task_audit: + if entry.skipped: + status = "SKIPPED" + elif entry.checked: + status = "DONE" + else: + status = "**UNCHECKED**" + lines.append(f"| {entry.group_number} | {entry.subtask_id} | {entry.title} | {status} |") + unchecked = [e for e in checklist.task_audit if not e.checked and not e.skipped] + lines.append("") + if unchecked: + lines.append( + f"**{len(unchecked)} unchecked subtask(s).** Each must be completed or documented in an erratum." + ) + else: + lines.append("All subtasks completed or intentionally skipped.") + else: + lines.append("No tasks found.") + lines.append("") + + # Errata notice + if checklist.has_errata: + lines.append( + "**Note:** Errata exist for this spec. Check `docs/errata/` " + "and the errata DB table for documented deviations." + ) + lines.append("") + + # Requirement-to-test coverage + lines.append("### Requirement-to-Test Coverage") + lines.append("") + if checklist.requirement_coverage: + lines.append("| Requirement | Status | Test Files |") + lines.append("|-------------|--------|------------|") + for mapping in checklist.requirement_coverage: + if mapping.covered: + status = "COVERED" + files = ", ".join(mapping.test_files) + else: + status = "**UNCOVERED**" + files = "-" + lines.append(f"| {mapping.requirement_id} | {status} | {files} |") + uncovered = [m for m in checklist.requirement_coverage if not m.covered] + lines.append("") + if uncovered: + lines.append( + f"**{len(uncovered)} requirement(s) without test coverage.** " + f"Each uncovered requirement is a critical finding." + ) + else: + lines.append("All requirements have test coverage.") + else: + lines.append("No requirements found to map.") + lines.append("") + + # Enforcement rules + lines.append("### Enforcement Rules") + lines.append("") + lines.append("- Any **UNCHECKED** subtask without a corresponding erratum → FAIL verdict.") + lines.append("- Any **UNCOVERED** requirement without test coverage → FAIL verdict.") + lines.append("- Errata document intentional deviations — verify they are legitimate.") + + return "\n".join(lines) diff --git a/docs/memory.md b/docs/memory.md index 496859f9..c78b2fa4 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -2,6 +2,15 @@ _3176 facts | last updated: 2026-04-23_ +**2026-04-23 verifier checklist enforcement (issue #521):** Added +`spec/verification_checklist.py` — builds a structured verification +checklist from tasks.md checkboxes, requirements.md acceptance criteria, +and errata. Injected into verifier context via `assemble_context()` +(new `archetype` parameter). Verifier profile updated with hard gates: +unchecked subtasks without errata → FAIL, uncovered requirements → FAIL. +Requirement-to-test mapping scans test files for req ID references +(string match + normalized function name match). +22 tests (4364 total pass). + **2026-04-23 simplification pass 3:** Inlined core/llm_validation.py into session/review_parser.py (single consumer), core/retry.py into core/client.py (single consumer), merged nightshift/fix_types.py into nightshift/fix_pipeline.py, diff --git a/tests/unit/session/test_context_verifier.py b/tests/unit/session/test_context_verifier.py new file mode 100644 index 00000000..2f52d0b8 --- /dev/null +++ b/tests/unit/session/test_context_verifier.py @@ -0,0 +1,120 @@ +"""Tests for verification checklist injection into verifier context. + +Verifies that assemble_context() includes the verification checklist +section when archetype is 'verifier' and omits it for other archetypes. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import duckdb + +from agent_fox.knowledge.migrations import run_migrations +from agent_fox.session.context import assemble_context + + +def _make_conn() -> duckdb.DuckDBPyConnection: + conn = duckdb.connect(":memory:") + run_migrations(conn) + return conn + + +def _setup_spec(tmp_path: Path) -> Path: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + (spec_dir / "requirements.md").write_text( + "## Requirements\n\n**[10-REQ-1.1]** The system SHALL do X.\n", + encoding="utf-8", + ) + (spec_dir / "design.md").write_text("## Overview\n\nDesign doc.\n", encoding="utf-8") + (spec_dir / "test_spec.md").write_text("## Test Cases\n\nTest spec.\n", encoding="utf-8") + (spec_dir / "tasks.md").write_text( + "- [x] 1. Write tests\n - [x] 1.1 Unit tests\n - [ ] 1.2 Integration tests\n", + encoding="utf-8", + ) + return spec_dir + + +class TestVerifierChecklistInjection: + def test_verifier_context_includes_checklist(self, tmp_path: Path) -> None: + spec_dir = _setup_spec(tmp_path) + conn = _make_conn() + context = assemble_context( + spec_dir, + task_group=1, + conn=conn, + project_root=tmp_path, + archetype="verifier", + ) + assert "Verification Checklist" in context + assert "Task Completion Audit" in context + assert "1.1" in context + assert "1.2" in context + + def test_coder_context_excludes_checklist(self, tmp_path: Path) -> None: + spec_dir = _setup_spec(tmp_path) + conn = _make_conn() + context = assemble_context( + spec_dir, + task_group=1, + conn=conn, + project_root=tmp_path, + archetype="coder", + ) + assert "Verification Checklist" not in context + + def test_no_archetype_excludes_checklist(self, tmp_path: Path) -> None: + spec_dir = _setup_spec(tmp_path) + conn = _make_conn() + context = assemble_context( + spec_dir, + task_group=1, + conn=conn, + project_root=tmp_path, + ) + assert "Verification Checklist" not in context + + def test_checklist_includes_unchecked_warning(self, tmp_path: Path) -> None: + spec_dir = _setup_spec(tmp_path) + conn = _make_conn() + context = assemble_context( + spec_dir, + task_group=1, + conn=conn, + project_root=tmp_path, + archetype="verifier", + ) + assert "UNCHECKED" in context + + def test_checklist_includes_requirement_coverage(self, tmp_path: Path) -> None: + spec_dir = _setup_spec(tmp_path) + conn = _make_conn() + context = assemble_context( + spec_dir, + task_group=1, + conn=conn, + project_root=tmp_path, + archetype="verifier", + ) + assert "Requirement-to-Test Coverage" in context + assert "10-REQ-1.1" in context + + def test_checklist_failure_does_not_crash(self, tmp_path: Path) -> None: + """If checklist building fails, context assembly continues.""" + spec_dir = _setup_spec(tmp_path) + conn = _make_conn() + with patch( + "agent_fox.spec.verification_checklist.build_verification_checklist", + side_effect=RuntimeError("boom"), + ): + context = assemble_context( + spec_dir, + task_group=1, + conn=conn, + project_root=tmp_path, + archetype="verifier", + ) + assert "Verification Checklist" not in context + assert "Requirements" in context diff --git a/tests/unit/spec/test_verification_checklist.py b/tests/unit/spec/test_verification_checklist.py new file mode 100644 index 00000000..29cdb078 --- /dev/null +++ b/tests/unit/spec/test_verification_checklist.py @@ -0,0 +1,302 @@ +"""Tests for the verification checklist builder. + +Verifies that the checklist correctly audits task completion, maps +requirements to test functions, and renders a structured markdown +document for verifier context injection. +""" + +from __future__ import annotations + +from pathlib import Path + +import duckdb +import pytest + +from agent_fox.knowledge.migrations import run_migrations +from agent_fox.spec.verification_checklist import ( + RequirementMapping, + SubtaskAuditEntry, + VerificationChecklist, + build_verification_checklist, + render_checklist_markdown, + scan_requirement_test_coverage, +) + + +def _make_conn() -> duckdb.DuckDBPyConnection: + conn = duckdb.connect(":memory:") + run_migrations(conn) + return conn + + +def _write_tasks(spec_dir: Path, content: str) -> None: + (spec_dir / "tasks.md").write_text(content, encoding="utf-8") + + +def _write_requirements(spec_dir: Path, content: str) -> None: + (spec_dir / "requirements.md").write_text(content, encoding="utf-8") + + +class TestSubtaskAudit: + def test_all_checked_returns_no_unchecked(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_tasks( + spec_dir, + "- [x] 1. Write failing tests\n" + " - [x] 1.1 Write unit tests\n" + " - [x] 1.2 Write integration tests\n" + " - [x] 1.V Verify task group 1\n", + ) + conn = _make_conn() + checklist = build_verification_checklist(spec_dir, conn) + unchecked = [e for e in checklist.task_audit if not e.checked] + assert unchecked == [] + + def test_unchecked_subtasks_flagged(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_tasks( + spec_dir, + "- [ ] 1. Write failing tests\n" + " - [x] 1.1 Write unit tests\n" + " - [ ] 1.2 Write integration tests\n" + " - [ ] 1.V Verify task group 1\n", + ) + conn = _make_conn() + checklist = build_verification_checklist(spec_dir, conn) + unchecked = [e for e in checklist.task_audit if not e.checked] + assert len(unchecked) == 2 + ids = {e.subtask_id for e in unchecked} + assert "1.2" in ids + assert "1.V" in ids + + def test_erratum_covers_unchecked_subtask(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_tasks( + spec_dir, + "- [ ] 1. Write failing tests\n" + " - [ ] 1.1 Skipped subtask\n", + ) + conn = _make_conn() + conn.execute( + "INSERT INTO errata " + "(id, spec_name, task_group, finding_summary, created_at) " + "VALUES ('e1', '10_my_spec', '1', 'Documented deviation for 1.1', CURRENT_TIMESTAMP)", + ) + checklist = build_verification_checklist(spec_dir, conn) + assert checklist.has_errata is True + + def test_multiple_groups_audited(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_tasks( + spec_dir, + "- [x] 1. Write failing tests\n" + " - [x] 1.1 Unit tests\n" + "- [ ] 2. Implement\n" + " - [x] 2.1 Core logic\n" + " - [ ] 2.2 Edge cases\n", + ) + conn = _make_conn() + checklist = build_verification_checklist(spec_dir, conn) + groups = {e.group_number for e in checklist.task_audit} + assert 1 in groups + assert 2 in groups + + def test_skipped_subtasks_excluded(self, tmp_path: Path) -> None: + """Subtasks marked with [-] or [~] are intentionally skipped.""" + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_tasks( + spec_dir, + "- [-] 1. Partially done\n" + " - [x] 1.1 Done\n" + " - [-] 1.2 Skipped intentionally\n" + " - [~] 1.3 Not applicable\n", + ) + conn = _make_conn() + checklist = build_verification_checklist(spec_dir, conn) + unchecked = [e for e in checklist.task_audit if not e.checked and not e.skipped] + assert unchecked == [] + + +class TestRequirementTestCoverage: + def test_requirement_found_in_test_docstring(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_requirements( + spec_dir, + "## Requirements\n\n" + "### Requirement 1: Core Feature\n" + "**[10-REQ-1.1]** The system SHALL do X.\n", + ) + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "test_core.py").write_text( + '"""Tests for core feature.\n\nRequirements: 10-REQ-1.1\n"""\n\n' + "def test_core_does_x():\n pass\n", + encoding="utf-8", + ) + mappings = scan_requirement_test_coverage(spec_dir, tests_dir) + mapped = {m.requirement_id: m for m in mappings} + assert "10-REQ-1.1" in mapped + assert mapped["10-REQ-1.1"].covered is True + + def test_requirement_found_in_function_name(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_requirements( + spec_dir, + "## Requirements\n\n" + "**[10-REQ-1.1]** The system SHALL do X.\n", + ) + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "test_feature.py").write_text( + "def test_req_10_1_1_something():\n pass\n", + encoding="utf-8", + ) + mappings = scan_requirement_test_coverage(spec_dir, tests_dir) + mapped = {m.requirement_id: m for m in mappings} + assert "10-REQ-1.1" in mapped + assert mapped["10-REQ-1.1"].covered is True + + def test_unmapped_requirement_flagged(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_requirements( + spec_dir, + "## Requirements\n\n" + "**[10-REQ-1.1]** The system SHALL do X.\n" + "**[10-REQ-1.2]** The system SHALL do Y.\n", + ) + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "test_feature.py").write_text( + "# Tests for 10-REQ-1.1\ndef test_x():\n pass\n", + encoding="utf-8", + ) + mappings = scan_requirement_test_coverage(spec_dir, tests_dir) + mapped = {m.requirement_id: m for m in mappings} + assert mapped["10-REQ-1.1"].covered is True + assert mapped["10-REQ-1.2"].covered is False + + def test_no_requirements_file(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + mappings = scan_requirement_test_coverage(spec_dir, tests_dir) + assert mappings == [] + + def test_no_tests_dir(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_requirements( + spec_dir, + "**[10-REQ-1.1]** The system SHALL do X.\n", + ) + tests_dir = tmp_path / "nonexistent" + mappings = scan_requirement_test_coverage(spec_dir, tests_dir) + mapped = {m.requirement_id: m for m in mappings} + assert mapped["10-REQ-1.1"].covered is False + + +class TestRenderChecklistMarkdown: + def test_renders_task_audit_section(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_tasks( + spec_dir, + "- [ ] 1. Write tests\n" + " - [x] 1.1 Unit tests\n" + " - [ ] 1.2 Integration tests\n", + ) + conn = _make_conn() + checklist = build_verification_checklist(spec_dir, conn) + md = render_checklist_markdown(checklist) + assert "## Verification Checklist" in md + assert "Task Completion Audit" in md + assert "1.1" in md + assert "1.2" in md + + def test_renders_requirement_coverage(self, tmp_path: Path) -> None: + checklist = VerificationChecklist( + spec_name="10_my_spec", + task_audit=[], + requirement_coverage=[ + RequirementMapping("10-REQ-1.1", True, ["test_core.py"]), + RequirementMapping("10-REQ-1.2", False, []), + ], + has_errata=False, + ) + md = render_checklist_markdown(checklist) + assert "Requirement-to-Test Coverage" in md + assert "10-REQ-1.1" in md + assert "10-REQ-1.2" in md + assert "UNCOVERED" in md + + def test_renders_errata_notice(self, tmp_path: Path) -> None: + checklist = VerificationChecklist( + spec_name="10_my_spec", + task_audit=[ + SubtaskAuditEntry(1, "1.1", "Do stuff", False, False), + ], + requirement_coverage=[], + has_errata=True, + ) + md = render_checklist_markdown(checklist) + assert "errata" in md.lower() + + def test_empty_checklist_renders_cleanly(self) -> None: + checklist = VerificationChecklist( + spec_name="10_my_spec", + task_audit=[], + requirement_coverage=[], + has_errata=False, + ) + md = render_checklist_markdown(checklist) + assert "## Verification Checklist" in md + + +class TestBuildVerificationChecklist: + def test_full_checklist_integration(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + _write_tasks( + spec_dir, + "- [x] 1. Write failing tests\n" + " - [x] 1.1 Unit tests\n" + " - [x] 1.V Verify\n" + "- [x] 2. Implement\n" + " - [x] 2.1 Core\n" + " - [x] 2.V Verify\n", + ) + _write_requirements( + spec_dir, + "## Requirements\n\n" + "**[10-REQ-1.1]** The system SHALL do X.\n", + ) + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "test_core.py").write_text( + "# 10-REQ-1.1\ndef test_x():\n pass\n", + encoding="utf-8", + ) + conn = _make_conn() + checklist = build_verification_checklist( + spec_dir, conn, tests_dir=tests_dir + ) + assert checklist.spec_name == "10_my_spec" + assert len(checklist.task_audit) > 0 + assert len(checklist.requirement_coverage) == 1 + assert checklist.requirement_coverage[0].covered is True + + def test_missing_tasks_file_returns_empty_audit(self, tmp_path: Path) -> None: + spec_dir = tmp_path / "10_my_spec" + spec_dir.mkdir() + conn = _make_conn() + checklist = build_verification_checklist(spec_dir, conn) + assert checklist.task_audit == []