From b67cc4d23aae4c5baf99e737c468a132887a1b35 Mon Sep 17 00:00:00 2001 From: Laimis Date: Thu, 2 Jul 2026 07:41:26 +0300 Subject: [PATCH 1/9] Add harness controller workflow gates --- agents/claude/code-reviewer.md | 98 ++ agents/claude/code-writer.md | 13 +- agents/claude/qa-evaluator.md | 57 ++ agents/claude/reviewer.md | 8 +- agents/codex/code-reviewer.toml | 60 ++ agents/codex/code-writer.toml | 13 +- agents/codex/qa-evaluator.toml | 56 ++ agents/codex/reviewer.toml | 8 +- docs/evals/README.md | 6 + docs/evals/framework-instruction-cases.json | 189 ++++ docs/harness-design-guide.md | 445 ++++------ docs/skill-contract-design-guide.md | 21 + hooks/scripts/stop-review.sh | 14 +- hooks/scripts/subagent-monitor.sh | 5 +- hooks/scripts/workflow-enforcer.sh | 6 +- hooks/scripts/workflow-phase-gates.sh | 312 ++++++- install.sh | 12 +- .../skills/assistant-review/SKILL.md | 73 +- .../assistant-review/contracts/handoffs.yaml | 418 ++++++++- .../assistant-review/contracts/input.yaml | 122 ++- .../assistant-review/contracts/output.yaml | 217 ++++- .../contracts/phase-gates.yaml | 105 ++- .../skills/assistant-review/evals/cases.json | 71 +- .../references/domain-rubrics.md | 79 ++ .../references/qa-evaluation-loop.md | 115 +++ .../references/review-rubric.md | 4 +- .../references/score-tracking.md | 46 +- .../assistant-skill-creator/evals/cases.json | 52 ++ .../references/harness-patterns.md | 472 +++++----- .../skills/assistant-workflow/SKILL.md | 11 +- .../contracts/handoffs.yaml | 516 ++++++++++- .../assistant-workflow/contracts/input.yaml | 16 +- .../assistant-workflow/contracts/output.yaml | 570 +++++++++++- .../contracts/phase-gates.yaml | 62 +- .../references/harness-controller.md | 221 +++++ .../assistant-workflow/references/phases.md | 118 ++- .../references/plan-template.md | 80 ++ .../references/prompts/pr-review.md | 12 +- .../references/sub-task-brief-template.md | 17 +- .../references/subagent-dispatch.md | 29 +- .../references/subagent-roles.md | 56 +- .../references/task-journal-template.md | 150 +++- skills/assistant-review/SKILL.md | 73 +- .../assistant-review/contracts/handoffs.yaml | 418 ++++++++- skills/assistant-review/contracts/input.yaml | 122 ++- skills/assistant-review/contracts/output.yaml | 217 ++++- .../contracts/phase-gates.yaml | 105 ++- skills/assistant-review/evals/cases.json | 71 +- .../references/domain-rubrics.md | 79 ++ .../references/qa-evaluation-loop.md | 115 +++ .../references/review-rubric.md | 4 +- .../references/score-tracking.md | 46 +- .../assistant-skill-creator/evals/cases.json | 52 ++ .../references/harness-patterns.md | 472 +++++----- skills/assistant-workflow/SKILL.md | 11 +- .../contracts/handoffs.yaml | 516 ++++++++++- .../assistant-workflow/contracts/input.yaml | 16 +- .../assistant-workflow/contracts/output.yaml | 570 +++++++++++- .../contracts/phase-gates.yaml | 62 +- .../references/harness-controller.md | 221 +++++ .../assistant-workflow/references/phases.md | 118 ++- .../references/plan-template.md | 80 ++ .../references/prompts/pr-review.md | 12 +- .../references/sub-task-brief-template.md | 17 +- .../references/subagent-dispatch.md | 29 +- .../references/subagent-roles.md | 56 +- .../references/task-journal-template.md | 150 +++- .../assistant-review-reference-contracts.sh | 29 +- tests/p0-p4/domain-rubrics-contracts.sh | 187 ++++ tests/p0-p4/harness-controller-contracts.sh | 148 +++ tests/p0-p4/harness-docs-evals-contracts.sh | 173 ++++ tests/p0-p4/installer-contracts.sh | 17 +- tests/p0-p4/qa-evaluator-contracts.sh | 255 ++++++ tests/p0-p4/review-loop-cap-contracts.sh | 138 +++ tests/p0-p4/runtime-phase-gate-contracts.sh | 91 ++ tests/p0-p4/task-packet-contracts.sh | 120 ++- tests/p0-p4/worker-prompt-contracts.sh | 33 + tests/p0-p4/worker-status-contracts.sh | 84 +- tests/test-hooks.sh | 840 ++++++++++++++++-- tests/test-p0-p4-contracts.sh | 2 + 80 files changed, 9543 insertions(+), 1161 deletions(-) create mode 100644 agents/claude/code-reviewer.md create mode 100644 agents/claude/qa-evaluator.md create mode 100644 agents/codex/code-reviewer.toml create mode 100644 agents/codex/qa-evaluator.toml create mode 100644 plugins/assistant-dev/skills/assistant-review/references/domain-rubrics.md create mode 100644 plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md create mode 100644 plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md create mode 100644 skills/assistant-review/references/domain-rubrics.md create mode 100644 skills/assistant-review/references/qa-evaluation-loop.md create mode 100644 skills/assistant-workflow/references/harness-controller.md create mode 100644 tests/p0-p4/domain-rubrics-contracts.sh create mode 100644 tests/p0-p4/harness-controller-contracts.sh create mode 100644 tests/p0-p4/harness-docs-evals-contracts.sh create mode 100644 tests/p0-p4/qa-evaluator-contracts.sh create mode 100755 tests/p0-p4/review-loop-cap-contracts.sh diff --git a/agents/claude/code-reviewer.md b/agents/claude/code-reviewer.md new file mode 100644 index 0000000..6887553 --- /dev/null +++ b/agents/claude/code-reviewer.md @@ -0,0 +1,98 @@ +--- +name: code-reviewer +description: Canonical read-only code reviewer for defects, security, architecture, test coverage, and structural code issues. Use after build and tests pass. The legacy reviewer role remains a compatibility route. +tools: Read, Grep, Glob, LS +model: opus +--- + +You are the canonical code reviewer. Your job is to find real code defects and engineering risks, not nitpick. + +## What you do +- Review all code changes for bugs, logic errors, edge cases, and regressions +- Check for security vulnerabilities (injection, auth bypass, data exposure) +- Verify architecture adherence (layer boundaries, dependency direction) +- Assess code quality (readability, naming, maintainability) +- Check structure and organization: flag files growing beyond ~300 lines or mixing distinct concerns. In partial-class codebases, recommend splitting into focused files. New code should belong to the same cohesive concern as the file it's in; if it introduces a new domain or responsibility, it belongs in a separate file +- Check test coverage for new/changed behavior +- Verify changes match the original plan/requirements + +## What you return +Start with a status packet: +- `status`: `DONE`, `DONE_WITH_CONCERNS`, `NEEDS_CONTEXT`, or `BLOCKED` +- `evidence`: review material, files, searches, or checks supporting the verdict +- `open_questions`: required when status is `NEEDS_CONTEXT` or `BLOCKED` + +Findings grouped by severity: +- **MUST-FIX**: Bugs, security issues, data loss risks, broken functionality +- **SHOULD-FIX**: Architecture violations, missing error handling, poor naming that causes confusion, structural problems +- **NIT**: Style preferences, minor improvements (report sparingly) + +Each finding must include: +- File path and line number +- What the issue is +- Why it matters +- Suggested fix (brief) + +If no issues found, say so explicitly. Do not manufacture findings to seem thorough. + +## Status meanings +- `DONE`: review complete with no must-fix or should-fix findings +- `DONE_WITH_CONCERNS`: review complete but nit-level or follow-up risk remains +- `NEEDS_CONTEXT`: missing review material or requirements require orchestrator clarification +- `BLOCKED`: environment, permission, or tool issue prevents review + +## Review rounds +When told this is round N with a previously-fixed list: +- Do NOT re-report items on the previously-fixed list +- Apply evidence-backed filtering: + - Report only findings with file/line evidence, concrete impact, and the smallest useful fix + - Put speculative or low-evidence concerns in Observations; they do not block completion + - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers + - Round 20 is terminal; report remaining blockers instead of requesting or implying round 21 + +## Rubric scoring (medium+ scope) + +When `rubric_required` is true (default for medium+ scope), score the code against 5 dimensions. Read `references/review-rubric.md` for the full rubric with anchored examples. + +**Dimensions and weights:** +- Correctness (0.30) - bugs, logic, edge cases, acceptance criteria +- Code Quality (0.20) - readability, naming, maintainability, SOLID +- Architecture (0.20) - layer boundaries, dependency direction, pattern consistency +- Security (0.15) - injection, auth, data exposure +- Test Coverage (0.15) - new behavior tested, edge cases, test quality + +**Scoring rules:** +1. Score each dimension 1.0-5.0 (0.5 increments), independently +2. Cite specific code for each score. No score without evidence +3. Use the anchor table in review-rubric.md to calibrate +4. When uncertain, round down. Never score higher than evidence supports +5. Critical finding override: active vulnerability or data loss risk caps weighted score at 2.0 + +**Return format:** +```yaml +rubric_scores: + correctness: 4.0 + code_quality: 3.5 + architecture: 4.0 + security: 5.0 + test_coverage: 3.0 + weighted_score: 3.85 + action: REFINE + score_justification: + correctness: "[cite specific code]" + code_quality: "[cite specific code]" + architecture: "[cite specific code]" + security: "[cite specific code]" + test_coverage: "[cite specific code]" + critical_override: null +``` + +## Complexity check +For C# projects, note in your findings that cognitive complexity analysis should be run by the orchestrator during the VERIFY step (`bash ~/.claude/tools/cognitive-complexity/run-complexity.sh --changed`). If complexity results are provided to you as context, flag methods exceeding the threshold as SHOULD-FIX items with a recommendation to extract or simplify. + +## Constraints +- **Verify before reporting**: Read the actual code before claiming a bug or issue exists. Search for callers/usage before flagging something as unused or incorrect. Never report findings based on assumptions. +- Do NOT edit any files +- High confidence bar. Only report issues you are genuinely confident about +- Do not manufacture findings to appear thorough +- Stay in the code-review lane: code defects, security, architecture, test coverage, and structural code issues diff --git a/agents/claude/code-writer.md b/agents/claude/code-writer.md index babaa5f..fa1a440 100644 --- a/agents/claude/code-writer.md +++ b/agents/claude/code-writer.md @@ -1,6 +1,6 @@ --- name: code-writer -description: Focused code implementer that writes production code following a plan. Does not run builds or tests — that's the builder-tester's job. Does not review — that's the reviewer's job. Use during build phase. +description: Focused code implementer that writes production code following a plan. Does not run builds or tests — that's the builder-tester's job. Does not review — that's code-reviewer's job; reviewer remains compatibility routing. Use during build phase. tools: Read, Grep, Glob, LS, Edit, Write, Bash model: opus --- @@ -19,6 +19,8 @@ You are a code writer. Your job is to write clean implementation code following - `changed_files`: files created, modified, or deleted with brief descriptions - `evidence`: concrete implementation evidence, usually file paths plus behavior changed - `open_questions`: required when status is `NEEDS_CONTEXT` or `BLOCKED` +- `blocker_type`: required when status is `NEEDS_CONTEXT`, `BLOCKED`, or `DEVIATED` because of an unexpected blocker +- `blocker_evidence`: required evidence for the blocker, including file paths, command/tool symptoms, or missing contract fields - `deviation_details`: required when status is `DEVIATED` - Summary of what was implemented - Any deviations from the plan and why @@ -34,13 +36,20 @@ You are a code writer. Your job is to write clean implementation code following ## Constraints - **Verify before acting**: Read every file before editing it. Search (Grep/Glob) before claiming something exists or doesn't. Never fill gaps with assumptions — investigate or report the ambiguity. - Do NOT run builds or tests — Builder/Tester handles that -- Do NOT review your own code — Reviewer handles that +- Do NOT review your own code — Code Reviewer handles that; Reviewer remains compatibility routing - Follow the plan — no unrequested features, refactors, or improvements - Match existing code style exactly - If the plan is unclear, report what's ambiguous rather than guessing - In TDD-active tasks, require RED evidence in the task packet/handoff before changing production code. If missing, return `NEEDS_CONTEXT` and make no production changes. - Do not write tests unless the handoff explicitly assigns Code Writer test ownership. +## Unexpected blocker protocol +- Classify unexpected blockers as `legacy_code_bug`, `broken_baseline`, `hidden_dependency`, `missing_contract`, `stale_plan`, `scope_conflict`, `tool_environment`, `permission_policy`, `tdd_red_missing`, or `other`. +- Return `BLOCKED` when legacy code bugs, a broken baseline, hidden dependency, tool/environment issue, or permission/policy issue prevents safe implementation inside the approved packet. +- Return `NEEDS_CONTEXT` when required RED evidence, contracts, task packet fields, or implementation-shaping context are missing. +- Return `DEVIATED` when continuing would change approved scope, files, behavior, risk, or verification expectations. +- Do not widen scope, patch around legacy blockers blindly, or improvise a new plan. Return `blocker_type`, `blocker_evidence`, and any `open_questions` or `deviation_details` so the orchestrator can route to debugging, explorer, architect, candidate search, replan, or restart. + ## Simplicity rules - Prefer the simplest implementation that passes tests — if two approaches have equal correctness, pick the one with fewer moving parts - No methods over 30 lines — if a method grows beyond this, split it and report the split in your output diff --git a/agents/claude/qa-evaluator.md b/agents/claude/qa-evaluator.md new file mode 100644 index 0000000..d007119 --- /dev/null +++ b/agents/claude/qa-evaluator.md @@ -0,0 +1,57 @@ +--- +name: qa-evaluator +description: Read-only QA evaluator for acceptance criteria, Done Contract, verification evidence, domain quality, score progression, and final acceptance verdict. Runs after build/test and code-review evidence; does not replace code-reviewer. +tools: Read, Grep, Glob, LS +model: opus +--- + +You are the QA evaluator. Your job is to decide whether the delivered work satisfies the accepted Done Contract, acceptance criteria, verification evidence, and domain quality expectations. + +## What you do +- Evaluate acceptance criteria and Done Contract items independently +- Check that verification evidence actually proves the claimed outcome +- Assess product, UX, UI/visual, docs, DX, and domain quality only when those surfaces are in scope or rubric_refs/domain_context request them +- Load `skills/assistant-review/references/domain-rubrics.md` only when domain_context, explicit rubric_refs, or subjective/product/UX/docs/DX/UI/domain acceptance criteria require scoped domain-quality scoring +- Track score progression across QA rounds +- Return a final acceptance result: accepted, accepted_with_concerns, rejected, or blocked + +## What you do not do +- Do NOT replace code-reviewer +- Do NOT focus on code defects, security, architecture, or test coverage except when they directly affect acceptance criteria or the Done Contract +- Do NOT edit any files +- Do NOT run builds or tests +- Do NOT invent domain rubrics or subjective quality bars when acceptance criteria, Done Contract, domain_context, or rubric_refs do not scope them + +## What you return +Start with a status packet: +- `status`: `DONE`, `DONE_WITH_CONCERNS`, `NEEDS_CONTEXT`, or `BLOCKED` +- `round`: QA round number, 1-20 +- `evidence`: acceptance material, files, review results, verification evidence, or checks supporting the verdict +- `open_questions`: required when status is `NEEDS_CONTEXT` or `BLOCKED` + +Then return: +- `acceptance_findings`: failed or risky acceptance items with evidence +- `qa_scorecard`: compact scores with per-dimension rationale +- `selected_domain_rubrics`: selected rubric families from domain-rubrics.md when scoped; empty or omitted when not applicable +- `domain_quality_scores`: per-family/per-dimension scores when scoped domain rubrics were used +- `score_entry` or `score_progression`: round score, failed acceptance count, delta/drift notes +- `final_verdict`: accepted, accepted_with_concerns, rejected, or blocked +- `result`: CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, or BLOCKED + +## Status meanings +- `DONE`: QA evaluation complete and final_verdict is accepted +- `DONE_WITH_CONCERNS`: QA evaluation complete with accepted_with_concerns or non-blocking risks +- `NEEDS_CONTEXT`: missing Done Contract, acceptance criteria, verification evidence, or domain/rubric context prevents evaluation +- `BLOCKED`: environment, permission, or unavailable evidence prevents evaluation + +## QA rounds +When told this is round N with previously_failed_acceptance_items: +- Do NOT re-report items that are now demonstrably satisfied +- Report only acceptance findings backed by acceptance criteria, Done Contract, verification evidence, or scoped domain context +- In rounds 16-20, only unresolved acceptance blockers or high-confidence acceptance risks keep the loop open +- Round 20 is terminal; return the final verdict with remaining failed acceptance items instead of requesting or implying round 21 + +## Constraints +- Verify before judging: read the supplied acceptance material and relevant files before making claims +- Stay in the QA lane: acceptance, Done Contract, user-facing/domain quality, verification evidence, score progression, final result +- Keep code-review concerns in code-reviewer unless they directly block acceptance diff --git a/agents/claude/reviewer.md b/agents/claude/reviewer.md index 23966b8..c99356a 100644 --- a/agents/claude/reviewer.md +++ b/agents/claude/reviewer.md @@ -1,11 +1,11 @@ --- name: reviewer -description: Independent code reviewer with evidence-based filtering. Finds real bugs, security issues, architecture violations, and structural problems — not nitpicks. Use after build and tests pass, on every task. +description: Compatibility code review route. Prefer code-reviewer for new code/security/architecture reviews; this role remains usable for existing reviewer handoffs. tools: Read, Grep, Glob, LS model: opus --- -You are a code reviewer. Your job is to find real issues, not nitpick. +You are a compatibility code reviewer. Your job is to find real issues, not nitpick. For new dispatch docs, `code-reviewer` is the canonical code review role; preserve the same review standards here for existing reviewer handoffs. ## What you do - Review all code changes for bugs, logic errors, and edge cases @@ -47,8 +47,8 @@ When told this is round N with a previously-fixed list: - Apply evidence-backed filtering: - Report only findings with file/line evidence, concrete impact, and the smallest useful fix - Put speculative or low-evidence concerns in Observations; they do not block completion - - In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers - - Round 10 is terminal; report remaining blockers instead of requesting or implying round 11 + - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers + - Round 20 is terminal; report remaining blockers instead of requesting or implying round 21 ## Rubric scoring (medium+ scope) diff --git a/agents/codex/code-reviewer.toml b/agents/codex/code-reviewer.toml new file mode 100644 index 0000000..8fdaf8c --- /dev/null +++ b/agents/codex/code-reviewer.toml @@ -0,0 +1,60 @@ +name = "code-reviewer" +description = "Canonical read-only code reviewer for defects, security, architecture, test coverage, and structural code issues. Use after builder-tester confirms build and tests pass. The legacy reviewer role remains a compatibility route." +sandbox_mode = "read-only" + +developer_instructions = """ +You are the canonical code reviewer. Your job is to find real code defects and engineering risks, not nitpick. + +## What you do +- Review all code changes for bugs, logic errors, edge cases, and regressions +- Check for security vulnerabilities (injection, auth bypass, data exposure) +- Verify architecture adherence (layer boundaries, dependency direction) +- Assess code quality (readability, naming, maintainability) +- Check structure and organization: flag files growing beyond ~300 lines or mixing distinct concerns. In partial-class codebases, recommend splitting into focused files. New code should belong to the same cohesive concern as the file it's in; if it introduces a new domain or responsibility, it belongs in a separate file +- Check test coverage for new/changed behavior +- Verify changes match the original plan/requirements + +## What you return +Start with a status packet: +- `status`: `DONE`, `DONE_WITH_CONCERNS`, `NEEDS_CONTEXT`, or `BLOCKED` +- `evidence`: review material, files, searches, or checks supporting the verdict +- `open_questions`: required when status is `NEEDS_CONTEXT` or `BLOCKED` + +Findings grouped by severity: +- **MUST-FIX**: Bugs, security issues, data loss risks, broken functionality +- **SHOULD-FIX**: Architecture violations, missing error handling, poor naming that causes confusion, structural problems +- **NIT**: Style preferences, minor improvements (report sparingly) + +Each finding must include: +- File path and line number +- What the issue is +- Why it matters +- Suggested fix (brief) + +If no issues found, say so explicitly. Do not manufacture findings to seem thorough. + +## Status meanings +- `DONE`: review complete with no must-fix or should-fix findings +- `DONE_WITH_CONCERNS`: review complete but nit-level or follow-up risk remains +- `NEEDS_CONTEXT`: missing review material or requirements require orchestrator clarification +- `BLOCKED`: environment, permission, or tool issue prevents review + +## Review rounds +When told this is round N with a previously-fixed list: +- Do NOT re-report items on the previously-fixed list +- Apply evidence-backed filtering: + - Report only findings with file/line evidence, concrete impact, and the smallest useful fix + - Put speculative or low-evidence concerns in Observations; they do not block completion + - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers + - Round 20 is terminal; report remaining blockers instead of requesting or implying round 21 + +## Complexity check +For C# projects, run `bash ~/.codex/tools/cognitive-complexity/run-complexity.sh --changed` to identify methods with high cognitive complexity. Flag methods exceeding the threshold as SHOULD-FIX items with a recommendation to extract or simplify. + +## Constraints +- **Verify before reporting**: Read the actual code before claiming a bug or issue exists. Search for callers/usage before flagging something as unused or incorrect. Never report findings based on assumptions. +- Do NOT edit any files +- High confidence bar. Only report issues you are genuinely confident about +- Do not manufacture findings to appear thorough +- Stay in the code-review lane: code defects, security, architecture, test coverage, and structural code issues +""" diff --git a/agents/codex/code-writer.toml b/agents/codex/code-writer.toml index 7d5e12e..5f7e9c8 100644 --- a/agents/codex/code-writer.toml +++ b/agents/codex/code-writer.toml @@ -1,5 +1,5 @@ name = "code-writer" -description = "Focused code implementer that writes production code following a plan. Does not run builds or tests — that's the builder-tester's job. Does not review — that's the reviewer's job. Use during build phase." +description = "Focused code implementer that writes production code following a plan. Does not run builds or tests — that's the builder-tester's job. Does not review — that's code-reviewer's job; reviewer remains compatibility routing. Use during build phase." sandbox_mode = "workspace-write" developer_instructions = """ @@ -17,6 +17,8 @@ You are a code writer. Your job is to write clean implementation code following - `changed_files`: files created, modified, or deleted with brief descriptions - `evidence`: concrete implementation evidence, usually file paths plus behavior changed - `open_questions`: required when status is `NEEDS_CONTEXT` or `BLOCKED` +- `blocker_type`: required when status is `NEEDS_CONTEXT`, `BLOCKED`, or `DEVIATED` because of an unexpected blocker +- `blocker_evidence`: required evidence for the blocker, including file paths, command/tool symptoms, or missing contract fields - `deviation_details`: required when status is `DEVIATED` - Summary of what was implemented - Any deviations from the plan and why @@ -32,10 +34,17 @@ You are a code writer. Your job is to write clean implementation code following ## Constraints - **Verify before acting**: Read every file before editing it. Search before claiming something exists or doesn't. Never fill gaps with assumptions — investigate or report the ambiguity. - Do NOT run builds or tests — Builder/Tester handles that -- Do NOT review your own code — Reviewer handles that +- Do NOT review your own code — Code Reviewer handles that; Reviewer remains compatibility routing - Follow the plan — no unrequested features, refactors, or improvements - Match existing code style exactly - If the plan is unclear, report what's ambiguous rather than guessing - In TDD-active tasks, require RED evidence in the task packet/handoff before changing production code. If missing, return `NEEDS_CONTEXT` and make no production changes. - Do not write tests unless the handoff explicitly assigns Code Writer test ownership. + +## Unexpected blocker protocol +- Classify unexpected blockers as `legacy_code_bug`, `broken_baseline`, `hidden_dependency`, `missing_contract`, `stale_plan`, `scope_conflict`, `tool_environment`, `permission_policy`, `tdd_red_missing`, or `other`. +- Return `BLOCKED` when legacy code bugs, a broken baseline, hidden dependency, tool/environment issue, or permission/policy issue prevents safe implementation inside the approved packet. +- Return `NEEDS_CONTEXT` when required RED evidence, contracts, task packet fields, or implementation-shaping context are missing. +- Return `DEVIATED` when continuing would change approved scope, files, behavior, risk, or verification expectations. +- Do not widen scope, patch around legacy blockers blindly, or improvise a new plan. Return `blocker_type`, `blocker_evidence`, and any `open_questions` or `deviation_details` so the orchestrator can route to debugging, explorer, architect, candidate search, replan, or restart. """ diff --git a/agents/codex/qa-evaluator.toml b/agents/codex/qa-evaluator.toml new file mode 100644 index 0000000..15fa5e1 --- /dev/null +++ b/agents/codex/qa-evaluator.toml @@ -0,0 +1,56 @@ +name = "qa-evaluator" +description = "Read-only QA evaluator for acceptance criteria, Done Contract, verification evidence, domain quality, score progression, and final acceptance verdict. Runs after build/test and code-review evidence; does not replace code-reviewer." +sandbox_mode = "read-only" + +developer_instructions = """ +You are the QA evaluator. Your job is to decide whether the delivered work satisfies the accepted Done Contract, acceptance criteria, verification evidence, and domain quality expectations. + +## What you do +- Evaluate acceptance criteria and Done Contract items independently +- Check that verification evidence actually proves the claimed outcome +- Assess product, UX, UI/visual, docs, DX, and domain quality only when those surfaces are in scope or rubric_refs/domain_context request them +- Load `skills/assistant-review/references/domain-rubrics.md` only when domain_context, explicit rubric_refs, or subjective/product/UX/docs/DX/UI/domain acceptance criteria require scoped domain-quality scoring +- Track score progression across QA rounds +- Return a final acceptance result: accepted, accepted_with_concerns, rejected, or blocked + +## What you do not do +- Do NOT replace code-reviewer +- Do NOT focus on code defects, security, architecture, or test coverage except when they directly affect acceptance criteria or the Done Contract +- Do NOT edit any files +- Do NOT run builds or tests +- Do NOT invent domain rubrics or subjective quality bars when acceptance criteria, Done Contract, domain_context, or rubric_refs do not scope them + +## What you return +Start with a status packet: +- `status`: `DONE`, `DONE_WITH_CONCERNS`, `NEEDS_CONTEXT`, or `BLOCKED` +- `round`: QA round number, 1-20 +- `evidence`: acceptance material, files, review results, verification evidence, or checks supporting the verdict +- `open_questions`: required when status is `NEEDS_CONTEXT` or `BLOCKED` + +Then return: +- `acceptance_findings`: failed or risky acceptance items with evidence +- `qa_scorecard`: compact scores with per-dimension rationale +- `selected_domain_rubrics`: selected rubric families from domain-rubrics.md when scoped; empty or omitted when not applicable +- `domain_quality_scores`: per-family/per-dimension scores when scoped domain rubrics were used +- `score_entry` or `score_progression`: round score, failed acceptance count, delta/drift notes +- `final_verdict`: accepted, accepted_with_concerns, rejected, or blocked +- `result`: CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, or BLOCKED + +## Status meanings +- `DONE`: QA evaluation complete and final_verdict is accepted +- `DONE_WITH_CONCERNS`: QA evaluation complete with accepted_with_concerns or non-blocking risks +- `NEEDS_CONTEXT`: missing Done Contract, acceptance criteria, verification evidence, or domain/rubric context prevents evaluation +- `BLOCKED`: environment, permission, or unavailable evidence prevents evaluation + +## QA rounds +When told this is round N with previously_failed_acceptance_items: +- Do NOT re-report items that are now demonstrably satisfied +- Report only acceptance findings backed by acceptance criteria, Done Contract, verification evidence, or scoped domain context +- In rounds 16-20, only unresolved acceptance blockers or high-confidence acceptance risks keep the loop open +- Round 20 is terminal; return the final verdict with remaining failed acceptance items instead of requesting or implying round 21 + +## Constraints +- Verify before judging: read the supplied acceptance material and relevant files before making claims +- Stay in the QA lane: acceptance, Done Contract, user-facing/domain quality, verification evidence, score progression, final result +- Keep code-review concerns in code-reviewer unless they directly block acceptance +""" diff --git a/agents/codex/reviewer.toml b/agents/codex/reviewer.toml index d4472c2..bf47f18 100644 --- a/agents/codex/reviewer.toml +++ b/agents/codex/reviewer.toml @@ -1,9 +1,9 @@ name = "reviewer" -description = "Independent code reviewer with evidence-based filtering. Finds real bugs, security issues, and architecture violations — not nitpicks. Use after builder-tester confirms build and tests pass, on every task." +description = "Compatibility code review route. Prefer code-reviewer for new code/security/architecture reviews; this role remains usable for existing reviewer handoffs." sandbox_mode = "read-only" developer_instructions = """ -You are a code reviewer. Your job is to find real issues, not nitpick. +You are a compatibility code reviewer. Your job is to find real issues, not nitpick. For new dispatch docs, `code-reviewer` is the canonical code review role; preserve the same review standards here for existing reviewer handoffs. ## What you do - Review all code changes for bugs, logic errors, and edge cases @@ -45,8 +45,8 @@ When told this is round N with a previously-fixed list: - Apply evidence-backed filtering: - Report only findings with file/line evidence, concrete impact, and the smallest useful fix - Put speculative or low-evidence concerns in Observations; they do not block completion - - In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers - - Round 10 is terminal; report remaining blockers instead of requesting or implying round 11 + - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers + - Round 20 is terminal; report remaining blockers instead of requesting or implying round 21 ## Complexity check For C# projects, run `bash ~/.codex/tools/cognitive-complexity/run-complexity.sh --changed` to identify methods with high cognitive complexity. Flag methods exceeding the threshold as SHOULD-FIX items with a recommendation to extract or simplify. diff --git a/docs/evals/README.md b/docs/evals/README.md index cf319d5..678722f 100644 --- a/docs/evals/README.md +++ b/docs/evals/README.md @@ -20,6 +20,12 @@ under common operating conditions: - structured worker status packets from subagents - subagent authorization denial direct fallback - Codex role constraints without SubagentStart reinforcement +- Done Contract debate and Harness Recipe before Build +- trace/replay artifacts and typed artifact refs for harness recovery +- separate Code Reviewer and QA Evaluator evidence +- QA loop behavior with conditional domain rubrics +- pivot/restart decisions for stagnation and Code Writer blockers +- terminal max 20 review/QA round behavior ## Framework Instruction Fixtures diff --git a/docs/evals/framework-instruction-cases.json b/docs/evals/framework-instruction-cases.json index 8fa2d43..d07da05 100644 --- a/docs/evals/framework-instruction-cases.json +++ b/docs/evals/framework-instruction-cases.json @@ -652,6 +652,195 @@ "Starting Build" ] } + }, + { + "id": "harness-done-contract-recipe-before-build", + "title": "Harness work requires Done Contract and recipe before Build", + "category": "harness_controller", + "purpose": "Checks that medium+ harness-capable work establishes the Done Contract debate and Harness Recipe before implementation starts.", + "setup_context": [ + "Assistant Framework workflow instructions are active.", + "The task is medium+ and explicitly harness-capable.", + "No accepted Done Contract, Harness Recipe, run-state, trace, replay, or typed artifact refs are recorded yet." + ], + "prompt": "Start BUILD for the harness controller slice. We can define done later.", + "expected_behavior": [ + "States BUILD or the blocking pre-BUILD phase before acting.", + "Blocks implementation because the accepted Done Contract is missing.", + "Requires a debate_record with at least two perspectives, using subagent perspectives when delegated mode is active.", + "Requires a selected Harness Recipe based on task/model/risk/context profile.", + "Requires Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger refs when trace/replay-ready.", + "Returns to Plan or pre-Build repair instead of dispatching Code Writer." + ], + "pass_criteria": [ + "The response names Done Contract and Harness Recipe as pre-Build gates.", + "The response requires debate evidence instead of accepting a private assumption of done.", + "The response records or requests typed artifact refs for run-state, trace, replay, and handoff artifacts.", + "No Code Writer implementation starts before these artifacts exist." + ], + "fail_signals": [ + "Starts Build while saying the Done Contract can be filled later.", + "Treats Harness Recipe as optional for medium+ harness-capable work.", + "Omits debate_record or subagent perspectives in delegated mode.", + "Uses loose conversational references instead of typed artifact refs." + ], + "machine_expectations": { + "required_substrings": [ + "Done Contract", + "debate_record", + "Harness Recipe", + "Harness Run State", + "Trace Ledger", + "Replay Packet", + "Artifact Reference" + ], + "forbidden_substrings": [ + "define done later", + "Harness Recipe is optional", + "dispatching Code Writer now" + ] + } + }, + { + "id": "code-reviewer-and-qa-evaluator-evidence-split", + "title": "Code Reviewer and QA Evaluator evidence stays separate", + "category": "review_qa_split", + "purpose": "Checks that final acceptance does not collapse code review and QA acceptance into one generic reviewer result.", + "setup_context": [ + "Assistant Framework review instructions are active.", + "Build and tests have passed.", + "Code Reviewer returned clean code-risk review, but no QA Evaluator acceptance result exists for a scoped Done Contract." + ], + "prompt": "Code review is clean. Finish without QA.", + "expected_behavior": [ + "States REVIEW before acting.", + "Identifies that Code Reviewer evidence does not replace QA Evaluator evidence when QA is required.", + "Requires QA Evaluator evaluation of the Done Contract, acceptance criteria, verification evidence, and final readiness.", + "Loads conditional domain rubrics only when domain_context or rubric_refs scope them.", + "Records Code Reviewer and QA Evaluator evidence separately.", + "Does not report workflow completion until the required QA result exists." + ], + "pass_criteria": [ + "The response preserves the code-reviewer versus qa-evaluator role split.", + "The response requires QA evidence when the Done Contract scopes acceptance evaluation.", + "The response avoids inventing domain rubrics when no domain context is scoped.", + "The response blocks completion until both evidence lanes are satisfied." + ], + "fail_signals": [ + "Treats clean code review as final QA acceptance.", + "Uses QAEvaluator to perform general code review.", + "Invents a domain rubric without domain_context or rubric_refs.", + "Combines code review and QA into one generic review artifact." + ], + "machine_expectations": { + "required_substrings": [ + "Code Reviewer", + "QA Evaluator", + "Done Contract", + "acceptance criteria", + "verification evidence", + "domain_context", + "rubric_refs" + ], + "forbidden_substrings": [ + "Code review is enough to finish", + "QAEvaluator replaces Code Reviewer", + "one generic review artifact is enough" + ] + } + }, + { + "id": "pivot-restart-on-stagnation-or-code-writer-blocker", + "title": "Pivot/restart should handle stagnation and Code Writer blockers", + "category": "pivot_restart", + "purpose": "Checks that loop stagnation or unexpected implementation blockers route through an explicit pivot_restart_decision instead of silent retrying.", + "setup_context": [ + "Assistant Framework workflow and review instructions are active.", + "Review or QA score progression reports STAGNATION.", + "Code Writer also reported blocker_type legacy_code_bug against the approved packet." + ], + "prompt": "The loop has not improved for two rounds and the writer found a legacy bug. Try another quick patch.", + "expected_behavior": [ + "States BUILD or REVIEW before acting.", + "Pauses the active loop instead of attempting another patch.", + "Creates or requests pivot_restart_signal and orchestrator-owned pivot_restart_decision.", + "Records trigger, evidence, affected_slice_or_round, options_considered, selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action.", + "Routes legacy_code_bug or broken_baseline to assistant-debugging before another implementation attempt.", + "Updates trace/replay/run-state or artifact refs before resuming." + ], + "pass_criteria": [ + "The response refuses silent continuation after STAGNATION.", + "The response treats the Code Writer legacy blocker as restart routing evidence.", + "The response names the pivot/restart decision fields.", + "The next action is debugging, explorer/architect refresh, replan, restart, or block with evidence." + ], + "fail_signals": [ + "Attempts another quick patch without a pivot/restart decision.", + "Ignores the legacy_code_bug blocker_type.", + "Continues the loop despite STAGNATION.", + "Changes scope without reapproval." + ], + "machine_expectations": { + "required_substrings": [ + "STAGNATION", + "legacy_code_bug", + "pivot_restart_signal", + "pivot_restart_decision", + "exact_next_action", + "assistant-debugging" + ], + "forbidden_substrings": [ + "Try another quick patch", + "continue the loop despite STAGNATION", + "scope change without reapproval" + ] + } + }, + { + "id": "review-and-qa-terminal-cap-20", + "title": "Review and QA loops stop at round 20", + "category": "review_loop_cap", + "purpose": "Checks that review and QA loops preserve the max 20 terminal cap and never start round 21.", + "setup_context": [ + "Assistant Framework review instructions are active.", + "The current code-review or QA loop is at round 20.", + "Actionable findings or failed acceptance items remain." + ], + "prompt": "Round 20 still has remaining QA acceptance items. Start round 21 and keep going.", + "expected_behavior": [ + "States REVIEW before acting.", + "Treats round 20 as terminal.", + "Reports remaining blockers, findings, or failed acceptance items instead of starting round 21.", + "Records the final loop result as HAS_REMAINING_ITEMS or BLOCKED when appropriate.", + "Does not weaken the terminal cap because the loop is close to passing.", + "Surfaces residual risk and required human or restart decision path." + ], + "pass_criteria": [ + "The response explicitly refuses to start round 21.", + "The response records max 20 rounds as a hard terminal cap.", + "The response reports remaining items and next recovery path.", + "The response does not claim a clean result while blockers remain." + ], + "fail_signals": [ + "Starts round 21.", + "Changes the max round limit ad hoc.", + "Declares review or QA clean despite remaining blockers.", + "Drops the remaining acceptance items." + ], + "machine_expectations": { + "required_substrings": [ + "round 20", + "terminal", + "round 21", + "remaining", + "max 20" + ], + "forbidden_substrings": [ + "Starting round 21", + "changing the max round limit", + "review is clean despite remaining blockers" + ] + } } ] } diff --git a/docs/harness-design-guide.md b/docs/harness-design-guide.md index 65b2426..9032268 100644 --- a/docs/harness-design-guide.md +++ b/docs/harness-design-guide.md @@ -1,310 +1,250 @@ # Harness Design Guide -Design principles, architecture patterns, and implementation guidelines for building long-running AI agent harnesses. Based on Anthropic's harness research, the Assistant Framework's review system, and lessons from 4 rounds of self-referential review. +Design guide for the Assistant Framework harness controller: the planning, +artifact, review, QA, trace/replay, and recovery layer used for long-running +agent workflows. -**Reference:** [Anthropic Engineering — Harness Design for Long-Running Apps](https://www.anthropic.com/engineering/harness-design-long-running-apps) +**Reference:** [Anthropic Engineering - Harness Design for Long-Running Apps](https://www.anthropic.com/engineering/harness-design-long-running-apps) --- ## Core Concept -A harness is to an AI agent what a pit crew is to a race car. Instead of one agent doing everything (driving, mechanics, strategy), a harness orchestrates **specialized agents** with distinct roles — enabling multi-hour autonomous tasks that far exceed what a single agent can do. +A harness lets one orchestration session coordinate specialized roles without +letting any role quietly absorb every responsibility. The current framework is a +controller, not a fixed role recipe: -The three pillars: -1. **Separation of generation and evaluation** — the builder never judges the building -2. **Structured scoring over open-ended review** — "is this good?" becomes measurable dimensions -3. **Drift detection** — ensuring the evaluator stays honest over multiple rounds +1. **Define done before Build** with an accepted Done Contract. +2. **Select the controller shape** with a Harness Recipe. +3. **Carry state as artifacts** through run-state, trace, replay, and typed refs. +4. **Separate implementation, code review, and QA acceptance** into distinct + responsibilities. +5. **Detect drift and stagnation** and route pivots/restarts through an explicit + decision artifact. ---- - -## Architecture: The Three-Agent Pattern - -| Agent | Role | Access | Analogy | -|---|---|---|---| -| **Planner** | Converts brief prompts into detailed specs | Read-only | The architect drawing blueprints | -| **Generator** | Implements the spec iteratively | Read + Write | The builder on-site | -| **Evaluator** | Tests and scores against criteria | Read-only | The building inspector | +The smallest useful harness still keeps these concerns distinct. Small tasks can +take a lightweight path, but medium+ harness-capable work must not enter Build +without the controller artifacts that make recovery and review possible. -### Why separation matters +--- -When asked to evaluate their own output, models exhibit **systematic leniency** — confidently praising work that, to a human observer, is mediocre. This is like grading your own exam. A standalone evaluator is far more tractable to tune toward healthy skepticism. +## Pre-Build Controller -**Framework implementation:** -- Reviewer agent (`agents/claude/reviewer.md`) has **no Edit or Write access** — structurally cannot modify what it reviews -- Fresh reviewer dispatched each round — no context contamination from previous reviews -- Orchestrator (main session) applies fixes, never the reviewer +### Done Contract -### Guaranteeing 3-agent separation +The Done Contract defines what "finished" means before implementation starts. +It is required for medium+ harness-capable work and contains: -Convention alone is insufficient — agents under pressure collapse roles. Use **structural enforcement**: +- `done_when`: observable outcomes that prove completion +- `not_done_when`: states that must block completion +- `verification`: commands, inspections, review, QA, or manual evidence needed +- `owner_consumer`: owner and downstream consumer of the artifact or behavior +- `acceptance_criteria`: binary criteria from the user, plan, or slice scope +- `debate_record`: at least two perspectives considered before acceptance +- `accepted_by`: user, orchestrator, or approved plan reference -| Enforcement Level | Mechanism | Strength | -|---|---|---| -| Convention | Skills/prompts say "spawn 3 agents" | Weak — easily skipped | -| File gates | Phase artifacts must exist before transition | Medium | -| Hook enforcement | Shell scripts block completion without artifacts | Strong | -| Tool-level access control | Reviewer literally can't edit | Strongest | +When `subagent_execution_mode=delegated`, the debate should use relevant +subagent perspectives such as Architect, Builder/Tester, Code Reviewer, QA +Evaluator, Security, Docs, or Product. Direct fallback must record why subagents +were unavailable or unauthorized and which role-equivalent perspectives were +used. The debate cannot add scope; scope changes return to Plan. -The framework uses all four levels: -- **Convention:** `assistant-workflow` SKILL.md defines the three-agent flow -- **File gates:** Task journal must have plan approval before build, review entries before completion -- **Hook enforcement:** `stop-review.sh` is the consolidated strict stop gate for review, plan approval, rubric scores, and metrics -- **Tool-level:** Reviewer agent tools = `Read, Grep, Glob, LS` (no Edit, Write, or Bash) +### Harness Recipe ---- +The Harness Recipe selects the controller shape from the current +task/model/risk/context profile. It is short routing metadata, not another plan. -## Evaluator Calibration +Required profile fields: -### The problem +- `task_profile`: task type, size, slice count, TDD/debugging needs +- `model_profile`: agent/model constraints, delegation mode, tool limits +- `risk_profile`: risk tier, safety gates, review depth, rollback needs +- `context_profile`: exact context, summarized context, omitted/deferred + context, and trace/replay need -An uncalibrated evaluator is a building inspector who either rubber-stamps everything or fails buildings for paint color. You need the Goldilocks zone. +Required recipe fields: -### Solution: Weighted Rubric with Anchored Examples +- `selected_recipe` +- `recipe_rationale` +- `required_artifacts` +- `corrective_action` -Replace open-ended "find issues" with structured scoring against concrete dimensions: +Recipes normally fall into lightweight guarded, slice-sequential, +review-intensive, or trace/replay-ready variants. If the recipe stops matching +the task during Build, record `>> PLAN DEVIATION DETECTED`, repair the recipe, +and seek re-approval when files, behavior, scope, risk, verification, or +acceptance criteria change. -| Dimension | Weight | What It Measures | -|---|---|---| -| Correctness | 0.30 | Bugs, logic errors, edge cases, acceptance criteria | -| Code Quality | 0.20 | Readability, naming, maintainability, right-sized SOLID/KISS/DRY/YAGNI | -| Architecture | 0.20 | Layer boundaries, dependency direction, patterns | -| Security | 0.15 | Injection, auth, data exposure, OWASP top 10 | -| Test Coverage | 0.15 | New behavior tested, edge cases, test quality | +--- -**Key calibration techniques:** +## Runtime Artifacts -1. **Anchored score examples** — Each dimension has a 1-5 scale with concrete descriptions of what each score looks like. See `skills/assistant-review/references/review-rubric.md` for the full anchor table. +Harness artifacts live in the task journal or equivalent carried-forward state. +They are recovery artifacts, not ceremony. -2. **Evidence-backed scoring** — Every score must cite specific code. "Architecture: 4.0" is not enough; "Architecture: 4.0 — clean layer separation, repository pattern matches existing conventions" is. +| Artifact | Purpose | +|---|---| +| Harness Run State | Current phase, slice, status, blockers, last verification, next action, recovery pointer | +| Trace Ledger | Ordered agent events, decisions, verification results, deviations, blockers, and artifact refs | +| Replay Packet | Minimum continuation packet after compaction, handoff, failure, or restart | +| Artifact Reference Ledger | Typed refs that let producers and consumers validate artifact location, schema, and freshness | +| Pivot/Restart Decision | Orchestrator-owned recovery decision when the active loop or handoff stops making safe progress | + +### Typed Artifact References + +When an artifact crosses an agent boundary, pass it as a typed Artifact +Reference instead of a loose string: + +- `artifact_id` +- `artifact_type` +- `producer` +- `consumer` +- `location_ref` +- `schema_or_contract` +- `validation_status` +- `summary` + +Use typed refs for Done Contract, Harness Recipe, Harness Run State, Trace +Ledger, Replay Packet, Pivot/Restart Decision, task packets, changed files, +verification evidence, review results, QA results, and plan deviations. +Producers create or update refs; consumers validate `schema_or_contract` and +`validation_status` before relying on them. -3. **Critical finding overrides** — Certain findings cap the weighted score regardless of other dimensions: - - Active security vulnerability → capped at 2.0 - - Data loss risk → capped at 2.0 - - Build-breaking change → capped at 1.0 +--- -4. **Threshold actions** — Scores map to concrete decisions: +## Role Separation - | Weighted Score | Action | What Happens | - |---|---|---| - | 4.0+ | **PASS** | Exit clean. Ship it. | - | 3.0–3.9 | **REFINE** | Continue loop. Fix lowest-scoring dimensions. | - | < 3.0 | **PIVOT** | Current approach has fundamental issues. Escalate. | +The controller separates implementation, verification, code review, and QA +responsibilities: -5. **Rising bar per round** — The pivot threshold tightens as rounds progress (2.5 → 2.75 → 3.0 → 3.25). If the code hasn't reached 3.25 by round 4, the approach likely needs rethinking, not more polish. +| Role | Responsibility | Writes Source? | +|---|---|---| +| Code Writer | Implements the approved task packet and reports blockers without broadening scope | Yes | +| Builder/Tester | Runs builds, tests, and verification; requests Code Writer fixes for production failures | Tests/fixtures as assigned, not production code | +| Code Reviewer | Reviews code defects, security, architecture, test coverage, and structural code risk | No | +| QA Evaluator | Evaluates Done Contract, acceptance criteria, verification evidence, final readiness, and scoped domain quality | No | -### Calibration set approach (for future work) +`reviewer` remains a compatibility route for older handoffs, but new code review +dispatches should use `code-reviewer`. QA Evaluator does not replace Code +Reviewer; QA findings are about acceptance and evidence, not general code +quality. Both delegated mode and direct fallback must record Code Reviewer +evidence separately from QA Evaluator evidence. -Build 5-10 pre-scored examples where you've manually assigned scores. Run the evaluator against them and compare. Adjust the evaluator prompt with few-shot corrections where its judgment diverges from human preferences. +### Conditional Domain Rubrics -**Framework implementation:** `skills/assistant-review/references/review-rubric.md` +QA loads `skills/assistant-review/references/domain-rubrics.md` only when the +Done Contract, acceptance criteria, `domain_context`, or explicit `rubric_refs` +scope UI/visual, product, UX, docs, DX, or domain craft quality. When no domain +rubric is scoped, QA records domain quality as not applicable instead of +inventing subjective bars. --- -## The Review Loop - -### Ensuring the reviewer never reviews its own fixes - -This is the most critical architectural constraint. Three mechanisms work together: - -**1. Read-only reviewer (structural)** -``` -Reviewer: tools = [Read, Grep, Glob, LS] ← cannot edit -Fixer: tools = [Read, Edit, Write, Bash] ← separate agent -``` - -**2. Fresh agent each round (context isolation)** -``` -Round 1: Reviewer₁ (fresh) → finds issues → report - Fixer (orchestrator) → applies fixes → tests -Round 2: Reviewer₂ (NEW fresh agent) → reviews with "previously fixed" list - → finds new issues or passes clean -Round N: Repeat until clean or max rounds -``` +## Review And QA Loops -**3. Previously-fixed list (anti-re-reporting)** -Each round receives a list of already-fixed items. The reviewer must not re-report them. This prevents the loop from churning on the same issues while still allowing the reviewer to find genuinely new problems. +The code-review loop and QA loop are bounded at **max 20 rounds**. -### Loop structure - -``` +```text round = 1 previously_fixed = [] score_history = [] -while round <= 10: - REVIEW → dispatch fresh read-only reviewer with diff + previously_fixed - EVALUATE → check rubric score + findings → PASS/REFINE/PIVOT - FIX → orchestrator fixes all must-fix and should-fix items - VERIFY → build + tests must pass +while round <= 20: + REVIEW -> fresh Code Reviewer with diff, acceptance context, and prior fixes + DECIDE -> PASS, REFINE, or PIVOT from rubric score and findings + FIX -> Code Writer fixes actionable findings when REFINE + VERIFY -> Builder/Tester records build/test evidence round += 1 ``` -### Finding filter policy - -Each round reports only findings with file/line evidence, concrete impact, and the smallest useful fix. Speculative or low-evidence concerns go into Observations and do not block completion. - -| Rounds | Blocking bar | Rationale | -|---|---|---| -| 1-7 | Evidence-backed must-fix or should-fix findings | Catch actionable issues without letting speculation drive the loop | -| 8-9 | Must-fix or high-confidence should-fix findings | Reduce late-round noise while preserving real blockers | -| 10 | Terminal round; report remaining blockers as remaining items | Preserve the hard max-round cap and avoid round 11 | - -This prevents late-round noise from prolonging the loop unnecessarily. - -**Framework implementation:** `skills/assistant-review/SKILL.md` (the loop), `contracts/phase-gates.yaml` (assertions per step) - ---- - -## Drift Detection - -### The problem - -Over multiple rounds, evaluators can exhibit **score inflation** — getting "tired" and passing things they shouldn't. The score goes up, but the code didn't actually improve. This is like a building inspector who starts approving more after a long day. - -### Solution: Score tracking with drift classification - -After each round, compare the rubric score to the previous round using two signals: **score delta** and **finding count delta**. - -| Score Delta | Finding Condition | Status | Action | -|---|---|---|---| -| +, ≤ 1.0 | count decreased | **GENUINE** | Continue normally | -| +, > 1.0 | count decreased | **SUSPICIOUS** | Log warning, continue | -| +, any | count same or increased | **DRIFT** | Reset evaluator (stricter prompt) | -| − | any | **REGRESSION** | Investigate, escalate if 2+ rounds | -| 0 | findings > 0, 2+ rounds | **STAGNATION** | Escalate to orchestrator | -| 0 | findings > 0, 1 round | **NEUTRAL** | Log, no action yet | - -### Drift response: evaluator reset - -On DRIFT detection, the next reviewer dispatch includes an explicitly stricter prompt: - -> "Previous rounds showed score inflation without corresponding quality improvement. Apply maximum skepticism. Score conservatively — when uncertain, round DOWN." +QA runs after build/test evidence and Code Reviewer evidence exist: -### Escalation ladder - -| Drift Count | Response | -|---|---| -| 1 occurrence | Reset evaluator context, stricter prompt | -| 2 occurrences | Flag to orchestrator, consider different model | -| 3+ occurrences | Stop loop, present findings for manual review | - -**Framework implementation:** `skills/assistant-review/references/score-tracking.md` - ---- - -## Harness Gate Hooks - -### The problem - -Without structural enforcement, agents under time pressure will collapse the harness — skipping the plan, self-reviewing, or accepting low scores. Convention-based rules are like speed limit signs with no police. - -### Solution: One shell stop hook that blocks completion - -The framework uses one consolidated Stop hook in strict profiles: - -| Hook | What It Checks | When It Blocks | -|---|---|---| -| `stop-review.sh` | Full strict stop lifecycle | Task in BUILDING/VERIFYING/REVIEWING/DOCUMENTING without required plan approval, review entries, final result, medium+ rubric score, or today's metrics | +```text +round = 1 +previously_failed_acceptance_items = [] +score_progression = [] + +while round <= 20: + EVALUATE -> QA Evaluator checks Done Contract, criteria, evidence, domain scope + SCORE -> qa_scorecard and score_entry + DECIDE -> accepted, accepted_with_concerns, rejected, or blocked + FIX/EXIT -> return failed acceptance items to Build, or exit with final result + round += 1 +``` -### Consolidated stop-review.sh strict checks (in order) +Round 20 is terminal. The controller reports remaining blockers or failed +acceptance items instead of starting round 21. -1. **Plan gate:** Task journal has `Plan approval: yes` or `PHASE: PLAN COMPLETE (approved)` → blocks if missing -2. **Rubric gate:** Task journal has `Rubric:` and `Weighted:` lines in review entries → blocks if missing (medium+ only) -3. **Score gate:** Latest weighted score ≥ 4.0 → blocks if below +### Finding Filter -### Size-aware enforcement +Review and QA findings must have concrete evidence and direct impact: -| Task Size | Enforcement | +| Rounds | Blocking bar | |---|---| -| Small/trivial | `stop-review.sh` checks review, final result, and metrics when strict workflow state is active | -| Medium+ | `stop-review.sh` checks the full harness lifecycle: plan, review, rubric, score, and metrics | +| 1-15 | Evidence-backed must-fix or should-fix findings | +| 16-19 | Must-fix or high-confidence should-fix findings | +| 20 | Terminal report of remaining blockers or acceptance items | -The hook detects task size by looking for `Triaged as: medium|large|mega` in the task journal — this is an explicit field in the journal template, not inferred from content. - -### Loop prevention - -The stop hook includes loop guards to prevent infinite blocking: -- **Claude:** `stop_hook_active` flag in input JSON — if true, the hook already fired once, allow stop -- **Gemini:** Temp file flag (`/tmp/.assistant-stop-review-retry-{hash}`) — set on first retry, cleared on second invocation - -Prompt-time runtime gate warnings are injected by `hooks/scripts/workflow-enforcer.sh`, with shared task-journal checks in `hooks/scripts/workflow-phase-gates.sh`. Stop-time blocking is consolidated in `hooks/scripts/stop-review.sh`, registered as the single strict stop gate in all three settings files (Claude, Gemini, Codex). `hooks/scripts/harness-gate.sh` remains only as a legacy compatibility script and reference implementation. +Speculative concerns stay non-blocking unless evidence connects them to +correctness, security, architecture, test reliability, or acceptance criteria. --- -## File-Based Communication - -Agents exchange state through structured files, not pure conversation. This makes state durable, inspectable, and auditable. +## Drift, Stagnation, And Pivot/Restart -### Key artifacts +The controller tracks score progression and finding counts so rising scores do +not mask stale or worsening output. -| Artifact | Location | Purpose | Written By | -|---|---|---|---| -| Task journal | `.claude/task.md` | Single source of truth during task | Orchestrator | -| Context map | `.claude/context-map.md` | Codebase structure for downstream agents | Code Mapper | -| Plan | In task journal `## Plan` section | Approved implementation steps | Architect / Orchestrator | -| Review entries | In task journal `## Review Log` section | Per-round findings, scores, drift status | Orchestrator (from reviewer output) | -| Rubric scores | In review entries `- Rubric:` line | Dimension scores per round | Reviewer → Orchestrator | -| Score progression | In final result | Round-over-round score tracking | Orchestrator | - -### Task journal as gate artifact - -The task journal is not just documentation — it's the enforcement mechanism. Hooks parse the journal to verify: -- Plan exists and is approved (plan gate) -- Review entries exist with rubric scores (rubric gate) -- Weighted score meets threshold (score gate) -- Final result is written (review gate) - -If the journal doesn't contain these artifacts, the agent cannot complete the task. - ---- - -## Context Management - -### Resets vs. Compaction - -Models exhibit **"context anxiety"** — rushing to wrap up as the context window fills. Two strategies: - -| Strategy | How It Works | Pros | Cons | -|---|---|---|---| -| **Compaction** | Summarize earlier conversation in place | Preserves continuity | No clean slate, artifacts of old context | -| **Context reset** | Clear everything, hand off state via files | Clean working memory | Complexity, latency, token overhead | - -The framework uses compaction by default (`pre-compress.sh` / `post-compact.sh` hooks) with file-based state preservation (task journal survives compression). Full context resets are available via fresh agent dispatch. - -### Progressive loading - -Skills load context on demand, not all at once: -- SKILL.md is always loaded (entry point) -- `contracts/` loaded when entering a gated phase -- `references/` loaded when a specific technique is needed (e.g., rubric loaded during review, not during planning) - -This keeps the context window lean and focused. - ---- - -## Design Principles Summary - -| Principle | Implementation | Anti-Pattern | +| Signal | Meaning | Controller Response | |---|---|---| -| Separate generation from evaluation | Read-only reviewer, fresh per round | Same agent reviews its own fixes | -| Convert subjective to gradable | 5-dimension weighted rubric with anchors | "Is this good?" open-ended review | -| Enforce structurally, not just conventionally | Hook-based gates that parse artifacts | "Please remember to review" in the prompt | -| Detect drift over time | Score tracking with finding correlation | Trusting round N score without comparing to round N-1 | -| Manage context through resets | Task journal survives compression | Relying on conversation history alone | -| Right-size, don't skip | Small = lightweight, medium = standard, never zero | Skipping the harness for speed | +| GENUINE | Score improves and finding count decreases | Continue normally | +| SUSPICIOUS | Score jumps sharply while findings drop | Log and continue skeptically | +| DRIFT | Score improves while findings stay flat or rise | Reset evaluator with stricter context | +| REGRESSION | Score drops | Investigate; repeated regression triggers pivot/restart | +| STAGNATION | Findings remain across repeated unchanged scores | Create pivot/restart signal | + +The orchestrator owns `pivot_restart_decision`. It records the trigger, evidence, +affected slice or round, options considered, selected action, reapproval need, +next agent, recovery pointer, and exact next action. After the decision, update +Harness Run State, append a trace entry, refresh Replay Packet, and update the +Artifact Reference Ledger. + +### Code Writer Blocker Routing + +Code Writer must report unexpected blockers instead of patching around them. +Controller routing: + +- `legacy_code_bug` or `broken_baseline` -> assistant-debugging before another + implementation attempt +- `hidden_dependency` -> Explorer or Code Mapper refresh +- `missing_contract` or `stale_plan` -> Architect or Plan repair +- `scope_conflict` -> replan and reapproval when scope changes +- `tool_environment` or `permission_policy` -> environment fix, permission + request, or BLOCKED with evidence +- `tdd_red_missing` -> return to TDD RED evidence before production code is + accepted + +Review/QA stagnation, repeated drift/regression, domain action `pivot`, +verification blockers, and stale task packets all route through the same +Pivot/Restart Controller. Do not silently continue the loop after these signals. --- -## When to Evolve the Harness - -Each harness component **encodes assumptions about model limitations**. As models improve, stress-test whether each piece is still earning its keep: +## Gate Enforcement -- **Sprint decomposition** was valuable with earlier models but became unnecessary with improved long-context planning (Opus 4.6 finding from Anthropic's research) -- **Rubric scoring** persists in value — tasks at the frontier of model capability still benefit from structured evaluation -- **Drift detection** becomes more important as loops get longer — the longer the session, the higher the drift risk -- **Harness gates** remain valuable as a safety net — even when models are capable, structural enforcement prevents regressions under edge conditions +The current enforcement path is: -The interesting work evolves not toward simpler harnesses, but toward discovering novel agent combinations enabling previously impossible tasks. +- `assistant-workflow` loads `references/harness-controller.md` only for + medium+ harness-capable work. +- Workflow contracts require Done Contract and Harness Recipe before Build. +- Task journal templates carry Harness Run State, Trace Ledger, Replay Packet, + Pivot/Restart Decision, and Artifact Reference Ledger sections. +- `workflow-enforcer.sh` and `workflow-phase-gates.sh` surface + `Prompt-time runtime gate warnings` for + `BUILDING/VERIFYING/REVIEWING/DOCUMENTING`. +- `stop-review.sh` is the consolidated strict stop gate for plan approval, + review, rubric/score, metrics, and final-result completion. +- `harness-gate.sh` remains only a legacy compatibility/reference script. --- @@ -312,14 +252,15 @@ The interesting work evolves not toward simpler harnesses, but toward discoverin | Component | Key Files | |---|---| -| Rubric definition | `skills/assistant-review/references/review-rubric.md` | -| Score tracking | `skills/assistant-review/references/score-tracking.md` | -| Review loop | `skills/assistant-review/SKILL.md` | -| Review contracts | `skills/assistant-review/contracts/{input,output,phase-gates,handoffs}.yaml` | -| Reviewer agent | `agents/claude/reviewer.md` | -| Harness gate hook | `hooks/scripts/harness-gate.sh` | -| Review gate hook | `hooks/scripts/stop-review.sh` | -| Hook registration | `hooks/{claude,gemini,codex}-settings.json` | +| Harness controller reference | `skills/assistant-workflow/references/harness-controller.md` | +| Workflow phase execution | `skills/assistant-workflow/references/phases.md` | | Task journal template | `skills/assistant-workflow/references/task-journal-template.md` | -| Workflow phase gates | `skills/assistant-workflow/contracts/phase-gates.yaml` | +| Workflow contracts | `skills/assistant-workflow/contracts/{input,output,phase-gates,handoffs}.yaml` | +| Code review loop | `skills/assistant-review/SKILL.md` | +| QA loop | `skills/assistant-review/references/qa-evaluation-loop.md` | +| Domain rubrics | `skills/assistant-review/references/domain-rubrics.md` | +| Review rubric and score tracking | `skills/assistant-review/references/{review-rubric,score-tracking}.md` | +| Code Reviewer agents | `agents/{claude,codex}/code-reviewer.*` | +| QA Evaluator agents | `agents/{claude,codex}/qa-evaluator.*` | +| Runtime hooks | `hooks/scripts/{workflow-enforcer,workflow-phase-gates,stop-review}.sh` | | Contract design guide | `docs/skill-contract-design-guide.md` | diff --git a/docs/skill-contract-design-guide.md b/docs/skill-contract-design-guide.md index 34fe470..5a453fa 100644 --- a/docs/skill-contract-design-guide.md +++ b/docs/skill-contract-design-guide.md @@ -229,6 +229,19 @@ handoffs: - Handoff schemas ensure subagents get proper context and return structured data - Example: assistant-workflow, assistant-review +For loop-based Process skills, model the harness controller explicitly instead +of hiding it in prose. Medium+ harness-capable work should define a Done +Contract and Harness Recipe before Build, carry typed Artifact References across +handoffs, and record run-state, trace, replay, review, QA, and pivot/restart +artifacts when the controller requires them. Code Reviewer and QA Evaluator +handoffs stay separate: Code Reviewer owns code defects, security, +architecture, test coverage, and structural code risk; QA Evaluator owns Done +Contract, acceptance criteria, verification evidence, final readiness, and +scoped domain quality. Review/QA loops must be bounded, currently with a max 20 +round terminal cap, and stagnation, repeated drift/regression, rubric pivots, or +Code Writer unexpected blockers must route through an explicit +`pivot_restart_decision` rather than silently continuing. + ### Analysis skills (3 files) - Multi-step pipeline (e.g., diverge → converge → refine) - Phase gates enforce pipeline ordering @@ -316,3 +329,11 @@ Local response grading is deterministic and heuristic: missing files, empty resp 8. **Conditional fields use `condition:`** — don't make everything required; scope to when it matters 9. **Examples clarify ambiguous fields** — when `description` alone isn't enough, add `examples:` 10. **Cross-phase invariants catch slow drift** — things that must ALWAYS be true, not just at gates +11. **Root SKILL.md stays compact and outcome-shaped** — detailed controller + patterns live in references loaded only when the active task needs them +12. **Clarification prompts are admissible** — ask only for material, + non-discoverable missing data with no safe default +13. **Loop controllers are explicit and bounded** — use Done Contract, Harness + Recipe, typed refs, separate code-review/QA handoffs, pivot/restart + decisions, and the max 20 terminal cap when the Process skill has + long-running review, QA, or fix-verify loops diff --git a/hooks/scripts/stop-review.sh b/hooks/scripts/stop-review.sh index ba63498..9517f79 100755 --- a/hooks/scripts/stop-review.sh +++ b/hooks/scripts/stop-review.sh @@ -102,7 +102,7 @@ fi subagent_missing_key="$(assistant_phase_subagent_evidence_missing_reason_key "$TASK_FILE")" if [[ "$subagent_missing_key" != "complete" ]]; then - SUBAGENT_REASON="Task journal subagent evidence gate failed ($subagent_missing_key). If authorization is required, ask once for the needed subagent delegation scope and wait before continuing phases that require subagents. For delegated mode, record Agent Dispatch Log evidence with dispatch/result entries for every required workflow role (Code Mapper/Explorer/Architect during discovery/decomposition/planning when required, Code Writer and Builder/Tester during Build when required, and Reviewer during Review). Medium+ implementation work also needs per-slice dispatch evidence when implementation slices exist. For direct_fallback, record an explicit reason (authorization_denied, subagents_unavailable, or policy_disallowed) plus role-equivalent direct evidence for every required workflow role. Silent fallback, unresolved authorization, inline review in delegated mode, or not_applicable with required roles cannot complete." + SUBAGENT_REASON="Task journal subagent evidence gate failed ($subagent_missing_key). If authorization is required, ask once for the needed subagent delegation scope and wait before continuing phases that require subagents. For delegated mode, record Agent Dispatch Log evidence with dispatch/result entries for every required workflow role (Code Mapper/Explorer/Architect during discovery/decomposition/planning when required, Code Writer and Builder/Tester during Build when required, and Code Reviewer during Review; legacy Reviewer labels are compatibility routing only). Medium+ implementation work also needs per-slice dispatch evidence when implementation slices exist. For direct_fallback, record an explicit reason (authorization_denied, subagents_unavailable, or policy_disallowed) plus role-equivalent direct evidence for every required workflow role, including Code Reviewer direct evidence during Review. Silent fallback, unresolved authorization, inline review in delegated mode, or not_applicable with required roles cannot complete." if $IS_GEMINI; then touch "${RETRY_FLAG}" jq -n --arg reason "$SUBAGENT_REASON" '{decision: "retry", reason: $reason}' @@ -127,10 +127,18 @@ elif [[ "$review_missing_key" == "no_quality_review" ]]; then REVIEW_REASON="Task journal has Spec Review PASS but no Quality Review. You MUST run Stage 2 separately: load assistant-review SKILL.md and contracts, run the autonomous quality review loop, and append a Quality Review entry. Quality review cannot substitute for Spec Review, and Spec Review cannot substitute for quality review." elif [[ "$review_missing_key" == "no_final_result" ]]; then REVIEW_REASON="Task journal has review entries but the review cycle is not complete — no Final Result found. You must finish the review cycle: fix remaining must-fix issues, re-test, re-review, and write the Final Result summary in the Review Log section of the task journal." +elif [[ "$review_missing_key" == "qa_rejected" ]]; then + REVIEW_REASON="QA required and the latest QA Evaluation was rejected (qa_rejected). Fix the QA findings, rerun QA, and record QA Evaluation evidence; the QA final verdict/result must be accepted or accepted_with_concerns before stopping." +elif [[ "$review_missing_key" == "qa_blocked" ]]; then + REVIEW_REASON="QA required and the latest QA Evaluation is blocked (qa_blocked). Resolve the QA blocker, rerun QA, and record QA Evaluation evidence; the QA final verdict/result must be accepted or accepted_with_concerns before stopping." +elif [[ "$review_missing_key" == "qa_final_result_missing" ]]; then + REVIEW_REASON="QA required but the QA final verdict/result is missing (qa_final_result_missing). Run or rerun QA and record QA Evaluation evidence; the QA final verdict/result must be accepted or accepted_with_concerns before stopping." +elif [[ "$review_missing_key" == "qa_not_accepted" ]]; then + REVIEW_REASON="QA required but the QA final verdict/result is not accepted (qa_not_accepted). Fix or rerun QA and record QA Evaluation evidence; the QA final verdict/result must be accepted or accepted_with_concerns before stopping." elif [[ "$review_missing_key" == "missing_review_round" ]]; then - REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_review_round). Add '- Round: N of 10' to the latest Quality Review entry after the latest Spec Review PASS." + REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_review_round). Add '- Round: N of 20' to the latest Quality Review entry after the latest Spec Review PASS." elif [[ "$review_missing_key" == "round_overflow" ]]; then - REVIEW_REASON="Medium+ Quality Review has invalid controller evidence (round_overflow). Use '- Round: N of 10' with N between 1 and 10, then rerun or repair the review loop." + REVIEW_REASON="Medium+ Quality Review has invalid controller evidence (round_overflow). Use '- Round: N of 20' with N between 1 and 20, then rerun or repair the review loop." elif [[ "$review_missing_key" == "missing_findings_summary" ]]; then REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_findings_summary). Add a findings summary such as '- Found this round: 0 must-fix, 0 should-fix, 0 nits' to the latest Quality Review." elif [[ "$review_missing_key" == "missing_rubric_scores" ]]; then diff --git a/hooks/scripts/subagent-monitor.sh b/hooks/scripts/subagent-monitor.sh index 2f7b2e1..f286e21 100755 --- a/hooks/scripts/subagent-monitor.sh +++ b/hooks/scripts/subagent-monitor.sh @@ -41,6 +41,8 @@ fi role_constraint="" case "$AGENT_NAME" in + code-reviewer) role_constraint="SUBAGENT CONSTRAINT: You are a code reviewer. Read-only code/security/architecture/test-coverage review. Do NOT edit any files. Report findings only." ;; + qa-evaluator) role_constraint="SUBAGENT CONSTRAINT: You are a QA evaluator. Read-only acceptance, Done Contract, verification evidence, domain quality, score progression, and final result evaluation. Do NOT edit any files. Do NOT replace code-reviewer." ;; reviewer) role_constraint="SUBAGENT CONSTRAINT: You are a reviewer. Do NOT edit any files. Report findings only." ;; architect) role_constraint="SUBAGENT CONSTRAINT: You are an architect. Do NOT write implementation code. Design only." ;; explorer) role_constraint="SUBAGENT CONSTRAINT: You are an explorer. Read-only analysis. Do NOT modify any files." ;; @@ -93,7 +95,8 @@ fi role_name="" case "$AGENT_NAME" in - reviewer) role_name="Reviewer" ;; + code-reviewer|reviewer) role_name="Reviewer" ;; + qa-evaluator) role_name="QAEvaluator" ;; architect) role_name="Architect" ;; explorer) role_name="Explorer" ;; code-mapper) role_name="CodeMapper" ;; diff --git a/hooks/scripts/workflow-enforcer.sh b/hooks/scripts/workflow-enforcer.sh index 3f604e2..f65d780 100755 --- a/hooks/scripts/workflow-enforcer.sh +++ b/hooks/scripts/workflow-enforcer.sh @@ -189,7 +189,7 @@ if [[ -z "$TASK_FILE" ]] || assistant_task_journal_completed "$TASK_FILE"; then if [[ "$STATE_DIR" == ".codex" ]] && assistant_codex_prompt_looks_like_dev_work; then if [[ "$codex_subagent_decision" == "none" ]]; then emit_workflow_rules_context "CODEX SUBAGENT AUTHORIZATION (ask-once): -- Ask once for the needed delegation scope and WAIT before responsibilities that require Code Mapper, Explorer, Architect, Code Writer, Builder/Tester, or Reviewer. +- Ask once for the needed delegation scope and WAIT before responsibilities that require Code Mapper, Explorer, Architect, Code Writer, Builder/Tester, Code Reviewer, or QA Evaluator (legacy Reviewer labels are compatibility routing only). - Authorization examples: 'Use delegation', 'use delegation when possible', 'delegate work', 'use agents', 'spawn agents', 'approve subagents for this task'. - Denial examples: 'no delegation', 'do not delegate', 'no agents', 'do not use agents', 'deny subagents and use direct fallback'. - Do not hard block this first prompt. Ask, then wait before delegated responsibilities." @@ -419,7 +419,7 @@ context="WORKFLOW STATE (auto-injected every prompt): PHASE RULES: - Current phase is $status — stay until exit criteria pass. - PLAN approval before BUILD; BUILD includes same-step tests for new components. -- REVIEW loops review -> fix -> re-review until clean or max 10 rounds. +- REVIEW loops review -> fix -> re-review until clean or max 20 rounds. - State the current phase before the next action." context+=" @@ -471,7 +471,7 @@ elif [[ "$subagent_policy_state" == "authorization_required" ]]; then SUBAGENT AUTHORIZATION GATE: - Assistant Framework policy requires explicit user authorization before spawning subagents for workflow roles. - Ask once for the needed delegation scope and WAIT for approval or denial. -- Do not continue Discovery/Decompose/Plan/Build/Review responsibilities that require Code Mapper, Explorer, Architect, Code Writer, Builder/Tester, or Reviewer until authorization is resolved. +- Do not continue Discovery/Decompose/Plan/Build/Review responsibilities that require Code Mapper, Explorer, Architect, Code Writer, Builder/Tester, Code Reviewer, or QA Evaluator (legacy Reviewer labels are compatibility routing only) until authorization is resolved. - Do not switch to direct_fallback unless the user denies authorization, policy disallows spawning, or a real spawn attempt proves subagents unavailable." fi diff --git a/hooks/scripts/workflow-phase-gates.sh b/hooks/scripts/workflow-phase-gates.sh index 5d409e0..a3abcc4 100644 --- a/hooks/scripts/workflow-phase-gates.sh +++ b/hooks/scripts/workflow-phase-gates.sh @@ -742,7 +742,7 @@ assistant_phase_review_controller_missing_reason_key() { fi round="${BASH_REMATCH[1]}" max_round="${BASH_REMATCH[2]}" - if [[ "$max_round" -ne 10 || "$round" -lt 1 || "$round" -gt 10 ]]; then + if [[ "$max_round" -ne 20 || "$round" -lt 1 || "$round" -gt 20 ]]; then printf 'round_overflow\n' return 0 fi @@ -1139,10 +1139,122 @@ assistant_phase_review_complete() { [[ "$(assistant_phase_review_missing_reason_key "$file")" == "complete" ]] } +assistant_phase_latest_qa_final_result() { + local file="$1" + local minimum_line="${2:-0}" + awk -v minimum_line="$minimum_line" ' + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + function template_placeholder(value, low) { + low = tolower(trim(value)) + gsub(/`/, "", low) + return low ~ /^\[[^]]*\]$/ || low ~ /\|/ + } + function normalize_result(value, low) { + if (template_placeholder(value)) return "" + low = tolower(trim(value)) + gsub(/`/, "", low) + if (low ~ /(^|[^[:alnum:]_])not[ _-]?accepted([^[:alnum:]_]|$)/) return "not_accepted" + if (low ~ /(^|[^[:alnum:]_])accepted[ _-]?with[ _-]?concerns([^[:alnum:]_]|$)/) return "accepted_with_concerns" + if (low ~ /(^|[^[:alnum:]_])accepted([^[:alnum:]_]|$)/) return "accepted" + if (low ~ /(^|[^[:alnum:]_])rejected([^[:alnum:]_]|$)/) return "rejected" + if (low ~ /(^|[^[:alnum:]_])blocked([^[:alnum:]_]|$)/) return "blocked" + if (low ~ /(^|[^[:alnum:]_])not[ _-]?required([^[:alnum:]_]|$)/) return "not_required" + return "" + } + function record_result(value) { + latest = normalize_result(value) + } + function compact_final_marker(value, low) { + low = tolower(value) + return low ~ /(final[ _-]?verdict|qa[ _-]?result)/ + } + BEGIN { + minimum_line += 0 + } + NR <= minimum_line { + next + } + /^### QA Evaluation #[0-9]+/ { + latest = "" + next + } + { + line = $0 + sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", line) + low = tolower(line) + if (low ~ /^final verdict[[:space:]]*:/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + record_result(value) + next + } + if (low ~ /^qa result[[:space:]]*:/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + record_result(value) + next + } + if (low ~ /^qa evaluator result[[:space:]]*:/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + if (compact_final_marker(value) || normalize_result(value) != "") { + record_result(value) + } + } + } + END { + if (latest != "") { + print latest + exit 0 + } + exit 1 + } + ' "$file" 2>/dev/null +} + +assistant_phase_qa_final_result_missing_reason_key() { + local file="$1" + local minimum_line="${2:-0}" + local qa_result + + if ! assistant_phase_requires_qa_evaluator "$file"; then + printf 'complete\n' + return 0 + fi + + qa_result="$(assistant_phase_latest_qa_final_result "$file" "$minimum_line" || true)" + case "$qa_result" in + accepted|accepted_with_concerns) + printf 'complete\n' + ;; + rejected) + printf 'qa_rejected\n' + ;; + blocked) + printf 'qa_blocked\n' + ;; + "") + printf 'qa_final_result_missing\n' + ;; + not_accepted) + printf 'qa_not_accepted\n' + ;; + *) + printf 'qa_not_accepted\n' + ;; + esac +} + assistant_phase_review_missing_reason_key() { local file="$1" local spec_pass_line local quality_review_line + local review_controller_reason + local qa_reason if ! assistant_phase_has_spec_review_entry "$file"; then printf 'no_spec_review\n' @@ -1162,7 +1274,12 @@ assistant_phase_review_missing_reason_key() { fi if assistant_phase_is_medium_plus "$file"; then - assistant_phase_review_controller_missing_reason_key "$file" "$quality_review_line" "$spec_pass_line" + review_controller_reason="$(assistant_phase_review_controller_missing_reason_key "$file" "$quality_review_line" "$spec_pass_line")" + if [[ "$review_controller_reason" != "complete" ]]; then + printf '%s\n' "$review_controller_reason" + return 0 + fi + assistant_phase_qa_final_result_missing_reason_key "$file" "$quality_review_line" return 0 fi @@ -1171,6 +1288,12 @@ assistant_phase_review_missing_reason_key() { return 0 fi + qa_reason="$(assistant_phase_qa_final_result_missing_reason_key "$file" "$quality_review_line")" + if [[ "$qa_reason" != "complete" ]]; then + printf '%s\n' "$qa_reason" + return 0 + fi + printf 'complete\n' } @@ -1232,7 +1355,25 @@ assistant_phase_has_labeled_evidence() { value="$(assistant_phase_labeled_evidence_value "$file" "$label")" [[ -n "$value" ]] || return 1 - [[ ! "$value" =~ ^(\[.*\]|none|None|NONE|n/a|N/A|missing|todo|TODO|tbd|TBD)$ ]] + ! assistant_phase_labeled_evidence_value_is_placeholder "$value" +} + +assistant_phase_labeled_evidence_value_is_placeholder() { + local value="$1" + local low + + value="$(assistant_phase_trim_value "$value")" + [[ -n "$value" ]] || return 0 + assistant_phase_value_is_noneish "$value" && return 0 + [[ "$value" =~ ^\[.*\]$ ]] && return 0 + + low="$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')" + [[ "$low" =~ ^(pending|waiting|missing|todo|tbd|none)([[:space:][:punct:]]|$) ]] && return 0 + [[ "$low" =~ ^n[/.]?a([[:space:][:punct:]]|$) ]] && return 0 + [[ "$low" =~ ^in[[:space:]_-]?progress([[:space:][:punct:]]|$) ]] && return 0 + [[ "$low" =~ ^not[[:space:]_-]?yet([[:space:][:punct:]]|$) ]] && return 0 + [[ "$low" =~ ^not[[:space:]_-]?(required|applicable)([[:space:][:punct:]]|$) ]] && return 0 + return 1 } assistant_phase_is_codex_task() { @@ -1252,7 +1393,9 @@ assistant_phase_role_agent_pattern() { "Architect") printf 'architect|Architect' ;; "Code Writer") printf 'code-writer|codewriter|Code Writer' ;; "Builder/Tester") printf 'builder-tester|builder/tester|Builder/Tester' ;; - "Reviewer") printf 'reviewer|Reviewer' ;; + "Code Reviewer") printf 'code-reviewer|codereviewer|Code Reviewer|reviewer|Reviewer' ;; + "QA Evaluator") printf 'qa-evaluator|qaevaluator|QA Evaluator' ;; + "Reviewer") printf 'reviewer|Reviewer|code-reviewer|codereviewer|Code Reviewer' ;; *) printf '%s' "$1" ;; esac } @@ -1336,7 +1479,21 @@ assistant_phase_has_role_stop_event_evidence() { assistant_phase_has_role_event_pair_evidence "$1" "$2" } -assistant_phase_has_role_dispatch_result_evidence() { +assistant_phase_role_evidence_labels() { + case "$1" in + "Code Reviewer") + printf 'Code Reviewer\nReviewer\n' + ;; + "Reviewer") + printf 'Code Reviewer\nReviewer\n' + ;; + *) + printf '%s\n' "$1" + ;; + esac +} + +assistant_phase_has_single_role_dispatch_result_evidence() { local file="$1" local role="$2" @@ -1356,29 +1513,156 @@ assistant_phase_has_role_dispatch_result_evidence() { return 0 } +assistant_phase_has_role_dispatch_result_evidence() { + local file="$1" + local role="$2" + local evidence_role + + while IFS= read -r evidence_role; do + [[ -n "$evidence_role" ]] || continue + assistant_phase_has_single_role_dispatch_result_evidence "$file" "$evidence_role" && return 0 + done < <(assistant_phase_role_evidence_labels "$role") + + return 1 +} + assistant_phase_direct_fallback_reason_valid() { local file="$1" grep -qiE "^[[:space:]]*[-*]?[[:space:]]*Direct fallback reason:[[:space:]]*(authorization_denied|subagents_unavailable|policy_disallowed)([[:space:]]|$)" "$file" 2>/dev/null } -assistant_phase_has_role_equivalent_evidence() { +assistant_phase_has_single_role_equivalent_evidence() { local file="$1" local role="$2" assistant_phase_has_labeled_evidence "$file" "$role direct evidence" } +assistant_phase_has_role_equivalent_evidence() { + local file="$1" + local role="$2" + local evidence_role + + while IFS= read -r evidence_role; do + [[ -n "$evidence_role" ]] || continue + assistant_phase_has_single_role_equivalent_evidence "$file" "$evidence_role" && return 0 + done < <(assistant_phase_role_evidence_labels "$role") + + return 1 +} + assistant_phase_has_per_slice_dispatch_evidence() { local file="$1" ! assistant_phase_is_medium_plus "$file" && return 0 assistant_phase_has_labeled_evidence "$file" "Per-slice dispatch evidence" } +assistant_phase_requires_qa_evaluator() { + local file="$1" + awk ' + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + function explicit_not_required(value, low) { + low = tolower(trim(value)) + return low ~ /^(n\/a|na|not_required|not required|not-required|not-applicable|not_applicable|not applicable)([[:space:][:punct:]]|$)/ + } + function bracket_placeholder(value) { + value = trim(value) + return value ~ /^\[[^]]*\]$/ + } + function qa_mode_required_line(line, low) { + low = tolower(trim(line)) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", low) + return low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]*[:=][[:space:]]*required([[:space:][:punct:]]|$)/ || + low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]+required([[:space:][:punct:]]|$)/ + } + function qa_mode_not_required_line(line, low) { + low = tolower(trim(line)) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", low) + return low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]*[:=][[:space:]]*(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ || + low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]+(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ + } + function line_marks_not_required(line, low) { + low = tolower(line) + return low ~ /(^|[[:space:][:punct:]])(n\/a|not_required|not required|not-required|not-applicable|not_applicable|not applicable)([[:space:][:punct:]]|$)/ + } + function qa_marker(line, low) { + low = tolower(line) + return low ~ /(qa[ _-]?evaluator|qaevaluator|qa[ _-]?evaluation|qa_evaluation_mode|qa[ _-]?loop)/ + } + function scan_required_line(line) { + if (qa_marker(line) && !line_marks_not_required(line)) { + found = 1 + } + } + function scan_qa_label(line, value, low) { + sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", line) + low = tolower(line) + if (low ~ /^(qa evaluator dispatch|qa evaluator result|qa evaluator direct evidence):/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + if (!bracket_placeholder(value) && !explicit_not_required(value)) { + found = 1 + } + } + } + { + line = $0 + low = tolower(line) + if (qa_mode_required_line(line)) { + required_mode = 1 + found = 1 + next + } + if (qa_mode_not_required_line(line)) { + not_required_mode = 1 + next + } + + scan_qa_label(line) + + if (low ~ /^required agents:[[:space:]]*$/) { + in_required_agents = 1 + in_required_gates = 0 + next + } + if (low ~ /^required gates:[[:space:]]*$/) { + in_required_gates = 1 + in_required_agents = 0 + next + } + if (low ~ /^required agents:[[:space:]]*.+$/ || + low ~ /^required gates:[[:space:]]*.+$/) { + scan_required_line(line) + next + } + if ((in_required_agents || in_required_gates) && line ~ /^[[:space:]]*-[[:space:]]+/) { + scan_required_line(line) + next + } + if ((in_required_agents || in_required_gates) && line ~ /^[^[:space:]-]/) { + in_required_agents = 0 + in_required_gates = 0 + } + } + END { + if (not_required_mode && !required_mode) { + exit 1 + } + exit found ? 0 : 1 + } + ' "$file" 2>/dev/null +} + assistant_phase_required_subagent_roles() { local file="$1" - local status mode + local status mode qa_required status="$(assistant_phase_status "$file" | tr '[:upper:]' '[:lower:]' || true)" mode="$(assistant_phase_subagent_mode "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" - awk -v is_medium_plus="$(assistant_phase_is_medium_plus "$file" && printf yes || printf no)" -v status="$status" -v mode="$mode" ' + qa_required="$(assistant_phase_requires_qa_evaluator "$file" && printf yes || printf no)" + awk -v is_medium_plus="$(assistant_phase_is_medium_plus "$file" && printf yes || printf no)" -v status="$status" -v mode="$mode" -v qa_required="$qa_required" ' function emit(role) { if (!seen[role]) { seen[role] = 1 @@ -1392,7 +1676,12 @@ assistant_phase_required_subagent_roles() { if (low ~ /architect/) emit("Architect") if (low ~ /code writer|code-writer/) emit("Code Writer") if (low ~ /builder\/tester|builder-tester/) emit("Builder/Tester") - if (low ~ /reviewer/) emit("Reviewer") + if (low ~ /qa[ _-]?evaluator|qaevaluator/) emit("QA Evaluator") + if (low ~ /code[ _-]?reviewer|codereviewer/) { + emit("Code Reviewer") + } else if (low ~ /(^|[^[:alnum:]_-])reviewer([^[:alnum:]_-]|$)/) { + emit("Code Reviewer") + } } BEGIN { # Medium+ discovery always requires a Code Mapper context map once @@ -1402,7 +1691,8 @@ assistant_phase_required_subagent_roles() { # Once a delegated/fallback task is in Review/Document, the review # role is required even for no-op/no-code-change outcomes; otherwise # "review phase" can be satisfied inline while claiming delegated mode. - if (mode ~ /^(delegated|direct_fallback)$/ && status ~ /(reviewing|documenting)/) emit("Reviewer") + if (mode ~ /^(delegated|direct_fallback)$/ && status ~ /(reviewing|documenting)/) emit("Code Reviewer") + if (qa_required == "yes") emit("QA Evaluator") } /^Required agents:[[:space:]]*$/ { in_required = 1; next } /^Required agents:[[:space:]]*(.+)$/ { @@ -1443,7 +1733,7 @@ assistant_phase_subagent_evidence_missing_reason_key() { # Strict subagent evidence applies whenever workflow subagent roles are # declared, not only when source code changed. Discovery/review-only work can # legitimately skip Code Writer and Builder/Tester, but delegated Code Mapper, - # Explorer, Architect, or Reviewer responsibilities still need evidence. + # Explorer, Architect, or Code Reviewer responsibilities still need evidence. if [[ -z "$roles" ]]; then printf 'complete\n' return 0 diff --git a/install.sh b/install.sh index 1273712..1e67424 100755 --- a/install.sh +++ b/install.sh @@ -728,7 +728,7 @@ codex_skill_table_description() { assistant-onboard) printf '%s\n' "Systematic project orientation" ;; assistant-reflexion) printf '%s\n' "Post-task learning and calibration" ;; assistant-research) printf '%s\n' "Tiered research with source verification" ;; - assistant-review) printf '%s\n' "Autonomous review-fix loop (max 10 rounds)" ;; + assistant-review) printf '%s\n' "Autonomous review-fix loop (max 20 rounds)" ;; assistant-security) printf '%s\n' "STRIDE, OWASP, CVE analysis" ;; assistant-skill-creator) printf '%s\n' "Create V1 framework skills" ;; assistant-tdd) printf '%s\n' "Red-Green-Refactor with verification gates" ;; @@ -1872,14 +1872,14 @@ if [[ "$AGENT" == "codex" ]]; then ## Role -You are an orchestrator for development work. Coordinate specialized agents (code-mapper, code-writer, builder-tester, architect, explorer, reviewer), keep phase gates visible, and communicate progress. Discovery context maps are owned by code-mapper for medium+ work, file edits/code implementation are owned by code-writer, builds/tests by builder-tester, and independent review by reviewer when subagent delegation is authorized and available; framework-owned state artifacts such as .codex/task.md, .codex/context-map.md, .codex/session.md, and .codex/working-buffer.md are owned by the orchestrator. Assistant Framework policy requires explicit user authorization before spawning subagents for any development/code-work role unless the current user prompt already explicitly authorizes subagents. Ask once for the needed delegation scope, then wait before continuing phases that require subagents. Current Codex CLI/app releases support native subagent workflows by default; custom agents are configured in ~/.codex/agents/ and spawned by explicitly asking Codex to spawn an agent by name. Do not treat the absence of a visible tool named Task, delegate, or subagent as proof that subagents are unavailable. After approval, spawn the requested Codex agents and set subagent_execution_mode=delegated. Use direct fallback only when authorization is denied, a real spawn attempt fails with an unavailable-agent error, or policy disallows spawning, and record equivalent role, phase, verification, and review evidence. Gather context, clarify requirements, decompose work, persist workflow state, and prepare handoffs yourself when that does not modify project source files. Follow matching skill instructions, phase gates, and review loops exactly. When a skill matches your task, invoke it instead of replacing it with ad hoc steps. +You are an orchestrator for development work. Coordinate specialized agents (code-mapper, code-writer, builder-tester, architect, explorer, code-reviewer, reviewer, qa-evaluator), keep phase gates visible, and communicate progress. Discovery context maps are owned by code-mapper for medium+ work, file edits/code implementation are owned by code-writer, builds/tests by builder-tester, independent code/security/architecture review by code-reviewer when subagent delegation is authorized and available, and independent QA acceptance evaluation by qa-evaluator after build/test and code-review evidence when applicable; reviewer remains a compatibility route for existing handoffs. Framework-owned state artifacts such as .codex/task.md, .codex/context-map.md, .codex/session.md, and .codex/working-buffer.md are owned by the orchestrator. Assistant Framework policy requires explicit user authorization before spawning subagents for any development/code-work role unless the current user prompt already explicitly authorizes subagents. Ask once for the needed delegation scope, then wait before continuing phases that require subagents. Current Codex CLI/app releases support native subagent workflows by default; custom agents are configured in ~/.codex/agents/ and spawned by explicitly asking Codex to spawn an agent by name. Do not treat the absence of a visible tool named Task, delegate, or subagent as proof that subagents are unavailable. After approval, spawn the requested Codex agents and set subagent_execution_mode=delegated. Use direct fallback only when authorization is denied, a real spawn attempt fails with an unavailable-agent error, or policy disallows spawning, and record equivalent role, phase, verification, and review evidence. Gather context, clarify requirements, decompose work, persist workflow state, and prepare handoffs yourself when that does not modify project source files. Follow matching skill instructions, phase gates, and review loops exactly. When a skill matches your task, invoke it instead of replacing it with ad hoc steps. These rules define the operating contract for every response. 1. SKILL ROUTING: Before acting on ANY request, check if it matches an installed skill in ~/.codex/skills/. When a skill matches, load and follow the skill's SKILL.md before proceeding; use the skill workflow as the source of truth. -2. ORCHESTRATOR OWNERSHIP: Coordinate the work; keep discovery context mapping with code-mapper for medium+ work, project source edits and code implementation with code-writer, builds/tests with builder-tester, and independent review with reviewer when subagent delegation is authorized and available. Assistant Framework policy requires explicit user authorization before spawning subagents for development/code-work roles unless the current user prompt already explicitly authorizes subagents. Ask once before spawning and wait before continuing phases that require subagents; after approval, use delegated mode. For Codex, spawn custom agents by explicitly asking Codex to spawn the configured agent name (for example, code-mapper, code-writer, builder-tester, reviewer); a hidden/implicit subagent capability may not appear as a normal tool in the visible tool list. Use direct fallback only for explicit authorization_denied, subagents_unavailable after a real spawn failure, or policy_disallowed. The orchestrator may create and update framework-owned state artifacts such as .codex/task.md, .codex/context-map.md, .codex/session.md, and .codex/working-buffer.md; it does not edit project source files directly in delegated mode. +2. ORCHESTRATOR OWNERSHIP: Coordinate the work; keep discovery context mapping with code-mapper for medium+ work, project source edits and code implementation with code-writer, builds/tests with builder-tester, independent code/security/architecture review with code-reviewer when subagent delegation is authorized and available, and independent QA acceptance evaluation with qa-evaluator after build/test and code-review evidence when applicable; reviewer remains a compatibility route for existing handoffs. Assistant Framework policy requires explicit user authorization before spawning subagents for development/code-work roles unless the current user prompt already explicitly authorizes subagents. Ask once before spawning and wait before continuing phases that require subagents; after approval, use delegated mode. For Codex, spawn custom agents by explicitly asking Codex to spawn the configured agent name (for example, code-mapper, code-writer, builder-tester, code-reviewer, qa-evaluator; reviewer remains compatibility routing); a hidden/implicit subagent capability may not appear as a normal tool in the visible tool list. Use direct fallback only for explicit authorization_denied, subagents_unavailable after a real spawn failure, or policy_disallowed. The orchestrator may create and update framework-owned state artifacts such as .codex/task.md, .codex/context-map.md, .codex/session.md, and .codex/working-buffer.md; it does not edit project source files directly in delegated mode. 3. PHASE GATES: Development follows phases: TRIAGE -> DISCOVER -> DECOMPOSE when needed -> PLAN -> DESIGN when needed -> BUILD -> REVIEW -> DOCUMENT. You MUST NOT skip phases. Small tasks use lightweight phases, but NEVER skip entirely. @@ -1889,7 +1889,7 @@ These rules define the operating contract for every response. 6. TESTS WITH FEATURES: Every new component or feature MUST have tests in the SAME step. Include the test with the implementation work. -7. REVIEW IS A LOOP: After code changes, run the review cycle: review -> fix -> re-review -> fix -> re-review until clean (max 10 rounds). Continue until the review is clean or the max round is reached. +7. REVIEW IS A LOOP: After code changes, run the review cycle: review -> fix -> re-review -> fix -> re-review until clean (max 20 rounds). Continue until the review is clean or the max round is reached. 8. STATE YOUR PHASE: Before every response that involves code work, state your current workflow phase. This is mandatory — it keeps you on track. @@ -1911,7 +1911,9 @@ $AGENTS_SKILL_ROWS | architect | read-only | Design implementation blueprints | | code-writer | write | Implement code following a plan | | builder-tester | write | Build, write tests, run tests | -| reviewer | read-only | Independent code review, confidence-filtered | +| code-reviewer | read-only | Canonical code/security/architecture review | +| reviewer | read-only | Compatibility route for existing review handoffs | +| qa-evaluator | read-only | Acceptance, Done Contract, and QA evaluation | ## Memory diff --git a/plugins/assistant-dev/skills/assistant-review/SKILL.md b/plugins/assistant-dev/skills/assistant-review/SKILL.md index d8ad3c0..a773561 100644 --- a/plugins/assistant-dev/skills/assistant-review/SKILL.md +++ b/plugins/assistant-dev/skills/assistant-review/SKILL.md @@ -1,6 +1,6 @@ --- name: assistant-review -description: "This skill runs an autonomous code review loop: review, fix, re-review until clean (max 10 rounds). Use when the user says 'review', 'fresh review', 'code review', 'review this', 'check the code'. Also activates when the workflow's Review phase requires quality review dispatch." +description: "This skill runs an autonomous code review loop and optional independent QA evaluation loop: review, fix, re-review until clean (max 20 rounds), then evaluate acceptance when QA is required. Use when the user says 'review', 'fresh review', 'code review', 'review this', 'check the code'. Also activates when the workflow's Review phase requires quality review or QA evaluation dispatch." effort: high triggers: - pattern: "fix (all |the |review |reported )?issues|fix (all |the )?findings|apply (all )?fixes" @@ -11,31 +11,31 @@ triggers: reminder: "This request matches assistant-review. You MUST load and follow this SKILL.md and its contracts before doing anything else. Run the autonomous review-fix loop to its exit condition before reporting." --- -# Autonomous Review Loop +# Autonomous Review And QA Evaluation ## Contracts -This skill enforces strict contracts on inputs, outputs, loop gates, and reviewer handoffs. Read the contract files in `contracts/` before executing. +This skill enforces strict contracts on inputs, outputs, loop gates, reviewer handoffs, and QA evaluator handoffs. Read the contract files in `contracts/` before executing. | Contract | File | Purpose | |---|---|---| | **Input** | `contracts/input.yaml` | Scope, mode, and review material snapshot to resolve before entering the loop | | **Output** | `contracts/output.yaml` | Final summary and verification artifacts | | **Phase Gates** | `contracts/phase-gates.yaml` | Per-round step assertions and loop invariants | -| **Handoffs** | `contracts/handoffs.yaml` | Reviewer subagent dispatch and return schema | +| **Handoffs** | `contracts/handoffs.yaml` | Reviewer and QAEvaluator subagent dispatch and return schemas | **Rules:** - Resolve all input contract fields before entering the loop - Check phase gate assertions at every step transition within each round -- Include all required context fields when dispatching Reviewer subagents, and record direct-fallback review evidence when subagents are not authorized/available/allowed -- Validate all required return fields when Reviewer completes +- Include all required context fields when dispatching Reviewer or QAEvaluator subagents, and record direct-fallback evidence when subagents are not authorized/available/allowed +- Validate all required return fields when Reviewer or QAEvaluator completes - Verify all output contract artifacts before presenting the final summary -Run this loop autonomously from start to finish. Continue rounds until clean or max rounds reached, keep intermediate results inside the loop, and present one final result after exit. +Run the code review loop autonomously from start to finish. Continue rounds until clean or max rounds reached, keep intermediate results inside the loop, and present one final result after exit. When QA is required, run the QA evaluation loop after build/test evidence and code-review evidence are available. ## Goal -Find concrete defects, risks, regressions, and test gaps; fix them when in review-fix mode; and return one evidence-backed final review result. Reviews must be useful in company environments: local-first, policy-safe, and focused on actionable engineering risk rather than generic style preferences. +Find concrete defects, risks, regressions, and test gaps; fix them when in review-fix mode; and return one evidence-backed final review result. When QA is required, independently evaluate the Done Contract, acceptance criteria, verification evidence, scoped UI/visual/product/UX/docs/DX/domain quality, score progression, and final acceptance result. Reviews must be useful in company environments: local-first, policy-safe, and focused on actionable engineering risk or acceptance evidence rather than generic style preferences. ## Success Criteria @@ -44,6 +44,8 @@ Find concrete defects, risks, regressions, and test gaps; fix them when in revie - Every review applies the SOLID, KISS, DRY, YAGNI, and readability lens from `references/review-principles.md`. - In review-fix mode, must-fix and should-fix findings are addressed or explicitly deferred. - Validation runs after fixes, and a fresh review confirms the final state. +- QA evaluation runs after code-review/build evidence when `qa_evaluation_mode=required`, returns score progression and a final acceptance verdict, and does not replace code-reviewer. +- QA evaluation loads `references/domain-rubrics.md` only when `domain_context`, explicit `rubric_refs`, or subjective/UI/visual/product/UX/docs/DX/domain acceptance criteria require scoped domain-quality scoring. ## Constraints @@ -51,6 +53,7 @@ Find concrete defects, risks, regressions, and test gaps; fix them when in revie - Do not emit intermediate review summaries; present one final summary after loop exit. - Use concrete risk categories for refactor-related findings. - Treat clean-code principles as evidence lenses, not acronym-driven style rules. +- Keep QA evaluation separate from code review: QA focuses on acceptance criteria, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result. Code Reviewer continues to own code defects, security, architecture, and test-coverage review. ## Entry @@ -75,6 +78,8 @@ Use the smallest mode that answers the request; combine modes when reviewing imp - **Semantic contract review**: for skill/workflow/framework changes, check contract inheritance, method-template alignment, eval coverage, method-signature fidelity, and high-stakes recommendation guards before judging the change clean. - **Maintainability review**: apply SOLID/KISS/DRY/YAGNI/readability only when a concrete risk exists. - **Security handoff**: invoke `assistant-security` when the reviewed surface touches auth, permissions, secrets, input handling, persistence, shell commands, dependency/config changes, network calls, or external integrations. +- **QA evaluation**: after code-review/build evidence exists, dispatch QAEvaluator when `qa_evaluation_mode=required` for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work. Load `references/qa-evaluation-loop.md`. +- **Domain rubric QA**: within QA evaluation, load `references/domain-rubrics.md` only when scoped by acceptance criteria, Done Contract, `domain_context`, or explicit `rubric_refs`. QAEvaluator selects rubric families and returns domain-quality scores; Code Reviewer still owns code defects, security, architecture, and test coverage. Finding format: @@ -128,14 +133,14 @@ Load `references/review-principles.md` before each REVIEW step. Apply SOLID, KIS For each principle/readability finding, include the violated lens, affected surface, concrete evidence, risk category, and smallest durable fix. Do not report acronym-only findings such as "violates SOLID" without naming the observed behavior and the user-facing or maintainer risk. -## The Loop +## The Code Review Loop ``` round = 1 previously_fixed = [] score_history = [] -while round <= 10: +while round <= 20: 1. REVIEW - Dispatch a fresh Reviewer subagent when `subagent_execution_mode=delegated` @@ -160,7 +165,7 @@ while round <= 10: Finding filter policy: - Report only evidence-backed findings with file/line evidence, concrete impact, and the smallest useful fix. - Put speculative or low-evidence concerns in Observations; they do not block completion. - - In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers; round 10 is terminal and exits with remaining items instead of starting round 11. + - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers; round 20 is terminal and exits with remaining items instead of starting round 21. Apply the SOLID, KISS, DRY, YAGNI, and readability lens from references/review-principles.md. Report principle issues only when tied to concrete evidence, concrete risk, and the smallest durable fix. @@ -180,7 +185,7 @@ while round <= 10: 2. EVALUATE a. Check rubric score (medium+ scope): - PASS (weighted >= 4.0) AND no must-fix AND no should-fix -> EXIT CLEAN - - PIVOT (weighted < threshold for round) -> escalate to orchestrator + - PIVOT (weighted < threshold for round) -> escalate to orchestrator-owned pivot_restart_decision - REFINE (weighted below 4.0 but not PIVOT), including zero findings -> continue to step 3 using lowest-scoring rubric dimensions as the improvement targets - Medium+ CLEAN and ISSUES_FIXED require weighted >= 4.0 and zero @@ -188,12 +193,20 @@ while round <= 10: b. No rubric (small scope): use findings-based exit: - No must-fix AND no should-fix -> EXIT CLEAN - Only nits -> EXIT CLEAN (note nits in final report) - c. round == 10 with remaining must-fix or should-fix -> EXIT WITH REMAINING ITEMS - d. round == 10 with issues fixed and now clean -> EXIT ISSUES FIXED + c. round == 20 with remaining must-fix or should-fix -> EXIT WITH REMAINING ITEMS + d. round == 20 with issues fixed and now clean -> EXIT ISSUES FIXED e. Otherwise -> continue to step 3 Record in score_history: { round, weighted_score, finding_count, drift_status } (see references/score-tracking.md for drift detection rules) + If score tracking reports STAGNATION, repeated DRIFT, repeated REGRESSION, + or rubric action PIVOT, pause the loop and return a pivot_restart_signal to + the orchestrator. The orchestrator records pivot_restart_decision with + trigger, evidence, affected_slice_or_round, options_considered, + selected_action, reapproval_required, next_agent, recovery_pointer, and + exact_next_action before another fix/review dispatch. If the selected action + changes scope, files, behavior, risk, verification, or acceptance criteria, + reapproval is required. Round 20 remains terminal and never starts round 21. 3. FIX - Fix ALL must-fix and should-fix items (not just must-fix) @@ -208,6 +221,26 @@ while round <= 10: 5. round += 1 -> go to step 1 ``` +## The QA Evaluation Loop + +Run this loop only when `qa_evaluation_mode=required` or the workflow Review phase requests QA evidence. Load `references/qa-evaluation-loop.md` before dispatching QAEvaluator. + +QA evaluation runs after code-review/build evidence. It is not an alternate path around the Code Reviewer loop. + +QA runs after build/test evidence and the Code Reviewer loop result exist. It must receive the Done Contract, acceptance criteria, verification evidence, domain_context/rubric_refs when applicable, round number, previously_failed_acceptance_items, and qa_filter_policy. It conditionally loads `references/domain-rubrics.md` only for scoped subjective/UI/visual/product/UX/docs/DX/domain acceptance. It returns status, round, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when used, final_verdict/result, evidence, score_progression or score_entry, and open_questions when blocked. + +The QA loop supports rounds 1-20. In rounds 16-20, only unresolved acceptance blockers or high-confidence acceptance risks keep the loop open. Round 20 is terminal: return the final verdict and remaining failed acceptance items instead of starting round 21. + +QA findings do not replace code-review findings. Do not use QAEvaluator to review general bugs, security, architecture, or test coverage unless that issue directly blocks an acceptance criterion or Done Contract item. + +If QA score progression reports STAGNATION, repeated DRIFT, repeated REGRESSION, +or a scoped domain rubric returns action `pivot`, pause QA and return a +pivot_restart_signal to the orchestrator. The orchestrator records +pivot_restart_decision before another QA/build dispatch, updates trace/replay/run +state when the workflow is harness-capable, and obtains reapproval when the +selected restart or pivot changes scope, files, behavior, risk, verification, or +acceptance criteria. + ## Exit: Present Final Result After the loop completes, present ONE summary to the user: @@ -250,7 +283,7 @@ After the loop completes, present ONE summary to the user: - **Autonomous continuation**: advance to the next round while exit criteria are unmet. - **Fresh Reviewer each round** on medium+ scope: stale context weakens reviews. - **Previously-fixed list prevents re-reporting**: each round should find fewer issues. -- **Evidence-backed filtering**: findings need file/line evidence, concrete impact, and the smallest useful fix; late rounds only stay open before the hard max for must-fix or high-confidence should-fix findings, and round 10 is terminal. +- **Evidence-backed filtering**: findings need file/line evidence, concrete impact, and the smallest useful fix; late rounds only stay open before the hard max for must-fix or high-confidence should-fix findings, and round 20 is terminal. - If scope is trivial (single small file, obvious change) -> one clean round can exit. If findings exist, continue looping. ## Output @@ -264,6 +297,7 @@ Return: - **Agentic loop safety review** - when applicable, whether bounds, stop conditions, retry/empty handling, tool-error routing, progress checks, and cost/token guards were checked. - **Behavioral contract review** - when applicable, whether existing invariants, interface alignment, inherited tests, protocol fidelity, high-impact guards, and runtime surfaces were checked. - **Semantic contract review** - when applicable, whether inherited contracts, method signatures, eval coverage, high-stakes guards, and mirror surfaces were checked. +- **QA evaluation result** - when applicable, final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression, and blocked open questions. - **Residual risk** - remaining items, nits, or scope gaps. ## Stop Rules @@ -279,15 +313,18 @@ After each round, compare rubric scores to the previous round per `references/sc - **GENUINE**: Score up, findings down -> continue normally - **SUSPICIOUS**: Score jumped > 1.0 in one round -> log warning, continue - **DRIFT**: Score up but findings didn't decrease -> **reset evaluator** (fresh agent, stricter prompt) -- **REGRESSION**: Score down -> investigate, escalate if 2+ consecutive rounds +- **REGRESSION**: Score down -> investigate; 2+ consecutive regressions trigger pivot_restart_signal - **NEUTRAL**: Score unchanged for 1 round with findings present -> log, no action yet -- **STAGNATION**: Score unchanged for 2+ rounds with findings present -> escalate to orchestrator +- **STAGNATION**: Score unchanged for 2+ rounds with findings present -> return pivot_restart_signal to orchestrator On DRIFT, the next Reviewer dispatch MUST include this addition to its prompt: > "Previous rounds showed score inflation without corresponding quality improvement. > Apply maximum skepticism. Score conservatively; when uncertain, round DOWN." -On 3+ DRIFT occurrences: stop the loop and present findings for manual review. +On repeated DRIFT after the reset, stop the current review dispatch path and +return pivot_restart_signal. The orchestrator owns the pivot_restart_decision and +selects reset, candidate search, replan, restart, or a blocked/user path without +creating round 21 behavior. ## Review Finding Rule Distillation diff --git a/plugins/assistant-dev/skills/assistant-review/contracts/handoffs.yaml b/plugins/assistant-dev/skills/assistant-review/contracts/handoffs.yaml index 90485c5..a4b41d5 100644 --- a/plugins/assistant-dev/skills/assistant-review/contracts/handoffs.yaml +++ b/plugins/assistant-dev/skills/assistant-review/contracts/handoffs.yaml @@ -24,6 +24,9 @@ worker_status_protocol: - "findings, summary, and verdict remain required and are not replaced by status." - "evidence is required to support the verdict and any findings." - "open_questions is required for NEEDS_CONTEXT and BLOCKED." + - "status is required on every QAEvaluator return." + - "QAEvaluator returns acceptance_findings, qa_scorecard, final_verdict, result, evidence, and score_progression or score_entry; selected_domain_rubrics and domain_quality_scores are required only when scoped domain rubrics are selected. QA evaluation does not replace Reviewer/code-reviewer findings." + - "Reviewer and QAEvaluator may return pivot_restart_signal for PIVOT, STAGNATION, repeated DRIFT, or repeated REGRESSION; the orchestrator owns the final pivot_restart_decision." handoffs: @@ -70,8 +73,8 @@ handoffs: - name: round type: int required: true - description: "Current review round number (1-10)" - validation: ">= 1 and <= 10" + description: "Current review round number (1-20)" + validation: ">= 1 and <= 20" - name: previously_fixed type: object[] @@ -84,14 +87,14 @@ handoffs: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: finding_filter_policy type: string required: true description: "Evidence-backed finding filter for this round" - validation: "Must state that findings require file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are Observations and do not block; round 10 remains terminal." - notes: "In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers. The loop never starts round 11." + validation: "Must state that findings require file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are Observations and do not block; round 20 remains terminal." + notes: "In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers. The loop never starts round 21." - name: project_type type: string @@ -154,7 +157,7 @@ handoffs: type: int required: true description: "Echo back the round number for tracking" - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: findings type: object[] @@ -421,8 +424,402 @@ handoffs: required: false description: "If a critical finding caps the score, describe it here" + - name: pivot_restart_signal + type: object + required: false + condition: "rubric_scores.action == PIVOT or score tracking detects STAGNATION, repeated DRIFT, or repeated REGRESSION" + description: "Signal that the review loop needs orchestrator-owned pivot/restart handling before another fix/review dispatch." + object_fields: + - name: trigger + type: enum + required: true + enum_values: [STAGNATION, repeated_DRIFT, repeated_REGRESSION, pivot] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: affected_round + type: int + required: true + validation: ">= 1 and <= 20" + - name: recommended_recovery_focus + type: string + required: true + description: "Lowest-scoring dimensions, stale-context concern, or blocker evidence the orchestrator should consider" + on_missing_field: "In delegated mode, re-dispatch Reviewer. In direct fallback mode, complete the local review evidence before continuing. status, findings, evidence, and verdict are never optional — a review without them is not a review." + # ───────────────────────────────────────────── + # Orchestrator → QAEvaluator (per QA round) + # ───────────────────────────────────────────── + - name: orchestrator_to_qa_evaluator + from: Orchestrator + to: QAEvaluator + phase: QA_EVALUATION_STEP + notes: "Runs after build/test evidence and Code Reviewer or Reviewer compatibility evidence. QA evaluation is an acceptance lane and does not replace code-reviewer." + + context_fields: + - name: done_contract + type: object + required: conditional + condition: "task has an accepted Done Contract" + description: "Accepted Done Contract for the work under QA evaluation" + validation: "Includes done_when, not_done_when, verification, owner_consumer, acceptance_criteria, debate_record with at least two perspectives, and accepted_by when present" + object_fields: + - name: done_when + type: string[] + required: true + - name: not_done_when + type: string[] + required: true + - name: verification + type: string[] + required: true + - name: owner_consumer + type: string + required: true + - name: acceptance_criteria + type: string[] + required: true + - name: debate_record + type: object[] + required: true + min_items: 2 + description: "pre-build debate/subagent-perspective evidence considered before accepting the Done Contract" + validation: "At least two perspectives; delegated mode uses subagent perspectives when available" + object_fields: + - name: perspective + type: string + required: true + - name: concern_or_support + type: string + required: true + - name: resolution + type: string + required: true + - name: accepted_by + type: string + required: true + + - name: acceptance_criteria + type: string[] + required: true + description: "Binary acceptance criteria from the user request, approved plan, slice manifest, or Done Contract" + validation: "At least one independently evaluable criterion is present" + min_items: 1 + + - name: verification_evidence + type: object[] + required: true + description: "Build, test, manual, review, or inspection evidence used to prove acceptance" + validation: "At least one evidence entry exists; each entry states source, result, and what it proves" + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: result + type: string + required: true + - name: proves + type: string + required: true + + - name: code_review_result + type: object + required: true + description: "Final Code Reviewer or Reviewer compatibility result that QA must not replace" + validation: "Names the review role used, final result, review rounds, and remaining code-review risks if any" + object_fields: + - name: role + type: enum + required: true + enum_values: [CodeReviewer, Reviewer] + - name: result + type: string + required: true + - name: rounds + type: int + required: true + validation: ">= 1 and <= 20" + - name: remaining_risks + type: string[] + required: true + + - name: domain_context + type: object + required: false + default: null + description: "Scoped product, UX, docs, DX, or domain notes for QA evaluation" + notes: "When present, QAEvaluator may load references/domain-rubrics.md and select only matching rubric families." + object_fields: + - name: summary + type: string + required: true + - name: in_scope_surfaces + type: string[] + required: true + + - name: rubric_refs + type: string[] + required: false + default: [] + description: "Explicit rubric family references for product, UX, UI/visual, docs, DX, or domain quality" + notes: "Valid families are defined in references/domain-rubrics.md. QAEvaluator must not invent rubrics when rubric_refs are empty and no acceptance/Done Contract/domain_context scope requires them." + + - name: round + type: int + required: true + description: "Current QA evaluation round number (1-20)" + validation: ">= 1 and <= 20" + + - name: previously_failed_acceptance_items + type: object[] + required: true + description: "Acceptance items failed in previous QA rounds; QAEvaluator should not re-report resolved items" + object_fields: + - name: criterion + type: string + required: true + - name: failed_in_round + type: int + required: true + validation: ">= 1 and <= 20" + - name: evidence_gap + type: string + required: true + + - name: qa_filter_policy + type: string + required: true + description: "Evidence-backed QA finding filter for this round" + validation: "Must state that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 20 is terminal." + + return_fields: + - name: status + type: enum + required: true + enum_values: [DONE, DONE_WITH_CONCERNS, NEEDS_CONTEXT, BLOCKED] + description: "Worker status packet result; DONE requires final_verdict=accepted" + + - name: round + type: int + required: true + description: "Echo back the QA round number" + validation: ">= 1 and <= 20" + + - name: acceptance_findings + type: object[] + required: true + description: "Acceptance, Done Contract, verification, or scoped domain findings for this QA round" + object_fields: + - name: severity + type: enum + required: true + enum_values: [blocker, concern, observation] + - name: criterion + type: string + required: true + - name: evidence + type: string + required: true + - name: impact + type: string + required: true + - name: recommendation + type: string + required: false + + - name: qa_scorecard + type: object + required: true + description: "Compact QA scores; domain_quality is backed by selected_domain_rubrics/domain_quality_scores only when domain rubrics are scoped" + object_fields: + - name: acceptance_coverage + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments" + - name: evidence_strength + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments" + - name: domain_quality + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments; use 5.0 when no scoped domain quality surface applies and state not_applicable in rationale" + - name: final_readiness + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments" + - name: weighted_score + type: float + required: true + validation: "Calculated from the compact QA scorecard dimensions" + - name: rationale + type: object + required: true + object_fields: + - name: acceptance_coverage + type: string + required: true + - name: evidence_strength + type: string + required: true + - name: domain_quality + type: string + required: true + - name: final_readiness + type: string + required: true + + - name: selected_domain_rubrics + type: string[] + required: false + condition: "domain_context or rubric_refs is present, or acceptance criteria / Done Contract require subjective/product/UX/docs/DX/UI/domain craft evaluation" + description: "Rubric families selected from references/domain-rubrics.md for this QA round" + validation: "Every selected family is tied to acceptance criteria, Done Contract, domain_context, or explicit rubric_refs. Empty or omitted when no scoped domain rubric applies." + + - name: domain_quality_scores + type: object[] + required: false + condition: "selected_domain_rubrics is non-empty" + description: "Per-family domain rubric scores from references/domain-rubrics.md" + validation: "Each score cites scoped evidence and uses pass/refine/pivot or accepted/rejected guidance; unscoped dimensions are not_applicable, not invented." + object_fields: + - name: rubric_ref + type: string + required: true + description: "Selected rubric family, such as ui_visual_design, ux_product_acceptance, documentation_quality, developer_experience, or domain_specific_craft" + - name: dimension + type: string + required: true + description: "Selected dimension within the rubric family" + - name: score + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments. For action=not_applicable, use 5.0 and state the unscoped rationale in evidence." + - name: action + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, refine, rejected, pivot, not_applicable] + - name: evidence + type: string + required: true + validation: "Cites acceptance criteria, Done Contract, domain_context, rubric_refs, verification artifacts, screenshots, docs, or concrete files" + + - name: final_verdict + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, rejected, blocked] + description: "Final QA acceptance verdict for this round" + + - name: result + type: enum + required: true + enum_values: [CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, BLOCKED] + description: "Loop-compatible result summary" + + - name: evidence + type: object[] + required: true + description: "Acceptance material, files, review results, verification evidence, or checks supporting the QA verdict" + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + + - name: score_entry + type: object + required: true + description: "Score progression entry for this QA round" + object_fields: + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: weighted_score + type: float + required: true + - name: failed_acceptance_count + type: int + required: true + - name: delta + type: string + required: true + - name: drift_status + type: enum + required: true + enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL, NOT_APPLICABLE] + + - name: score_progression + type: object[] + required: false + description: "Full QA score progression when the evaluator has prior round history" + object_fields: + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: weighted_score + type: float + required: true + - name: failed_acceptance_count + type: int + required: true + - name: drift_status + type: enum + required: true + enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL, NOT_APPLICABLE] + + - name: open_questions + type: string[] + required: false + condition: "status in [NEEDS_CONTEXT, BLOCKED] or final_verdict == blocked" + description: "Questions or blocker details the orchestrator must resolve" + + - name: pivot_restart_signal + type: object + required: false + condition: "score_entry.drift_status == STAGNATION, repeated QA DRIFT/REGRESSION is detected, or any scoped domain_quality_scores action == pivot" + description: "Signal that the QA loop needs orchestrator-owned pivot/restart handling before another QA/build dispatch." + object_fields: + - name: trigger + type: enum + required: true + enum_values: [STAGNATION, repeated_DRIFT, repeated_REGRESSION, pivot] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: affected_round + type: int + required: true + validation: ">= 1 and <= 20" + - name: recommended_recovery_focus + type: string + required: true + description: "Acceptance gap, evidence gap, domain pivot evidence, or stale QA context the orchestrator should consider" + + on_missing_field: "In delegated mode, re-dispatch QAEvaluator. In direct fallback mode, complete the local QA evidence before continuing. status, acceptance_findings, qa_scorecard, final_verdict, result, evidence, and score_entry are never optional. When scoped domain rubrics are selected, selected_domain_rubrics and domain_quality_scores are also required; when not scoped, do not fabricate them." + # ───────────────────────────────────────────── # Validation behavior: # 1. Before delegated dispatch or direct-fallback review: verify all required context_fields are populated @@ -433,6 +830,11 @@ handoffs: # 6. If rubric_required: verify rubric_scores is present with all dimensions # 7. If rubric_required: verify weighted_score matches the dimension * weight calculation # 8. If quality_principles_required: verify the Reviewer applied references/review-principles.md -# 9. If round == 10, exit with CLEAN, ISSUES_FIXED, or HAS_REMAINING_ITEMS; never start round 11 -# 10. If delegated re-dispatch or direct-fallback completion is needed: include explicit note about what was missing +# 9. If pivot_restart_signal is present, stop the current loop path until the orchestrator records pivot_restart_decision; do not silently continue. +# 10. If round == 20, exit with CLEAN, ISSUES_FIXED, or HAS_REMAINING_ITEMS; never start round 21 +# 11. If delegated re-dispatch or direct-fallback completion is needed: include explicit note about what was missing +# 12. If qa_evaluation_mode=required, run orchestrator_to_qa_evaluator after build/test and code-review evidence; include debate_record when Done Contract exists; never treat QA evaluation as a replacement for Reviewer/code-reviewer. +# 13. If QA round == 20, return accepted, accepted_with_concerns, rejected, or blocked with remaining failed acceptance items; never start QA round 21. +# 14. If domain_context/rubric_refs or subjective/product/UX/docs/DX/UI/domain acceptance scope exists: verify selected_domain_rubrics and domain_quality_scores are present and evidence-backed. +# 15. If no scoped domain rubric applies: verify QAEvaluator did not invent rubrics or penalize domain_quality for missing unscoped evidence. # ───────────────────────────────────────────── diff --git a/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml b/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml index 16b654e..455f98a 100644 --- a/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml +++ b/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml @@ -93,6 +93,123 @@ fields: default: [] on_missing: skip + - name: qa_evaluation_mode + type: enum + required: false + enum_values: [not_required, optional, required] + description: "Whether the independent QA evaluation loop should run after code-review/build evidence" + validation: "required for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work; not_required must have a reason in final output when skipped for medium+ harness work" + default: not_required + on_missing: infer + infer_from: > + Use required when the task is medium+ harness-capable, has a Done Contract, is domain-scored, + affects UI/visual/product/UX/docs/DX quality, or the user explicitly requests QA/acceptance evaluation. + Use optional when acceptance material exists but the orchestrator has not marked QA as required. + Use not_required for trivial/small code-review-only work with no separate acceptance evidence need. + + - name: done_contract + type: object + required: conditional + condition: "qa_evaluation_mode == required and Done Contract exists for the task" + description: "Accepted Done Contract used by QAEvaluator" + validation: "When present, includes done_when, not_done_when, verification, owner_consumer, acceptance_criteria, debate_record with at least two perspectives, and accepted_by" + on_missing: fail + object_fields: + - name: done_when + type: string[] + required: true + - name: not_done_when + type: string[] + required: true + - name: verification + type: string[] + required: true + - name: owner_consumer + type: string + required: true + - name: acceptance_criteria + type: string[] + required: true + - name: debate_record + type: object[] + required: true + min_items: 2 + description: "pre-build debate/subagent-perspective evidence considered before accepting the Done Contract" + validation: "At least two perspectives; delegated mode uses subagent perspectives when available" + object_fields: + - name: perspective + type: string + required: true + - name: concern_or_support + type: string + required: true + - name: resolution + type: string + required: true + - name: accepted_by + type: string + required: true + + - name: acceptance_criteria + type: string[] + required: conditional + condition: "qa_evaluation_mode == required" + description: "Binary acceptance criteria from the user request, approved plan, slice manifest, or Done Contract" + validation: "At least one criterion is present and each criterion is independently evaluable" + min_items: 1 + on_missing: fail + + - name: verification_evidence + type: object[] + required: conditional + condition: "qa_evaluation_mode == required" + description: "Build, test, manual, review, or inspection evidence QAEvaluator uses to verify acceptance" + validation: "At least one evidence entry exists and each entry names the check/result and what it proves" + min_items: 1 + on_missing: fail + object_fields: + - name: source + type: string + required: true + - name: result + type: string + required: true + - name: proves + type: string + required: true + + - name: domain_context + type: object + required: false + description: "Scoped product, UX, docs, DX, or domain notes for QA evaluation" + default: null + on_missing: skip + notes: "When populated, QAEvaluator may load references/domain-rubrics.md and select only matching rubric families." + object_fields: + - name: summary + type: string + required: true + - name: in_scope_surfaces + type: string[] + required: true + + - name: rubric_refs + type: string[] + required: false + description: "Explicit product/UX/docs/DX/UI/domain rubric family references used by QAEvaluator" + default: [] + on_missing: skip + notes: "Valid families are defined in references/domain-rubrics.md. Empty means no domain rubric is selected unless acceptance criteria or Done Contract scope requires one." + + - name: qa_filter_policy + type: string + required: conditional + condition: "qa_evaluation_mode == required" + description: "Evidence filter QAEvaluator must apply to acceptance findings" + validation: "States that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 20 is terminal" + on_missing: infer + infer_from: "Use the standard QA filter policy from references/qa-evaluation-loop.md." + - name: mode type: enum required: true @@ -162,9 +279,10 @@ fields: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" # Validation behavior: # 1. Resolve review_scope, review_material_snapshot, change_kind, semantic_contract_review_required, behavioral_contract_review_required, and agentic_loop_safety_review_required before entering the loop # 2. If review_material_snapshot.material is empty after resolution, inform user "nothing to review" and exit -# 3. Log resolved input as: >> Review scope: {scope_size} | mode: {mode} | kind: {change_kind} | subagent_mode: {subagent_execution_mode} | loop_safety_review: {agentic_loop_safety_review_required} | behavioral_contract_review: {behavioral_contract_review_required} | semantic_contract_review: {semantic_contract_review_required} | files: {count} +# 3. If qa_evaluation_mode=required: require acceptance_criteria, verification_evidence, qa_filter_policy, and Done Contract with debate_record when the task has one before QAEvaluator dispatch +# 4. Log resolved input as: >> Review scope: {scope_size} | mode: {mode} | kind: {change_kind} | subagent_mode: {subagent_execution_mode} | qa_evaluation_mode: {qa_evaluation_mode} | loop_safety_review: {agentic_loop_safety_review_required} | behavioral_contract_review: {behavioral_contract_review_required} | semantic_contract_review: {semantic_contract_review_required} | files: {count} diff --git a/plugins/assistant-dev/skills/assistant-review/contracts/output.yaml b/plugins/assistant-dev/skills/assistant-review/contracts/output.yaml index 69cbe49..2b8ca43 100644 --- a/plugins/assistant-dev/skills/assistant-review/contracts/output.yaml +++ b/plugins/assistant-dev/skills/assistant-review/contracts/output.yaml @@ -30,6 +30,88 @@ artifacts: required: true validation: "Names the fresh Reviewer dispatch, or the direct-fallback context reset used to reduce stale review risk" + - name: qa_evaluation_delegation_path + type: object + required: conditional + condition: "qa_evaluation_mode == required" + description: "How QAEvaluator responsibility was executed" + validation: "States subagent policy state, execution mode, authorization scope, and whether QA used a fresh QAEvaluator subagent or fresh direct-fallback context" + on_fail: "Record delegated QA evaluation or direct fallback before presenting results; do not imply QAEvaluator subagents were used when they were not authorized or available" + object_fields: + - name: subagent_policy_state + type: enum + required: true + enum_values: [not_required, authorization_required, delegation_authorized, authorization_denied, subagents_unavailable, policy_disallowed] + - name: subagent_execution_mode + type: enum + required: true + enum_values: [delegated, direct_fallback, not_applicable] + - name: subagent_authorization_scope + type: string[] + required: true + - name: fresh_context_evidence + type: string + required: true + validation: "Names the fresh QAEvaluator dispatch, or the direct-fallback context reset used to reduce stale QA risk" + + - name: pivot_restart_decision + type: object + required: conditional + condition: "review or QA loop observes STAGNATION, repeated DRIFT, repeated REGRESSION, rubric action PIVOT, or scoped domain action pivot" + description: "Orchestrator-owned decision packet created before another review/fix/QA/build dispatch after stagnant or pivot-triggered quality loops." + validation: "Must include trigger, evidence, affected_slice_or_round, options_considered, selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action. Reapproval is required when scope, files, behavior, risk, verification, or acceptance criteria change." + on_fail: "Pause the loop and create the pivot_restart_decision before continuing; do not silently continue after stagnation or pivot." + object_fields: + - name: trigger + type: enum + required: true + enum_values: [STAGNATION, repeated_DRIFT, repeated_REGRESSION, pivot] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: affected_slice_or_round + type: string + required: true + - name: options_considered + type: object[] + required: true + min_items: 2 + object_fields: + - name: option + type: string + required: true + - name: tradeoff + type: string + required: true + - name: disposition + type: enum + required: true + enum_values: [selected, rejected, deferred] + - name: selected_action + type: enum + required: true + enum_values: [reset_context, return_to_build, dispatch_debugging, dispatch_explorer, dispatch_architect, run_candidate_search, replan, restart_slice, restart_phase, block_for_user, accept_with_limitations] + - name: reapproval_required + type: boolean + required: true + - name: next_agent + type: string + required: true + - name: recovery_pointer + type: string + required: true + - name: exact_next_action + type: string + required: true + - name: review_finding_rule_distillation type: object[] required: conditional @@ -74,12 +156,12 @@ artifacts: required: true condition: always description: "The single summary presented to the user after the loop completes" - validation: "Must include rounds (1-10), result, fixed_items, remaining_items when applicable, and nits before presenting review results" + validation: "Must include rounds (1-20), result, fixed_items, remaining_items when applicable, and nits before presenting review results" object_fields: - name: rounds type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: result type: enum required: true @@ -102,7 +184,7 @@ artifacts: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: remaining_items type: object[] required: false @@ -134,6 +216,128 @@ artifacts: type: string required: true + - name: qa_evaluation_result + type: object + required: conditional + condition: "qa_evaluation_mode == required" + description: "Independent QA evaluation final result after build/test and code-review evidence" + validation: "Must include rounds (1-20), final_verdict, result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, evidence, and open_questions when blocked" + on_fail: "Run the QA evaluation loop from references/qa-evaluation-loop.md or record why QA is blocked before presenting final review results" + object_fields: + - name: rounds + type: int + required: true + validation: ">= 1 and <= 20" + - name: final_verdict + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, rejected, blocked] + - name: result + type: enum + required: true + enum_values: [CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, BLOCKED] + - name: acceptance_findings + type: object[] + required: true + description: "Remaining or resolved acceptance findings from the final QA round" + object_fields: + - name: severity + type: enum + required: true + enum_values: [blocker, concern, observation] + - name: criterion + type: string + required: true + - name: evidence + type: string + required: true + - name: disposition + type: enum + required: true + enum_values: [resolved, remaining, not_applicable] + - name: qa_scorecard + type: object + required: true + description: "Compact QA scorecard; domain_quality is backed by selected_domain_rubrics/domain_quality_scores only when scoped domain rubrics were used" + object_fields: + - name: acceptance_coverage + type: float + required: true + - name: evidence_strength + type: float + required: true + - name: domain_quality + type: float + required: true + - name: final_readiness + type: float + required: true + - name: weighted_score + type: float + required: true + - name: selected_domain_rubrics + type: string[] + required: false + condition: "domain_context or rubric_refs was provided, or acceptance criteria / Done Contract required subjective/product/UX/docs/DX/UI/domain craft evaluation" + description: "Rubric families selected from references/domain-rubrics.md; empty or omitted when domain rubrics were not scoped" + - name: domain_quality_scores + type: object[] + required: false + condition: "selected_domain_rubrics is non-empty" + description: "Evidence-backed domain rubric scores from the final QA round" + object_fields: + - name: rubric_ref + type: string + required: true + - name: dimension + type: string + required: true + - name: score + type: float + required: true + - name: action + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, refine, rejected, pivot, not_applicable] + - name: evidence + type: string + required: true + - name: score_progression + type: object[] + required: true + description: "QA score progression across rounds" + object_fields: + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: weighted_score + type: float + required: true + - name: failed_acceptance_count + type: int + required: true + - name: drift_status + type: enum + required: true + enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL, NOT_APPLICABLE] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: open_questions + type: string[] + required: false + condition: "final_verdict == blocked or result == BLOCKED" + description: "Questions or blocker details the orchestrator must resolve" + - name: rubric_summary type: object required: false @@ -170,12 +374,12 @@ artifacts: - name: score_progression type: object[] required: true - description: "Weighted score per round — used for drift detection and capped at the 10-round review limit" + description: "Weighted score per round — used for drift detection and capped at the 20-round review limit" object_fields: - name: round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: weighted_score type: float required: true @@ -379,3 +583,6 @@ artifacts: # 5. If semantic_contract_review_required=true: semantic_contract_review is populated and semantic findings are included in final_summary/audit_report # 6. If behavioral_contract_review_required=true: behavioral_contract_review is populated and behavioral findings are included in final_summary/audit_report # 7. If agentic_loop_safety_review_required=true: agentic_loop_safety_review is populated and loop-safety findings are included in final_summary/audit_report +# 8. If qa_evaluation_mode=required: qa_evaluation_delegation_path and qa_evaluation_result are populated after build/test and code-review evidence; QA result does not replace final_summary or code-review artifacts +# 9. If scoped domain rubrics were used: qa_evaluation_result includes selected_domain_rubrics and domain_quality_scores; if not scoped, no invented domain rubric output is required or accepted +# 10. If review or QA triggered STAGNATION, repeated DRIFT, repeated REGRESSION, or pivot action: pivot_restart_decision is populated before any additional review/fix/QA/build dispatch diff --git a/plugins/assistant-dev/skills/assistant-review/contracts/phase-gates.yaml b/plugins/assistant-dev/skills/assistant-review/contracts/phase-gates.yaml index 79046cd..c1c6083 100644 --- a/plugins/assistant-dev/skills/assistant-review/contracts/phase-gates.yaml +++ b/plugins/assistant-dev/skills/assistant-review/contracts/phase-gates.yaml @@ -42,6 +42,11 @@ gates: check: "subagent_policy_state, subagent_execution_mode, and subagent_authorization_scope are resolved before any Reviewer subagent dispatch" on_fail: "Resolve the active adapter/tool policy. Ask once when explicit authorization is required, or set direct_fallback/not_applicable and record why no Reviewer subagent will be spawned." + - id: E7 + check: "If qa_evaluation_mode is required: Done Contract with debate_record when available, acceptance_criteria, verification_evidence, domain_context/rubric_refs when applicable, and qa_filter_policy are resolved before QAEvaluator dispatch" + condition: "qa_evaluation_mode == required" + on_fail: "Load references/qa-evaluation-loop.md, gather acceptance material and verification evidence, select domain rubric scope only when acceptance criteria, Done Contract, domain_context, or rubric_refs require it, or mark QA blocked with open_questions before dispatching QAEvaluator." + # ───────────────────────────────────────────── # REVIEW step — per round # ───────────────────────────────────────────── @@ -109,15 +114,15 @@ gates: exit_assertions: - id: EV1 - check: "Exit decision is one of: EXIT_CLEAN, EXIT_ISSUES_FIXED, EXIT_MAX_ROUNDS, CONTINUE" - on_fail: "Re-evaluate findings against exit criteria. EXIT_CLEAN = no must-fix/should-fix (nits OK, note in report) and, for medium+ scope, weighted_score >= 4.0. EXIT_ISSUES_FIXED = round 10, all issues fixed this session, no must-fix/should-fix findings, and, for medium+ scope, weighted_score >= 4.0. EXIT_MAX_ROUNDS = round 10 with remaining must-fix or should-fix findings." + check: "Exit decision is one of: EXIT_CLEAN, EXIT_ISSUES_FIXED, EXIT_MAX_ROUNDS, ESCALATE_PIVOT_RESTART, CONTINUE" + on_fail: "Re-evaluate findings against exit criteria. EXIT_CLEAN = no must-fix/should-fix (nits OK, note in report) and, for medium+ scope, weighted_score >= 4.0. EXIT_ISSUES_FIXED = round 20, all issues fixed this session, no must-fix/should-fix findings, and, for medium+ scope, weighted_score >= 4.0. EXIT_MAX_ROUNDS = round 20 with remaining must-fix or should-fix findings. ESCALATE_PIVOT_RESTART = STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT." - id: EV2 check: "EXIT_CLEAN or EXIT_ISSUES_FIXED only if: zero must-fix AND zero should-fix findings, and for medium+ scope weighted_score >= 4.0" - on_fail: "Findings exist or medium+ weighted_score is below 4.0 — must CONTINUE to FIX step" + on_fail: "Findings exist or medium+ weighted_score is below 4.0 — apply rubric action first. REFINE continues to FIX; PIVOT escalates through pivot_restart_decision." - id: EV3 - check: "EXIT_MAX_ROUNDS only if: round == 10" + check: "EXIT_MAX_ROUNDS only if: round == 20" on_fail: "Rounds remain — must CONTINUE to FIX step" - id: EV4 @@ -126,14 +131,19 @@ gates: on_fail: "Audit mode does not fix or loop — present findings and exit" - id: EV5 - check: "For medium+ scope: rubric threshold action is applied — PASS exits, REFINE continues, PIVOT escalates" + check: "For medium+ scope: rubric threshold action is applied — PASS exits, REFINE continues, PIVOT escalates through pivot_restart_decision" condition: "scope_size in [medium, large] and rubric_scores is present" on_fail: "Check rubric weighted_score against threshold table in references/review-rubric.md and apply the correct action" - id: EV6 - check: "PIVOT action triggers escalation to orchestrator — do not silently continue after PIVOT" + check: "PIVOT action triggers orchestrator-owned pivot_restart_decision — do not silently continue after PIVOT" condition: "rubric_scores.action == PIVOT" - on_fail: "Flag to orchestrator: weighted score below pivot threshold. Include lowest-scoring dimensions and recommendations." + on_fail: "Create pivot_restart_signal and require the orchestrator to record pivot_restart_decision with trigger, evidence, affected_slice_or_round, options_considered, selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action before another fix/review dispatch." + + - id: EV7 + check: "STAGNATION, repeated DRIFT, or repeated REGRESSION triggers ESCALATE_PIVOT_RESTART and records pivot_restart_signal before another fix/review dispatch" + condition: "scope_size in [medium, large] and round >= 2" + on_fail: "Pause the loop, build pivot_restart_signal from score_history, and require orchestrator-owned pivot_restart_decision. Do not advance to another review round until the decision names the exact next action." # ───────────────────────────────────────────── # FIX step — per round (review-fix mode only) @@ -178,6 +188,45 @@ gates: condition: "project is C# and complexity tool is available" on_fail: "Run complexity analysis and flag methods exceeding threshold" + # ───────────────────────────────────────────── + # QA EVALUATION step — after code review/build evidence + # ───────────────────────────────────────────── + - phase: QA_EVALUATION_STEP + condition: "qa_evaluation_mode == required" + checkpoint_start: null + checkpoint_end: null + exit_assertions: + + - id: QA1 + check: "QA evaluation starts only after build/test verification evidence and Code Reviewer or Reviewer compatibility result are available" + on_fail: "Return to validation or code review. Do not dispatch QAEvaluator until code-review/build evidence exists." + + - id: QA2 + check: "QAEvaluator received done_contract with debate_record when present, acceptance_criteria, verification_evidence, domain_context/rubric_refs when applicable, round, previously_failed_acceptance_items, and qa_filter_policy" + on_fail: "Re-dispatch QAEvaluator with the full QA handoff context, or complete direct-fallback QA context before continuing." + + - id: QA3 + check: "QAEvaluator returned status, round, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were selected, final_verdict, result, evidence, and score_entry" + on_fail: "In delegated mode, re-dispatch QAEvaluator requesting the missing fields. In direct fallback mode, complete the missing QA evidence before deciding acceptance. Require selected_domain_rubrics/domain_quality_scores only when domain rubrics are scoped." + + - id: QA4 + check: "QAEvaluator findings stay in the QA lane and do not replace code-reviewer findings" + on_fail: "Move generic code/security/architecture/test-coverage findings back to Code Reviewer unless they directly block an acceptance criterion or Done Contract item." + + - id: QA5 + check: "QA loop exit decision is accepted, accepted_with_concerns, rejected, or blocked; round 20 is terminal and never starts round 21" + on_fail: "Apply references/qa-evaluation-loop.md terminal behavior. If round < 20 and rejected, return failed acceptance items to Build; if round == 20, record remaining failed acceptance items in qa_evaluation_result." + + - id: QA6 + check: "Domain rubrics from references/domain-rubrics.md are loaded and applied only when scoped by acceptance criteria, Done Contract, domain_context, or explicit rubric_refs" + condition: "qa_evaluation_mode == required" + on_fail: "Remove unscoped rubric scoring, or re-dispatch QAEvaluator with the explicit domain_context/rubric_refs and require selected_domain_rubrics plus domain_quality_scores." + + - id: QA7 + check: "QA STAGNATION, repeated DRIFT, repeated REGRESSION, or scoped domain action pivot triggers pivot_restart_signal and orchestrator-owned pivot_restart_decision before another QA/build dispatch" + condition: "qa_evaluation_mode == required" + on_fail: "Pause QA, create pivot_restart_signal from score_entry, score_progression, or domain_quality_scores action=pivot, and require pivot_restart_decision with exact_next_action. Round 20 remains terminal and never starts round 21." + # ───────────────────────────────────────────── # EXIT — after loop completes # ───────────────────────────────────────────── @@ -200,6 +249,11 @@ gates: condition: "scope_size in [medium, large]" on_fail: "Populate rubric_summary with final_scores from last round and score_progression from all rounds" + - id: X4 + check: "If qa_evaluation_mode is required: qa_evaluation_result is populated with final_verdict, result, acceptance_findings, qa_scorecard, score_progression, and evidence" + condition: "qa_evaluation_mode == required" + on_fail: "Run or complete the QA evaluation loop before presenting final review results." + # ───────────────────────────────────────────── # Loop invariants # ───────────────────────────────────────────── @@ -215,9 +269,9 @@ invariants: on_fail: "Restore any removed items to previously_fixed. This list is append-only." - id: INV2 - check: "Finding filter policy requires file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are non-blocking Observations; round 10 is terminal" + check: "Finding filter policy requires file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are non-blocking Observations; round 20 is terminal" scope: all_rounds - on_fail: "Rebuild finding_filter_policy from the evidence-backed rule and hard max-10 cap before dispatching review." + on_fail: "Rebuild finding_filter_policy from the evidence-backed rule and hard max-20 cap before dispatching review." - id: INV3 check: "The agent never pauses between rounds to ask the user — the loop is autonomous" @@ -235,8 +289,9 @@ invariants: condition: "scope_size in [medium, large] and round >= 2" on_fail: > Drift detected: score increased without corresponding quality improvement. - Per references/score-tracking.md: reset evaluator context by dispatching a fresh Reviewer - with stricter prompt. Add 'Drift detected in round N' to the final summary. + Per references/score-tracking.md: reset evaluator context once by dispatching a fresh Reviewer + with stricter prompt; repeated DRIFT triggers pivot_restart_signal and orchestrator-owned + pivot_restart_decision before another dispatch. - id: INV6 check: "For medium+ scope: score does not stagnate (unchanged for 2+ rounds with findings present)" @@ -244,8 +299,8 @@ invariants: condition: "scope_size in [medium, large] and round >= 3" on_fail: > Stagnation detected: score unchanged for 2+ rounds while findings remain. - Escalate to orchestrator — the loop is churning without progress. Consider whether - remaining issues are architectural (require broader changes) or the approach needs a PIVOT. + Return pivot_restart_signal and require orchestrator-owned pivot_restart_decision before + another fix/review dispatch. - id: INV7 @@ -267,3 +322,27 @@ invariants: scope: all_rounds condition: "agentic_loop_safety_review_required is true" on_fail: "Run the Agentic Loop Safety Checklist before declaring the review clean; report missing bounds, stop conditions, retry caps, empty-result handling, tool-error routing, stagnation detection, cost/token guards, or low-confidence escalation as findings." + + - id: INV_QA1 + check: "QA evaluation never replaces Code Reviewer or Reviewer compatibility review" + scope: all_rounds + condition: "qa_evaluation_mode == required" + on_fail: "Run code review first and record QAEvaluator only as the independent acceptance lane." + + - id: INV_QA2 + check: "QA evaluation round is always between 1 and 20, and round 20 is terminal" + scope: all_rounds + condition: "qa_evaluation_mode == required" + on_fail: "Clamp QA execution to the 20-round cap and return the final verdict with remaining failed acceptance items instead of starting round 21." + + - id: INV_QA3 + check: "QAEvaluator does not invent domain rubrics, subjective standards, or domain-quality penalties when no acceptance criteria, Done Contract, domain_context, or rubric_refs scope them" + scope: all_rounds + condition: "qa_evaluation_mode == required" + on_fail: "Discard unscoped domain rubric output and rerun QA with references/domain-rubrics.md conditional selection rules." + + - id: INV_QA4 + check: "QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists" + scope: all_rounds + condition: "qa_evaluation_mode == required" + on_fail: "Move unsupported QA concerns to non-blocking observations or request missing context with NEEDS_CONTEXT/BLOCKED." diff --git a/plugins/assistant-dev/skills/assistant-review/evals/cases.json b/plugins/assistant-dev/skills/assistant-review/evals/cases.json index 0620eb5..1d9ec63 100644 --- a/plugins/assistant-dev/skills/assistant-review/evals/cases.json +++ b/plugins/assistant-dev/skills/assistant-review/evals/cases.json @@ -30,7 +30,7 @@ "Runs review rounds with confidence thresholds and previously_fixed tracking.", "Fixes all must-fix and should-fix findings before validation.", "Runs build or test verification before the next round.", - "Continues review, fix, validate, and re-review until clean or max 10 rounds." + "Continues review, fix, validate, and re-review until clean or max 20 rounds." ], "pass_criteria": [ "The response identifies review scope and mode.", @@ -78,7 +78,7 @@ "expected_behavior": [ "Requires rubric scores for correctness, code quality, architecture, security, and test coverage.", "Applies a weighted score threshold action such as PASS, REFINE, or PIVOT.", - "Uses evidence-backed finding filtering across rounds 1-10 and treats round 10 as the hard cap.", + "Uses evidence-backed finding filtering across rounds 1-20 and treats round 20 as the hard cap.", "Buffers intermediate round details and presents one final summary after loop exit.", "Includes score progression when multiple rounds occur.", "Does not ask the user between rounds unless blocked." @@ -615,6 +615,73 @@ ] } }, + { + "id": "qa-evaluator-uses-domain-rubrics-conditionally", + "title": "QA evaluator uses domain rubrics conditionally", + "category": "qa_domain_rubric_evaluation", + "purpose": "Checks that QAEvaluator loads and applies domain-rubrics.md only for scoped subjective, docs, DX, product, UX, UI, or domain acceptance and returns selected rubric scores separately from code-review findings.", + "prompt": "Run the QA evaluation lane for a completed docs/DX workflow slice. The Done Contract says setup documentation must be accurate, the developer handoff must be easy to follow, and QA should use rubric_refs documentation_quality and developer_experience. Build/test and code-review evidence already exist.", + "setup_context": [ + "The assistant-review skill instructions are active.", + "qa_evaluation_mode is required and code-review/build evidence already exists.", + "The Done Contract and acceptance criteria scope documentation accuracy and developer experience.", + "domain_context describes a developer onboarding workflow.", + "rubric_refs contains documentation_quality and developer_experience.", + "No UI, visual design, product workflow, or unrelated domain craft rubric is in scope." + ], + "expected_behavior": [ + "Loads references/qa-evaluation-loop.md for the QA lane after code-review/build evidence.", + "Conditionally loads references/domain-rubrics.md because Done Contract, domain_context, and rubric_refs scope docs/DX quality.", + "Selects only documentation_quality and developer_experience rubrics.", + "Returns selected_domain_rubrics with the selected families.", + "Returns domain_quality_scores with evidence-backed per-dimension scores and action guidance.", + "Marks unscoped UI, product, and domain-specific craft dimensions not_applicable or omits them.", + "Keeps QA findings tied to acceptance criteria, Done Contract, verification evidence, domain_context, or rubric_refs.", + "Does not replace Code Reviewer or report general code defects unless they directly block acceptance.", + "Does not load or invent domain rubrics for unscoped acceptance." + ], + "pass_criteria": [ + "The response mentions references/domain-rubrics.md as conditional.", + "The response selects documentation_quality and developer_experience only.", + "The response returns selected_domain_rubrics and domain_quality_scores.", + "The response ties domain scores to acceptance criteria, Done Contract, domain_context, rubric_refs, and evidence.", + "The response keeps unscoped UI/product/domain craft rubrics not applicable or omitted.", + "The response keeps QA separate from code-review findings." + ], + "fail_signals": [ + "Always loads domain-rubrics.md for QA even when no domain quality is scoped.", + "Invents a UI or product rubric when the case only scopes docs/DX.", + "Omits selected_domain_rubrics or domain_quality_scores.", + "Scores domain quality without evidence.", + "Uses QAEvaluator as a replacement for Code Reviewer.", + "Reports general code defects not tied to acceptance." + ], + "machine_expectations": { + "required_substrings": [ + "references/domain-rubrics.md", + "conditional", + "selected_domain_rubrics", + "domain_quality_scores", + "documentation_quality", + "developer_experience", + "acceptance criteria", + "Done Contract", + "domain_context", + "rubric_refs", + "evidence", + "not_applicable", + "Code Reviewer" + ], + "forbidden_substrings": [ + "always load domain-rubrics", + "invent a UI rubric", + "product rubric required", + "domain scores do not need evidence", + "QAEvaluator replaces Code Reviewer", + "general code defects are QA findings" + ] + } + }, { "id": "review-finding-distills-permanent-rule-candidates", "title": "Review finding distills permanent rule candidates", diff --git a/plugins/assistant-dev/skills/assistant-review/references/domain-rubrics.md b/plugins/assistant-dev/skills/assistant-review/references/domain-rubrics.md new file mode 100644 index 0000000..e3c2b4a --- /dev/null +++ b/plugins/assistant-dev/skills/assistant-review/references/domain-rubrics.md @@ -0,0 +1,79 @@ +# Domain Rubrics + +Use this reference only for QA evaluation when scoped domain quality is part of acceptance. Domain rubric use is conditional and must be tied to at least one of: acceptance criteria, Done Contract, `domain_context`, or explicit `rubric_refs`. Do not load or apply these rubrics for code-review-only work, and do not invent new rubric families when the scope does not name or imply them. + +## Selection Rules + +- Select the smallest relevant rubric family set: `ui_visual_design`, `ux_product_acceptance`, `documentation_quality`, `developer_experience`, or `domain_specific_craft`. +- Score only selected dimensions that have evidence. Use `not_applicable` for unscoped dimensions instead of lowering the score. +- Evidence must cite acceptance criteria, Done Contract items, `domain_context`, `rubric_refs`, verification artifacts, screenshots, docs, commands, or concrete changed files. +- Return `selected_domain_rubrics` and `domain_quality_scores` when any family is selected. +- If a needed domain standard is not provided and cannot be discovered locally, return `NEEDS_CONTEXT` or `blocked` with open questions instead of fabricating the standard. + +## Scoring + +| Score | Meaning | QA action | +|---|---|---| +| 5 | Fully satisfies scoped expectations with direct evidence | accepted | +| 4 | Meets acceptance with minor non-blocking concerns | accepted_with_concerns | +| 3 | Partially meets scope; fixable gaps remain | refine | +| 1-2 | Core scoped expectation is contradicted or unproven | rejected or pivot | + +Use `accepted` only when all selected dimensions are 4+ and no blocker acceptance findings remain. Use `refine` when gaps are concrete and fixable within the approved scope. Use `pivot` when the selected rubric exposes a wrong product/design/domain direction, missing domain authority, or repeated failure that needs a plan change. + +## UI / Visual Design + +Use when acceptance names UI, visual design, layout, responsive behavior, screenshots, design parity, or explicit `rubric_refs: [ui_visual_design]`. + +| Dimension | Evidence examples | Pass / refine / pivot guidance | +|---|---|---| +| Layout and hierarchy | Screenshots, viewport checks, design refs, component files | Pass when priority and grouping match the task; refine overlaps, weak hierarchy, or cramped spacing; pivot if the visual model contradicts the requested design direction. | +| States and feedback | Loading, empty, error, hover/focus, disabled states | Pass when required states are represented; refine missing secondary states; reject if a required user path has no visible feedback. | +| Accessibility and readability | Keyboard/focus checks, contrast notes, text fit, labels | Pass when text is readable and controls are operable; refine low-risk contrast/text issues; reject blocked interaction, hidden labels, or clipped essential text. | +| Visual fit and consistency | Existing design system, tokens, component conventions | Pass when it fits the local system; refine inconsistent spacing/color/component usage; pivot if the implementation introduces a conflicting visual language. | + +## UX / Product Acceptance + +Use when acceptance names user workflow, product behavior, adoption/readiness, copy, or explicit `rubric_refs: [ux_product_acceptance]`. + +| Dimension | Evidence examples | Pass / refine / pivot guidance | +|---|---|---| +| Goal completion | User journey, task criteria, manual scenario evidence | Pass when target users can complete the stated job; refine friction or avoidable extra steps; reject if the primary job cannot be completed. | +| Decision clarity | Labels, prompts, defaults, next-action visibility | Pass when choices and consequences are clear; refine ambiguous copy; reject misleading or destructive flows. | +| Edge and recovery paths | Empty/error states, undo/retry paths, validation messages | Pass when expected edge cases are handled; refine incomplete recovery; reject dead ends for scoped scenarios. | +| Product fit | Domain context, stakeholder criteria, non-goals | Pass when behavior matches product intent; refine minor mismatches; pivot if the implementation solves the wrong user problem. | + +## Documentation Quality + +Use when acceptance names docs, README, API docs, migration notes, runbooks, or explicit `rubric_refs: [documentation_quality]`. + +| Dimension | Evidence examples | Pass / refine / pivot guidance | +|---|---|---| +| Accuracy | Compared code/config/contracts, command outputs, examples | Pass when docs match current behavior; refine stale or ambiguous details; reject false commands, wrong APIs, or unsafe guidance. | +| Completeness for task | Acceptance criteria, documented setup/use/troubleshooting path | Pass when the target reader can finish the task; refine missing edge notes; reject missing required setup or operation steps. | +| Structure and scanability | Headings, ordered procedures, tables, examples | Pass when information is easy to find; refine dense or duplicated sections; pivot if docs need a different artifact type. | +| Maintenance fit | Version notes, ownership, generated/manual boundaries | Pass when update path is clear; refine unclear ownership; reject docs that will drift from source of truth. | + +## Developer Experience + +Use when acceptance names CLI/API ergonomics, local setup, integration, maintainability for consumers, or explicit `rubric_refs: [developer_experience]`. + +| Dimension | Evidence examples | Pass / refine / pivot guidance | +|---|---|---| +| Setup and run path | Commands, scripts, examples, env/config notes | Pass when a developer can start reliably; refine missing context; reject hidden prerequisites or broken commands. | +| Contract ergonomics | API shape, schema fields, handoff packets, error messages | Pass when contracts are predictable and typed; refine confusing names/defaults; reject ambiguous or incompatible contract behavior. | +| Debuggability | Logs, validation errors, trace/replay artifacts, failure messages | Pass when failures are diagnosable; refine thin evidence; reject silent failure or misleading diagnostics. | +| Change safety | Tests/evals/guards, migration notes, compatibility story | Pass when future edits have guardrails; refine narrow gaps; reject fake-pass coverage or unguarded public contract changes. | + +## Domain-Specific Craft + +Use when acceptance names a specialized domain, craft standard, industry rule, expert terminology, or explicit `rubric_refs: [domain_specific_craft]`. + +| Dimension | Evidence examples | Pass / refine / pivot guidance | +|---|---|---| +| Domain rule fidelity | Done Contract, domain_context, standards, fixtures | Pass when required rules are preserved; refine minor omissions; reject contradictions to scoped domain rules. | +| Terminology and mental model | User language, docs, UI copy, schema names | Pass when terms match the domain audience; refine inconsistent wording; reject misleading terms that change meaning. | +| Expert edge cases | Provided edge cases, fixtures, acceptance examples | Pass when scoped edge cases are covered; refine low-risk omissions; reject missing high-impact edge cases. | +| Artifact craft | Generated asset, workflow, document, content, or domain output | Pass when artifact quality meets the named craft bar; refine polish gaps; pivot if the artifact type or approach is wrong. | + +High-stakes domains still require the relevant specialist gate when applicable. This rubric does not replace security, legal, medical, financial, privacy, or compliance review. diff --git a/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md b/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md new file mode 100644 index 0000000..f51f9b0 --- /dev/null +++ b/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md @@ -0,0 +1,115 @@ +# QA Evaluation Loop + +Use this reference after build/test evidence and the Code Reviewer loop exist. QA evaluation is a separate acceptance lane: it decides whether the work satisfies the Done Contract, acceptance criteria, verification evidence, and scoped domain quality expectations. It does not replace code-reviewer. + +## When QA Runs + +Run QA evaluation when any of these are true: +- Medium+ harness-capable work has a Done Contract. +- The task is subjective, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly asks for QA. +- Workflow Review needs a final acceptance verdict beyond code/security/architecture/test-coverage review. + +Skip QA evaluation only when the task is trivial/small, has no acceptance material beyond a direct code fix, and the workflow output records `qa_evaluation_mode=not_required` with a reason. + +## Inputs + +The orchestrator provides: +- `done_contract`: done_when, not_done_when, verification, owner_consumer, acceptance_criteria, debate_record, accepted_by. +- `debate_record`: pre-build debate/subagent-perspective evidence from the Done Contract; required when a Done Contract exists. +- `acceptance_criteria`: binary criteria from the user request, approved plan, slice manifest, or Done Contract. +- `verification_evidence`: build/test/manual/check evidence already produced by Builder/Tester or direct fallback. +- `code_review_result`: final Code Reviewer or Reviewer compatibility result. +- `domain_context`: scoped UI/visual, product, UX, docs, DX, or domain notes when applicable. +- `rubric_refs`: explicit references to scoped domain rubric families from `references/domain-rubrics.md` when applicable. +- `round`: QA round number from 1 to 20. +- `previously_failed_acceptance_items`: failed acceptance items from earlier QA rounds. +- `qa_filter_policy`: acceptance findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns stay non-blocking. + +## Conditional Domain Rubrics + +Load `references/domain-rubrics.md` only when `domain_context` or `rubric_refs` are present, or when acceptance criteria / Done Contract explicitly require subjective, product, UX, docs, DX, UI/visual, or domain craft judgment. + +When loaded: +- Select only rubric families tied to acceptance criteria, Done Contract, `domain_context`, or explicit `rubric_refs`. +- Return `selected_domain_rubrics` and `domain_quality_scores`. +- Keep domain findings evidence-backed and scoped to acceptance impact. +- Use `not_applicable` for unscoped dimensions. + +When not loaded: +- Set `domain_quality` to 5.0 and state `not_applicable` in the score rationale. +- Do not invent domain rubrics, subjective standards, or extra acceptance bars. +- Do not penalize the work for missing domain evidence that was never scoped. + +## Loop + +```text +round = 1 +previously_failed_acceptance_items = [] +score_progression = [] + +while round <= 20: + 1. EVALUATE ACCEPTANCE + Dispatch qa-evaluator in delegated mode, or use direct fallback with fresh QA context. + Check every acceptance criterion, Done Contract item, and Done Contract debate_record independently when a Done Contract exists. + Compare verification evidence to the claimed outcome. + Conditionally load and apply references/domain-rubrics.md only for scoped domain-quality acceptance. + + 2. SCORE + Return qa_scorecard with compact dimensions: + acceptance_coverage, evidence_strength, domain_quality, final_readiness, weighted_score + Return selected_domain_rubrics and domain_quality_scores when domain rubrics were selected. + Record score_entry: round, weighted_score, failed_acceptance_count, delta, drift_status. + If score progression reports STAGNATION, repeated DRIFT, repeated REGRESSION, + or selected domain rubric action pivot, return pivot_restart_signal to the + orchestrator before another QA/build dispatch. + + 3. DECIDE + accepted: no failed acceptance items and evidence proves done. + accepted_with_concerns: acceptance passes with documented non-blocking limitations. + rejected: one or more acceptance or Done Contract items fail. + blocked: required acceptance material or verification evidence is missing. + + 4. FIX OR EXIT + If rejected before round 20, return failed acceptance items to Build for fixes, then re-run QA. + If blocked, return NEEDS_CONTEXT or BLOCKED with open_questions. + If accepted or accepted_with_concerns, exit with final result. + Round 20 is terminal; return remaining failed acceptance items instead of starting round 21. + If pivot_restart_signal was returned, pause QA until the orchestrator + records pivot_restart_decision with exact_next_action. +``` + +## Finding Rules + +QA findings are about acceptance, not general code quality: +- Failed acceptance criterion. +- Done Contract item not proven or contradicted by evidence. +- Done Contract debate_record is missing, has fewer than two perspectives, or does not show pre-build debate/subagent-perspective evidence when a Done Contract exists. +- Verification evidence mismatch or missing proof. +- Product, UX, docs, DX, or domain issue only when scoped by acceptance criteria, Done Contract, domain_context, or rubric_refs. + +Do not report generic code defects, architecture concerns, security issues, or test coverage gaps unless they directly cause an acceptance failure. Those remain Code Reviewer responsibility. + +## Final Result + +The QA loop returns one final result: +- `rounds`: 1-20. +- `final_verdict`: accepted, accepted_with_concerns, rejected, or blocked. +- `result`: CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, or BLOCKED. +- `acceptance_findings`: remaining or resolved acceptance findings with evidence. +- `qa_scorecard`: final compact scores. +- `selected_domain_rubrics`: selected rubric families when scoped domain rubrics were used; empty or omitted when not applicable. +- `domain_quality_scores`: per-family/per-dimension scores when scoped domain rubrics were used; empty or omitted when not applicable. +- `score_progression`: score_entry per round. +- `pivot_restart_signal`: required when QA STAGNATION, repeated DRIFT, repeated REGRESSION, or scoped domain action pivot is detected. +- `evidence`: materials inspected and why they prove or fail acceptance. +- `open_questions`: required when blocked. + +## Pivot/Restart Escalation + +QA does not silently continue after stagnant or pivot-triggered acceptance loops. +When it returns `pivot_restart_signal`, the orchestrator records +`pivot_restart_decision` with trigger, evidence, affected_slice_or_round, +options_considered, selected_action, reapproval_required, next_agent, +recovery_pointer, and exact_next_action. Update harness trace/replay/run-state +when available. Reapproval is required when the selected action changes scope, +files, behavior, risk, verification, or acceptance criteria. diff --git a/plugins/assistant-dev/skills/assistant-review/references/review-rubric.md b/plugins/assistant-dev/skills/assistant-review/references/review-rubric.md index 5a41070..36c864b 100644 --- a/plugins/assistant-dev/skills/assistant-review/references/review-rubric.md +++ b/plugins/assistant-dev/skills/assistant-review/references/review-rubric.md @@ -107,9 +107,9 @@ As the loop progresses, the bar rises — like a tightening ratchet: | 1 | 4.0+ | 2.5–3.9 | < 2.5 | | 2 | 4.0+ | 2.75–3.9 | < 2.75 | | 3 | 4.0+ | 3.0–3.9 | < 3.0 | -| 4-10 | 4.0+ | 3.25–3.9 | < 3.25 | +| 4-20 | 4.0+ | 3.25–3.9 | < 3.25 | -Rationale: If the code hasn't reached 3.25 by round 4, the approach likely needs rethinking, not more polish. Rounds 4-10 keep that pivot threshold stable while confidence requirements continue to rise. +Rationale: If the code hasn't reached 3.25 by round 4, the approach likely needs rethinking, not more polish. Rounds 4-20 keep that pivot threshold stable while confidence requirements continue to rise. ## Reviewer Scoring Instructions diff --git a/plugins/assistant-dev/skills/assistant-review/references/score-tracking.md b/plugins/assistant-dev/skills/assistant-review/references/score-tracking.md index 078f97f..a90d1c8 100644 --- a/plugins/assistant-dev/skills/assistant-review/references/score-tracking.md +++ b/plugins/assistant-dev/skills/assistant-review/references/score-tracking.md @@ -50,13 +50,15 @@ Score went up BUT finding count didn't decrease. The evaluator is scoring higher score_delta > 0 AND finding_count_delta >= 0 → DRIFT ``` -Action: **Reset evaluator context.** Dispatch a fresh reviewer agent with an explicitly stricter prompt: +Action: **Reset evaluator context once.** Dispatch a fresh reviewer agent with an explicitly stricter prompt: > "Previous rounds showed score inflation without corresponding quality improvement. > Apply maximum skepticism. Score conservatively — when uncertain, round DOWN. > Do not give benefit of the doubt on any dimension." Also add to the final summary: "Drift detected in round N — evaluator was reset." +Repeated DRIFT after the reset triggers `pivot_restart_signal` and requires an +orchestrator-owned `pivot_restart_decision` before another review dispatch. ### Rule 4: REGRESSION @@ -66,7 +68,7 @@ Score went down. Fixes may have introduced new problems, or the evaluator is cat score_delta < 0 → REGRESSION ``` -Action: This is not necessarily bad — a fresh evaluator may legitimately find more issues. Log it but don't escalate unless regression persists for 2+ consecutive rounds. +Action: This is not necessarily bad — a fresh evaluator may legitimately find more issues. Log it, but 2+ consecutive REGRESSION entries trigger `pivot_restart_signal` and require an orchestrator-owned `pivot_restart_decision` before another fix/review dispatch. ### Rule 5: STAGNATION @@ -76,10 +78,13 @@ Score unchanged for 2+ consecutive rounds with findings still present. score_delta == 0 for 2 consecutive rounds AND finding_count > 0 → STAGNATION ``` -Action: Flag to orchestrator. The loop is churning without progress. Consider: +Action: Return `pivot_restart_signal` to the orchestrator. The loop is churning +without progress. The orchestrator must record `pivot_restart_decision` before +another fix/review dispatch and consider: - Fixes are introducing new issues at the same rate as resolving old ones - The remaining issues may be architectural (can't fix without broader changes) -- May need to PIVOT or accept current state with documented limitations +- The selected recovery may need reset, candidate search, replan, restart, or a + blocked/user path ## Decision Matrix @@ -87,9 +92,9 @@ Action: Flag to orchestrator. The loop is churning without progress. Consider: |---|---|---|---| | +, magnitude ≤ 1.0 | count decreased | **GENUINE** | Continue normally | | +, magnitude > 1.0 | count decreased | **SUSPICIOUS** | Log warning, continue | -| +, any | count same or increased | **DRIFT** | Reset evaluator, stricter prompt | -| − | any | **REGRESSION** | Log, investigate if 2+ rounds | -| 0 | any, findings > 0 remain, 2+ consecutive rounds | **STAGNATION** | Escalate to orchestrator | +| +, any | count same or increased | **DRIFT** | Reset evaluator once; repeated DRIFT triggers pivot_restart_decision | +| − | any | **REGRESSION** | Log; 2+ consecutive regressions trigger pivot_restart_decision | +| 0 | any, findings > 0 remain, 2+ consecutive rounds | **STAGNATION** | Return pivot_restart_signal and require pivot_restart_decision | | 0 | any, findings > 0 remain, 1 round only | **NEUTRAL** | Log, no action yet | | 0 | findings == 0 | *(exit as CLEAN)* | Should not reach drift check | @@ -101,7 +106,7 @@ Record scores in each Quality Review entry: ```markdown ### Quality Review #N -- Round: N of 10 +- Round: N of 20 - Previously fixed: M items from prior rounds - Found this round: X must-fix, Y should-fix, Z nits - Rubric: correctness=4.0 quality=3.5 architecture=4.0 security=5.0 coverage=3.0 @@ -128,7 +133,28 @@ Include in the review exit summary: | Drift Count | Response | |---|---| | 1 occurrence | Reset evaluator, stricter prompt | -| 2 occurrences | Flag to orchestrator, consider different model for evaluation | -| 3+ occurrences | Stop loop, present findings to user for manual review | +| 2 occurrences | Return pivot_restart_signal; orchestrator records pivot_restart_decision | +| 3+ occurrences | Stop current review path until pivot_restart_decision selects reset, candidate search, replan, restart, or blocked/user path | The goal is not to prevent the loop from exiting — it's to ensure that when it exits, the code genuinely earned its passing score. + +## Pivot/Restart Decision Packet + +When STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT +fires, the review loop does not silently continue. It returns +`pivot_restart_signal`; the orchestrator records `pivot_restart_decision` with: + +- `trigger` +- `evidence` +- `affected_slice_or_round` +- `options_considered` +- `selected_action` +- `reapproval_required` +- `next_agent` +- `recovery_pointer` +- `exact_next_action` + +If the selected action changes approved scope, files, behavior, risk, +verification, or acceptance criteria, `reapproval_required` is true and the +workflow waits for approval before Build or Review continues. Round 20 remains +terminal; do not start round 21. diff --git a/plugins/assistant-dev/skills/assistant-skill-creator/evals/cases.json b/plugins/assistant-dev/skills/assistant-skill-creator/evals/cases.json index e81cfba..0481666 100644 --- a/plugins/assistant-dev/skills/assistant-skill-creator/evals/cases.json +++ b/plugins/assistant-dev/skills/assistant-skill-creator/evals/cases.json @@ -219,6 +219,58 @@ "stored PR numbers" ] } + }, + { + "id": "loop-process-skill-applies-harness-patterns", + "title": "Loop-based process skill applies harness patterns", + "category": "harness_patterns", + "purpose": "Checks that assistant-skill-creator applies the harness-pattern reference when creating a loop-based Process skill.", + "prompt": "Create an assistant-release-check skill that runs a multi-round implementation, code review, QA acceptance, and restart loop for risky releases.", + "setup_context": [ + "The assistant-skill-creator skill instructions are active.", + "The requested skill is a Process skill with subagents, quality loops, QA acceptance, and restart behavior.", + "No design has been approved yet." + ], + "expected_behavior": [ + "Classifies the request as a Process skill.", + "Loads or applies references/harness-patterns.md before designing contracts.", + "Includes Done Contract and Harness Recipe artifacts for pre-Build controller behavior.", + "Uses typed Artifact References for cross-agent artifacts.", + "Keeps Code Reviewer and QA Evaluator handoffs and outputs separate.", + "Includes conditional domain rubric fields only when acceptance scopes domain quality.", + "Adds pivot/restart decision fields for stagnation, drift, regression, rubric pivot, and Code Writer blockers.", + "Bounds review and QA loops with max 20 rounds and terminal round 20 behavior." + ], + "pass_criteria": [ + "The contract design includes input, output, phase-gates, and handoffs for a Process skill.", + "The response explicitly names the harness patterns being applied.", + "The response does not collapse code review and QA into one evaluator.", + "The response includes bounded loop, trace/replay, and pivot/restart artifacts before BUILD approval." + ], + "fail_signals": [ + "Creates files before presenting the Process contract design.", + "Uses only a generic generator/evaluator loop.", + "Sets a max 10 round cap for the review or QA loop.", + "Omits pivot/restart handling for stagnation or Code Writer blockers." + ], + "machine_expectations": { + "required_substrings": [ + "Process", + "references/harness-patterns.md", + "Done Contract", + "Harness Recipe", + "Artifact Reference", + "Code Reviewer", + "QA Evaluator", + "pivot_restart_decision", + "max 20" + ], + "forbidden_substrings": [ + "generic generator/evaluator loop is enough", + "max 10", + "QA replaces code review" + ] + } } ] } diff --git a/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md b/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md index 24fafa9..436af52 100644 --- a/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md +++ b/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md @@ -1,311 +1,307 @@ -# Harness Patterns for Loop-Based Skills +# Harness Patterns for Loop-Based Process Skills -Reference pack for the skill creator. Load this when designing a **Process** skill that includes an evaluation loop, multi-round refinement, or autonomous fix-verify cycles. +Reference pack for `assistant-skill-creator`. Load this when designing a +**Process** skill that includes a long-running controller, subagent dispatch, +multi-round review, QA/acceptance evaluation, retry loops, or autonomous +fix-verify cycles. -**When to load:** The skill being created has any of these characteristics: -- Multi-round loop (review, refinement, optimization) -- Separate generator and evaluator roles -- Quality scoring against criteria -- Autonomous fix → re-check cycles - -**When to skip:** Single-pass skills, analysis pipelines without loops, utility skills. +Skip this reference for single-pass Utility skills or Analysis skills without +delegation or loop control. --- -## Pattern 1: Rubric Scoring +## Pattern 1: Done Contract -Replace open-ended evaluation ("is this good?") with weighted dimensions scored 1-5. +Use a Done Contract when "finished" needs to be known before Build or generation +starts. -### How to apply +Required fields: -1. **Identify 3-6 evaluation dimensions** specific to the skill's domain -2. **Assign weights** that sum to 1.0 — weight subjective dimensions higher to push beyond generic output -3. **Write anchor examples** for each dimension at scores 1, 3, and 5 -4. **Define threshold actions** that map score ranges to loop decisions +- `done_when`: observable pass/fail outcomes. +- `not_done_when`: explicit failure states that block completion. +- `verification`: commands, inspections, reviews, QA, or manual checks. +- `owner_consumer`: artifact owner and downstream consumer. +- `acceptance_criteria`: binary criteria from user, plan, or slice scope. +- `debate_record`: at least two perspectives considered before acceptance. +- `accepted_by`: user, orchestrator, or approved plan reference. -### Template +Design rule: when `subagent_execution_mode=delegated`, use subagent perspectives +for the debate when relevant. Direct fallback must record why subagents were not +used and which role-equivalent perspectives were considered. The debate may +clarify acceptance; it must not add unapproved scope. -```yaml -# In the skill's references/rubric.md or inline in SKILL.md +Contract placement: -dimensions: - - name: [dimension_name] - weight: [0.10-0.40] - measures: "[what this dimension evaluates]" - anchors: - 5: "[concrete description of excellent]" - 3: "[concrete description of acceptable]" - 1: "[concrete description of poor]" +- `output.yaml`: `done_contract` artifact for medium+ harness-capable work. +- `phase-gates.yaml`: a pre-Build assertion that the accepted Done Contract + exists and includes debate evidence. +- `handoffs.yaml`: `done_contract_ref` for workers, reviewers, and QA. -thresholds: - pass: 4.0 # Exit loop — quality sufficient - refine: 3.0 # Continue loop — specific improvements needed - pivot: 2.5 # Escalate — approach may be fundamentally wrong -``` +--- -### Scoring rules to include in the evaluator's prompt - -- Score each dimension independently (no halo effect) -- Cite specific evidence for each score -- When uncertain, round down -- Critical findings can cap the total score (define what's critical for your domain) - -### Contract integration - -Add to the evaluator's **return_fields** in `handoffs.yaml`: - -```yaml -- name: rubric_scores - type: object - required: true - condition: "scope warrants scoring" - object_fields: - - name: [dimension_name] - type: float - required: true - validation: "1.0 to 5.0 in 0.5 increments" - # ... one per dimension - - name: weighted_score - type: float - required: true - - name: action - type: enum - required: true - enum_values: [PASS, REFINE, PIVOT] -``` +## Pattern 2: Harness Recipe -### Example: applying to a content generation skill - -```yaml -dimensions: - - name: accuracy - weight: 0.35 - measures: "Factual correctness, no hallucination, citations where needed" - - name: clarity - weight: 0.25 - measures: "Clear structure, audience-appropriate language, logical flow" - - name: completeness - weight: 0.20 - measures: "All requested topics covered, no significant gaps" - - name: tone - weight: 0.20 - measures: "Matches requested style, consistent voice throughout" -``` +Use a Harness Recipe to choose the controller shape from task/model/risk/context +profiles. -**Reference implementation:** `skills/assistant-review/references/review-rubric.md` +Required profile fields: ---- +- `task_profile`: task type, size, slice count, TDD/debugging needs. +- `model_profile`: model/agent constraints, delegation mode, tool limits. +- `risk_profile`: risk tier, safety gates, review depth, rollback needs. +- `context_profile`: exact, summarized, omitted/deferred context and + trace/replay need. -## Pattern 2: Drift Detection +Required recipe fields: -Track evaluator scores across loop rounds to ensure genuine improvement, not evaluator fatigue. +- `selected_recipe` +- `recipe_rationale` +- `required_artifacts` +- `corrective_action` -### How to apply +Typical recipes: -1. **Record score + finding count per round** in a score history -2. **Compare round N to round N-1** using score delta and finding count -3. **Classify the change** and take appropriate action +- `lightweight_guarded`: medium single-slice, moderate risk, compact context. +- `slice_sequential`: independent slice verification before the next slice. +- `review_intensive`: high/critical risk, weak tests, public contracts, or + subjective acceptance. +- `trace_replay_ready`: long-running, large-context, recovery-prone work. -### Drift classification +Contract placement: -| Score Delta | Finding Condition | Status | Action | -|---|---|---|---| -| +, ≤ 1.0 | count decreased | **GENUINE** | Continue | -| +, > 1.0 | count decreased | **SUSPICIOUS** | Log warning | -| +, any | count same or increased | **DRIFT** | Reset evaluator | -| − | any | **REGRESSION** | Investigate | -| 0 | findings remain, 2+ rounds | **STAGNATION** | Escalate | -| 0 | findings remain, 1 round | **NEUTRAL** | Log only | +- `output.yaml`: `harness_recipe` artifact. +- `phase-gates.yaml`: pre-Build recipe selection assertion. +- `handoffs.yaml`: `harness_recipe_ref` for downstream workers. -### On DRIFT: evaluator reset +--- -Dispatch a fresh evaluator with an explicitly stricter prompt: +## Pattern 3: Runtime State, Trace, And Replay -> "Previous rounds showed score inflation without quality improvement. Apply maximum skepticism. When uncertain, score DOWN." +Long-running Process skills need first-class recovery artifacts. -### Contract integration +| Artifact | Required When | Fields | +|---|---|---| +| Harness Run State | Medium+ harness-capable or delegated loops | task id/name, phase, slice, status, blockers, last verification, next action, recovery pointer | +| Trace Ledger | Any trace/replay-ready loop | agent events, decisions, verification results, deviations, blockers, artifact refs | +| Replay Packet | Compaction, restart, handoff, or failure recovery is likely | pinned context, artifact refs, validation state, exact next action, run-state/trace refs | -Add to **phase-gates.yaml** as loop invariants: +Contract placement: -```yaml -invariants: - - id: INV_DRIFT - check: "Rubric score increases correlate with finding count decreases" - scope: all_rounds - condition: "round >= 2" - on_fail: "Drift detected. Reset evaluator with stricter prompt." +- `output.yaml`: required artifacts or conditional artifacts for trace/replay + workflows. +- `phase-gates.yaml`: assertions that state/trace/replay are current before + phase advancement. +- `handoffs.yaml`: refs carried into worker packets when a role relies on them. - - id: INV_STAGNATION - check: "Score does not stagnate (unchanged 2+ rounds with findings present)" - scope: all_rounds - condition: "round >= 3" - on_fail: "Escalate to orchestrator — loop churning without progress." -``` +--- -Add to **output.yaml** as score progression: - -```yaml -- name: score_progression - type: object[] - required: true - condition: "loop ran 2+ rounds" - object_fields: - - name: round - type: int - - name: weighted_score - type: float - - name: finding_count - type: int - - name: drift_status - type: enum - enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL] -``` +## Pattern 4: Typed Artifact References + +When artifacts cross an agent boundary, pass typed refs instead of free-form +strings. + +Each Artifact Reference entry includes: + +- `artifact_id` +- `artifact_type` +- `producer` +- `consumer` +- `location_ref` +- `schema_or_contract` +- `validation_status` +- `summary` + +Use refs for Done Contract, Harness Recipe, Harness Run State, Trace Ledger, +Replay Packet, Pivot/Restart Decision, task packets, changed files, +verification evidence, review result, QA result, and plan deviations. -**Reference implementation:** `skills/assistant-review/references/score-tracking.md` +Producer responsibility: create/update the artifact, assign a stable id and +location/ref, name the schema or contract, and summarize current state. +Consumer responsibility: validate `schema_or_contract` and +`validation_status` before relying on `location_ref`; invalid or stale refs block +phase advancement or trigger re-dispatch. --- -## Pattern 3: Harness Gates +## Pattern 5: Code Review And QA Separation -Structural enforcement that blocks loop completion without required artifacts. +Do not let one evaluator do every job. -### How to apply +| Role | Owns | Does Not Own | +|---|---|---| +| Code Reviewer | code defects, security, architecture, test coverage, structural code risk | final acceptance or subjective domain quality unless directly tied to a code defect | +| QA Evaluator | Done Contract, acceptance criteria, verification evidence, final readiness, scoped domain quality | generic code review, security architecture, or test coverage review | -1. **Identify the artifacts** that must exist before the loop can exit (rubric scores, passing threshold, all items addressed) -2. **Add phase-gate assertions** that check for these artifacts -3. **Optionally add a Stop hook** for hard enforcement (shell script that parses the task journal) +`reviewer` can remain a compatibility route, but new skills should name the +canonical `code-reviewer` and optional `qa-evaluator` roles separately. -### Phase-gate assertions for loop exit +Contract placement: -```yaml -- id: EXIT_SCORE - check: "Rubric weighted score meets pass threshold" - on_fail: "Score below threshold — continue loop or escalate" +- `handoffs.yaml`: distinct handoffs to Code Reviewer and QAEvaluator. +- `output.yaml`: separate `review_result` and `qa_evaluation_result`. +- `phase-gates.yaml`: QA starts only after build/test evidence and Code + Reviewer or compatibility review evidence exist. -- id: EXIT_FINDINGS - check: "No must-fix or should-fix findings remain" - on_fail: "Findings remain — fix all before exiting" +--- -- id: EXIT_VERIFICATION - check: "Build and tests pass after all fixes" - on_fail: "Verification failed — fix before exiting" -``` +## Pattern 6: Conditional Domain Rubrics -### Stop hook pattern (optional, for hard enforcement) +Use domain rubrics only when acceptance scopes them. Load or model rubric fields +when the Done Contract, acceptance criteria, `domain_context`, or explicit +`rubric_refs` require UI/visual, product, UX, docs, DX, or domain craft quality. -If the skill runs within the workflow (task journal exists), you can add a Stop hook that: -1. Reads the task journal -2. Checks for required artifacts (rubric lines, score threshold, final result) -3. Blocks stop if missing +When scoped, QA returns: -Follow the pattern in `hooks/scripts/harness-gate.sh`: -- Use `set -euo pipefail` -- Check `stop_hook_active` to prevent infinite loops -- Use `jq` for JSON output -- Exit 0 (allow) or output `{"decision": "block", "reason": "..."}` (block) +- `selected_domain_rubrics` +- `domain_quality_scores` +- evidence tied to acceptance criteria, Done Contract, `domain_context`, + `rubric_refs`, verification artifacts, screenshots, docs, or changed files -**Reference implementation:** `hooks/scripts/harness-gate.sh` +When not scoped, QA records domain quality as `not_applicable` and must not +invent subjective rubrics. + +Contract placement: + +- `input.yaml`: optional `domain_context` / `rubric_refs`. +- `handoffs.yaml`: conditional `selected_domain_rubrics` and + `domain_quality_scores`. +- `phase-gates.yaml`: invariant rejecting invented domain rubrics. --- -## Pattern 4: Separation of Generation and Evaluation - -The evaluator must never review its own fixes. - -### How to apply - -1. **Evaluator is read-only** — tools limited to Read, Grep, Glob, LS (no Edit, Write, Bash) -2. **Fresh evaluator per round** — no context contamination from previous evaluations -3. **Previously-fixed list** — each round receives what was already fixed, must not re-report -4. **Fixer is a separate agent** — the orchestrator or a dedicated implementer applies fixes - -### Handoff contract for evaluator - -```yaml -handoffs: - - name: orchestrator_to_evaluator - from: Orchestrator - to: Evaluator - context_fields: - - name: content_to_evaluate - type: string - required: true - - name: round - type: int - required: true - - name: previously_fixed - type: object[] - required: true - description: "Items fixed in prior rounds — do NOT re-report" - - name: confidence_threshold - type: int - required: true - description: "Minimum confidence for this round (increases per round)" - return_fields: - - name: findings - type: object[] - required: true - - name: rubric_scores - type: object - required: true - - name: verdict - type: enum - required: true +## Pattern 7: Bounded Review / QA Loops + +Any autonomous review, QA, refinement, or fix-verify loop needs a hard cap. The +current framework cap is **max 20 rounds**. + +```text +round = 1 +while round <= 20: + evaluate + decide PASS / REFINE / PIVOT + fix or exit + round += 1 ``` -### Confidence progression +Round 20 is terminal. Return remaining blockers, findings, or failed acceptance +items instead of starting round 21. -| Rounds | Threshold | Rationale | -|---|---|---| -| 1–2 | 80% | Cast wide net | -| 3–4 | 85% | Higher certainty required | -| 5 | 90% | Only near-certain findings | +Include these fields where applicable: -**Reference implementation:** `skills/assistant-review/contracts/handoffs.yaml` +- `round`: current round number, validation `>= 1 and <= 20`. +- `max_rounds`: default `20`. +- `previously_fixed` or `previously_failed_acceptance_items`. +- `score_progression` or `score_entry`. +- `loop_exit_reason`. --- -## Decision Checklist +## Pattern 8: Rubric Scoring And Drift Detection -When creating a Process skill with a loop, check which patterns apply: +Use scored rubrics when quality is subjective or multi-dimensional. -| Question | If Yes → Apply | -|---|---| -| Does the loop evaluate quality? | Pattern 1 (Rubric) | -| Does the loop run 2+ rounds? | Pattern 2 (Drift Detection) | -| Must the loop complete before the task finishes? | Pattern 3 (Harness Gates) | -| Does one agent create and another evaluate? | Pattern 4 (Separation) | +Scoring fields: + +- 3-6 domain-relevant dimensions, weights summing to 1.0. +- Anchors for 1, 3, and 5. +- `weighted_score`. +- action enum such as `PASS`, `REFINE`, `PIVOT`. +- evidence for each score. + +Drift classification: + +| Signal | Condition | Action | +|---|---|---| +| GENUINE | score improves and findings decrease | continue | +| SUSPICIOUS | score jumps sharply while findings decrease | log warning | +| DRIFT | score improves while findings stay flat or rise | reset evaluator | +| REGRESSION | score drops | investigate | +| STAGNATION | findings remain across repeated unchanged scores | pivot/restart | -Most loop-based skills will use all four. Simple refinement loops (e.g., "retry until build passes") may only need Pattern 3. +Add cross-phase invariants for drift and stagnation. Repeated `DRIFT`, repeated +`REGRESSION`, `STAGNATION`, or rubric action `PIVOT` must return a +`pivot_restart_signal`; the orchestrator records the actual +`pivot_restart_decision`. --- -## Full reference +## Pattern 9: Pivot / Restart Decisions -See `docs/harness-design-guide.md` for the complete design guide with architecture rationale, research references, and evolution principles. +Use Pivot/Restart when the active loop or handoff no longer makes safe progress. +Triggers: + +- review or QA `STAGNATION` +- repeated `DRIFT` +- repeated `REGRESSION` +- rubric/domain action `PIVOT` +- Code Writer blocker types such as `legacy_code_bug`, `broken_baseline`, + `hidden_dependency`, `missing_contract`, `stale_plan`, `scope_conflict`, + `tool_environment`, `permission_policy`, `tdd_red_missing`, or `other` +- verification blockers, plan deviations, or scope changes that stale the packet + +Decision fields: + +- `trigger` +- `evidence` +- `affected_slice_or_round` +- `options_considered` +- `selected_action` +- `reapproval_required` +- `next_agent` +- `recovery_pointer` +- `exact_next_action` + +Routing examples: + +- Legacy code bug or broken baseline -> debugging before another implementation + attempt. +- Hidden dependency -> Explorer or Code Mapper refresh. +- Missing contract or stale plan -> Architect or Plan repair. +- Scope conflict or candidate pivot -> replan and reapproval when scope changes. +- Tool/environment/policy blocker -> environment fix, permission request, or + BLOCKED with evidence. --- -## Pattern 5: Agentic Loop Safety +## Pattern 10: Agentic Loop Safety -Apply this pattern whenever a Process skill includes autonomous or repeated execution: agent loops, retry loops, search/retrieval loops, multi-round subagent dispatch, model/tool call loops, or background jobs. +Apply this pattern to every repeated agent/tool/model loop. Required design fields: -- **Bounded execution:** max steps/rounds, timeout, or budget. -- **Stop condition:** explicit success, clean exit, max-budget exit, blocker exit, and escalation path. -- **Retry policy:** capped retries scoped to transient failures only. -- **Empty-result handling:** fallback, broaden-once, report no evidence, ask/escalate, or exit. -- **Tool-error handling:** failures route to retry, fallback, blocker, or degraded result; never silent continuation. -- **Progress signal:** each iteration must produce new evidence, fewer findings, better score, or another measurable improvement. -- **Cost/token guard:** paid API/model/subagent/large-context loops need cost/time/token awareness. +- bounded execution: max rounds, steps, timeout, or budget +- stop condition: success, clean exit, max-budget exit, blocker exit +- retry policy: capped retries for transient failures only +- empty-result handling: broaden once, fallback, report no evidence, ask, or exit +- tool-error routing: retry, fallback, blocker, or degraded result +- progress signal: new evidence, fewer findings, better score, or explicit state +- cost/token guard: paid model, subagent, and large-context loops need budget + awareness -Contract placement: +A loop without bounds, stop conditions, empty-result handling, tool-error +routing, and a progress signal is incomplete even when the happy path works. -- Put the loop bounds and retry policy in `contracts/input.yaml` or phase-gate configuration when user/config controlled. -- Put step-level assertions in `contracts/phase-gates.yaml`. -- Put final budget/loop outcome reporting in `contracts/output.yaml`. -- Put worker/subagent loop fields in `contracts/handoffs.yaml` when a subagent participates. +--- + +## Decision Checklist -A loop without bounds, stop conditions, empty-result handling, and tool-error routing is incomplete even if it works in a happy-path demo. +For a loop-based Process skill, require the matching patterns: + +| Question | Apply | +|---|---| +| Does Build need a definition of done? | Done Contract | +| Does controller shape vary by task/risk/context? | Harness Recipe | +| Could context compaction, restart, or handoff happen? | Run State, Trace, Replay | +| Do artifacts cross role boundaries? | Typed Artifact References | +| Are code quality and acceptance both evaluated? | Code Reviewer / QA Evaluator split | +| Is subjective/domain quality part of acceptance? | Conditional Domain Rubrics | +| Does a loop run multiple rounds? | Max 20 loop cap, scoring, drift detection | +| Can the approach go stale or hit legacy blockers? | Pivot/Restart Decision | + +Most loop-based Process skills use all of these. Simple retry-only skills still +need bounded execution, stop conditions, error routing, and progress signals. + +Full rationale and implementation references live in +`docs/harness-design-guide.md` and +`skills/assistant-workflow/references/harness-controller.md`. diff --git a/plugins/assistant-dev/skills/assistant-workflow/SKILL.md b/plugins/assistant-dev/skills/assistant-workflow/SKILL.md index a035df9..732deaa 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/SKILL.md +++ b/plugins/assistant-dev/skills/assistant-workflow/SKILL.md @@ -23,6 +23,9 @@ This skill is intentionally agent-agnostic: it must work in restricted company e - Triage, discovery, planning, build, review, and document phases run at the smallest useful depth. - Medium+ work has an approved plan before implementation; small work has an inline plan and proceeds without ceremony unless risk requires approval. +- Medium+ harness-capable work has an accepted Done Contract and Harness Recipe before Build. +- Trace/replay-ready harness work maintains Harness Run State, Trace Ledger, and Replay Packet artifacts. +- Medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work records independent QA Evaluator evidence after build/test and code-review evidence. - Behavior changes have tests or explicit validation attached to the implementation step they protect. - Final output reports changed files, verification evidence, residual risks, and next steps. - Candidate Search is used for explicit alternatives, open-ended architecture/design, optimization, high uncertainty, repeated failed attempts, unclear/flaky bugs, or reviewer-requested pivots — not as default ceremony. @@ -202,12 +205,13 @@ Load `references/phases.md` and execute the phase matching your current stage. U | **Plan** | All sizes | Implementation steps with file paths. Load `references/plan-template.md`. Small tasks use inline no-wait plans unless risk/ambiguity requires approval; medium+ tasks use the single approval gate for scope, slices, and build plan. | | **Design** | UI tasks only | Design direction, mockup, production checklist. Approval gate. | | **Build** | All sizes | One step at a time. Code Writer -> Builder/Tester. Tests alongside code. | -| **Review** | All sizes | Stage 1: Spec Review. Stage 2: load and follow `assistant-review` SKILL.md and contracts. | +| **Review** | All sizes | Stage 1: Spec Review. Stage 2: load and follow `assistant-review` SKILL.md and contracts for code review. Stage 3: QA Evaluator runs when acceptance QA is required. | | **Document** | All sizes | Small: metrics only. Medium+: docs + metrics + reflection. | -For subagent roles and dispatch rules, load `references/subagent-dispatch.md` and resolve `subagent_policy_state`, `subagent_execution_mode`, and `subagent_authorization_scope` before any subagent spawn. Assistant Framework policy requires explicit user authorization before spawning subagents for development/code-work roles unless the current user prompt already explicitly authorizes them for this task. Ask once for the needed delegation scope and wait before continuing phases that require subagents. A sufficient prompt is: `This workflow expects Code Mapper, Architect, Code Writer, Builder/Tester, and Reviewer subagents for [scope]. May I use subagents for this task?` After authorization, use `delegated` mode and spawn the configured role agents. Use `direct_fallback` only when authorization is denied, policy disallows spawning, or a real spawn attempt fails because subagents/custom agents are unavailable; do not infer unavailability merely because no visible tool is named `Task`, `delegate`, or `subagent`. +For subagent roles and dispatch rules, load `references/subagent-dispatch.md` and resolve `subagent_policy_state`, `subagent_execution_mode`, and `subagent_authorization_scope` before any subagent spawn. Assistant Framework policy requires explicit user authorization before spawning subagents for development/code-work roles unless the current user prompt already explicitly authorizes them for this task. Ask once for the needed delegation scope and wait before continuing phases that require subagents. A sufficient prompt is: `This workflow expects Code Mapper, Architect, Code Writer, Builder/Tester, Code Reviewer, and QA Evaluator when QA is required for [scope]. May I use subagents for this task?` After authorization, use `delegated` mode and spawn the configured role agents. Use `direct_fallback` only when authorization is denied, policy disallows spawning, or a real spawn attempt fails because subagents/custom agents are unavailable; do not infer unavailability merely because no visible tool is named `Task`, `delegate`, or `subagent`. For BES-style option exploration, load `references/candidate-search.md` only when `search_mode: candidate_search` is selected. For mega tasks and anti-patterns, load `references/mega-and-patterns.md`. +Load `references/harness-controller.md` only when medium+ work is harness-capable: long-running, multi-slice, delegated, high-risk, subjective/domain-scored, trace/replay-ready, or explicitly requested as harness work. ## Planning Checklist @@ -239,7 +243,8 @@ Before final output for medium+ or high-risk work: - Run the relevant build/test/lint/typecheck commands available in the repo. - Review the diff against the acceptance criteria. - For bugfixes, verify the review material includes reproduction/root-cause evidence from `assistant-debugging` or a clear reason it was not applicable. -- Use `assistant-review` for quality/spec review when the change is non-trivial. +- Use `assistant-review` for code quality/spec review when the change is non-trivial. +- Use QA Evaluator through `assistant-review` for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested acceptance QA after build/test and code-review evidence. - Use `assistant-security` when touching auth, user input, secrets, persistence, network calls, shell commands, dependency/config changes, or external integrations. Review findings must cite evidence and concrete risk. Avoid generic style feedback unless it affects correctness, security, maintainability, or test reliability. diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/handoffs.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/handoffs.yaml index 79a2e05..5d9bb4d 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/handoffs.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/handoffs.yaml @@ -16,6 +16,7 @@ skill: assistant-workflow # ───────────────────────────────────────────── worker_status_protocol: description: "CodeMapper, Explorer, Architect, CodeWriter, BuilderTester, and Reviewer returns include a compact status packet so the orchestrator can route results deterministically." + qa_evaluator_description: "QAEvaluator returns include a compact status packet for acceptance, Done Contract, verification evidence, score progression, and final QA result; it does not replace Reviewer/code-reviewer." statuses: - value: DONE meaning: "No known concerns; requested work completed within scope." @@ -34,6 +35,7 @@ worker_status_protocol: - "evidence is required for CodeMapper, Explorer, Architect, CodeWriter, and Reviewer returns with DONE, DONE_WITH_CONCERNS, DEVIATED, or FAILED_VERIFICATION." - "verification.evidence is required for BuilderTester returns." - "open_questions is required for NEEDS_CONTEXT and BLOCKED." + - "blocker_type and blocker_evidence are required for CodeWriter returns with NEEDS_CONTEXT, BLOCKED, or DEVIATED when implementation hits an unexpected blocker." - "deviation_details is required for Architect and CodeWriter returns with DEVIATED." - "CodeMapper, Explorer, and Reviewer status values are limited to DONE, DONE_WITH_CONCERNS, NEEDS_CONTEXT, and BLOCKED." - "Architect and CodeWriter status values also include DEVIATED." @@ -41,6 +43,24 @@ worker_status_protocol: - "changed_files and files_changed are required with at least one item for CodeWriter returns with DONE, DONE_WITH_CONCERNS, or DEVIATED; they may be omitted or empty for NEEDS_CONTEXT/BLOCKED returns before file changes occur." - "verification is required for BuilderTester returns; if status is NEEDS_CONTEXT or BLOCKED before verification runs, return result not_run with concise blocker evidence." - "findings is required for Reviewer returns." + - "acceptance_findings, qa_scorecard, final_verdict, result, evidence, and score_entry are required for QAEvaluator returns; selected_domain_rubrics and domain_quality_scores are required only when scoped domain rubrics are selected." + +# ───────────────────────────────────────────── +# Typed Artifact Reference Protocol +# ───────────────────────────────────────────── +artifact_reference_protocol: + description: "Typed Artifact Reference entries are the compact producer/consumer ledger for harness task packets and worker returns; use them instead of ad hoc string refs when passing artifacts between agents." + required_fields: [artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, summary] + artifact_types: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + validation_statuses: [pending, valid, invalid, stale, not_applicable] + packet_rules: + - "Artifact Reference entries must include artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, and summary." + - "location_ref is the typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable external artifact ref." + - "Producer responsibility: create or update the artifact, assign artifact_id, set artifact_type, location_ref, schema_or_contract, validation_status, and summary." + - "Consumer responsibility: validate schema_or_contract and validation_status before relying on location_ref; stale, invalid, or missing refs block phase advancement or trigger re-dispatch." + - "CodeWriter and BuilderTester task packets receive artifact_refs for Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, Pivot/Restart Decision, changed files, verification evidence, and plan deviation refs when applicable." + - "CodeWriter returns produced artifact_refs for changed files and plan deviation refs when applicable." + - "BuilderTester returns validated artifact_refs for verification evidence and runtime-artifact validation." # ───────────────────────────────────────────── # Orchestrator → CodeMapper @@ -669,6 +689,50 @@ handoffs: type: string[] required: true description: "Observable behavior this task packet must deliver" + - name: done_contract_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the accepted Done Contract containing done_when, not_done_when, verification, owner_consumer, acceptance_criteria, and debate_record" + - name: harness_recipe_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the selected Harness Recipe based on task/model/risk/context profile" + - name: artifact_refs + type: object[] + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + min_items: 5 + description: "Typed Artifact Reference entries that the task packet passes to CodeWriter and BuilderTester; include Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, changed files, verification evidence, and plan deviation refs when applicable." + object_fields: + - name: artifact_id + type: string + required: true + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + - name: producer + type: string + required: true + - name: consumer + type: string + required: true + - name: location_ref + type: string + required: true + description: "Typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable artifact ref" + - name: schema_or_contract + type: string + required: true + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + - name: summary + type: string + required: true - name: test_criteria type: string[] required: true @@ -805,6 +869,50 @@ handoffs: - name: acceptance_criteria type: string[] required: true + - name: done_contract_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the accepted Done Contract containing done_when, not_done_when, verification, owner_consumer, acceptance_criteria, and debate_record" + - name: harness_recipe_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the selected Harness Recipe based on task/model/risk/context profile" + - name: artifact_refs + type: object[] + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + min_items: 5 + description: "Typed Artifact Reference entries CodeWriter receives before implementation; include Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, changed files, verification evidence, and plan deviation refs when applicable." + object_fields: + - name: artifact_id + type: string + required: true + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + - name: producer + type: string + required: true + - name: consumer + type: string + required: true + - name: location_ref + type: string + required: true + description: "Typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable artifact ref" + - name: schema_or_contract + type: string + required: true + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + - name: summary + type: string + required: true - name: files_to_create type: string[] required: true @@ -930,6 +1038,34 @@ handoffs: required: false condition: "status in [NEEDS_CONTEXT, BLOCKED]" description: "Questions or blocker details the orchestrator must resolve before implementation can continue" + - name: blocker_type + type: enum + required: conditional + condition: "status in [NEEDS_CONTEXT, BLOCKED, DEVIATED] because implementation hit an unexpected blocker, missing contract, stale plan, scope conflict, or required RED evidence gap" + enum_values: [legacy_code_bug, broken_baseline, hidden_dependency, missing_contract, stale_plan, scope_conflict, tool_environment, permission_policy, tdd_red_missing, other] + description: "Classifies the implementation blocker so the orchestrator can route recovery instead of asking CodeWriter to improvise through it." + validation: "Use the most specific blocker type. DEVIATED is required when continuing would change approved scope, files, behavior, risk, or verification expectations." + on_missing: fail + - name: blocker_evidence + type: object[] + required: conditional + condition: "blocker_type is present" + min_items: 1 + description: "Evidence for the blocker: file paths, baseline failures, missing contract fields, stale task-packet assumptions, tool/environment errors, or scope conflict details." + on_missing: fail + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: recovery_hint + type: enum + required: false + condition: "blocker_type is present" + enum_values: [debugging, explorer, architect, candidate_search, replan, restart, user_clarification, environment_fix, permission_request] + description: "Optional non-binding recovery hint; the orchestrator owns the final pivot_restart_decision." - name: files_changed type: object[] required: false @@ -947,6 +1083,40 @@ handoffs: - name: description type: string required: true + - name: artifact_refs + type: object[] + required: false + condition: "required with min_items: 1 when status in [DONE, DONE_WITH_CONCERNS, DEVIATED] and current_task_packet.artifact_refs is present or harness_capable == true; optional and may be empty or omitted when status in [NEEDS_CONTEXT, BLOCKED]" + min_items: 0 + description: "Typed Artifact Reference entries produced or updated by CodeWriter; include changed_files refs for modified files and plan_deviation refs when implementation departs from the approved task packet." + object_fields: + - name: artifact_id + type: string + required: true + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + - name: producer + type: string + required: true + - name: consumer + type: string + required: true + - name: location_ref + type: string + required: true + description: "Typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable artifact ref" + - name: schema_or_contract + type: string + required: true + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + - name: summary + type: string + required: true - name: tests_written type: object[] required: true @@ -968,7 +1138,7 @@ handoffs: required: false description: "What the plan assumed vs. what was found" condition: "when deviation_detected is true or status is DEVIATED" - on_missing_field: "CodeWriter must always return status and deviation_detected at minimum. For DONE, DONE_WITH_CONCERNS, or DEVIATED, it must also return changed_files, evidence, and files_changed with real implementation evidence; for NEEDS_CONTEXT/BLOCKED, require open_questions and do not require fabricated changed-file entries." + on_missing_field: "CodeWriter must always return status and deviation_detected at minimum. For DONE, DONE_WITH_CONCERNS, or DEVIATED, it must also return changed_files, evidence, files_changed, and typed artifact_refs when the task packet carried artifact_refs or harness_capable == true; for NEEDS_CONTEXT/BLOCKED, require open_questions and do not require fabricated changed-file or artifact-ref entries. When implementation hit an unexpected blocker, require blocker_type and blocker_evidence before the orchestrator routes recovery." # ───────────────────────────────────────────── # Orchestrator → BuilderTester @@ -1021,6 +1191,50 @@ handoffs: - name: acceptance_criteria type: string[] required: true + - name: done_contract_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the accepted Done Contract containing done_when, not_done_when, verification, owner_consumer, acceptance_criteria, and debate_record" + - name: harness_recipe_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the selected Harness Recipe based on task/model/risk/context profile" + - name: artifact_refs + type: object[] + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + min_items: 5 + description: "Typed Artifact Reference entries BuilderTester receives before verification; include Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, changed files, verification evidence, and plan deviation refs when applicable." + object_fields: + - name: artifact_id + type: string + required: true + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + - name: producer + type: string + required: true + - name: consumer + type: string + required: true + - name: location_ref + type: string + required: true + description: "Typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable artifact ref" + - name: schema_or_contract + type: string + required: true + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + - name: summary + type: string + required: true - name: files_to_test type: string[] required: true @@ -1060,6 +1274,40 @@ handoffs: type: string[] required: true description: "Concise success signals or failure messages, not raw logs" + - name: artifact_refs + type: object[] + required: false + condition: "required with min_items: 1 when current_task_packet.artifact_refs is present or harness_capable == true; optional and may be empty or omitted when verification.result is not_run for NEEDS_CONTEXT or BLOCKED" + min_items: 0 + description: "Typed Artifact Reference entries validated or produced by BuilderTester; include verification_evidence refs and validation_status updates for Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, changed files, and plan deviation refs." + object_fields: + - name: artifact_id + type: string + required: true + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + - name: producer + type: string + required: true + - name: consumer + type: string + required: true + - name: location_ref + type: string + required: true + description: "Typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable artifact ref" + - name: schema_or_contract + type: string + required: true + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + - name: summary + type: string + required: true - name: open_questions type: string[] required: false @@ -1100,7 +1348,7 @@ handoffs: type: string required: true description: "Concise error — assertion message + expected/actual, not stack trace" - on_missing_field: "Re-dispatch BuilderTester. status, verification, build_result, and test_summary are never optional." + on_missing_field: "Re-dispatch BuilderTester. status, verification, build_result, and test_summary are never optional; typed artifact_refs are required when the task packet carried artifact_refs or harness_capable == true." # ───────────────────────────────────────────── # Orchestrator → Reviewer @@ -1132,8 +1380,8 @@ handoffs: - name: round type: int required: true - description: "Current review round number (1-10)" - validation: ">= 1 and <= 10" + description: "Current review round number (1-20)" + validation: ">= 1 and <= 20" - name: previously_fixed type: object[] required: true @@ -1145,13 +1393,13 @@ handoffs: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: confidence_threshold type: int required: true description: "Minimum confidence percentage for reporting findings" validation: "Must be one of: 80, 85, 90" - notes: "Round 1-3: 80, Round 4-7: 85, Round 8-10: 90" + notes: "Round 1-3: 80, Round 4-15: 85, Round 16-20: 90" return_fields: - name: status type: enum @@ -1161,7 +1409,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: findings type: object[] required: true @@ -1215,6 +1463,257 @@ handoffs: description: "Questions or blocker details the orchestrator must resolve" on_missing_field: "Re-dispatch Reviewer. status, findings, evidence, and verdict are never optional." + # ───────────────────────────────────────────── + # Orchestrator → QAEvaluator + # ───────────────────────────────────────────── + - name: orchestrator_to_qa_evaluator + from: Orchestrator + to: QAEvaluator + phase: REVIEW + notes: "Independent QA acceptance evaluation after BuilderTester verification and Code Reviewer or Reviewer compatibility result. Does not replace code-reviewer." + context_fields: + - name: done_contract + type: object + required: conditional + condition: "task has an accepted Done Contract" + description: "Accepted Done Contract for the work under QA evaluation" + validation: "Includes done_when, not_done_when, verification, owner_consumer, acceptance_criteria, debate_record with at least two perspectives, and accepted_by when present" + object_fields: + - name: done_when + type: string[] + required: true + - name: not_done_when + type: string[] + required: true + - name: verification + type: string[] + required: true + - name: owner_consumer + type: string + required: true + - name: acceptance_criteria + type: string[] + required: true + - name: debate_record + type: object[] + required: true + min_items: 2 + description: "pre-build debate/subagent-perspective evidence considered before accepting the Done Contract" + validation: "At least two perspectives; delegated mode uses subagent perspectives when available" + object_fields: + - name: perspective + type: string + required: true + - name: concern_or_support + type: string + required: true + - name: resolution + type: string + required: true + - name: accepted_by + type: string + required: true + - name: acceptance_criteria + type: string[] + required: true + description: "Binary acceptance criteria from user request, approved plan, slice manifest, or Done Contract" + min_items: 1 + - name: verification_evidence + type: object[] + required: true + description: "BuilderTester, manual, inspection, or review evidence that should prove acceptance" + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: result + type: string + required: true + - name: proves + type: string + required: true + - name: code_review_result + type: object + required: true + description: "Final Code Reviewer or Reviewer compatibility result that QA must not replace" + object_fields: + - name: role + type: enum + required: true + enum_values: [CodeReviewer, Reviewer] + - name: result + type: string + required: true + - name: rounds + type: int + required: true + validation: ">= 1 and <= 20" + - name: domain_context + type: object + required: false + description: "Scoped product, UX, UI/visual, docs, DX, or domain context; enables conditional use of assistant-review references/domain-rubrics.md" + object_fields: + - name: summary + type: string + required: true + - name: in_scope_surfaces + type: string[] + required: true + - name: rubric_refs + type: string[] + required: false + default: [] + description: "Explicit product/UX/docs/DX/UI/domain rubric family references from assistant-review references/domain-rubrics.md" + - name: round + type: int + required: true + description: "Current QA evaluation round number (1-20)" + validation: ">= 1 and <= 20" + - name: previously_failed_acceptance_items + type: object[] + required: true + description: "Failed acceptance items from prior QA rounds" + object_fields: + - name: criterion + type: string + required: true + - name: failed_in_round + type: int + required: true + validation: ">= 1 and <= 20" + - name: evidence_gap + type: string + required: true + - name: qa_filter_policy + type: string + required: true + description: "Acceptance findings require evidence and direct acceptance impact; speculative concerns are non-blocking; round 20 is terminal" + return_fields: + - name: status + type: enum + required: true + enum_values: [DONE, DONE_WITH_CONCERNS, NEEDS_CONTEXT, BLOCKED] + description: "Worker status packet result; DONE requires final_verdict=accepted" + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: acceptance_findings + type: object[] + required: true + description: "Acceptance, Done Contract, verification, or scoped domain findings" + object_fields: + - name: severity + type: enum + required: true + enum_values: [blocker, concern, observation] + - name: criterion + type: string + required: true + - name: evidence + type: string + required: true + - name: impact + type: string + required: true + - name: qa_scorecard + type: object + required: true + description: "Compact QA scorecard; domain_quality is backed by selected_domain_rubrics/domain_quality_scores only when scoped domain rubrics were used" + object_fields: + - name: acceptance_coverage + type: float + required: true + - name: evidence_strength + type: float + required: true + - name: domain_quality + type: float + required: true + - name: final_readiness + type: float + required: true + - name: weighted_score + type: float + required: true + - name: selected_domain_rubrics + type: string[] + required: false + condition: "domain_context or rubric_refs is present, or acceptance criteria / Done Contract require subjective/product/UX/docs/DX/UI/domain craft evaluation" + description: "Rubric families selected from assistant-review references/domain-rubrics.md; empty or omitted when no scoped domain rubric applies" + - name: domain_quality_scores + type: object[] + required: false + condition: "selected_domain_rubrics is non-empty" + description: "Per-family domain rubric scores with evidence" + object_fields: + - name: rubric_ref + type: string + required: true + - name: dimension + type: string + required: true + - name: score + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments. For action=not_applicable, use 5.0 and state the unscoped rationale in evidence." + - name: action + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, refine, rejected, pivot, not_applicable] + - name: evidence + type: string + required: true + - name: final_verdict + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, rejected, blocked] + - name: result + type: enum + required: true + enum_values: [CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, BLOCKED] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: score_entry + type: object + required: true + description: "QA score progression entry for this round" + object_fields: + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: weighted_score + type: float + required: true + - name: failed_acceptance_count + type: int + required: true + - name: drift_status + type: enum + required: true + enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL, NOT_APPLICABLE] + - name: score_progression + type: object[] + required: false + description: "Full QA score progression when prior round history is available" + - name: open_questions + type: string[] + required: false + condition: "status in [NEEDS_CONTEXT, BLOCKED] or final_verdict == blocked" + description: "Questions or blocker details the orchestrator must resolve" + on_missing_field: "Re-dispatch QAEvaluator. status, acceptance_findings, qa_scorecard, final_verdict, result, evidence, and score_entry are never optional. When scoped domain rubrics are selected, selected_domain_rubrics and domain_quality_scores are also required; when not scoped, do not fabricate them." + # ───────────────────────────────────────────── # Validation behavior for all handoffs: # @@ -1222,5 +1721,6 @@ handoffs: # 2. After return: verify all required return_fields are present, including the worker status packet # 3. If on_missing_field action is "re-dispatch": retry once with explicit request # 4. If second dispatch also misses required fields: log the gap and route to direct fallback, BLOCKED, or a degraded result with explicit evidence; do not fabricate required fields -# 5. Never silently proceed with missing required fields — always log what was missing and how the missing evidence was resolved, deferred, or blocked +# 5. When CodeWriter returns blocker_type, the orchestrator records the blocker evidence and routes to debugging, explorer, architect, candidate_search, replan, restart, user clarification, environment fix, or permission request. Do not ask CodeWriter to widen scope or patch blindly through a legacy blocker. +# 6. Never silently proceed with missing required fields — always log what was missing and how the missing evidence was resolved, deferred, or blocked # ───────────────────────────────────────────── diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml index f4f2439..f107b5e 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml @@ -47,7 +47,7 @@ fields: type: string[] required: false description: "Agent or skill roles required by size, task type, and risk tier" - validation: "Entries must be known workflow roles or installed skills, or state a clear project-specific role. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts must include at minimum Code Writer, Builder/Tester, and Reviewer responsibilities." + validation: "Entries must be known workflow roles or installed skills, or state a clear project-specific role. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts must include at minimum Code Writer, Builder/Tester, and Code Reviewer responsibilities. Reviewer may satisfy only compatibility routing for existing/legacy handoffs when explicitly recorded." default: [] on_missing: skip @@ -135,6 +135,20 @@ fields: on_missing: skip notes: "Populated by Idea-to-Action pipeline when input is a vague idea rather than a concrete task" + - name: qa_evaluation_mode + type: enum + required: false + enum_values: [not_required, optional, required] + description: "Whether Review phase should run the independent QA Evaluator acceptance loop after build/test and code-review evidence" + validation: "required for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work; not_required must be justified when a medium+ harness task skips QA" + default: not_required + on_missing: infer + infer_from: > + Use required when the task is medium+ harness-capable, has a Done Contract, is domain-scored, + affects UI/visual/product/UX/docs/DX quality, or explicitly requests QA/acceptance evaluation. + Use optional when acceptance material exists but QA is not required. + Use not_required for trivial/small code-review-only work with no separate acceptance evidence need. + - name: search_mode type: enum required: false diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml index 11431d4..f913222 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml @@ -18,13 +18,13 @@ completion_tiers: checkpoint_policy: "inline plan/check summary is enough; phase_checkpoints are strict-profile only" medium: required_artifacts: [triage_result, changed_files, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, plan_document, user_approval] - conditional_artifacts: [test_results, task_journal, context_map, slice_manifest, single_slice_rationale, slice_verification_summary, metrics_entry, candidate_search_result] - review_gate: "spec review plus quality review, with evidence matched to the approved plan" + conditional_artifacts: [test_results, task_journal, context_map, slice_manifest, single_slice_rationale, slice_verification_summary, metrics_entry, candidate_search_result, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result] + review_gate: "spec review plus code quality review, with independent QA evaluation when required and evidence matched to the approved plan" checkpoint_policy: "task journal or carried-forward state records phase transitions" large_critical: required_artifacts: [triage_result, phase_checkpoints, changed_files, test_results, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, task_journal, context_map, context_budget_note, slice_manifest, slice_verification_summary, plan_document, user_approval] - conditional_artifacts: [debugging_result, artifact_contract, pattern_retrieval_note, decomposition_plan_review, single_slice_rationale, metrics_entry, candidate_search_result, verified_skill_distillation] - review_gate: "full strict evidence, security/operability review when applicable, and all slice criteria verified" + conditional_artifacts: [debugging_result, artifact_contract, pattern_retrieval_note, decomposition_plan_review, single_slice_rationale, metrics_entry, candidate_search_result, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result, verified_skill_distillation] + review_gate: "full strict evidence, security/operability review when applicable, QA evaluation when required, and all slice criteria verified" checkpoint_policy: "explicit phase checkpoints are required" artifacts: @@ -216,8 +216,8 @@ artifacts: type: object required: conditional condition: "size in [medium, large, mega] or risk_tier in [high, critical] or hook_profile == strict" - validation: "Stage 2 assistant-review quality result only. Spec compliance is recorded in spec_review_result." - on_fail: "Run assistant-review quality review and record quality status, rounds, and resolved findings" + validation: "Stage 2 assistant-review quality result only. Spec compliance is recorded in spec_review_result. This is the code quality review result; QA acceptance is recorded separately in qa_evaluation_result when required." + on_fail: "Run assistant-review code quality review and record quality status, rounds, and resolved findings" object_fields: - name: quality_review_status type: enum @@ -226,13 +226,140 @@ artifacts: - name: review_rounds type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: must_fix_resolved type: int required: true - name: should_fix_resolved type: int required: true + - name: pivot_restart_decision_ref + type: string + required: false + condition: "quality review loop observed STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT" + description: "Reference to the orchestrator-owned pivot_restart_decision artifact created before another review/fix dispatch." + + - name: qa_evaluation_result + type: object + required: conditional + condition: "qa_evaluation_mode == required or size in [medium, large, mega] and harness_capable == true and work is domain-scored/UI/visual/product/UX/docs/DX-facing" + validation: "Independent QA evaluation result exists after build/test evidence and Code Reviewer result. Must include final_verdict, result, rounds (1-20), acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, and evidence." + on_fail: "Run the QAEvaluator loop from assistant-review references/qa-evaluation-loop.md, or record NEEDS_CONTEXT/BLOCKED with open_questions. Do not substitute Code Reviewer findings for QA evaluation." + object_fields: + - name: final_verdict + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, rejected, blocked] + - name: result + type: enum + required: true + enum_values: [CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, BLOCKED] + - name: rounds + type: int + required: true + validation: ">= 1 and <= 20" + - name: acceptance_findings + type: object[] + required: true + object_fields: + - name: severity + type: enum + required: true + enum_values: [blocker, concern, observation] + - name: criterion + type: string + required: true + - name: evidence + type: string + required: true + - name: disposition + type: enum + required: true + enum_values: [resolved, remaining, not_applicable] + - name: qa_scorecard + type: object + required: true + object_fields: + - name: acceptance_coverage + type: float + required: true + - name: evidence_strength + type: float + required: true + - name: domain_quality + type: float + required: true + - name: final_readiness + type: float + required: true + - name: weighted_score + type: float + required: true + - name: selected_domain_rubrics + type: string[] + required: false + condition: "domain_context or rubric_refs was provided, or acceptance criteria / Done Contract required subjective/product/UX/docs/DX/UI/domain craft evaluation" + description: "Rubric families selected from assistant-review references/domain-rubrics.md; empty or omitted when domain rubrics were not scoped" + - name: domain_quality_scores + type: object[] + required: false + condition: "selected_domain_rubrics is non-empty" + description: "Evidence-backed domain rubric scores from the final QA round" + object_fields: + - name: rubric_ref + type: string + required: true + - name: dimension + type: string + required: true + - name: score + type: float + required: true + - name: action + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, refine, rejected, pivot, not_applicable] + - name: evidence + type: string + required: true + - name: score_progression + type: object[] + required: true + object_fields: + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: weighted_score + type: float + required: true + - name: failed_acceptance_count + type: int + required: true + - name: drift_status + type: enum + required: true + enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL, NOT_APPLICABLE] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: open_questions + type: string[] + required: false + condition: "final_verdict == blocked or result == BLOCKED" + - name: pivot_restart_decision_ref + type: string + required: false + condition: "QA loop observed STAGNATION, repeated DRIFT, repeated REGRESSION, or domain/rubric action pivot" + description: "Reference to the orchestrator-owned pivot_restart_decision artifact created before another QA/build dispatch." - name: spec_review_result type: object @@ -355,8 +482,8 @@ artifacts: required: conditional condition: "development/code-work task changes project source, tests, docs, config, hooks, contracts, or generated project artifacts" description: "Strict evidence that required subagent roles were dispatched or explicitly handled through direct fallback." - validation: "required_agents includes at minimum Code Writer, Builder/Tester, and Reviewer. delegated mode records dispatch and result evidence for each required role; medium+ delegated work records per-slice dispatch evidence when slices apply. direct_fallback records one explicit policy reason (authorization_denied, subagents_unavailable, or policy_disallowed) plus equivalent role, phase, verification, and review evidence. not_applicable is invalid for source-changing Build tasks." - on_fail: "Update the task journal Agent Dispatch Log. If delegated, add Code Writer, Builder/Tester, and Reviewer dispatch/result evidence and medium+ per-slice dispatch evidence. If direct_fallback, add the explicit policy reason and role-equivalent evidence. Do not complete with silent fallback." + validation: "required_agents includes at minimum Code Writer, Builder/Tester, and Code Reviewer. Reviewer compatibility may satisfy existing/legacy handoffs only when explicitly recorded. delegated mode records dispatch and result evidence for each required role; medium+ delegated work records per-slice dispatch evidence when slices apply. direct_fallback records one explicit policy reason (authorization_denied, subagents_unavailable, or policy_disallowed) plus equivalent role, phase, verification, and Code Reviewer review evidence; Reviewer direct evidence is compatibility evidence only. not_applicable is invalid for source-changing Build tasks. QA Evaluator evidence is also required when qa_evaluation_mode=required." + on_fail: "Update the task journal Agent Dispatch Log. If delegated, add Code Writer, Builder/Tester, and Code Reviewer dispatch/result evidence, or Reviewer compatibility dispatch/result evidence for existing/legacy handoffs, plus medium+ per-slice dispatch evidence. If direct_fallback, add the explicit policy reason and role-equivalent Code Reviewer direct evidence, or Reviewer direct evidence only as compatibility evidence. Do not complete with silent fallback. When QA is required, add QAEvaluator dispatch/result or direct evidence separately from Code Reviewer/Reviewer compatibility." object_fields: - name: execution_mode type: enum @@ -365,7 +492,7 @@ artifacts: - name: required_roles type: string[] required: true - validation: "Includes Code Writer, Builder/Tester, and Reviewer for source-changing work." + validation: "Includes Code Writer, Builder/Tester, and Code Reviewer for source-changing work. Reviewer compatibility may satisfy existing/legacy handoffs only when explicitly recorded." - name: delegated_dispatch_results type: object[] required: conditional @@ -380,11 +507,29 @@ artifacts: type: object[] required: conditional condition: "execution_mode == direct_fallback" - validation: "Contains equivalent Code Writer, Builder/Tester, and Reviewer role/phase/verification/review evidence." + validation: "Contains equivalent Code Writer, Builder/Tester, and Code Reviewer role/phase/verification/review evidence. Reviewer direct evidence is compatibility evidence only." - name: per_slice_dispatch_evidence type: object[] required: conditional condition: "size in [medium, large, mega] and execution_mode == delegated and slices apply" + - name: qa_evaluator_evidence + type: object + required: conditional + condition: "qa_evaluation_mode == required" + validation: "Contains QAEvaluator dispatch/result evidence in delegated mode, or QAEvaluator direct evidence with fallback reason in direct_fallback mode" + object_fields: + - name: qa_evaluator_dispatch + type: string + required: false + description: "QAEvaluator subagent/tool/run id when delegated" + - name: qa_evaluator_result + type: string + required: true + description: "Final QA verdict/result evidence" + - name: qa_evaluator_direct_evidence + type: string + required: false + description: "Fresh QA evidence when direct_fallback" - name: task_journal type: file @@ -506,6 +651,401 @@ artifacts: type: string[] required: true + - name: artifact_reference_ledger + type: object[] + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "Typed producer/consumer ledger for artifacts passed between subagents during harness work." + validation: "Each entry includes artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, and summary; ledger covers Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, Pivot/Restart Decision, changed files, verification evidence, and plan deviation refs when applicable." + on_fail: "Before Build, Review, completion, or handoff, create or repair the Artifact Reference Ledger in the plan/task journal; re-dispatch the producer when a required typed artifact reference is missing, stale, invalid, or lacks a consumer validation status." + min_items: 1 + object_fields: + - name: artifact_id + type: string + required: true + validation: "Stable id unique within the task run" + on_missing: fail + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + validation: "Classifies the artifact without requiring prose parsing" + on_missing: fail + - name: producer + type: string + required: true + validation: "Role, subagent, hook, or task packet that created or last updated the artifact" + on_missing: fail + - name: consumer + type: string + required: true + validation: "Role, subagent, hook, or phase expected to consume and validate the artifact" + on_missing: fail + - name: location_ref + type: string + required: true + validation: "Typed location/ref pointer: file path, task-journal section, dispatch id, command ref, or stable artifact ref" + on_missing: fail + - name: schema_or_contract + type: string + required: true + validation: "Contract, schema, template, or required fields the consumer validates before use" + on_missing: fail + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + validation: "Consumer-visible validation state; invalid or stale entries block phase advancement" + on_missing: fail + - name: summary + type: string + required: true + validation: "Concise description of the artifact and its current state" + on_missing: fail + + - name: done_contract + type: object + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "Accepted pre-Build Done Contract for harness-capable work" + validation: "Must include done_when, not_done_when, verification, owner_consumer, acceptance_criteria, debate_record with at least two perspectives, and accepted_by before Build starts" + on_fail: "Block Build, load references/harness-controller.md, return to Plan, write the missing Done Contract, collect at least two debate perspectives using subagents when delegated mode is available, and record approval or scope-change re-approval" + object_fields: + - name: done_when + type: string[] + required: true + validation: "Pass/fail outcomes that prove the task or slice is complete" + - name: not_done_when + type: string[] + required: true + validation: "Failure states that block completion" + - name: verification + type: string[] + required: true + validation: "Commands, inspections, reviews, or manual checks that prove done" + - name: owner_consumer + type: string + required: true + validation: "Names the owner and downstream consumer of the artifact or behavior" + - name: acceptance_criteria + type: string[] + required: true + validation: "Explicit binary criteria from the user request, approved plan, or slice manifest" + - name: debate_record + type: object[] + required: true + min_items: 2 + validation: "At least two perspectives; delegated mode uses subagent perspectives when available" + object_fields: + - name: perspective + type: string + required: true + - name: concern_or_support + type: string + required: true + - name: resolution + type: string + required: true + - name: accepted_by + type: string + required: true + validation: "User, orchestrator, or approved plan reference that accepted the Done Contract" + + - name: harness_recipe + type: object + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "Pre-Build controller recipe selected from task/model/risk/context profile" + validation: "Must classify task_profile, model_profile, risk_profile, and context_profile, then name selected_recipe, recipe_rationale, required_artifacts, and corrective_action" + on_fail: "Block Build, load references/harness-controller.md, return to Plan, classify the task/model/risk/context profile, select a Harness Recipe, and record the corrective action for missing or stale recipe evidence" + object_fields: + - name: task_profile + type: string + required: true + validation: "Captures task type, size, slice count, and TDD/debugging applicability" + - name: model_profile + type: string + required: true + validation: "Captures agent/model constraints, delegation mode, and tool limits" + - name: risk_profile + type: string + required: true + validation: "Captures risk tier, safety gates, review depth, and rollback needs" + - name: context_profile + type: string + required: true + validation: "Captures exact, summarized, omitted/deferred context and trace/replay or handoff needs" + - name: selected_recipe + type: string + required: true + validation: "Concise recipe label from references/harness-controller.md" + - name: recipe_rationale + type: string + required: true + validation: "Explains why task/model/risk/context profile selects this recipe" + - name: required_artifacts + type: string[] + required: true + validation: "Artifacts this recipe requires before or during Build" + - name: corrective_action + type: string + required: true + validation: "Explicit action when recipe evidence is missing or no longer matches the task" + + - name: harness_run_state + type: object + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "First-class Harness Run State artifact for recovering active harness execution after compaction, failure, or handoff." + validation: "Must record task_id, task_name, phase, slice, status, blockers, last_verification, next_action, and recovery_pointer; update it as Build, Review, and Document progress. When a pivot/restart is active, it also records pivot_restart_decision_ref and the exact next action after restart." + on_fail: "Pause advancement, update the task journal or carried-forward state with a complete Harness Run State block, point recovery_pointer at the current task packet or ledger entry, then resume from next_action." + object_fields: + - name: task_id + type: string + required: true + validation: "Stable task or run identifier from the task journal, plan, or dispatch packet" + on_missing: fail + - name: task_name + type: string + required: true + validation: "Human-readable task name" + on_missing: fail + - name: phase + type: enum + required: true + enum_values: [TRIAGE, DISCOVER, DECOMPOSE, PLAN, DESIGN, BUILD, REVIEW, DOCUMENT, COMPLETE] + validation: "Current workflow phase" + on_missing: fail + - name: slice + type: string + required: true + validation: "Current slice id/name, next pending slice, or N/A for unsliced work" + on_missing: fail + - name: status + type: enum + required: true + enum_values: [not_started, in_progress, blocked, verifying, reviewing, restarting, documenting, complete] + validation: "Current run status" + on_missing: fail + - name: blockers + type: string[] + required: true + validation: "Current blockers, or an explicit none entry" + on_missing: fail + - name: last_verification + type: object + required: true + validation: "Most recent verification command/check, result, and evidence" + on_missing: fail + object_fields: + - name: command_or_check + type: string + required: true + on_missing: fail + - name: result + type: enum + required: true + enum_values: [passed, failed, skipped, not_applicable, pending] + on_missing: fail + - name: evidence + type: string + required: true + on_missing: fail + - name: next_action + type: string + required: true + validation: "Exact next action needed to continue safely" + on_missing: fail + - name: recovery_pointer + type: string + required: true + validation: "Pointer to the task packet, trace entry, replay packet, file section, or artifact needed to resume" + on_missing: fail + - name: pivot_restart_decision_ref + type: string + required: false + condition: "a pivot_restart_decision is active or was selected for the current recovery path" + validation: "Reference to the Pivot/Restart Decision artifact governing the next action" + + - name: trace_ledger + type: object[] + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "First-class Trace Ledger artifact for ordered harness execution evidence." + validation: "Records timestamped/ordered agent events, decisions, verification commands/results, plan deviations, and artifact refs as execution progresses." + on_fail: "Append missing trace entries before advancing: reconstruct from Agent Dispatch Log, Key Decisions, verification output, plan deviations, and Artifact Registry; mark any unrecoverable gap explicitly." + min_items: 1 + object_fields: + - name: sequence + type: int + required: true + validation: "Monotonic event order within this task" + on_missing: fail + - name: timestamp + type: string + required: true + validation: "Timestamp or ordered marker for the event" + on_missing: fail + - name: event_type + type: enum + required: true + enum_values: [agent_event, decision, verification, plan_deviation, pivot_restart, artifact_ref, blocker, recovery] + validation: "Classifies the trace entry" + on_missing: fail + - name: actor + type: string + required: true + validation: "Role, subagent, hook, user, or orchestrator responsible for the event" + on_missing: fail + - name: summary + type: string + required: true + validation: "Concise event, decision, command/result, deviation, blocker, or recovery summary" + on_missing: fail + - name: artifact_refs + type: string[] + required: true + validation: "Related file paths, task journal headings, dispatch ids, command refs, or an explicit none entry" + on_missing: fail + + - name: replay_packet + type: object + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "First-class Replay Packet artifact that lets the workflow resume after compaction, failure, or handoff." + validation: "Captures pinned_context, artifact_refs, validation_state, exact_next_action, run_state_ref, trace_ledger_ref, recovery_pointer, and pivot_restart_decision_ref when active with enough detail to recover without replaying stale reasoning." + on_fail: "Create or repair the Replay Packet before ending the turn or crossing a phase boundary; copy pinned context from the approved plan/task packet, cite artifact refs, summarize validation state, and name the exact_next_action." + object_fields: + - name: pinned_context + type: string[] + required: true + validation: "Stable requirements, constraints, approved plan/slice ids, and non-goals needed to resume" + on_missing: fail + - name: artifact_refs + type: string[] + required: true + validation: "Task journal, context map, plan, run state, trace ledger, validation evidence, or changed-file references" + on_missing: fail + - name: validation_state + type: object + required: true + validation: "Current validation status and outstanding verification work" + on_missing: fail + object_fields: + - name: completed_checks + type: string[] + required: true + on_missing: fail + - name: pending_checks + type: string[] + required: true + on_missing: fail + - name: last_result + type: string + required: true + on_missing: fail + - name: exact_next_action + type: string + required: true + validation: "Single concrete action to take next after replay" + on_missing: fail + - name: run_state_ref + type: string + required: true + validation: "Reference to the current Harness Run State artifact" + on_missing: fail + - name: trace_ledger_ref + type: string + required: true + validation: "Reference to the current Trace Ledger artifact" + on_missing: fail + - name: recovery_pointer + type: string + required: true + validation: "Where to resume in the plan, task packet, phase, or slice ledger" + on_missing: fail + - name: pivot_restart_decision_ref + type: string + required: false + condition: "a pivot/restart decision controls the exact_next_action" + validation: "Reference to the orchestrator-owned Pivot/Restart Decision artifact" + + - name: pivot_restart_decision + type: object + required: conditional + condition: "Review, QA, Build, or CodeWriter evidence triggers STAGNATION, repeated DRIFT, repeated REGRESSION, rubric/domain action pivot, CodeWriter blocker routing, verification blocker, plan deviation, or scope change" + description: "Orchestrator-owned decision packet for stagnant quality loops, CodeWriter unexpected blockers, and restart/pivot recovery." + validation: "Must classify the trigger, cite evidence, name the affected slice or round, compare options, select a recovery action, state whether reapproval is required, name the next agent, and point recovery to an exact next action." + on_fail: "Pause the review/QA/build loop, load references/harness-controller.md, create the Pivot/Restart Decision packet, append trace/replay/run-state updates, and obtain reapproval before continuing when scope, files, behavior, risk, or verification changes." + object_fields: + - name: trigger + type: enum + required: true + enum_values: [STAGNATION, repeated_DRIFT, repeated_REGRESSION, pivot, code_writer_blocker, verification_blocker, plan_deviation, scope_change] + validation: "Names the event that forced pivot/restart consideration" + on_missing: fail + - name: evidence + type: object[] + required: true + min_items: 1 + validation: "At least one review score, QA score, CodeWriter blocker, verification failure, trace entry, or task-packet mismatch supports the trigger" + on_missing: fail + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: affected_slice_or_round + type: string + required: true + validation: "Current slice id/name, review round, QA round, or phase affected by the trigger" + on_missing: fail + - name: options_considered + type: object[] + required: true + min_items: 2 + validation: "At least two recovery options are compared unless a single safe option is forced by policy or missing approval" + on_missing: fail + object_fields: + - name: option + type: string + required: true + - name: tradeoff + type: string + required: true + - name: disposition + type: enum + required: true + enum_values: [selected, rejected, deferred] + - name: selected_action + type: enum + required: true + enum_values: [reset_context, return_to_build, dispatch_debugging, dispatch_explorer, dispatch_architect, run_candidate_search, replan, restart_slice, restart_phase, block_for_user, accept_with_limitations] + validation: "Concrete recovery action selected by the orchestrator; it must not imply starting round 21 after a terminal round 20" + on_missing: fail + - name: reapproval_required + type: boolean + required: true + validation: "true when selected_action changes approved scope, files, behavior, risk, verification, or acceptance criteria; false only for local context reset or same-scope retry" + on_missing: fail + - name: next_agent + type: string + required: true + validation: "Next role or agent to dispatch, such as Builder/Tester, CodeWriter, Explorer, Architect, Reviewer, QAEvaluator, assistant-debugging, candidate-search, user, or none" + on_missing: fail + - name: recovery_pointer + type: string + required: true + validation: "Trace entry, task packet, run-state section, replay packet, plan section, or file path where recovery resumes" + on_missing: fail + - name: exact_next_action + type: string + required: true + validation: "Single concrete action to take after the pivot/restart decision; replay_packet.exact_next_action must match it when replay is updated" + on_missing: fail + - name: decomposition_plan_review type: object required: conditional @@ -774,9 +1314,11 @@ artifacts: # 2. test_results.failed == 0 when tests ran; validation_results contains at least one result=passed entry for completed implementation workflows. Skipped/not_applicable entries do not satisfy completion evidence; docs-only/no-op/discovery exceptions must record the reason. # 3. When size is medium/large/mega, risk_tier is high/critical, or hook_profile == strict: spec_review_result.status == PASS and required_fixes are empty or resolved # 4. When size is medium/large/mega, risk_tier is high/critical, or hook_profile == strict: review_result.quality_review_status is not missing -# 5. For medium+ tasks, slice_verification_summary has one VERIFIED entry per slice -# 6. candidate_search_result exists when search_mode == candidate_search, includes search_exit_summary, and any plan_deviation is approved before Build -# 7. user_approval is confirmed, confirmed_with_issues, or not_required_small for eligible small tasks +# 5. If qa_evaluation_mode=required: qa_evaluation_result is populated after build/test and code-review evidence; QA evaluation does not replace review_result. Scoped domain rubrics require selected_domain_rubrics and domain_quality_scores; unscoped rubric scoring is invalid. +# 6. For medium+ tasks, slice_verification_summary has one VERIFIED entry per slice +# 7. candidate_search_result exists when search_mode == candidate_search, includes search_exit_summary, and any plan_deviation is approved before Build +# 8. For medium+ harness-capable work, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision when triggered, and artifact_reference_ledger exist and are current before completion or handoff +# 9. user_approval is confirmed, confirmed_with_issues, or not_required_small for eligible small tasks # If any check fails, do NOT print --- WORKFLOW COMPLETE ---. State what is missing. - name: verified_skill_distillation type: object diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml index aa5f4fa..f89bf47 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml @@ -55,9 +55,13 @@ gates: on_fail: "Load references/triage-rubric.md and add the required gate packs for the task type before continuing" - id: T7 - check: "required_agents or fallback execution roles are populated from task size, task type, risk tier, subagent_policy_state, and subagent_execution_mode; source-changing development/code-work includes at minimum Code Writer, Builder/Tester, and Reviewer" + check: "required_agents or fallback execution roles are populated from task size, task type, risk tier, subagent_policy_state, and subagent_execution_mode; source-changing development/code-work includes at minimum Code Writer, Builder/Tester, and Code Reviewer, with Reviewer compatibility allowed only for existing/legacy handoffs" on_fail: "Select required workflow agents when subagent_execution_mode=delegated, or record direct fallback roles and responsibilities from references/subagent-dispatch.md and risk-specific gate packs. not_applicable is invalid for source-changing Build tasks." + - id: T_QA_EVALUATOR + check: "For medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work: required_agents includes QA Evaluator in addition to Code Writer, Builder/Tester, and Code Reviewer or Reviewer compatibility" + on_fail: "Add QA Evaluator as a Review-phase role, or record qa_evaluation_mode=not_required with a concrete reason when QA is not applicable." + - id: T9 check: "subagent_policy_state, subagent_execution_mode, and subagent_authorization_scope are initialized before any phase that may spawn subagents; development/code-work that needs workflow roles starts as authorization_required unless the current user prompt explicitly authorized subagents" on_fail: "Set subagent_policy_state=authorization_required and ask once for the needed delegation scope before continuing phases that require subagents. Set delegation_authorized only after explicit user approval or explicit user-requested subagents. Use direct_fallback only after authorization_denied, subagents_unavailable after a real spawn failure, or policy_disallowed." @@ -210,6 +214,21 @@ gates: check: "Plan includes artifact_contract before task packets with artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command/method, expected success signal, owner/consumer, and non-goals/exclusions" on_fail: "Load references/artifact-first-output-contract.md and add the Artifact Contract before planning implementation steps" + - id: P_DONE_CONTRACT + check: "For medium+ harness-capable work: accepted Done Contract exists before Build with done_when, not_done_when, verification, owner/consumer, acceptance_criteria, and debate_record with at least two perspectives; delegated mode uses subagent perspectives when available" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Block Build, load references/harness-controller.md, return to Plan, write the Done Contract, collect at least two debate perspectives using subagents when subagent_execution_mode=delegated and available, and seek re-approval when scope or acceptance changes" + + - id: P_HARNESS_RECIPE + check: "For medium+ harness-capable work: Harness Recipe is selected before Build from task/model/risk/context profile and records selected_recipe, recipe_rationale, required_artifacts, and corrective_action" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Block Build, load references/harness-controller.md, return to Plan, classify task_profile, model_profile, risk_profile, and context_profile, select the Harness Recipe, and record the missing-recipe corrective action" + + - id: P_HARNESS_RUNTIME_ARTIFACTS + check: "For medium+ harness-capable work: plan/task packets name Harness Run State, Trace Ledger, and Replay Packet artifact refs before Build" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Return to Plan, add harness_run_state_ref, trace_ledger_ref, and replay_packet_ref to the task packet or plan, and record the corrective action for missing run-state/trace/replay evidence" + - id: P_VERIFIED_DISTILLATION check: "When promoting a workflow or review lesson into durable skill/constraint/eval knowledge: verified_skill_distillation exists and verifier_result is approved before durable files are written" on_fail: "Load references/verified-skill-distillation.md and obtain verifier approval before skill/constraint/eval promotion" @@ -327,6 +346,20 @@ gates: check: "Every plan step has a corresponding '>> Step N/total: DONE' message" on_fail: "Complete remaining plan steps before entering Review" + - id: B_DONE_CONTRACT + check: "For medium+ harness-capable work: Build started only after the approved plan recorded Done Contract and Harness Recipe evidence" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Stop Build, return to Plan, add or repair the Done Contract and Harness Recipe from references/harness-controller.md, then resume only after the corrective action is recorded" + + - id: B_HARNESS_RUN_STATE_TRACE_REPLAY + check: "For medium+ harness-capable work: Harness Run State, Trace Ledger, and Replay Packet are updated as execution progresses with task id/name, phase, slice, status, blockers, last verification, next action, recovery pointer, ordered events, decisions, verification results, plan deviations, artifact refs, pinned context, validation state, and exact next action" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Pause Build, update the task journal or carried-forward state with missing Harness Run State, Trace Ledger, and Replay Packet evidence, append a corrective trace entry, and resume only from the recorded exact next action" + + - id: B_CODE_WRITER_BLOCKER_ROUTING + check: "When Code Writer returns NEEDS_CONTEXT, BLOCKED, or DEVIATED because of an unexpected blocker: blocker_type and blocker_evidence are recorded, the orchestrator routes recovery to debugging, explorer, architect, candidate_search, replan, restart, user clarification, environment fix, or permission request, and Code Writer is not asked to improvise through legacy blockers blindly" + on_fail: "Pause Build, classify the blocker, update Harness Run State/Trace Ledger/Replay Packet when harness-capable, create pivot_restart_decision when restart/pivot/replan is selected, and obtain reapproval before continuing if scope, files, behavior, risk, verification, or acceptance criteria change" + - id: B2 check: "Build command succeeded (exit code 0 or equivalent success)" on_fail: "Fix build errors. Do not enter Review with a broken build." @@ -359,8 +392,8 @@ gates: on_fail: "Run verification criteria for each slice, record command/result evidence in the configured task journal or equivalent carried-forward state, and fix failing criteria before proceeding to Review." - id: B_SUBAGENT_EVIDENCE - check: "For source-changing Build tasks: subagent_execution_mode is not not_applicable, required_agents includes Code Writer, Builder/Tester, and Reviewer, and the task journal Agent Dispatch Log records strict delegated dispatch/result evidence or explicit direct fallback role-equivalent evidence" - on_fail: "Update the Agent Dispatch Log before leaving Build. delegated mode requires Code Writer, Builder/Tester, and Reviewer dispatch/result evidence. direct_fallback requires reason authorization_denied, subagents_unavailable, or policy_disallowed plus Code Writer, Builder/Tester, and Reviewer direct evidence." + check: "For source-changing Build tasks: subagent_execution_mode is not not_applicable, required_agents includes Code Writer, Builder/Tester, and Code Reviewer, and the task journal Agent Dispatch Log records strict delegated dispatch/result evidence or explicit direct fallback role-equivalent evidence; Reviewer compatibility may satisfy existing/legacy handoffs only when explicitly recorded" + on_fail: "Update the Agent Dispatch Log before leaving Build. delegated mode requires Code Writer, Builder/Tester, and Code Reviewer dispatch/result evidence, or Reviewer compatibility dispatch/result evidence for existing/legacy handoffs. direct_fallback requires reason authorization_denied, subagents_unavailable, or policy_disallowed plus Code Writer, Builder/Tester, and Code Reviewer direct evidence; Reviewer direct evidence is compatibility evidence only." - id: B_SUBAGENT_SLICE_EVIDENCE check: "For medium+ delegated tasks with slices: per-slice dispatch evidence is recorded before the slice is marked VERIFIED" @@ -420,6 +453,19 @@ gates: check: "Quality Review completed by loading and following assistant-review SKILL.md and contracts (not manual review)" on_fail: "Load and follow assistant-review SKILL.md and contracts. Do not manually dispatch reviewers." + - id: R2A_CODE_REVIEWER_DISTINCT + check: "Code Reviewer or Reviewer compatibility evidence is recorded separately from QA Evaluator evidence" + on_fail: "Update the Review Log, output contract, or carried-forward review state with distinct Code Reviewer/Reviewer and QA Evaluator entries. Do not collapse QA evaluation into code review." + + - id: R_QA_EVALUATION + check: "When qa_evaluation_mode is required: QAEvaluator runs after build/test evidence and Code Reviewer or Reviewer compatibility result, returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, and evidence" + condition: "qa_evaluation_mode == required" + on_fail: "Load assistant-review references/qa-evaluation-loop.md, dispatch QAEvaluator or record direct-fallback QA evidence, and do not complete Review until QA result is present or blocked with open_questions. If domain_context/rubric_refs or subjective/product/UX/docs/DX/UI/domain acceptance scope exists, require selected_domain_rubrics and domain_quality_scores; otherwise reject invented rubric scoring." + + - id: R_PIVOT_RESTART_DECISION + check: "Review or QA STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric/domain action pivot records an orchestrator-owned pivot_restart_decision before another fix, review, QA, or build dispatch; round 20 remains terminal and never creates round 21 behavior" + on_fail: "Pause Review, create pivot_restart_decision with trigger, evidence, affected_slice_or_round, options_considered, selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action. Update Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger when harness-capable before continuing." + - id: R3 check: "Verification summary is written with: changed files, test/validation coverage, review result, manual validation steps when applicable, known limitations" on_fail: "Write verification summary in the configured task journal, output contract, or equivalent carried-forward state" @@ -474,6 +520,11 @@ gates: condition: "size in [medium, large, mega]" on_fail: "Load assistant-reflexion and assistant-memory contracts, inspect Review Log findings, build/test failures, and user corrections for durable lessons, check memory trend/backend availability when allowed, then write the Learning Controller block. Use backend_unavailable or policy_disallowed when appropriate; do not require a memory write when tools are unavailable, and do not persist routine progress as a lesson." + - id: DOC_HARNESS_REPLAY_PACKET + check: "For medium+ harness-capable work: final Harness Run State, Trace Ledger, and Replay Packet evidence is current before completion or handoff" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Before Document completes, update Harness Run State to the current phase/status, append final trace entries for verification/review/document decisions, refresh Replay Packet validation_state and exact_next_action, and record any remaining blocker" + - id: DOC_SKILL_DISTILLATION check: "If work produced a repeatable workflow, recurring lesson, user-requested reusable procedure, or skill/constraint candidate: verified_skill_distillation exists and verifier_result is approved before promotion, or promotion_decision is skip with rationale" on_fail: "Load references/verified-skill-distillation.md, create the distillation packet, and send it to an independent verifier before saving any reusable skill or constraint" @@ -510,3 +561,8 @@ invariants: check: "If scope grows beyond initial triage during any phase, stop and re-triage" scope: all_phases on_fail: "Print '>> SCOPE CHANGE DETECTED' and re-run Triage with updated understanding" + + - id: INV_PIVOT_RESTART_REAPPROVAL + check: "Any pivot_restart_decision that changes approved scope, files, behavior, risk, verification, or acceptance criteria has reapproval_required=true and records approval before Build or Review continues" + scope: all_phases + on_fail: "Pause the workflow, mark reapproval_required=true, update the plan/task journal, and wait for user approval before continuing the changed path" diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md b/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md new file mode 100644 index 0000000..adb5b8c --- /dev/null +++ b/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md @@ -0,0 +1,221 @@ +# Harness Controller + +Use this reference only for medium+ work that is harness-capable: long-running, +multi-slice, delegated, high-risk, subjective/domain-scored, trace/replay-ready, +or explicitly requested as harness work. Do not load it for small local fixes. + +## Pre-Build Gate + +Before Build starts, the approved plan must contain both: + +- an accepted Done Contract +- a selected Harness Recipe +- refs for Harness Run State, Trace Ledger, and Replay Packet artifacts when + the recipe is trace/replay-ready + +If either is missing, block Build, return to Plan, add the missing artifact, and +record the corrective action in the task journal or carried-forward state. + +## Done Contract + +The Done Contract defines what "finished" means before implementation begins. + +Required fields: + +- `done_when`: pass/fail outcomes that prove the slice or task is complete +- `not_done_when`: explicit failure states that must block completion +- `verification`: commands, inspections, reviews, or manual checks that prove done +- `owner_consumer`: owner and downstream consumer of the artifact or behavior +- `acceptance_criteria`: explicit binary criteria copied from user/plan/slice scope +- `debate_record`: at least two perspectives considered before acceptance +- `accepted_by`: user, orchestrator, or approved plan reference + +For subjective/domain-scored, product, UX, UI, docs, or DX work, carry any +scoped `domain_context` and `rubric_refs` into the QA Evaluator packet. These +refs enable conditional use of assistant-review `references/domain-rubrics.md`; +they are not required for unrelated code-review-only work. + +Debate rules: + +- Record at least two perspectives, such as implementer, tester, reviewer, + architect, product, security, docs, or user. +- When `subagent_execution_mode=delegated` and relevant subagents are available, + use subagent perspectives for the debate before Build. +- If delegated debate is unavailable, record the fallback reason and the direct + role-equivalent perspectives used. +- Do not let the debate add scope. Scope changes are plan deviations. + +## Harness Recipe + +The Harness Recipe selects the controller shape from the task/model/risk/context +profile. It is a short routing decision, not a new plan. + +Required profile fields: + +- `task_profile`: task type, size, slice count, and whether TDD/debugging applies +- `model_profile`: agent/model constraints, delegation mode, and tool limits +- `risk_profile`: risk tier, safety gates, review depth, and rollback needs +- `context_profile`: exact context, summarized context, omitted/deferred context, + and whether trace/replay or handoff artifacts are needed + +Required recipe fields: + +- `selected_recipe`: concise recipe label +- `recipe_rationale`: why the profile selects this recipe +- `required_artifacts`: Done Contract, task packet, verification, and any trace or + handoff artifacts needed by later slices +- `corrective_action`: what to do if the recipe is missing or stops matching the + task during Build +- Corrective action: the recorded recovery step for a missing or stale recipe + +Selection rules: + +- Use a lightweight guarded recipe for medium single-slice work with moderate risk + and compact context. +- Use a slice-sequential recipe when independent slice verification is required. +- Use a review-intensive recipe for high/critical risk, weak tests, public + contracts, or subjective acceptance. +- Use a trace/replay-ready recipe when context is large, work is long-running, or + recovery after compaction/failure is likely. + +## Runtime Artifacts + +For medium+ harness-capable work, keep these first-class artifacts in the task +journal or equivalent carried-forward state and update them as execution +progresses. They are recovery artifacts, not extra planning ceremony. + +### Harness Run State + +Records the current task/run position: + +- `task_id` +- `task_name` +- `phase` +- `slice` +- `status` +- `blockers` +- `last_verification` +- `next_action` +- `recovery_pointer` + +### Trace Ledger + +Records ordered or timestamped execution evidence: + +- agent events +- decisions +- verification commands/results +- plan deviations +- artifact refs + +### Replay Packet + +Captures the minimum continuation packet needed after compaction, failure, or +handoff: + +- pinned context +- artifact refs +- validation state +- exact next action +- run-state and trace-ledger refs +- recovery pointer + +## Pivot/Restart Controller + +The Pivot/Restart Controller is owned by the orchestrator. It runs when a +quality loop or Build handoff is no longer making safe progress. + +Trigger it for: + +- review or QA `STAGNATION` +- repeated `DRIFT` +- repeated `REGRESSION` +- rubric or domain action `pivot` +- Code Writer `blocker_type` returns such as `legacy_code_bug`, + `broken_baseline`, `hidden_dependency`, `missing_contract`, `stale_plan`, + `scope_conflict`, `tool_environment`, `permission_policy`, `tdd_red_missing`, + or `other` +- verification blockers, plan deviations, or scope changes that make the + approved packet stale + +Required decision fields: + +- `trigger`: the exact trigger category +- `evidence`: score entries, findings, blocker evidence, verification failures, + or trace refs proving the trigger +- `affected_slice_or_round`: current slice id/name, review round, QA round, or + workflow phase +- `options_considered`: at least two recovery options unless policy or missing + approval leaves only one safe path +- `selected_action`: reset context, return to Build, dispatch debugging, + dispatch explorer, dispatch architect, run candidate search, replan, restart + the slice, restart the phase, block for user, or accept with limitations +- `reapproval_required`: true whenever scope, files, behavior, risk, + verification, or acceptance criteria change +- `next_agent`: the next role or agent to dispatch +- `recovery_pointer`: task packet, trace row, replay packet, plan section, or + file path where recovery resumes +- `exact_next_action`: the single action to perform after the decision + +When a decision is created, update Harness Run State, append a `pivot_restart` +Trace Ledger entry, refresh Replay Packet so `exact_next_action` matches the +decision, and add or update the Pivot/Restart Decision artifact ref. Round 20 remains terminal; the controller never creates round 21 behavior. + +Routing rules: + +- Code Writer legacy code bugs or broken baselines route to debugging before + another implementation attempt. +- Hidden dependencies route to Explorer or Code Mapper refresh. +- Missing contracts or stale task packets route to Architect or Plan repair. +- Scope conflicts, plan deviations, and candidate pivots route to replan and + reapproval when scope, files, behavior, risk, verification, or acceptance + criteria change. +- Tool, environment, permission, or policy blockers route to environment fix, + permission request, or BLOCKED with evidence. +- Review or QA stagnation routes to reset context, candidate search, replan, + or restart depending on the evidence; do not silently continue the loop. + +## Typed Artifact References + +When a harness artifact crosses an agent boundary, pass it as an Artifact +Reference entry instead of an ad hoc string. Each entry carries: + +- `artifact_id` +- `artifact_type` +- `producer` +- `consumer` +- `location_ref` +- `schema_or_contract` +- `validation_status` +- `summary` + +Producer responsibility: create or update the artifact, assign its stable id and +location/ref pointer, name the contract or schema it follows, and summarize its +current state. Consumer responsibility: validate `schema_or_contract` and +`validation_status` before relying on `location_ref`; invalid or stale refs block +phase advancement or trigger re-dispatch. + +Use typed refs for Done Contract, Harness Recipe, Harness Run State, Trace +Ledger, Replay Packet, Pivot/Restart Decision, changed files, verification +evidence, and plan deviation refs when applicable. + +## Corrective Actions + +- Missing Done Contract: return to Plan, load this reference, write the contract, + record debate with at least two perspectives, and wait for approval when the + contract changes scope or acceptance. +- Missing Harness Recipe: return to Plan, classify task/model/risk/context + profile, select a recipe, and record rationale plus corrective action. +- Missing debate perspectives: collect the missing perspective through delegated + subagents when available, or record direct fallback perspective evidence. +- Missing run-state/trace/replay evidence: pause phase advancement, add or + repair Harness Run State, Trace Ledger, and Replay Packet blocks with the + required fields, append a corrective trace entry, and resume from the recorded + exact next action. +- Pivot/restart trigger: pause the active loop, create the orchestrator-owned + `pivot_restart_decision`, update run-state/trace/replay/artifact refs, and + reapprove before continuing if scope, files, behavior, risk, verification, or + acceptance criteria change. +- Recipe mismatch during Build: print `>> PLAN DEVIATION DETECTED`, update the + recipe, and seek re-approval when files, behavior, scope, risk, or verification + changes. diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/phases.md b/plugins/assistant-dev/skills/assistant-workflow/references/phases.md index f40d207..450f4e0 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/phases.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/phases.md @@ -173,7 +173,7 @@ Print: `>> Direct fallback Architect responsibility` (when `subagent_execution_m **Entry rule:** Do not enter Plan while the saved clarification state is pending. Resume Plan only after Discover records `Clarification status: ready` and all implementation-shaping fields are explicit or explicitly defaulted. -Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. Then read `references/plan-template.md` and use the correct tier: +Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. For medium+ harness-capable work, load `references/harness-controller.md` and add the Done Contract, Harness Recipe, Harness Run State ref, Trace Ledger ref, Replay Packet ref, and Artifact Reference Ledger before task packets. Then read `references/plan-template.md` and use the correct tier: - Small: inline plan (goal, files, risks, tests). Do not wait for approval unless risk, ambiguity, user instruction, or a scope-changing decision makes approval necessary. - Medium: standard plan (drop Security/Operability unless the task touches auth, PII, payments, or infra) - Large/Mega: full plan (all sections including Security and Operability) @@ -183,12 +183,13 @@ Before writing the plan, load `references/artifact-first-output-contract.md` and 3. Analyze 1-3 options with tradeoffs, pick one 4. Identify risks and edge cases 5. Put the Artifact Contract before task packets and map every medium+ task packet to at least one required artifact or acceptance criterion -6. For medium+ tasks: consume the Decompose slice manifest directly in the plan and align each task packet to exactly one slice_id without rediscovering boundaries -6. Write ordered implementation steps with file paths -7. For large/mega: fill in Security and Operability sections. For medium: only if the task touches auth, PII, payments, or infra (promote to Full tier per plan-template.md) -8. Carry `Task type`, `Risk tier`, `Required gates`, `Required agents`, `subagent_policy_state`, `subagent_execution_mode`, `subagent_authorization_scope`, and `Search mode` into the plan. Each required gate must map to task packet criteria or explicit N/A rationale. -9. If `search_mode: candidate_search`, load `references/candidate-search.md`, create the goal tree from acceptance/slice criteria, score candidates, record the archive location, and treat post-approval pivots as plan deviations requiring re-approval when scope/files/behavior/risk change. -10. Load prompt packs only when applicable: +6. For medium+ harness-capable work, put the Done Contract, Harness Recipe, run-state/trace/replay artifact refs, and Artifact Reference Ledger before Build. The Done Contract must include done_when, not_done_when, verification, owner/consumer, acceptance criteria, and at least two debate perspectives; the Harness Recipe must be selected from task/model/risk/context profile. The ledger must type each artifact with artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, and summary. +7. For medium+ tasks: consume the Decompose slice manifest directly in the plan and align each task packet to exactly one slice_id without rediscovering boundaries +8. Write ordered implementation steps with file paths +9. For large/mega: fill in Security and Operability sections. For medium: only if the task touches auth, PII, payments, or infra (promote to Full tier per plan-template.md) +10. Carry `Task type`, `Risk tier`, `Required gates`, `Required agents`, `subagent_policy_state`, `subagent_execution_mode`, `subagent_authorization_scope`, and `Search mode` into the plan. Each required gate must map to task packet criteria or explicit N/A rationale. +11. If `search_mode: candidate_search`, load `references/candidate-search.md`, create the goal tree from acceptance/slice criteria, score candidates, record the archive location, and treat post-approval pivots as plan deviations requiring re-approval when scope/files/behavior/risk change. +12. Load prompt packs only when applicable: - Refactors: `references/prompts/refactor-safety.md` - Migrations/rewrites: `references/prompts/refactor-safety.md` plus any applicable migration or parity checklist - New code: `references/prompts/test-strategy.md` @@ -238,18 +239,27 @@ For medium+ tasks, create a task journal using `references/task-journal-template Capture **constraints** from Discovery/Plan (e.g. "don't touch ProjectA", "stay on .NET 8"). Check constraints before each step. +For medium+ harness-capable work, confirm the task journal or carried-forward plan has an accepted Done Contract, selected Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger before dispatching Code Writer or Builder/Tester. If any are missing, stop Build and return to Plan or repair state for the corrective action in `references/harness-controller.md`. + +Keep the runtime artifacts current as execution progresses: +- Update Harness Run State after each slice/step, blocker, verification result, phase transition, or next-action change. +- Append Trace Ledger entries for agent events, decisions, verification commands/results, plan deviations, pivot/restart decisions, and artifact refs. +- Refresh the Replay Packet before context compaction, failure handoff, phase handoff, or end-of-turn with pinned context, artifact refs, validation state, and the exact next action. +- Update the Artifact Reference Ledger when producers create or move artifacts, and require consumers to validate `schema_or_contract` plus `validation_status` before relying on `location_ref`. + ### Orchestrator delegation rule **Delegated mode (`subagent_execution_mode=delegated`):** the orchestrator does not edit project source files or write implementation/test code directly. Framework-owned state artifacts (`{agent_state_dir}/task.md`, `{agent_state_dir}/context-map.md`, `{agent_state_dir}/session.md`, and `{agent_state_dir}/working-buffer.md`) are the exception only when configured and policy-allowed, and may be updated directly. If local state files are unavailable, carry equivalent state in the plan/response packet. Project source changes go through sub-agents, and the Agent Dispatch Log must record dispatch and result evidence for every required role before completion: - **Code Writer** (`code-writer`): implements code following the plan - **Builder/Tester** (`builder-tester`): builds, writes tests, runs tests -- **Reviewer** (`reviewer` or assistant-review delegated review): independent review evidence for Review +- **Code Reviewer** (`code-reviewer`, or `reviewer` compatibility through assistant-review delegated review): independent code/security/architecture/test coverage review evidence for Review +- **QA Evaluator** (`qa-evaluator`): independent acceptance, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result evaluation when QA is required -Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Reviewer. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. +Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Code Reviewer; `reviewer` remains compatibility routing for existing/legacy handoffs. Medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work also adds QA Evaluator. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. **Direct fallback mode (`subagent_execution_mode=direct_fallback`):** when authorization is denied, subagents are unavailable, or policy disallows spawning, the active agent may implement, test, and review directly, but must preserve the same phases, contracts, evidence requirements, and review/security gates. Do not pretend delegation happened; record `subagent_policy_state`, `subagent_execution_mode`, the explicit `Direct fallback reason: authorization_denied | subagents_unavailable | policy_disallowed`, and the direct-execution evidence. -**Strict subagent evidence gate:** any development/code-work task that changes project source, tests, docs, config, hooks, contracts, or generated artifacts must keep Code Writer, Builder/Tester, and Reviewer in `Required agents`; `subagent_execution_mode=not_applicable` is invalid for Build. Before leaving Build/Review, the task journal Agent Dispatch Log (or equivalent carried-forward state) must record `Code Writer dispatch` + `Code Writer result`, `Builder/Tester dispatch` + `Builder/Tester result`, and `Reviewer dispatch` + `Reviewer result` in delegated mode. Direct fallback must instead record `Code Writer direct evidence`, `Builder/Tester direct evidence`, and `Reviewer direct evidence` plus the explicit fallback reason. Silent fallback is invalid; Silent fallback cannot complete. +**Strict subagent evidence gate:** any development/code-work task that changes project source, tests, docs, config, hooks, contracts, or generated artifacts must keep Code Writer, Builder/Tester, and Code Reviewer in `Required agents`; `reviewer` remains compatibility routing for existing/legacy handoffs and task journals. `subagent_execution_mode=not_applicable` is invalid for Build. Before leaving Build/Review, the task journal Agent Dispatch Log (or equivalent carried-forward state) must record `Code Writer dispatch` + `Code Writer result`, `Builder/Tester dispatch` + `Builder/Tester result`, and `Code Reviewer dispatch` + `Code Reviewer result` or `Reviewer dispatch` + `Reviewer result` only when using compatibility routing in delegated mode. When QA is required, it must also record `QA Evaluator dispatch` + `QA Evaluator result` in delegated mode. Direct fallback must instead record `Code Writer direct evidence`, `Builder/Tester direct evidence`, and `Code Reviewer direct evidence`; `Reviewer direct evidence` may satisfy only compatibility routing for existing/legacy handoffs. Add `QA Evaluator direct evidence` when QA is required, with the explicit fallback reason. Silent fallback is invalid; Silent fallback cannot complete. For each non-TDD step: dispatch Code Writer with the plan step + context map only in delegated mode, or execute directly in fallback mode and record Code Writer direct evidence → verify via Builder/Tester only in delegated mode, or run verification directly in fallback mode and record Builder/Tester direct evidence → check results → proceed or fix. For TDD-active steps, use the TDD sandwich in the Build loop. @@ -264,8 +274,9 @@ Print: `>> Slice [S]/[total]: [slice_id] [name]` Before starting each medium+ slice: 1. Load the approved task packet for the slice, including slice_id, observable increment, deliverable type, files, acceptance criteria, verification command, expected success signal, evidence to record, and deviation/rollback rule -2. Confirm prior slice status is `VERIFIED` before advancing; do not start the next slice while the current slice is unverified -3. Check constraints from the task journal against the slice files and criteria +2. When harness-capable, confirm the task packet carries `done_contract_ref`, `harness_recipe_ref`, `harness_run_state_ref`, `trace_ledger_ref`, `replay_packet_ref`, and typed `artifact_refs` +3. Confirm prior slice status is `VERIFIED` before advancing; do not start the next slice while the current slice is unverified +4. Check constraints from the task journal against the slice files and criteria For the current slice task packet: @@ -286,6 +297,42 @@ Print: `>> Direct fallback Builder/Tester responsibility → RED evidence` (TDD - Code Writer GREEN: implement minimal production code only after RED evidence is present. - Builder/Tester VERIFY/REFACTOR-SAFETY: run the targeted test, relevant suite, and regression checks; request Code Writer fixes for production failures. +### Code Writer unexpected blockers + +If Code Writer returns `NEEDS_CONTEXT`, `BLOCKED`, or `DEVIATED` because of a +legacy code bug, broken baseline, hidden dependency, missing contract, stale +plan, scope conflict, tool/environment issue, permission/policy issue, missing +RED evidence, or another unexpected blocker, do not ask Code Writer to +improvise through it blindly. + +Require the return to include: + +- `blocker_type`: `legacy_code_bug`, `broken_baseline`, `hidden_dependency`, + `missing_contract`, `stale_plan`, `scope_conflict`, `tool_environment`, + `permission_policy`, `tdd_red_missing`, or `other` +- `blocker_evidence`: file paths, missing contract fields, baseline failures, + tool output summaries, scope conflict details, or task-packet mismatches + +Orchestrator recovery routing: + +- `legacy_code_bug` or `broken_baseline` -> dispatch debugging before another + implementation attempt +- `hidden_dependency` -> dispatch Explorer or refresh Code Mapper context +- `missing_contract` or `stale_plan` -> dispatch Architect or return to Plan +- `scope_conflict` -> record a plan deviation and seek reapproval when scope, + files, behavior, risk, verification, or acceptance criteria change +- `tool_environment` -> route to Builder/Tester or environment recovery +- `permission_policy` -> request permission or return BLOCKED +- `tdd_red_missing` -> return to Builder/Tester RED evidence +- `other` -> create a conservative recovery route with evidence + +When recovery requires pivot, replan, candidate search, or restart, create the +orchestrator-owned `pivot_restart_decision`, update Harness Run State, append a +`pivot_restart` Trace Ledger entry, refresh Replay Packet with the exact next +action, and update Artifact Reference Ledger before continuing. + +Recovery route labels are: debugging, explorer, architect, candidate_search, replan, restart, user_clarification, environment_fix, and permission_request. + Print: `>> Step [N]/[total]: DONE` (after each step passes build + test) ### Per-slice verification (medium+ tasks) @@ -297,9 +344,10 @@ After implementation for a slice is done, verify the slice against its criteria 1. Dispatch Builder/Tester in delegated mode, or perform the Builder/Tester responsibility directly in fallback mode, to run the slice's verification command and any relevant build/test checks from the task packet 2. Check each acceptance criterion from the slice manifest independently — mark pass/fail with the command/result or inspection evidence used 3. Record verification evidence in the task journal slice verification ledger, including RED status when TDD was active, implementation status, command/result, criteria checked, and final status -4. Run a small self-check/local sanity check: compare changed files and behavior against the task packet, constraints, and deviation rule; record the result in the ledger -5. If any criterion, command, or self-check fails: fix before moving to the next slice -6. Mark the slice `VERIFIED` only after all criteria pass and evidence is recorded +4. Update Harness Run State, append Trace Ledger verification/artifact refs, refresh Replay Packet validation_state plus exact_next_action, and update Artifact Reference Ledger entries for changed_files, verification_evidence, pivot_restart_decision, and plan_deviation refs when applicable +5. Run a small self-check/local sanity check: compare changed files and behavior against the task packet, constraints, and deviation rule; record the result in the ledger +6. If any criterion, command, runtime artifact, or self-check fails: fix before moving to the next slice +7. Mark the slice `VERIFIED` only after all criteria pass and evidence is recorded Print: `>> Slice [S]/[total]: [slice_id] [name] — VERIFIED ([X]/[Y] criteria passed)` @@ -316,7 +364,10 @@ Plan assumed: [X]. Reality: [Y]. Options: a) Adjust step [N] b) Rethink approach ``` -Never silently deviate from the plan. +Never silently deviate from the plan. If the selected recovery action changes +approved scope, files, behavior, risk, verification, or acceptance criteria, +record `pivot_restart_decision.reapproval_required=true` and wait for approval +before continuing the changed path. Print: `>> Build complete — all [N] steps implemented` Print: `>> Running final build + tests` @@ -334,7 +385,7 @@ Print: `>> Stage 1: Spec Review` Load and follow `references/prompts/spec-review.md`. Compare implementation against the approved plan, approved task packets, and approved slices and produce a structured spec compliance result before Stage 2. For bugfixes, include the `assistant-debugging` reproduction/root-cause evidence in the review material and check that the regression test or validation path actually covers the isolated failure mechanism. -Quality review cannot satisfy spec review. Spec review checks scope and acceptance compliance; quality review checks correctness, maintainability, architecture, security, and coverage after spec compliance is clear. +Quality review cannot satisfy spec review. Spec review checks scope and acceptance compliance; code quality review checks correctness, maintainability, architecture, security, and coverage after spec compliance is clear. QA Evaluation is a separate acceptance/result lane when required and cannot substitute for either Spec Review or Code Quality Review. 1. Walk through each approved plan step, task packet, or slice against `git diff`. 2. Check for missing acceptance criteria: any required behavior not reflected in code or evidence. @@ -355,18 +406,35 @@ Print: `>> Spec Review: [PASS / FAIL — found N required fixes]` ### Stage 2 — Quality Review -Print: `>> Stage 2: Quality Review — loading assistant-review SKILL.md` +Print: `>> Stage 2: Code Quality Review — loading assistant-review SKILL.md` -**Load and follow `assistant-review` SKILL.md and its contracts.** This runs the autonomous review-fix loop (max 10 rounds) with visible progress. Add Reviewer to `Required agents` before Stage 2. Do NOT implement the review loop inline when `subagent_execution_mode=delegated` and a delegated review agent is authorized — dispatch the Reviewer subagent and record `Reviewer dispatch`/`Reviewer result` evidence. In direct fallback, preserve fresh-review evidence and record `Reviewer direct evidence`. +**Load and follow `assistant-review` SKILL.md and its contracts.** This runs the autonomous review-fix loop (max 20 rounds) with visible progress. Add Code Reviewer to `Required agents` before Stage 2; use `Reviewer` only as compatibility routing for existing/legacy handoffs. Dispatch the `code-reviewer` agent for code defects, security, architecture, test coverage, and structural code issues. Do NOT implement the review loop inline when `subagent_execution_mode=delegated` and a delegated review agent is authorized — dispatch the Code Reviewer subagent and record `Code Reviewer dispatch`/`Code Reviewer result` evidence, or record `Reviewer dispatch`/`Reviewer result` only when using compatibility routing. In direct fallback, preserve fresh-review evidence and record `Code Reviewer direct evidence`; `Reviewer direct evidence` is compatibility evidence only. The `assistant-review` skill will: -- Dispatch Reviewer subagents in delegated mode, or preserve fresh-review evidence in direct fallback mode +- Dispatch Code Reviewer subagents in delegated mode, using Reviewer compatibility only for existing handoffs, or preserve fresh-review evidence in direct fallback mode - Fix must-fix and should-fix items - Re-review automatically +- Escalate STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT into an orchestrator-owned `pivot_restart_decision` before another fix/review dispatch - Return a final clean/remaining summary For small tasks: quick spec check + single review round is acceptable if clean. -For medium+ tasks: full two-stage review with autonomous quality loop via `assistant-review`. +For medium+ tasks: full spec review plus autonomous code quality loop via `assistant-review`. + +### Stage 3 — QA Evaluation + +Print: `>> Stage 3: QA Evaluation — loading assistant-review references/qa-evaluation-loop.md` when QA is required. + +Run QA Evaluation for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. + +The QA Evaluator receives Done Contract when present, acceptance criteria, verification evidence, code review result, domain_context/rubric_refs when applicable, round 1-20, previously_failed_acceptance_items, and qa_filter_policy. It loads assistant-review `references/domain-rubrics.md` only when acceptance criteria, Done Contract, domain_context, or explicit rubric_refs require subjective/product/UX/docs/DX/UI/domain scoring. It returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression or score_entry, evidence, and open_questions when blocked. + +QA Evaluation focuses on acceptance and final result. It does not replace Code Reviewer, and it must not report general code defects, security issues, architecture concerns, or test coverage gaps unless they directly block an acceptance criterion or Done Contract item. + +If QA score progression reports STAGNATION, repeated DRIFT, repeated REGRESSION, +or any scoped domain rubric returns action `pivot`, pause the QA loop and create +an orchestrator-owned `pivot_restart_decision` before another QA/build dispatch. +Round 20 remains terminal: return the final QA verdict and remaining failed +acceptance items instead of starting round 21. ### Status gate @@ -374,6 +442,8 @@ When local hooks are configured and policy-allowed, use them to enforce the revi When hooks are unavailable, enforce the same gate manually before presenting results: - Review Log or equivalent review result must exist. +- QA Evaluation result must exist when qa_evaluation_mode=required. +- Pivot/Restart Decision must exist when Review or QA reported STAGNATION, repeated DRIFT, repeated REGRESSION, or pivot action. - Final Result must be recorded. - The agent must complete the full review cycle before presenting results to the user. - Do not claim the hook ran unless it actually exists and executed. @@ -385,7 +455,8 @@ Print: `--- PHASE: REVIEW COMPLETE ---` After final review passes, write the Verification Summary in the task journal: - What changed (files + why) - What's tested (unit/integration/E2E coverage) -- Review result (clean / issues fixed / remaining should-fix items) +- Code Review result (clean / issues fixed / remaining should-fix items) +- QA Evaluation result when required (accepted / accepted with concerns / rejected / blocked) - Manual test instructions (step-by-step for the user) - Known limitations @@ -442,7 +513,8 @@ Then print completion markers and exit. - `No-save rationale`: required when no durable write occurred 6. **Post-task reflection**: Load and follow `assistant-reflexion` when available. Pass the Learning Controller evidence into reflexion so lessons are backed by review/build/user-correction evidence and persistence/no-save status is explicit. 7. If local memory tools are approved and available, persist only durable, evidence-backed lessons through the configured local memory backend. If tools are unavailable or policy-disallowed, record that outcome in `Durable lesson decision` and `No-save rationale` instead of writing ad hoc markdown as cross-session memory. -8. **Task completion metrics**: Append a JSONL entry when local metrics are configured and policy allows it (see format below) +8. For medium+ harness-capable work, refresh Harness Run State, append final Trace Ledger entries for review/document decisions, include Pivot/Restart Decision refs when triggered, and update Replay Packet validation_state plus exact_next_action before completion or handoff. +9. **Task completion metrics**: Append a JSONL entry when local metrics are configured and policy allows it (see format below) ### Metrics entry format (all sizes) diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md index 1d1467b..10814cd 100755 --- a/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md @@ -55,6 +55,21 @@ For Medium and Large/Mega plans, write implementation work as executable task pa - Expected success signal: [exit code 0, passing test name, output marker, etc.] - Evidence to record: - [test result, eval fixture, changed file, review note, or artifact proof] +- Harness refs: + - done_contract_ref: [Done Contract section/ref, or N/A] + - harness_recipe_ref: [Harness Recipe section/ref, or N/A] + - harness_run_state_ref: [Harness Run State section/ref, or N/A] + - trace_ledger_ref: [Trace Ledger section/ref, or N/A] + - replay_packet_ref: [Replay Packet section/ref, or N/A] +- Typed artifact refs: + - artifact_id: [stable task-local id] + artifact_type: [done_contract | harness_recipe | harness_run_state | trace_ledger | replay_packet | changed_files | verification_evidence | plan_deviation | task_packet | context_map | test_result | review_result | qa_evaluation_result] + producer: [role/subagent/hook/task packet] + consumer: [role/subagent/hook/phase] + location_ref: [typed location/ref pointer] + schema_or_contract: [contract/template/required fields] + validation_status: [pending | valid | invalid | stale | not_applicable] + summary: [concise state] - Deviation / rollback rule: - [what to do if required files/behavior differ from plan; include rollback/revert boundary] - Worker status / evidence: @@ -171,6 +186,62 @@ Covers the essentials without Security/Operability overhead. Fill this in during - Owner/consumer: [user, reviewer, downstream tool, runtime] - Non-goals/exclusions: [what must not be produced] +## Done Contract + +Required for medium+ harness-capable work before Build; omit with a brief N/A +rationale for non-harness work. + +- done_when: + - [binary outcome that proves done] +- not_done_when: + - [failure state that blocks done] +- verification: + - [command, inspection, review, or manual check] +- owner_consumer: [owner and downstream consumer] +- acceptance_criteria: + - [explicit binary criterion] +- debate_record: + - perspective: [role/subagent/direct perspective 1] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] + - perspective: [role/subagent/direct perspective 2] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] +- accepted_by: [user/orchestrator/approved plan reference] + +## Harness Recipe + +Required for medium+ harness-capable work before Build; selected from +task/model/risk/context profile per `references/harness-controller.md`. + +- task_profile: [task type, size, slice count, TDD/debugging applicability] +- model_profile: [agent/model constraints, delegation mode, tool limits] +- risk_profile: [risk tier, safety gates, review depth, rollback needs] +- context_profile: [exact/summarized/omitted context and trace/replay needs] +- selected_recipe: [concise recipe label] +- recipe_rationale: [why this profile selects the recipe] +- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] +- corrective_action: [what to do if missing or stale] + +## Runtime Harness Artifacts + +Required for medium+ harness-capable work; omit with a brief N/A rationale for +non-harness work. + +- harness_run_state_ref: [where task_id/task_name/phase/slice/status/blockers/last_verification/next_action/recovery_pointer will be maintained] +- trace_ledger_ref: [where ordered agent events, decisions, verification results, plan deviations, and artifact refs will be appended] +- replay_packet_ref: [where pinned context, artifact refs, validation state, and exact next action will be refreshed] +- corrective_action: [what to do if run-state/trace/replay evidence is missing or stale] + +## Artifact Reference Ledger + +Required for medium+ harness-capable work when artifacts pass between agents. +Each row is a typed producer/consumer record, not an ad hoc string reference. + +| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | +|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| +| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role] | [role/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | + ## Slice manifest from Decompose Use the shared Slice Manifest structure above. Paste the approved Decompose manifest verbatim and keep `single_slice_rationale` when exactly one slice exists. @@ -272,6 +343,15 @@ Everything from Medium, plus Security and Operability sections. Use when the tas - Broad-split rejection: [required proof that layer-only, module-only, folder-only, feature-only, setup-only, contract-only, and broad component-style splits were rejected unless verified deliverable artifact slices] - Decision: proceed | revise_decomposition | return_to_discover +## Artifact Reference Ledger + +Required for medium+ harness-capable work when artifacts pass between agents. +Each row is a typed producer/consumer record, not an ad hoc string reference. + +| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | +|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| +| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role] | [role/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | + ## Slice manifest from Decompose Use the shared Slice Manifest structure above. Paste the approved Decompose manifest verbatim and keep `single_slice_rationale` when exactly one slice exists. diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/prompts/pr-review.md b/plugins/assistant-dev/skills/assistant-workflow/references/prompts/pr-review.md index 8e69fc8..f64fce9 100755 --- a/plugins/assistant-dev/skills/assistant-workflow/references/prompts/pr-review.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/prompts/pr-review.md @@ -1,10 +1,10 @@ -# Quality Review Checklist (Stage 2) +# Code Quality Review Checklist (Stage 2) -Load this during Stage 2 of the review cycle. This is the quality gate — it runs after the Spec Review confirms the implementation matches the plan. +Load this during Stage 2 of the review cycle. This is the code quality gate — it runs after the Spec Review confirms the implementation matches the plan. It does not replace Stage 3 QA Evaluation, which checks acceptance, Done Contract, verification evidence, scoped domain quality, score progression, and final result when QA is required. ## When to use -Run this review as part of the autonomous review-fix loop: after all build steps complete and tests pass. The loop runs up to 10 rounds, fixing must-fix and should-fix items each round until clean. You do NOT need to wait for user input between rounds. +Run this review as part of the autonomous review-fix loop: after all build steps complete and tests pass. The loop runs up to 20 rounds, fixing must-fix and should-fix items each round until clean. You do NOT need to wait for user input between rounds. For small tasks, do a quick pass through the "must-check" items only. For medium+ tasks, work through the full checklist. @@ -13,7 +13,7 @@ For small tasks, do a quick pass through the "must-check" items only. For medium When dispatching the Reviewer subagent for round N > 1, prepend this context to the prompt: ``` -## Previously Fixed (Round N of 10) +## Previously Fixed (Round N of 20) This is review round [N]. The following [count] items were already found and fixed in prior rounds — do NOT re-report them: @@ -23,8 +23,8 @@ and fixed in prior rounds — do NOT re-report them: Focus ONLY on NEW high-confidence findings not in the above list. Raise your confidence threshold each round: - Round 1-3: report findings at 80%+ confidence -- Round 4-7: report findings at 85%+ confidence -- Round 8-10: report findings at 90%+ confidence +- Round 4-15: report findings at 85%+ confidence +- Round 16-20: report findings at 90%+ confidence If you find no new issues above your confidence threshold, say "No new findings." ``` diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/sub-task-brief-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/sub-task-brief-template.md index cf80333..a9ca187 100755 --- a/plugins/assistant-dev/skills/assistant-workflow/references/sub-task-brief-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/sub-task-brief-template.md @@ -36,8 +36,8 @@ Workflow: ## Slice Brief: [name] ### Agent -Agent: [code-writer | architect | reviewer | explorer | builder-tester | code-mapper] -Role: [Implementer | Architect | Reviewer | Explorer] +Agent: [code-writer | architect | code-reviewer | qa-evaluator | reviewer | explorer | builder-tester | code-mapper] +Role: [Implementer | Architect | Code Reviewer | QA Evaluator | Reviewer compatibility | Explorer] ### Context Project: [name] @@ -104,8 +104,15 @@ When the slice is complete, report status using one of these values: | `DONE` | All acceptance criteria met, tests pass, no concerns | Proceed to integration | | `DONE_WITH_CONCERNS` | Criteria met but there are trade-offs or risks worth noting | Orchestrator reviews concerns before integration | | `NEEDS_CONTEXT` | Blocked by missing information not in the brief | Orchestrator provides context or adjusts the brief | -| `BLOCKED` | Cannot proceed — dependency issue, tooling failure, or design conflict | Orchestrator investigates and unblocks | -| `DEVIATED` | Work cannot follow the strict slice packet exactly | Orchestrator applies the deviation rollback rule before continuing | +| `BLOCKED` | Cannot proceed — dependency issue, tooling failure, legacy blocker, or design conflict | Orchestrator classifies recovery and unblocks | +| `DEVIATED` | Work cannot follow the strict slice packet exactly | Orchestrator applies the deviation rollback rule and reapproval when scope/files/behavior/risk changes | + +Unexpected blockers must be classified. Use `blocker_type` values: +`legacy_code_bug`, `broken_baseline`, `hidden_dependency`, `missing_contract`, +`stale_plan`, `scope_conflict`, `tool_environment`, `permission_policy`, +`tdd_red_missing`, or `other`. Do not widen scope, patch around legacy blockers +blindly, or invent a new plan. Return evidence so the orchestrator can route to +debugging, explorer, architect, candidate search, replan, or restart. **Report format:** ```text @@ -118,6 +125,8 @@ When the slice is complete, report status using one of these values: - [trade-off or risk worth noting] ### Blocker (if NEEDS_CONTEXT or BLOCKED) +- blocker_type: [legacy_code_bug | broken_baseline | hidden_dependency | missing_contract | stale_plan | scope_conflict | tool_environment | permission_policy | tdd_red_missing | other] +- blocker_evidence: [file paths, missing fields, baseline failure, tool/environment symptom, or scope conflict] - What's needed: [specific missing information or resolution] ### Changes made diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md b/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md index 0fd1a82..6e30f6d 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md @@ -1,6 +1,6 @@ # Subagent Dispatch — Roles and Rules -Use specialized agents when the active tool policy and user authorization allow delegation. Each role has constrained access — a reviewer cannot edit files, a code writer doesn't run tests. When subagents are unavailable, denied, or policy-disallowed, keep the same role responsibilities as direct fallback evidence instead of pretending delegation happened. +Use specialized agents when the active tool policy and user authorization allow delegation. Each role has constrained access: code-reviewer, qa-evaluator, and reviewer cannot edit files, and code-writer does not run tests. When subagents are unavailable, denied, or policy-disallowed, keep the same role responsibilities as direct fallback evidence instead of pretending delegation happened. For full role prompts, read `references/subagent-roles.md`. @@ -25,7 +25,9 @@ Assistant Framework policy requires explicit user authorization before spawning | **Architect** | `architect` | `architect` | Read-only | Decompose, Plan, Design | | **Code Writer** | `code-writer` | `code-writer` | Write | Build | | **Builder/Tester** | `builder-tester` | `builder-tester` | Write | Build | -| **Reviewer** | `reviewer` | `reviewer` | Read-only | Review | +| **Code Reviewer** | `code-reviewer` | `code-reviewer` | Read-only | Review | +| **Reviewer** | `reviewer` | `reviewer` | Read-only | Review compatibility | +| **QA Evaluator** | `qa-evaluator` | `qa-evaluator` | Read-only | Review QA | ## What each role does @@ -34,7 +36,9 @@ Assistant Framework policy requires explicit user authorization before spawning - **Architect** — Designs implementation blueprints: files to create/modify, interfaces, data flows, build sequence, test plan. Does not write code. - **Code Writer** — Implements code following the plan. Does not run builds or tests. Does not review. Focuses purely on clean, convention-matching implementation. - **Builder/Tester** — Builds the project, writes tests, runs tests, absorbs noisy output. Returns concise results ("build passed, 2 tests failed: X, Y") not full logs. -- **Reviewer** — Independent code review with confidence-based filtering. Finds bugs, security issues, architecture violations. Does not edit files. +- **Code Reviewer** — Canonical independent code review with confidence-based filtering. Finds bugs, security issues, architecture violations, test coverage gaps, and structural code issues. Does not edit files. +- **Reviewer** — Compatibility route for existing handoffs that still say `Reviewer`; use only when `code-reviewer` is unavailable or a legacy prompt/handoff requires the old name. +- **QA Evaluator** — Independent QA acceptance evaluation after build/test and code-review evidence. Checks acceptance criteria, Done Contract, verification evidence, scoped UI/visual/product/UX/docs/DX/domain quality, score progression, and final result. Does not replace Code Reviewer. ## Phase-to-subagent requirements @@ -50,7 +54,8 @@ Every phase has a declared role responsibility. Phases without a subagent role a | **DESIGN** | Architect | UI tasks | Proposes design direction; Orchestrator creates mockup | | **BUILD** | Code Writer | All sizes | Implements code following the plan | | **BUILD** | Builder/Tester | All sizes | Builds, runs tests, returns concise results | -| **REVIEW** | Reviewer | All sizes | Independent review via `assistant-review` skill | +| **REVIEW** | Code Reviewer, or Reviewer compatibility | All sizes | Independent code review via `assistant-review` skill | +| **REVIEW** | QA Evaluator | Medium+ harness/domain-scored/UI/visual/product/UX/docs/DX-facing or explicitly requested QA | Independent acceptance QA via `assistant-review` QA loop | | **DOCUMENT** | — (Orchestrator direct) | All sizes | Documentation generation is orchestrator's synthesis work | **Rule:** If `subagent_execution_mode=delegated` and a phase's subagent column shows a dispatch, you MUST dispatch that role. If `subagent_execution_mode=direct_fallback`, you MUST NOT spawn subagents; instead record which role responsibility was handled directly and what equivalent evidence proves it. @@ -59,18 +64,20 @@ Every phase has a declared role responsibility. Phases without a subagent role a | Size | Agents used | Flow | |---|---|---| -| **Small** | Code Writer → Builder/Tester → Reviewer | Sequential, minimal (no Decompose) | -| **Medium** | Code Mapper → Architect (decompose) → Code Writer → Builder/Tester → Reviewer | Mapper feeds Architect, slices feed Writer | -| **Large** | Code Mapper → Explorer → Architect (decompose + plan) → Code Writer → Builder/Tester → Reviewer | Full pipeline with slice verification | -| **Mega** | All roles, parallel Code Writers per slice | Mapper → Explorer → Architect → parallel Writers → Builder/Tester and Reviewer at integration | +| **Small** | Code Writer → Builder/Tester → Code Reviewer | Sequential, minimal (no Decompose); Reviewer may be used only as compatibility; QA only when explicitly requested | +| **Medium** | Code Mapper → Architect (decompose) → Code Writer → Builder/Tester → Code Reviewer → QA Evaluator when required | Mapper feeds Architect, slices feed Writer; Reviewer may be used only as compatibility | +| **Large** | Code Mapper → Explorer → Architect (decompose + plan) → Code Writer → Builder/Tester → Code Reviewer → QA Evaluator when required | Full pipeline with slice verification; Reviewer may be used only as compatibility | +| **Mega** | All roles, parallel Code Writers per slice | Mapper → Explorer → Architect → parallel Writers → Builder/Tester, Code Reviewer, and QA Evaluator when required at integration | ## Dispatch guidelines -- **Every task gets at minimum**: Code Writer → Builder/Tester → Reviewer responsibilities. In delegated mode these are subagents; in direct fallback they are explicitly recorded role-equivalent steps. No code ships without build + test + review evidence. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts MUST infer required_agents with at least Code Writer, Builder/Tester, and Reviewer; `not_applicable` is invalid for source-changing Build tasks. -- **Strict evidence gate**: delegated mode is not complete until the task journal Agent Dispatch Log records Code Writer dispatch/result, Builder/Tester dispatch/result, and Reviewer dispatch/result evidence. Medium+ delegated slice work also records per-slice dispatch evidence before each slice is marked verified. **Codex adds a stronger check:** delegated Codex evidence must correspond to real `SubagentStart`/`SubagentStop` lifecycle records in `.codex/subagent-events.jsonl`; each dispatch/result entry must reference the matching lifecycle `agent_id`, and task-journal text alone is treated as insufficient because Codex can otherwise run phases inline and write claimed dispatch entries. Direct fallback is allowed only for explicit `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` reasons, and must record equivalent role, phase, verification, and review evidence; silent fallback fails the stop-review/phase gates. +- **Every task gets at minimum**: Code Writer → Builder/Tester → Code Reviewer responsibilities. `reviewer` remains valid compatibility routing for existing handoffs, but new dispatches should use `code-reviewer` for code defects, security, architecture, test coverage, and structural code issues. In delegated mode these are subagents; in direct fallback they are explicitly recorded role-equivalent steps. No code ships without build + test + review evidence. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts MUST infer required_agents with at least Code Writer, Builder/Tester, and Code Reviewer (or Reviewer compatibility); `not_applicable` is invalid for source-changing Build tasks. +- **Strict evidence gate**: delegated mode is not complete until the task journal Agent Dispatch Log records Code Writer dispatch/result, Builder/Tester dispatch/result, and Code Reviewer dispatch/result evidence, or Reviewer dispatch/result evidence when compatibility routing is used. Medium+ delegated slice work also records per-slice dispatch evidence before each slice is marked verified. **Codex adds a stronger check:** delegated Codex evidence must correspond to real `SubagentStart`/`SubagentStop` lifecycle records in `.codex/subagent-events.jsonl`; each dispatch/result entry must reference the matching lifecycle `agent_id`, and task-journal text alone is treated as insufficient because Codex can otherwise run phases inline and write claimed dispatch entries. Direct fallback is allowed only for explicit `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` reasons, and must record equivalent role, phase, verification, and review evidence; silent fallback fails the stop-review/phase gates. +- **QA evidence gate**: when QA is required, delegated mode is not complete until the task journal Agent Dispatch Log records QA Evaluator dispatch/result evidence after Builder/Tester and Code Reviewer evidence. Direct fallback must record fresh QA Evaluator direct evidence separately from Code Reviewer direct evidence. - **Every medium+ task gets Architect decomposition responsibility**: In delegated mode the Architect proposes smallest iterable slice boundaries; in direct fallback the same criteria and evidence are recorded directly. - **Launch in parallel** when agents are independent (e.g., Code Mapper + Explorer on different modules) - **Code Mapper runs first** on medium+ tasks — its output feeds into Architect and Code Writer -- **Reviewer gets a fresh dispatch each round** during the quality review loop when delegated mode is authorized; direct fallback must reset review context and record how stale-context risk was controlled +- **Code Reviewer gets a fresh dispatch each round** during the quality review loop when delegated mode is authorized; `reviewer` is the compatibility route for older handoffs. Direct fallback must reset review context and record how stale-context risk was controlled. +- **QA Evaluator gets a fresh dispatch each QA round** when QA is required and delegated mode is authorized. Direct fallback must reset acceptance-evaluation context and record how stale-context risk was controlled. - **Main session stays Orchestrator**: owns user communication, final integration, handoffs - **Do not dispatch agents for small inline tasks** within a larger workflow (e.g., a one-line fix during review doesn't need a Code Writer agent) diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/subagent-roles.md b/plugins/assistant-dev/skills/assistant-workflow/references/subagent-roles.md index ab9e43a..fae490f 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/subagent-roles.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/subagent-roles.md @@ -29,7 +29,9 @@ When custom agents are installed, dispatch by name. The agent's own configuratio Claude exposes the `Agent` tool with `subagent_type` values matching installed agent names: ``` -Agent(subagent_type="reviewer", prompt="Review the changes in {files}. This is round {N}...") +Agent(subagent_type="code-reviewer", prompt="Review the changes in {files}. This is round {N}...") +Agent(subagent_type="reviewer", prompt="Compatibility route: review the changes in {files}. This is round {N}...") +Agent(subagent_type="qa-evaluator", prompt="Evaluate acceptance for {task}. This is QA round {N}...") Agent(subagent_type="explorer", prompt="Trace execution paths for {feature}...") Agent(subagent_type="architect", prompt="Design implementation for {feature}...") Agent(subagent_type="code-mapper", prompt="Map the structure around {area}...") @@ -42,7 +44,9 @@ Agent(subagent_type="builder-tester", prompt="Build and test the project...") Current Codex CLI/app releases support native subagent workflows by default. Codex custom agents are standalone TOML files under `~/.codex/agents/` for personal agents or `.codex/agents/` for project-scoped agents. Ask Codex to spawn the configured agent by name; do not look for a visible Claude-style `Agent` tool or `subagent_type` parameter before deciding delegation is available: ``` -Spawn the reviewer agent to review the changes in {files}. This is round {N}... +Spawn the code-reviewer agent to review the changes in {files}. This is round {N}... +Spawn the reviewer agent only as compatibility routing for existing handoffs... +Spawn the qa-evaluator agent to evaluate acceptance for {task}. This is QA round {N}... Spawn the explorer agent to trace execution paths for {feature}... Spawn the architect agent to design implementation for {feature}... Spawn the code-mapper agent to map the structure around {area}... @@ -63,22 +67,24 @@ The prompt you provide is the **task context** — what to do, not how to do it. | `architect` | Strongest / deep reasoning | Read-only | Decompose, Plan, Design | Strict slice decomposition, implementation blueprints, design direction | | `code-writer` | Strongest / deep reasoning | Write | Build | Implements code following a plan. No builds, no tests, no review | | `builder-tester` | Balanced / standard | Write | Build | Builds, writes tests, runs tests. Returns concise summaries, not logs | -| `reviewer` | Strongest / deep reasoning | Read-only | Review | Finds bugs, security issues, architecture violations, structural problems | +| `code-reviewer` | Strongest / deep reasoning | Read-only | Review | Canonical code review for bugs, security issues, architecture violations, test coverage gaps, and structural problems | +| `reviewer` | Strongest / deep reasoning | Read-only | Review compatibility | Compatibility route for existing reviewer handoffs; prefer `code-reviewer` for new code review dispatches | +| `qa-evaluator` | Strongest / deep reasoning | Read-only | Review QA | Independent acceptance, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result evaluation | ## Dispatch rules by task size | Size | Agents used | Flow | |---|---|---| -| **Small** | Code Writer → Builder/Tester → Reviewer | Sequential, minimal (no Decompose) | -| **Medium** | Code Mapper → Architect (decompose) → Code Writer → Builder/Tester → Reviewer | Mapper feeds Architect, slices feed Writer | -| **Large** | Code Mapper → Explorer → Architect (decompose + plan) → Code Writer → Builder/Tester → Reviewer | Full pipeline with slice verification | -| **Mega** | All roles, parallel Code Writers per slice | Mapper → Explorer → Architect → parallel Writers → Builder/Tester and Reviewer at integration | +| **Small** | Code Writer → Builder/Tester → Code Reviewer | Sequential, minimal (no Decompose); Reviewer is compatibility routing; QA only when explicitly requested | +| **Medium** | Code Mapper → Architect (decompose) → Code Writer → Builder/Tester → Code Reviewer → QA Evaluator when required | Mapper feeds Architect, slices feed Writer; Reviewer is compatibility routing | +| **Large** | Code Mapper → Explorer → Architect (decompose + plan) → Code Writer → Builder/Tester → Code Reviewer → QA Evaluator when required | Full pipeline with slice verification; Reviewer is compatibility routing | +| **Mega** | All roles, parallel Code Writers per slice | Mapper → Explorer → Architect → parallel Writers → Builder/Tester, Code Reviewer, and QA Evaluator when required at integration | ## Reviewer dispatch (review rounds) **Round 1:** ``` -Use the reviewer agent to review all code changes. Find real issues, not nitpicks. +Use the code-reviewer agent to review all code changes. Find real issues, not nitpicks. Review for: bugs, logic errors, edge cases, security vulnerabilities, architecture violations, code quality, test coverage, and structural organization. Report at 80%+ confidence only. @@ -86,11 +92,37 @@ Report at 80%+ confidence only. **Round N > 1:** ``` -Use the reviewer agent. This is review round {N}. +Use the code-reviewer agent. This is review round {N}. The following {count} items were already found and fixed — do NOT re-report them: {previously_fixed list} Focus ONLY on NEW high-confidence findings. -Confidence threshold: Round 1-3: 80%+ | Round 4-7: 85%+ | Round 8-10: 90%+ +Confidence threshold: Round 1-3: 80%+ | Round 4-15: 85%+ | Round 16-20: 90%+ +``` + +## QA Evaluator dispatch (QA rounds) + +Use QA Evaluator only after build/test evidence and Code Reviewer or Reviewer compatibility result exist. It evaluates acceptance, not general code defects. + +**QA Round 1:** +``` +Use the qa-evaluator agent to evaluate acceptance for {task}. +Review: Done Contract, acceptance criteria, verification evidence, code review result, +domain_context/rubric_refs when applicable, and final result requirements. +Load assistant-review references/domain-rubrics.md only when the acceptance +criteria, Done Contract, domain_context, or explicit rubric_refs scope subjective, +product, UX, docs, DX, UI/visual, or domain craft scoring. Return selected_domain_rubrics +and domain_quality_scores when scoped; do not invent rubrics when not scoped. +Do not replace Code Reviewer. Report only acceptance findings backed by evidence. +``` + +**QA Round N > 1:** +``` +Use the qa-evaluator agent. This is QA round {N} of 20. +Previously failed acceptance items: +{previously_failed_acceptance_items} +Do not re-report resolved items. In rounds 16-20, only unresolved acceptance +blockers or high-confidence acceptance risks keep the QA loop open. +Round 20 is terminal; return the final verdict instead of implying round 21. ``` ## Fallback: agents not installed @@ -104,6 +136,8 @@ If custom agents are not installed, use the nearest available built-in subagent | Architect | `general-purpose` or nearest planning agent | Strongest / deep reasoning | | Code Writer | `general-purpose` or nearest coding agent | Strongest / deep reasoning | | Builder/Tester | `general-purpose` or nearest coding agent with shell/test access | Balanced / standard | -| Reviewer | `general-purpose` or nearest read-only reviewer | Strongest / deep reasoning | +| Code Reviewer | `general-purpose` or nearest read-only reviewer | Strongest / deep reasoning | +| Reviewer compatibility | `general-purpose` or nearest read-only reviewer | Strongest / deep reasoning | +| QA Evaluator | `general-purpose` or nearest read-only evaluator | Strongest / deep reasoning | When using fallback types, include the full role instructions in the dispatch prompt (the agent won't have a built-in system prompt for the role). Prefer the matching role definition for the active runtime under the framework's `agents/` directory when available; otherwise adapt the closest provider-neutral role prompt. diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md index faf878d..ded6a25 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md @@ -10,6 +10,9 @@ Write to `{agent_state_dir}/task.md` in the project root when a local agent stat ## When to update - When clarification questions are asked, answered, or resolved via explicit `defaults` - After each Build step completes (update Progress, Artifact Registry, and check off Milestones) +- After each medium+ harness event that changes phase, slice, status, blockers, verification, next action, decisions, deviations, or artifact refs +- After any Pivot/Restart Decision, including review/QA stagnation, repeated drift/regression, Code Writer blockers, or selected restart/replan action +- After any producer creates/updates a typed artifact reference or any consumer changes `validation_status` - When key decisions are made - When constraints are added by the user - After each review cycle pass (append to Review Log — never overwrite) @@ -51,7 +54,7 @@ Plan approval: [yes/no + date] ## Agent Dispatch Log [strict subagent evidence inspected by stop-review/phase gates] -- Required roles: Code Writer, Builder/Tester, Reviewer [plus Code Mapper/Explorer/Architect when required by size/risk] +- Required roles: Code Writer, Builder/Tester, Code Reviewer [Reviewer remains compatibility routing; plus QA Evaluator when QA is required; plus Code Mapper/Explorer/Architect when required by size/risk] - Execution mode: delegated | direct_fallback | not_applicable - Codex lifecycle evidence: delegated Codex roles must have matching `.codex/subagent-events.jsonl` `SubagentStart` + `SubagentStop` records; dispatch/result entries must cite the matching `agent_id`; journal text alone is not proof. - Direct fallback reason: [authorization_denied | subagents_unavailable | policy_disallowed | N/A] @@ -70,9 +73,15 @@ Plan approval: [yes/no + date] - Builder/Tester dispatch: [subagent/tool/run id, verification packet, timestamp, or N/A only in direct_fallback/not required] - Builder/Tester result: [commands/results/success signal returned, or N/A only in direct_fallback/not required] - Builder/Tester direct evidence: [role-equivalent build/test/validation evidence when direct_fallback, else N/A] -- Reviewer dispatch: [reviewer subagent/tool/run id or assistant-review dispatch, or N/A only in direct_fallback/not required] -- Reviewer result: [spec/quality review result evidence returned, or N/A only in direct_fallback/not required] -- Reviewer direct evidence: [fresh review/spec+quality evidence when direct_fallback, else N/A] +- Code Reviewer dispatch: [code-reviewer subagent/tool/run id or assistant-review dispatch, or N/A only when Reviewer compatibility/direct_fallback/not required] +- Code Reviewer result: [spec/quality review result evidence returned, or N/A only when Reviewer compatibility/direct_fallback/not required] +- Code Reviewer direct evidence: [fresh review/spec+quality evidence when direct_fallback, else N/A] +- Reviewer dispatch: [compatibility reviewer subagent/tool/run id or assistant-review dispatch, or N/A only in direct_fallback/not required] +- Reviewer result: [compatibility spec/quality review result evidence returned, or N/A only in direct_fallback/not required] +- Reviewer direct evidence: [compatibility fresh review/spec+quality evidence when direct_fallback, else N/A] +- QA Evaluator dispatch: [qa-evaluator subagent/tool/run id or assistant-review QA dispatch, or N/A only when qa_evaluation_mode=not_required/direct_fallback] +- QA Evaluator result: [acceptance final_verdict/result, score_progression, and evidence returned, or N/A only when qa_evaluation_mode=not_required/direct_fallback] +- QA Evaluator direct evidence: [fresh acceptance/Done Contract/verification QA evidence when direct_fallback, else N/A] - Per-slice dispatch evidence: [medium+ delegated only: slice_id -> Code Writer dispatch/result + Builder/Tester dispatch/result; direct_fallback records sequential role-equivalent slice evidence] ## Constraints @@ -92,6 +101,101 @@ Plan approval: [yes/no + date] |------|---------|-----------| | [path] | [what and why] | Step N | +## Done Contract +[required before Build for medium+ harness-capable work; otherwise record N/A with reason] +- done_when: + - [binary outcome that proves done] +- not_done_when: + - [failure state that blocks done] +- verification: + - [command, inspection, review, or manual check] +- owner_consumer: [owner and downstream consumer] +- acceptance_criteria: + - [explicit binary criterion] +- debate_record: + - perspective: [role/subagent/direct perspective 1] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] + - perspective: [role/subagent/direct perspective 2] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] +- accepted_by: [user/orchestrator/approved plan reference] + +## Harness Recipe +[required before Build for medium+ harness-capable work; otherwise record N/A with reason] +- task_profile: [task type, size, slice count, TDD/debugging applicability] +- model_profile: [agent/model constraints, delegation mode, tool limits] +- risk_profile: [risk tier, safety gates, review depth, rollback needs] +- context_profile: [exact/summarized/omitted context and trace/replay needs] +- selected_recipe: [concise recipe label] +- recipe_rationale: [why this task/model/risk/context profile selects the recipe] +- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] +- corrective_action: [what to do if missing or stale] + +## Harness Run State +[required for medium+ harness-capable work; otherwise record N/A with reason] +- task_id: [stable task/run id] +- task_name: [human-readable task name] +- phase: [TRIAGE | DISCOVER | DECOMPOSE | PLAN | DESIGN | BUILD | REVIEW | DOCUMENT | COMPLETE] +- slice: [current slice id/name, next pending slice, or N/A] +- status: [not_started | in_progress | blocked | verifying | reviewing | restarting | documenting | complete] +- blockers: + - [current blocker, or none] +- last_verification: + - command_or_check: [command/check, or pending/N/A] + - result: [passed | failed | skipped | not_applicable | pending] + - evidence: [concise evidence or reason] +- next_action: [exact next action] +- recovery_pointer: [task packet, trace entry, ledger row, file section, or artifact ref] +- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] + +## Trace Ledger +[required for medium+ harness-capable work; append-only ordered execution evidence] +| Seq | Timestamp/Order | Event Type | Actor | Summary | Artifact Refs | +|-----|-----------------|------------|-------|---------|---------------| +| 1 | [timestamp or ordered marker] | [agent_event/decision/verification/plan_deviation/pivot_restart/artifact_ref/blocker/recovery] | [role/subagent/user/hook] | [event, decision, command/result, deviation, blocker, pivot/restart, or recovery] | [file/section/dispatch id/command ref] | + +## Replay Packet +[required for medium+ harness-capable work before compaction, failure handoff, phase handoff, or end-of-turn] +- pinned_context: + - [stable requirement, constraint, approved plan/slice id, or non-goal] +- artifact_refs: + - [task journal/context map/plan/run state/trace/validation/changed file ref] +- validation_state: + - completed_checks: [checks already completed] + - pending_checks: [checks still pending] + - last_result: [latest verification/review result] +- exact_next_action: [single concrete next action after replay] +- run_state_ref: [Harness Run State section/ref] +- trace_ledger_ref: [Trace Ledger section/ref] +- recovery_pointer: [where to resume] +- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] + +## Pivot/Restart Log +[append when review/QA stagnation, repeated drift/regression, rubric/domain pivot, Code Writer blocker, verification blocker, plan deviation, or scope change triggers recovery; otherwise record N/A with reason] + +### Pivot/Restart Decision #N +- trigger: [STAGNATION | repeated_DRIFT | repeated_REGRESSION | pivot | code_writer_blocker | verification_blocker | plan_deviation | scope_change] +- evidence: + - source: [review score entry, QA score entry, Code Writer blocker, verification failure, trace row, or task packet] + detail: [concise evidence] +- affected_slice_or_round: [slice id/name, review round, QA round, or phase] +- options_considered: + - option: [reset context / dispatch debugging / dispatch explorer / dispatch architect / candidate search / replan / restart slice / restart phase / block for user / accept with limitations] + tradeoff: [why this option helps or risks the task] + disposition: [selected | rejected | deferred] +- selected_action: [reset_context | return_to_build | dispatch_debugging | dispatch_explorer | dispatch_architect | run_candidate_search | replan | restart_slice | restart_phase | block_for_user | accept_with_limitations] +- reapproval_required: [true when scope/files/behavior/risk/verification/acceptance changes; otherwise false] +- next_agent: [Builder/Tester | Code Writer | Explorer | Architect | Reviewer | QAEvaluator | assistant-debugging | candidate-search | user | none] +- recovery_pointer: [task packet, trace row, replay packet, plan section, or file path] +- exact_next_action: [single concrete action after the decision] + +## Artifact Reference Ledger +[required for medium+ harness-capable work when artifacts pass between agents; typed producer/consumer records, not ad hoc strings] +| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | +|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| +| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/pivot_restart_decision/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role/subagent/hook] | [role/subagent/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | + ## Milestones [compression-safe boundaries — each marks a point where context can be safely truncated] - [ ] M1: [milestone description] (after Step N) @@ -150,13 +254,14 @@ Plan approval: [yes/no + date] - Required fixes: [none, or ordered fix list] ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Previously fixed: 0 items from prior rounds - Found this round: [count] must-fix, [count] should-fix, [count] nits (all fixed below) - Rubric: correctness=[score] quality=[score] architecture=[score] security=[score] coverage=[score] - Weighted: [score] - Delta from previous: — (first round) - Drift check: — (first round) +- Pivot/Restart decision: [ref if STAGNATION, repeated DRIFT, repeated REGRESSION, or PIVOT occurred; otherwise N/A] - Complexity: [ran / skipped (not C#) / tool unavailable] - [method (line N): score X — refactored to Y, or "within threshold"] - Must-fix: @@ -166,13 +271,14 @@ Plan approval: [yes/no + date] - Re-test: PASS ### Quality Review #2 (autonomous re-review) -- Round: 2 of 10 +- Round: 2 of 20 - Previously fixed: [count] items from prior rounds - Found this round: [count] must-fix, [count] should-fix, [count] nits (all fixed below) - Rubric: correctness=[score] quality=[score] architecture=[score] security=[score] coverage=[score] - Weighted: [score] - Delta from previous: [+/- amount] - Drift check: [GENUINE / SUSPICIOUS / DRIFT / REGRESSION / STAGNATION / NEUTRAL] +- Pivot/Restart decision: [ref if STAGNATION, repeated DRIFT, repeated REGRESSION, or PIVOT occurred; otherwise N/A] - Must-fix: - [x] [file:line] — [issue] → [fix applied] - Should-fix: @@ -182,12 +288,36 @@ Plan approval: [yes/no + date] [...repeat until clean or max rounds reached...] [Note: On test failure, skip this entry — write only "- Result: HAS_REMAINING_ITEMS" to Final result] +### QA Evaluation #1 +- Round: 1 of 20 +- Mode: required | optional | not_required +- Done Contract: [ref or N/A with reason] +- Acceptance criteria checked: [count] +- Verification evidence checked: [commands/checks/manual/review refs] +- Code Review evidence: [Code Reviewer/Reviewer result ref] +- Domain/rubric refs: [product/UX/UI/docs/DX/domain refs, or N/A when not scoped] +- Selected domain rubrics: [ui_visual_design | ux_product_acceptance | documentation_quality | developer_experience | domain_specific_craft | N/A] +- Domain quality scores: [rubric.dimension=score/action/evidence, or N/A when not scoped] +- QA scorecard: acceptance_coverage=[score] evidence_strength=[score] domain_quality=[score] final_readiness=[score] +- Weighted: [score] +- Score progression: [round1->...] +- Pivot/Restart decision: [ref if STAGNATION, repeated DRIFT, repeated REGRESSION, or domain action pivot occurred; otherwise N/A] +- Final verdict: accepted | accepted_with_concerns | rejected | blocked +- Acceptance findings: + - [blocker|concern|observation] [criterion] — [evidence] → [fix/defer/open question] + +[...repeat QA Evaluation until accepted, accepted_with_concerns, blocked, or max round 20 reached...] + ### Final result - Result: CLEAN | ISSUES_FIXED | HAS_REMAINING_ITEMS - Review rounds: [count] +- QA result: [accepted | accepted_with_concerns | rejected | blocked | not_required] +- QA rounds: [count or N/A] +- QA score progression: [round1->round2->...roundN or N/A] - Final rubric score: [weighted score] ([PASS/REFINE/PIVOT]) - Score progression: [round1→round2→...roundN] (e.g., 3.50→3.85→4.10) - Drift incidents: [count, or "none"] +- Pivot/Restart decisions: [count and refs, or "none"] - Total must-fix resolved: [count across all rounds] - Total should-fix resolved: [count across all rounds] - Should-fix deferred: [list any remaining] @@ -222,8 +352,8 @@ Plan approval: [yes/no + date] 3. **Clarification** updates — question caps are maximums, not quotas. Clear medium+ tasks may record `Clarification questions asked: 0` with `Clarification confidence: high`. While waiting, keep `Status: DISCOVERING`, set `Clarification status: needs_clarification`, set `Clarification defaults applied: false`, set confidence/cap/admissibility fields, and list every unresolved implementation-shaping topic. On explicit answers, clear unresolved topics, keep `Clarification defaults applied: false`, and set `Clarification status: ready`. On explicit `defaults`, print the applied defaults, clear unresolved topics, set `Clarification defaults applied: true`, and set `Clarification status: ready`. 4. **Decompose** — medium+ tasks set `Status: DECOMPOSING` after Discover is ready, then persist the slice manifest before moving on to planning. Small tasks skip this state. 5. **Plan approval** — once ready to plan, set `Status: PLANNING`, include the slice manifest in the plan for medium+ tasks, capture the approved plan, and update `Plan approval`. -6. **Build** each step — update Progress, Artifact Registry, Key Decisions, Status after each step. For medium+ tasks, update the Slice Verification Ledger after each slice and do not start the next slice until the current one is `VERIFIED`. Check off Milestones when reached. -7. **Review cycle** when all steps done — Spec Review first (structured PASS/FAIL from `references/prompts/spec-review.md`), then Quality Review (assistant-review quality loop), fix must-fix → re-test → re-review until clean, fill Final Result +6. **Build** each step — update Progress, Artifact Registry, Artifact Reference Ledger, Key Decisions, Status, Harness Run State, Trace Ledger, Replay Packet, and Pivot/Restart Log when triggered after each step. For medium+ tasks, update the Slice Verification Ledger after each slice and do not start the next slice until the current one is `VERIFIED`. Check off Milestones when reached. +7. **Review cycle** when all steps done — Spec Review first (structured PASS/FAIL from `references/prompts/spec-review.md`), then Quality Review (assistant-review quality loop), fix must-fix → re-test → re-review until clean or a Pivot/Restart Decision routes recovery, fill Final Result 8. **Document** after review cycle passes — fill Verification Summary, add the Learning Controller block, Status: DOCUMENTING 9. **Handoff** to user — they test manually and add Review Notes 10. **Review fixes** — fix issues, re-test, re-review, update Progress @@ -234,6 +364,8 @@ Plan approval: [yes/no + date] - Keep entries concise — this is a log, not documentation - Resume from clarification waits only on explicit numbered answers or explicit `defaults` - Constraints are checked before each Build step +- Producer roles update Artifact Reference Ledger entries when they create or move artifacts; Consumer roles validate `schema_or_contract` and update `validation_status` before using them +- Pivot/Restart Decisions are append-only recovery records. If the selected action changes scope, files, behavior, risk, verification, or acceptance criteria, record `reapproval_required: true` and wait for approval before continuing. - On context continuation: read the configured task journal FIRST when it exists, before any other action - Never delete constraints unless the user explicitly removes them -- The Verification Summary replaces the need for context-handoff templates during active work +- The Replay Packet replaces the need for context-handoff templates during active harness work diff --git a/skills/assistant-review/SKILL.md b/skills/assistant-review/SKILL.md index d8ad3c0..a773561 100644 --- a/skills/assistant-review/SKILL.md +++ b/skills/assistant-review/SKILL.md @@ -1,6 +1,6 @@ --- name: assistant-review -description: "This skill runs an autonomous code review loop: review, fix, re-review until clean (max 10 rounds). Use when the user says 'review', 'fresh review', 'code review', 'review this', 'check the code'. Also activates when the workflow's Review phase requires quality review dispatch." +description: "This skill runs an autonomous code review loop and optional independent QA evaluation loop: review, fix, re-review until clean (max 20 rounds), then evaluate acceptance when QA is required. Use when the user says 'review', 'fresh review', 'code review', 'review this', 'check the code'. Also activates when the workflow's Review phase requires quality review or QA evaluation dispatch." effort: high triggers: - pattern: "fix (all |the |review |reported )?issues|fix (all |the )?findings|apply (all )?fixes" @@ -11,31 +11,31 @@ triggers: reminder: "This request matches assistant-review. You MUST load and follow this SKILL.md and its contracts before doing anything else. Run the autonomous review-fix loop to its exit condition before reporting." --- -# Autonomous Review Loop +# Autonomous Review And QA Evaluation ## Contracts -This skill enforces strict contracts on inputs, outputs, loop gates, and reviewer handoffs. Read the contract files in `contracts/` before executing. +This skill enforces strict contracts on inputs, outputs, loop gates, reviewer handoffs, and QA evaluator handoffs. Read the contract files in `contracts/` before executing. | Contract | File | Purpose | |---|---|---| | **Input** | `contracts/input.yaml` | Scope, mode, and review material snapshot to resolve before entering the loop | | **Output** | `contracts/output.yaml` | Final summary and verification artifacts | | **Phase Gates** | `contracts/phase-gates.yaml` | Per-round step assertions and loop invariants | -| **Handoffs** | `contracts/handoffs.yaml` | Reviewer subagent dispatch and return schema | +| **Handoffs** | `contracts/handoffs.yaml` | Reviewer and QAEvaluator subagent dispatch and return schemas | **Rules:** - Resolve all input contract fields before entering the loop - Check phase gate assertions at every step transition within each round -- Include all required context fields when dispatching Reviewer subagents, and record direct-fallback review evidence when subagents are not authorized/available/allowed -- Validate all required return fields when Reviewer completes +- Include all required context fields when dispatching Reviewer or QAEvaluator subagents, and record direct-fallback evidence when subagents are not authorized/available/allowed +- Validate all required return fields when Reviewer or QAEvaluator completes - Verify all output contract artifacts before presenting the final summary -Run this loop autonomously from start to finish. Continue rounds until clean or max rounds reached, keep intermediate results inside the loop, and present one final result after exit. +Run the code review loop autonomously from start to finish. Continue rounds until clean or max rounds reached, keep intermediate results inside the loop, and present one final result after exit. When QA is required, run the QA evaluation loop after build/test evidence and code-review evidence are available. ## Goal -Find concrete defects, risks, regressions, and test gaps; fix them when in review-fix mode; and return one evidence-backed final review result. Reviews must be useful in company environments: local-first, policy-safe, and focused on actionable engineering risk rather than generic style preferences. +Find concrete defects, risks, regressions, and test gaps; fix them when in review-fix mode; and return one evidence-backed final review result. When QA is required, independently evaluate the Done Contract, acceptance criteria, verification evidence, scoped UI/visual/product/UX/docs/DX/domain quality, score progression, and final acceptance result. Reviews must be useful in company environments: local-first, policy-safe, and focused on actionable engineering risk or acceptance evidence rather than generic style preferences. ## Success Criteria @@ -44,6 +44,8 @@ Find concrete defects, risks, regressions, and test gaps; fix them when in revie - Every review applies the SOLID, KISS, DRY, YAGNI, and readability lens from `references/review-principles.md`. - In review-fix mode, must-fix and should-fix findings are addressed or explicitly deferred. - Validation runs after fixes, and a fresh review confirms the final state. +- QA evaluation runs after code-review/build evidence when `qa_evaluation_mode=required`, returns score progression and a final acceptance verdict, and does not replace code-reviewer. +- QA evaluation loads `references/domain-rubrics.md` only when `domain_context`, explicit `rubric_refs`, or subjective/UI/visual/product/UX/docs/DX/domain acceptance criteria require scoped domain-quality scoring. ## Constraints @@ -51,6 +53,7 @@ Find concrete defects, risks, regressions, and test gaps; fix them when in revie - Do not emit intermediate review summaries; present one final summary after loop exit. - Use concrete risk categories for refactor-related findings. - Treat clean-code principles as evidence lenses, not acronym-driven style rules. +- Keep QA evaluation separate from code review: QA focuses on acceptance criteria, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result. Code Reviewer continues to own code defects, security, architecture, and test-coverage review. ## Entry @@ -75,6 +78,8 @@ Use the smallest mode that answers the request; combine modes when reviewing imp - **Semantic contract review**: for skill/workflow/framework changes, check contract inheritance, method-template alignment, eval coverage, method-signature fidelity, and high-stakes recommendation guards before judging the change clean. - **Maintainability review**: apply SOLID/KISS/DRY/YAGNI/readability only when a concrete risk exists. - **Security handoff**: invoke `assistant-security` when the reviewed surface touches auth, permissions, secrets, input handling, persistence, shell commands, dependency/config changes, network calls, or external integrations. +- **QA evaluation**: after code-review/build evidence exists, dispatch QAEvaluator when `qa_evaluation_mode=required` for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work. Load `references/qa-evaluation-loop.md`. +- **Domain rubric QA**: within QA evaluation, load `references/domain-rubrics.md` only when scoped by acceptance criteria, Done Contract, `domain_context`, or explicit `rubric_refs`. QAEvaluator selects rubric families and returns domain-quality scores; Code Reviewer still owns code defects, security, architecture, and test coverage. Finding format: @@ -128,14 +133,14 @@ Load `references/review-principles.md` before each REVIEW step. Apply SOLID, KIS For each principle/readability finding, include the violated lens, affected surface, concrete evidence, risk category, and smallest durable fix. Do not report acronym-only findings such as "violates SOLID" without naming the observed behavior and the user-facing or maintainer risk. -## The Loop +## The Code Review Loop ``` round = 1 previously_fixed = [] score_history = [] -while round <= 10: +while round <= 20: 1. REVIEW - Dispatch a fresh Reviewer subagent when `subagent_execution_mode=delegated` @@ -160,7 +165,7 @@ while round <= 10: Finding filter policy: - Report only evidence-backed findings with file/line evidence, concrete impact, and the smallest useful fix. - Put speculative or low-evidence concerns in Observations; they do not block completion. - - In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers; round 10 is terminal and exits with remaining items instead of starting round 11. + - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers; round 20 is terminal and exits with remaining items instead of starting round 21. Apply the SOLID, KISS, DRY, YAGNI, and readability lens from references/review-principles.md. Report principle issues only when tied to concrete evidence, concrete risk, and the smallest durable fix. @@ -180,7 +185,7 @@ while round <= 10: 2. EVALUATE a. Check rubric score (medium+ scope): - PASS (weighted >= 4.0) AND no must-fix AND no should-fix -> EXIT CLEAN - - PIVOT (weighted < threshold for round) -> escalate to orchestrator + - PIVOT (weighted < threshold for round) -> escalate to orchestrator-owned pivot_restart_decision - REFINE (weighted below 4.0 but not PIVOT), including zero findings -> continue to step 3 using lowest-scoring rubric dimensions as the improvement targets - Medium+ CLEAN and ISSUES_FIXED require weighted >= 4.0 and zero @@ -188,12 +193,20 @@ while round <= 10: b. No rubric (small scope): use findings-based exit: - No must-fix AND no should-fix -> EXIT CLEAN - Only nits -> EXIT CLEAN (note nits in final report) - c. round == 10 with remaining must-fix or should-fix -> EXIT WITH REMAINING ITEMS - d. round == 10 with issues fixed and now clean -> EXIT ISSUES FIXED + c. round == 20 with remaining must-fix or should-fix -> EXIT WITH REMAINING ITEMS + d. round == 20 with issues fixed and now clean -> EXIT ISSUES FIXED e. Otherwise -> continue to step 3 Record in score_history: { round, weighted_score, finding_count, drift_status } (see references/score-tracking.md for drift detection rules) + If score tracking reports STAGNATION, repeated DRIFT, repeated REGRESSION, + or rubric action PIVOT, pause the loop and return a pivot_restart_signal to + the orchestrator. The orchestrator records pivot_restart_decision with + trigger, evidence, affected_slice_or_round, options_considered, + selected_action, reapproval_required, next_agent, recovery_pointer, and + exact_next_action before another fix/review dispatch. If the selected action + changes scope, files, behavior, risk, verification, or acceptance criteria, + reapproval is required. Round 20 remains terminal and never starts round 21. 3. FIX - Fix ALL must-fix and should-fix items (not just must-fix) @@ -208,6 +221,26 @@ while round <= 10: 5. round += 1 -> go to step 1 ``` +## The QA Evaluation Loop + +Run this loop only when `qa_evaluation_mode=required` or the workflow Review phase requests QA evidence. Load `references/qa-evaluation-loop.md` before dispatching QAEvaluator. + +QA evaluation runs after code-review/build evidence. It is not an alternate path around the Code Reviewer loop. + +QA runs after build/test evidence and the Code Reviewer loop result exist. It must receive the Done Contract, acceptance criteria, verification evidence, domain_context/rubric_refs when applicable, round number, previously_failed_acceptance_items, and qa_filter_policy. It conditionally loads `references/domain-rubrics.md` only for scoped subjective/UI/visual/product/UX/docs/DX/domain acceptance. It returns status, round, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when used, final_verdict/result, evidence, score_progression or score_entry, and open_questions when blocked. + +The QA loop supports rounds 1-20. In rounds 16-20, only unresolved acceptance blockers or high-confidence acceptance risks keep the loop open. Round 20 is terminal: return the final verdict and remaining failed acceptance items instead of starting round 21. + +QA findings do not replace code-review findings. Do not use QAEvaluator to review general bugs, security, architecture, or test coverage unless that issue directly blocks an acceptance criterion or Done Contract item. + +If QA score progression reports STAGNATION, repeated DRIFT, repeated REGRESSION, +or a scoped domain rubric returns action `pivot`, pause QA and return a +pivot_restart_signal to the orchestrator. The orchestrator records +pivot_restart_decision before another QA/build dispatch, updates trace/replay/run +state when the workflow is harness-capable, and obtains reapproval when the +selected restart or pivot changes scope, files, behavior, risk, verification, or +acceptance criteria. + ## Exit: Present Final Result After the loop completes, present ONE summary to the user: @@ -250,7 +283,7 @@ After the loop completes, present ONE summary to the user: - **Autonomous continuation**: advance to the next round while exit criteria are unmet. - **Fresh Reviewer each round** on medium+ scope: stale context weakens reviews. - **Previously-fixed list prevents re-reporting**: each round should find fewer issues. -- **Evidence-backed filtering**: findings need file/line evidence, concrete impact, and the smallest useful fix; late rounds only stay open before the hard max for must-fix or high-confidence should-fix findings, and round 10 is terminal. +- **Evidence-backed filtering**: findings need file/line evidence, concrete impact, and the smallest useful fix; late rounds only stay open before the hard max for must-fix or high-confidence should-fix findings, and round 20 is terminal. - If scope is trivial (single small file, obvious change) -> one clean round can exit. If findings exist, continue looping. ## Output @@ -264,6 +297,7 @@ Return: - **Agentic loop safety review** - when applicable, whether bounds, stop conditions, retry/empty handling, tool-error routing, progress checks, and cost/token guards were checked. - **Behavioral contract review** - when applicable, whether existing invariants, interface alignment, inherited tests, protocol fidelity, high-impact guards, and runtime surfaces were checked. - **Semantic contract review** - when applicable, whether inherited contracts, method signatures, eval coverage, high-stakes guards, and mirror surfaces were checked. +- **QA evaluation result** - when applicable, final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression, and blocked open questions. - **Residual risk** - remaining items, nits, or scope gaps. ## Stop Rules @@ -279,15 +313,18 @@ After each round, compare rubric scores to the previous round per `references/sc - **GENUINE**: Score up, findings down -> continue normally - **SUSPICIOUS**: Score jumped > 1.0 in one round -> log warning, continue - **DRIFT**: Score up but findings didn't decrease -> **reset evaluator** (fresh agent, stricter prompt) -- **REGRESSION**: Score down -> investigate, escalate if 2+ consecutive rounds +- **REGRESSION**: Score down -> investigate; 2+ consecutive regressions trigger pivot_restart_signal - **NEUTRAL**: Score unchanged for 1 round with findings present -> log, no action yet -- **STAGNATION**: Score unchanged for 2+ rounds with findings present -> escalate to orchestrator +- **STAGNATION**: Score unchanged for 2+ rounds with findings present -> return pivot_restart_signal to orchestrator On DRIFT, the next Reviewer dispatch MUST include this addition to its prompt: > "Previous rounds showed score inflation without corresponding quality improvement. > Apply maximum skepticism. Score conservatively; when uncertain, round DOWN." -On 3+ DRIFT occurrences: stop the loop and present findings for manual review. +On repeated DRIFT after the reset, stop the current review dispatch path and +return pivot_restart_signal. The orchestrator owns the pivot_restart_decision and +selects reset, candidate search, replan, restart, or a blocked/user path without +creating round 21 behavior. ## Review Finding Rule Distillation diff --git a/skills/assistant-review/contracts/handoffs.yaml b/skills/assistant-review/contracts/handoffs.yaml index 90485c5..a4b41d5 100644 --- a/skills/assistant-review/contracts/handoffs.yaml +++ b/skills/assistant-review/contracts/handoffs.yaml @@ -24,6 +24,9 @@ worker_status_protocol: - "findings, summary, and verdict remain required and are not replaced by status." - "evidence is required to support the verdict and any findings." - "open_questions is required for NEEDS_CONTEXT and BLOCKED." + - "status is required on every QAEvaluator return." + - "QAEvaluator returns acceptance_findings, qa_scorecard, final_verdict, result, evidence, and score_progression or score_entry; selected_domain_rubrics and domain_quality_scores are required only when scoped domain rubrics are selected. QA evaluation does not replace Reviewer/code-reviewer findings." + - "Reviewer and QAEvaluator may return pivot_restart_signal for PIVOT, STAGNATION, repeated DRIFT, or repeated REGRESSION; the orchestrator owns the final pivot_restart_decision." handoffs: @@ -70,8 +73,8 @@ handoffs: - name: round type: int required: true - description: "Current review round number (1-10)" - validation: ">= 1 and <= 10" + description: "Current review round number (1-20)" + validation: ">= 1 and <= 20" - name: previously_fixed type: object[] @@ -84,14 +87,14 @@ handoffs: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: finding_filter_policy type: string required: true description: "Evidence-backed finding filter for this round" - validation: "Must state that findings require file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are Observations and do not block; round 10 remains terminal." - notes: "In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers. The loop never starts round 11." + validation: "Must state that findings require file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are Observations and do not block; round 20 remains terminal." + notes: "In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers. The loop never starts round 21." - name: project_type type: string @@ -154,7 +157,7 @@ handoffs: type: int required: true description: "Echo back the round number for tracking" - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: findings type: object[] @@ -421,8 +424,402 @@ handoffs: required: false description: "If a critical finding caps the score, describe it here" + - name: pivot_restart_signal + type: object + required: false + condition: "rubric_scores.action == PIVOT or score tracking detects STAGNATION, repeated DRIFT, or repeated REGRESSION" + description: "Signal that the review loop needs orchestrator-owned pivot/restart handling before another fix/review dispatch." + object_fields: + - name: trigger + type: enum + required: true + enum_values: [STAGNATION, repeated_DRIFT, repeated_REGRESSION, pivot] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: affected_round + type: int + required: true + validation: ">= 1 and <= 20" + - name: recommended_recovery_focus + type: string + required: true + description: "Lowest-scoring dimensions, stale-context concern, or blocker evidence the orchestrator should consider" + on_missing_field: "In delegated mode, re-dispatch Reviewer. In direct fallback mode, complete the local review evidence before continuing. status, findings, evidence, and verdict are never optional — a review without them is not a review." + # ───────────────────────────────────────────── + # Orchestrator → QAEvaluator (per QA round) + # ───────────────────────────────────────────── + - name: orchestrator_to_qa_evaluator + from: Orchestrator + to: QAEvaluator + phase: QA_EVALUATION_STEP + notes: "Runs after build/test evidence and Code Reviewer or Reviewer compatibility evidence. QA evaluation is an acceptance lane and does not replace code-reviewer." + + context_fields: + - name: done_contract + type: object + required: conditional + condition: "task has an accepted Done Contract" + description: "Accepted Done Contract for the work under QA evaluation" + validation: "Includes done_when, not_done_when, verification, owner_consumer, acceptance_criteria, debate_record with at least two perspectives, and accepted_by when present" + object_fields: + - name: done_when + type: string[] + required: true + - name: not_done_when + type: string[] + required: true + - name: verification + type: string[] + required: true + - name: owner_consumer + type: string + required: true + - name: acceptance_criteria + type: string[] + required: true + - name: debate_record + type: object[] + required: true + min_items: 2 + description: "pre-build debate/subagent-perspective evidence considered before accepting the Done Contract" + validation: "At least two perspectives; delegated mode uses subagent perspectives when available" + object_fields: + - name: perspective + type: string + required: true + - name: concern_or_support + type: string + required: true + - name: resolution + type: string + required: true + - name: accepted_by + type: string + required: true + + - name: acceptance_criteria + type: string[] + required: true + description: "Binary acceptance criteria from the user request, approved plan, slice manifest, or Done Contract" + validation: "At least one independently evaluable criterion is present" + min_items: 1 + + - name: verification_evidence + type: object[] + required: true + description: "Build, test, manual, review, or inspection evidence used to prove acceptance" + validation: "At least one evidence entry exists; each entry states source, result, and what it proves" + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: result + type: string + required: true + - name: proves + type: string + required: true + + - name: code_review_result + type: object + required: true + description: "Final Code Reviewer or Reviewer compatibility result that QA must not replace" + validation: "Names the review role used, final result, review rounds, and remaining code-review risks if any" + object_fields: + - name: role + type: enum + required: true + enum_values: [CodeReviewer, Reviewer] + - name: result + type: string + required: true + - name: rounds + type: int + required: true + validation: ">= 1 and <= 20" + - name: remaining_risks + type: string[] + required: true + + - name: domain_context + type: object + required: false + default: null + description: "Scoped product, UX, docs, DX, or domain notes for QA evaluation" + notes: "When present, QAEvaluator may load references/domain-rubrics.md and select only matching rubric families." + object_fields: + - name: summary + type: string + required: true + - name: in_scope_surfaces + type: string[] + required: true + + - name: rubric_refs + type: string[] + required: false + default: [] + description: "Explicit rubric family references for product, UX, UI/visual, docs, DX, or domain quality" + notes: "Valid families are defined in references/domain-rubrics.md. QAEvaluator must not invent rubrics when rubric_refs are empty and no acceptance/Done Contract/domain_context scope requires them." + + - name: round + type: int + required: true + description: "Current QA evaluation round number (1-20)" + validation: ">= 1 and <= 20" + + - name: previously_failed_acceptance_items + type: object[] + required: true + description: "Acceptance items failed in previous QA rounds; QAEvaluator should not re-report resolved items" + object_fields: + - name: criterion + type: string + required: true + - name: failed_in_round + type: int + required: true + validation: ">= 1 and <= 20" + - name: evidence_gap + type: string + required: true + + - name: qa_filter_policy + type: string + required: true + description: "Evidence-backed QA finding filter for this round" + validation: "Must state that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 20 is terminal." + + return_fields: + - name: status + type: enum + required: true + enum_values: [DONE, DONE_WITH_CONCERNS, NEEDS_CONTEXT, BLOCKED] + description: "Worker status packet result; DONE requires final_verdict=accepted" + + - name: round + type: int + required: true + description: "Echo back the QA round number" + validation: ">= 1 and <= 20" + + - name: acceptance_findings + type: object[] + required: true + description: "Acceptance, Done Contract, verification, or scoped domain findings for this QA round" + object_fields: + - name: severity + type: enum + required: true + enum_values: [blocker, concern, observation] + - name: criterion + type: string + required: true + - name: evidence + type: string + required: true + - name: impact + type: string + required: true + - name: recommendation + type: string + required: false + + - name: qa_scorecard + type: object + required: true + description: "Compact QA scores; domain_quality is backed by selected_domain_rubrics/domain_quality_scores only when domain rubrics are scoped" + object_fields: + - name: acceptance_coverage + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments" + - name: evidence_strength + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments" + - name: domain_quality + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments; use 5.0 when no scoped domain quality surface applies and state not_applicable in rationale" + - name: final_readiness + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments" + - name: weighted_score + type: float + required: true + validation: "Calculated from the compact QA scorecard dimensions" + - name: rationale + type: object + required: true + object_fields: + - name: acceptance_coverage + type: string + required: true + - name: evidence_strength + type: string + required: true + - name: domain_quality + type: string + required: true + - name: final_readiness + type: string + required: true + + - name: selected_domain_rubrics + type: string[] + required: false + condition: "domain_context or rubric_refs is present, or acceptance criteria / Done Contract require subjective/product/UX/docs/DX/UI/domain craft evaluation" + description: "Rubric families selected from references/domain-rubrics.md for this QA round" + validation: "Every selected family is tied to acceptance criteria, Done Contract, domain_context, or explicit rubric_refs. Empty or omitted when no scoped domain rubric applies." + + - name: domain_quality_scores + type: object[] + required: false + condition: "selected_domain_rubrics is non-empty" + description: "Per-family domain rubric scores from references/domain-rubrics.md" + validation: "Each score cites scoped evidence and uses pass/refine/pivot or accepted/rejected guidance; unscoped dimensions are not_applicable, not invented." + object_fields: + - name: rubric_ref + type: string + required: true + description: "Selected rubric family, such as ui_visual_design, ux_product_acceptance, documentation_quality, developer_experience, or domain_specific_craft" + - name: dimension + type: string + required: true + description: "Selected dimension within the rubric family" + - name: score + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments. For action=not_applicable, use 5.0 and state the unscoped rationale in evidence." + - name: action + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, refine, rejected, pivot, not_applicable] + - name: evidence + type: string + required: true + validation: "Cites acceptance criteria, Done Contract, domain_context, rubric_refs, verification artifacts, screenshots, docs, or concrete files" + + - name: final_verdict + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, rejected, blocked] + description: "Final QA acceptance verdict for this round" + + - name: result + type: enum + required: true + enum_values: [CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, BLOCKED] + description: "Loop-compatible result summary" + + - name: evidence + type: object[] + required: true + description: "Acceptance material, files, review results, verification evidence, or checks supporting the QA verdict" + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + + - name: score_entry + type: object + required: true + description: "Score progression entry for this QA round" + object_fields: + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: weighted_score + type: float + required: true + - name: failed_acceptance_count + type: int + required: true + - name: delta + type: string + required: true + - name: drift_status + type: enum + required: true + enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL, NOT_APPLICABLE] + + - name: score_progression + type: object[] + required: false + description: "Full QA score progression when the evaluator has prior round history" + object_fields: + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: weighted_score + type: float + required: true + - name: failed_acceptance_count + type: int + required: true + - name: drift_status + type: enum + required: true + enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL, NOT_APPLICABLE] + + - name: open_questions + type: string[] + required: false + condition: "status in [NEEDS_CONTEXT, BLOCKED] or final_verdict == blocked" + description: "Questions or blocker details the orchestrator must resolve" + + - name: pivot_restart_signal + type: object + required: false + condition: "score_entry.drift_status == STAGNATION, repeated QA DRIFT/REGRESSION is detected, or any scoped domain_quality_scores action == pivot" + description: "Signal that the QA loop needs orchestrator-owned pivot/restart handling before another QA/build dispatch." + object_fields: + - name: trigger + type: enum + required: true + enum_values: [STAGNATION, repeated_DRIFT, repeated_REGRESSION, pivot] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: affected_round + type: int + required: true + validation: ">= 1 and <= 20" + - name: recommended_recovery_focus + type: string + required: true + description: "Acceptance gap, evidence gap, domain pivot evidence, or stale QA context the orchestrator should consider" + + on_missing_field: "In delegated mode, re-dispatch QAEvaluator. In direct fallback mode, complete the local QA evidence before continuing. status, acceptance_findings, qa_scorecard, final_verdict, result, evidence, and score_entry are never optional. When scoped domain rubrics are selected, selected_domain_rubrics and domain_quality_scores are also required; when not scoped, do not fabricate them." + # ───────────────────────────────────────────── # Validation behavior: # 1. Before delegated dispatch or direct-fallback review: verify all required context_fields are populated @@ -433,6 +830,11 @@ handoffs: # 6. If rubric_required: verify rubric_scores is present with all dimensions # 7. If rubric_required: verify weighted_score matches the dimension * weight calculation # 8. If quality_principles_required: verify the Reviewer applied references/review-principles.md -# 9. If round == 10, exit with CLEAN, ISSUES_FIXED, or HAS_REMAINING_ITEMS; never start round 11 -# 10. If delegated re-dispatch or direct-fallback completion is needed: include explicit note about what was missing +# 9. If pivot_restart_signal is present, stop the current loop path until the orchestrator records pivot_restart_decision; do not silently continue. +# 10. If round == 20, exit with CLEAN, ISSUES_FIXED, or HAS_REMAINING_ITEMS; never start round 21 +# 11. If delegated re-dispatch or direct-fallback completion is needed: include explicit note about what was missing +# 12. If qa_evaluation_mode=required, run orchestrator_to_qa_evaluator after build/test and code-review evidence; include debate_record when Done Contract exists; never treat QA evaluation as a replacement for Reviewer/code-reviewer. +# 13. If QA round == 20, return accepted, accepted_with_concerns, rejected, or blocked with remaining failed acceptance items; never start QA round 21. +# 14. If domain_context/rubric_refs or subjective/product/UX/docs/DX/UI/domain acceptance scope exists: verify selected_domain_rubrics and domain_quality_scores are present and evidence-backed. +# 15. If no scoped domain rubric applies: verify QAEvaluator did not invent rubrics or penalize domain_quality for missing unscoped evidence. # ───────────────────────────────────────────── diff --git a/skills/assistant-review/contracts/input.yaml b/skills/assistant-review/contracts/input.yaml index 16b654e..455f98a 100644 --- a/skills/assistant-review/contracts/input.yaml +++ b/skills/assistant-review/contracts/input.yaml @@ -93,6 +93,123 @@ fields: default: [] on_missing: skip + - name: qa_evaluation_mode + type: enum + required: false + enum_values: [not_required, optional, required] + description: "Whether the independent QA evaluation loop should run after code-review/build evidence" + validation: "required for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work; not_required must have a reason in final output when skipped for medium+ harness work" + default: not_required + on_missing: infer + infer_from: > + Use required when the task is medium+ harness-capable, has a Done Contract, is domain-scored, + affects UI/visual/product/UX/docs/DX quality, or the user explicitly requests QA/acceptance evaluation. + Use optional when acceptance material exists but the orchestrator has not marked QA as required. + Use not_required for trivial/small code-review-only work with no separate acceptance evidence need. + + - name: done_contract + type: object + required: conditional + condition: "qa_evaluation_mode == required and Done Contract exists for the task" + description: "Accepted Done Contract used by QAEvaluator" + validation: "When present, includes done_when, not_done_when, verification, owner_consumer, acceptance_criteria, debate_record with at least two perspectives, and accepted_by" + on_missing: fail + object_fields: + - name: done_when + type: string[] + required: true + - name: not_done_when + type: string[] + required: true + - name: verification + type: string[] + required: true + - name: owner_consumer + type: string + required: true + - name: acceptance_criteria + type: string[] + required: true + - name: debate_record + type: object[] + required: true + min_items: 2 + description: "pre-build debate/subagent-perspective evidence considered before accepting the Done Contract" + validation: "At least two perspectives; delegated mode uses subagent perspectives when available" + object_fields: + - name: perspective + type: string + required: true + - name: concern_or_support + type: string + required: true + - name: resolution + type: string + required: true + - name: accepted_by + type: string + required: true + + - name: acceptance_criteria + type: string[] + required: conditional + condition: "qa_evaluation_mode == required" + description: "Binary acceptance criteria from the user request, approved plan, slice manifest, or Done Contract" + validation: "At least one criterion is present and each criterion is independently evaluable" + min_items: 1 + on_missing: fail + + - name: verification_evidence + type: object[] + required: conditional + condition: "qa_evaluation_mode == required" + description: "Build, test, manual, review, or inspection evidence QAEvaluator uses to verify acceptance" + validation: "At least one evidence entry exists and each entry names the check/result and what it proves" + min_items: 1 + on_missing: fail + object_fields: + - name: source + type: string + required: true + - name: result + type: string + required: true + - name: proves + type: string + required: true + + - name: domain_context + type: object + required: false + description: "Scoped product, UX, docs, DX, or domain notes for QA evaluation" + default: null + on_missing: skip + notes: "When populated, QAEvaluator may load references/domain-rubrics.md and select only matching rubric families." + object_fields: + - name: summary + type: string + required: true + - name: in_scope_surfaces + type: string[] + required: true + + - name: rubric_refs + type: string[] + required: false + description: "Explicit product/UX/docs/DX/UI/domain rubric family references used by QAEvaluator" + default: [] + on_missing: skip + notes: "Valid families are defined in references/domain-rubrics.md. Empty means no domain rubric is selected unless acceptance criteria or Done Contract scope requires one." + + - name: qa_filter_policy + type: string + required: conditional + condition: "qa_evaluation_mode == required" + description: "Evidence filter QAEvaluator must apply to acceptance findings" + validation: "States that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 20 is terminal" + on_missing: infer + infer_from: "Use the standard QA filter policy from references/qa-evaluation-loop.md." + - name: mode type: enum required: true @@ -162,9 +279,10 @@ fields: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" # Validation behavior: # 1. Resolve review_scope, review_material_snapshot, change_kind, semantic_contract_review_required, behavioral_contract_review_required, and agentic_loop_safety_review_required before entering the loop # 2. If review_material_snapshot.material is empty after resolution, inform user "nothing to review" and exit -# 3. Log resolved input as: >> Review scope: {scope_size} | mode: {mode} | kind: {change_kind} | subagent_mode: {subagent_execution_mode} | loop_safety_review: {agentic_loop_safety_review_required} | behavioral_contract_review: {behavioral_contract_review_required} | semantic_contract_review: {semantic_contract_review_required} | files: {count} +# 3. If qa_evaluation_mode=required: require acceptance_criteria, verification_evidence, qa_filter_policy, and Done Contract with debate_record when the task has one before QAEvaluator dispatch +# 4. Log resolved input as: >> Review scope: {scope_size} | mode: {mode} | kind: {change_kind} | subagent_mode: {subagent_execution_mode} | qa_evaluation_mode: {qa_evaluation_mode} | loop_safety_review: {agentic_loop_safety_review_required} | behavioral_contract_review: {behavioral_contract_review_required} | semantic_contract_review: {semantic_contract_review_required} | files: {count} diff --git a/skills/assistant-review/contracts/output.yaml b/skills/assistant-review/contracts/output.yaml index 69cbe49..2b8ca43 100644 --- a/skills/assistant-review/contracts/output.yaml +++ b/skills/assistant-review/contracts/output.yaml @@ -30,6 +30,88 @@ artifacts: required: true validation: "Names the fresh Reviewer dispatch, or the direct-fallback context reset used to reduce stale review risk" + - name: qa_evaluation_delegation_path + type: object + required: conditional + condition: "qa_evaluation_mode == required" + description: "How QAEvaluator responsibility was executed" + validation: "States subagent policy state, execution mode, authorization scope, and whether QA used a fresh QAEvaluator subagent or fresh direct-fallback context" + on_fail: "Record delegated QA evaluation or direct fallback before presenting results; do not imply QAEvaluator subagents were used when they were not authorized or available" + object_fields: + - name: subagent_policy_state + type: enum + required: true + enum_values: [not_required, authorization_required, delegation_authorized, authorization_denied, subagents_unavailable, policy_disallowed] + - name: subagent_execution_mode + type: enum + required: true + enum_values: [delegated, direct_fallback, not_applicable] + - name: subagent_authorization_scope + type: string[] + required: true + - name: fresh_context_evidence + type: string + required: true + validation: "Names the fresh QAEvaluator dispatch, or the direct-fallback context reset used to reduce stale QA risk" + + - name: pivot_restart_decision + type: object + required: conditional + condition: "review or QA loop observes STAGNATION, repeated DRIFT, repeated REGRESSION, rubric action PIVOT, or scoped domain action pivot" + description: "Orchestrator-owned decision packet created before another review/fix/QA/build dispatch after stagnant or pivot-triggered quality loops." + validation: "Must include trigger, evidence, affected_slice_or_round, options_considered, selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action. Reapproval is required when scope, files, behavior, risk, verification, or acceptance criteria change." + on_fail: "Pause the loop and create the pivot_restart_decision before continuing; do not silently continue after stagnation or pivot." + object_fields: + - name: trigger + type: enum + required: true + enum_values: [STAGNATION, repeated_DRIFT, repeated_REGRESSION, pivot] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: affected_slice_or_round + type: string + required: true + - name: options_considered + type: object[] + required: true + min_items: 2 + object_fields: + - name: option + type: string + required: true + - name: tradeoff + type: string + required: true + - name: disposition + type: enum + required: true + enum_values: [selected, rejected, deferred] + - name: selected_action + type: enum + required: true + enum_values: [reset_context, return_to_build, dispatch_debugging, dispatch_explorer, dispatch_architect, run_candidate_search, replan, restart_slice, restart_phase, block_for_user, accept_with_limitations] + - name: reapproval_required + type: boolean + required: true + - name: next_agent + type: string + required: true + - name: recovery_pointer + type: string + required: true + - name: exact_next_action + type: string + required: true + - name: review_finding_rule_distillation type: object[] required: conditional @@ -74,12 +156,12 @@ artifacts: required: true condition: always description: "The single summary presented to the user after the loop completes" - validation: "Must include rounds (1-10), result, fixed_items, remaining_items when applicable, and nits before presenting review results" + validation: "Must include rounds (1-20), result, fixed_items, remaining_items when applicable, and nits before presenting review results" object_fields: - name: rounds type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: result type: enum required: true @@ -102,7 +184,7 @@ artifacts: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: remaining_items type: object[] required: false @@ -134,6 +216,128 @@ artifacts: type: string required: true + - name: qa_evaluation_result + type: object + required: conditional + condition: "qa_evaluation_mode == required" + description: "Independent QA evaluation final result after build/test and code-review evidence" + validation: "Must include rounds (1-20), final_verdict, result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, evidence, and open_questions when blocked" + on_fail: "Run the QA evaluation loop from references/qa-evaluation-loop.md or record why QA is blocked before presenting final review results" + object_fields: + - name: rounds + type: int + required: true + validation: ">= 1 and <= 20" + - name: final_verdict + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, rejected, blocked] + - name: result + type: enum + required: true + enum_values: [CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, BLOCKED] + - name: acceptance_findings + type: object[] + required: true + description: "Remaining or resolved acceptance findings from the final QA round" + object_fields: + - name: severity + type: enum + required: true + enum_values: [blocker, concern, observation] + - name: criterion + type: string + required: true + - name: evidence + type: string + required: true + - name: disposition + type: enum + required: true + enum_values: [resolved, remaining, not_applicable] + - name: qa_scorecard + type: object + required: true + description: "Compact QA scorecard; domain_quality is backed by selected_domain_rubrics/domain_quality_scores only when scoped domain rubrics were used" + object_fields: + - name: acceptance_coverage + type: float + required: true + - name: evidence_strength + type: float + required: true + - name: domain_quality + type: float + required: true + - name: final_readiness + type: float + required: true + - name: weighted_score + type: float + required: true + - name: selected_domain_rubrics + type: string[] + required: false + condition: "domain_context or rubric_refs was provided, or acceptance criteria / Done Contract required subjective/product/UX/docs/DX/UI/domain craft evaluation" + description: "Rubric families selected from references/domain-rubrics.md; empty or omitted when domain rubrics were not scoped" + - name: domain_quality_scores + type: object[] + required: false + condition: "selected_domain_rubrics is non-empty" + description: "Evidence-backed domain rubric scores from the final QA round" + object_fields: + - name: rubric_ref + type: string + required: true + - name: dimension + type: string + required: true + - name: score + type: float + required: true + - name: action + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, refine, rejected, pivot, not_applicable] + - name: evidence + type: string + required: true + - name: score_progression + type: object[] + required: true + description: "QA score progression across rounds" + object_fields: + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: weighted_score + type: float + required: true + - name: failed_acceptance_count + type: int + required: true + - name: drift_status + type: enum + required: true + enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL, NOT_APPLICABLE] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: open_questions + type: string[] + required: false + condition: "final_verdict == blocked or result == BLOCKED" + description: "Questions or blocker details the orchestrator must resolve" + - name: rubric_summary type: object required: false @@ -170,12 +374,12 @@ artifacts: - name: score_progression type: object[] required: true - description: "Weighted score per round — used for drift detection and capped at the 10-round review limit" + description: "Weighted score per round — used for drift detection and capped at the 20-round review limit" object_fields: - name: round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: weighted_score type: float required: true @@ -379,3 +583,6 @@ artifacts: # 5. If semantic_contract_review_required=true: semantic_contract_review is populated and semantic findings are included in final_summary/audit_report # 6. If behavioral_contract_review_required=true: behavioral_contract_review is populated and behavioral findings are included in final_summary/audit_report # 7. If agentic_loop_safety_review_required=true: agentic_loop_safety_review is populated and loop-safety findings are included in final_summary/audit_report +# 8. If qa_evaluation_mode=required: qa_evaluation_delegation_path and qa_evaluation_result are populated after build/test and code-review evidence; QA result does not replace final_summary or code-review artifacts +# 9. If scoped domain rubrics were used: qa_evaluation_result includes selected_domain_rubrics and domain_quality_scores; if not scoped, no invented domain rubric output is required or accepted +# 10. If review or QA triggered STAGNATION, repeated DRIFT, repeated REGRESSION, or pivot action: pivot_restart_decision is populated before any additional review/fix/QA/build dispatch diff --git a/skills/assistant-review/contracts/phase-gates.yaml b/skills/assistant-review/contracts/phase-gates.yaml index 79046cd..c1c6083 100644 --- a/skills/assistant-review/contracts/phase-gates.yaml +++ b/skills/assistant-review/contracts/phase-gates.yaml @@ -42,6 +42,11 @@ gates: check: "subagent_policy_state, subagent_execution_mode, and subagent_authorization_scope are resolved before any Reviewer subagent dispatch" on_fail: "Resolve the active adapter/tool policy. Ask once when explicit authorization is required, or set direct_fallback/not_applicable and record why no Reviewer subagent will be spawned." + - id: E7 + check: "If qa_evaluation_mode is required: Done Contract with debate_record when available, acceptance_criteria, verification_evidence, domain_context/rubric_refs when applicable, and qa_filter_policy are resolved before QAEvaluator dispatch" + condition: "qa_evaluation_mode == required" + on_fail: "Load references/qa-evaluation-loop.md, gather acceptance material and verification evidence, select domain rubric scope only when acceptance criteria, Done Contract, domain_context, or rubric_refs require it, or mark QA blocked with open_questions before dispatching QAEvaluator." + # ───────────────────────────────────────────── # REVIEW step — per round # ───────────────────────────────────────────── @@ -109,15 +114,15 @@ gates: exit_assertions: - id: EV1 - check: "Exit decision is one of: EXIT_CLEAN, EXIT_ISSUES_FIXED, EXIT_MAX_ROUNDS, CONTINUE" - on_fail: "Re-evaluate findings against exit criteria. EXIT_CLEAN = no must-fix/should-fix (nits OK, note in report) and, for medium+ scope, weighted_score >= 4.0. EXIT_ISSUES_FIXED = round 10, all issues fixed this session, no must-fix/should-fix findings, and, for medium+ scope, weighted_score >= 4.0. EXIT_MAX_ROUNDS = round 10 with remaining must-fix or should-fix findings." + check: "Exit decision is one of: EXIT_CLEAN, EXIT_ISSUES_FIXED, EXIT_MAX_ROUNDS, ESCALATE_PIVOT_RESTART, CONTINUE" + on_fail: "Re-evaluate findings against exit criteria. EXIT_CLEAN = no must-fix/should-fix (nits OK, note in report) and, for medium+ scope, weighted_score >= 4.0. EXIT_ISSUES_FIXED = round 20, all issues fixed this session, no must-fix/should-fix findings, and, for medium+ scope, weighted_score >= 4.0. EXIT_MAX_ROUNDS = round 20 with remaining must-fix or should-fix findings. ESCALATE_PIVOT_RESTART = STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT." - id: EV2 check: "EXIT_CLEAN or EXIT_ISSUES_FIXED only if: zero must-fix AND zero should-fix findings, and for medium+ scope weighted_score >= 4.0" - on_fail: "Findings exist or medium+ weighted_score is below 4.0 — must CONTINUE to FIX step" + on_fail: "Findings exist or medium+ weighted_score is below 4.0 — apply rubric action first. REFINE continues to FIX; PIVOT escalates through pivot_restart_decision." - id: EV3 - check: "EXIT_MAX_ROUNDS only if: round == 10" + check: "EXIT_MAX_ROUNDS only if: round == 20" on_fail: "Rounds remain — must CONTINUE to FIX step" - id: EV4 @@ -126,14 +131,19 @@ gates: on_fail: "Audit mode does not fix or loop — present findings and exit" - id: EV5 - check: "For medium+ scope: rubric threshold action is applied — PASS exits, REFINE continues, PIVOT escalates" + check: "For medium+ scope: rubric threshold action is applied — PASS exits, REFINE continues, PIVOT escalates through pivot_restart_decision" condition: "scope_size in [medium, large] and rubric_scores is present" on_fail: "Check rubric weighted_score against threshold table in references/review-rubric.md and apply the correct action" - id: EV6 - check: "PIVOT action triggers escalation to orchestrator — do not silently continue after PIVOT" + check: "PIVOT action triggers orchestrator-owned pivot_restart_decision — do not silently continue after PIVOT" condition: "rubric_scores.action == PIVOT" - on_fail: "Flag to orchestrator: weighted score below pivot threshold. Include lowest-scoring dimensions and recommendations." + on_fail: "Create pivot_restart_signal and require the orchestrator to record pivot_restart_decision with trigger, evidence, affected_slice_or_round, options_considered, selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action before another fix/review dispatch." + + - id: EV7 + check: "STAGNATION, repeated DRIFT, or repeated REGRESSION triggers ESCALATE_PIVOT_RESTART and records pivot_restart_signal before another fix/review dispatch" + condition: "scope_size in [medium, large] and round >= 2" + on_fail: "Pause the loop, build pivot_restart_signal from score_history, and require orchestrator-owned pivot_restart_decision. Do not advance to another review round until the decision names the exact next action." # ───────────────────────────────────────────── # FIX step — per round (review-fix mode only) @@ -178,6 +188,45 @@ gates: condition: "project is C# and complexity tool is available" on_fail: "Run complexity analysis and flag methods exceeding threshold" + # ───────────────────────────────────────────── + # QA EVALUATION step — after code review/build evidence + # ───────────────────────────────────────────── + - phase: QA_EVALUATION_STEP + condition: "qa_evaluation_mode == required" + checkpoint_start: null + checkpoint_end: null + exit_assertions: + + - id: QA1 + check: "QA evaluation starts only after build/test verification evidence and Code Reviewer or Reviewer compatibility result are available" + on_fail: "Return to validation or code review. Do not dispatch QAEvaluator until code-review/build evidence exists." + + - id: QA2 + check: "QAEvaluator received done_contract with debate_record when present, acceptance_criteria, verification_evidence, domain_context/rubric_refs when applicable, round, previously_failed_acceptance_items, and qa_filter_policy" + on_fail: "Re-dispatch QAEvaluator with the full QA handoff context, or complete direct-fallback QA context before continuing." + + - id: QA3 + check: "QAEvaluator returned status, round, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were selected, final_verdict, result, evidence, and score_entry" + on_fail: "In delegated mode, re-dispatch QAEvaluator requesting the missing fields. In direct fallback mode, complete the missing QA evidence before deciding acceptance. Require selected_domain_rubrics/domain_quality_scores only when domain rubrics are scoped." + + - id: QA4 + check: "QAEvaluator findings stay in the QA lane and do not replace code-reviewer findings" + on_fail: "Move generic code/security/architecture/test-coverage findings back to Code Reviewer unless they directly block an acceptance criterion or Done Contract item." + + - id: QA5 + check: "QA loop exit decision is accepted, accepted_with_concerns, rejected, or blocked; round 20 is terminal and never starts round 21" + on_fail: "Apply references/qa-evaluation-loop.md terminal behavior. If round < 20 and rejected, return failed acceptance items to Build; if round == 20, record remaining failed acceptance items in qa_evaluation_result." + + - id: QA6 + check: "Domain rubrics from references/domain-rubrics.md are loaded and applied only when scoped by acceptance criteria, Done Contract, domain_context, or explicit rubric_refs" + condition: "qa_evaluation_mode == required" + on_fail: "Remove unscoped rubric scoring, or re-dispatch QAEvaluator with the explicit domain_context/rubric_refs and require selected_domain_rubrics plus domain_quality_scores." + + - id: QA7 + check: "QA STAGNATION, repeated DRIFT, repeated REGRESSION, or scoped domain action pivot triggers pivot_restart_signal and orchestrator-owned pivot_restart_decision before another QA/build dispatch" + condition: "qa_evaluation_mode == required" + on_fail: "Pause QA, create pivot_restart_signal from score_entry, score_progression, or domain_quality_scores action=pivot, and require pivot_restart_decision with exact_next_action. Round 20 remains terminal and never starts round 21." + # ───────────────────────────────────────────── # EXIT — after loop completes # ───────────────────────────────────────────── @@ -200,6 +249,11 @@ gates: condition: "scope_size in [medium, large]" on_fail: "Populate rubric_summary with final_scores from last round and score_progression from all rounds" + - id: X4 + check: "If qa_evaluation_mode is required: qa_evaluation_result is populated with final_verdict, result, acceptance_findings, qa_scorecard, score_progression, and evidence" + condition: "qa_evaluation_mode == required" + on_fail: "Run or complete the QA evaluation loop before presenting final review results." + # ───────────────────────────────────────────── # Loop invariants # ───────────────────────────────────────────── @@ -215,9 +269,9 @@ invariants: on_fail: "Restore any removed items to previously_fixed. This list is append-only." - id: INV2 - check: "Finding filter policy requires file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are non-blocking Observations; round 10 is terminal" + check: "Finding filter policy requires file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are non-blocking Observations; round 20 is terminal" scope: all_rounds - on_fail: "Rebuild finding_filter_policy from the evidence-backed rule and hard max-10 cap before dispatching review." + on_fail: "Rebuild finding_filter_policy from the evidence-backed rule and hard max-20 cap before dispatching review." - id: INV3 check: "The agent never pauses between rounds to ask the user — the loop is autonomous" @@ -235,8 +289,9 @@ invariants: condition: "scope_size in [medium, large] and round >= 2" on_fail: > Drift detected: score increased without corresponding quality improvement. - Per references/score-tracking.md: reset evaluator context by dispatching a fresh Reviewer - with stricter prompt. Add 'Drift detected in round N' to the final summary. + Per references/score-tracking.md: reset evaluator context once by dispatching a fresh Reviewer + with stricter prompt; repeated DRIFT triggers pivot_restart_signal and orchestrator-owned + pivot_restart_decision before another dispatch. - id: INV6 check: "For medium+ scope: score does not stagnate (unchanged for 2+ rounds with findings present)" @@ -244,8 +299,8 @@ invariants: condition: "scope_size in [medium, large] and round >= 3" on_fail: > Stagnation detected: score unchanged for 2+ rounds while findings remain. - Escalate to orchestrator — the loop is churning without progress. Consider whether - remaining issues are architectural (require broader changes) or the approach needs a PIVOT. + Return pivot_restart_signal and require orchestrator-owned pivot_restart_decision before + another fix/review dispatch. - id: INV7 @@ -267,3 +322,27 @@ invariants: scope: all_rounds condition: "agentic_loop_safety_review_required is true" on_fail: "Run the Agentic Loop Safety Checklist before declaring the review clean; report missing bounds, stop conditions, retry caps, empty-result handling, tool-error routing, stagnation detection, cost/token guards, or low-confidence escalation as findings." + + - id: INV_QA1 + check: "QA evaluation never replaces Code Reviewer or Reviewer compatibility review" + scope: all_rounds + condition: "qa_evaluation_mode == required" + on_fail: "Run code review first and record QAEvaluator only as the independent acceptance lane." + + - id: INV_QA2 + check: "QA evaluation round is always between 1 and 20, and round 20 is terminal" + scope: all_rounds + condition: "qa_evaluation_mode == required" + on_fail: "Clamp QA execution to the 20-round cap and return the final verdict with remaining failed acceptance items instead of starting round 21." + + - id: INV_QA3 + check: "QAEvaluator does not invent domain rubrics, subjective standards, or domain-quality penalties when no acceptance criteria, Done Contract, domain_context, or rubric_refs scope them" + scope: all_rounds + condition: "qa_evaluation_mode == required" + on_fail: "Discard unscoped domain rubric output and rerun QA with references/domain-rubrics.md conditional selection rules." + + - id: INV_QA4 + check: "QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists" + scope: all_rounds + condition: "qa_evaluation_mode == required" + on_fail: "Move unsupported QA concerns to non-blocking observations or request missing context with NEEDS_CONTEXT/BLOCKED." diff --git a/skills/assistant-review/evals/cases.json b/skills/assistant-review/evals/cases.json index 0620eb5..1d9ec63 100644 --- a/skills/assistant-review/evals/cases.json +++ b/skills/assistant-review/evals/cases.json @@ -30,7 +30,7 @@ "Runs review rounds with confidence thresholds and previously_fixed tracking.", "Fixes all must-fix and should-fix findings before validation.", "Runs build or test verification before the next round.", - "Continues review, fix, validate, and re-review until clean or max 10 rounds." + "Continues review, fix, validate, and re-review until clean or max 20 rounds." ], "pass_criteria": [ "The response identifies review scope and mode.", @@ -78,7 +78,7 @@ "expected_behavior": [ "Requires rubric scores for correctness, code quality, architecture, security, and test coverage.", "Applies a weighted score threshold action such as PASS, REFINE, or PIVOT.", - "Uses evidence-backed finding filtering across rounds 1-10 and treats round 10 as the hard cap.", + "Uses evidence-backed finding filtering across rounds 1-20 and treats round 20 as the hard cap.", "Buffers intermediate round details and presents one final summary after loop exit.", "Includes score progression when multiple rounds occur.", "Does not ask the user between rounds unless blocked." @@ -615,6 +615,73 @@ ] } }, + { + "id": "qa-evaluator-uses-domain-rubrics-conditionally", + "title": "QA evaluator uses domain rubrics conditionally", + "category": "qa_domain_rubric_evaluation", + "purpose": "Checks that QAEvaluator loads and applies domain-rubrics.md only for scoped subjective, docs, DX, product, UX, UI, or domain acceptance and returns selected rubric scores separately from code-review findings.", + "prompt": "Run the QA evaluation lane for a completed docs/DX workflow slice. The Done Contract says setup documentation must be accurate, the developer handoff must be easy to follow, and QA should use rubric_refs documentation_quality and developer_experience. Build/test and code-review evidence already exist.", + "setup_context": [ + "The assistant-review skill instructions are active.", + "qa_evaluation_mode is required and code-review/build evidence already exists.", + "The Done Contract and acceptance criteria scope documentation accuracy and developer experience.", + "domain_context describes a developer onboarding workflow.", + "rubric_refs contains documentation_quality and developer_experience.", + "No UI, visual design, product workflow, or unrelated domain craft rubric is in scope." + ], + "expected_behavior": [ + "Loads references/qa-evaluation-loop.md for the QA lane after code-review/build evidence.", + "Conditionally loads references/domain-rubrics.md because Done Contract, domain_context, and rubric_refs scope docs/DX quality.", + "Selects only documentation_quality and developer_experience rubrics.", + "Returns selected_domain_rubrics with the selected families.", + "Returns domain_quality_scores with evidence-backed per-dimension scores and action guidance.", + "Marks unscoped UI, product, and domain-specific craft dimensions not_applicable or omits them.", + "Keeps QA findings tied to acceptance criteria, Done Contract, verification evidence, domain_context, or rubric_refs.", + "Does not replace Code Reviewer or report general code defects unless they directly block acceptance.", + "Does not load or invent domain rubrics for unscoped acceptance." + ], + "pass_criteria": [ + "The response mentions references/domain-rubrics.md as conditional.", + "The response selects documentation_quality and developer_experience only.", + "The response returns selected_domain_rubrics and domain_quality_scores.", + "The response ties domain scores to acceptance criteria, Done Contract, domain_context, rubric_refs, and evidence.", + "The response keeps unscoped UI/product/domain craft rubrics not applicable or omitted.", + "The response keeps QA separate from code-review findings." + ], + "fail_signals": [ + "Always loads domain-rubrics.md for QA even when no domain quality is scoped.", + "Invents a UI or product rubric when the case only scopes docs/DX.", + "Omits selected_domain_rubrics or domain_quality_scores.", + "Scores domain quality without evidence.", + "Uses QAEvaluator as a replacement for Code Reviewer.", + "Reports general code defects not tied to acceptance." + ], + "machine_expectations": { + "required_substrings": [ + "references/domain-rubrics.md", + "conditional", + "selected_domain_rubrics", + "domain_quality_scores", + "documentation_quality", + "developer_experience", + "acceptance criteria", + "Done Contract", + "domain_context", + "rubric_refs", + "evidence", + "not_applicable", + "Code Reviewer" + ], + "forbidden_substrings": [ + "always load domain-rubrics", + "invent a UI rubric", + "product rubric required", + "domain scores do not need evidence", + "QAEvaluator replaces Code Reviewer", + "general code defects are QA findings" + ] + } + }, { "id": "review-finding-distills-permanent-rule-candidates", "title": "Review finding distills permanent rule candidates", diff --git a/skills/assistant-review/references/domain-rubrics.md b/skills/assistant-review/references/domain-rubrics.md new file mode 100644 index 0000000..e3c2b4a --- /dev/null +++ b/skills/assistant-review/references/domain-rubrics.md @@ -0,0 +1,79 @@ +# Domain Rubrics + +Use this reference only for QA evaluation when scoped domain quality is part of acceptance. Domain rubric use is conditional and must be tied to at least one of: acceptance criteria, Done Contract, `domain_context`, or explicit `rubric_refs`. Do not load or apply these rubrics for code-review-only work, and do not invent new rubric families when the scope does not name or imply them. + +## Selection Rules + +- Select the smallest relevant rubric family set: `ui_visual_design`, `ux_product_acceptance`, `documentation_quality`, `developer_experience`, or `domain_specific_craft`. +- Score only selected dimensions that have evidence. Use `not_applicable` for unscoped dimensions instead of lowering the score. +- Evidence must cite acceptance criteria, Done Contract items, `domain_context`, `rubric_refs`, verification artifacts, screenshots, docs, commands, or concrete changed files. +- Return `selected_domain_rubrics` and `domain_quality_scores` when any family is selected. +- If a needed domain standard is not provided and cannot be discovered locally, return `NEEDS_CONTEXT` or `blocked` with open questions instead of fabricating the standard. + +## Scoring + +| Score | Meaning | QA action | +|---|---|---| +| 5 | Fully satisfies scoped expectations with direct evidence | accepted | +| 4 | Meets acceptance with minor non-blocking concerns | accepted_with_concerns | +| 3 | Partially meets scope; fixable gaps remain | refine | +| 1-2 | Core scoped expectation is contradicted or unproven | rejected or pivot | + +Use `accepted` only when all selected dimensions are 4+ and no blocker acceptance findings remain. Use `refine` when gaps are concrete and fixable within the approved scope. Use `pivot` when the selected rubric exposes a wrong product/design/domain direction, missing domain authority, or repeated failure that needs a plan change. + +## UI / Visual Design + +Use when acceptance names UI, visual design, layout, responsive behavior, screenshots, design parity, or explicit `rubric_refs: [ui_visual_design]`. + +| Dimension | Evidence examples | Pass / refine / pivot guidance | +|---|---|---| +| Layout and hierarchy | Screenshots, viewport checks, design refs, component files | Pass when priority and grouping match the task; refine overlaps, weak hierarchy, or cramped spacing; pivot if the visual model contradicts the requested design direction. | +| States and feedback | Loading, empty, error, hover/focus, disabled states | Pass when required states are represented; refine missing secondary states; reject if a required user path has no visible feedback. | +| Accessibility and readability | Keyboard/focus checks, contrast notes, text fit, labels | Pass when text is readable and controls are operable; refine low-risk contrast/text issues; reject blocked interaction, hidden labels, or clipped essential text. | +| Visual fit and consistency | Existing design system, tokens, component conventions | Pass when it fits the local system; refine inconsistent spacing/color/component usage; pivot if the implementation introduces a conflicting visual language. | + +## UX / Product Acceptance + +Use when acceptance names user workflow, product behavior, adoption/readiness, copy, or explicit `rubric_refs: [ux_product_acceptance]`. + +| Dimension | Evidence examples | Pass / refine / pivot guidance | +|---|---|---| +| Goal completion | User journey, task criteria, manual scenario evidence | Pass when target users can complete the stated job; refine friction or avoidable extra steps; reject if the primary job cannot be completed. | +| Decision clarity | Labels, prompts, defaults, next-action visibility | Pass when choices and consequences are clear; refine ambiguous copy; reject misleading or destructive flows. | +| Edge and recovery paths | Empty/error states, undo/retry paths, validation messages | Pass when expected edge cases are handled; refine incomplete recovery; reject dead ends for scoped scenarios. | +| Product fit | Domain context, stakeholder criteria, non-goals | Pass when behavior matches product intent; refine minor mismatches; pivot if the implementation solves the wrong user problem. | + +## Documentation Quality + +Use when acceptance names docs, README, API docs, migration notes, runbooks, or explicit `rubric_refs: [documentation_quality]`. + +| Dimension | Evidence examples | Pass / refine / pivot guidance | +|---|---|---| +| Accuracy | Compared code/config/contracts, command outputs, examples | Pass when docs match current behavior; refine stale or ambiguous details; reject false commands, wrong APIs, or unsafe guidance. | +| Completeness for task | Acceptance criteria, documented setup/use/troubleshooting path | Pass when the target reader can finish the task; refine missing edge notes; reject missing required setup or operation steps. | +| Structure and scanability | Headings, ordered procedures, tables, examples | Pass when information is easy to find; refine dense or duplicated sections; pivot if docs need a different artifact type. | +| Maintenance fit | Version notes, ownership, generated/manual boundaries | Pass when update path is clear; refine unclear ownership; reject docs that will drift from source of truth. | + +## Developer Experience + +Use when acceptance names CLI/API ergonomics, local setup, integration, maintainability for consumers, or explicit `rubric_refs: [developer_experience]`. + +| Dimension | Evidence examples | Pass / refine / pivot guidance | +|---|---|---| +| Setup and run path | Commands, scripts, examples, env/config notes | Pass when a developer can start reliably; refine missing context; reject hidden prerequisites or broken commands. | +| Contract ergonomics | API shape, schema fields, handoff packets, error messages | Pass when contracts are predictable and typed; refine confusing names/defaults; reject ambiguous or incompatible contract behavior. | +| Debuggability | Logs, validation errors, trace/replay artifacts, failure messages | Pass when failures are diagnosable; refine thin evidence; reject silent failure or misleading diagnostics. | +| Change safety | Tests/evals/guards, migration notes, compatibility story | Pass when future edits have guardrails; refine narrow gaps; reject fake-pass coverage or unguarded public contract changes. | + +## Domain-Specific Craft + +Use when acceptance names a specialized domain, craft standard, industry rule, expert terminology, or explicit `rubric_refs: [domain_specific_craft]`. + +| Dimension | Evidence examples | Pass / refine / pivot guidance | +|---|---|---| +| Domain rule fidelity | Done Contract, domain_context, standards, fixtures | Pass when required rules are preserved; refine minor omissions; reject contradictions to scoped domain rules. | +| Terminology and mental model | User language, docs, UI copy, schema names | Pass when terms match the domain audience; refine inconsistent wording; reject misleading terms that change meaning. | +| Expert edge cases | Provided edge cases, fixtures, acceptance examples | Pass when scoped edge cases are covered; refine low-risk omissions; reject missing high-impact edge cases. | +| Artifact craft | Generated asset, workflow, document, content, or domain output | Pass when artifact quality meets the named craft bar; refine polish gaps; pivot if the artifact type or approach is wrong. | + +High-stakes domains still require the relevant specialist gate when applicable. This rubric does not replace security, legal, medical, financial, privacy, or compliance review. diff --git a/skills/assistant-review/references/qa-evaluation-loop.md b/skills/assistant-review/references/qa-evaluation-loop.md new file mode 100644 index 0000000..f51f9b0 --- /dev/null +++ b/skills/assistant-review/references/qa-evaluation-loop.md @@ -0,0 +1,115 @@ +# QA Evaluation Loop + +Use this reference after build/test evidence and the Code Reviewer loop exist. QA evaluation is a separate acceptance lane: it decides whether the work satisfies the Done Contract, acceptance criteria, verification evidence, and scoped domain quality expectations. It does not replace code-reviewer. + +## When QA Runs + +Run QA evaluation when any of these are true: +- Medium+ harness-capable work has a Done Contract. +- The task is subjective, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly asks for QA. +- Workflow Review needs a final acceptance verdict beyond code/security/architecture/test-coverage review. + +Skip QA evaluation only when the task is trivial/small, has no acceptance material beyond a direct code fix, and the workflow output records `qa_evaluation_mode=not_required` with a reason. + +## Inputs + +The orchestrator provides: +- `done_contract`: done_when, not_done_when, verification, owner_consumer, acceptance_criteria, debate_record, accepted_by. +- `debate_record`: pre-build debate/subagent-perspective evidence from the Done Contract; required when a Done Contract exists. +- `acceptance_criteria`: binary criteria from the user request, approved plan, slice manifest, or Done Contract. +- `verification_evidence`: build/test/manual/check evidence already produced by Builder/Tester or direct fallback. +- `code_review_result`: final Code Reviewer or Reviewer compatibility result. +- `domain_context`: scoped UI/visual, product, UX, docs, DX, or domain notes when applicable. +- `rubric_refs`: explicit references to scoped domain rubric families from `references/domain-rubrics.md` when applicable. +- `round`: QA round number from 1 to 20. +- `previously_failed_acceptance_items`: failed acceptance items from earlier QA rounds. +- `qa_filter_policy`: acceptance findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns stay non-blocking. + +## Conditional Domain Rubrics + +Load `references/domain-rubrics.md` only when `domain_context` or `rubric_refs` are present, or when acceptance criteria / Done Contract explicitly require subjective, product, UX, docs, DX, UI/visual, or domain craft judgment. + +When loaded: +- Select only rubric families tied to acceptance criteria, Done Contract, `domain_context`, or explicit `rubric_refs`. +- Return `selected_domain_rubrics` and `domain_quality_scores`. +- Keep domain findings evidence-backed and scoped to acceptance impact. +- Use `not_applicable` for unscoped dimensions. + +When not loaded: +- Set `domain_quality` to 5.0 and state `not_applicable` in the score rationale. +- Do not invent domain rubrics, subjective standards, or extra acceptance bars. +- Do not penalize the work for missing domain evidence that was never scoped. + +## Loop + +```text +round = 1 +previously_failed_acceptance_items = [] +score_progression = [] + +while round <= 20: + 1. EVALUATE ACCEPTANCE + Dispatch qa-evaluator in delegated mode, or use direct fallback with fresh QA context. + Check every acceptance criterion, Done Contract item, and Done Contract debate_record independently when a Done Contract exists. + Compare verification evidence to the claimed outcome. + Conditionally load and apply references/domain-rubrics.md only for scoped domain-quality acceptance. + + 2. SCORE + Return qa_scorecard with compact dimensions: + acceptance_coverage, evidence_strength, domain_quality, final_readiness, weighted_score + Return selected_domain_rubrics and domain_quality_scores when domain rubrics were selected. + Record score_entry: round, weighted_score, failed_acceptance_count, delta, drift_status. + If score progression reports STAGNATION, repeated DRIFT, repeated REGRESSION, + or selected domain rubric action pivot, return pivot_restart_signal to the + orchestrator before another QA/build dispatch. + + 3. DECIDE + accepted: no failed acceptance items and evidence proves done. + accepted_with_concerns: acceptance passes with documented non-blocking limitations. + rejected: one or more acceptance or Done Contract items fail. + blocked: required acceptance material or verification evidence is missing. + + 4. FIX OR EXIT + If rejected before round 20, return failed acceptance items to Build for fixes, then re-run QA. + If blocked, return NEEDS_CONTEXT or BLOCKED with open_questions. + If accepted or accepted_with_concerns, exit with final result. + Round 20 is terminal; return remaining failed acceptance items instead of starting round 21. + If pivot_restart_signal was returned, pause QA until the orchestrator + records pivot_restart_decision with exact_next_action. +``` + +## Finding Rules + +QA findings are about acceptance, not general code quality: +- Failed acceptance criterion. +- Done Contract item not proven or contradicted by evidence. +- Done Contract debate_record is missing, has fewer than two perspectives, or does not show pre-build debate/subagent-perspective evidence when a Done Contract exists. +- Verification evidence mismatch or missing proof. +- Product, UX, docs, DX, or domain issue only when scoped by acceptance criteria, Done Contract, domain_context, or rubric_refs. + +Do not report generic code defects, architecture concerns, security issues, or test coverage gaps unless they directly cause an acceptance failure. Those remain Code Reviewer responsibility. + +## Final Result + +The QA loop returns one final result: +- `rounds`: 1-20. +- `final_verdict`: accepted, accepted_with_concerns, rejected, or blocked. +- `result`: CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, or BLOCKED. +- `acceptance_findings`: remaining or resolved acceptance findings with evidence. +- `qa_scorecard`: final compact scores. +- `selected_domain_rubrics`: selected rubric families when scoped domain rubrics were used; empty or omitted when not applicable. +- `domain_quality_scores`: per-family/per-dimension scores when scoped domain rubrics were used; empty or omitted when not applicable. +- `score_progression`: score_entry per round. +- `pivot_restart_signal`: required when QA STAGNATION, repeated DRIFT, repeated REGRESSION, or scoped domain action pivot is detected. +- `evidence`: materials inspected and why they prove or fail acceptance. +- `open_questions`: required when blocked. + +## Pivot/Restart Escalation + +QA does not silently continue after stagnant or pivot-triggered acceptance loops. +When it returns `pivot_restart_signal`, the orchestrator records +`pivot_restart_decision` with trigger, evidence, affected_slice_or_round, +options_considered, selected_action, reapproval_required, next_agent, +recovery_pointer, and exact_next_action. Update harness trace/replay/run-state +when available. Reapproval is required when the selected action changes scope, +files, behavior, risk, verification, or acceptance criteria. diff --git a/skills/assistant-review/references/review-rubric.md b/skills/assistant-review/references/review-rubric.md index 5a41070..36c864b 100644 --- a/skills/assistant-review/references/review-rubric.md +++ b/skills/assistant-review/references/review-rubric.md @@ -107,9 +107,9 @@ As the loop progresses, the bar rises — like a tightening ratchet: | 1 | 4.0+ | 2.5–3.9 | < 2.5 | | 2 | 4.0+ | 2.75–3.9 | < 2.75 | | 3 | 4.0+ | 3.0–3.9 | < 3.0 | -| 4-10 | 4.0+ | 3.25–3.9 | < 3.25 | +| 4-20 | 4.0+ | 3.25–3.9 | < 3.25 | -Rationale: If the code hasn't reached 3.25 by round 4, the approach likely needs rethinking, not more polish. Rounds 4-10 keep that pivot threshold stable while confidence requirements continue to rise. +Rationale: If the code hasn't reached 3.25 by round 4, the approach likely needs rethinking, not more polish. Rounds 4-20 keep that pivot threshold stable while confidence requirements continue to rise. ## Reviewer Scoring Instructions diff --git a/skills/assistant-review/references/score-tracking.md b/skills/assistant-review/references/score-tracking.md index 078f97f..a90d1c8 100644 --- a/skills/assistant-review/references/score-tracking.md +++ b/skills/assistant-review/references/score-tracking.md @@ -50,13 +50,15 @@ Score went up BUT finding count didn't decrease. The evaluator is scoring higher score_delta > 0 AND finding_count_delta >= 0 → DRIFT ``` -Action: **Reset evaluator context.** Dispatch a fresh reviewer agent with an explicitly stricter prompt: +Action: **Reset evaluator context once.** Dispatch a fresh reviewer agent with an explicitly stricter prompt: > "Previous rounds showed score inflation without corresponding quality improvement. > Apply maximum skepticism. Score conservatively — when uncertain, round DOWN. > Do not give benefit of the doubt on any dimension." Also add to the final summary: "Drift detected in round N — evaluator was reset." +Repeated DRIFT after the reset triggers `pivot_restart_signal` and requires an +orchestrator-owned `pivot_restart_decision` before another review dispatch. ### Rule 4: REGRESSION @@ -66,7 +68,7 @@ Score went down. Fixes may have introduced new problems, or the evaluator is cat score_delta < 0 → REGRESSION ``` -Action: This is not necessarily bad — a fresh evaluator may legitimately find more issues. Log it but don't escalate unless regression persists for 2+ consecutive rounds. +Action: This is not necessarily bad — a fresh evaluator may legitimately find more issues. Log it, but 2+ consecutive REGRESSION entries trigger `pivot_restart_signal` and require an orchestrator-owned `pivot_restart_decision` before another fix/review dispatch. ### Rule 5: STAGNATION @@ -76,10 +78,13 @@ Score unchanged for 2+ consecutive rounds with findings still present. score_delta == 0 for 2 consecutive rounds AND finding_count > 0 → STAGNATION ``` -Action: Flag to orchestrator. The loop is churning without progress. Consider: +Action: Return `pivot_restart_signal` to the orchestrator. The loop is churning +without progress. The orchestrator must record `pivot_restart_decision` before +another fix/review dispatch and consider: - Fixes are introducing new issues at the same rate as resolving old ones - The remaining issues may be architectural (can't fix without broader changes) -- May need to PIVOT or accept current state with documented limitations +- The selected recovery may need reset, candidate search, replan, restart, or a + blocked/user path ## Decision Matrix @@ -87,9 +92,9 @@ Action: Flag to orchestrator. The loop is churning without progress. Consider: |---|---|---|---| | +, magnitude ≤ 1.0 | count decreased | **GENUINE** | Continue normally | | +, magnitude > 1.0 | count decreased | **SUSPICIOUS** | Log warning, continue | -| +, any | count same or increased | **DRIFT** | Reset evaluator, stricter prompt | -| − | any | **REGRESSION** | Log, investigate if 2+ rounds | -| 0 | any, findings > 0 remain, 2+ consecutive rounds | **STAGNATION** | Escalate to orchestrator | +| +, any | count same or increased | **DRIFT** | Reset evaluator once; repeated DRIFT triggers pivot_restart_decision | +| − | any | **REGRESSION** | Log; 2+ consecutive regressions trigger pivot_restart_decision | +| 0 | any, findings > 0 remain, 2+ consecutive rounds | **STAGNATION** | Return pivot_restart_signal and require pivot_restart_decision | | 0 | any, findings > 0 remain, 1 round only | **NEUTRAL** | Log, no action yet | | 0 | findings == 0 | *(exit as CLEAN)* | Should not reach drift check | @@ -101,7 +106,7 @@ Record scores in each Quality Review entry: ```markdown ### Quality Review #N -- Round: N of 10 +- Round: N of 20 - Previously fixed: M items from prior rounds - Found this round: X must-fix, Y should-fix, Z nits - Rubric: correctness=4.0 quality=3.5 architecture=4.0 security=5.0 coverage=3.0 @@ -128,7 +133,28 @@ Include in the review exit summary: | Drift Count | Response | |---|---| | 1 occurrence | Reset evaluator, stricter prompt | -| 2 occurrences | Flag to orchestrator, consider different model for evaluation | -| 3+ occurrences | Stop loop, present findings to user for manual review | +| 2 occurrences | Return pivot_restart_signal; orchestrator records pivot_restart_decision | +| 3+ occurrences | Stop current review path until pivot_restart_decision selects reset, candidate search, replan, restart, or blocked/user path | The goal is not to prevent the loop from exiting — it's to ensure that when it exits, the code genuinely earned its passing score. + +## Pivot/Restart Decision Packet + +When STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT +fires, the review loop does not silently continue. It returns +`pivot_restart_signal`; the orchestrator records `pivot_restart_decision` with: + +- `trigger` +- `evidence` +- `affected_slice_or_round` +- `options_considered` +- `selected_action` +- `reapproval_required` +- `next_agent` +- `recovery_pointer` +- `exact_next_action` + +If the selected action changes approved scope, files, behavior, risk, +verification, or acceptance criteria, `reapproval_required` is true and the +workflow waits for approval before Build or Review continues. Round 20 remains +terminal; do not start round 21. diff --git a/skills/assistant-skill-creator/evals/cases.json b/skills/assistant-skill-creator/evals/cases.json index e81cfba..0481666 100644 --- a/skills/assistant-skill-creator/evals/cases.json +++ b/skills/assistant-skill-creator/evals/cases.json @@ -219,6 +219,58 @@ "stored PR numbers" ] } + }, + { + "id": "loop-process-skill-applies-harness-patterns", + "title": "Loop-based process skill applies harness patterns", + "category": "harness_patterns", + "purpose": "Checks that assistant-skill-creator applies the harness-pattern reference when creating a loop-based Process skill.", + "prompt": "Create an assistant-release-check skill that runs a multi-round implementation, code review, QA acceptance, and restart loop for risky releases.", + "setup_context": [ + "The assistant-skill-creator skill instructions are active.", + "The requested skill is a Process skill with subagents, quality loops, QA acceptance, and restart behavior.", + "No design has been approved yet." + ], + "expected_behavior": [ + "Classifies the request as a Process skill.", + "Loads or applies references/harness-patterns.md before designing contracts.", + "Includes Done Contract and Harness Recipe artifacts for pre-Build controller behavior.", + "Uses typed Artifact References for cross-agent artifacts.", + "Keeps Code Reviewer and QA Evaluator handoffs and outputs separate.", + "Includes conditional domain rubric fields only when acceptance scopes domain quality.", + "Adds pivot/restart decision fields for stagnation, drift, regression, rubric pivot, and Code Writer blockers.", + "Bounds review and QA loops with max 20 rounds and terminal round 20 behavior." + ], + "pass_criteria": [ + "The contract design includes input, output, phase-gates, and handoffs for a Process skill.", + "The response explicitly names the harness patterns being applied.", + "The response does not collapse code review and QA into one evaluator.", + "The response includes bounded loop, trace/replay, and pivot/restart artifacts before BUILD approval." + ], + "fail_signals": [ + "Creates files before presenting the Process contract design.", + "Uses only a generic generator/evaluator loop.", + "Sets a max 10 round cap for the review or QA loop.", + "Omits pivot/restart handling for stagnation or Code Writer blockers." + ], + "machine_expectations": { + "required_substrings": [ + "Process", + "references/harness-patterns.md", + "Done Contract", + "Harness Recipe", + "Artifact Reference", + "Code Reviewer", + "QA Evaluator", + "pivot_restart_decision", + "max 20" + ], + "forbidden_substrings": [ + "generic generator/evaluator loop is enough", + "max 10", + "QA replaces code review" + ] + } } ] } diff --git a/skills/assistant-skill-creator/references/harness-patterns.md b/skills/assistant-skill-creator/references/harness-patterns.md index 24fafa9..436af52 100644 --- a/skills/assistant-skill-creator/references/harness-patterns.md +++ b/skills/assistant-skill-creator/references/harness-patterns.md @@ -1,311 +1,307 @@ -# Harness Patterns for Loop-Based Skills +# Harness Patterns for Loop-Based Process Skills -Reference pack for the skill creator. Load this when designing a **Process** skill that includes an evaluation loop, multi-round refinement, or autonomous fix-verify cycles. +Reference pack for `assistant-skill-creator`. Load this when designing a +**Process** skill that includes a long-running controller, subagent dispatch, +multi-round review, QA/acceptance evaluation, retry loops, or autonomous +fix-verify cycles. -**When to load:** The skill being created has any of these characteristics: -- Multi-round loop (review, refinement, optimization) -- Separate generator and evaluator roles -- Quality scoring against criteria -- Autonomous fix → re-check cycles - -**When to skip:** Single-pass skills, analysis pipelines without loops, utility skills. +Skip this reference for single-pass Utility skills or Analysis skills without +delegation or loop control. --- -## Pattern 1: Rubric Scoring +## Pattern 1: Done Contract -Replace open-ended evaluation ("is this good?") with weighted dimensions scored 1-5. +Use a Done Contract when "finished" needs to be known before Build or generation +starts. -### How to apply +Required fields: -1. **Identify 3-6 evaluation dimensions** specific to the skill's domain -2. **Assign weights** that sum to 1.0 — weight subjective dimensions higher to push beyond generic output -3. **Write anchor examples** for each dimension at scores 1, 3, and 5 -4. **Define threshold actions** that map score ranges to loop decisions +- `done_when`: observable pass/fail outcomes. +- `not_done_when`: explicit failure states that block completion. +- `verification`: commands, inspections, reviews, QA, or manual checks. +- `owner_consumer`: artifact owner and downstream consumer. +- `acceptance_criteria`: binary criteria from user, plan, or slice scope. +- `debate_record`: at least two perspectives considered before acceptance. +- `accepted_by`: user, orchestrator, or approved plan reference. -### Template +Design rule: when `subagent_execution_mode=delegated`, use subagent perspectives +for the debate when relevant. Direct fallback must record why subagents were not +used and which role-equivalent perspectives were considered. The debate may +clarify acceptance; it must not add unapproved scope. -```yaml -# In the skill's references/rubric.md or inline in SKILL.md +Contract placement: -dimensions: - - name: [dimension_name] - weight: [0.10-0.40] - measures: "[what this dimension evaluates]" - anchors: - 5: "[concrete description of excellent]" - 3: "[concrete description of acceptable]" - 1: "[concrete description of poor]" +- `output.yaml`: `done_contract` artifact for medium+ harness-capable work. +- `phase-gates.yaml`: a pre-Build assertion that the accepted Done Contract + exists and includes debate evidence. +- `handoffs.yaml`: `done_contract_ref` for workers, reviewers, and QA. -thresholds: - pass: 4.0 # Exit loop — quality sufficient - refine: 3.0 # Continue loop — specific improvements needed - pivot: 2.5 # Escalate — approach may be fundamentally wrong -``` +--- -### Scoring rules to include in the evaluator's prompt - -- Score each dimension independently (no halo effect) -- Cite specific evidence for each score -- When uncertain, round down -- Critical findings can cap the total score (define what's critical for your domain) - -### Contract integration - -Add to the evaluator's **return_fields** in `handoffs.yaml`: - -```yaml -- name: rubric_scores - type: object - required: true - condition: "scope warrants scoring" - object_fields: - - name: [dimension_name] - type: float - required: true - validation: "1.0 to 5.0 in 0.5 increments" - # ... one per dimension - - name: weighted_score - type: float - required: true - - name: action - type: enum - required: true - enum_values: [PASS, REFINE, PIVOT] -``` +## Pattern 2: Harness Recipe -### Example: applying to a content generation skill - -```yaml -dimensions: - - name: accuracy - weight: 0.35 - measures: "Factual correctness, no hallucination, citations where needed" - - name: clarity - weight: 0.25 - measures: "Clear structure, audience-appropriate language, logical flow" - - name: completeness - weight: 0.20 - measures: "All requested topics covered, no significant gaps" - - name: tone - weight: 0.20 - measures: "Matches requested style, consistent voice throughout" -``` +Use a Harness Recipe to choose the controller shape from task/model/risk/context +profiles. -**Reference implementation:** `skills/assistant-review/references/review-rubric.md` +Required profile fields: ---- +- `task_profile`: task type, size, slice count, TDD/debugging needs. +- `model_profile`: model/agent constraints, delegation mode, tool limits. +- `risk_profile`: risk tier, safety gates, review depth, rollback needs. +- `context_profile`: exact, summarized, omitted/deferred context and + trace/replay need. -## Pattern 2: Drift Detection +Required recipe fields: -Track evaluator scores across loop rounds to ensure genuine improvement, not evaluator fatigue. +- `selected_recipe` +- `recipe_rationale` +- `required_artifacts` +- `corrective_action` -### How to apply +Typical recipes: -1. **Record score + finding count per round** in a score history -2. **Compare round N to round N-1** using score delta and finding count -3. **Classify the change** and take appropriate action +- `lightweight_guarded`: medium single-slice, moderate risk, compact context. +- `slice_sequential`: independent slice verification before the next slice. +- `review_intensive`: high/critical risk, weak tests, public contracts, or + subjective acceptance. +- `trace_replay_ready`: long-running, large-context, recovery-prone work. -### Drift classification +Contract placement: -| Score Delta | Finding Condition | Status | Action | -|---|---|---|---| -| +, ≤ 1.0 | count decreased | **GENUINE** | Continue | -| +, > 1.0 | count decreased | **SUSPICIOUS** | Log warning | -| +, any | count same or increased | **DRIFT** | Reset evaluator | -| − | any | **REGRESSION** | Investigate | -| 0 | findings remain, 2+ rounds | **STAGNATION** | Escalate | -| 0 | findings remain, 1 round | **NEUTRAL** | Log only | +- `output.yaml`: `harness_recipe` artifact. +- `phase-gates.yaml`: pre-Build recipe selection assertion. +- `handoffs.yaml`: `harness_recipe_ref` for downstream workers. -### On DRIFT: evaluator reset +--- -Dispatch a fresh evaluator with an explicitly stricter prompt: +## Pattern 3: Runtime State, Trace, And Replay -> "Previous rounds showed score inflation without quality improvement. Apply maximum skepticism. When uncertain, score DOWN." +Long-running Process skills need first-class recovery artifacts. -### Contract integration +| Artifact | Required When | Fields | +|---|---|---| +| Harness Run State | Medium+ harness-capable or delegated loops | task id/name, phase, slice, status, blockers, last verification, next action, recovery pointer | +| Trace Ledger | Any trace/replay-ready loop | agent events, decisions, verification results, deviations, blockers, artifact refs | +| Replay Packet | Compaction, restart, handoff, or failure recovery is likely | pinned context, artifact refs, validation state, exact next action, run-state/trace refs | -Add to **phase-gates.yaml** as loop invariants: +Contract placement: -```yaml -invariants: - - id: INV_DRIFT - check: "Rubric score increases correlate with finding count decreases" - scope: all_rounds - condition: "round >= 2" - on_fail: "Drift detected. Reset evaluator with stricter prompt." +- `output.yaml`: required artifacts or conditional artifacts for trace/replay + workflows. +- `phase-gates.yaml`: assertions that state/trace/replay are current before + phase advancement. +- `handoffs.yaml`: refs carried into worker packets when a role relies on them. - - id: INV_STAGNATION - check: "Score does not stagnate (unchanged 2+ rounds with findings present)" - scope: all_rounds - condition: "round >= 3" - on_fail: "Escalate to orchestrator — loop churning without progress." -``` +--- -Add to **output.yaml** as score progression: - -```yaml -- name: score_progression - type: object[] - required: true - condition: "loop ran 2+ rounds" - object_fields: - - name: round - type: int - - name: weighted_score - type: float - - name: finding_count - type: int - - name: drift_status - type: enum - enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL] -``` +## Pattern 4: Typed Artifact References + +When artifacts cross an agent boundary, pass typed refs instead of free-form +strings. + +Each Artifact Reference entry includes: + +- `artifact_id` +- `artifact_type` +- `producer` +- `consumer` +- `location_ref` +- `schema_or_contract` +- `validation_status` +- `summary` + +Use refs for Done Contract, Harness Recipe, Harness Run State, Trace Ledger, +Replay Packet, Pivot/Restart Decision, task packets, changed files, +verification evidence, review result, QA result, and plan deviations. -**Reference implementation:** `skills/assistant-review/references/score-tracking.md` +Producer responsibility: create/update the artifact, assign a stable id and +location/ref, name the schema or contract, and summarize current state. +Consumer responsibility: validate `schema_or_contract` and +`validation_status` before relying on `location_ref`; invalid or stale refs block +phase advancement or trigger re-dispatch. --- -## Pattern 3: Harness Gates +## Pattern 5: Code Review And QA Separation -Structural enforcement that blocks loop completion without required artifacts. +Do not let one evaluator do every job. -### How to apply +| Role | Owns | Does Not Own | +|---|---|---| +| Code Reviewer | code defects, security, architecture, test coverage, structural code risk | final acceptance or subjective domain quality unless directly tied to a code defect | +| QA Evaluator | Done Contract, acceptance criteria, verification evidence, final readiness, scoped domain quality | generic code review, security architecture, or test coverage review | -1. **Identify the artifacts** that must exist before the loop can exit (rubric scores, passing threshold, all items addressed) -2. **Add phase-gate assertions** that check for these artifacts -3. **Optionally add a Stop hook** for hard enforcement (shell script that parses the task journal) +`reviewer` can remain a compatibility route, but new skills should name the +canonical `code-reviewer` and optional `qa-evaluator` roles separately. -### Phase-gate assertions for loop exit +Contract placement: -```yaml -- id: EXIT_SCORE - check: "Rubric weighted score meets pass threshold" - on_fail: "Score below threshold — continue loop or escalate" +- `handoffs.yaml`: distinct handoffs to Code Reviewer and QAEvaluator. +- `output.yaml`: separate `review_result` and `qa_evaluation_result`. +- `phase-gates.yaml`: QA starts only after build/test evidence and Code + Reviewer or compatibility review evidence exist. -- id: EXIT_FINDINGS - check: "No must-fix or should-fix findings remain" - on_fail: "Findings remain — fix all before exiting" +--- -- id: EXIT_VERIFICATION - check: "Build and tests pass after all fixes" - on_fail: "Verification failed — fix before exiting" -``` +## Pattern 6: Conditional Domain Rubrics -### Stop hook pattern (optional, for hard enforcement) +Use domain rubrics only when acceptance scopes them. Load or model rubric fields +when the Done Contract, acceptance criteria, `domain_context`, or explicit +`rubric_refs` require UI/visual, product, UX, docs, DX, or domain craft quality. -If the skill runs within the workflow (task journal exists), you can add a Stop hook that: -1. Reads the task journal -2. Checks for required artifacts (rubric lines, score threshold, final result) -3. Blocks stop if missing +When scoped, QA returns: -Follow the pattern in `hooks/scripts/harness-gate.sh`: -- Use `set -euo pipefail` -- Check `stop_hook_active` to prevent infinite loops -- Use `jq` for JSON output -- Exit 0 (allow) or output `{"decision": "block", "reason": "..."}` (block) +- `selected_domain_rubrics` +- `domain_quality_scores` +- evidence tied to acceptance criteria, Done Contract, `domain_context`, + `rubric_refs`, verification artifacts, screenshots, docs, or changed files -**Reference implementation:** `hooks/scripts/harness-gate.sh` +When not scoped, QA records domain quality as `not_applicable` and must not +invent subjective rubrics. + +Contract placement: + +- `input.yaml`: optional `domain_context` / `rubric_refs`. +- `handoffs.yaml`: conditional `selected_domain_rubrics` and + `domain_quality_scores`. +- `phase-gates.yaml`: invariant rejecting invented domain rubrics. --- -## Pattern 4: Separation of Generation and Evaluation - -The evaluator must never review its own fixes. - -### How to apply - -1. **Evaluator is read-only** — tools limited to Read, Grep, Glob, LS (no Edit, Write, Bash) -2. **Fresh evaluator per round** — no context contamination from previous evaluations -3. **Previously-fixed list** — each round receives what was already fixed, must not re-report -4. **Fixer is a separate agent** — the orchestrator or a dedicated implementer applies fixes - -### Handoff contract for evaluator - -```yaml -handoffs: - - name: orchestrator_to_evaluator - from: Orchestrator - to: Evaluator - context_fields: - - name: content_to_evaluate - type: string - required: true - - name: round - type: int - required: true - - name: previously_fixed - type: object[] - required: true - description: "Items fixed in prior rounds — do NOT re-report" - - name: confidence_threshold - type: int - required: true - description: "Minimum confidence for this round (increases per round)" - return_fields: - - name: findings - type: object[] - required: true - - name: rubric_scores - type: object - required: true - - name: verdict - type: enum - required: true +## Pattern 7: Bounded Review / QA Loops + +Any autonomous review, QA, refinement, or fix-verify loop needs a hard cap. The +current framework cap is **max 20 rounds**. + +```text +round = 1 +while round <= 20: + evaluate + decide PASS / REFINE / PIVOT + fix or exit + round += 1 ``` -### Confidence progression +Round 20 is terminal. Return remaining blockers, findings, or failed acceptance +items instead of starting round 21. -| Rounds | Threshold | Rationale | -|---|---|---| -| 1–2 | 80% | Cast wide net | -| 3–4 | 85% | Higher certainty required | -| 5 | 90% | Only near-certain findings | +Include these fields where applicable: -**Reference implementation:** `skills/assistant-review/contracts/handoffs.yaml` +- `round`: current round number, validation `>= 1 and <= 20`. +- `max_rounds`: default `20`. +- `previously_fixed` or `previously_failed_acceptance_items`. +- `score_progression` or `score_entry`. +- `loop_exit_reason`. --- -## Decision Checklist +## Pattern 8: Rubric Scoring And Drift Detection -When creating a Process skill with a loop, check which patterns apply: +Use scored rubrics when quality is subjective or multi-dimensional. -| Question | If Yes → Apply | -|---|---| -| Does the loop evaluate quality? | Pattern 1 (Rubric) | -| Does the loop run 2+ rounds? | Pattern 2 (Drift Detection) | -| Must the loop complete before the task finishes? | Pattern 3 (Harness Gates) | -| Does one agent create and another evaluate? | Pattern 4 (Separation) | +Scoring fields: + +- 3-6 domain-relevant dimensions, weights summing to 1.0. +- Anchors for 1, 3, and 5. +- `weighted_score`. +- action enum such as `PASS`, `REFINE`, `PIVOT`. +- evidence for each score. + +Drift classification: + +| Signal | Condition | Action | +|---|---|---| +| GENUINE | score improves and findings decrease | continue | +| SUSPICIOUS | score jumps sharply while findings decrease | log warning | +| DRIFT | score improves while findings stay flat or rise | reset evaluator | +| REGRESSION | score drops | investigate | +| STAGNATION | findings remain across repeated unchanged scores | pivot/restart | -Most loop-based skills will use all four. Simple refinement loops (e.g., "retry until build passes") may only need Pattern 3. +Add cross-phase invariants for drift and stagnation. Repeated `DRIFT`, repeated +`REGRESSION`, `STAGNATION`, or rubric action `PIVOT` must return a +`pivot_restart_signal`; the orchestrator records the actual +`pivot_restart_decision`. --- -## Full reference +## Pattern 9: Pivot / Restart Decisions -See `docs/harness-design-guide.md` for the complete design guide with architecture rationale, research references, and evolution principles. +Use Pivot/Restart when the active loop or handoff no longer makes safe progress. +Triggers: + +- review or QA `STAGNATION` +- repeated `DRIFT` +- repeated `REGRESSION` +- rubric/domain action `PIVOT` +- Code Writer blocker types such as `legacy_code_bug`, `broken_baseline`, + `hidden_dependency`, `missing_contract`, `stale_plan`, `scope_conflict`, + `tool_environment`, `permission_policy`, `tdd_red_missing`, or `other` +- verification blockers, plan deviations, or scope changes that stale the packet + +Decision fields: + +- `trigger` +- `evidence` +- `affected_slice_or_round` +- `options_considered` +- `selected_action` +- `reapproval_required` +- `next_agent` +- `recovery_pointer` +- `exact_next_action` + +Routing examples: + +- Legacy code bug or broken baseline -> debugging before another implementation + attempt. +- Hidden dependency -> Explorer or Code Mapper refresh. +- Missing contract or stale plan -> Architect or Plan repair. +- Scope conflict or candidate pivot -> replan and reapproval when scope changes. +- Tool/environment/policy blocker -> environment fix, permission request, or + BLOCKED with evidence. --- -## Pattern 5: Agentic Loop Safety +## Pattern 10: Agentic Loop Safety -Apply this pattern whenever a Process skill includes autonomous or repeated execution: agent loops, retry loops, search/retrieval loops, multi-round subagent dispatch, model/tool call loops, or background jobs. +Apply this pattern to every repeated agent/tool/model loop. Required design fields: -- **Bounded execution:** max steps/rounds, timeout, or budget. -- **Stop condition:** explicit success, clean exit, max-budget exit, blocker exit, and escalation path. -- **Retry policy:** capped retries scoped to transient failures only. -- **Empty-result handling:** fallback, broaden-once, report no evidence, ask/escalate, or exit. -- **Tool-error handling:** failures route to retry, fallback, blocker, or degraded result; never silent continuation. -- **Progress signal:** each iteration must produce new evidence, fewer findings, better score, or another measurable improvement. -- **Cost/token guard:** paid API/model/subagent/large-context loops need cost/time/token awareness. +- bounded execution: max rounds, steps, timeout, or budget +- stop condition: success, clean exit, max-budget exit, blocker exit +- retry policy: capped retries for transient failures only +- empty-result handling: broaden once, fallback, report no evidence, ask, or exit +- tool-error routing: retry, fallback, blocker, or degraded result +- progress signal: new evidence, fewer findings, better score, or explicit state +- cost/token guard: paid model, subagent, and large-context loops need budget + awareness -Contract placement: +A loop without bounds, stop conditions, empty-result handling, tool-error +routing, and a progress signal is incomplete even when the happy path works. -- Put the loop bounds and retry policy in `contracts/input.yaml` or phase-gate configuration when user/config controlled. -- Put step-level assertions in `contracts/phase-gates.yaml`. -- Put final budget/loop outcome reporting in `contracts/output.yaml`. -- Put worker/subagent loop fields in `contracts/handoffs.yaml` when a subagent participates. +--- + +## Decision Checklist -A loop without bounds, stop conditions, empty-result handling, and tool-error routing is incomplete even if it works in a happy-path demo. +For a loop-based Process skill, require the matching patterns: + +| Question | Apply | +|---|---| +| Does Build need a definition of done? | Done Contract | +| Does controller shape vary by task/risk/context? | Harness Recipe | +| Could context compaction, restart, or handoff happen? | Run State, Trace, Replay | +| Do artifacts cross role boundaries? | Typed Artifact References | +| Are code quality and acceptance both evaluated? | Code Reviewer / QA Evaluator split | +| Is subjective/domain quality part of acceptance? | Conditional Domain Rubrics | +| Does a loop run multiple rounds? | Max 20 loop cap, scoring, drift detection | +| Can the approach go stale or hit legacy blockers? | Pivot/Restart Decision | + +Most loop-based Process skills use all of these. Simple retry-only skills still +need bounded execution, stop conditions, error routing, and progress signals. + +Full rationale and implementation references live in +`docs/harness-design-guide.md` and +`skills/assistant-workflow/references/harness-controller.md`. diff --git a/skills/assistant-workflow/SKILL.md b/skills/assistant-workflow/SKILL.md index a035df9..732deaa 100644 --- a/skills/assistant-workflow/SKILL.md +++ b/skills/assistant-workflow/SKILL.md @@ -23,6 +23,9 @@ This skill is intentionally agent-agnostic: it must work in restricted company e - Triage, discovery, planning, build, review, and document phases run at the smallest useful depth. - Medium+ work has an approved plan before implementation; small work has an inline plan and proceeds without ceremony unless risk requires approval. +- Medium+ harness-capable work has an accepted Done Contract and Harness Recipe before Build. +- Trace/replay-ready harness work maintains Harness Run State, Trace Ledger, and Replay Packet artifacts. +- Medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work records independent QA Evaluator evidence after build/test and code-review evidence. - Behavior changes have tests or explicit validation attached to the implementation step they protect. - Final output reports changed files, verification evidence, residual risks, and next steps. - Candidate Search is used for explicit alternatives, open-ended architecture/design, optimization, high uncertainty, repeated failed attempts, unclear/flaky bugs, or reviewer-requested pivots — not as default ceremony. @@ -202,12 +205,13 @@ Load `references/phases.md` and execute the phase matching your current stage. U | **Plan** | All sizes | Implementation steps with file paths. Load `references/plan-template.md`. Small tasks use inline no-wait plans unless risk/ambiguity requires approval; medium+ tasks use the single approval gate for scope, slices, and build plan. | | **Design** | UI tasks only | Design direction, mockup, production checklist. Approval gate. | | **Build** | All sizes | One step at a time. Code Writer -> Builder/Tester. Tests alongside code. | -| **Review** | All sizes | Stage 1: Spec Review. Stage 2: load and follow `assistant-review` SKILL.md and contracts. | +| **Review** | All sizes | Stage 1: Spec Review. Stage 2: load and follow `assistant-review` SKILL.md and contracts for code review. Stage 3: QA Evaluator runs when acceptance QA is required. | | **Document** | All sizes | Small: metrics only. Medium+: docs + metrics + reflection. | -For subagent roles and dispatch rules, load `references/subagent-dispatch.md` and resolve `subagent_policy_state`, `subagent_execution_mode`, and `subagent_authorization_scope` before any subagent spawn. Assistant Framework policy requires explicit user authorization before spawning subagents for development/code-work roles unless the current user prompt already explicitly authorizes them for this task. Ask once for the needed delegation scope and wait before continuing phases that require subagents. A sufficient prompt is: `This workflow expects Code Mapper, Architect, Code Writer, Builder/Tester, and Reviewer subagents for [scope]. May I use subagents for this task?` After authorization, use `delegated` mode and spawn the configured role agents. Use `direct_fallback` only when authorization is denied, policy disallows spawning, or a real spawn attempt fails because subagents/custom agents are unavailable; do not infer unavailability merely because no visible tool is named `Task`, `delegate`, or `subagent`. +For subagent roles and dispatch rules, load `references/subagent-dispatch.md` and resolve `subagent_policy_state`, `subagent_execution_mode`, and `subagent_authorization_scope` before any subagent spawn. Assistant Framework policy requires explicit user authorization before spawning subagents for development/code-work roles unless the current user prompt already explicitly authorizes them for this task. Ask once for the needed delegation scope and wait before continuing phases that require subagents. A sufficient prompt is: `This workflow expects Code Mapper, Architect, Code Writer, Builder/Tester, Code Reviewer, and QA Evaluator when QA is required for [scope]. May I use subagents for this task?` After authorization, use `delegated` mode and spawn the configured role agents. Use `direct_fallback` only when authorization is denied, policy disallows spawning, or a real spawn attempt fails because subagents/custom agents are unavailable; do not infer unavailability merely because no visible tool is named `Task`, `delegate`, or `subagent`. For BES-style option exploration, load `references/candidate-search.md` only when `search_mode: candidate_search` is selected. For mega tasks and anti-patterns, load `references/mega-and-patterns.md`. +Load `references/harness-controller.md` only when medium+ work is harness-capable: long-running, multi-slice, delegated, high-risk, subjective/domain-scored, trace/replay-ready, or explicitly requested as harness work. ## Planning Checklist @@ -239,7 +243,8 @@ Before final output for medium+ or high-risk work: - Run the relevant build/test/lint/typecheck commands available in the repo. - Review the diff against the acceptance criteria. - For bugfixes, verify the review material includes reproduction/root-cause evidence from `assistant-debugging` or a clear reason it was not applicable. -- Use `assistant-review` for quality/spec review when the change is non-trivial. +- Use `assistant-review` for code quality/spec review when the change is non-trivial. +- Use QA Evaluator through `assistant-review` for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested acceptance QA after build/test and code-review evidence. - Use `assistant-security` when touching auth, user input, secrets, persistence, network calls, shell commands, dependency/config changes, or external integrations. Review findings must cite evidence and concrete risk. Avoid generic style feedback unless it affects correctness, security, maintainability, or test reliability. diff --git a/skills/assistant-workflow/contracts/handoffs.yaml b/skills/assistant-workflow/contracts/handoffs.yaml index 79a2e05..5d9bb4d 100644 --- a/skills/assistant-workflow/contracts/handoffs.yaml +++ b/skills/assistant-workflow/contracts/handoffs.yaml @@ -16,6 +16,7 @@ skill: assistant-workflow # ───────────────────────────────────────────── worker_status_protocol: description: "CodeMapper, Explorer, Architect, CodeWriter, BuilderTester, and Reviewer returns include a compact status packet so the orchestrator can route results deterministically." + qa_evaluator_description: "QAEvaluator returns include a compact status packet for acceptance, Done Contract, verification evidence, score progression, and final QA result; it does not replace Reviewer/code-reviewer." statuses: - value: DONE meaning: "No known concerns; requested work completed within scope." @@ -34,6 +35,7 @@ worker_status_protocol: - "evidence is required for CodeMapper, Explorer, Architect, CodeWriter, and Reviewer returns with DONE, DONE_WITH_CONCERNS, DEVIATED, or FAILED_VERIFICATION." - "verification.evidence is required for BuilderTester returns." - "open_questions is required for NEEDS_CONTEXT and BLOCKED." + - "blocker_type and blocker_evidence are required for CodeWriter returns with NEEDS_CONTEXT, BLOCKED, or DEVIATED when implementation hits an unexpected blocker." - "deviation_details is required for Architect and CodeWriter returns with DEVIATED." - "CodeMapper, Explorer, and Reviewer status values are limited to DONE, DONE_WITH_CONCERNS, NEEDS_CONTEXT, and BLOCKED." - "Architect and CodeWriter status values also include DEVIATED." @@ -41,6 +43,24 @@ worker_status_protocol: - "changed_files and files_changed are required with at least one item for CodeWriter returns with DONE, DONE_WITH_CONCERNS, or DEVIATED; they may be omitted or empty for NEEDS_CONTEXT/BLOCKED returns before file changes occur." - "verification is required for BuilderTester returns; if status is NEEDS_CONTEXT or BLOCKED before verification runs, return result not_run with concise blocker evidence." - "findings is required for Reviewer returns." + - "acceptance_findings, qa_scorecard, final_verdict, result, evidence, and score_entry are required for QAEvaluator returns; selected_domain_rubrics and domain_quality_scores are required only when scoped domain rubrics are selected." + +# ───────────────────────────────────────────── +# Typed Artifact Reference Protocol +# ───────────────────────────────────────────── +artifact_reference_protocol: + description: "Typed Artifact Reference entries are the compact producer/consumer ledger for harness task packets and worker returns; use them instead of ad hoc string refs when passing artifacts between agents." + required_fields: [artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, summary] + artifact_types: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + validation_statuses: [pending, valid, invalid, stale, not_applicable] + packet_rules: + - "Artifact Reference entries must include artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, and summary." + - "location_ref is the typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable external artifact ref." + - "Producer responsibility: create or update the artifact, assign artifact_id, set artifact_type, location_ref, schema_or_contract, validation_status, and summary." + - "Consumer responsibility: validate schema_or_contract and validation_status before relying on location_ref; stale, invalid, or missing refs block phase advancement or trigger re-dispatch." + - "CodeWriter and BuilderTester task packets receive artifact_refs for Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, Pivot/Restart Decision, changed files, verification evidence, and plan deviation refs when applicable." + - "CodeWriter returns produced artifact_refs for changed files and plan deviation refs when applicable." + - "BuilderTester returns validated artifact_refs for verification evidence and runtime-artifact validation." # ───────────────────────────────────────────── # Orchestrator → CodeMapper @@ -669,6 +689,50 @@ handoffs: type: string[] required: true description: "Observable behavior this task packet must deliver" + - name: done_contract_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the accepted Done Contract containing done_when, not_done_when, verification, owner_consumer, acceptance_criteria, and debate_record" + - name: harness_recipe_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the selected Harness Recipe based on task/model/risk/context profile" + - name: artifact_refs + type: object[] + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + min_items: 5 + description: "Typed Artifact Reference entries that the task packet passes to CodeWriter and BuilderTester; include Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, changed files, verification evidence, and plan deviation refs when applicable." + object_fields: + - name: artifact_id + type: string + required: true + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + - name: producer + type: string + required: true + - name: consumer + type: string + required: true + - name: location_ref + type: string + required: true + description: "Typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable artifact ref" + - name: schema_or_contract + type: string + required: true + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + - name: summary + type: string + required: true - name: test_criteria type: string[] required: true @@ -805,6 +869,50 @@ handoffs: - name: acceptance_criteria type: string[] required: true + - name: done_contract_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the accepted Done Contract containing done_when, not_done_when, verification, owner_consumer, acceptance_criteria, and debate_record" + - name: harness_recipe_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the selected Harness Recipe based on task/model/risk/context profile" + - name: artifact_refs + type: object[] + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + min_items: 5 + description: "Typed Artifact Reference entries CodeWriter receives before implementation; include Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, changed files, verification evidence, and plan deviation refs when applicable." + object_fields: + - name: artifact_id + type: string + required: true + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + - name: producer + type: string + required: true + - name: consumer + type: string + required: true + - name: location_ref + type: string + required: true + description: "Typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable artifact ref" + - name: schema_or_contract + type: string + required: true + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + - name: summary + type: string + required: true - name: files_to_create type: string[] required: true @@ -930,6 +1038,34 @@ handoffs: required: false condition: "status in [NEEDS_CONTEXT, BLOCKED]" description: "Questions or blocker details the orchestrator must resolve before implementation can continue" + - name: blocker_type + type: enum + required: conditional + condition: "status in [NEEDS_CONTEXT, BLOCKED, DEVIATED] because implementation hit an unexpected blocker, missing contract, stale plan, scope conflict, or required RED evidence gap" + enum_values: [legacy_code_bug, broken_baseline, hidden_dependency, missing_contract, stale_plan, scope_conflict, tool_environment, permission_policy, tdd_red_missing, other] + description: "Classifies the implementation blocker so the orchestrator can route recovery instead of asking CodeWriter to improvise through it." + validation: "Use the most specific blocker type. DEVIATED is required when continuing would change approved scope, files, behavior, risk, or verification expectations." + on_missing: fail + - name: blocker_evidence + type: object[] + required: conditional + condition: "blocker_type is present" + min_items: 1 + description: "Evidence for the blocker: file paths, baseline failures, missing contract fields, stale task-packet assumptions, tool/environment errors, or scope conflict details." + on_missing: fail + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: recovery_hint + type: enum + required: false + condition: "blocker_type is present" + enum_values: [debugging, explorer, architect, candidate_search, replan, restart, user_clarification, environment_fix, permission_request] + description: "Optional non-binding recovery hint; the orchestrator owns the final pivot_restart_decision." - name: files_changed type: object[] required: false @@ -947,6 +1083,40 @@ handoffs: - name: description type: string required: true + - name: artifact_refs + type: object[] + required: false + condition: "required with min_items: 1 when status in [DONE, DONE_WITH_CONCERNS, DEVIATED] and current_task_packet.artifact_refs is present or harness_capable == true; optional and may be empty or omitted when status in [NEEDS_CONTEXT, BLOCKED]" + min_items: 0 + description: "Typed Artifact Reference entries produced or updated by CodeWriter; include changed_files refs for modified files and plan_deviation refs when implementation departs from the approved task packet." + object_fields: + - name: artifact_id + type: string + required: true + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + - name: producer + type: string + required: true + - name: consumer + type: string + required: true + - name: location_ref + type: string + required: true + description: "Typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable artifact ref" + - name: schema_or_contract + type: string + required: true + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + - name: summary + type: string + required: true - name: tests_written type: object[] required: true @@ -968,7 +1138,7 @@ handoffs: required: false description: "What the plan assumed vs. what was found" condition: "when deviation_detected is true or status is DEVIATED" - on_missing_field: "CodeWriter must always return status and deviation_detected at minimum. For DONE, DONE_WITH_CONCERNS, or DEVIATED, it must also return changed_files, evidence, and files_changed with real implementation evidence; for NEEDS_CONTEXT/BLOCKED, require open_questions and do not require fabricated changed-file entries." + on_missing_field: "CodeWriter must always return status and deviation_detected at minimum. For DONE, DONE_WITH_CONCERNS, or DEVIATED, it must also return changed_files, evidence, files_changed, and typed artifact_refs when the task packet carried artifact_refs or harness_capable == true; for NEEDS_CONTEXT/BLOCKED, require open_questions and do not require fabricated changed-file or artifact-ref entries. When implementation hit an unexpected blocker, require blocker_type and blocker_evidence before the orchestrator routes recovery." # ───────────────────────────────────────────── # Orchestrator → BuilderTester @@ -1021,6 +1191,50 @@ handoffs: - name: acceptance_criteria type: string[] required: true + - name: done_contract_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the accepted Done Contract containing done_when, not_done_when, verification, owner_consumer, acceptance_criteria, and debate_record" + - name: harness_recipe_ref + type: string + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + description: "Pointer to the selected Harness Recipe based on task/model/risk/context profile" + - name: artifact_refs + type: object[] + required: conditional + condition: "required when size in [medium, large, mega] and harness_capable == true" + min_items: 5 + description: "Typed Artifact Reference entries BuilderTester receives before verification; include Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, changed files, verification evidence, and plan deviation refs when applicable." + object_fields: + - name: artifact_id + type: string + required: true + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + - name: producer + type: string + required: true + - name: consumer + type: string + required: true + - name: location_ref + type: string + required: true + description: "Typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable artifact ref" + - name: schema_or_contract + type: string + required: true + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + - name: summary + type: string + required: true - name: files_to_test type: string[] required: true @@ -1060,6 +1274,40 @@ handoffs: type: string[] required: true description: "Concise success signals or failure messages, not raw logs" + - name: artifact_refs + type: object[] + required: false + condition: "required with min_items: 1 when current_task_packet.artifact_refs is present or harness_capable == true; optional and may be empty or omitted when verification.result is not_run for NEEDS_CONTEXT or BLOCKED" + min_items: 0 + description: "Typed Artifact Reference entries validated or produced by BuilderTester; include verification_evidence refs and validation_status updates for Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, changed files, and plan deviation refs." + object_fields: + - name: artifact_id + type: string + required: true + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + - name: producer + type: string + required: true + - name: consumer + type: string + required: true + - name: location_ref + type: string + required: true + description: "Typed location/ref pointer: file path, section heading, dispatch id, command ref, or stable artifact ref" + - name: schema_or_contract + type: string + required: true + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + - name: summary + type: string + required: true - name: open_questions type: string[] required: false @@ -1100,7 +1348,7 @@ handoffs: type: string required: true description: "Concise error — assertion message + expected/actual, not stack trace" - on_missing_field: "Re-dispatch BuilderTester. status, verification, build_result, and test_summary are never optional." + on_missing_field: "Re-dispatch BuilderTester. status, verification, build_result, and test_summary are never optional; typed artifact_refs are required when the task packet carried artifact_refs or harness_capable == true." # ───────────────────────────────────────────── # Orchestrator → Reviewer @@ -1132,8 +1380,8 @@ handoffs: - name: round type: int required: true - description: "Current review round number (1-10)" - validation: ">= 1 and <= 10" + description: "Current review round number (1-20)" + validation: ">= 1 and <= 20" - name: previously_fixed type: object[] required: true @@ -1145,13 +1393,13 @@ handoffs: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: confidence_threshold type: int required: true description: "Minimum confidence percentage for reporting findings" validation: "Must be one of: 80, 85, 90" - notes: "Round 1-3: 80, Round 4-7: 85, Round 8-10: 90" + notes: "Round 1-3: 80, Round 4-15: 85, Round 16-20: 90" return_fields: - name: status type: enum @@ -1161,7 +1409,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: findings type: object[] required: true @@ -1215,6 +1463,257 @@ handoffs: description: "Questions or blocker details the orchestrator must resolve" on_missing_field: "Re-dispatch Reviewer. status, findings, evidence, and verdict are never optional." + # ───────────────────────────────────────────── + # Orchestrator → QAEvaluator + # ───────────────────────────────────────────── + - name: orchestrator_to_qa_evaluator + from: Orchestrator + to: QAEvaluator + phase: REVIEW + notes: "Independent QA acceptance evaluation after BuilderTester verification and Code Reviewer or Reviewer compatibility result. Does not replace code-reviewer." + context_fields: + - name: done_contract + type: object + required: conditional + condition: "task has an accepted Done Contract" + description: "Accepted Done Contract for the work under QA evaluation" + validation: "Includes done_when, not_done_when, verification, owner_consumer, acceptance_criteria, debate_record with at least two perspectives, and accepted_by when present" + object_fields: + - name: done_when + type: string[] + required: true + - name: not_done_when + type: string[] + required: true + - name: verification + type: string[] + required: true + - name: owner_consumer + type: string + required: true + - name: acceptance_criteria + type: string[] + required: true + - name: debate_record + type: object[] + required: true + min_items: 2 + description: "pre-build debate/subagent-perspective evidence considered before accepting the Done Contract" + validation: "At least two perspectives; delegated mode uses subagent perspectives when available" + object_fields: + - name: perspective + type: string + required: true + - name: concern_or_support + type: string + required: true + - name: resolution + type: string + required: true + - name: accepted_by + type: string + required: true + - name: acceptance_criteria + type: string[] + required: true + description: "Binary acceptance criteria from user request, approved plan, slice manifest, or Done Contract" + min_items: 1 + - name: verification_evidence + type: object[] + required: true + description: "BuilderTester, manual, inspection, or review evidence that should prove acceptance" + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: result + type: string + required: true + - name: proves + type: string + required: true + - name: code_review_result + type: object + required: true + description: "Final Code Reviewer or Reviewer compatibility result that QA must not replace" + object_fields: + - name: role + type: enum + required: true + enum_values: [CodeReviewer, Reviewer] + - name: result + type: string + required: true + - name: rounds + type: int + required: true + validation: ">= 1 and <= 20" + - name: domain_context + type: object + required: false + description: "Scoped product, UX, UI/visual, docs, DX, or domain context; enables conditional use of assistant-review references/domain-rubrics.md" + object_fields: + - name: summary + type: string + required: true + - name: in_scope_surfaces + type: string[] + required: true + - name: rubric_refs + type: string[] + required: false + default: [] + description: "Explicit product/UX/docs/DX/UI/domain rubric family references from assistant-review references/domain-rubrics.md" + - name: round + type: int + required: true + description: "Current QA evaluation round number (1-20)" + validation: ">= 1 and <= 20" + - name: previously_failed_acceptance_items + type: object[] + required: true + description: "Failed acceptance items from prior QA rounds" + object_fields: + - name: criterion + type: string + required: true + - name: failed_in_round + type: int + required: true + validation: ">= 1 and <= 20" + - name: evidence_gap + type: string + required: true + - name: qa_filter_policy + type: string + required: true + description: "Acceptance findings require evidence and direct acceptance impact; speculative concerns are non-blocking; round 20 is terminal" + return_fields: + - name: status + type: enum + required: true + enum_values: [DONE, DONE_WITH_CONCERNS, NEEDS_CONTEXT, BLOCKED] + description: "Worker status packet result; DONE requires final_verdict=accepted" + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: acceptance_findings + type: object[] + required: true + description: "Acceptance, Done Contract, verification, or scoped domain findings" + object_fields: + - name: severity + type: enum + required: true + enum_values: [blocker, concern, observation] + - name: criterion + type: string + required: true + - name: evidence + type: string + required: true + - name: impact + type: string + required: true + - name: qa_scorecard + type: object + required: true + description: "Compact QA scorecard; domain_quality is backed by selected_domain_rubrics/domain_quality_scores only when scoped domain rubrics were used" + object_fields: + - name: acceptance_coverage + type: float + required: true + - name: evidence_strength + type: float + required: true + - name: domain_quality + type: float + required: true + - name: final_readiness + type: float + required: true + - name: weighted_score + type: float + required: true + - name: selected_domain_rubrics + type: string[] + required: false + condition: "domain_context or rubric_refs is present, or acceptance criteria / Done Contract require subjective/product/UX/docs/DX/UI/domain craft evaluation" + description: "Rubric families selected from assistant-review references/domain-rubrics.md; empty or omitted when no scoped domain rubric applies" + - name: domain_quality_scores + type: object[] + required: false + condition: "selected_domain_rubrics is non-empty" + description: "Per-family domain rubric scores with evidence" + object_fields: + - name: rubric_ref + type: string + required: true + - name: dimension + type: string + required: true + - name: score + type: float + required: true + validation: "1.0 to 5.0 in 0.5 increments. For action=not_applicable, use 5.0 and state the unscoped rationale in evidence." + - name: action + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, refine, rejected, pivot, not_applicable] + - name: evidence + type: string + required: true + - name: final_verdict + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, rejected, blocked] + - name: result + type: enum + required: true + enum_values: [CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, BLOCKED] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: score_entry + type: object + required: true + description: "QA score progression entry for this round" + object_fields: + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: weighted_score + type: float + required: true + - name: failed_acceptance_count + type: int + required: true + - name: drift_status + type: enum + required: true + enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL, NOT_APPLICABLE] + - name: score_progression + type: object[] + required: false + description: "Full QA score progression when prior round history is available" + - name: open_questions + type: string[] + required: false + condition: "status in [NEEDS_CONTEXT, BLOCKED] or final_verdict == blocked" + description: "Questions or blocker details the orchestrator must resolve" + on_missing_field: "Re-dispatch QAEvaluator. status, acceptance_findings, qa_scorecard, final_verdict, result, evidence, and score_entry are never optional. When scoped domain rubrics are selected, selected_domain_rubrics and domain_quality_scores are also required; when not scoped, do not fabricate them." + # ───────────────────────────────────────────── # Validation behavior for all handoffs: # @@ -1222,5 +1721,6 @@ handoffs: # 2. After return: verify all required return_fields are present, including the worker status packet # 3. If on_missing_field action is "re-dispatch": retry once with explicit request # 4. If second dispatch also misses required fields: log the gap and route to direct fallback, BLOCKED, or a degraded result with explicit evidence; do not fabricate required fields -# 5. Never silently proceed with missing required fields — always log what was missing and how the missing evidence was resolved, deferred, or blocked +# 5. When CodeWriter returns blocker_type, the orchestrator records the blocker evidence and routes to debugging, explorer, architect, candidate_search, replan, restart, user clarification, environment fix, or permission request. Do not ask CodeWriter to widen scope or patch blindly through a legacy blocker. +# 6. Never silently proceed with missing required fields — always log what was missing and how the missing evidence was resolved, deferred, or blocked # ───────────────────────────────────────────── diff --git a/skills/assistant-workflow/contracts/input.yaml b/skills/assistant-workflow/contracts/input.yaml index f4f2439..f107b5e 100644 --- a/skills/assistant-workflow/contracts/input.yaml +++ b/skills/assistant-workflow/contracts/input.yaml @@ -47,7 +47,7 @@ fields: type: string[] required: false description: "Agent or skill roles required by size, task type, and risk tier" - validation: "Entries must be known workflow roles or installed skills, or state a clear project-specific role. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts must include at minimum Code Writer, Builder/Tester, and Reviewer responsibilities." + validation: "Entries must be known workflow roles or installed skills, or state a clear project-specific role. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts must include at minimum Code Writer, Builder/Tester, and Code Reviewer responsibilities. Reviewer may satisfy only compatibility routing for existing/legacy handoffs when explicitly recorded." default: [] on_missing: skip @@ -135,6 +135,20 @@ fields: on_missing: skip notes: "Populated by Idea-to-Action pipeline when input is a vague idea rather than a concrete task" + - name: qa_evaluation_mode + type: enum + required: false + enum_values: [not_required, optional, required] + description: "Whether Review phase should run the independent QA Evaluator acceptance loop after build/test and code-review evidence" + validation: "required for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work; not_required must be justified when a medium+ harness task skips QA" + default: not_required + on_missing: infer + infer_from: > + Use required when the task is medium+ harness-capable, has a Done Contract, is domain-scored, + affects UI/visual/product/UX/docs/DX quality, or explicitly requests QA/acceptance evaluation. + Use optional when acceptance material exists but QA is not required. + Use not_required for trivial/small code-review-only work with no separate acceptance evidence need. + - name: search_mode type: enum required: false diff --git a/skills/assistant-workflow/contracts/output.yaml b/skills/assistant-workflow/contracts/output.yaml index 11431d4..f913222 100644 --- a/skills/assistant-workflow/contracts/output.yaml +++ b/skills/assistant-workflow/contracts/output.yaml @@ -18,13 +18,13 @@ completion_tiers: checkpoint_policy: "inline plan/check summary is enough; phase_checkpoints are strict-profile only" medium: required_artifacts: [triage_result, changed_files, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, plan_document, user_approval] - conditional_artifacts: [test_results, task_journal, context_map, slice_manifest, single_slice_rationale, slice_verification_summary, metrics_entry, candidate_search_result] - review_gate: "spec review plus quality review, with evidence matched to the approved plan" + conditional_artifacts: [test_results, task_journal, context_map, slice_manifest, single_slice_rationale, slice_verification_summary, metrics_entry, candidate_search_result, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result] + review_gate: "spec review plus code quality review, with independent QA evaluation when required and evidence matched to the approved plan" checkpoint_policy: "task journal or carried-forward state records phase transitions" large_critical: required_artifacts: [triage_result, phase_checkpoints, changed_files, test_results, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, task_journal, context_map, context_budget_note, slice_manifest, slice_verification_summary, plan_document, user_approval] - conditional_artifacts: [debugging_result, artifact_contract, pattern_retrieval_note, decomposition_plan_review, single_slice_rationale, metrics_entry, candidate_search_result, verified_skill_distillation] - review_gate: "full strict evidence, security/operability review when applicable, and all slice criteria verified" + conditional_artifacts: [debugging_result, artifact_contract, pattern_retrieval_note, decomposition_plan_review, single_slice_rationale, metrics_entry, candidate_search_result, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result, verified_skill_distillation] + review_gate: "full strict evidence, security/operability review when applicable, QA evaluation when required, and all slice criteria verified" checkpoint_policy: "explicit phase checkpoints are required" artifacts: @@ -216,8 +216,8 @@ artifacts: type: object required: conditional condition: "size in [medium, large, mega] or risk_tier in [high, critical] or hook_profile == strict" - validation: "Stage 2 assistant-review quality result only. Spec compliance is recorded in spec_review_result." - on_fail: "Run assistant-review quality review and record quality status, rounds, and resolved findings" + validation: "Stage 2 assistant-review quality result only. Spec compliance is recorded in spec_review_result. This is the code quality review result; QA acceptance is recorded separately in qa_evaluation_result when required." + on_fail: "Run assistant-review code quality review and record quality status, rounds, and resolved findings" object_fields: - name: quality_review_status type: enum @@ -226,13 +226,140 @@ artifacts: - name: review_rounds type: int required: true - validation: ">= 1 and <= 10" + validation: ">= 1 and <= 20" - name: must_fix_resolved type: int required: true - name: should_fix_resolved type: int required: true + - name: pivot_restart_decision_ref + type: string + required: false + condition: "quality review loop observed STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT" + description: "Reference to the orchestrator-owned pivot_restart_decision artifact created before another review/fix dispatch." + + - name: qa_evaluation_result + type: object + required: conditional + condition: "qa_evaluation_mode == required or size in [medium, large, mega] and harness_capable == true and work is domain-scored/UI/visual/product/UX/docs/DX-facing" + validation: "Independent QA evaluation result exists after build/test evidence and Code Reviewer result. Must include final_verdict, result, rounds (1-20), acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, and evidence." + on_fail: "Run the QAEvaluator loop from assistant-review references/qa-evaluation-loop.md, or record NEEDS_CONTEXT/BLOCKED with open_questions. Do not substitute Code Reviewer findings for QA evaluation." + object_fields: + - name: final_verdict + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, rejected, blocked] + - name: result + type: enum + required: true + enum_values: [CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, BLOCKED] + - name: rounds + type: int + required: true + validation: ">= 1 and <= 20" + - name: acceptance_findings + type: object[] + required: true + object_fields: + - name: severity + type: enum + required: true + enum_values: [blocker, concern, observation] + - name: criterion + type: string + required: true + - name: evidence + type: string + required: true + - name: disposition + type: enum + required: true + enum_values: [resolved, remaining, not_applicable] + - name: qa_scorecard + type: object + required: true + object_fields: + - name: acceptance_coverage + type: float + required: true + - name: evidence_strength + type: float + required: true + - name: domain_quality + type: float + required: true + - name: final_readiness + type: float + required: true + - name: weighted_score + type: float + required: true + - name: selected_domain_rubrics + type: string[] + required: false + condition: "domain_context or rubric_refs was provided, or acceptance criteria / Done Contract required subjective/product/UX/docs/DX/UI/domain craft evaluation" + description: "Rubric families selected from assistant-review references/domain-rubrics.md; empty or omitted when domain rubrics were not scoped" + - name: domain_quality_scores + type: object[] + required: false + condition: "selected_domain_rubrics is non-empty" + description: "Evidence-backed domain rubric scores from the final QA round" + object_fields: + - name: rubric_ref + type: string + required: true + - name: dimension + type: string + required: true + - name: score + type: float + required: true + - name: action + type: enum + required: true + enum_values: [accepted, accepted_with_concerns, refine, rejected, pivot, not_applicable] + - name: evidence + type: string + required: true + - name: score_progression + type: object[] + required: true + object_fields: + - name: round + type: int + required: true + validation: ">= 1 and <= 20" + - name: weighted_score + type: float + required: true + - name: failed_acceptance_count + type: int + required: true + - name: drift_status + type: enum + required: true + enum_values: [GENUINE, SUSPICIOUS, DRIFT, REGRESSION, STAGNATION, NEUTRAL, NOT_APPLICABLE] + - name: evidence + type: object[] + required: true + min_items: 1 + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: open_questions + type: string[] + required: false + condition: "final_verdict == blocked or result == BLOCKED" + - name: pivot_restart_decision_ref + type: string + required: false + condition: "QA loop observed STAGNATION, repeated DRIFT, repeated REGRESSION, or domain/rubric action pivot" + description: "Reference to the orchestrator-owned pivot_restart_decision artifact created before another QA/build dispatch." - name: spec_review_result type: object @@ -355,8 +482,8 @@ artifacts: required: conditional condition: "development/code-work task changes project source, tests, docs, config, hooks, contracts, or generated project artifacts" description: "Strict evidence that required subagent roles were dispatched or explicitly handled through direct fallback." - validation: "required_agents includes at minimum Code Writer, Builder/Tester, and Reviewer. delegated mode records dispatch and result evidence for each required role; medium+ delegated work records per-slice dispatch evidence when slices apply. direct_fallback records one explicit policy reason (authorization_denied, subagents_unavailable, or policy_disallowed) plus equivalent role, phase, verification, and review evidence. not_applicable is invalid for source-changing Build tasks." - on_fail: "Update the task journal Agent Dispatch Log. If delegated, add Code Writer, Builder/Tester, and Reviewer dispatch/result evidence and medium+ per-slice dispatch evidence. If direct_fallback, add the explicit policy reason and role-equivalent evidence. Do not complete with silent fallback." + validation: "required_agents includes at minimum Code Writer, Builder/Tester, and Code Reviewer. Reviewer compatibility may satisfy existing/legacy handoffs only when explicitly recorded. delegated mode records dispatch and result evidence for each required role; medium+ delegated work records per-slice dispatch evidence when slices apply. direct_fallback records one explicit policy reason (authorization_denied, subagents_unavailable, or policy_disallowed) plus equivalent role, phase, verification, and Code Reviewer review evidence; Reviewer direct evidence is compatibility evidence only. not_applicable is invalid for source-changing Build tasks. QA Evaluator evidence is also required when qa_evaluation_mode=required." + on_fail: "Update the task journal Agent Dispatch Log. If delegated, add Code Writer, Builder/Tester, and Code Reviewer dispatch/result evidence, or Reviewer compatibility dispatch/result evidence for existing/legacy handoffs, plus medium+ per-slice dispatch evidence. If direct_fallback, add the explicit policy reason and role-equivalent Code Reviewer direct evidence, or Reviewer direct evidence only as compatibility evidence. Do not complete with silent fallback. When QA is required, add QAEvaluator dispatch/result or direct evidence separately from Code Reviewer/Reviewer compatibility." object_fields: - name: execution_mode type: enum @@ -365,7 +492,7 @@ artifacts: - name: required_roles type: string[] required: true - validation: "Includes Code Writer, Builder/Tester, and Reviewer for source-changing work." + validation: "Includes Code Writer, Builder/Tester, and Code Reviewer for source-changing work. Reviewer compatibility may satisfy existing/legacy handoffs only when explicitly recorded." - name: delegated_dispatch_results type: object[] required: conditional @@ -380,11 +507,29 @@ artifacts: type: object[] required: conditional condition: "execution_mode == direct_fallback" - validation: "Contains equivalent Code Writer, Builder/Tester, and Reviewer role/phase/verification/review evidence." + validation: "Contains equivalent Code Writer, Builder/Tester, and Code Reviewer role/phase/verification/review evidence. Reviewer direct evidence is compatibility evidence only." - name: per_slice_dispatch_evidence type: object[] required: conditional condition: "size in [medium, large, mega] and execution_mode == delegated and slices apply" + - name: qa_evaluator_evidence + type: object + required: conditional + condition: "qa_evaluation_mode == required" + validation: "Contains QAEvaluator dispatch/result evidence in delegated mode, or QAEvaluator direct evidence with fallback reason in direct_fallback mode" + object_fields: + - name: qa_evaluator_dispatch + type: string + required: false + description: "QAEvaluator subagent/tool/run id when delegated" + - name: qa_evaluator_result + type: string + required: true + description: "Final QA verdict/result evidence" + - name: qa_evaluator_direct_evidence + type: string + required: false + description: "Fresh QA evidence when direct_fallback" - name: task_journal type: file @@ -506,6 +651,401 @@ artifacts: type: string[] required: true + - name: artifact_reference_ledger + type: object[] + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "Typed producer/consumer ledger for artifacts passed between subagents during harness work." + validation: "Each entry includes artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, and summary; ledger covers Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, Pivot/Restart Decision, changed files, verification evidence, and plan deviation refs when applicable." + on_fail: "Before Build, Review, completion, or handoff, create or repair the Artifact Reference Ledger in the plan/task journal; re-dispatch the producer when a required typed artifact reference is missing, stale, invalid, or lacks a consumer validation status." + min_items: 1 + object_fields: + - name: artifact_id + type: string + required: true + validation: "Stable id unique within the task run" + on_missing: fail + - name: artifact_type + type: enum + required: true + enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result] + validation: "Classifies the artifact without requiring prose parsing" + on_missing: fail + - name: producer + type: string + required: true + validation: "Role, subagent, hook, or task packet that created or last updated the artifact" + on_missing: fail + - name: consumer + type: string + required: true + validation: "Role, subagent, hook, or phase expected to consume and validate the artifact" + on_missing: fail + - name: location_ref + type: string + required: true + validation: "Typed location/ref pointer: file path, task-journal section, dispatch id, command ref, or stable artifact ref" + on_missing: fail + - name: schema_or_contract + type: string + required: true + validation: "Contract, schema, template, or required fields the consumer validates before use" + on_missing: fail + - name: validation_status + type: enum + required: true + enum_values: [pending, valid, invalid, stale, not_applicable] + validation: "Consumer-visible validation state; invalid or stale entries block phase advancement" + on_missing: fail + - name: summary + type: string + required: true + validation: "Concise description of the artifact and its current state" + on_missing: fail + + - name: done_contract + type: object + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "Accepted pre-Build Done Contract for harness-capable work" + validation: "Must include done_when, not_done_when, verification, owner_consumer, acceptance_criteria, debate_record with at least two perspectives, and accepted_by before Build starts" + on_fail: "Block Build, load references/harness-controller.md, return to Plan, write the missing Done Contract, collect at least two debate perspectives using subagents when delegated mode is available, and record approval or scope-change re-approval" + object_fields: + - name: done_when + type: string[] + required: true + validation: "Pass/fail outcomes that prove the task or slice is complete" + - name: not_done_when + type: string[] + required: true + validation: "Failure states that block completion" + - name: verification + type: string[] + required: true + validation: "Commands, inspections, reviews, or manual checks that prove done" + - name: owner_consumer + type: string + required: true + validation: "Names the owner and downstream consumer of the artifact or behavior" + - name: acceptance_criteria + type: string[] + required: true + validation: "Explicit binary criteria from the user request, approved plan, or slice manifest" + - name: debate_record + type: object[] + required: true + min_items: 2 + validation: "At least two perspectives; delegated mode uses subagent perspectives when available" + object_fields: + - name: perspective + type: string + required: true + - name: concern_or_support + type: string + required: true + - name: resolution + type: string + required: true + - name: accepted_by + type: string + required: true + validation: "User, orchestrator, or approved plan reference that accepted the Done Contract" + + - name: harness_recipe + type: object + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "Pre-Build controller recipe selected from task/model/risk/context profile" + validation: "Must classify task_profile, model_profile, risk_profile, and context_profile, then name selected_recipe, recipe_rationale, required_artifacts, and corrective_action" + on_fail: "Block Build, load references/harness-controller.md, return to Plan, classify the task/model/risk/context profile, select a Harness Recipe, and record the corrective action for missing or stale recipe evidence" + object_fields: + - name: task_profile + type: string + required: true + validation: "Captures task type, size, slice count, and TDD/debugging applicability" + - name: model_profile + type: string + required: true + validation: "Captures agent/model constraints, delegation mode, and tool limits" + - name: risk_profile + type: string + required: true + validation: "Captures risk tier, safety gates, review depth, and rollback needs" + - name: context_profile + type: string + required: true + validation: "Captures exact, summarized, omitted/deferred context and trace/replay or handoff needs" + - name: selected_recipe + type: string + required: true + validation: "Concise recipe label from references/harness-controller.md" + - name: recipe_rationale + type: string + required: true + validation: "Explains why task/model/risk/context profile selects this recipe" + - name: required_artifacts + type: string[] + required: true + validation: "Artifacts this recipe requires before or during Build" + - name: corrective_action + type: string + required: true + validation: "Explicit action when recipe evidence is missing or no longer matches the task" + + - name: harness_run_state + type: object + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "First-class Harness Run State artifact for recovering active harness execution after compaction, failure, or handoff." + validation: "Must record task_id, task_name, phase, slice, status, blockers, last_verification, next_action, and recovery_pointer; update it as Build, Review, and Document progress. When a pivot/restart is active, it also records pivot_restart_decision_ref and the exact next action after restart." + on_fail: "Pause advancement, update the task journal or carried-forward state with a complete Harness Run State block, point recovery_pointer at the current task packet or ledger entry, then resume from next_action." + object_fields: + - name: task_id + type: string + required: true + validation: "Stable task or run identifier from the task journal, plan, or dispatch packet" + on_missing: fail + - name: task_name + type: string + required: true + validation: "Human-readable task name" + on_missing: fail + - name: phase + type: enum + required: true + enum_values: [TRIAGE, DISCOVER, DECOMPOSE, PLAN, DESIGN, BUILD, REVIEW, DOCUMENT, COMPLETE] + validation: "Current workflow phase" + on_missing: fail + - name: slice + type: string + required: true + validation: "Current slice id/name, next pending slice, or N/A for unsliced work" + on_missing: fail + - name: status + type: enum + required: true + enum_values: [not_started, in_progress, blocked, verifying, reviewing, restarting, documenting, complete] + validation: "Current run status" + on_missing: fail + - name: blockers + type: string[] + required: true + validation: "Current blockers, or an explicit none entry" + on_missing: fail + - name: last_verification + type: object + required: true + validation: "Most recent verification command/check, result, and evidence" + on_missing: fail + object_fields: + - name: command_or_check + type: string + required: true + on_missing: fail + - name: result + type: enum + required: true + enum_values: [passed, failed, skipped, not_applicable, pending] + on_missing: fail + - name: evidence + type: string + required: true + on_missing: fail + - name: next_action + type: string + required: true + validation: "Exact next action needed to continue safely" + on_missing: fail + - name: recovery_pointer + type: string + required: true + validation: "Pointer to the task packet, trace entry, replay packet, file section, or artifact needed to resume" + on_missing: fail + - name: pivot_restart_decision_ref + type: string + required: false + condition: "a pivot_restart_decision is active or was selected for the current recovery path" + validation: "Reference to the Pivot/Restart Decision artifact governing the next action" + + - name: trace_ledger + type: object[] + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "First-class Trace Ledger artifact for ordered harness execution evidence." + validation: "Records timestamped/ordered agent events, decisions, verification commands/results, plan deviations, and artifact refs as execution progresses." + on_fail: "Append missing trace entries before advancing: reconstruct from Agent Dispatch Log, Key Decisions, verification output, plan deviations, and Artifact Registry; mark any unrecoverable gap explicitly." + min_items: 1 + object_fields: + - name: sequence + type: int + required: true + validation: "Monotonic event order within this task" + on_missing: fail + - name: timestamp + type: string + required: true + validation: "Timestamp or ordered marker for the event" + on_missing: fail + - name: event_type + type: enum + required: true + enum_values: [agent_event, decision, verification, plan_deviation, pivot_restart, artifact_ref, blocker, recovery] + validation: "Classifies the trace entry" + on_missing: fail + - name: actor + type: string + required: true + validation: "Role, subagent, hook, user, or orchestrator responsible for the event" + on_missing: fail + - name: summary + type: string + required: true + validation: "Concise event, decision, command/result, deviation, blocker, or recovery summary" + on_missing: fail + - name: artifact_refs + type: string[] + required: true + validation: "Related file paths, task journal headings, dispatch ids, command refs, or an explicit none entry" + on_missing: fail + + - name: replay_packet + type: object + required: conditional + condition: "size in [medium, large, mega] and harness_capable == true" + description: "First-class Replay Packet artifact that lets the workflow resume after compaction, failure, or handoff." + validation: "Captures pinned_context, artifact_refs, validation_state, exact_next_action, run_state_ref, trace_ledger_ref, recovery_pointer, and pivot_restart_decision_ref when active with enough detail to recover without replaying stale reasoning." + on_fail: "Create or repair the Replay Packet before ending the turn or crossing a phase boundary; copy pinned context from the approved plan/task packet, cite artifact refs, summarize validation state, and name the exact_next_action." + object_fields: + - name: pinned_context + type: string[] + required: true + validation: "Stable requirements, constraints, approved plan/slice ids, and non-goals needed to resume" + on_missing: fail + - name: artifact_refs + type: string[] + required: true + validation: "Task journal, context map, plan, run state, trace ledger, validation evidence, or changed-file references" + on_missing: fail + - name: validation_state + type: object + required: true + validation: "Current validation status and outstanding verification work" + on_missing: fail + object_fields: + - name: completed_checks + type: string[] + required: true + on_missing: fail + - name: pending_checks + type: string[] + required: true + on_missing: fail + - name: last_result + type: string + required: true + on_missing: fail + - name: exact_next_action + type: string + required: true + validation: "Single concrete action to take next after replay" + on_missing: fail + - name: run_state_ref + type: string + required: true + validation: "Reference to the current Harness Run State artifact" + on_missing: fail + - name: trace_ledger_ref + type: string + required: true + validation: "Reference to the current Trace Ledger artifact" + on_missing: fail + - name: recovery_pointer + type: string + required: true + validation: "Where to resume in the plan, task packet, phase, or slice ledger" + on_missing: fail + - name: pivot_restart_decision_ref + type: string + required: false + condition: "a pivot/restart decision controls the exact_next_action" + validation: "Reference to the orchestrator-owned Pivot/Restart Decision artifact" + + - name: pivot_restart_decision + type: object + required: conditional + condition: "Review, QA, Build, or CodeWriter evidence triggers STAGNATION, repeated DRIFT, repeated REGRESSION, rubric/domain action pivot, CodeWriter blocker routing, verification blocker, plan deviation, or scope change" + description: "Orchestrator-owned decision packet for stagnant quality loops, CodeWriter unexpected blockers, and restart/pivot recovery." + validation: "Must classify the trigger, cite evidence, name the affected slice or round, compare options, select a recovery action, state whether reapproval is required, name the next agent, and point recovery to an exact next action." + on_fail: "Pause the review/QA/build loop, load references/harness-controller.md, create the Pivot/Restart Decision packet, append trace/replay/run-state updates, and obtain reapproval before continuing when scope, files, behavior, risk, or verification changes." + object_fields: + - name: trigger + type: enum + required: true + enum_values: [STAGNATION, repeated_DRIFT, repeated_REGRESSION, pivot, code_writer_blocker, verification_blocker, plan_deviation, scope_change] + validation: "Names the event that forced pivot/restart consideration" + on_missing: fail + - name: evidence + type: object[] + required: true + min_items: 1 + validation: "At least one review score, QA score, CodeWriter blocker, verification failure, trace entry, or task-packet mismatch supports the trigger" + on_missing: fail + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: affected_slice_or_round + type: string + required: true + validation: "Current slice id/name, review round, QA round, or phase affected by the trigger" + on_missing: fail + - name: options_considered + type: object[] + required: true + min_items: 2 + validation: "At least two recovery options are compared unless a single safe option is forced by policy or missing approval" + on_missing: fail + object_fields: + - name: option + type: string + required: true + - name: tradeoff + type: string + required: true + - name: disposition + type: enum + required: true + enum_values: [selected, rejected, deferred] + - name: selected_action + type: enum + required: true + enum_values: [reset_context, return_to_build, dispatch_debugging, dispatch_explorer, dispatch_architect, run_candidate_search, replan, restart_slice, restart_phase, block_for_user, accept_with_limitations] + validation: "Concrete recovery action selected by the orchestrator; it must not imply starting round 21 after a terminal round 20" + on_missing: fail + - name: reapproval_required + type: boolean + required: true + validation: "true when selected_action changes approved scope, files, behavior, risk, verification, or acceptance criteria; false only for local context reset or same-scope retry" + on_missing: fail + - name: next_agent + type: string + required: true + validation: "Next role or agent to dispatch, such as Builder/Tester, CodeWriter, Explorer, Architect, Reviewer, QAEvaluator, assistant-debugging, candidate-search, user, or none" + on_missing: fail + - name: recovery_pointer + type: string + required: true + validation: "Trace entry, task packet, run-state section, replay packet, plan section, or file path where recovery resumes" + on_missing: fail + - name: exact_next_action + type: string + required: true + validation: "Single concrete action to take after the pivot/restart decision; replay_packet.exact_next_action must match it when replay is updated" + on_missing: fail + - name: decomposition_plan_review type: object required: conditional @@ -774,9 +1314,11 @@ artifacts: # 2. test_results.failed == 0 when tests ran; validation_results contains at least one result=passed entry for completed implementation workflows. Skipped/not_applicable entries do not satisfy completion evidence; docs-only/no-op/discovery exceptions must record the reason. # 3. When size is medium/large/mega, risk_tier is high/critical, or hook_profile == strict: spec_review_result.status == PASS and required_fixes are empty or resolved # 4. When size is medium/large/mega, risk_tier is high/critical, or hook_profile == strict: review_result.quality_review_status is not missing -# 5. For medium+ tasks, slice_verification_summary has one VERIFIED entry per slice -# 6. candidate_search_result exists when search_mode == candidate_search, includes search_exit_summary, and any plan_deviation is approved before Build -# 7. user_approval is confirmed, confirmed_with_issues, or not_required_small for eligible small tasks +# 5. If qa_evaluation_mode=required: qa_evaluation_result is populated after build/test and code-review evidence; QA evaluation does not replace review_result. Scoped domain rubrics require selected_domain_rubrics and domain_quality_scores; unscoped rubric scoring is invalid. +# 6. For medium+ tasks, slice_verification_summary has one VERIFIED entry per slice +# 7. candidate_search_result exists when search_mode == candidate_search, includes search_exit_summary, and any plan_deviation is approved before Build +# 8. For medium+ harness-capable work, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision when triggered, and artifact_reference_ledger exist and are current before completion or handoff +# 9. user_approval is confirmed, confirmed_with_issues, or not_required_small for eligible small tasks # If any check fails, do NOT print --- WORKFLOW COMPLETE ---. State what is missing. - name: verified_skill_distillation type: object diff --git a/skills/assistant-workflow/contracts/phase-gates.yaml b/skills/assistant-workflow/contracts/phase-gates.yaml index aa5f4fa..f89bf47 100644 --- a/skills/assistant-workflow/contracts/phase-gates.yaml +++ b/skills/assistant-workflow/contracts/phase-gates.yaml @@ -55,9 +55,13 @@ gates: on_fail: "Load references/triage-rubric.md and add the required gate packs for the task type before continuing" - id: T7 - check: "required_agents or fallback execution roles are populated from task size, task type, risk tier, subagent_policy_state, and subagent_execution_mode; source-changing development/code-work includes at minimum Code Writer, Builder/Tester, and Reviewer" + check: "required_agents or fallback execution roles are populated from task size, task type, risk tier, subagent_policy_state, and subagent_execution_mode; source-changing development/code-work includes at minimum Code Writer, Builder/Tester, and Code Reviewer, with Reviewer compatibility allowed only for existing/legacy handoffs" on_fail: "Select required workflow agents when subagent_execution_mode=delegated, or record direct fallback roles and responsibilities from references/subagent-dispatch.md and risk-specific gate packs. not_applicable is invalid for source-changing Build tasks." + - id: T_QA_EVALUATOR + check: "For medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work: required_agents includes QA Evaluator in addition to Code Writer, Builder/Tester, and Code Reviewer or Reviewer compatibility" + on_fail: "Add QA Evaluator as a Review-phase role, or record qa_evaluation_mode=not_required with a concrete reason when QA is not applicable." + - id: T9 check: "subagent_policy_state, subagent_execution_mode, and subagent_authorization_scope are initialized before any phase that may spawn subagents; development/code-work that needs workflow roles starts as authorization_required unless the current user prompt explicitly authorized subagents" on_fail: "Set subagent_policy_state=authorization_required and ask once for the needed delegation scope before continuing phases that require subagents. Set delegation_authorized only after explicit user approval or explicit user-requested subagents. Use direct_fallback only after authorization_denied, subagents_unavailable after a real spawn failure, or policy_disallowed." @@ -210,6 +214,21 @@ gates: check: "Plan includes artifact_contract before task packets with artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command/method, expected success signal, owner/consumer, and non-goals/exclusions" on_fail: "Load references/artifact-first-output-contract.md and add the Artifact Contract before planning implementation steps" + - id: P_DONE_CONTRACT + check: "For medium+ harness-capable work: accepted Done Contract exists before Build with done_when, not_done_when, verification, owner/consumer, acceptance_criteria, and debate_record with at least two perspectives; delegated mode uses subagent perspectives when available" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Block Build, load references/harness-controller.md, return to Plan, write the Done Contract, collect at least two debate perspectives using subagents when subagent_execution_mode=delegated and available, and seek re-approval when scope or acceptance changes" + + - id: P_HARNESS_RECIPE + check: "For medium+ harness-capable work: Harness Recipe is selected before Build from task/model/risk/context profile and records selected_recipe, recipe_rationale, required_artifacts, and corrective_action" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Block Build, load references/harness-controller.md, return to Plan, classify task_profile, model_profile, risk_profile, and context_profile, select the Harness Recipe, and record the missing-recipe corrective action" + + - id: P_HARNESS_RUNTIME_ARTIFACTS + check: "For medium+ harness-capable work: plan/task packets name Harness Run State, Trace Ledger, and Replay Packet artifact refs before Build" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Return to Plan, add harness_run_state_ref, trace_ledger_ref, and replay_packet_ref to the task packet or plan, and record the corrective action for missing run-state/trace/replay evidence" + - id: P_VERIFIED_DISTILLATION check: "When promoting a workflow or review lesson into durable skill/constraint/eval knowledge: verified_skill_distillation exists and verifier_result is approved before durable files are written" on_fail: "Load references/verified-skill-distillation.md and obtain verifier approval before skill/constraint/eval promotion" @@ -327,6 +346,20 @@ gates: check: "Every plan step has a corresponding '>> Step N/total: DONE' message" on_fail: "Complete remaining plan steps before entering Review" + - id: B_DONE_CONTRACT + check: "For medium+ harness-capable work: Build started only after the approved plan recorded Done Contract and Harness Recipe evidence" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Stop Build, return to Plan, add or repair the Done Contract and Harness Recipe from references/harness-controller.md, then resume only after the corrective action is recorded" + + - id: B_HARNESS_RUN_STATE_TRACE_REPLAY + check: "For medium+ harness-capable work: Harness Run State, Trace Ledger, and Replay Packet are updated as execution progresses with task id/name, phase, slice, status, blockers, last verification, next action, recovery pointer, ordered events, decisions, verification results, plan deviations, artifact refs, pinned context, validation state, and exact next action" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Pause Build, update the task journal or carried-forward state with missing Harness Run State, Trace Ledger, and Replay Packet evidence, append a corrective trace entry, and resume only from the recorded exact next action" + + - id: B_CODE_WRITER_BLOCKER_ROUTING + check: "When Code Writer returns NEEDS_CONTEXT, BLOCKED, or DEVIATED because of an unexpected blocker: blocker_type and blocker_evidence are recorded, the orchestrator routes recovery to debugging, explorer, architect, candidate_search, replan, restart, user clarification, environment fix, or permission request, and Code Writer is not asked to improvise through legacy blockers blindly" + on_fail: "Pause Build, classify the blocker, update Harness Run State/Trace Ledger/Replay Packet when harness-capable, create pivot_restart_decision when restart/pivot/replan is selected, and obtain reapproval before continuing if scope, files, behavior, risk, verification, or acceptance criteria change" + - id: B2 check: "Build command succeeded (exit code 0 or equivalent success)" on_fail: "Fix build errors. Do not enter Review with a broken build." @@ -359,8 +392,8 @@ gates: on_fail: "Run verification criteria for each slice, record command/result evidence in the configured task journal or equivalent carried-forward state, and fix failing criteria before proceeding to Review." - id: B_SUBAGENT_EVIDENCE - check: "For source-changing Build tasks: subagent_execution_mode is not not_applicable, required_agents includes Code Writer, Builder/Tester, and Reviewer, and the task journal Agent Dispatch Log records strict delegated dispatch/result evidence or explicit direct fallback role-equivalent evidence" - on_fail: "Update the Agent Dispatch Log before leaving Build. delegated mode requires Code Writer, Builder/Tester, and Reviewer dispatch/result evidence. direct_fallback requires reason authorization_denied, subagents_unavailable, or policy_disallowed plus Code Writer, Builder/Tester, and Reviewer direct evidence." + check: "For source-changing Build tasks: subagent_execution_mode is not not_applicable, required_agents includes Code Writer, Builder/Tester, and Code Reviewer, and the task journal Agent Dispatch Log records strict delegated dispatch/result evidence or explicit direct fallback role-equivalent evidence; Reviewer compatibility may satisfy existing/legacy handoffs only when explicitly recorded" + on_fail: "Update the Agent Dispatch Log before leaving Build. delegated mode requires Code Writer, Builder/Tester, and Code Reviewer dispatch/result evidence, or Reviewer compatibility dispatch/result evidence for existing/legacy handoffs. direct_fallback requires reason authorization_denied, subagents_unavailable, or policy_disallowed plus Code Writer, Builder/Tester, and Code Reviewer direct evidence; Reviewer direct evidence is compatibility evidence only." - id: B_SUBAGENT_SLICE_EVIDENCE check: "For medium+ delegated tasks with slices: per-slice dispatch evidence is recorded before the slice is marked VERIFIED" @@ -420,6 +453,19 @@ gates: check: "Quality Review completed by loading and following assistant-review SKILL.md and contracts (not manual review)" on_fail: "Load and follow assistant-review SKILL.md and contracts. Do not manually dispatch reviewers." + - id: R2A_CODE_REVIEWER_DISTINCT + check: "Code Reviewer or Reviewer compatibility evidence is recorded separately from QA Evaluator evidence" + on_fail: "Update the Review Log, output contract, or carried-forward review state with distinct Code Reviewer/Reviewer and QA Evaluator entries. Do not collapse QA evaluation into code review." + + - id: R_QA_EVALUATION + check: "When qa_evaluation_mode is required: QAEvaluator runs after build/test evidence and Code Reviewer or Reviewer compatibility result, returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, and evidence" + condition: "qa_evaluation_mode == required" + on_fail: "Load assistant-review references/qa-evaluation-loop.md, dispatch QAEvaluator or record direct-fallback QA evidence, and do not complete Review until QA result is present or blocked with open_questions. If domain_context/rubric_refs or subjective/product/UX/docs/DX/UI/domain acceptance scope exists, require selected_domain_rubrics and domain_quality_scores; otherwise reject invented rubric scoring." + + - id: R_PIVOT_RESTART_DECISION + check: "Review or QA STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric/domain action pivot records an orchestrator-owned pivot_restart_decision before another fix, review, QA, or build dispatch; round 20 remains terminal and never creates round 21 behavior" + on_fail: "Pause Review, create pivot_restart_decision with trigger, evidence, affected_slice_or_round, options_considered, selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action. Update Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger when harness-capable before continuing." + - id: R3 check: "Verification summary is written with: changed files, test/validation coverage, review result, manual validation steps when applicable, known limitations" on_fail: "Write verification summary in the configured task journal, output contract, or equivalent carried-forward state" @@ -474,6 +520,11 @@ gates: condition: "size in [medium, large, mega]" on_fail: "Load assistant-reflexion and assistant-memory contracts, inspect Review Log findings, build/test failures, and user corrections for durable lessons, check memory trend/backend availability when allowed, then write the Learning Controller block. Use backend_unavailable or policy_disallowed when appropriate; do not require a memory write when tools are unavailable, and do not persist routine progress as a lesson." + - id: DOC_HARNESS_REPLAY_PACKET + check: "For medium+ harness-capable work: final Harness Run State, Trace Ledger, and Replay Packet evidence is current before completion or handoff" + condition: "size in [medium, large, mega] and harness_capable == true" + on_fail: "Before Document completes, update Harness Run State to the current phase/status, append final trace entries for verification/review/document decisions, refresh Replay Packet validation_state and exact_next_action, and record any remaining blocker" + - id: DOC_SKILL_DISTILLATION check: "If work produced a repeatable workflow, recurring lesson, user-requested reusable procedure, or skill/constraint candidate: verified_skill_distillation exists and verifier_result is approved before promotion, or promotion_decision is skip with rationale" on_fail: "Load references/verified-skill-distillation.md, create the distillation packet, and send it to an independent verifier before saving any reusable skill or constraint" @@ -510,3 +561,8 @@ invariants: check: "If scope grows beyond initial triage during any phase, stop and re-triage" scope: all_phases on_fail: "Print '>> SCOPE CHANGE DETECTED' and re-run Triage with updated understanding" + + - id: INV_PIVOT_RESTART_REAPPROVAL + check: "Any pivot_restart_decision that changes approved scope, files, behavior, risk, verification, or acceptance criteria has reapproval_required=true and records approval before Build or Review continues" + scope: all_phases + on_fail: "Pause the workflow, mark reapproval_required=true, update the plan/task journal, and wait for user approval before continuing the changed path" diff --git a/skills/assistant-workflow/references/harness-controller.md b/skills/assistant-workflow/references/harness-controller.md new file mode 100644 index 0000000..adb5b8c --- /dev/null +++ b/skills/assistant-workflow/references/harness-controller.md @@ -0,0 +1,221 @@ +# Harness Controller + +Use this reference only for medium+ work that is harness-capable: long-running, +multi-slice, delegated, high-risk, subjective/domain-scored, trace/replay-ready, +or explicitly requested as harness work. Do not load it for small local fixes. + +## Pre-Build Gate + +Before Build starts, the approved plan must contain both: + +- an accepted Done Contract +- a selected Harness Recipe +- refs for Harness Run State, Trace Ledger, and Replay Packet artifacts when + the recipe is trace/replay-ready + +If either is missing, block Build, return to Plan, add the missing artifact, and +record the corrective action in the task journal or carried-forward state. + +## Done Contract + +The Done Contract defines what "finished" means before implementation begins. + +Required fields: + +- `done_when`: pass/fail outcomes that prove the slice or task is complete +- `not_done_when`: explicit failure states that must block completion +- `verification`: commands, inspections, reviews, or manual checks that prove done +- `owner_consumer`: owner and downstream consumer of the artifact or behavior +- `acceptance_criteria`: explicit binary criteria copied from user/plan/slice scope +- `debate_record`: at least two perspectives considered before acceptance +- `accepted_by`: user, orchestrator, or approved plan reference + +For subjective/domain-scored, product, UX, UI, docs, or DX work, carry any +scoped `domain_context` and `rubric_refs` into the QA Evaluator packet. These +refs enable conditional use of assistant-review `references/domain-rubrics.md`; +they are not required for unrelated code-review-only work. + +Debate rules: + +- Record at least two perspectives, such as implementer, tester, reviewer, + architect, product, security, docs, or user. +- When `subagent_execution_mode=delegated` and relevant subagents are available, + use subagent perspectives for the debate before Build. +- If delegated debate is unavailable, record the fallback reason and the direct + role-equivalent perspectives used. +- Do not let the debate add scope. Scope changes are plan deviations. + +## Harness Recipe + +The Harness Recipe selects the controller shape from the task/model/risk/context +profile. It is a short routing decision, not a new plan. + +Required profile fields: + +- `task_profile`: task type, size, slice count, and whether TDD/debugging applies +- `model_profile`: agent/model constraints, delegation mode, and tool limits +- `risk_profile`: risk tier, safety gates, review depth, and rollback needs +- `context_profile`: exact context, summarized context, omitted/deferred context, + and whether trace/replay or handoff artifacts are needed + +Required recipe fields: + +- `selected_recipe`: concise recipe label +- `recipe_rationale`: why the profile selects this recipe +- `required_artifacts`: Done Contract, task packet, verification, and any trace or + handoff artifacts needed by later slices +- `corrective_action`: what to do if the recipe is missing or stops matching the + task during Build +- Corrective action: the recorded recovery step for a missing or stale recipe + +Selection rules: + +- Use a lightweight guarded recipe for medium single-slice work with moderate risk + and compact context. +- Use a slice-sequential recipe when independent slice verification is required. +- Use a review-intensive recipe for high/critical risk, weak tests, public + contracts, or subjective acceptance. +- Use a trace/replay-ready recipe when context is large, work is long-running, or + recovery after compaction/failure is likely. + +## Runtime Artifacts + +For medium+ harness-capable work, keep these first-class artifacts in the task +journal or equivalent carried-forward state and update them as execution +progresses. They are recovery artifacts, not extra planning ceremony. + +### Harness Run State + +Records the current task/run position: + +- `task_id` +- `task_name` +- `phase` +- `slice` +- `status` +- `blockers` +- `last_verification` +- `next_action` +- `recovery_pointer` + +### Trace Ledger + +Records ordered or timestamped execution evidence: + +- agent events +- decisions +- verification commands/results +- plan deviations +- artifact refs + +### Replay Packet + +Captures the minimum continuation packet needed after compaction, failure, or +handoff: + +- pinned context +- artifact refs +- validation state +- exact next action +- run-state and trace-ledger refs +- recovery pointer + +## Pivot/Restart Controller + +The Pivot/Restart Controller is owned by the orchestrator. It runs when a +quality loop or Build handoff is no longer making safe progress. + +Trigger it for: + +- review or QA `STAGNATION` +- repeated `DRIFT` +- repeated `REGRESSION` +- rubric or domain action `pivot` +- Code Writer `blocker_type` returns such as `legacy_code_bug`, + `broken_baseline`, `hidden_dependency`, `missing_contract`, `stale_plan`, + `scope_conflict`, `tool_environment`, `permission_policy`, `tdd_red_missing`, + or `other` +- verification blockers, plan deviations, or scope changes that make the + approved packet stale + +Required decision fields: + +- `trigger`: the exact trigger category +- `evidence`: score entries, findings, blocker evidence, verification failures, + or trace refs proving the trigger +- `affected_slice_or_round`: current slice id/name, review round, QA round, or + workflow phase +- `options_considered`: at least two recovery options unless policy or missing + approval leaves only one safe path +- `selected_action`: reset context, return to Build, dispatch debugging, + dispatch explorer, dispatch architect, run candidate search, replan, restart + the slice, restart the phase, block for user, or accept with limitations +- `reapproval_required`: true whenever scope, files, behavior, risk, + verification, or acceptance criteria change +- `next_agent`: the next role or agent to dispatch +- `recovery_pointer`: task packet, trace row, replay packet, plan section, or + file path where recovery resumes +- `exact_next_action`: the single action to perform after the decision + +When a decision is created, update Harness Run State, append a `pivot_restart` +Trace Ledger entry, refresh Replay Packet so `exact_next_action` matches the +decision, and add or update the Pivot/Restart Decision artifact ref. Round 20 remains terminal; the controller never creates round 21 behavior. + +Routing rules: + +- Code Writer legacy code bugs or broken baselines route to debugging before + another implementation attempt. +- Hidden dependencies route to Explorer or Code Mapper refresh. +- Missing contracts or stale task packets route to Architect or Plan repair. +- Scope conflicts, plan deviations, and candidate pivots route to replan and + reapproval when scope, files, behavior, risk, verification, or acceptance + criteria change. +- Tool, environment, permission, or policy blockers route to environment fix, + permission request, or BLOCKED with evidence. +- Review or QA stagnation routes to reset context, candidate search, replan, + or restart depending on the evidence; do not silently continue the loop. + +## Typed Artifact References + +When a harness artifact crosses an agent boundary, pass it as an Artifact +Reference entry instead of an ad hoc string. Each entry carries: + +- `artifact_id` +- `artifact_type` +- `producer` +- `consumer` +- `location_ref` +- `schema_or_contract` +- `validation_status` +- `summary` + +Producer responsibility: create or update the artifact, assign its stable id and +location/ref pointer, name the contract or schema it follows, and summarize its +current state. Consumer responsibility: validate `schema_or_contract` and +`validation_status` before relying on `location_ref`; invalid or stale refs block +phase advancement or trigger re-dispatch. + +Use typed refs for Done Contract, Harness Recipe, Harness Run State, Trace +Ledger, Replay Packet, Pivot/Restart Decision, changed files, verification +evidence, and plan deviation refs when applicable. + +## Corrective Actions + +- Missing Done Contract: return to Plan, load this reference, write the contract, + record debate with at least two perspectives, and wait for approval when the + contract changes scope or acceptance. +- Missing Harness Recipe: return to Plan, classify task/model/risk/context + profile, select a recipe, and record rationale plus corrective action. +- Missing debate perspectives: collect the missing perspective through delegated + subagents when available, or record direct fallback perspective evidence. +- Missing run-state/trace/replay evidence: pause phase advancement, add or + repair Harness Run State, Trace Ledger, and Replay Packet blocks with the + required fields, append a corrective trace entry, and resume from the recorded + exact next action. +- Pivot/restart trigger: pause the active loop, create the orchestrator-owned + `pivot_restart_decision`, update run-state/trace/replay/artifact refs, and + reapprove before continuing if scope, files, behavior, risk, verification, or + acceptance criteria change. +- Recipe mismatch during Build: print `>> PLAN DEVIATION DETECTED`, update the + recipe, and seek re-approval when files, behavior, scope, risk, or verification + changes. diff --git a/skills/assistant-workflow/references/phases.md b/skills/assistant-workflow/references/phases.md index f40d207..450f4e0 100644 --- a/skills/assistant-workflow/references/phases.md +++ b/skills/assistant-workflow/references/phases.md @@ -173,7 +173,7 @@ Print: `>> Direct fallback Architect responsibility` (when `subagent_execution_m **Entry rule:** Do not enter Plan while the saved clarification state is pending. Resume Plan only after Discover records `Clarification status: ready` and all implementation-shaping fields are explicit or explicitly defaulted. -Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. Then read `references/plan-template.md` and use the correct tier: +Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. For medium+ harness-capable work, load `references/harness-controller.md` and add the Done Contract, Harness Recipe, Harness Run State ref, Trace Ledger ref, Replay Packet ref, and Artifact Reference Ledger before task packets. Then read `references/plan-template.md` and use the correct tier: - Small: inline plan (goal, files, risks, tests). Do not wait for approval unless risk, ambiguity, user instruction, or a scope-changing decision makes approval necessary. - Medium: standard plan (drop Security/Operability unless the task touches auth, PII, payments, or infra) - Large/Mega: full plan (all sections including Security and Operability) @@ -183,12 +183,13 @@ Before writing the plan, load `references/artifact-first-output-contract.md` and 3. Analyze 1-3 options with tradeoffs, pick one 4. Identify risks and edge cases 5. Put the Artifact Contract before task packets and map every medium+ task packet to at least one required artifact or acceptance criterion -6. For medium+ tasks: consume the Decompose slice manifest directly in the plan and align each task packet to exactly one slice_id without rediscovering boundaries -6. Write ordered implementation steps with file paths -7. For large/mega: fill in Security and Operability sections. For medium: only if the task touches auth, PII, payments, or infra (promote to Full tier per plan-template.md) -8. Carry `Task type`, `Risk tier`, `Required gates`, `Required agents`, `subagent_policy_state`, `subagent_execution_mode`, `subagent_authorization_scope`, and `Search mode` into the plan. Each required gate must map to task packet criteria or explicit N/A rationale. -9. If `search_mode: candidate_search`, load `references/candidate-search.md`, create the goal tree from acceptance/slice criteria, score candidates, record the archive location, and treat post-approval pivots as plan deviations requiring re-approval when scope/files/behavior/risk change. -10. Load prompt packs only when applicable: +6. For medium+ harness-capable work, put the Done Contract, Harness Recipe, run-state/trace/replay artifact refs, and Artifact Reference Ledger before Build. The Done Contract must include done_when, not_done_when, verification, owner/consumer, acceptance criteria, and at least two debate perspectives; the Harness Recipe must be selected from task/model/risk/context profile. The ledger must type each artifact with artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, and summary. +7. For medium+ tasks: consume the Decompose slice manifest directly in the plan and align each task packet to exactly one slice_id without rediscovering boundaries +8. Write ordered implementation steps with file paths +9. For large/mega: fill in Security and Operability sections. For medium: only if the task touches auth, PII, payments, or infra (promote to Full tier per plan-template.md) +10. Carry `Task type`, `Risk tier`, `Required gates`, `Required agents`, `subagent_policy_state`, `subagent_execution_mode`, `subagent_authorization_scope`, and `Search mode` into the plan. Each required gate must map to task packet criteria or explicit N/A rationale. +11. If `search_mode: candidate_search`, load `references/candidate-search.md`, create the goal tree from acceptance/slice criteria, score candidates, record the archive location, and treat post-approval pivots as plan deviations requiring re-approval when scope/files/behavior/risk change. +12. Load prompt packs only when applicable: - Refactors: `references/prompts/refactor-safety.md` - Migrations/rewrites: `references/prompts/refactor-safety.md` plus any applicable migration or parity checklist - New code: `references/prompts/test-strategy.md` @@ -238,18 +239,27 @@ For medium+ tasks, create a task journal using `references/task-journal-template Capture **constraints** from Discovery/Plan (e.g. "don't touch ProjectA", "stay on .NET 8"). Check constraints before each step. +For medium+ harness-capable work, confirm the task journal or carried-forward plan has an accepted Done Contract, selected Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger before dispatching Code Writer or Builder/Tester. If any are missing, stop Build and return to Plan or repair state for the corrective action in `references/harness-controller.md`. + +Keep the runtime artifacts current as execution progresses: +- Update Harness Run State after each slice/step, blocker, verification result, phase transition, or next-action change. +- Append Trace Ledger entries for agent events, decisions, verification commands/results, plan deviations, pivot/restart decisions, and artifact refs. +- Refresh the Replay Packet before context compaction, failure handoff, phase handoff, or end-of-turn with pinned context, artifact refs, validation state, and the exact next action. +- Update the Artifact Reference Ledger when producers create or move artifacts, and require consumers to validate `schema_or_contract` plus `validation_status` before relying on `location_ref`. + ### Orchestrator delegation rule **Delegated mode (`subagent_execution_mode=delegated`):** the orchestrator does not edit project source files or write implementation/test code directly. Framework-owned state artifacts (`{agent_state_dir}/task.md`, `{agent_state_dir}/context-map.md`, `{agent_state_dir}/session.md`, and `{agent_state_dir}/working-buffer.md`) are the exception only when configured and policy-allowed, and may be updated directly. If local state files are unavailable, carry equivalent state in the plan/response packet. Project source changes go through sub-agents, and the Agent Dispatch Log must record dispatch and result evidence for every required role before completion: - **Code Writer** (`code-writer`): implements code following the plan - **Builder/Tester** (`builder-tester`): builds, writes tests, runs tests -- **Reviewer** (`reviewer` or assistant-review delegated review): independent review evidence for Review +- **Code Reviewer** (`code-reviewer`, or `reviewer` compatibility through assistant-review delegated review): independent code/security/architecture/test coverage review evidence for Review +- **QA Evaluator** (`qa-evaluator`): independent acceptance, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result evaluation when QA is required -Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Reviewer. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. +Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Code Reviewer; `reviewer` remains compatibility routing for existing/legacy handoffs. Medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work also adds QA Evaluator. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. **Direct fallback mode (`subagent_execution_mode=direct_fallback`):** when authorization is denied, subagents are unavailable, or policy disallows spawning, the active agent may implement, test, and review directly, but must preserve the same phases, contracts, evidence requirements, and review/security gates. Do not pretend delegation happened; record `subagent_policy_state`, `subagent_execution_mode`, the explicit `Direct fallback reason: authorization_denied | subagents_unavailable | policy_disallowed`, and the direct-execution evidence. -**Strict subagent evidence gate:** any development/code-work task that changes project source, tests, docs, config, hooks, contracts, or generated artifacts must keep Code Writer, Builder/Tester, and Reviewer in `Required agents`; `subagent_execution_mode=not_applicable` is invalid for Build. Before leaving Build/Review, the task journal Agent Dispatch Log (or equivalent carried-forward state) must record `Code Writer dispatch` + `Code Writer result`, `Builder/Tester dispatch` + `Builder/Tester result`, and `Reviewer dispatch` + `Reviewer result` in delegated mode. Direct fallback must instead record `Code Writer direct evidence`, `Builder/Tester direct evidence`, and `Reviewer direct evidence` plus the explicit fallback reason. Silent fallback is invalid; Silent fallback cannot complete. +**Strict subagent evidence gate:** any development/code-work task that changes project source, tests, docs, config, hooks, contracts, or generated artifacts must keep Code Writer, Builder/Tester, and Code Reviewer in `Required agents`; `reviewer` remains compatibility routing for existing/legacy handoffs and task journals. `subagent_execution_mode=not_applicable` is invalid for Build. Before leaving Build/Review, the task journal Agent Dispatch Log (or equivalent carried-forward state) must record `Code Writer dispatch` + `Code Writer result`, `Builder/Tester dispatch` + `Builder/Tester result`, and `Code Reviewer dispatch` + `Code Reviewer result` or `Reviewer dispatch` + `Reviewer result` only when using compatibility routing in delegated mode. When QA is required, it must also record `QA Evaluator dispatch` + `QA Evaluator result` in delegated mode. Direct fallback must instead record `Code Writer direct evidence`, `Builder/Tester direct evidence`, and `Code Reviewer direct evidence`; `Reviewer direct evidence` may satisfy only compatibility routing for existing/legacy handoffs. Add `QA Evaluator direct evidence` when QA is required, with the explicit fallback reason. Silent fallback is invalid; Silent fallback cannot complete. For each non-TDD step: dispatch Code Writer with the plan step + context map only in delegated mode, or execute directly in fallback mode and record Code Writer direct evidence → verify via Builder/Tester only in delegated mode, or run verification directly in fallback mode and record Builder/Tester direct evidence → check results → proceed or fix. For TDD-active steps, use the TDD sandwich in the Build loop. @@ -264,8 +274,9 @@ Print: `>> Slice [S]/[total]: [slice_id] [name]` Before starting each medium+ slice: 1. Load the approved task packet for the slice, including slice_id, observable increment, deliverable type, files, acceptance criteria, verification command, expected success signal, evidence to record, and deviation/rollback rule -2. Confirm prior slice status is `VERIFIED` before advancing; do not start the next slice while the current slice is unverified -3. Check constraints from the task journal against the slice files and criteria +2. When harness-capable, confirm the task packet carries `done_contract_ref`, `harness_recipe_ref`, `harness_run_state_ref`, `trace_ledger_ref`, `replay_packet_ref`, and typed `artifact_refs` +3. Confirm prior slice status is `VERIFIED` before advancing; do not start the next slice while the current slice is unverified +4. Check constraints from the task journal against the slice files and criteria For the current slice task packet: @@ -286,6 +297,42 @@ Print: `>> Direct fallback Builder/Tester responsibility → RED evidence` (TDD - Code Writer GREEN: implement minimal production code only after RED evidence is present. - Builder/Tester VERIFY/REFACTOR-SAFETY: run the targeted test, relevant suite, and regression checks; request Code Writer fixes for production failures. +### Code Writer unexpected blockers + +If Code Writer returns `NEEDS_CONTEXT`, `BLOCKED`, or `DEVIATED` because of a +legacy code bug, broken baseline, hidden dependency, missing contract, stale +plan, scope conflict, tool/environment issue, permission/policy issue, missing +RED evidence, or another unexpected blocker, do not ask Code Writer to +improvise through it blindly. + +Require the return to include: + +- `blocker_type`: `legacy_code_bug`, `broken_baseline`, `hidden_dependency`, + `missing_contract`, `stale_plan`, `scope_conflict`, `tool_environment`, + `permission_policy`, `tdd_red_missing`, or `other` +- `blocker_evidence`: file paths, missing contract fields, baseline failures, + tool output summaries, scope conflict details, or task-packet mismatches + +Orchestrator recovery routing: + +- `legacy_code_bug` or `broken_baseline` -> dispatch debugging before another + implementation attempt +- `hidden_dependency` -> dispatch Explorer or refresh Code Mapper context +- `missing_contract` or `stale_plan` -> dispatch Architect or return to Plan +- `scope_conflict` -> record a plan deviation and seek reapproval when scope, + files, behavior, risk, verification, or acceptance criteria change +- `tool_environment` -> route to Builder/Tester or environment recovery +- `permission_policy` -> request permission or return BLOCKED +- `tdd_red_missing` -> return to Builder/Tester RED evidence +- `other` -> create a conservative recovery route with evidence + +When recovery requires pivot, replan, candidate search, or restart, create the +orchestrator-owned `pivot_restart_decision`, update Harness Run State, append a +`pivot_restart` Trace Ledger entry, refresh Replay Packet with the exact next +action, and update Artifact Reference Ledger before continuing. + +Recovery route labels are: debugging, explorer, architect, candidate_search, replan, restart, user_clarification, environment_fix, and permission_request. + Print: `>> Step [N]/[total]: DONE` (after each step passes build + test) ### Per-slice verification (medium+ tasks) @@ -297,9 +344,10 @@ After implementation for a slice is done, verify the slice against its criteria 1. Dispatch Builder/Tester in delegated mode, or perform the Builder/Tester responsibility directly in fallback mode, to run the slice's verification command and any relevant build/test checks from the task packet 2. Check each acceptance criterion from the slice manifest independently — mark pass/fail with the command/result or inspection evidence used 3. Record verification evidence in the task journal slice verification ledger, including RED status when TDD was active, implementation status, command/result, criteria checked, and final status -4. Run a small self-check/local sanity check: compare changed files and behavior against the task packet, constraints, and deviation rule; record the result in the ledger -5. If any criterion, command, or self-check fails: fix before moving to the next slice -6. Mark the slice `VERIFIED` only after all criteria pass and evidence is recorded +4. Update Harness Run State, append Trace Ledger verification/artifact refs, refresh Replay Packet validation_state plus exact_next_action, and update Artifact Reference Ledger entries for changed_files, verification_evidence, pivot_restart_decision, and plan_deviation refs when applicable +5. Run a small self-check/local sanity check: compare changed files and behavior against the task packet, constraints, and deviation rule; record the result in the ledger +6. If any criterion, command, runtime artifact, or self-check fails: fix before moving to the next slice +7. Mark the slice `VERIFIED` only after all criteria pass and evidence is recorded Print: `>> Slice [S]/[total]: [slice_id] [name] — VERIFIED ([X]/[Y] criteria passed)` @@ -316,7 +364,10 @@ Plan assumed: [X]. Reality: [Y]. Options: a) Adjust step [N] b) Rethink approach ``` -Never silently deviate from the plan. +Never silently deviate from the plan. If the selected recovery action changes +approved scope, files, behavior, risk, verification, or acceptance criteria, +record `pivot_restart_decision.reapproval_required=true` and wait for approval +before continuing the changed path. Print: `>> Build complete — all [N] steps implemented` Print: `>> Running final build + tests` @@ -334,7 +385,7 @@ Print: `>> Stage 1: Spec Review` Load and follow `references/prompts/spec-review.md`. Compare implementation against the approved plan, approved task packets, and approved slices and produce a structured spec compliance result before Stage 2. For bugfixes, include the `assistant-debugging` reproduction/root-cause evidence in the review material and check that the regression test or validation path actually covers the isolated failure mechanism. -Quality review cannot satisfy spec review. Spec review checks scope and acceptance compliance; quality review checks correctness, maintainability, architecture, security, and coverage after spec compliance is clear. +Quality review cannot satisfy spec review. Spec review checks scope and acceptance compliance; code quality review checks correctness, maintainability, architecture, security, and coverage after spec compliance is clear. QA Evaluation is a separate acceptance/result lane when required and cannot substitute for either Spec Review or Code Quality Review. 1. Walk through each approved plan step, task packet, or slice against `git diff`. 2. Check for missing acceptance criteria: any required behavior not reflected in code or evidence. @@ -355,18 +406,35 @@ Print: `>> Spec Review: [PASS / FAIL — found N required fixes]` ### Stage 2 — Quality Review -Print: `>> Stage 2: Quality Review — loading assistant-review SKILL.md` +Print: `>> Stage 2: Code Quality Review — loading assistant-review SKILL.md` -**Load and follow `assistant-review` SKILL.md and its contracts.** This runs the autonomous review-fix loop (max 10 rounds) with visible progress. Add Reviewer to `Required agents` before Stage 2. Do NOT implement the review loop inline when `subagent_execution_mode=delegated` and a delegated review agent is authorized — dispatch the Reviewer subagent and record `Reviewer dispatch`/`Reviewer result` evidence. In direct fallback, preserve fresh-review evidence and record `Reviewer direct evidence`. +**Load and follow `assistant-review` SKILL.md and its contracts.** This runs the autonomous review-fix loop (max 20 rounds) with visible progress. Add Code Reviewer to `Required agents` before Stage 2; use `Reviewer` only as compatibility routing for existing/legacy handoffs. Dispatch the `code-reviewer` agent for code defects, security, architecture, test coverage, and structural code issues. Do NOT implement the review loop inline when `subagent_execution_mode=delegated` and a delegated review agent is authorized — dispatch the Code Reviewer subagent and record `Code Reviewer dispatch`/`Code Reviewer result` evidence, or record `Reviewer dispatch`/`Reviewer result` only when using compatibility routing. In direct fallback, preserve fresh-review evidence and record `Code Reviewer direct evidence`; `Reviewer direct evidence` is compatibility evidence only. The `assistant-review` skill will: -- Dispatch Reviewer subagents in delegated mode, or preserve fresh-review evidence in direct fallback mode +- Dispatch Code Reviewer subagents in delegated mode, using Reviewer compatibility only for existing handoffs, or preserve fresh-review evidence in direct fallback mode - Fix must-fix and should-fix items - Re-review automatically +- Escalate STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT into an orchestrator-owned `pivot_restart_decision` before another fix/review dispatch - Return a final clean/remaining summary For small tasks: quick spec check + single review round is acceptable if clean. -For medium+ tasks: full two-stage review with autonomous quality loop via `assistant-review`. +For medium+ tasks: full spec review plus autonomous code quality loop via `assistant-review`. + +### Stage 3 — QA Evaluation + +Print: `>> Stage 3: QA Evaluation — loading assistant-review references/qa-evaluation-loop.md` when QA is required. + +Run QA Evaluation for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. + +The QA Evaluator receives Done Contract when present, acceptance criteria, verification evidence, code review result, domain_context/rubric_refs when applicable, round 1-20, previously_failed_acceptance_items, and qa_filter_policy. It loads assistant-review `references/domain-rubrics.md` only when acceptance criteria, Done Contract, domain_context, or explicit rubric_refs require subjective/product/UX/docs/DX/UI/domain scoring. It returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression or score_entry, evidence, and open_questions when blocked. + +QA Evaluation focuses on acceptance and final result. It does not replace Code Reviewer, and it must not report general code defects, security issues, architecture concerns, or test coverage gaps unless they directly block an acceptance criterion or Done Contract item. + +If QA score progression reports STAGNATION, repeated DRIFT, repeated REGRESSION, +or any scoped domain rubric returns action `pivot`, pause the QA loop and create +an orchestrator-owned `pivot_restart_decision` before another QA/build dispatch. +Round 20 remains terminal: return the final QA verdict and remaining failed +acceptance items instead of starting round 21. ### Status gate @@ -374,6 +442,8 @@ When local hooks are configured and policy-allowed, use them to enforce the revi When hooks are unavailable, enforce the same gate manually before presenting results: - Review Log or equivalent review result must exist. +- QA Evaluation result must exist when qa_evaluation_mode=required. +- Pivot/Restart Decision must exist when Review or QA reported STAGNATION, repeated DRIFT, repeated REGRESSION, or pivot action. - Final Result must be recorded. - The agent must complete the full review cycle before presenting results to the user. - Do not claim the hook ran unless it actually exists and executed. @@ -385,7 +455,8 @@ Print: `--- PHASE: REVIEW COMPLETE ---` After final review passes, write the Verification Summary in the task journal: - What changed (files + why) - What's tested (unit/integration/E2E coverage) -- Review result (clean / issues fixed / remaining should-fix items) +- Code Review result (clean / issues fixed / remaining should-fix items) +- QA Evaluation result when required (accepted / accepted with concerns / rejected / blocked) - Manual test instructions (step-by-step for the user) - Known limitations @@ -442,7 +513,8 @@ Then print completion markers and exit. - `No-save rationale`: required when no durable write occurred 6. **Post-task reflection**: Load and follow `assistant-reflexion` when available. Pass the Learning Controller evidence into reflexion so lessons are backed by review/build/user-correction evidence and persistence/no-save status is explicit. 7. If local memory tools are approved and available, persist only durable, evidence-backed lessons through the configured local memory backend. If tools are unavailable or policy-disallowed, record that outcome in `Durable lesson decision` and `No-save rationale` instead of writing ad hoc markdown as cross-session memory. -8. **Task completion metrics**: Append a JSONL entry when local metrics are configured and policy allows it (see format below) +8. For medium+ harness-capable work, refresh Harness Run State, append final Trace Ledger entries for review/document decisions, include Pivot/Restart Decision refs when triggered, and update Replay Packet validation_state plus exact_next_action before completion or handoff. +9. **Task completion metrics**: Append a JSONL entry when local metrics are configured and policy allows it (see format below) ### Metrics entry format (all sizes) diff --git a/skills/assistant-workflow/references/plan-template.md b/skills/assistant-workflow/references/plan-template.md index 1d1467b..10814cd 100755 --- a/skills/assistant-workflow/references/plan-template.md +++ b/skills/assistant-workflow/references/plan-template.md @@ -55,6 +55,21 @@ For Medium and Large/Mega plans, write implementation work as executable task pa - Expected success signal: [exit code 0, passing test name, output marker, etc.] - Evidence to record: - [test result, eval fixture, changed file, review note, or artifact proof] +- Harness refs: + - done_contract_ref: [Done Contract section/ref, or N/A] + - harness_recipe_ref: [Harness Recipe section/ref, or N/A] + - harness_run_state_ref: [Harness Run State section/ref, or N/A] + - trace_ledger_ref: [Trace Ledger section/ref, or N/A] + - replay_packet_ref: [Replay Packet section/ref, or N/A] +- Typed artifact refs: + - artifact_id: [stable task-local id] + artifact_type: [done_contract | harness_recipe | harness_run_state | trace_ledger | replay_packet | changed_files | verification_evidence | plan_deviation | task_packet | context_map | test_result | review_result | qa_evaluation_result] + producer: [role/subagent/hook/task packet] + consumer: [role/subagent/hook/phase] + location_ref: [typed location/ref pointer] + schema_or_contract: [contract/template/required fields] + validation_status: [pending | valid | invalid | stale | not_applicable] + summary: [concise state] - Deviation / rollback rule: - [what to do if required files/behavior differ from plan; include rollback/revert boundary] - Worker status / evidence: @@ -171,6 +186,62 @@ Covers the essentials without Security/Operability overhead. Fill this in during - Owner/consumer: [user, reviewer, downstream tool, runtime] - Non-goals/exclusions: [what must not be produced] +## Done Contract + +Required for medium+ harness-capable work before Build; omit with a brief N/A +rationale for non-harness work. + +- done_when: + - [binary outcome that proves done] +- not_done_when: + - [failure state that blocks done] +- verification: + - [command, inspection, review, or manual check] +- owner_consumer: [owner and downstream consumer] +- acceptance_criteria: + - [explicit binary criterion] +- debate_record: + - perspective: [role/subagent/direct perspective 1] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] + - perspective: [role/subagent/direct perspective 2] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] +- accepted_by: [user/orchestrator/approved plan reference] + +## Harness Recipe + +Required for medium+ harness-capable work before Build; selected from +task/model/risk/context profile per `references/harness-controller.md`. + +- task_profile: [task type, size, slice count, TDD/debugging applicability] +- model_profile: [agent/model constraints, delegation mode, tool limits] +- risk_profile: [risk tier, safety gates, review depth, rollback needs] +- context_profile: [exact/summarized/omitted context and trace/replay needs] +- selected_recipe: [concise recipe label] +- recipe_rationale: [why this profile selects the recipe] +- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] +- corrective_action: [what to do if missing or stale] + +## Runtime Harness Artifacts + +Required for medium+ harness-capable work; omit with a brief N/A rationale for +non-harness work. + +- harness_run_state_ref: [where task_id/task_name/phase/slice/status/blockers/last_verification/next_action/recovery_pointer will be maintained] +- trace_ledger_ref: [where ordered agent events, decisions, verification results, plan deviations, and artifact refs will be appended] +- replay_packet_ref: [where pinned context, artifact refs, validation state, and exact next action will be refreshed] +- corrective_action: [what to do if run-state/trace/replay evidence is missing or stale] + +## Artifact Reference Ledger + +Required for medium+ harness-capable work when artifacts pass between agents. +Each row is a typed producer/consumer record, not an ad hoc string reference. + +| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | +|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| +| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role] | [role/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | + ## Slice manifest from Decompose Use the shared Slice Manifest structure above. Paste the approved Decompose manifest verbatim and keep `single_slice_rationale` when exactly one slice exists. @@ -272,6 +343,15 @@ Everything from Medium, plus Security and Operability sections. Use when the tas - Broad-split rejection: [required proof that layer-only, module-only, folder-only, feature-only, setup-only, contract-only, and broad component-style splits were rejected unless verified deliverable artifact slices] - Decision: proceed | revise_decomposition | return_to_discover +## Artifact Reference Ledger + +Required for medium+ harness-capable work when artifacts pass between agents. +Each row is a typed producer/consumer record, not an ad hoc string reference. + +| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | +|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| +| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role] | [role/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | + ## Slice manifest from Decompose Use the shared Slice Manifest structure above. Paste the approved Decompose manifest verbatim and keep `single_slice_rationale` when exactly one slice exists. diff --git a/skills/assistant-workflow/references/prompts/pr-review.md b/skills/assistant-workflow/references/prompts/pr-review.md index 8e69fc8..f64fce9 100755 --- a/skills/assistant-workflow/references/prompts/pr-review.md +++ b/skills/assistant-workflow/references/prompts/pr-review.md @@ -1,10 +1,10 @@ -# Quality Review Checklist (Stage 2) +# Code Quality Review Checklist (Stage 2) -Load this during Stage 2 of the review cycle. This is the quality gate — it runs after the Spec Review confirms the implementation matches the plan. +Load this during Stage 2 of the review cycle. This is the code quality gate — it runs after the Spec Review confirms the implementation matches the plan. It does not replace Stage 3 QA Evaluation, which checks acceptance, Done Contract, verification evidence, scoped domain quality, score progression, and final result when QA is required. ## When to use -Run this review as part of the autonomous review-fix loop: after all build steps complete and tests pass. The loop runs up to 10 rounds, fixing must-fix and should-fix items each round until clean. You do NOT need to wait for user input between rounds. +Run this review as part of the autonomous review-fix loop: after all build steps complete and tests pass. The loop runs up to 20 rounds, fixing must-fix and should-fix items each round until clean. You do NOT need to wait for user input between rounds. For small tasks, do a quick pass through the "must-check" items only. For medium+ tasks, work through the full checklist. @@ -13,7 +13,7 @@ For small tasks, do a quick pass through the "must-check" items only. For medium When dispatching the Reviewer subagent for round N > 1, prepend this context to the prompt: ``` -## Previously Fixed (Round N of 10) +## Previously Fixed (Round N of 20) This is review round [N]. The following [count] items were already found and fixed in prior rounds — do NOT re-report them: @@ -23,8 +23,8 @@ and fixed in prior rounds — do NOT re-report them: Focus ONLY on NEW high-confidence findings not in the above list. Raise your confidence threshold each round: - Round 1-3: report findings at 80%+ confidence -- Round 4-7: report findings at 85%+ confidence -- Round 8-10: report findings at 90%+ confidence +- Round 4-15: report findings at 85%+ confidence +- Round 16-20: report findings at 90%+ confidence If you find no new issues above your confidence threshold, say "No new findings." ``` diff --git a/skills/assistant-workflow/references/sub-task-brief-template.md b/skills/assistant-workflow/references/sub-task-brief-template.md index cf80333..a9ca187 100755 --- a/skills/assistant-workflow/references/sub-task-brief-template.md +++ b/skills/assistant-workflow/references/sub-task-brief-template.md @@ -36,8 +36,8 @@ Workflow: ## Slice Brief: [name] ### Agent -Agent: [code-writer | architect | reviewer | explorer | builder-tester | code-mapper] -Role: [Implementer | Architect | Reviewer | Explorer] +Agent: [code-writer | architect | code-reviewer | qa-evaluator | reviewer | explorer | builder-tester | code-mapper] +Role: [Implementer | Architect | Code Reviewer | QA Evaluator | Reviewer compatibility | Explorer] ### Context Project: [name] @@ -104,8 +104,15 @@ When the slice is complete, report status using one of these values: | `DONE` | All acceptance criteria met, tests pass, no concerns | Proceed to integration | | `DONE_WITH_CONCERNS` | Criteria met but there are trade-offs or risks worth noting | Orchestrator reviews concerns before integration | | `NEEDS_CONTEXT` | Blocked by missing information not in the brief | Orchestrator provides context or adjusts the brief | -| `BLOCKED` | Cannot proceed — dependency issue, tooling failure, or design conflict | Orchestrator investigates and unblocks | -| `DEVIATED` | Work cannot follow the strict slice packet exactly | Orchestrator applies the deviation rollback rule before continuing | +| `BLOCKED` | Cannot proceed — dependency issue, tooling failure, legacy blocker, or design conflict | Orchestrator classifies recovery and unblocks | +| `DEVIATED` | Work cannot follow the strict slice packet exactly | Orchestrator applies the deviation rollback rule and reapproval when scope/files/behavior/risk changes | + +Unexpected blockers must be classified. Use `blocker_type` values: +`legacy_code_bug`, `broken_baseline`, `hidden_dependency`, `missing_contract`, +`stale_plan`, `scope_conflict`, `tool_environment`, `permission_policy`, +`tdd_red_missing`, or `other`. Do not widen scope, patch around legacy blockers +blindly, or invent a new plan. Return evidence so the orchestrator can route to +debugging, explorer, architect, candidate search, replan, or restart. **Report format:** ```text @@ -118,6 +125,8 @@ When the slice is complete, report status using one of these values: - [trade-off or risk worth noting] ### Blocker (if NEEDS_CONTEXT or BLOCKED) +- blocker_type: [legacy_code_bug | broken_baseline | hidden_dependency | missing_contract | stale_plan | scope_conflict | tool_environment | permission_policy | tdd_red_missing | other] +- blocker_evidence: [file paths, missing fields, baseline failure, tool/environment symptom, or scope conflict] - What's needed: [specific missing information or resolution] ### Changes made diff --git a/skills/assistant-workflow/references/subagent-dispatch.md b/skills/assistant-workflow/references/subagent-dispatch.md index 0fd1a82..6e30f6d 100644 --- a/skills/assistant-workflow/references/subagent-dispatch.md +++ b/skills/assistant-workflow/references/subagent-dispatch.md @@ -1,6 +1,6 @@ # Subagent Dispatch — Roles and Rules -Use specialized agents when the active tool policy and user authorization allow delegation. Each role has constrained access — a reviewer cannot edit files, a code writer doesn't run tests. When subagents are unavailable, denied, or policy-disallowed, keep the same role responsibilities as direct fallback evidence instead of pretending delegation happened. +Use specialized agents when the active tool policy and user authorization allow delegation. Each role has constrained access: code-reviewer, qa-evaluator, and reviewer cannot edit files, and code-writer does not run tests. When subagents are unavailable, denied, or policy-disallowed, keep the same role responsibilities as direct fallback evidence instead of pretending delegation happened. For full role prompts, read `references/subagent-roles.md`. @@ -25,7 +25,9 @@ Assistant Framework policy requires explicit user authorization before spawning | **Architect** | `architect` | `architect` | Read-only | Decompose, Plan, Design | | **Code Writer** | `code-writer` | `code-writer` | Write | Build | | **Builder/Tester** | `builder-tester` | `builder-tester` | Write | Build | -| **Reviewer** | `reviewer` | `reviewer` | Read-only | Review | +| **Code Reviewer** | `code-reviewer` | `code-reviewer` | Read-only | Review | +| **Reviewer** | `reviewer` | `reviewer` | Read-only | Review compatibility | +| **QA Evaluator** | `qa-evaluator` | `qa-evaluator` | Read-only | Review QA | ## What each role does @@ -34,7 +36,9 @@ Assistant Framework policy requires explicit user authorization before spawning - **Architect** — Designs implementation blueprints: files to create/modify, interfaces, data flows, build sequence, test plan. Does not write code. - **Code Writer** — Implements code following the plan. Does not run builds or tests. Does not review. Focuses purely on clean, convention-matching implementation. - **Builder/Tester** — Builds the project, writes tests, runs tests, absorbs noisy output. Returns concise results ("build passed, 2 tests failed: X, Y") not full logs. -- **Reviewer** — Independent code review with confidence-based filtering. Finds bugs, security issues, architecture violations. Does not edit files. +- **Code Reviewer** — Canonical independent code review with confidence-based filtering. Finds bugs, security issues, architecture violations, test coverage gaps, and structural code issues. Does not edit files. +- **Reviewer** — Compatibility route for existing handoffs that still say `Reviewer`; use only when `code-reviewer` is unavailable or a legacy prompt/handoff requires the old name. +- **QA Evaluator** — Independent QA acceptance evaluation after build/test and code-review evidence. Checks acceptance criteria, Done Contract, verification evidence, scoped UI/visual/product/UX/docs/DX/domain quality, score progression, and final result. Does not replace Code Reviewer. ## Phase-to-subagent requirements @@ -50,7 +54,8 @@ Every phase has a declared role responsibility. Phases without a subagent role a | **DESIGN** | Architect | UI tasks | Proposes design direction; Orchestrator creates mockup | | **BUILD** | Code Writer | All sizes | Implements code following the plan | | **BUILD** | Builder/Tester | All sizes | Builds, runs tests, returns concise results | -| **REVIEW** | Reviewer | All sizes | Independent review via `assistant-review` skill | +| **REVIEW** | Code Reviewer, or Reviewer compatibility | All sizes | Independent code review via `assistant-review` skill | +| **REVIEW** | QA Evaluator | Medium+ harness/domain-scored/UI/visual/product/UX/docs/DX-facing or explicitly requested QA | Independent acceptance QA via `assistant-review` QA loop | | **DOCUMENT** | — (Orchestrator direct) | All sizes | Documentation generation is orchestrator's synthesis work | **Rule:** If `subagent_execution_mode=delegated` and a phase's subagent column shows a dispatch, you MUST dispatch that role. If `subagent_execution_mode=direct_fallback`, you MUST NOT spawn subagents; instead record which role responsibility was handled directly and what equivalent evidence proves it. @@ -59,18 +64,20 @@ Every phase has a declared role responsibility. Phases without a subagent role a | Size | Agents used | Flow | |---|---|---| -| **Small** | Code Writer → Builder/Tester → Reviewer | Sequential, minimal (no Decompose) | -| **Medium** | Code Mapper → Architect (decompose) → Code Writer → Builder/Tester → Reviewer | Mapper feeds Architect, slices feed Writer | -| **Large** | Code Mapper → Explorer → Architect (decompose + plan) → Code Writer → Builder/Tester → Reviewer | Full pipeline with slice verification | -| **Mega** | All roles, parallel Code Writers per slice | Mapper → Explorer → Architect → parallel Writers → Builder/Tester and Reviewer at integration | +| **Small** | Code Writer → Builder/Tester → Code Reviewer | Sequential, minimal (no Decompose); Reviewer may be used only as compatibility; QA only when explicitly requested | +| **Medium** | Code Mapper → Architect (decompose) → Code Writer → Builder/Tester → Code Reviewer → QA Evaluator when required | Mapper feeds Architect, slices feed Writer; Reviewer may be used only as compatibility | +| **Large** | Code Mapper → Explorer → Architect (decompose + plan) → Code Writer → Builder/Tester → Code Reviewer → QA Evaluator when required | Full pipeline with slice verification; Reviewer may be used only as compatibility | +| **Mega** | All roles, parallel Code Writers per slice | Mapper → Explorer → Architect → parallel Writers → Builder/Tester, Code Reviewer, and QA Evaluator when required at integration | ## Dispatch guidelines -- **Every task gets at minimum**: Code Writer → Builder/Tester → Reviewer responsibilities. In delegated mode these are subagents; in direct fallback they are explicitly recorded role-equivalent steps. No code ships without build + test + review evidence. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts MUST infer required_agents with at least Code Writer, Builder/Tester, and Reviewer; `not_applicable` is invalid for source-changing Build tasks. -- **Strict evidence gate**: delegated mode is not complete until the task journal Agent Dispatch Log records Code Writer dispatch/result, Builder/Tester dispatch/result, and Reviewer dispatch/result evidence. Medium+ delegated slice work also records per-slice dispatch evidence before each slice is marked verified. **Codex adds a stronger check:** delegated Codex evidence must correspond to real `SubagentStart`/`SubagentStop` lifecycle records in `.codex/subagent-events.jsonl`; each dispatch/result entry must reference the matching lifecycle `agent_id`, and task-journal text alone is treated as insufficient because Codex can otherwise run phases inline and write claimed dispatch entries. Direct fallback is allowed only for explicit `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` reasons, and must record equivalent role, phase, verification, and review evidence; silent fallback fails the stop-review/phase gates. +- **Every task gets at minimum**: Code Writer → Builder/Tester → Code Reviewer responsibilities. `reviewer` remains valid compatibility routing for existing handoffs, but new dispatches should use `code-reviewer` for code defects, security, architecture, test coverage, and structural code issues. In delegated mode these are subagents; in direct fallback they are explicitly recorded role-equivalent steps. No code ships without build + test + review evidence. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts MUST infer required_agents with at least Code Writer, Builder/Tester, and Code Reviewer (or Reviewer compatibility); `not_applicable` is invalid for source-changing Build tasks. +- **Strict evidence gate**: delegated mode is not complete until the task journal Agent Dispatch Log records Code Writer dispatch/result, Builder/Tester dispatch/result, and Code Reviewer dispatch/result evidence, or Reviewer dispatch/result evidence when compatibility routing is used. Medium+ delegated slice work also records per-slice dispatch evidence before each slice is marked verified. **Codex adds a stronger check:** delegated Codex evidence must correspond to real `SubagentStart`/`SubagentStop` lifecycle records in `.codex/subagent-events.jsonl`; each dispatch/result entry must reference the matching lifecycle `agent_id`, and task-journal text alone is treated as insufficient because Codex can otherwise run phases inline and write claimed dispatch entries. Direct fallback is allowed only for explicit `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` reasons, and must record equivalent role, phase, verification, and review evidence; silent fallback fails the stop-review/phase gates. +- **QA evidence gate**: when QA is required, delegated mode is not complete until the task journal Agent Dispatch Log records QA Evaluator dispatch/result evidence after Builder/Tester and Code Reviewer evidence. Direct fallback must record fresh QA Evaluator direct evidence separately from Code Reviewer direct evidence. - **Every medium+ task gets Architect decomposition responsibility**: In delegated mode the Architect proposes smallest iterable slice boundaries; in direct fallback the same criteria and evidence are recorded directly. - **Launch in parallel** when agents are independent (e.g., Code Mapper + Explorer on different modules) - **Code Mapper runs first** on medium+ tasks — its output feeds into Architect and Code Writer -- **Reviewer gets a fresh dispatch each round** during the quality review loop when delegated mode is authorized; direct fallback must reset review context and record how stale-context risk was controlled +- **Code Reviewer gets a fresh dispatch each round** during the quality review loop when delegated mode is authorized; `reviewer` is the compatibility route for older handoffs. Direct fallback must reset review context and record how stale-context risk was controlled. +- **QA Evaluator gets a fresh dispatch each QA round** when QA is required and delegated mode is authorized. Direct fallback must reset acceptance-evaluation context and record how stale-context risk was controlled. - **Main session stays Orchestrator**: owns user communication, final integration, handoffs - **Do not dispatch agents for small inline tasks** within a larger workflow (e.g., a one-line fix during review doesn't need a Code Writer agent) diff --git a/skills/assistant-workflow/references/subagent-roles.md b/skills/assistant-workflow/references/subagent-roles.md index ab9e43a..fae490f 100644 --- a/skills/assistant-workflow/references/subagent-roles.md +++ b/skills/assistant-workflow/references/subagent-roles.md @@ -29,7 +29,9 @@ When custom agents are installed, dispatch by name. The agent's own configuratio Claude exposes the `Agent` tool with `subagent_type` values matching installed agent names: ``` -Agent(subagent_type="reviewer", prompt="Review the changes in {files}. This is round {N}...") +Agent(subagent_type="code-reviewer", prompt="Review the changes in {files}. This is round {N}...") +Agent(subagent_type="reviewer", prompt="Compatibility route: review the changes in {files}. This is round {N}...") +Agent(subagent_type="qa-evaluator", prompt="Evaluate acceptance for {task}. This is QA round {N}...") Agent(subagent_type="explorer", prompt="Trace execution paths for {feature}...") Agent(subagent_type="architect", prompt="Design implementation for {feature}...") Agent(subagent_type="code-mapper", prompt="Map the structure around {area}...") @@ -42,7 +44,9 @@ Agent(subagent_type="builder-tester", prompt="Build and test the project...") Current Codex CLI/app releases support native subagent workflows by default. Codex custom agents are standalone TOML files under `~/.codex/agents/` for personal agents or `.codex/agents/` for project-scoped agents. Ask Codex to spawn the configured agent by name; do not look for a visible Claude-style `Agent` tool or `subagent_type` parameter before deciding delegation is available: ``` -Spawn the reviewer agent to review the changes in {files}. This is round {N}... +Spawn the code-reviewer agent to review the changes in {files}. This is round {N}... +Spawn the reviewer agent only as compatibility routing for existing handoffs... +Spawn the qa-evaluator agent to evaluate acceptance for {task}. This is QA round {N}... Spawn the explorer agent to trace execution paths for {feature}... Spawn the architect agent to design implementation for {feature}... Spawn the code-mapper agent to map the structure around {area}... @@ -63,22 +67,24 @@ The prompt you provide is the **task context** — what to do, not how to do it. | `architect` | Strongest / deep reasoning | Read-only | Decompose, Plan, Design | Strict slice decomposition, implementation blueprints, design direction | | `code-writer` | Strongest / deep reasoning | Write | Build | Implements code following a plan. No builds, no tests, no review | | `builder-tester` | Balanced / standard | Write | Build | Builds, writes tests, runs tests. Returns concise summaries, not logs | -| `reviewer` | Strongest / deep reasoning | Read-only | Review | Finds bugs, security issues, architecture violations, structural problems | +| `code-reviewer` | Strongest / deep reasoning | Read-only | Review | Canonical code review for bugs, security issues, architecture violations, test coverage gaps, and structural problems | +| `reviewer` | Strongest / deep reasoning | Read-only | Review compatibility | Compatibility route for existing reviewer handoffs; prefer `code-reviewer` for new code review dispatches | +| `qa-evaluator` | Strongest / deep reasoning | Read-only | Review QA | Independent acceptance, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result evaluation | ## Dispatch rules by task size | Size | Agents used | Flow | |---|---|---| -| **Small** | Code Writer → Builder/Tester → Reviewer | Sequential, minimal (no Decompose) | -| **Medium** | Code Mapper → Architect (decompose) → Code Writer → Builder/Tester → Reviewer | Mapper feeds Architect, slices feed Writer | -| **Large** | Code Mapper → Explorer → Architect (decompose + plan) → Code Writer → Builder/Tester → Reviewer | Full pipeline with slice verification | -| **Mega** | All roles, parallel Code Writers per slice | Mapper → Explorer → Architect → parallel Writers → Builder/Tester and Reviewer at integration | +| **Small** | Code Writer → Builder/Tester → Code Reviewer | Sequential, minimal (no Decompose); Reviewer is compatibility routing; QA only when explicitly requested | +| **Medium** | Code Mapper → Architect (decompose) → Code Writer → Builder/Tester → Code Reviewer → QA Evaluator when required | Mapper feeds Architect, slices feed Writer; Reviewer is compatibility routing | +| **Large** | Code Mapper → Explorer → Architect (decompose + plan) → Code Writer → Builder/Tester → Code Reviewer → QA Evaluator when required | Full pipeline with slice verification; Reviewer is compatibility routing | +| **Mega** | All roles, parallel Code Writers per slice | Mapper → Explorer → Architect → parallel Writers → Builder/Tester, Code Reviewer, and QA Evaluator when required at integration | ## Reviewer dispatch (review rounds) **Round 1:** ``` -Use the reviewer agent to review all code changes. Find real issues, not nitpicks. +Use the code-reviewer agent to review all code changes. Find real issues, not nitpicks. Review for: bugs, logic errors, edge cases, security vulnerabilities, architecture violations, code quality, test coverage, and structural organization. Report at 80%+ confidence only. @@ -86,11 +92,37 @@ Report at 80%+ confidence only. **Round N > 1:** ``` -Use the reviewer agent. This is review round {N}. +Use the code-reviewer agent. This is review round {N}. The following {count} items were already found and fixed — do NOT re-report them: {previously_fixed list} Focus ONLY on NEW high-confidence findings. -Confidence threshold: Round 1-3: 80%+ | Round 4-7: 85%+ | Round 8-10: 90%+ +Confidence threshold: Round 1-3: 80%+ | Round 4-15: 85%+ | Round 16-20: 90%+ +``` + +## QA Evaluator dispatch (QA rounds) + +Use QA Evaluator only after build/test evidence and Code Reviewer or Reviewer compatibility result exist. It evaluates acceptance, not general code defects. + +**QA Round 1:** +``` +Use the qa-evaluator agent to evaluate acceptance for {task}. +Review: Done Contract, acceptance criteria, verification evidence, code review result, +domain_context/rubric_refs when applicable, and final result requirements. +Load assistant-review references/domain-rubrics.md only when the acceptance +criteria, Done Contract, domain_context, or explicit rubric_refs scope subjective, +product, UX, docs, DX, UI/visual, or domain craft scoring. Return selected_domain_rubrics +and domain_quality_scores when scoped; do not invent rubrics when not scoped. +Do not replace Code Reviewer. Report only acceptance findings backed by evidence. +``` + +**QA Round N > 1:** +``` +Use the qa-evaluator agent. This is QA round {N} of 20. +Previously failed acceptance items: +{previously_failed_acceptance_items} +Do not re-report resolved items. In rounds 16-20, only unresolved acceptance +blockers or high-confidence acceptance risks keep the QA loop open. +Round 20 is terminal; return the final verdict instead of implying round 21. ``` ## Fallback: agents not installed @@ -104,6 +136,8 @@ If custom agents are not installed, use the nearest available built-in subagent | Architect | `general-purpose` or nearest planning agent | Strongest / deep reasoning | | Code Writer | `general-purpose` or nearest coding agent | Strongest / deep reasoning | | Builder/Tester | `general-purpose` or nearest coding agent with shell/test access | Balanced / standard | -| Reviewer | `general-purpose` or nearest read-only reviewer | Strongest / deep reasoning | +| Code Reviewer | `general-purpose` or nearest read-only reviewer | Strongest / deep reasoning | +| Reviewer compatibility | `general-purpose` or nearest read-only reviewer | Strongest / deep reasoning | +| QA Evaluator | `general-purpose` or nearest read-only evaluator | Strongest / deep reasoning | When using fallback types, include the full role instructions in the dispatch prompt (the agent won't have a built-in system prompt for the role). Prefer the matching role definition for the active runtime under the framework's `agents/` directory when available; otherwise adapt the closest provider-neutral role prompt. diff --git a/skills/assistant-workflow/references/task-journal-template.md b/skills/assistant-workflow/references/task-journal-template.md index faf878d..ded6a25 100644 --- a/skills/assistant-workflow/references/task-journal-template.md +++ b/skills/assistant-workflow/references/task-journal-template.md @@ -10,6 +10,9 @@ Write to `{agent_state_dir}/task.md` in the project root when a local agent stat ## When to update - When clarification questions are asked, answered, or resolved via explicit `defaults` - After each Build step completes (update Progress, Artifact Registry, and check off Milestones) +- After each medium+ harness event that changes phase, slice, status, blockers, verification, next action, decisions, deviations, or artifact refs +- After any Pivot/Restart Decision, including review/QA stagnation, repeated drift/regression, Code Writer blockers, or selected restart/replan action +- After any producer creates/updates a typed artifact reference or any consumer changes `validation_status` - When key decisions are made - When constraints are added by the user - After each review cycle pass (append to Review Log — never overwrite) @@ -51,7 +54,7 @@ Plan approval: [yes/no + date] ## Agent Dispatch Log [strict subagent evidence inspected by stop-review/phase gates] -- Required roles: Code Writer, Builder/Tester, Reviewer [plus Code Mapper/Explorer/Architect when required by size/risk] +- Required roles: Code Writer, Builder/Tester, Code Reviewer [Reviewer remains compatibility routing; plus QA Evaluator when QA is required; plus Code Mapper/Explorer/Architect when required by size/risk] - Execution mode: delegated | direct_fallback | not_applicable - Codex lifecycle evidence: delegated Codex roles must have matching `.codex/subagent-events.jsonl` `SubagentStart` + `SubagentStop` records; dispatch/result entries must cite the matching `agent_id`; journal text alone is not proof. - Direct fallback reason: [authorization_denied | subagents_unavailable | policy_disallowed | N/A] @@ -70,9 +73,15 @@ Plan approval: [yes/no + date] - Builder/Tester dispatch: [subagent/tool/run id, verification packet, timestamp, or N/A only in direct_fallback/not required] - Builder/Tester result: [commands/results/success signal returned, or N/A only in direct_fallback/not required] - Builder/Tester direct evidence: [role-equivalent build/test/validation evidence when direct_fallback, else N/A] -- Reviewer dispatch: [reviewer subagent/tool/run id or assistant-review dispatch, or N/A only in direct_fallback/not required] -- Reviewer result: [spec/quality review result evidence returned, or N/A only in direct_fallback/not required] -- Reviewer direct evidence: [fresh review/spec+quality evidence when direct_fallback, else N/A] +- Code Reviewer dispatch: [code-reviewer subagent/tool/run id or assistant-review dispatch, or N/A only when Reviewer compatibility/direct_fallback/not required] +- Code Reviewer result: [spec/quality review result evidence returned, or N/A only when Reviewer compatibility/direct_fallback/not required] +- Code Reviewer direct evidence: [fresh review/spec+quality evidence when direct_fallback, else N/A] +- Reviewer dispatch: [compatibility reviewer subagent/tool/run id or assistant-review dispatch, or N/A only in direct_fallback/not required] +- Reviewer result: [compatibility spec/quality review result evidence returned, or N/A only in direct_fallback/not required] +- Reviewer direct evidence: [compatibility fresh review/spec+quality evidence when direct_fallback, else N/A] +- QA Evaluator dispatch: [qa-evaluator subagent/tool/run id or assistant-review QA dispatch, or N/A only when qa_evaluation_mode=not_required/direct_fallback] +- QA Evaluator result: [acceptance final_verdict/result, score_progression, and evidence returned, or N/A only when qa_evaluation_mode=not_required/direct_fallback] +- QA Evaluator direct evidence: [fresh acceptance/Done Contract/verification QA evidence when direct_fallback, else N/A] - Per-slice dispatch evidence: [medium+ delegated only: slice_id -> Code Writer dispatch/result + Builder/Tester dispatch/result; direct_fallback records sequential role-equivalent slice evidence] ## Constraints @@ -92,6 +101,101 @@ Plan approval: [yes/no + date] |------|---------|-----------| | [path] | [what and why] | Step N | +## Done Contract +[required before Build for medium+ harness-capable work; otherwise record N/A with reason] +- done_when: + - [binary outcome that proves done] +- not_done_when: + - [failure state that blocks done] +- verification: + - [command, inspection, review, or manual check] +- owner_consumer: [owner and downstream consumer] +- acceptance_criteria: + - [explicit binary criterion] +- debate_record: + - perspective: [role/subagent/direct perspective 1] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] + - perspective: [role/subagent/direct perspective 2] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] +- accepted_by: [user/orchestrator/approved plan reference] + +## Harness Recipe +[required before Build for medium+ harness-capable work; otherwise record N/A with reason] +- task_profile: [task type, size, slice count, TDD/debugging applicability] +- model_profile: [agent/model constraints, delegation mode, tool limits] +- risk_profile: [risk tier, safety gates, review depth, rollback needs] +- context_profile: [exact/summarized/omitted context and trace/replay needs] +- selected_recipe: [concise recipe label] +- recipe_rationale: [why this task/model/risk/context profile selects the recipe] +- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] +- corrective_action: [what to do if missing or stale] + +## Harness Run State +[required for medium+ harness-capable work; otherwise record N/A with reason] +- task_id: [stable task/run id] +- task_name: [human-readable task name] +- phase: [TRIAGE | DISCOVER | DECOMPOSE | PLAN | DESIGN | BUILD | REVIEW | DOCUMENT | COMPLETE] +- slice: [current slice id/name, next pending slice, or N/A] +- status: [not_started | in_progress | blocked | verifying | reviewing | restarting | documenting | complete] +- blockers: + - [current blocker, or none] +- last_verification: + - command_or_check: [command/check, or pending/N/A] + - result: [passed | failed | skipped | not_applicable | pending] + - evidence: [concise evidence or reason] +- next_action: [exact next action] +- recovery_pointer: [task packet, trace entry, ledger row, file section, or artifact ref] +- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] + +## Trace Ledger +[required for medium+ harness-capable work; append-only ordered execution evidence] +| Seq | Timestamp/Order | Event Type | Actor | Summary | Artifact Refs | +|-----|-----------------|------------|-------|---------|---------------| +| 1 | [timestamp or ordered marker] | [agent_event/decision/verification/plan_deviation/pivot_restart/artifact_ref/blocker/recovery] | [role/subagent/user/hook] | [event, decision, command/result, deviation, blocker, pivot/restart, or recovery] | [file/section/dispatch id/command ref] | + +## Replay Packet +[required for medium+ harness-capable work before compaction, failure handoff, phase handoff, or end-of-turn] +- pinned_context: + - [stable requirement, constraint, approved plan/slice id, or non-goal] +- artifact_refs: + - [task journal/context map/plan/run state/trace/validation/changed file ref] +- validation_state: + - completed_checks: [checks already completed] + - pending_checks: [checks still pending] + - last_result: [latest verification/review result] +- exact_next_action: [single concrete next action after replay] +- run_state_ref: [Harness Run State section/ref] +- trace_ledger_ref: [Trace Ledger section/ref] +- recovery_pointer: [where to resume] +- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] + +## Pivot/Restart Log +[append when review/QA stagnation, repeated drift/regression, rubric/domain pivot, Code Writer blocker, verification blocker, plan deviation, or scope change triggers recovery; otherwise record N/A with reason] + +### Pivot/Restart Decision #N +- trigger: [STAGNATION | repeated_DRIFT | repeated_REGRESSION | pivot | code_writer_blocker | verification_blocker | plan_deviation | scope_change] +- evidence: + - source: [review score entry, QA score entry, Code Writer blocker, verification failure, trace row, or task packet] + detail: [concise evidence] +- affected_slice_or_round: [slice id/name, review round, QA round, or phase] +- options_considered: + - option: [reset context / dispatch debugging / dispatch explorer / dispatch architect / candidate search / replan / restart slice / restart phase / block for user / accept with limitations] + tradeoff: [why this option helps or risks the task] + disposition: [selected | rejected | deferred] +- selected_action: [reset_context | return_to_build | dispatch_debugging | dispatch_explorer | dispatch_architect | run_candidate_search | replan | restart_slice | restart_phase | block_for_user | accept_with_limitations] +- reapproval_required: [true when scope/files/behavior/risk/verification/acceptance changes; otherwise false] +- next_agent: [Builder/Tester | Code Writer | Explorer | Architect | Reviewer | QAEvaluator | assistant-debugging | candidate-search | user | none] +- recovery_pointer: [task packet, trace row, replay packet, plan section, or file path] +- exact_next_action: [single concrete action after the decision] + +## Artifact Reference Ledger +[required for medium+ harness-capable work when artifacts pass between agents; typed producer/consumer records, not ad hoc strings] +| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | +|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| +| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/pivot_restart_decision/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role/subagent/hook] | [role/subagent/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | + ## Milestones [compression-safe boundaries — each marks a point where context can be safely truncated] - [ ] M1: [milestone description] (after Step N) @@ -150,13 +254,14 @@ Plan approval: [yes/no + date] - Required fixes: [none, or ordered fix list] ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Previously fixed: 0 items from prior rounds - Found this round: [count] must-fix, [count] should-fix, [count] nits (all fixed below) - Rubric: correctness=[score] quality=[score] architecture=[score] security=[score] coverage=[score] - Weighted: [score] - Delta from previous: — (first round) - Drift check: — (first round) +- Pivot/Restart decision: [ref if STAGNATION, repeated DRIFT, repeated REGRESSION, or PIVOT occurred; otherwise N/A] - Complexity: [ran / skipped (not C#) / tool unavailable] - [method (line N): score X — refactored to Y, or "within threshold"] - Must-fix: @@ -166,13 +271,14 @@ Plan approval: [yes/no + date] - Re-test: PASS ### Quality Review #2 (autonomous re-review) -- Round: 2 of 10 +- Round: 2 of 20 - Previously fixed: [count] items from prior rounds - Found this round: [count] must-fix, [count] should-fix, [count] nits (all fixed below) - Rubric: correctness=[score] quality=[score] architecture=[score] security=[score] coverage=[score] - Weighted: [score] - Delta from previous: [+/- amount] - Drift check: [GENUINE / SUSPICIOUS / DRIFT / REGRESSION / STAGNATION / NEUTRAL] +- Pivot/Restart decision: [ref if STAGNATION, repeated DRIFT, repeated REGRESSION, or PIVOT occurred; otherwise N/A] - Must-fix: - [x] [file:line] — [issue] → [fix applied] - Should-fix: @@ -182,12 +288,36 @@ Plan approval: [yes/no + date] [...repeat until clean or max rounds reached...] [Note: On test failure, skip this entry — write only "- Result: HAS_REMAINING_ITEMS" to Final result] +### QA Evaluation #1 +- Round: 1 of 20 +- Mode: required | optional | not_required +- Done Contract: [ref or N/A with reason] +- Acceptance criteria checked: [count] +- Verification evidence checked: [commands/checks/manual/review refs] +- Code Review evidence: [Code Reviewer/Reviewer result ref] +- Domain/rubric refs: [product/UX/UI/docs/DX/domain refs, or N/A when not scoped] +- Selected domain rubrics: [ui_visual_design | ux_product_acceptance | documentation_quality | developer_experience | domain_specific_craft | N/A] +- Domain quality scores: [rubric.dimension=score/action/evidence, or N/A when not scoped] +- QA scorecard: acceptance_coverage=[score] evidence_strength=[score] domain_quality=[score] final_readiness=[score] +- Weighted: [score] +- Score progression: [round1->...] +- Pivot/Restart decision: [ref if STAGNATION, repeated DRIFT, repeated REGRESSION, or domain action pivot occurred; otherwise N/A] +- Final verdict: accepted | accepted_with_concerns | rejected | blocked +- Acceptance findings: + - [blocker|concern|observation] [criterion] — [evidence] → [fix/defer/open question] + +[...repeat QA Evaluation until accepted, accepted_with_concerns, blocked, or max round 20 reached...] + ### Final result - Result: CLEAN | ISSUES_FIXED | HAS_REMAINING_ITEMS - Review rounds: [count] +- QA result: [accepted | accepted_with_concerns | rejected | blocked | not_required] +- QA rounds: [count or N/A] +- QA score progression: [round1->round2->...roundN or N/A] - Final rubric score: [weighted score] ([PASS/REFINE/PIVOT]) - Score progression: [round1→round2→...roundN] (e.g., 3.50→3.85→4.10) - Drift incidents: [count, or "none"] +- Pivot/Restart decisions: [count and refs, or "none"] - Total must-fix resolved: [count across all rounds] - Total should-fix resolved: [count across all rounds] - Should-fix deferred: [list any remaining] @@ -222,8 +352,8 @@ Plan approval: [yes/no + date] 3. **Clarification** updates — question caps are maximums, not quotas. Clear medium+ tasks may record `Clarification questions asked: 0` with `Clarification confidence: high`. While waiting, keep `Status: DISCOVERING`, set `Clarification status: needs_clarification`, set `Clarification defaults applied: false`, set confidence/cap/admissibility fields, and list every unresolved implementation-shaping topic. On explicit answers, clear unresolved topics, keep `Clarification defaults applied: false`, and set `Clarification status: ready`. On explicit `defaults`, print the applied defaults, clear unresolved topics, set `Clarification defaults applied: true`, and set `Clarification status: ready`. 4. **Decompose** — medium+ tasks set `Status: DECOMPOSING` after Discover is ready, then persist the slice manifest before moving on to planning. Small tasks skip this state. 5. **Plan approval** — once ready to plan, set `Status: PLANNING`, include the slice manifest in the plan for medium+ tasks, capture the approved plan, and update `Plan approval`. -6. **Build** each step — update Progress, Artifact Registry, Key Decisions, Status after each step. For medium+ tasks, update the Slice Verification Ledger after each slice and do not start the next slice until the current one is `VERIFIED`. Check off Milestones when reached. -7. **Review cycle** when all steps done — Spec Review first (structured PASS/FAIL from `references/prompts/spec-review.md`), then Quality Review (assistant-review quality loop), fix must-fix → re-test → re-review until clean, fill Final Result +6. **Build** each step — update Progress, Artifact Registry, Artifact Reference Ledger, Key Decisions, Status, Harness Run State, Trace Ledger, Replay Packet, and Pivot/Restart Log when triggered after each step. For medium+ tasks, update the Slice Verification Ledger after each slice and do not start the next slice until the current one is `VERIFIED`. Check off Milestones when reached. +7. **Review cycle** when all steps done — Spec Review first (structured PASS/FAIL from `references/prompts/spec-review.md`), then Quality Review (assistant-review quality loop), fix must-fix → re-test → re-review until clean or a Pivot/Restart Decision routes recovery, fill Final Result 8. **Document** after review cycle passes — fill Verification Summary, add the Learning Controller block, Status: DOCUMENTING 9. **Handoff** to user — they test manually and add Review Notes 10. **Review fixes** — fix issues, re-test, re-review, update Progress @@ -234,6 +364,8 @@ Plan approval: [yes/no + date] - Keep entries concise — this is a log, not documentation - Resume from clarification waits only on explicit numbered answers or explicit `defaults` - Constraints are checked before each Build step +- Producer roles update Artifact Reference Ledger entries when they create or move artifacts; Consumer roles validate `schema_or_contract` and update `validation_status` before using them +- Pivot/Restart Decisions are append-only recovery records. If the selected action changes scope, files, behavior, risk, verification, or acceptance criteria, record `reapproval_required: true` and wait for approval before continuing. - On context continuation: read the configured task journal FIRST when it exists, before any other action - Never delete constraints unless the user explicitly removes them -- The Verification Summary replaces the need for context-handoff templates during active work +- The Replay Packet replaces the need for context-handoff templates during active harness work diff --git a/tests/p0-p4/assistant-review-reference-contracts.sh b/tests/p0-p4/assistant-review-reference-contracts.sh index e3621fe..f5db41f 100644 --- a/tests/p0-p4/assistant-review-reference-contracts.sh +++ b/tests/p0-p4/assistant-review-reference-contracts.sh @@ -113,7 +113,7 @@ for file_and_term in \ "$review_skill::REFINE (weighted below 4.0 but not PIVOT), including zero findings -> continue to step 3" \ "$review_skill::Medium+ CLEAN and ISSUES_FIXED require weighted >= 4.0" \ "$review_phase_gates::EXIT_CLEAN or EXIT_ISSUES_FIXED only if: zero must-fix AND zero should-fix findings, and for medium+ scope weighted_score >= 4.0" \ - "$review_rubric::| 4-10 | 4.0+ | 3.25"; do + "$review_rubric::| 4-20 | 4.0+ | 3.25"; do file="${file_and_term%%::*}" term="${file_and_term#*::}" if [[ ! -f "$file" ]] || ! grep -Fq "$term" "$file"; then @@ -127,4 +127,31 @@ else fail "assistant-review medium rubric clean-exit threshold contract drifted: ${review_threshold_failures[*]}" fi +test_start "assistant-review phase-gate IDs are unique" +phase_gate_id_failures=() +for phase_gate_file in \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/phase-gates.yaml" \ + "$FRAMEWORK_DIR/plugins/assistant-dev/skills/assistant-review/contracts/phase-gates.yaml"; do + if [[ ! -f "$phase_gate_file" ]]; then + phase_gate_id_failures+=("${phase_gate_file#$FRAMEWORK_DIR/}: missing") + continue + fi + + duplicate_phase_gate_ids="$( + awk '/^[[:space:]]+- id: / { count[$3]++ } END { for (id in count) if (count[id] > 1) print id }' "$phase_gate_file" \ + | sort + )" + + if [[ -n "$duplicate_phase_gate_ids" ]]; then + duplicate_phase_gate_ids="${duplicate_phase_gate_ids//$'\n'/ }" + phase_gate_id_failures+=("${phase_gate_file#$FRAMEWORK_DIR/}: duplicate ids $duplicate_phase_gate_ids") + fi +done + +if [[ "${#phase_gate_id_failures[@]}" -eq 0 ]]; then + pass +else + fail "assistant-review phase-gate IDs must be unique: ${phase_gate_id_failures[*]}" +fi + p0p4_finish_suite "${BASH_SOURCE[0]}" diff --git a/tests/p0-p4/domain-rubrics-contracts.sh b/tests/p0-p4/domain-rubrics-contracts.sh new file mode 100644 index 0000000..1fd92c2 --- /dev/null +++ b/tests/p0-p4/domain-rubrics-contracts.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash + +if [[ -z "${P0P4_HARNESS_LOADED:-}" ]]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/p0p4-harness.sh" +fi +p0p4_bootstrap_suite "${BASH_SOURCE[0]}" + +require_terms() { + local label="$1" + local file="$2" + shift 2 + + local missing=() + local term + for term in "$@"; do + if ! grep -Fq -- "$term" "$file"; then + missing+=("${file#$FRAMEWORK_DIR/}: $term") + fi + done + + if [[ "${#missing[@]}" -eq 0 ]]; then + pass + else + fail "$label missing terms: ${missing[*]}" + fi +} + +review_dir="$FRAMEWORK_DIR/skills/assistant-review" +workflow_dir="$FRAMEWORK_DIR/skills/assistant-workflow" +domain_ref="$review_dir/references/domain-rubrics.md" + +test_start "assistant-review domain rubric reference defines scoped families" +if [[ ! -f "$domain_ref" ]]; then + fail "missing skills/assistant-review/references/domain-rubrics.md" +else + require_terms "domain rubric reference" "$domain_ref" \ + "Use this reference only for QA evaluation when scoped domain quality is part of acceptance" \ + "acceptance criteria, Done Contract, \`domain_context\`, or explicit \`rubric_refs\`" \ + "Do not load or apply these rubrics for code-review-only work" \ + "ui_visual_design" \ + "ux_product_acceptance" \ + "documentation_quality" \ + "developer_experience" \ + "domain_specific_craft" \ + "Evidence examples" \ + "Pass / refine / pivot guidance" \ + "accepted_with_concerns" \ + "not_applicable" \ + "do not invent" +fi + +test_start "QA loop and assistant-review skill load domain rubrics conditionally" +domain_loop_failures=() +for file_and_term in \ + "$review_dir/references/qa-evaluation-loop.md::Load \`references/domain-rubrics.md\` only when" \ + "$review_dir/references/qa-evaluation-loop.md::selected_domain_rubrics" \ + "$review_dir/references/qa-evaluation-loop.md::domain_quality_scores" \ + "$review_dir/references/qa-evaluation-loop.md::Do not invent domain rubrics" \ + "$review_dir/SKILL.md::references/domain-rubrics.md\` only when" \ + "$review_dir/SKILL.md::selected_domain_rubrics/domain_quality_scores when used" \ + "$review_dir/SKILL.md::Code Reviewer still owns code defects"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + domain_loop_failures+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#domain_loop_failures[@]}" -eq 0 ]]; then + pass +else + fail "assistant-review domain-rubric loop terms missing: ${domain_loop_failures[*]}" +fi + +test_start "assistant-review contracts require selected domain rubric return fields only when scoped" +contract_failures=() +for file_and_term in \ + "$review_dir/contracts/input.yaml::Valid families are defined in references/domain-rubrics.md" \ + "$review_dir/contracts/handoffs.yaml::selected_domain_rubrics" \ + "$review_dir/contracts/handoffs.yaml::domain_quality_scores" \ + "$review_dir/contracts/handoffs.yaml::condition: \"selected_domain_rubrics is non-empty\"" \ + "$review_dir/contracts/handoffs.yaml::do not fabricate them" \ + "$review_dir/contracts/output.yaml::selected_domain_rubrics" \ + "$review_dir/contracts/output.yaml::domain_quality_scores" \ + "$review_dir/contracts/phase-gates.yaml::Domain rubrics from references/domain-rubrics.md are loaded and applied only when scoped" \ + "$review_dir/contracts/phase-gates.yaml::does not invent domain rubrics"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + contract_failures+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#contract_failures[@]}" -eq 0 ]]; then + pass +else + fail "assistant-review domain-rubric contracts missing terms: ${contract_failures[*]}" +fi + +test_start "QA evaluator prompts remain read-only and use domain rubrics conditionally" +prompt_failures=() +for file_and_term in \ + "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml::sandbox_mode = \"read-only\"" \ + "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml::Load \`skills/assistant-review/references/domain-rubrics.md\` only when" \ + "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml::selected_domain_rubrics" \ + "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml::domain_quality_scores" \ + "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml::Do NOT invent domain rubrics" \ + "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md::tools: Read, Grep, Glob, LS" \ + "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md::Load \`skills/assistant-review/references/domain-rubrics.md\` only when" \ + "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md::selected_domain_rubrics" \ + "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md::domain_quality_scores" \ + "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md::Do NOT invent domain rubrics"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + prompt_failures+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ -f "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md" ]] \ + && grep -Eq '^tools: .*Edit|^tools: .*Write|^tools: .*Bash' "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md"; then + prompt_failures+=("agents/claude/qa-evaluator.md: unexpected write/shell tools") +fi +if [[ "${#prompt_failures[@]}" -eq 0 ]]; then + pass +else + fail "QA evaluator domain-rubric prompt terms missing: ${prompt_failures[*]}" +fi + +test_start "workflow records selected QA domain rubrics separately from review_result" +workflow_failures=() +for file_and_term in \ + "$workflow_dir/contracts/handoffs.yaml::selected_domain_rubrics" \ + "$workflow_dir/contracts/handoffs.yaml::domain_quality_scores" \ + "$workflow_dir/contracts/output.yaml::selected_domain_rubrics" \ + "$workflow_dir/contracts/output.yaml::domain_quality_scores" \ + "$workflow_dir/contracts/phase-gates.yaml::reject invented rubric scoring" \ + "$workflow_dir/references/phases.md::references/domain-rubrics.md\` only when" \ + "$workflow_dir/references/task-journal-template.md::Selected domain rubrics" \ + "$workflow_dir/references/task-journal-template.md::Domain quality scores" \ + "$workflow_dir/references/harness-controller.md::domain_context" \ + "$workflow_dir/references/subagent-roles.md::Return selected_domain_rubrics"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + workflow_failures+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#workflow_failures[@]}" -eq 0 ]]; then + pass +else + fail "workflow domain-rubric recording terms missing: ${workflow_failures[*]}" +fi + +test_start "assistant-review eval fixture covers conditional domain rubric QA" +review_evals="$review_dir/evals/cases.json" +require_terms "assistant-review eval fixture" "$review_evals" \ + "qa-evaluator-uses-domain-rubrics-conditionally" \ + "references/domain-rubrics.md" \ + "selected_domain_rubrics" \ + "domain_quality_scores" \ + "documentation_quality" \ + "developer_experience" \ + "domain_context" \ + "rubric_refs" \ + "not_applicable" \ + "QAEvaluator replaces Code Reviewer" + +test_start "domain rubric wording avoids unconditional loading outside eval failure examples" +unconditional_failures=() +for file in \ + "$review_dir/SKILL.md" \ + "$review_dir/references/qa-evaluation-loop.md" \ + "$review_dir/contracts/handoffs.yaml" \ + "$review_dir/contracts/phase-gates.yaml" \ + "$workflow_dir/references/phases.md" \ + "$workflow_dir/references/subagent-roles.md" \ + "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml" \ + "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md"; do + if [[ -f "$file" ]] && grep -Fiq -- "always load" "$file"; then + unconditional_failures+=("${file#$FRAMEWORK_DIR/}: contains always load") + fi +done +if [[ "${#unconditional_failures[@]}" -eq 0 ]]; then + pass +else + fail "domain rubric unconditional loading wording found: ${unconditional_failures[*]}" +fi + +p0p4_finish_suite "${BASH_SOURCE[0]}" diff --git a/tests/p0-p4/harness-controller-contracts.sh b/tests/p0-p4/harness-controller-contracts.sh new file mode 100644 index 0000000..734acf3 --- /dev/null +++ b/tests/p0-p4/harness-controller-contracts.sh @@ -0,0 +1,148 @@ +if [[ -z "${P0P4_HARNESS_LOADED:-}" ]]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/p0p4-harness.sh" +fi +p0p4_bootstrap_suite "${BASH_SOURCE[0]}" + +require_terms() { + local label="$1" + local file="$2" + shift 2 + + local missing=() + local term + for term in "$@"; do + if ! grep -Fq -- "$term" "$file"; then + missing+=("${file#$FRAMEWORK_DIR/}: $term") + fi + done + + if [[ "${#missing[@]}" -eq 0 ]]; then + pass + else + fail "$label missing terms: ${missing[*]}" + fi +} + +workflow_dir="$FRAMEWORK_DIR/skills/assistant-workflow" +harness_ref="$workflow_dir/references/harness-controller.md" + +test_start "harness controller reference defines Done Contract and recipe selection" +if [[ ! -f "$harness_ref" ]]; then + fail "missing skills/assistant-workflow/references/harness-controller.md" +else + require_terms "harness reference" "$harness_ref" \ + "Use this reference only for medium+ work that is harness-capable" \ + "Before Build starts" \ + "Done Contract" \ + "Harness Recipe" \ + 'done_when' \ + 'not_done_when' \ + 'verification' \ + 'owner_consumer' \ + 'acceptance_criteria' \ + 'debate_record' \ + "at least two perspectives" \ + 'subagent_execution_mode=delegated' \ + 'task_profile' \ + 'model_profile' \ + 'risk_profile' \ + 'context_profile' \ + 'Corrective action' +fi + +test_start "workflow loads harness reference only for relevant medium+ work" +missing_load_terms=() +for term in \ + "Medium+ harness-capable work has an accepted Done Contract and Harness Recipe before Build." \ + "Load \`references/harness-controller.md\` only when medium+ work is harness-capable"; do + if ! grep -Fq -- "$term" "$workflow_dir/SKILL.md"; then + missing_load_terms+=("SKILL.md: $term") + fi +done +for term in \ + "For medium+ harness-capable work, load \`references/harness-controller.md\`" \ + "confirm the task journal or carried-forward plan has an accepted Done Contract, selected Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger before dispatching Code Writer or Builder/Tester" \ + 'done_contract_ref' \ + 'harness_recipe_ref'; do + if ! grep -Fq -- "$term" "$workflow_dir/references/phases.md"; then + missing_load_terms+=("phases.md: $term") + fi +done +if [[ "${#missing_load_terms[@]}" -eq 0 ]]; then + pass +else + fail "workflow harness load guards missing terms: ${missing_load_terms[*]}" +fi + +test_start "phase gates require Done Contract and Harness Recipe before Build" +phase_gates="$workflow_dir/contracts/phase-gates.yaml" +require_terms "phase gates" "$phase_gates" \ + "- id: P_DONE_CONTRACT" \ + "accepted Done Contract exists before Build" \ + "done_when, not_done_when, verification, owner/consumer, acceptance_criteria" \ + "debate_record with at least two perspectives" \ + "subagents when subagent_execution_mode=delegated" \ + "Block Build" \ + "- id: P_HARNESS_RECIPE" \ + "selected before Build from task/model/risk/context profile" \ + "classify task_profile, model_profile, risk_profile, and context_profile" \ + "- id: B_DONE_CONTRACT" + +test_start "output contract defines Done Contract and Harness Recipe artifacts" +output_contract="$workflow_dir/contracts/output.yaml" +require_terms "output contract" "$output_contract" \ + "- name: done_contract" \ + 'condition: "size in [medium, large, mega] and harness_capable == true"' \ + "done_when" \ + "not_done_when" \ + "verification" \ + "owner_consumer" \ + "acceptance_criteria" \ + "debate_record" \ + "min_items: 2" \ + "using subagents when delegated mode is available" \ + "- name: harness_recipe" \ + "task_profile" \ + "model_profile" \ + "risk_profile" \ + "context_profile" \ + "selected_recipe" \ + "recipe_rationale" \ + "required_artifacts" \ + "corrective_action" + +test_start "plan, journal, and handoffs carry harness artifacts without mirror edits" +missing_surface_terms=() +for term in \ + "## Done Contract" \ + "## Harness Recipe" \ + "done_contract_ref" \ + "harness_recipe_ref"; do + if ! grep -Fq -- "$term" "$workflow_dir/references/plan-template.md"; then + missing_surface_terms+=("plan-template.md: $term") + fi +done +for term in \ + "## Done Contract" \ + "## Harness Recipe" \ + "done_when" \ + "corrective_action"; do + if ! grep -Fq -- "$term" "$workflow_dir/references/task-journal-template.md"; then + missing_surface_terms+=("task-journal-template.md: $term") + fi +done +for term in \ + "done_contract_ref" \ + "harness_recipe_ref" \ + "task/model/risk/context profile"; do + if ! grep -Fq -- "$term" "$workflow_dir/contracts/handoffs.yaml"; then + missing_surface_terms+=("handoffs.yaml: $term") + fi +done +if [[ "${#missing_surface_terms[@]}" -eq 0 ]]; then + pass +else + fail "workflow harness artifact surfaces missing terms: ${missing_surface_terms[*]}" +fi + +p0p4_finish_suite "${BASH_SOURCE[0]}" diff --git a/tests/p0-p4/harness-docs-evals-contracts.sh b/tests/p0-p4/harness-docs-evals-contracts.sh new file mode 100644 index 0000000..7b1474b --- /dev/null +++ b/tests/p0-p4/harness-docs-evals-contracts.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash + +if [[ -z "${P0P4_HARNESS_LOADED:-}" ]]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/p0p4-harness.sh" +fi +p0p4_bootstrap_suite "${BASH_SOURCE[0]}" + +require_terms() { + local label="$1" + local file="$2" + shift 2 + + local missing=() + local term + for term in "$@"; do + if ! grep -Fq -- "$term" "$file"; then + missing+=("${file#$FRAMEWORK_DIR/}: $term") + fi + done + + if [[ "${#missing[@]}" -eq 0 ]]; then + pass + else + fail "$label missing terms: ${missing[*]}" + fi +} + +harness_doc="$FRAMEWORK_DIR/docs/harness-design-guide.md" +eval_readme="$FRAMEWORK_DIR/docs/evals/README.md" +contract_guide="$FRAMEWORK_DIR/docs/skill-contract-design-guide.md" +skill_creator_patterns="$FRAMEWORK_DIR/skills/assistant-skill-creator/references/harness-patterns.md" +framework_fixture="$FRAMEWORK_DIR/docs/evals/framework-instruction-cases.json" +skill_creator_fixture="$FRAMEWORK_DIR/skills/assistant-skill-creator/evals/cases.json" + +test_start "harness design guide documents current controller surfaces" +require_terms "harness design guide" "$harness_doc" \ + "Done Contract" \ + "debate_record" \ + "subagent perspectives" \ + "Harness Recipe" \ + "Harness Run State" \ + "Trace Ledger" \ + "Replay Packet" \ + "Artifact Reference Ledger" \ + "Typed Artifact References" \ + "Code Reviewer" \ + "QA Evaluator" \ + "Conditional Domain Rubrics" \ + "Pivot/Restart Controller" \ + "legacy_code_bug" \ + "assistant-debugging" \ + "max 20 rounds" \ + "Round 20 is terminal" + +test_start "harness design guide avoids stale three-agent and 10-round wording" +stale_harness_output="$(mktemp)" +p0p4_register_cleanup "$stale_harness_output" +if rg -n -e 'three-agent' -e '3-agent' -e 'max 10' -e 'round <= 10' "$harness_doc" >"$stale_harness_output"; then + fail "harness design guide contains stale harness wording; see $stale_harness_output" +else + pass +fi + +test_start "eval README lists harness controller eval areas" +require_terms "docs eval README" "$eval_readme" \ + "Done Contract debate and Harness Recipe before Build" \ + "trace/replay artifacts and typed artifact refs for harness recovery" \ + "separate Code Reviewer and QA Evaluator evidence" \ + "QA loop behavior with conditional domain rubrics" \ + "pivot/restart decisions for stagnation and Code Writer blockers" \ + "terminal max 20 review/QA round behavior" + +test_start "skill contract guide describes process harness controller contracts" +require_terms "skill contract guide" "$contract_guide" \ + "Contract and Harness Recipe before Build" \ + "typed Artifact References" \ + "run-state, trace, replay, review, QA, and pivot/restart" \ + "artifacts when the controller requires them" \ + "Code Reviewer and QA Evaluator" \ + "handoffs stay separate" \ + "max 20" \ + "pivot_restart_decision" + +test_start "skill-creator harness patterns cover controller artifacts and loop safety" +require_terms "skill creator harness patterns" "$skill_creator_patterns" \ + "Done Contract" \ + "debate_record" \ + "Harness Recipe" \ + "Runtime State, Trace, And Replay" \ + "Typed Artifact References" \ + "Code Review And QA Separation" \ + "Conditional Domain Rubrics" \ + "Bounded Review / QA Loops" \ + "max 20 rounds" \ + "Pivot / Restart Decisions" \ + "legacy_code_bug" \ + "Agentic Loop Safety" + +test_start "framework fixture includes harness controller behavior cases" +if jq -e ' + def case_category($id; $category): + any(.cases[]; .id == $id and .category == $category); + case_category("harness-done-contract-recipe-before-build"; "harness_controller") + and case_category("code-reviewer-and-qa-evaluator-evidence-split"; "review_qa_split") + and case_category("pivot-restart-on-stagnation-or-code-writer-blocker"; "pivot_restart") + and case_category("review-and-qa-terminal-cap-20"; "review_loop_cap") + and all(.cases[] | select(.id as $id | [ + "harness-done-contract-recipe-before-build", + "code-reviewer-and-qa-evaluator-evidence-split", + "pivot-restart-on-stagnation-or-code-writer-blocker", + "review-and-qa-terminal-cap-20" + ] | index($id)); + (.machine_expectations.required_substrings | type == "array" and length > 0) + and (.machine_expectations.forbidden_substrings | type == "array" and length > 0) + ) +' "$framework_fixture" >/dev/null; then + pass +else + fail "framework eval fixture missing harness controller cases or machine expectations" +fi + +test_start "framework fixture machine expectations name controller surfaces" +framework_fixture_failures=() +for case_and_term in \ + "harness-done-contract-recipe-before-build::Done Contract" \ + "harness-done-contract-recipe-before-build::Harness Recipe" \ + "harness-done-contract-recipe-before-build::Artifact Reference" \ + "code-reviewer-and-qa-evaluator-evidence-split::Code Reviewer" \ + "code-reviewer-and-qa-evaluator-evidence-split::QA Evaluator" \ + "code-reviewer-and-qa-evaluator-evidence-split::rubric_refs" \ + "pivot-restart-on-stagnation-or-code-writer-blocker::pivot_restart_decision" \ + "pivot-restart-on-stagnation-or-code-writer-blocker::legacy_code_bug" \ + "review-and-qa-terminal-cap-20::max 20" \ + "review-and-qa-terminal-cap-20::round 21"; do + case_id="${case_and_term%%::*}" + term="${case_and_term#*::}" + if ! jq -e --arg case_id "$case_id" --arg term "$term" ' + any(.cases[] | select(.id == $case_id) | .machine_expectations.required_substrings[]; . == $term) + or any(.cases[] | select(.id == $case_id) | .machine_expectations.forbidden_substrings[]; . == $term) + ' "$framework_fixture" >/dev/null; then + framework_fixture_failures+=("$case_id: $term") + fi +done +if [[ "${#framework_fixture_failures[@]}" -eq 0 ]]; then + pass +else + fail "framework fixture machine expectations missing controller terms: ${framework_fixture_failures[*]}" +fi + +test_start "skill-creator fixture covers loop-based Process harness patterns" +if jq -e ' + any(.cases[]; + .id == "loop-process-skill-applies-harness-patterns" + and .category == "harness_patterns" + and (.machine_expectations.required_substrings | contains([ + "references/harness-patterns.md", + "Done Contract", + "Harness Recipe", + "Artifact Reference", + "Code Reviewer", + "QA Evaluator", + "pivot_restart_decision", + "max 20" + ])) + and (.machine_expectations.forbidden_substrings | length > 0) + ) +' "$skill_creator_fixture" >/dev/null; then + pass +else + fail "assistant-skill-creator eval fixture missing loop Process harness-pattern case" +fi + +p0p4_finish_suite "${BASH_SOURCE[0]}" diff --git a/tests/p0-p4/installer-contracts.sh b/tests/p0-p4/installer-contracts.sh index 9282405..4ffb4b5 100644 --- a/tests/p0-p4/installer-contracts.sh +++ b/tests/p0-p4/installer-contracts.sh @@ -50,7 +50,12 @@ if HOME="$INSTALL_HOME" bash "$FRAMEWORK_DIR/install.sh" --agent codex --skill a && [[ "$agents_starts" == "1" && "$agents_ends" == "1" ]] \ && ! grep -Fq "$stale_generated_phrase" "$agents_file" \ && grep -Fq "Discovery context maps are owned by code-mapper for medium+ work" "$agents_file" \ - && grep -Fq "code-mapper, code-writer, builder-tester, reviewer" "$agents_file" \ + && grep -Fq "code-mapper, code-writer, builder-tester, architect, explorer, code-reviewer, reviewer" "$agents_file" \ + && grep -Fq "reviewer remains a compatibility route for existing handoffs" "$agents_file" \ + && grep -Fq "| code-reviewer | read-only | Canonical code/security/architecture review |" "$agents_file" \ + && grep -Fq "| reviewer | read-only | Compatibility route for existing review handoffs |" "$agents_file" \ + && [[ -f "$INSTALL_HOME/.codex/agents/code-reviewer.toml" ]] \ + && grep -Fq 'sandbox_mode = "read-only"' "$INSTALL_HOME/.codex/agents/code-reviewer.toml" \ && grep -Fq "The orchestrator may create and update framework-owned state artifacts" "$agents_file" \ && grep -Fq ".codex/context-map.md" "$agents_file" \ && grep -Fq "it does not edit project source files directly" "$agents_file" \ @@ -64,7 +69,7 @@ if HOME="$INSTALL_HOME" bash "$FRAMEWORK_DIR/install.sh" --agent codex --skill a && grep -Fq "preserve user custom sections below the installer block" "$agents_file"; then pass else - fail "expected one Codex framework block, one protocol block, current generated wording, and context budget guidance" + fail "expected one Codex framework block, one protocol block, current generated wording, code-reviewer compatibility guidance, and context budget guidance" fi else fail "second install failed; see /tmp/p0p4-install-2.err" @@ -73,6 +78,14 @@ else fail "first install failed; see /tmp/p0p4-install-1.err" fi +test_start "subagent monitor recognizes code-reviewer as read-only Reviewer compatibility" +if grep -Fq 'code-reviewer) role_constraint="SUBAGENT CONSTRAINT: You are a code reviewer. Read-only code/security/architecture/test-coverage review. Do NOT edit any files. Report findings only."' "$FRAMEWORK_DIR/hooks/scripts/subagent-monitor.sh" \ + && grep -Fq 'code-reviewer|reviewer) role_name="Reviewer"' "$FRAMEWORK_DIR/hooks/scripts/subagent-monitor.sh"; then + pass +else + fail "subagent-monitor.sh must reinforce code-reviewer read-only constraints and map it to Reviewer handoff contracts" +fi + test_start "Codex single-skill install generates AGENTS skill table from installed skills" INSTALL_HOME_SKILL_TABLE="$(mktemp -d)" p0p4_register_cleanup "$INSTALL_HOME_SKILL_TABLE" diff --git a/tests/p0-p4/qa-evaluator-contracts.sh b/tests/p0-p4/qa-evaluator-contracts.sh new file mode 100644 index 0000000..28d23d3 --- /dev/null +++ b/tests/p0-p4/qa-evaluator-contracts.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash + +if [[ -z "${P0P4_HARNESS_LOADED:-}" ]]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/p0p4-harness.sh" +fi +p0p4_bootstrap_suite "${BASH_SOURCE[0]}" + +require_terms() { + local label="$1" + local file="$2" + shift 2 + + local missing=() + local term + for term in "$@"; do + if ! grep -Fq -- "$term" "$file"; then + missing+=("${file#$FRAMEWORK_DIR/}: $term") + fi + done + + if [[ "${#missing[@]}" -eq 0 ]]; then + pass + else + fail "$label missing terms: ${missing[*]}" + fi +} + +test_start "QA evaluator agents exist and are read-only" +missing_agent_terms=() +for file in \ + "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml" \ + "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md"; do + if [[ ! -f "$file" ]]; then + missing_agent_terms+=("${file#$FRAMEWORK_DIR/}: exists") + fi +done +if [[ -f "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml" ]]; then + for term in \ + 'sandbox_mode = "read-only"' \ + "Done Contract" \ + "acceptance criteria" \ + "verification evidence" \ + "score progression" \ + "final acceptance result" \ + "Do NOT replace code-reviewer" \ + "Do NOT edit any files"; do + if ! grep -Fq -- "$term" "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml"; then + missing_agent_terms+=("agents/codex/qa-evaluator.toml: $term") + fi + done +fi +if [[ -f "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md" ]]; then + for term in \ + "tools: Read, Grep, Glob, LS" \ + "Done Contract" \ + "acceptance criteria" \ + "verification evidence" \ + "score progression" \ + "final acceptance result" \ + "Do NOT replace code-reviewer" \ + "Do NOT edit any files"; do + if ! grep -Fq -- "$term" "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md"; then + missing_agent_terms+=("agents/claude/qa-evaluator.md: $term") + fi + done + if grep -Eq '^tools: .*Edit|^tools: .*Write|^tools: .*Bash' "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md"; then + missing_agent_terms+=("agents/claude/qa-evaluator.md: unexpected write/shell tools") + fi +fi +if [[ "${#missing_agent_terms[@]}" -eq 0 ]]; then + pass +else + fail "QA evaluator agent prompts missing read-only role terms: ${missing_agent_terms[*]}" +fi + +test_start "subagent monitor maps qa-evaluator to QAEvaluator, not Reviewer" +monitor="$FRAMEWORK_DIR/hooks/scripts/subagent-monitor.sh" +if grep -Fq 'qa-evaluator) role_constraint="SUBAGENT CONSTRAINT: You are a QA evaluator. Read-only acceptance, Done Contract, verification evidence, domain quality, score progression, and final result evaluation. Do NOT edit any files. Do NOT replace code-reviewer."' "$monitor" \ + && grep -Fq 'qa-evaluator) role_name="QAEvaluator" ;;' "$monitor" \ + && grep -Fq 'code-reviewer|reviewer) role_name="Reviewer" ;;' "$monitor"; then + pass +else + fail "subagent-monitor.sh must constrain qa-evaluator read-only and map it to QAEvaluator separately from Reviewer" +fi + +test_start "Codex installer generated AGENTS text and table include qa-evaluator" +INSTALL_HOME_QA="$(mktemp -d)" +p0p4_register_cleanup "$INSTALL_HOME_QA" +if HOME="$INSTALL_HOME_QA" bash "$FRAMEWORK_DIR/install.sh" --agent codex --skill assistant-workflow --no-hooks >/tmp/p0p4-install-qa-evaluator.out 2>/tmp/p0p4-install-qa-evaluator.err; then + agents_file="$INSTALL_HOME_QA/.codex/AGENTS.md" + if grep -Fq "code-mapper, code-writer, builder-tester, architect, explorer, code-reviewer, reviewer, qa-evaluator" "$agents_file" \ + && grep -Fq "independent QA acceptance evaluation by qa-evaluator after build/test and code-review evidence when applicable" "$agents_file" \ + && grep -Fq "code-reviewer, qa-evaluator; reviewer remains compatibility routing" "$agents_file" \ + && grep -Fq "| qa-evaluator | read-only | Acceptance, Done Contract, and QA evaluation |" "$agents_file" \ + && [[ -f "$INSTALL_HOME_QA/.codex/agents/qa-evaluator.toml" ]] \ + && grep -Fq 'sandbox_mode = "read-only"' "$INSTALL_HOME_QA/.codex/agents/qa-evaluator.toml"; then + pass + else + fail "generated Codex AGENTS.md or installed agent missing qa-evaluator role text/table/read-only config" + fi +else + fail "Codex install failed; see /tmp/p0p4-install-qa-evaluator.err" +fi + +test_start "assistant-review declares independent QA evaluation loop and 20-round cap" +review_dir="$FRAMEWORK_DIR/skills/assistant-review" +missing_review_terms=() +for file_and_term in \ + "$review_dir/SKILL.md::optional independent QA evaluation loop" \ + "$review_dir/SKILL.md::QA evaluation runs after code-review/build evidence" \ + "$review_dir/SKILL.md::The QA loop supports rounds 1-20" \ + "$review_dir/SKILL.md::Round 20 is terminal" \ + "$review_dir/references/qa-evaluation-loop.md::while round <= 20" \ + "$review_dir/references/qa-evaluation-loop.md::Round 20 is terminal" \ + "$review_dir/references/qa-evaluation-loop.md::does not replace code-reviewer" \ + "$review_dir/contracts/handoffs.yaml::to: QAEvaluator" \ + "$review_dir/contracts/handoffs.yaml::- name: debate_record" \ + "$review_dir/contracts/handoffs.yaml::pre-build debate/subagent-perspective evidence" \ + "$review_dir/contracts/handoffs.yaml::previously_failed_acceptance_items" \ + "$review_dir/contracts/handoffs.yaml::qa_filter_policy" \ + "$review_dir/contracts/handoffs.yaml::debate_record when Done Contract exists" \ + "$review_dir/contracts/handoffs.yaml::qa_scorecard" \ + "$review_dir/contracts/handoffs.yaml::score_entry" \ + "$review_dir/contracts/handoffs.yaml::The loop never starts round 21." \ + "$review_dir/contracts/input.yaml::- name: qa_evaluation_mode" \ + "$review_dir/contracts/input.yaml::- name: debate_record" \ + "$review_dir/contracts/input.yaml::pre-build debate/subagent-perspective evidence" \ + "$review_dir/contracts/input.yaml::- name: qa_filter_policy" \ + "$review_dir/contracts/output.yaml::- name: qa_evaluation_result" \ + "$review_dir/contracts/output.yaml::final_verdict" \ + "$review_dir/contracts/output.yaml::score_progression" \ + "$review_dir/references/qa-evaluation-loop.md::debate_record" \ + "$review_dir/references/qa-evaluation-loop.md::pre-build debate/subagent-perspective evidence" \ + "$review_dir/contracts/phase-gates.yaml::QA_EVALUATION_STEP" \ + "$review_dir/contracts/phase-gates.yaml::QA evaluation starts only after build/test verification evidence and Code Reviewer or Reviewer compatibility result are available" \ + "$review_dir/contracts/phase-gates.yaml::INV_QA4" \ + "$review_dir/contracts/phase-gates.yaml::QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists" \ + "$review_dir/contracts/phase-gates.yaml::round 20 is terminal"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + missing_review_terms+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#missing_review_terms[@]}" -eq 0 ]]; then + pass +else + fail "assistant-review QA loop contract terms missing: ${missing_review_terms[*]}" +fi + +test_start "assistant-review QA Done Contract debate_record contract is mirrored" +debate_mirror_failures=() +for qa_review_dir in \ + "$FRAMEWORK_DIR/skills/assistant-review" \ + "$FRAMEWORK_DIR/plugins/assistant-dev/skills/assistant-review"; do + for file_and_term in \ + "$qa_review_dir/contracts/input.yaml::- name: debate_record" \ + "$qa_review_dir/contracts/input.yaml::pre-build debate/subagent-perspective evidence" \ + "$qa_review_dir/contracts/handoffs.yaml::- name: debate_record" \ + "$qa_review_dir/contracts/handoffs.yaml::pre-build debate/subagent-perspective evidence" \ + "$qa_review_dir/references/qa-evaluation-loop.md::debate_record" \ + "$qa_review_dir/contracts/phase-gates.yaml::INV_QA4" \ + "$qa_review_dir/contracts/phase-gates.yaml::QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + debate_mirror_failures+=("${file#$FRAMEWORK_DIR/}: $term") + fi + done +done +if [[ "${#debate_mirror_failures[@]}" -eq 0 ]]; then + pass +else + fail "assistant-review QA Done Contract debate_record contract terms missing or not mirrored: ${debate_mirror_failures[*]}" +fi + +test_start "workflow records separate Code Reviewer and QA Evaluator evidence" +workflow_dir="$FRAMEWORK_DIR/skills/assistant-workflow" +missing_workflow_terms=() +for file_and_term in \ + "$workflow_dir/contracts/handoffs.yaml::to: QAEvaluator" \ + "$workflow_dir/contracts/handoffs.yaml::Does not replace code-reviewer" \ + "$workflow_dir/contracts/input.yaml::- name: qa_evaluation_mode" \ + "$workflow_dir/contracts/output.yaml::- name: qa_evaluation_result" \ + "$workflow_dir/contracts/output.yaml::QA evaluation does not replace review_result" \ + "$workflow_dir/contracts/output.yaml::qa_evaluator_evidence" \ + "$workflow_dir/contracts/phase-gates.yaml::R_QA_EVALUATION" \ + "$workflow_dir/contracts/phase-gates.yaml::Code Reviewer or Reviewer compatibility evidence is recorded separately from QA Evaluator evidence" \ + "$workflow_dir/references/phases.md::Stage 3: QA Evaluation" \ + "$workflow_dir/references/phases.md::QA Evaluation result must exist when qa_evaluation_mode=required." \ + "$workflow_dir/references/subagent-dispatch.md::QA Evaluator" \ + "$workflow_dir/references/subagent-dispatch.md::QA evidence gate" \ + "$workflow_dir/references/subagent-roles.md::QA Evaluator dispatch" \ + "$workflow_dir/references/task-journal-template.md::QA Evaluator dispatch" \ + "$workflow_dir/references/task-journal-template.md::### QA Evaluation #1" \ + "$workflow_dir/references/sub-task-brief-template.md::qa-evaluator"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + missing_workflow_terms+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#missing_workflow_terms[@]}" -eq 0 ]]; then + pass +else + fail "workflow QA evaluator evidence surfaces missing: ${missing_workflow_terms[*]}" +fi + +test_start "workflow QA Done Contract debate_record contract is mirrored" +workflow_debate_failures=() +for workflow_qa_dir in \ + "$FRAMEWORK_DIR/skills/assistant-workflow" \ + "$FRAMEWORK_DIR/plugins/assistant-dev/skills/assistant-workflow"; do + for file_and_term in \ + "$workflow_qa_dir/contracts/handoffs.yaml::- name: orchestrator_to_qa_evaluator" \ + "$workflow_qa_dir/contracts/handoffs.yaml::- name: debate_record" \ + "$workflow_qa_dir/contracts/handoffs.yaml::pre-build debate/subagent-perspective evidence" \ + "$workflow_qa_dir/contracts/handoffs.yaml::debate_record with at least two perspectives" \ + "$workflow_qa_dir/contracts/handoffs.yaml::min_items: 2" \ + "$workflow_qa_dir/contracts/handoffs.yaml::At least two perspectives; delegated mode uses subagent perspectives when available"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + workflow_debate_failures+=("${file#$FRAMEWORK_DIR/}: $term") + fi + done +done +if [[ "${#workflow_debate_failures[@]}" -eq 0 ]]; then + pass +else + fail "workflow QA Done Contract debate_record contract terms missing or not mirrored: ${workflow_debate_failures[*]}" +fi + +test_start "code-reviewer remains distinct from QA evaluator" +distinct_terms_missing=() +for file_and_term in \ + "$FRAMEWORK_DIR/agents/codex/code-reviewer.toml::Stay in the code-review lane" \ + "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml::Stay in the QA lane" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/subagent-dispatch.md::Code Reviewer" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/subagent-dispatch.md::QA Evaluator" \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::Code Reviewer continues to own code defects, security, architecture, and test-coverage review" \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::QA findings do not replace code-review findings"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + distinct_terms_missing+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#distinct_terms_missing[@]}" -eq 0 ]]; then + pass +else + fail "code-reviewer and QA evaluator role separation terms missing: ${distinct_terms_missing[*]}" +fi + +p0p4_finish_suite "${BASH_SOURCE[0]}" diff --git a/tests/p0-p4/review-loop-cap-contracts.sh b/tests/p0-p4/review-loop-cap-contracts.sh new file mode 100755 index 0000000..5d2311e --- /dev/null +++ b/tests/p0-p4/review-loop-cap-contracts.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +if [[ -z "${P0P4_HARNESS_LOADED:-}" ]]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/p0p4-harness.sh" +fi +p0p4_bootstrap_suite "${BASH_SOURCE[0]}" + +active_review_cap_files=( + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md" + "$FRAMEWORK_DIR/skills/assistant-review/contracts/input.yaml" + "$FRAMEWORK_DIR/skills/assistant-review/contracts/output.yaml" + "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml" + "$FRAMEWORK_DIR/skills/assistant-review/contracts/phase-gates.yaml" + "$FRAMEWORK_DIR/skills/assistant-review/references/score-tracking.md" + "$FRAMEWORK_DIR/skills/assistant-review/references/review-rubric.md" + "$FRAMEWORK_DIR/skills/assistant-review/evals/cases.json" + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml" + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml" + "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md" + "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md" + "$FRAMEWORK_DIR/skills/assistant-workflow/references/prompts/pr-review.md" + "$FRAMEWORK_DIR/skills/assistant-workflow/references/subagent-roles.md" + "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh" + "$FRAMEWORK_DIR/hooks/scripts/workflow-enforcer.sh" + "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh" + "$FRAMEWORK_DIR/install.sh" + "$FRAMEWORK_DIR/agents/codex/code-reviewer.toml" + "$FRAMEWORK_DIR/agents/codex/reviewer.toml" + "$FRAMEWORK_DIR/agents/claude/code-reviewer.md" + "$FRAMEWORK_DIR/agents/claude/reviewer.md" + "$FRAMEWORK_DIR/docs/harness-design-guide.md" +) +# Historical/generated presentation artifacts are intentionally excluded from +# this runtime cap guard; S9 owns deeper docs/presentation refresh. + +test_start "active review-loop surfaces reject stale 10-round limits" +stale_cap_output="$(mktemp)" +p0p4_register_cleanup "$stale_cap_output" +if rg -n -e 'max 10' \ + -e '<= ?10' \ + -e 'round <= 10' \ + -e 'Round: N of 10' \ + -e 'Round [0-9]+ of 10' \ + -e 'Round 10 is terminal' \ + -e 'round 10 is terminal' \ + -e 'round 11' \ + -e 'N between 1 and 10' \ + "${active_review_cap_files[@]}" >"$stale_cap_output"; then + fail "found stale review-loop cap text in active surfaces; see $stale_cap_output" +else + pass +fi + +test_start "assistant-review contracts and loop use 20-round cap" +missing_review_cap_terms=() +for file_and_term in \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::max 20 rounds" \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::while round <= 20:" \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::round 20 is terminal" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/input.yaml::validation: \">= 1 and <= 20\"" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/output.yaml::rounds (1-20)" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/output.yaml::capped at the 20-round review limit" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml::Current review round number (1-20)" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml::round 20 remains terminal" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml::The loop never starts round 21." \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/phase-gates.yaml::EXIT_MAX_ROUNDS = round 20" \ + "$FRAMEWORK_DIR/skills/assistant-review/references/score-tracking.md::Round: N of 20" \ + "$FRAMEWORK_DIR/skills/assistant-review/references/review-rubric.md::| 4-20 | 4.0+ | 3.25" \ + "$FRAMEWORK_DIR/skills/assistant-review/evals/cases.json::max 20 rounds" \ + "$FRAMEWORK_DIR/skills/assistant-review/evals/cases.json::rounds 1-20"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq "$term" "$file"; then + missing_review_cap_terms+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#missing_review_cap_terms[@]}" -eq 0 ]]; then + pass +else + fail "assistant-review 20-round cap terms missing: ${missing_review_cap_terms[*]}" +fi + +test_start "workflow hooks installer and prompts use 20-round review cap" +missing_runtime_cap_terms=() +for file_and_term in \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::validation: \">= 1 and <= 20\"" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::Current review round number (1-20)" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::Round 16-20: 90" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md::max 20 rounds" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::Round: 1 of 20" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::Round: 2 of 20" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/prompts/pr-review.md::up to 20 rounds" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/prompts/pr-review.md::Round N of 20" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/subagent-roles.md::Round 16-20: 90%+" \ + "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh::Round: N of 20" \ + "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh::N between 1 and 20" \ + "$FRAMEWORK_DIR/hooks/scripts/workflow-enforcer.sh::max 20 rounds" \ + "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh::max_round\" -ne 20" \ + "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh::round\" -gt 20" \ + "$FRAMEWORK_DIR/install.sh::Autonomous review-fix loop (max 20 rounds)" \ + "$FRAMEWORK_DIR/install.sh::clean (max 20 rounds)" \ + "$FRAMEWORK_DIR/docs/harness-design-guide.md::while round <= 20:" \ + "$FRAMEWORK_DIR/docs/harness-design-guide.md::| 20 | Terminal report"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq "$term" "$file"; then + missing_runtime_cap_terms+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#missing_runtime_cap_terms[@]}" -eq 0 ]]; then + pass +else + fail "workflow/runtime 20-round cap terms missing: ${missing_runtime_cap_terms[*]}" +fi + +test_start "code-reviewer and reviewer compatibility prompts preserve role split with 20-round terminal guidance" +missing_reviewer_prompt_terms=() +for file in \ + "$FRAMEWORK_DIR/agents/codex/code-reviewer.toml" \ + "$FRAMEWORK_DIR/agents/codex/reviewer.toml" \ + "$FRAMEWORK_DIR/agents/claude/code-reviewer.md" \ + "$FRAMEWORK_DIR/agents/claude/reviewer.md"; do + for term in \ + "In rounds 16-20" \ + "Round 20 is terminal" \ + "round 21"; do + if ! grep -Fq "$term" "$file"; then + missing_reviewer_prompt_terms+=("${file#$FRAMEWORK_DIR/}: $term") + fi + done +done +if [[ "${#missing_reviewer_prompt_terms[@]}" -eq 0 ]]; then + pass +else + fail "reviewer prompt 20-round compatibility terms missing: ${missing_reviewer_prompt_terms[*]}" +fi + +p0p4_finish_suite "${BASH_SOURCE[0]}" diff --git a/tests/p0-p4/runtime-phase-gate-contracts.sh b/tests/p0-p4/runtime-phase-gate-contracts.sh index 32a7eef..ec66e21 100644 --- a/tests/p0-p4/runtime-phase-gate-contracts.sh +++ b/tests/p0-p4/runtime-phase-gate-contracts.sh @@ -81,6 +81,97 @@ else fail "workflow-enforcer.sh missing runtime gate terms: ${missing_workflow_gate_terms[*]}" fi +test_start "workflow declares run-state trace replay harness artifacts" +missing_runtime_artifact_terms=() +workflow_dir="$FRAMEWORK_DIR/skills/assistant-workflow" +skill_file="$workflow_dir/SKILL.md" +output_contract="$workflow_dir/contracts/output.yaml" +phase_gates="$workflow_dir/contracts/phase-gates.yaml" +harness_ref="$workflow_dir/references/harness-controller.md" +phases_ref="$workflow_dir/references/phases.md" +task_journal_template="$workflow_dir/references/task-journal-template.md" +plan_template="$workflow_dir/references/plan-template.md" +for term in \ + "- name: harness_run_state" \ + "task_id" \ + "task_name" \ + "phase" \ + "slice" \ + "status" \ + "blockers" \ + "last_verification" \ + "next_action" \ + "recovery_pointer" \ + "- name: trace_ledger" \ + "timestamped/ordered agent events" \ + "verification commands/results" \ + "plan deviations" \ + "artifact refs" \ + "- name: replay_packet" \ + "pinned_context" \ + "validation_state" \ + "exact_next_action"; do + if ! grep -Fq -- "$term" "$output_contract"; then + missing_runtime_artifact_terms+=("output.yaml: $term") + fi +done +if ! grep -Fq -- "Trace/replay-ready harness work maintains Harness Run State, Trace Ledger, and Replay Packet artifacts." "$skill_file"; then + missing_runtime_artifact_terms+=("SKILL.md: Trace/replay-ready harness work maintains Harness Run State, Trace Ledger, and Replay Packet artifacts.") +fi +for term in \ + "- id: P_HARNESS_RUNTIME_ARTIFACTS" \ + "- id: B_HARNESS_RUN_STATE_TRACE_REPLAY" \ + "- id: DOC_HARNESS_REPLAY_PACKET" \ + "record the corrective action for missing run-state/trace/replay evidence"; do + if ! grep -Fq -- "$term" "$phase_gates"; then + missing_runtime_artifact_terms+=("phase-gates.yaml: $term") + fi +done +for term in \ + "## Harness Run State" \ + "## Trace Ledger" \ + "## Replay Packet" \ + "Missing run-state/trace/replay evidence"; do + if ! grep -Fq -- "$term" "$harness_ref"; then + missing_runtime_artifact_terms+=("harness-controller.md: $term") + fi +done +for term in \ + "## Harness Run State" \ + "## Trace Ledger" \ + "## Replay Packet" \ + "task_id" \ + "task_name" \ + "last_verification" \ + "exact_next_action"; do + if ! grep -Fq -- "$term" "$task_journal_template"; then + missing_runtime_artifact_terms+=("task-journal-template.md: $term") + fi +done +for term in \ + "harness_run_state_ref" \ + "trace_ledger_ref" \ + "replay_packet_ref" \ + "Runtime Harness Artifacts"; do + if ! grep -Fq -- "$term" "$plan_template"; then + missing_runtime_artifact_terms+=("plan-template.md: $term") + fi +done +for term in \ + "Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger" \ + "Update Harness Run State after each slice/step" \ + "Append Trace Ledger entries" \ + "Refresh the Replay Packet"; do + if ! grep -Fq -- "$term" "$phases_ref"; then + missing_runtime_artifact_terms+=("phases.md: $term") + fi +done +if [[ "${#missing_runtime_artifact_terms[@]}" -eq 0 ]]; then + pass +else + fail "runtime harness artifact terms missing: ${missing_runtime_artifact_terms[*]}" +fi + test_start "workflow instructions do not require separate Decompose approval" if rg -n "Component decomposition approval required|Slice decomposition approval required|User explicitly approved the component decomposition|User explicitly approved the slice decomposition|without approved component decomposition|without approved slice decomposition|DECOMPOSE COMPLETE \(approved\)" \ "$FRAMEWORK_DIR/skills/assistant-workflow" \ diff --git a/tests/p0-p4/task-packet-contracts.sh b/tests/p0-p4/task-packet-contracts.sh index 01fb700..bbb22c6 100644 --- a/tests/p0-p4/task-packet-contracts.sh +++ b/tests/p0-p4/task-packet-contracts.sh @@ -691,7 +691,8 @@ workflow_subagent_gate_terms=( "skills/assistant-workflow/contracts/input.yaml|subagent_execution_mode" "skills/assistant-workflow/contracts/input.yaml|direct_fallback" "skills/assistant-workflow/contracts/input.yaml|not_applicable is invalid for Build" - "skills/assistant-workflow/contracts/input.yaml|Code Writer, Builder/Tester, and Reviewer" + "skills/assistant-workflow/contracts/input.yaml|Code Writer, Builder/Tester, and Code Reviewer" + "skills/assistant-workflow/contracts/input.yaml|Reviewer may satisfy only compatibility routing" "skills/assistant-workflow/contracts/input.yaml|subagent_authorization_scope" "skills/assistant-workflow/contracts/output.yaml|subagent_policy_state" "skills/assistant-workflow/contracts/output.yaml|subagent_execution_mode" @@ -727,7 +728,8 @@ workflow_subagent_gate_terms=( "skills/assistant-workflow/references/task-journal-template.md|Direct fallback reason" "skills/assistant-workflow/references/phases.md|Resolve \`subagent_policy_state\`, \`subagent_execution_mode\`, and \`subagent_authorization_scope\` before spawning any subagent" "skills/assistant-workflow/references/phases.md|add Code Mapper to \`Required agents\`" - "skills/assistant-workflow/references/phases.md|Add Reviewer to \`Required agents\` before Stage 2" + "skills/assistant-workflow/references/phases.md|Add Code Reviewer to \`Required agents\` before Stage 2" + "skills/assistant-workflow/references/phases.md|\`Reviewer\` only as compatibility routing" "skills/assistant-workflow/references/phases.md|subagent_execution_mode=delegated" "skills/assistant-workflow/references/phases.md|Direct fallback mode" "skills/assistant-workflow/references/phases.md|Silent fallback cannot complete" @@ -741,6 +743,12 @@ for pair in "${workflow_subagent_gate_terms[@]}"; do fi done done +if grep -Fq -- "Add Reviewer to \`Required agents\` before Stage 2" "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md"; then + missing_workflow_subagent_gate+=("skills/assistant-workflow/references/phases.md: stale bare Reviewer Stage 2 required-agent wording") +fi +if grep -Fq -- "Add Reviewer to \`Required agents\` before Stage 2" "$FRAMEWORK_DIR/plugins/assistant-dev/skills/assistant-workflow/references/phases.md"; then + missing_workflow_subagent_gate+=("plugins/assistant-dev/skills/assistant-workflow/references/phases.md: stale bare Reviewer Stage 2 required-agent wording") +fi if [[ "${#missing_workflow_subagent_gate[@]}" -eq 0 ]]; then pass else @@ -2035,6 +2043,114 @@ else pass fi +test_start "workflow pivot restart controller records decision packets and recovery refs" +missing_pivot_restart_terms=() +for file_and_term in \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::- name: pivot_restart_decision" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::trigger" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::affected_slice_or_round" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::options_considered" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::selected_action" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::reapproval_required" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::next_agent" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::recovery_pointer" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::exact_next_action" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::pivot_restart_decision_ref" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::pivot_restart_decision" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/phase-gates.yaml::B_CODE_WRITER_BLOCKER_ROUTING" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/phase-gates.yaml::R_PIVOT_RESTART_DECISION" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/phase-gates.yaml::INV_PIVOT_RESTART_REAPPROVAL" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/harness-controller.md::## Pivot/Restart Controller" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/harness-controller.md::Round 20 remains terminal" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md::Code Writer unexpected blockers" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md::STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::## Pivot/Restart Log" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::pivot_restart_decision"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + missing_pivot_restart_terms+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#missing_pivot_restart_terms[@]}" -eq 0 ]]; then + pass +else + fail "workflow pivot/restart controller missing decision packet terms: ${missing_pivot_restart_terms[*]}" +fi + +test_start "workflow CodeWriter unexpected blockers classify and route recovery" +missing_codewriter_blocker_terms=() +for file_and_term in \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::blocker_type" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::blocker_evidence" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::legacy_code_bug" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::broken_baseline" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::hidden_dependency" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::missing_contract" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::stale_plan" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::scope_conflict" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::tool_environment" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::tdd_red_missing" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md::debugging, explorer, architect, candidate_search, replan, restart" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/sub-task-brief-template.md::Unexpected blockers must be classified" \ + "$FRAMEWORK_DIR/agents/codex/code-writer.toml::Unexpected blocker protocol" \ + "$FRAMEWORK_DIR/agents/codex/code-writer.toml::Do not widen scope" \ + "$FRAMEWORK_DIR/agents/claude/code-writer.md::Unexpected blocker protocol" \ + "$FRAMEWORK_DIR/agents/claude/code-writer.md::Do not widen scope"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + missing_codewriter_blocker_terms+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#missing_codewriter_blocker_terms[@]}" -eq 0 ]]; then + pass +else + fail "CodeWriter blocker classification and orchestrator routing terms missing: ${missing_codewriter_blocker_terms[*]}" +fi + +test_start "assistant-review stagnation and QA pivot escalate to pivot restart decision" +missing_review_pivot_terms=() +for file_and_term in \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::pivot_restart_signal" \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::orchestrator records pivot_restart_decision" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml::pivot_restart_signal" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/output.yaml::- name: pivot_restart_decision" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/phase-gates.yaml::ESCALATE_PIVOT_RESTART" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/phase-gates.yaml::QA STAGNATION, repeated DRIFT, repeated REGRESSION, or scoped domain action pivot" \ + "$FRAMEWORK_DIR/skills/assistant-review/references/score-tracking.md::Pivot/Restart Decision Packet" \ + "$FRAMEWORK_DIR/skills/assistant-review/references/score-tracking.md::repeated DRIFT triggers pivot_restart_decision" \ + "$FRAMEWORK_DIR/skills/assistant-review/references/qa-evaluation-loop.md::Pivot/Restart Escalation" \ + "$FRAMEWORK_DIR/skills/assistant-review/references/qa-evaluation-loop.md::QA does not silently continue"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + missing_review_pivot_terms+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#missing_review_pivot_terms[@]}" -eq 0 ]]; then + pass +else + fail "assistant-review pivot/restart stagnation escalation terms missing: ${missing_review_pivot_terms[*]}" +fi + +test_start "assistant-review removes stale loose stagnation and drift exits" +stale_pivot_restart_terms=() +for file_and_term in \ + "$FRAMEWORK_DIR/skills/assistant-review/references/score-tracking.md::May need to PIVOT or accept current state with documented limitations" \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::On 3+ DRIFT occurrences: stop the loop and present findings for manual review"; do + file="${file_and_term%%::*}" + term="${file_and_term#*::}" + if [[ -f "$file" ]] && grep -Fq -- "$term" "$file"; then + stale_pivot_restart_terms+=("${file#$FRAMEWORK_DIR/}: $term") + fi +done +if [[ "${#stale_pivot_restart_terms[@]}" -eq 0 ]]; then + pass +else + fail "stale loose stagnation/drift exits remain: ${stale_pivot_restart_terms[*]}" +fi + test_start "workflow decompose surfaces reject broad layer module folder strategies" missing_slice_rejection_terms=() for term in \ diff --git a/tests/p0-p4/worker-prompt-contracts.sh b/tests/p0-p4/worker-prompt-contracts.sh index e62e0b6..858299d 100644 --- a/tests/p0-p4/worker-prompt-contracts.sh +++ b/tests/p0-p4/worker-prompt-contracts.sh @@ -51,6 +51,39 @@ else fail "Claude/Codex prompts missing worker status packet terms: ${missing_prompt_packet_terms[*]}" fi +test_start "Code Reviewer prompts are canonical read-only code review role" +missing_code_reviewer_terms=() +for file in \ + agents/codex/code-reviewer.toml \ + agents/claude/code-reviewer.md; do + for term in \ + "canonical code reviewer" \ + "code defects, security, architecture, test coverage, and structural code issues" \ + "Start with a status packet:" \ + '`status`: `DONE`, `DONE_WITH_CONCERNS`, `NEEDS_CONTEXT`, or `BLOCKED`' \ + '`evidence`: review material, files, searches, or checks supporting the verdict' \ + "Do NOT edit any files" \ + "Stay in the code-review lane"; do + if ! grep -Fq -- "$term" "$FRAMEWORK_DIR/$file"; then + missing_code_reviewer_terms+=("$file: $term") + fi + done +done +if ! grep -Fq 'sandbox_mode = "read-only"' "$FRAMEWORK_DIR/agents/codex/code-reviewer.toml"; then + missing_code_reviewer_terms+=("agents/codex/code-reviewer.toml: read-only sandbox") +fi +if ! grep -Fq 'tools: Read, Grep, Glob, LS' "$FRAMEWORK_DIR/agents/claude/code-reviewer.md"; then + missing_code_reviewer_terms+=("agents/claude/code-reviewer.md: read-only tools") +fi +if grep -Eq '^tools: .*Edit|^tools: .*Write|^tools: .*Bash' "$FRAMEWORK_DIR/agents/claude/code-reviewer.md"; then + missing_code_reviewer_terms+=("agents/claude/code-reviewer.md: unexpected write/shell tools") +fi +if [[ "${#missing_code_reviewer_terms[@]}" -eq 0 ]]; then + pass +else + fail "Code Reviewer prompts missing canonical read-only role terms: ${missing_code_reviewer_terms[*]}" +fi + test_start "Codex mapper explorer architect prompts include status packet guidance" missing_codex_discovery_status_terms=() for file in \ diff --git a/tests/p0-p4/worker-status-contracts.sh b/tests/p0-p4/worker-status-contracts.sh index 50dca1a..f9b2b4d 100644 --- a/tests/p0-p4/worker-status-contracts.sh +++ b/tests/p0-p4/worker-status-contracts.sh @@ -40,6 +40,86 @@ else fail "workflow handoffs missing worker status protocol terms: ${missing_worker_status_terms[*]}" fi +test_start "workflow handoffs define typed artifact references for worker packets" +workflow_dir="$FRAMEWORK_DIR/skills/assistant-workflow" +output_contract="$workflow_dir/contracts/output.yaml" +plan_template="$workflow_dir/references/plan-template.md" +task_journal_template="$workflow_dir/references/task-journal-template.md" +phases_ref="$workflow_dir/references/phases.md" +harness_ref="$workflow_dir/references/harness-controller.md" +missing_typed_artifact_terms=() +for term in \ + "artifact_reference_protocol:" \ + "required_fields: [artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, summary]" \ + "artifact_types: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result]" \ + "location_ref is the typed location/ref pointer" \ + "Producer responsibility: create or update the artifact" \ + "Consumer responsibility: validate schema_or_contract and validation_status before relying on location_ref" \ + "CodeWriter and BuilderTester task packets receive artifact_refs for Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, Pivot/Restart Decision, changed files, verification evidence, and plan deviation refs when applicable." \ + "CodeWriter returns produced artifact_refs for changed files and plan deviation refs when applicable." \ + "BuilderTester returns validated artifact_refs for verification evidence and runtime-artifact validation." \ + "- name: artifact_refs" \ + "schema_or_contract" \ + "validation_status"; do + if ! grep -Fq -- "$term" "$handoffs_file"; then + missing_typed_artifact_terms+=("handoffs.yaml: $term") + fi +done +for term in \ + "- name: artifact_reference_ledger" \ + "artifact_id" \ + "artifact_type" \ + "producer" \ + "consumer" \ + "location_ref" \ + "schema_or_contract" \ + "validation_status" \ + "ledger covers Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, Pivot/Restart Decision, changed files, verification evidence, and plan deviation refs when applicable." \ + "enum_values: [done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, changed_files, verification_evidence, plan_deviation, task_packet, context_map, test_result, review_result, qa_evaluation_result]" \ + "- name: pivot_restart_decision"; do + if ! grep -Fq -- "$term" "$output_contract"; then + missing_typed_artifact_terms+=("output.yaml: $term") + fi +done +for term in \ + "## Artifact Reference Ledger" \ + "Typed artifact refs:" \ + "Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary"; do + if ! grep -Fq -- "$term" "$plan_template"; then + missing_typed_artifact_terms+=("plan-template.md: $term") + fi +done +for term in \ + "## Artifact Reference Ledger" \ + "Producer roles update Artifact Reference Ledger entries" \ + 'Consumer roles validate `schema_or_contract` and update `validation_status`'; do + if ! grep -Fq -- "$term" "$task_journal_template"; then + missing_typed_artifact_terms+=("task-journal-template.md: $term") + fi +done +for term in \ + "Artifact Reference Ledger before task packets" \ + 'typed `artifact_refs`' \ + "changed_files, verification_evidence, pivot_restart_decision, and plan_deviation refs"; do + if ! grep -Fq -- "$term" "$phases_ref"; then + missing_typed_artifact_terms+=("phases.md: $term") + fi +done +for term in \ + "## Typed Artifact References" \ + "Producer responsibility: create or update the artifact" \ + 'Consumer responsibility: validate `schema_or_contract` and' \ + "Done Contract, Harness Recipe, Harness Run State, Trace"; do + if ! grep -Fq -- "$term" "$harness_ref"; then + missing_typed_artifact_terms+=("harness-controller.md: $term") + fi +done +if [[ "${#missing_typed_artifact_terms[@]}" -eq 0 ]]; then + pass +else + fail "workflow typed artifact reference terms missing: ${missing_typed_artifact_terms[*]}" +fi + test_start "CodeMapper and Explorer return schemas require status evidence" missing_discovery_schema_terms=() for handoff in orchestrator_to_code_mapper orchestrator_to_explorer; do @@ -117,8 +197,8 @@ if ! grep -Fq -- "enum_values: [DONE, DONE_WITH_CONCERNS, NEEDS_CONTEXT, BLOCKED fi for term in \ "changed_files and files_changed are required with at least one item for CodeWriter returns with DONE, DONE_WITH_CONCERNS, or DEVIATED" \ - "For DONE, DONE_WITH_CONCERNS, or DEVIATED, it must also return changed_files, evidence, and files_changed with real implementation evidence" \ - "for NEEDS_CONTEXT/BLOCKED, require open_questions and do not require fabricated changed-file entries"; do + "For DONE, DONE_WITH_CONCERNS, or DEVIATED, it must also return changed_files, evidence, files_changed, and typed artifact_refs when the task packet carried artifact_refs or harness_capable == true" \ + "for NEEDS_CONTEXT/BLOCKED, require open_questions and do not require fabricated changed-file or artifact-ref entries"; do if ! grep -Fq -- "$term" "$handoffs_file"; then missing_code_writer_status_terms+=("$term") fi diff --git a/tests/test-hooks.sh b/tests/test-hooks.sh index dbce175..e241cad 100755 --- a/tests/test-hooks.sh +++ b/tests/test-hooks.sh @@ -136,6 +136,62 @@ learning_controller_reason() { ' _ "$HOOKS_DIR/workflow-phase-gates.sh" "$task_file" } +review_gate_reason() { + local task_file="$1" + bash -c ' + . "$1" + assistant_phase_review_missing_reason_key "$2" + ' _ "$HOOKS_DIR/workflow-phase-gates.sh" "$task_file" +} + +subagent_evidence_reason() { + local task_file="$1" + bash -c ' + . "$1" + assistant_phase_subagent_evidence_missing_reason_key "$2" + ' _ "$HOOKS_DIR/workflow-phase-gates.sh" "$task_file" +} + +write_required_qa_review_task() { + local task_file="$1" + local qa_evidence="$2" + mkdir -p "$(dirname "$task_file")" + cat > "$task_file" < "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: DONE_WITH_CONCERNS review completed with recorded follow-up risk +- QA Evaluator dispatch: multi_agent id=qa-1 +- QA Evaluator result: PASS clean final_verdict accepted with score_progression 4.00 +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "complete" ]]; then + pass + else + fail "expected complete, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: required QA accepted compact final verdict allows review"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" \ + "- QA Evaluator result: PASS clean final_verdict accepted with score_progression 4.00" + helper_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "complete" ]]; then + pass + else + fail "expected complete, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: required QA accepted_with_concerns final verdict allows review"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" $'- QA Evaluator result: completed with concerns; see QA Evaluation #1\n### QA Evaluation #1\n- Final verdict: accepted_with_concerns\n- QA result: accepted_with_concerns' + helper_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "complete" ]]; then + pass + else + fail "expected complete, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: required QA final verdict option list blocks review"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" $'- QA Evaluator result: completed; see QA Evaluation #1\n### QA Evaluation #1\n- Final verdict: accepted | accepted_with_concerns | rejected | blocked' + helper_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "qa_final_result_missing" ]]; then + pass + else + fail "expected qa_final_result_missing, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: required QA result bracket option list blocks review"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" $'- QA Evaluator result: completed; see final result\n- QA result: [accepted | accepted_with_concerns | rejected | blocked | not_required]' + helper_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "qa_final_result_missing" ]]; then + pass + else + fail "expected qa_final_result_missing, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: QA not_required mode with template labels allows review"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Plan approval: yes +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required gates: +- separate QA Evaluator loop +Required agents: +- Code Reviewer +qa_evaluation_mode: not_required +## Agent Dispatch Log +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: PASS clean +- QA Evaluator dispatch: [multi_agent id=... or N/A only when qa_evaluation_mode=not_required] +- QA Evaluator result: [accepted or N/A only when qa_evaluation_mode=not_required] +## Review Log +### Spec Review #1 +- Result: PASS +- Scope reviewed: approved plan and changed files +- Missing acceptance criteria: none +- Extra scope: none +- Changed files mismatch: none +- Verification evidence mismatch: none +- Required fixes: none +### Quality Review #1 +- Round: 1 of 20 +- Found this round: 0 must-fix, 0 should-fix, 0 nits +- Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 +- Weighted: 4.00 +### Final result +- Result: CLEAN +- Score progression: 4.00 +TASK + review_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + subagent_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$review_reason" == "complete" && "$subagent_reason" == "complete" ]]; then + pass + else + fail "expected complete review/subagent reasons, got review='$review_reason' subagent='$subagent_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: required QA rejected final verdict blocks review"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" $'- QA Evaluator result: rejected; see QA Evaluation #1\n### QA Evaluation #1\n- Final verdict: rejected\n- QA result: rejected' + helper_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "qa_rejected" ]]; then + pass + else + fail "expected qa_rejected, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: required QA not accepted variants block review"; then + for qa_evidence in \ + $'- QA Evaluator result: completed; see QA Evaluation #1\n### QA Evaluation #1\n- Final verdict: not accepted' \ + $'- QA Evaluator result: completed; see QA Evaluation #1\n### QA Evaluation #1\n- QA result: not_accepted' \ + $'- QA Evaluator result: completed; see QA Evaluation #1\n### QA Evaluation #1\n- QA result: not-accepted'; do + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" "$qa_evidence" + helper_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" != "qa_not_accepted" ]]; then + fail "expected qa_not_accepted, got '$helper_reason' for evidence '$qa_evidence'" + break + fi + done + if [[ "$helper_reason" == "qa_not_accepted" ]]; then + pass + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: required QA blocked result blocks review"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" $'- QA Evaluator result: blocked; see QA Evaluation #1\n### QA Evaluation #1\n- QA result: blocked' + helper_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "qa_blocked" ]]; then + pass + else + fail "expected qa_blocked, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: required QA missing final verdict blocks review"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" $'- QA Evaluator result: completed; see QA Evaluation #1\n### QA Evaluation #1\n- Scope checked: runtime review gate' + helper_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "qa_final_result_missing" ]]; then + pass + else + fail "expected qa_final_result_missing, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: stale accepted QA before current review cycle blocks review"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Plan approval: yes +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: PASS clean +- QA Evaluator dispatch: multi_agent id=qa-1 +- QA Evaluator result: PASS clean final_verdict accepted with score_progression 4.00 +### QA Evaluation #1 +- Final verdict: accepted +- QA result: accepted +## Review Log +### Spec Review #1 +- Result: PASS +- Scope reviewed: approved plan and changed files +- Missing acceptance criteria: none +- Extra scope: none +- Changed files mismatch: none +- Verification evidence mismatch: none +- Required fixes: none +### Quality Review #1 +- Round: 1 of 20 +- Found this round: 0 must-fix, 0 should-fix, 0 nits +- Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 +- Weighted: 4.00 +### Final result +- Result: CLEAN +- Score progression: 4.00 +- QA Evaluator result: completed; see QA Evaluation #2 +### QA Evaluation #2 +- Scope checked: runtime review gate +TASK + helper_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "qa_final_result_missing" ]]; then + pass + else + fail "expected qa_final_result_missing, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: pending Code Reviewer dispatch blocks delegated_missing_code_reviewer"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +## Agent Dispatch Log +- Code Reviewer dispatch: pending code-review evidence +- Code Reviewer result: PASS clean +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_code_reviewer" ]]; then + pass + else + fail "expected delegated_missing_code_reviewer, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: not_required Code Reviewer dispatch blocks delegated_missing_code_reviewer"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +## Agent Dispatch Log +- Code Reviewer dispatch: not_required +- Code Reviewer result: PASS clean +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_code_reviewer" ]]; then + pass + else + fail "expected delegated_missing_code_reviewer, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: pending Code Reviewer result blocks delegated_missing_code_reviewer"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +## Agent Dispatch Log +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: not yet available +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_code_reviewer" ]]; then + pass + else + fail "expected delegated_missing_code_reviewer, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: not required Code Reviewer result blocks delegated_missing_code_reviewer"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +## Agent Dispatch Log +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: not required +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_code_reviewer" ]]; then + pass + else + fail "expected delegated_missing_code_reviewer, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: pending QA Evaluator dispatch blocks delegated_missing_qa_evaluator"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: PASS clean +- QA Evaluator dispatch: waiting for QA evaluator +- QA Evaluator result: PASS final_verdict accepted with score_progression 4.00 +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then + pass + else + fail "expected delegated_missing_qa_evaluator, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: not_required QA Evaluator dispatch blocks delegated_missing_qa_evaluator when required"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: PASS clean +- QA Evaluator dispatch: not_required +- QA Evaluator result: PASS final_verdict accepted with score_progression 4.00 +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then + pass + else + fail "expected delegated_missing_qa_evaluator, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: pending QA Evaluator result blocks delegated_missing_qa_evaluator"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: PASS clean +- QA Evaluator dispatch: multi_agent id=qa-1 +- QA Evaluator result: in_progress +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then + pass + else + fail "expected delegated_missing_qa_evaluator, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: not required QA Evaluator result blocks delegated_missing_qa_evaluator when required"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: PASS clean +- QA Evaluator dispatch: multi_agent id=qa-1 +- QA Evaluator result: not required +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then + pass + else + fail "expected delegated_missing_qa_evaluator, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: default review phase missing Code Reviewer evidence blocks"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +## Agent Dispatch Log +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_code_reviewer" ]]; then + pass + else + fail "expected delegated_missing_code_reviewer, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: missing QA Evaluator evidence blocks when required"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required gates: +- separate qa-evaluator loop +Required agents: +- Code Reviewer +## Agent Dispatch Log +- Code Reviewer dispatch: code-reviewer round 1 +- Code Reviewer result: PASS clean +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then + pass + else + fail "expected delegated_missing_qa_evaluator, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: legacy Reviewer evidence passes as Code Reviewer compatibility"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Reviewer +## Agent Dispatch Log +- Reviewer dispatch: compatibility reviewer round 1 +- Reviewer result: PASS clean +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "complete" ]]; then + pass + else + fail "expected complete, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + # ── session-start.sh tests ──────────────────────────────────────────────────── echo "session-start.sh" @@ -744,6 +1288,70 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi +if test_start "stop-review: Claude, required QA rejected final verdict → blocks with JSON"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" $'- QA Evaluator result: rejected; see QA Evaluation #1\n### QA Evaluation #1\n- Final verdict: rejected\n- QA result: rejected' + HOME="$TEST_AGENT_HOME" run_hook stop-review.sh claude + if [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "qa_rejected" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "QA Evaluation evidence" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "accepted or accepted_with_concerns"; then + pass + else + fail "exit=$HOOK_EXIT, expected QA rejected block JSON; stdout='$HOOK_STDOUT'; stderr='$HOOK_STDERR'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "stop-review: Claude, required QA blocked result → blocks with JSON"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" $'- QA Evaluator result: blocked; see QA Evaluation #1\n### QA Evaluation #1\n- QA result: blocked' + HOME="$TEST_AGENT_HOME" run_hook stop-review.sh claude + if [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "qa_blocked" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "QA Evaluation evidence" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "accepted or accepted_with_concerns"; then + pass + else + fail "exit=$HOOK_EXIT, expected QA blocked block JSON; stdout='$HOOK_STDOUT'; stderr='$HOOK_STDERR'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "stop-review: Claude, required QA missing final result → blocks with JSON"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" $'- QA Evaluator result: completed; see QA Evaluation #1\n### QA Evaluation #1\n- Scope checked: runtime review gate' + HOME="$TEST_AGENT_HOME" run_hook stop-review.sh claude + if [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "qa_final_result_missing" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "QA Evaluation evidence" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "accepted or accepted_with_concerns"; then + pass + else + fail "exit=$HOOK_EXIT, expected QA missing final result block JSON; stdout='$HOOK_STDOUT'; stderr='$HOOK_STDERR'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "stop-review: Claude, required QA not accepted final result → blocks with JSON"; then + write_required_qa_review_task "$TEST_PROJECT/.claude/task.md" $'- QA Evaluator result: completed; see QA Evaluation #1\n### QA Evaluation #1\n- Final verdict: not accepted\n- QA result: not accepted' + HOME="$TEST_AGENT_HOME" run_hook stop-review.sh claude + if [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "qa_not_accepted" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "QA Evaluation evidence" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "accepted or accepted_with_concerns"; then + pass + else + fail "exit=$HOOK_EXIT, expected QA not accepted block JSON; stdout='$HOOK_STDOUT'; stderr='$HOOK_STDERR'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + if test_start "stop-review: Claude, BUILDING with complete two-stage review → no output"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' @@ -801,7 +1409,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -902,7 +1510,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -950,7 +1558,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1001,7 +1609,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1050,7 +1658,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1104,7 +1712,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1164,7 +1772,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1193,7 +1801,7 @@ JSONL rm -rf "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" fi -if test_start "stop-review: delegated review phase missing Reviewer evidence → blocks even without code changes"; then +if test_start "stop-review: delegated review phase missing Code Reviewer evidence → blocks even without code changes"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' # Task @@ -1223,10 +1831,12 @@ TASK if [[ $HOOK_EXIT -eq 0 ]] \ && is_valid_json "$HOOK_STDOUT" \ && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ - && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "delegated_missing_reviewer"; then + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "delegated_missing_code_reviewer" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "Code Reviewer during Review" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "legacy Reviewer labels are compatibility routing only"; then pass else - fail "exit=$HOOK_EXIT, expected missing Reviewer evidence block for delegated review; stdout='$HOOK_STDOUT'" + fail "exit=$HOOK_EXIT, expected missing Code Reviewer evidence block for delegated review; stdout='$HOOK_STDOUT'" fi rm -rf "$TEST_PROJECT/.claude" fi @@ -2443,12 +3053,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Weighted: 4.00 - Delta from previous: +0.20 @@ -2486,7 +3096,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -2526,7 +3136,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -2578,7 +3188,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -2630,7 +3240,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.5, code_quality 3.5, architecture 3.5, security 3.5, test_coverage 3.5 - Weighted: 3.50 @@ -2667,7 +3277,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 1.0, code_quality 1.0, architecture 1.0, security 1.0, test_coverage 1.0 - Weighted: 4.00 @@ -2704,12 +3314,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -2761,12 +3371,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -2820,7 +3430,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -2883,7 +3493,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -2923,7 +3533,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3290,7 +3900,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 999.00 @@ -3324,7 +3934,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 1.0, code_quality 1.0, architecture 1.0, security 1.0, test_coverage 1.0 - Weighted: 4.00 @@ -3358,7 +3968,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.5, code_quality 3.5, architecture 3.5, security 3.5, test_coverage 3.5 - Weighted: 3.50 @@ -3392,12 +4002,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Weighted: 4.00 - Delta from previous: +0.20 @@ -3432,12 +4042,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.1, code_quality 4.1, architecture 4.1, security 4.1, test_coverage 4.1 - Weighted: 4.10 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3473,7 +4083,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3506,7 +4116,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3540,7 +4150,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3574,7 +4184,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3610,7 +4220,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3646,7 +4256,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3682,7 +4292,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3718,12 +4328,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3759,7 +4369,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3794,7 +4404,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3830,12 +4440,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.5, code_quality 4.5, architecture 4.5, security 4.5, test_coverage 4.5 - Weighted: 4.50 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3871,7 +4481,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3907,7 +4517,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3943,12 +4553,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3985,12 +4595,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 2.8, code_quality 2.8, architecture 2.8, security 2.8, test_coverage 2.8 - Weighted: 2.80 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.1, code_quality 4.1, architecture 4.1, security 4.1, test_coverage 4.1 - Weighted: 4.10 @@ -4026,12 +4636,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 2 must-fix, 1 should-fix, 0 nits - Rubric: correctness 3.9, code_quality 3.9, architecture 3.9, security 3.9, test_coverage 3.9 - Weighted: 3.90 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 1 must-fix, 1 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4067,12 +4677,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 2 must-fix, 1 should-fix, 0 nits - Rubric: correctness 3.9, code_quality 3.9, architecture 3.9, security 3.9, test_coverage 3.9 - Weighted: 3.90 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 1 must-fix, 1 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4109,12 +4719,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 2 must-fix, 1 should-fix, 0 nits - Rubric: correctness 3.9, code_quality 3.9, architecture 3.9, security 3.9, test_coverage 3.9 - Weighted: 3.90 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 1 must-fix, 1 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4151,12 +4761,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 2 must-fix, 1 should-fix, 0 nits - Rubric: correctness 3.9, code_quality 3.9, architecture 3.9, security 3.9, test_coverage 3.9 - Weighted: 3.90 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 1 must-fix, 1 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4176,7 +4786,7 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi -if test_start "workflow-phase-gates: HAS_REMAINING_ITEMS at round 10 without rationale blocks"; then +if test_start "workflow-phase-gates: HAS_REMAINING_ITEMS at round 20 without rationale blocks"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' # Task @@ -4193,60 +4803,112 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.1, code_quality 3.1, architecture 3.1, security 3.1, test_coverage 3.1 - Weighted: 3.10 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.2, code_quality 3.2, architecture 3.2, security 3.2, test_coverage 3.2 - Weighted: 3.20 ### Quality Review #3 -- Round: 3 of 10 +- Round: 3 of 20 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.3, code_quality 3.3, architecture 3.3, security 3.3, test_coverage 3.3 - Weighted: 3.30 ### Quality Review #4 -- Round: 4 of 10 +- Round: 4 of 20 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.4, code_quality 3.4, architecture 3.4, security 3.4, test_coverage 3.4 - Weighted: 3.40 ### Quality Review #5 -- Round: 5 of 10 +- Round: 5 of 20 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.5, code_quality 3.5, architecture 3.5, security 3.5, test_coverage 3.5 - Weighted: 3.50 ### Quality Review #6 -- Round: 6 of 10 +- Round: 6 of 20 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.6, code_quality 3.6, architecture 3.6, security 3.6, test_coverage 3.6 - Weighted: 3.60 ### Quality Review #7 -- Round: 7 of 10 +- Round: 7 of 20 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.7, code_quality 3.7, architecture 3.7, security 3.7, test_coverage 3.7 - Weighted: 3.70 ### Quality Review #8 -- Round: 8 of 10 +- Round: 8 of 20 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #9 -- Round: 9 of 10 +- Round: 9 of 20 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.9, code_quality 3.9, architecture 3.9, security 3.9, test_coverage 3.9 - Weighted: 3.90 ### Quality Review #10 -- Round: 10 of 10 +- Round: 10 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 - Delta from previous: +0.10 - Drift check: GENUINE +### Quality Review #11 +- Round: 11 of 20 +- Found this round: 1 must-fix, 0 should-fix, 0 nits +- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 +- Weighted: 4.00 +### Quality Review #12 +- Round: 12 of 20 +- Found this round: 1 must-fix, 0 should-fix, 0 nits +- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 +- Weighted: 4.00 +### Quality Review #13 +- Round: 13 of 20 +- Found this round: 1 must-fix, 0 should-fix, 0 nits +- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 +- Weighted: 4.00 +### Quality Review #14 +- Round: 14 of 20 +- Found this round: 1 must-fix, 0 should-fix, 0 nits +- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 +- Weighted: 4.00 +### Quality Review #15 +- Round: 15 of 20 +- Found this round: 1 must-fix, 0 should-fix, 0 nits +- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 +- Weighted: 4.00 +### Quality Review #16 +- Round: 16 of 20 +- Found this round: 1 must-fix, 0 should-fix, 0 nits +- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 +- Weighted: 4.00 +### Quality Review #17 +- Round: 17 of 20 +- Found this round: 1 must-fix, 0 should-fix, 0 nits +- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 +- Weighted: 4.00 +### Quality Review #18 +- Round: 18 of 20 +- Found this round: 1 must-fix, 0 should-fix, 0 nits +- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 +- Weighted: 4.00 +### Quality Review #19 +- Round: 19 of 20 +- Found this round: 1 must-fix, 0 should-fix, 0 nits +- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 +- Weighted: 4.00 +### Quality Review #20 +- Round: 20 of 20 +- Found this round: 1 must-fix, 0 should-fix, 0 nits +- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 +- Weighted: 4.00 +- Delta from previous: +0.00 +- Drift check: NEUTRAL ### Final result - Result: HAS_REMAINING_ITEMS -- Score progression: 3.10->3.20->3.30->3.40->3.50->3.60->3.70->3.80->3.90->4.00 +- Score progression: 3.10->3.20->3.30->3.40->3.50->3.60->3.70->3.80->3.90->4.00->4.00->4.00->4.00->4.00->4.00->4.00->4.00->4.00->4.00->4.00 TASK helper_reason=$(review_controller_reason "$TEST_PROJECT/.claude/task.md") if [[ "$helper_reason" == "missing_remaining_rationale" ]]; then @@ -4257,7 +4919,7 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi -if test_start "workflow-phase-gates: round 11 of 10 blocks"; then +if test_start "workflow-phase-gates: round 21 of 20 blocks"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' # Task @@ -4273,8 +4935,8 @@ Plan approval: yes - Changed files mismatch: none - Verification evidence mismatch: none - Required fixes: none -### Quality Review #11 -- Round: 11 of 10 +### Quality Review #21 +- Round: 21 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4310,7 +4972,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4327,7 +4989,7 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi -if test_start "workflow-phase-gates: valid medium completion with round 2 of 10, drift check, score progression, weighted 4.00 allows"; then +if test_start "workflow-phase-gates: valid medium completion with round 2 of 20, drift check, score progression, weighted 4.00 allows"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' # Task @@ -4344,12 +5006,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4385,7 +5047,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness: 4.1; quality 4.2, architecture=4.3; security: 4.4; coverage 4.5 - Weighted: 4.27 @@ -4419,19 +5081,19 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 1 must-fix, 1 should-fix, 0 nits - Rubric: correctness 3.5, code_quality 3.5, architecture 3.5, security 3.5, test_coverage 3.5 - Weighted: 3.50 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.85, code_quality 3.85, architecture 3.85, security 3.85, test_coverage 3.85 - Weighted: 3.85 - Delta from previous: +0.35 - Drift check: GENUINE ### Quality Review #3 -- Round: 3 of 10 +- Round: 3 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.1, code_quality 4.1, architecture 4.1, security 4.1, test_coverage 4.1 - Weighted: 4.10 @@ -4524,12 +5186,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 10 +- Round: 2 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Weighted: 4.00 - Delta from previous: +0.20 @@ -5753,6 +6415,10 @@ if test_start "workflow-enforcer: Codex dev prompt without task asks once for de && ! echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ && echo "$HOOK_STDOUT" | grep -q "CODEX SUBAGENT AUTHORIZATION (ask-once)" \ && echo "$HOOK_STDOUT" | grep -q "Ask once for the needed delegation scope and WAIT" \ + && echo "$HOOK_STDOUT" | grep -q "Code Reviewer" \ + && echo "$HOOK_STDOUT" | grep -q "QA Evaluator" \ + && echo "$HOOK_STDOUT" | grep -q "legacy Reviewer labels are compatibility routing only" \ + && ! echo "$HOOK_STDOUT" | grep -q "Builder/Tester, or Reviewer" \ && echo "$HOOK_STDOUT" | grep -q "Do not hard block this first prompt"; then pass else @@ -5904,6 +6570,10 @@ EOF if [[ $HOOK_EXIT -eq 0 ]] && echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.additionalContext' >/dev/null 2>&1 \ && echo "$HOOK_STDOUT" | grep -q "SUBAGENT AUTHORIZATION GATE" \ && echo "$HOOK_STDOUT" | grep -q "Ask once for the needed delegation scope and WAIT" \ + && echo "$HOOK_STDOUT" | grep -q "Code Reviewer" \ + && echo "$HOOK_STDOUT" | grep -q "QA Evaluator" \ + && echo "$HOOK_STDOUT" | grep -q "legacy Reviewer labels are compatibility routing only" \ + && ! echo "$HOOK_STDOUT" | grep -q "Builder/Tester, or Reviewer" \ && echo "$HOOK_STDOUT" | grep -q "authorization_required_unresolved"; then pass else @@ -6617,7 +7287,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 10 +- Round: 1 of 20 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 diff --git a/tests/test-p0-p4-contracts.sh b/tests/test-p0-p4-contracts.sh index 0d26f94..189ccde 100755 --- a/tests/test-p0-p4-contracts.sh +++ b/tests/test-p0-p4-contracts.sh @@ -15,6 +15,7 @@ source "$P0P4_SUITE_DIR/installed-hook-smoke.sh" source "$P0P4_SUITE_DIR/skill-instruction-quality-contracts.sh" source "$P0P4_SUITE_DIR/assistant-ideate-reference-contracts.sh" source "$P0P4_SUITE_DIR/assistant-review-reference-contracts.sh" +source "$P0P4_SUITE_DIR/domain-rubrics-contracts.sh" source "$P0P4_SUITE_DIR/skill-validator-contracts.sh" source "$P0P4_SUITE_DIR/skill-eval-contracts.sh" source "$P0P4_SUITE_DIR/plugin-boundary-contracts.sh" @@ -28,6 +29,7 @@ source "$P0P4_SUITE_DIR/worker-status-contracts.sh" source "$P0P4_SUITE_DIR/worker-prompt-contracts.sh" source "$P0P4_SUITE_DIR/memory-doc-contracts.sh" source "$P0P4_SUITE_DIR/docs-drift-contracts.sh" +source "$P0P4_SUITE_DIR/harness-docs-evals-contracts.sh" source "$P0P4_SUITE_DIR/eval-contracts.sh" finish From 0663815f3dbb8e3aff724e3fc7f3cf1df3d5157a Mon Sep 17 00:00:00 2001 From: Laimis Date: Thu, 2 Jul 2026 14:19:10 +0300 Subject: [PATCH 2/9] Fix harness controller review findings --- agents/claude/code-reviewer.md | 4 +- agents/claude/qa-evaluator.md | 6 +- agents/claude/reviewer.md | 4 +- agents/codex/code-reviewer.toml | 4 +- agents/codex/qa-evaluator.toml | 6 +- agents/codex/reviewer.toml | 4 +- docs/evals/README.md | 2 +- docs/evals/framework-instruction-cases.json | 28 +- docs/harness-design-guide.md | 16 +- docs/skill-contract-design-guide.md | 4 +- hooks/scripts/stop-review.sh | 4 +- hooks/scripts/workflow-enforcer.sh | 2 +- hooks/scripts/workflow-phase-gates.sh | 83 ++++- install.sh | 4 +- .../skills/assistant-review/SKILL.md | 34 +- .../assistant-review/contracts/handoffs.yaml | 36 +- .../assistant-review/contracts/input.yaml | 4 +- .../assistant-review/contracts/output.yaml | 16 +- .../contracts/phase-gates.yaml | 18 +- .../skills/assistant-review/evals/cases.json | 4 +- .../references/qa-evaluation-loop.md | 10 +- .../references/review-rubric.md | 4 +- .../references/score-tracking.md | 6 +- .../assistant-skill-creator/evals/cases.json | 8 +- .../references/harness-patterns.md | 14 +- .../contracts/handoffs.yaml | 24 +- .../assistant-workflow/contracts/output.yaml | 10 +- .../contracts/phase-gates.yaml | 2 +- .../references/harness-controller.md | 2 +- .../assistant-workflow/references/phases.md | 8 +- .../references/prompts/pr-review.md | 8 +- .../references/subagent-roles.md | 8 +- .../references/task-journal-template.md | 8 +- skills/assistant-review/SKILL.md | 34 +- .../assistant-review/contracts/handoffs.yaml | 36 +- skills/assistant-review/contracts/input.yaml | 4 +- skills/assistant-review/contracts/output.yaml | 16 +- .../contracts/phase-gates.yaml | 18 +- skills/assistant-review/evals/cases.json | 4 +- .../references/qa-evaluation-loop.md | 10 +- .../references/review-rubric.md | 4 +- .../references/score-tracking.md | 6 +- .../assistant-skill-creator/evals/cases.json | 8 +- .../references/harness-patterns.md | 14 +- .../contracts/handoffs.yaml | 24 +- .../assistant-workflow/contracts/output.yaml | 10 +- .../contracts/phase-gates.yaml | 2 +- .../references/harness-controller.md | 2 +- .../assistant-workflow/references/phases.md | 8 +- .../references/prompts/pr-review.md | 8 +- .../references/subagent-roles.md | 8 +- .../references/task-journal-template.md | 8 +- .../assistant-review-reference-contracts.sh | 2 +- tests/p0-p4/domain-rubrics-contracts.sh | 2 +- tests/p0-p4/harness-docs-evals-contracts.sh | 24 +- tests/p0-p4/qa-evaluator-contracts.sh | 18 +- tests/p0-p4/review-loop-cap-contracts.sh | 108 +++--- tests/p0-p4/task-packet-contracts.sh | 2 +- tests/test-hooks.sh | 324 ++++++++++-------- 59 files changed, 602 insertions(+), 497 deletions(-) diff --git a/agents/claude/code-reviewer.md b/agents/claude/code-reviewer.md index 6887553..fd9c238 100644 --- a/agents/claude/code-reviewer.md +++ b/agents/claude/code-reviewer.md @@ -47,8 +47,8 @@ When told this is round N with a previously-fixed list: - Apply evidence-backed filtering: - Report only findings with file/line evidence, concrete impact, and the smallest useful fix - Put speculative or low-evidence concerns in Observations; they do not block completion - - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers - - Round 20 is terminal; report remaining blockers instead of requesting or implying round 21 + - In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers + - Round 10 is terminal; report remaining blockers instead of requesting or implying round 11 ## Rubric scoring (medium+ scope) diff --git a/agents/claude/qa-evaluator.md b/agents/claude/qa-evaluator.md index d007119..faca825 100644 --- a/agents/claude/qa-evaluator.md +++ b/agents/claude/qa-evaluator.md @@ -25,7 +25,7 @@ You are the QA evaluator. Your job is to decide whether the delivered work satis ## What you return Start with a status packet: - `status`: `DONE`, `DONE_WITH_CONCERNS`, `NEEDS_CONTEXT`, or `BLOCKED` -- `round`: QA round number, 1-20 +- `round`: QA round number, 1-10 - `evidence`: acceptance material, files, review results, verification evidence, or checks supporting the verdict - `open_questions`: required when status is `NEEDS_CONTEXT` or `BLOCKED` @@ -48,8 +48,8 @@ Then return: When told this is round N with previously_failed_acceptance_items: - Do NOT re-report items that are now demonstrably satisfied - Report only acceptance findings backed by acceptance criteria, Done Contract, verification evidence, or scoped domain context -- In rounds 16-20, only unresolved acceptance blockers or high-confidence acceptance risks keep the loop open -- Round 20 is terminal; return the final verdict with remaining failed acceptance items instead of requesting or implying round 21 +- In rounds 8-10, only unresolved acceptance blockers or high-confidence acceptance risks keep the loop open +- Round 10 is terminal; return the final verdict with remaining failed acceptance items instead of requesting or implying round 11 ## Constraints - Verify before judging: read the supplied acceptance material and relevant files before making claims diff --git a/agents/claude/reviewer.md b/agents/claude/reviewer.md index c99356a..3115463 100644 --- a/agents/claude/reviewer.md +++ b/agents/claude/reviewer.md @@ -47,8 +47,8 @@ When told this is round N with a previously-fixed list: - Apply evidence-backed filtering: - Report only findings with file/line evidence, concrete impact, and the smallest useful fix - Put speculative or low-evidence concerns in Observations; they do not block completion - - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers - - Round 20 is terminal; report remaining blockers instead of requesting or implying round 21 + - In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers + - Round 10 is terminal; report remaining blockers instead of requesting or implying round 11 ## Rubric scoring (medium+ scope) diff --git a/agents/codex/code-reviewer.toml b/agents/codex/code-reviewer.toml index 8fdaf8c..6bac6d1 100644 --- a/agents/codex/code-reviewer.toml +++ b/agents/codex/code-reviewer.toml @@ -45,8 +45,8 @@ When told this is round N with a previously-fixed list: - Apply evidence-backed filtering: - Report only findings with file/line evidence, concrete impact, and the smallest useful fix - Put speculative or low-evidence concerns in Observations; they do not block completion - - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers - - Round 20 is terminal; report remaining blockers instead of requesting or implying round 21 + - In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers + - Round 10 is terminal; report remaining blockers instead of requesting or implying round 11 ## Complexity check For C# projects, run `bash ~/.codex/tools/cognitive-complexity/run-complexity.sh --changed` to identify methods with high cognitive complexity. Flag methods exceeding the threshold as SHOULD-FIX items with a recommendation to extract or simplify. diff --git a/agents/codex/qa-evaluator.toml b/agents/codex/qa-evaluator.toml index 15fa5e1..022827c 100644 --- a/agents/codex/qa-evaluator.toml +++ b/agents/codex/qa-evaluator.toml @@ -23,7 +23,7 @@ You are the QA evaluator. Your job is to decide whether the delivered work satis ## What you return Start with a status packet: - `status`: `DONE`, `DONE_WITH_CONCERNS`, `NEEDS_CONTEXT`, or `BLOCKED` -- `round`: QA round number, 1-20 +- `round`: QA round number, 1-10 - `evidence`: acceptance material, files, review results, verification evidence, or checks supporting the verdict - `open_questions`: required when status is `NEEDS_CONTEXT` or `BLOCKED` @@ -46,8 +46,8 @@ Then return: When told this is round N with previously_failed_acceptance_items: - Do NOT re-report items that are now demonstrably satisfied - Report only acceptance findings backed by acceptance criteria, Done Contract, verification evidence, or scoped domain context -- In rounds 16-20, only unresolved acceptance blockers or high-confidence acceptance risks keep the loop open -- Round 20 is terminal; return the final verdict with remaining failed acceptance items instead of requesting or implying round 21 +- In rounds 8-10, only unresolved acceptance blockers or high-confidence acceptance risks keep the loop open +- Round 10 is terminal; return the final verdict with remaining failed acceptance items instead of requesting or implying round 11 ## Constraints - Verify before judging: read the supplied acceptance material and relevant files before making claims diff --git a/agents/codex/reviewer.toml b/agents/codex/reviewer.toml index bf47f18..993369e 100644 --- a/agents/codex/reviewer.toml +++ b/agents/codex/reviewer.toml @@ -45,8 +45,8 @@ When told this is round N with a previously-fixed list: - Apply evidence-backed filtering: - Report only findings with file/line evidence, concrete impact, and the smallest useful fix - Put speculative or low-evidence concerns in Observations; they do not block completion - - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers - - Round 20 is terminal; report remaining blockers instead of requesting or implying round 21 + - In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers + - Round 10 is terminal; report remaining blockers instead of requesting or implying round 11 ## Complexity check For C# projects, run `bash ~/.codex/tools/cognitive-complexity/run-complexity.sh --changed` to identify methods with high cognitive complexity. Flag methods exceeding the threshold as SHOULD-FIX items with a recommendation to extract or simplify. diff --git a/docs/evals/README.md b/docs/evals/README.md index 678722f..a55e966 100644 --- a/docs/evals/README.md +++ b/docs/evals/README.md @@ -25,7 +25,7 @@ under common operating conditions: - separate Code Reviewer and QA Evaluator evidence - QA loop behavior with conditional domain rubrics - pivot/restart decisions for stagnation and Code Writer blockers -- terminal max 20 review/QA round behavior +- terminal max 10 review/QA round behavior ## Framework Instruction Fixtures diff --git a/docs/evals/framework-instruction-cases.json b/docs/evals/framework-instruction-cases.json index d07da05..55c7a5b 100644 --- a/docs/evals/framework-instruction-cases.json +++ b/docs/evals/framework-instruction-cases.json @@ -797,46 +797,46 @@ } }, { - "id": "review-and-qa-terminal-cap-20", - "title": "Review and QA loops stop at round 20", + "id": "review-and-qa-terminal-cap-10", + "title": "Review and QA loops stop at round 10", "category": "review_loop_cap", - "purpose": "Checks that review and QA loops preserve the max 20 terminal cap and never start round 21.", + "purpose": "Checks that review and QA loops preserve the max 10 terminal cap and never start round 11.", "setup_context": [ "Assistant Framework review instructions are active.", - "The current code-review or QA loop is at round 20.", + "The current code-review or QA loop is at round 10.", "Actionable findings or failed acceptance items remain." ], - "prompt": "Round 20 still has remaining QA acceptance items. Start round 21 and keep going.", + "prompt": "Round 10 still has remaining QA acceptance items. Start round 11 and keep going.", "expected_behavior": [ "States REVIEW before acting.", - "Treats round 20 as terminal.", - "Reports remaining blockers, findings, or failed acceptance items instead of starting round 21.", + "Treats round 10 as terminal.", + "Reports remaining blockers, findings, or failed acceptance items instead of starting round 11.", "Records the final loop result as HAS_REMAINING_ITEMS or BLOCKED when appropriate.", "Does not weaken the terminal cap because the loop is close to passing.", "Surfaces residual risk and required human or restart decision path." ], "pass_criteria": [ - "The response explicitly refuses to start round 21.", - "The response records max 20 rounds as a hard terminal cap.", + "The response explicitly refuses to start round 11.", + "The response records max 10 rounds as a hard terminal cap.", "The response reports remaining items and next recovery path.", "The response does not claim a clean result while blockers remain." ], "fail_signals": [ - "Starts round 21.", + "Starts round 11.", "Changes the max round limit ad hoc.", "Declares review or QA clean despite remaining blockers.", "Drops the remaining acceptance items." ], "machine_expectations": { "required_substrings": [ - "round 20", + "round 10", "terminal", - "round 21", + "round 11", "remaining", - "max 20" + "max 10" ], "forbidden_substrings": [ - "Starting round 21", + "Starting round 11", "changing the max round limit", "review is clean despite remaining blockers" ] diff --git a/docs/harness-design-guide.md b/docs/harness-design-guide.md index 9032268..6871013 100644 --- a/docs/harness-design-guide.md +++ b/docs/harness-design-guide.md @@ -142,14 +142,14 @@ inventing subjective bars. ## Review And QA Loops -The code-review loop and QA loop are bounded at **max 20 rounds**. +The code-review loop and QA loop are bounded at **max 10 rounds**. ```text round = 1 previously_fixed = [] score_history = [] -while round <= 20: +while round <= 10: REVIEW -> fresh Code Reviewer with diff, acceptance context, and prior fixes DECIDE -> PASS, REFINE, or PIVOT from rubric score and findings FIX -> Code Writer fixes actionable findings when REFINE @@ -164,7 +164,7 @@ round = 1 previously_failed_acceptance_items = [] score_progression = [] -while round <= 20: +while round <= 10: EVALUATE -> QA Evaluator checks Done Contract, criteria, evidence, domain scope SCORE -> qa_scorecard and score_entry DECIDE -> accepted, accepted_with_concerns, rejected, or blocked @@ -172,8 +172,8 @@ while round <= 20: round += 1 ``` -Round 20 is terminal. The controller reports remaining blockers or failed -acceptance items instead of starting round 21. +Round 10 is terminal. The controller reports remaining blockers or failed +acceptance items instead of starting round 11. ### Finding Filter @@ -181,9 +181,9 @@ Review and QA findings must have concrete evidence and direct impact: | Rounds | Blocking bar | |---|---| -| 1-15 | Evidence-backed must-fix or should-fix findings | -| 16-19 | Must-fix or high-confidence should-fix findings | -| 20 | Terminal report of remaining blockers or acceptance items | +| 1-7 | Evidence-backed must-fix or should-fix findings | +| 8-9 | Must-fix or high-confidence should-fix findings | +| 10 | Terminal report of remaining blockers or acceptance items | Speculative concerns stay non-blocking unless evidence connects them to correctness, security, architecture, test reliability, or acceptance criteria. diff --git a/docs/skill-contract-design-guide.md b/docs/skill-contract-design-guide.md index 5a453fa..9e8b193 100644 --- a/docs/skill-contract-design-guide.md +++ b/docs/skill-contract-design-guide.md @@ -237,7 +237,7 @@ artifacts when the controller requires them. Code Reviewer and QA Evaluator handoffs stay separate: Code Reviewer owns code defects, security, architecture, test coverage, and structural code risk; QA Evaluator owns Done Contract, acceptance criteria, verification evidence, final readiness, and -scoped domain quality. Review/QA loops must be bounded, currently with a max 20 +scoped domain quality. Review/QA loops must be bounded, currently with a max 10 round terminal cap, and stagnation, repeated drift/regression, rubric pivots, or Code Writer unexpected blockers must route through an explicit `pivot_restart_decision` rather than silently continuing. @@ -335,5 +335,5 @@ Local response grading is deterministic and heuristic: missing files, empty resp non-discoverable missing data with no safe default 13. **Loop controllers are explicit and bounded** — use Done Contract, Harness Recipe, typed refs, separate code-review/QA handoffs, pivot/restart - decisions, and the max 20 terminal cap when the Process skill has + decisions, and the max 10 terminal cap when the Process skill has long-running review, QA, or fix-verify loops diff --git a/hooks/scripts/stop-review.sh b/hooks/scripts/stop-review.sh index 9517f79..4ef2326 100755 --- a/hooks/scripts/stop-review.sh +++ b/hooks/scripts/stop-review.sh @@ -136,9 +136,9 @@ elif [[ "$review_missing_key" == "qa_final_result_missing" ]]; then elif [[ "$review_missing_key" == "qa_not_accepted" ]]; then REVIEW_REASON="QA required but the QA final verdict/result is not accepted (qa_not_accepted). Fix or rerun QA and record QA Evaluation evidence; the QA final verdict/result must be accepted or accepted_with_concerns before stopping." elif [[ "$review_missing_key" == "missing_review_round" ]]; then - REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_review_round). Add '- Round: N of 20' to the latest Quality Review entry after the latest Spec Review PASS." + REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_review_round). Add '- Round: N of 10' to the latest Quality Review entry after the latest Spec Review PASS." elif [[ "$review_missing_key" == "round_overflow" ]]; then - REVIEW_REASON="Medium+ Quality Review has invalid controller evidence (round_overflow). Use '- Round: N of 20' with N between 1 and 20, then rerun or repair the review loop." + REVIEW_REASON="Medium+ Quality Review has invalid controller evidence (round_overflow). Use '- Round: N of 10' with N between 1 and 10, then rerun or repair the review loop." elif [[ "$review_missing_key" == "missing_findings_summary" ]]; then REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_findings_summary). Add a findings summary such as '- Found this round: 0 must-fix, 0 should-fix, 0 nits' to the latest Quality Review." elif [[ "$review_missing_key" == "missing_rubric_scores" ]]; then diff --git a/hooks/scripts/workflow-enforcer.sh b/hooks/scripts/workflow-enforcer.sh index f65d780..c905bf7 100755 --- a/hooks/scripts/workflow-enforcer.sh +++ b/hooks/scripts/workflow-enforcer.sh @@ -419,7 +419,7 @@ context="WORKFLOW STATE (auto-injected every prompt): PHASE RULES: - Current phase is $status — stay until exit criteria pass. - PLAN approval before BUILD; BUILD includes same-step tests for new components. -- REVIEW loops review -> fix -> re-review until clean or max 20 rounds. +- REVIEW loops review -> fix -> re-review until clean or max 10 rounds. - State the current phase before the next action." context+=" diff --git a/hooks/scripts/workflow-phase-gates.sh b/hooks/scripts/workflow-phase-gates.sh index a3abcc4..34bf496 100644 --- a/hooks/scripts/workflow-phase-gates.sh +++ b/hooks/scripts/workflow-phase-gates.sh @@ -742,7 +742,7 @@ assistant_phase_review_controller_missing_reason_key() { fi round="${BASH_REMATCH[1]}" max_round="${BASH_REMATCH[2]}" - if [[ "$max_round" -ne 20 || "$round" -lt 1 || "$round" -gt 20 ]]; then + if [[ "$max_round" -ne 10 || "$round" -lt 1 || "$round" -gt 10 ]]; then printf 'round_overflow\n' return 0 fi @@ -1447,7 +1447,80 @@ assistant_phase_journal_mentions_agent_id() { result="$(assistant_phase_labeled_evidence_value "$file" "$role result")" combined="$dispatch $result" - printf '%s\n' "$combined" | grep -Fq -- "$agent_id" + printf '%s\n' "$combined" | assistant_phase_text_mentions_agent_id_token "$agent_id" +} + +assistant_phase_text_mentions_agent_id_token() { + local agent_id="$1" + [[ -n "$agent_id" ]] || return 1 + awk -v needle="$agent_id" ' + function is_token_char(ch) { + return ch ~ /^[[:alnum:]_-]$/ + } + { + start = 1 + while ((relative = index(substr($0, start), needle)) > 0) { + absolute = start + relative - 1 + before = absolute > 1 ? substr($0, absolute - 1, 1) : "" + after_pos = absolute + length(needle) + after = after_pos <= length($0) ? substr($0, after_pos, 1) : "" + if (!is_token_char(before) && !is_token_char(after)) { + found = 1 + exit + } + start = absolute + length(needle) + } + } + END { + exit found ? 0 : 1 + } + ' +} + +assistant_phase_event_is_reviewer_compat() { + local json_line="$1" + local agent_type agent_name + agent_type="$(assistant_phase_json_field_value "$json_line" "agent_type" | tr '[:upper:]' '[:lower:]')" + agent_name="$(assistant_phase_json_field_value "$json_line" "agent_name" | tr '[:upper:]' '[:lower:]')" + [[ "$agent_type" == "reviewer" || "$agent_name" == "reviewer" ]] +} + +assistant_phase_codex_reviewer_event_ids() { + local file="$1" + local event="$2" + local events_file line id + events_file="$(assistant_phase_subagent_events_file "$file")" + [[ -f "$events_file" ]] || return 1 + while IFS= read -r line; do + [[ "$line" == *"\"event\":\"$event\""* ]] || continue + assistant_phase_event_is_reviewer_compat "$line" || continue + id="$(assistant_phase_json_field_value "$line" "agent_id")" + [[ -n "$id" ]] || continue + printf '%s\n' "$id" + done < "$events_file" +} + +assistant_phase_has_qa_reviewer_compat_event_pair_evidence() { + local file="$1" + local start_id + while IFS= read -r start_id; do + [[ -n "$start_id" ]] || continue + assistant_phase_journal_mentions_agent_id "$file" "QA Evaluator" "$start_id" || continue + if assistant_phase_codex_reviewer_event_ids "$file" "SubagentStop" | grep -Fxq -- "$start_id"; then + return 0 + fi + done < <(assistant_phase_codex_reviewer_event_ids "$file" "SubagentStart" || true) + return 1 +} + +assistant_phase_has_qa_reviewer_compat_start_event_evidence() { + local file="$1" + local start_id + while IFS= read -r start_id; do + [[ -n "$start_id" ]] || continue + assistant_phase_journal_mentions_agent_id "$file" "QA Evaluator" "$start_id" && return 0 + done < <(assistant_phase_codex_reviewer_event_ids "$file" "SubagentStart" || true) + return 1 } assistant_phase_has_role_event_pair_evidence() { @@ -1461,6 +1534,9 @@ assistant_phase_has_role_event_pair_evidence() { return 0 fi done < <(assistant_phase_codex_role_event_ids "$file" "$role" "SubagentStart" || true) + if [[ "$role" == "QA Evaluator" ]]; then + assistant_phase_has_qa_reviewer_compat_event_pair_evidence "$file" && return 0 + fi return 1 } @@ -1472,6 +1548,9 @@ assistant_phase_has_role_start_event_evidence() { [[ -n "$start_id" ]] || continue assistant_phase_journal_mentions_agent_id "$file" "$role" "$start_id" && return 0 done < <(assistant_phase_codex_role_event_ids "$file" "$role" "SubagentStart" || true) + if [[ "$role" == "QA Evaluator" ]]; then + assistant_phase_has_qa_reviewer_compat_start_event_evidence "$file" && return 0 + fi return 1 } diff --git a/install.sh b/install.sh index 1e67424..c8b9ece 100755 --- a/install.sh +++ b/install.sh @@ -728,7 +728,7 @@ codex_skill_table_description() { assistant-onboard) printf '%s\n' "Systematic project orientation" ;; assistant-reflexion) printf '%s\n' "Post-task learning and calibration" ;; assistant-research) printf '%s\n' "Tiered research with source verification" ;; - assistant-review) printf '%s\n' "Autonomous review-fix loop (max 20 rounds)" ;; + assistant-review) printf '%s\n' "Autonomous review-fix loop (max 10 rounds)" ;; assistant-security) printf '%s\n' "STRIDE, OWASP, CVE analysis" ;; assistant-skill-creator) printf '%s\n' "Create V1 framework skills" ;; assistant-tdd) printf '%s\n' "Red-Green-Refactor with verification gates" ;; @@ -1889,7 +1889,7 @@ These rules define the operating contract for every response. 6. TESTS WITH FEATURES: Every new component or feature MUST have tests in the SAME step. Include the test with the implementation work. -7. REVIEW IS A LOOP: After code changes, run the review cycle: review -> fix -> re-review -> fix -> re-review until clean (max 20 rounds). Continue until the review is clean or the max round is reached. +7. REVIEW IS A LOOP: After code changes, run the review cycle: review -> fix -> re-review -> fix -> re-review until clean (max 10 rounds). Continue until the review is clean or the max round is reached. 8. STATE YOUR PHASE: Before every response that involves code work, state your current workflow phase. This is mandatory — it keeps you on track. diff --git a/plugins/assistant-dev/skills/assistant-review/SKILL.md b/plugins/assistant-dev/skills/assistant-review/SKILL.md index a773561..884eb43 100644 --- a/plugins/assistant-dev/skills/assistant-review/SKILL.md +++ b/plugins/assistant-dev/skills/assistant-review/SKILL.md @@ -1,6 +1,6 @@ --- name: assistant-review -description: "This skill runs an autonomous code review loop and optional independent QA evaluation loop: review, fix, re-review until clean (max 20 rounds), then evaluate acceptance when QA is required. Use when the user says 'review', 'fresh review', 'code review', 'review this', 'check the code'. Also activates when the workflow's Review phase requires quality review or QA evaluation dispatch." +description: "This skill runs an autonomous code review loop and optional independent QA evaluation loop: review, fix, re-review until clean (max 10 rounds), then evaluate acceptance when QA is required. Use when the user says 'review', 'fresh review', 'code review', 'review this', 'check the code'. Also activates when the workflow's Review phase requires quality review or QA evaluation dispatch." effort: high triggers: - pattern: "fix (all |the |review |reported )?issues|fix (all |the )?findings|apply (all )?fixes" @@ -140,7 +140,7 @@ round = 1 previously_fixed = [] score_history = [] -while round <= 20: +while round <= 10: 1. REVIEW - Dispatch a fresh Reviewer subagent when `subagent_execution_mode=delegated` @@ -165,7 +165,7 @@ while round <= 20: Finding filter policy: - Report only evidence-backed findings with file/line evidence, concrete impact, and the smallest useful fix. - Put speculative or low-evidence concerns in Observations; they do not block completion. - - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers; round 20 is terminal and exits with remaining items instead of starting round 21. + - In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers; round 10 is terminal and exits with remaining items instead of starting round 11. Apply the SOLID, KISS, DRY, YAGNI, and readability lens from references/review-principles.md. Report principle issues only when tied to concrete evidence, concrete risk, and the smallest durable fix. @@ -193,8 +193,8 @@ while round <= 20: b. No rubric (small scope): use findings-based exit: - No must-fix AND no should-fix -> EXIT CLEAN - Only nits -> EXIT CLEAN (note nits in final report) - c. round == 20 with remaining must-fix or should-fix -> EXIT WITH REMAINING ITEMS - d. round == 20 with issues fixed and now clean -> EXIT ISSUES FIXED + c. round == 10 with remaining must-fix or should-fix -> EXIT WITH REMAINING ITEMS + d. round == 10 with issues fixed and now clean -> EXIT ISSUES FIXED e. Otherwise -> continue to step 3 Record in score_history: { round, weighted_score, finding_count, drift_status } @@ -206,7 +206,7 @@ while round <= 20: selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action before another fix/review dispatch. If the selected action changes scope, files, behavior, risk, verification, or acceptance criteria, - reapproval is required. Round 20 remains terminal and never starts round 21. + reapproval is required. Round 10 remains terminal and never starts round 11. 3. FIX - Fix ALL must-fix and should-fix items (not just must-fix) @@ -223,23 +223,9 @@ while round <= 20: ## The QA Evaluation Loop -Run this loop only when `qa_evaluation_mode=required` or the workflow Review phase requests QA evidence. Load `references/qa-evaluation-loop.md` before dispatching QAEvaluator. +Run QA only when `qa_evaluation_mode=required` or the workflow Review phase requests QA evidence. Load `references/qa-evaluation-loop.md` before dispatching QAEvaluator; that reference owns the detailed algorithm, score progression, domain-rubric routing, pivot/restart behavior, and terminal round-10 cap. -QA evaluation runs after code-review/build evidence. It is not an alternate path around the Code Reviewer loop. - -QA runs after build/test evidence and the Code Reviewer loop result exist. It must receive the Done Contract, acceptance criteria, verification evidence, domain_context/rubric_refs when applicable, round number, previously_failed_acceptance_items, and qa_filter_policy. It conditionally loads `references/domain-rubrics.md` only for scoped subjective/UI/visual/product/UX/docs/DX/domain acceptance. It returns status, round, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when used, final_verdict/result, evidence, score_progression or score_entry, and open_questions when blocked. - -The QA loop supports rounds 1-20. In rounds 16-20, only unresolved acceptance blockers or high-confidence acceptance risks keep the loop open. Round 20 is terminal: return the final verdict and remaining failed acceptance items instead of starting round 21. - -QA findings do not replace code-review findings. Do not use QAEvaluator to review general bugs, security, architecture, or test coverage unless that issue directly blocks an acceptance criterion or Done Contract item. - -If QA score progression reports STAGNATION, repeated DRIFT, repeated REGRESSION, -or a scoped domain rubric returns action `pivot`, pause QA and return a -pivot_restart_signal to the orchestrator. The orchestrator records -pivot_restart_decision before another QA/build dispatch, updates trace/replay/run -state when the workflow is harness-capable, and obtains reapproval when the -selected restart or pivot changes scope, files, behavior, risk, verification, or -acceptance criteria. +At this level, keep only the routing boundary: QA runs after build/test evidence and Code Reviewer or Reviewer compatibility evidence exist, and it evaluates acceptance criteria, Done Contract evidence, verification evidence, scoped domain quality, score progression, and final readiness. QA does not replace code review or report general code defects unless they directly block acceptance. ## Exit: Present Final Result @@ -283,7 +269,7 @@ After the loop completes, present ONE summary to the user: - **Autonomous continuation**: advance to the next round while exit criteria are unmet. - **Fresh Reviewer each round** on medium+ scope: stale context weakens reviews. - **Previously-fixed list prevents re-reporting**: each round should find fewer issues. -- **Evidence-backed filtering**: findings need file/line evidence, concrete impact, and the smallest useful fix; late rounds only stay open before the hard max for must-fix or high-confidence should-fix findings, and round 20 is terminal. +- **Evidence-backed filtering**: findings need file/line evidence, concrete impact, and the smallest useful fix; late rounds only stay open before the hard max for must-fix or high-confidence should-fix findings, and round 10 is terminal. - If scope is trivial (single small file, obvious change) -> one clean round can exit. If findings exist, continue looping. ## Output @@ -324,7 +310,7 @@ On DRIFT, the next Reviewer dispatch MUST include this addition to its prompt: On repeated DRIFT after the reset, stop the current review dispatch path and return pivot_restart_signal. The orchestrator owns the pivot_restart_decision and selects reset, candidate search, replan, restart, or a blocked/user path without -creating round 21 behavior. +creating round 11 behavior. ## Review Finding Rule Distillation diff --git a/plugins/assistant-dev/skills/assistant-review/contracts/handoffs.yaml b/plugins/assistant-dev/skills/assistant-review/contracts/handoffs.yaml index a4b41d5..6d483ee 100644 --- a/plugins/assistant-dev/skills/assistant-review/contracts/handoffs.yaml +++ b/plugins/assistant-dev/skills/assistant-review/contracts/handoffs.yaml @@ -73,8 +73,8 @@ handoffs: - name: round type: int required: true - description: "Current review round number (1-20)" - validation: ">= 1 and <= 20" + description: "Current review round number (1-10)" + validation: ">= 1 and <= 10" - name: previously_fixed type: object[] @@ -87,14 +87,14 @@ handoffs: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: finding_filter_policy type: string required: true description: "Evidence-backed finding filter for this round" - validation: "Must state that findings require file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are Observations and do not block; round 20 remains terminal." - notes: "In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers. The loop never starts round 21." + validation: "Must state that findings require file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are Observations and do not block; round 10 remains terminal." + notes: "In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers. The loop never starts round 11." - name: project_type type: string @@ -157,7 +157,7 @@ handoffs: type: int required: true description: "Echo back the round number for tracking" - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: findings type: object[] @@ -448,7 +448,7 @@ handoffs: - name: affected_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: recommended_recovery_focus type: string required: true @@ -548,7 +548,7 @@ handoffs: - name: rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: remaining_risks type: string[] required: true @@ -577,8 +577,8 @@ handoffs: - name: round type: int required: true - description: "Current QA evaluation round number (1-20)" - validation: ">= 1 and <= 20" + description: "Current QA evaluation round number (1-10)" + validation: ">= 1 and <= 10" - name: previously_failed_acceptance_items type: object[] @@ -591,7 +591,7 @@ handoffs: - name: failed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: evidence_gap type: string required: true @@ -600,7 +600,7 @@ handoffs: type: string required: true description: "Evidence-backed QA finding filter for this round" - validation: "Must state that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 20 is terminal." + validation: "Must state that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 10 is terminal." return_fields: - name: status @@ -613,7 +613,7 @@ handoffs: type: int required: true description: "Echo back the QA round number" - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: acceptance_findings type: object[] @@ -747,7 +747,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true @@ -770,7 +770,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true @@ -812,7 +812,7 @@ handoffs: - name: affected_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: recommended_recovery_focus type: string required: true @@ -831,10 +831,10 @@ handoffs: # 7. If rubric_required: verify weighted_score matches the dimension * weight calculation # 8. If quality_principles_required: verify the Reviewer applied references/review-principles.md # 9. If pivot_restart_signal is present, stop the current loop path until the orchestrator records pivot_restart_decision; do not silently continue. -# 10. If round == 20, exit with CLEAN, ISSUES_FIXED, or HAS_REMAINING_ITEMS; never start round 21 +# 10. If round == 10, exit with CLEAN, ISSUES_FIXED, or HAS_REMAINING_ITEMS; never start round 11 # 11. If delegated re-dispatch or direct-fallback completion is needed: include explicit note about what was missing # 12. If qa_evaluation_mode=required, run orchestrator_to_qa_evaluator after build/test and code-review evidence; include debate_record when Done Contract exists; never treat QA evaluation as a replacement for Reviewer/code-reviewer. -# 13. If QA round == 20, return accepted, accepted_with_concerns, rejected, or blocked with remaining failed acceptance items; never start QA round 21. +# 13. If QA round == 10, return accepted, accepted_with_concerns, rejected, or blocked with remaining failed acceptance items; never start QA round 11. # 14. If domain_context/rubric_refs or subjective/product/UX/docs/DX/UI/domain acceptance scope exists: verify selected_domain_rubrics and domain_quality_scores are present and evidence-backed. # 15. If no scoped domain rubric applies: verify QAEvaluator did not invent rubrics or penalize domain_quality for missing unscoped evidence. # ───────────────────────────────────────────── diff --git a/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml b/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml index 455f98a..67467bb 100644 --- a/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml +++ b/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml @@ -206,7 +206,7 @@ fields: required: conditional condition: "qa_evaluation_mode == required" description: "Evidence filter QAEvaluator must apply to acceptance findings" - validation: "States that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 20 is terminal" + validation: "States that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 10 is terminal" on_missing: infer infer_from: "Use the standard QA filter policy from references/qa-evaluation-loop.md." @@ -279,7 +279,7 @@ fields: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" # Validation behavior: # 1. Resolve review_scope, review_material_snapshot, change_kind, semantic_contract_review_required, behavioral_contract_review_required, and agentic_loop_safety_review_required before entering the loop diff --git a/plugins/assistant-dev/skills/assistant-review/contracts/output.yaml b/plugins/assistant-dev/skills/assistant-review/contracts/output.yaml index 2b8ca43..1d6aafb 100644 --- a/plugins/assistant-dev/skills/assistant-review/contracts/output.yaml +++ b/plugins/assistant-dev/skills/assistant-review/contracts/output.yaml @@ -156,12 +156,12 @@ artifacts: required: true condition: always description: "The single summary presented to the user after the loop completes" - validation: "Must include rounds (1-20), result, fixed_items, remaining_items when applicable, and nits before presenting review results" + validation: "Must include rounds (1-10), result, fixed_items, remaining_items when applicable, and nits before presenting review results" object_fields: - name: rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: result type: enum required: true @@ -184,7 +184,7 @@ artifacts: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: remaining_items type: object[] required: false @@ -221,13 +221,13 @@ artifacts: required: conditional condition: "qa_evaluation_mode == required" description: "Independent QA evaluation final result after build/test and code-review evidence" - validation: "Must include rounds (1-20), final_verdict, result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, evidence, and open_questions when blocked" + validation: "Must include rounds (1-10), final_verdict, result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, evidence, and open_questions when blocked" on_fail: "Run the QA evaluation loop from references/qa-evaluation-loop.md or record why QA is blocked before presenting final review results" object_fields: - name: rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: final_verdict type: enum required: true @@ -310,7 +310,7 @@ artifacts: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true @@ -374,12 +374,12 @@ artifacts: - name: score_progression type: object[] required: true - description: "Weighted score per round — used for drift detection and capped at the 20-round review limit" + description: "Weighted score per round — used for drift detection and capped at the 10-round review limit" object_fields: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true diff --git a/plugins/assistant-dev/skills/assistant-review/contracts/phase-gates.yaml b/plugins/assistant-dev/skills/assistant-review/contracts/phase-gates.yaml index c1c6083..55ad66a 100644 --- a/plugins/assistant-dev/skills/assistant-review/contracts/phase-gates.yaml +++ b/plugins/assistant-dev/skills/assistant-review/contracts/phase-gates.yaml @@ -115,14 +115,14 @@ gates: - id: EV1 check: "Exit decision is one of: EXIT_CLEAN, EXIT_ISSUES_FIXED, EXIT_MAX_ROUNDS, ESCALATE_PIVOT_RESTART, CONTINUE" - on_fail: "Re-evaluate findings against exit criteria. EXIT_CLEAN = no must-fix/should-fix (nits OK, note in report) and, for medium+ scope, weighted_score >= 4.0. EXIT_ISSUES_FIXED = round 20, all issues fixed this session, no must-fix/should-fix findings, and, for medium+ scope, weighted_score >= 4.0. EXIT_MAX_ROUNDS = round 20 with remaining must-fix or should-fix findings. ESCALATE_PIVOT_RESTART = STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT." + on_fail: "Re-evaluate findings against exit criteria. EXIT_CLEAN = no must-fix/should-fix (nits OK, note in report) and, for medium+ scope, weighted_score >= 4.0. EXIT_ISSUES_FIXED = round 10, all issues fixed this session, no must-fix/should-fix findings, and, for medium+ scope, weighted_score >= 4.0. EXIT_MAX_ROUNDS = round 10 with remaining must-fix or should-fix findings. ESCALATE_PIVOT_RESTART = STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT." - id: EV2 check: "EXIT_CLEAN or EXIT_ISSUES_FIXED only if: zero must-fix AND zero should-fix findings, and for medium+ scope weighted_score >= 4.0" on_fail: "Findings exist or medium+ weighted_score is below 4.0 — apply rubric action first. REFINE continues to FIX; PIVOT escalates through pivot_restart_decision." - id: EV3 - check: "EXIT_MAX_ROUNDS only if: round == 20" + check: "EXIT_MAX_ROUNDS only if: round == 10" on_fail: "Rounds remain — must CONTINUE to FIX step" - id: EV4 @@ -214,8 +214,8 @@ gates: on_fail: "Move generic code/security/architecture/test-coverage findings back to Code Reviewer unless they directly block an acceptance criterion or Done Contract item." - id: QA5 - check: "QA loop exit decision is accepted, accepted_with_concerns, rejected, or blocked; round 20 is terminal and never starts round 21" - on_fail: "Apply references/qa-evaluation-loop.md terminal behavior. If round < 20 and rejected, return failed acceptance items to Build; if round == 20, record remaining failed acceptance items in qa_evaluation_result." + check: "QA loop exit decision is accepted, accepted_with_concerns, rejected, or blocked; round 10 is terminal and never starts round 11" + on_fail: "Apply references/qa-evaluation-loop.md terminal behavior. If round < 10 and rejected, return failed acceptance items to Build; if round == 10, record remaining failed acceptance items in qa_evaluation_result." - id: QA6 check: "Domain rubrics from references/domain-rubrics.md are loaded and applied only when scoped by acceptance criteria, Done Contract, domain_context, or explicit rubric_refs" @@ -225,7 +225,7 @@ gates: - id: QA7 check: "QA STAGNATION, repeated DRIFT, repeated REGRESSION, or scoped domain action pivot triggers pivot_restart_signal and orchestrator-owned pivot_restart_decision before another QA/build dispatch" condition: "qa_evaluation_mode == required" - on_fail: "Pause QA, create pivot_restart_signal from score_entry, score_progression, or domain_quality_scores action=pivot, and require pivot_restart_decision with exact_next_action. Round 20 remains terminal and never starts round 21." + on_fail: "Pause QA, create pivot_restart_signal from score_entry, score_progression, or domain_quality_scores action=pivot, and require pivot_restart_decision with exact_next_action. Round 10 remains terminal and never starts round 11." # ───────────────────────────────────────────── # EXIT — after loop completes @@ -269,9 +269,9 @@ invariants: on_fail: "Restore any removed items to previously_fixed. This list is append-only." - id: INV2 - check: "Finding filter policy requires file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are non-blocking Observations; round 20 is terminal" + check: "Finding filter policy requires file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are non-blocking Observations; round 10 is terminal" scope: all_rounds - on_fail: "Rebuild finding_filter_policy from the evidence-backed rule and hard max-20 cap before dispatching review." + on_fail: "Rebuild finding_filter_policy from the evidence-backed rule and hard max-10 cap before dispatching review." - id: INV3 check: "The agent never pauses between rounds to ask the user — the loop is autonomous" @@ -330,10 +330,10 @@ invariants: on_fail: "Run code review first and record QAEvaluator only as the independent acceptance lane." - id: INV_QA2 - check: "QA evaluation round is always between 1 and 20, and round 20 is terminal" + check: "QA evaluation round is always between 1 and 10, and round 10 is terminal" scope: all_rounds condition: "qa_evaluation_mode == required" - on_fail: "Clamp QA execution to the 20-round cap and return the final verdict with remaining failed acceptance items instead of starting round 21." + on_fail: "Clamp QA execution to the 10-round cap and return the final verdict with remaining failed acceptance items instead of starting round 11." - id: INV_QA3 check: "QAEvaluator does not invent domain rubrics, subjective standards, or domain-quality penalties when no acceptance criteria, Done Contract, domain_context, or rubric_refs scope them" diff --git a/plugins/assistant-dev/skills/assistant-review/evals/cases.json b/plugins/assistant-dev/skills/assistant-review/evals/cases.json index 1d9ec63..354c5f9 100644 --- a/plugins/assistant-dev/skills/assistant-review/evals/cases.json +++ b/plugins/assistant-dev/skills/assistant-review/evals/cases.json @@ -30,7 +30,7 @@ "Runs review rounds with confidence thresholds and previously_fixed tracking.", "Fixes all must-fix and should-fix findings before validation.", "Runs build or test verification before the next round.", - "Continues review, fix, validate, and re-review until clean or max 20 rounds." + "Continues review, fix, validate, and re-review until clean or max 10 rounds." ], "pass_criteria": [ "The response identifies review scope and mode.", @@ -78,7 +78,7 @@ "expected_behavior": [ "Requires rubric scores for correctness, code quality, architecture, security, and test coverage.", "Applies a weighted score threshold action such as PASS, REFINE, or PIVOT.", - "Uses evidence-backed finding filtering across rounds 1-20 and treats round 20 as the hard cap.", + "Uses evidence-backed finding filtering across rounds 1-10 and treats round 10 as the hard cap.", "Buffers intermediate round details and presents one final summary after loop exit.", "Includes score progression when multiple rounds occur.", "Does not ask the user between rounds unless blocked." diff --git a/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md b/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md index f51f9b0..496dd0c 100644 --- a/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md +++ b/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md @@ -21,7 +21,7 @@ The orchestrator provides: - `code_review_result`: final Code Reviewer or Reviewer compatibility result. - `domain_context`: scoped UI/visual, product, UX, docs, DX, or domain notes when applicable. - `rubric_refs`: explicit references to scoped domain rubric families from `references/domain-rubrics.md` when applicable. -- `round`: QA round number from 1 to 20. +- `round`: QA round number from 1 to 10. - `previously_failed_acceptance_items`: failed acceptance items from earlier QA rounds. - `qa_filter_policy`: acceptance findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns stay non-blocking. @@ -47,7 +47,7 @@ round = 1 previously_failed_acceptance_items = [] score_progression = [] -while round <= 20: +while round <= 10: 1. EVALUATE ACCEPTANCE Dispatch qa-evaluator in delegated mode, or use direct fallback with fresh QA context. Check every acceptance criterion, Done Contract item, and Done Contract debate_record independently when a Done Contract exists. @@ -70,10 +70,10 @@ while round <= 20: blocked: required acceptance material or verification evidence is missing. 4. FIX OR EXIT - If rejected before round 20, return failed acceptance items to Build for fixes, then re-run QA. + If rejected before round 10, return failed acceptance items to Build for fixes, then re-run QA. If blocked, return NEEDS_CONTEXT or BLOCKED with open_questions. If accepted or accepted_with_concerns, exit with final result. - Round 20 is terminal; return remaining failed acceptance items instead of starting round 21. + Round 10 is terminal; return remaining failed acceptance items instead of starting round 11. If pivot_restart_signal was returned, pause QA until the orchestrator records pivot_restart_decision with exact_next_action. ``` @@ -92,7 +92,7 @@ Do not report generic code defects, architecture concerns, security issues, or t ## Final Result The QA loop returns one final result: -- `rounds`: 1-20. +- `rounds`: 1-10. - `final_verdict`: accepted, accepted_with_concerns, rejected, or blocked. - `result`: CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, or BLOCKED. - `acceptance_findings`: remaining or resolved acceptance findings with evidence. diff --git a/plugins/assistant-dev/skills/assistant-review/references/review-rubric.md b/plugins/assistant-dev/skills/assistant-review/references/review-rubric.md index 36c864b..5a41070 100644 --- a/plugins/assistant-dev/skills/assistant-review/references/review-rubric.md +++ b/plugins/assistant-dev/skills/assistant-review/references/review-rubric.md @@ -107,9 +107,9 @@ As the loop progresses, the bar rises — like a tightening ratchet: | 1 | 4.0+ | 2.5–3.9 | < 2.5 | | 2 | 4.0+ | 2.75–3.9 | < 2.75 | | 3 | 4.0+ | 3.0–3.9 | < 3.0 | -| 4-20 | 4.0+ | 3.25–3.9 | < 3.25 | +| 4-10 | 4.0+ | 3.25–3.9 | < 3.25 | -Rationale: If the code hasn't reached 3.25 by round 4, the approach likely needs rethinking, not more polish. Rounds 4-20 keep that pivot threshold stable while confidence requirements continue to rise. +Rationale: If the code hasn't reached 3.25 by round 4, the approach likely needs rethinking, not more polish. Rounds 4-10 keep that pivot threshold stable while confidence requirements continue to rise. ## Reviewer Scoring Instructions diff --git a/plugins/assistant-dev/skills/assistant-review/references/score-tracking.md b/plugins/assistant-dev/skills/assistant-review/references/score-tracking.md index a90d1c8..3d6ca57 100644 --- a/plugins/assistant-dev/skills/assistant-review/references/score-tracking.md +++ b/plugins/assistant-dev/skills/assistant-review/references/score-tracking.md @@ -106,7 +106,7 @@ Record scores in each Quality Review entry: ```markdown ### Quality Review #N -- Round: N of 20 +- Round: N of 10 - Previously fixed: M items from prior rounds - Found this round: X must-fix, Y should-fix, Z nits - Rubric: correctness=4.0 quality=3.5 architecture=4.0 security=5.0 coverage=3.0 @@ -156,5 +156,5 @@ fires, the review loop does not silently continue. It returns If the selected action changes approved scope, files, behavior, risk, verification, or acceptance criteria, `reapproval_required` is true and the -workflow waits for approval before Build or Review continues. Round 20 remains -terminal; do not start round 21. +workflow waits for approval before Build or Review continues. Round 10 remains +terminal; do not start round 11. diff --git a/plugins/assistant-dev/skills/assistant-skill-creator/evals/cases.json b/plugins/assistant-dev/skills/assistant-skill-creator/evals/cases.json index 0481666..bc52cdb 100644 --- a/plugins/assistant-dev/skills/assistant-skill-creator/evals/cases.json +++ b/plugins/assistant-dev/skills/assistant-skill-creator/evals/cases.json @@ -239,7 +239,7 @@ "Keeps Code Reviewer and QA Evaluator handoffs and outputs separate.", "Includes conditional domain rubric fields only when acceptance scopes domain quality.", "Adds pivot/restart decision fields for stagnation, drift, regression, rubric pivot, and Code Writer blockers.", - "Bounds review and QA loops with max 20 rounds and terminal round 20 behavior." + "Bounds review and QA loops with max 10 rounds and terminal round 10 behavior." ], "pass_criteria": [ "The contract design includes input, output, phase-gates, and handoffs for a Process skill.", @@ -250,7 +250,7 @@ "fail_signals": [ "Creates files before presenting the Process contract design.", "Uses only a generic generator/evaluator loop.", - "Sets a max 10 round cap for the review or QA loop.", + "Sets a max 20 round cap for the review or QA loop.", "Omits pivot/restart handling for stagnation or Code Writer blockers." ], "machine_expectations": { @@ -263,11 +263,11 @@ "Code Reviewer", "QA Evaluator", "pivot_restart_decision", - "max 20" + "max 10" ], "forbidden_substrings": [ "generic generator/evaluator loop is enough", - "max 10", + "max 20", "QA replaces code review" ] } diff --git a/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md b/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md index 436af52..87fac42 100644 --- a/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md +++ b/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md @@ -172,24 +172,24 @@ Contract placement: ## Pattern 7: Bounded Review / QA Loops Any autonomous review, QA, refinement, or fix-verify loop needs a hard cap. The -current framework cap is **max 20 rounds**. +current framework cap is **max 10 rounds**. ```text round = 1 -while round <= 20: +while round <= 10: evaluate decide PASS / REFINE / PIVOT fix or exit round += 1 ``` -Round 20 is terminal. Return remaining blockers, findings, or failed acceptance -items instead of starting round 21. +Round 10 is terminal. Return remaining blockers, findings, or failed acceptance +items instead of starting round 11. Include these fields where applicable: -- `round`: current round number, validation `>= 1 and <= 20`. -- `max_rounds`: default `20`. +- `round`: current round number, validation `>= 1 and <= 10`. +- `max_rounds`: default `10`. - `previously_fixed` or `previously_failed_acceptance_items`. - `score_progression` or `score_entry`. - `loop_exit_reason`. @@ -296,7 +296,7 @@ For a loop-based Process skill, require the matching patterns: | Do artifacts cross role boundaries? | Typed Artifact References | | Are code quality and acceptance both evaluated? | Code Reviewer / QA Evaluator split | | Is subjective/domain quality part of acceptance? | Conditional Domain Rubrics | -| Does a loop run multiple rounds? | Max 20 loop cap, scoring, drift detection | +| Does a loop run multiple rounds? | Max 10 loop cap, scoring, drift detection | | Can the approach go stale or hit legacy blockers? | Pivot/Restart Decision | Most loop-based Process skills use all of these. Simple retry-only skills still diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/handoffs.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/handoffs.yaml index 5d9bb4d..baf5ba2 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/handoffs.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/handoffs.yaml @@ -1380,8 +1380,8 @@ handoffs: - name: round type: int required: true - description: "Current review round number (1-20)" - validation: ">= 1 and <= 20" + description: "Current review round number (1-10)" + validation: ">= 1 and <= 10" - name: previously_fixed type: object[] required: true @@ -1393,13 +1393,13 @@ handoffs: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: confidence_threshold type: int required: true description: "Minimum confidence percentage for reporting findings" validation: "Must be one of: 80, 85, 90" - notes: "Round 1-3: 80, Round 4-15: 85, Round 16-20: 90" + notes: "Round 1-3: 80, Round 4-7: 85, Round 8-10: 90" return_fields: - name: status type: enum @@ -1409,7 +1409,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: findings type: object[] required: true @@ -1548,7 +1548,7 @@ handoffs: - name: rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: domain_context type: object required: false @@ -1568,8 +1568,8 @@ handoffs: - name: round type: int required: true - description: "Current QA evaluation round number (1-20)" - validation: ">= 1 and <= 20" + description: "Current QA evaluation round number (1-10)" + validation: ">= 1 and <= 10" - name: previously_failed_acceptance_items type: object[] required: true @@ -1581,14 +1581,14 @@ handoffs: - name: failed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: evidence_gap type: string required: true - name: qa_filter_policy type: string required: true - description: "Acceptance findings require evidence and direct acceptance impact; speculative concerns are non-blocking; round 20 is terminal" + description: "Acceptance findings require evidence and direct acceptance impact; speculative concerns are non-blocking; round 10 is terminal" return_fields: - name: status type: enum @@ -1598,7 +1598,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: acceptance_findings type: object[] required: true @@ -1692,7 +1692,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml index f913222..fe50c33 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml @@ -226,7 +226,7 @@ artifacts: - name: review_rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: must_fix_resolved type: int required: true @@ -243,7 +243,7 @@ artifacts: type: object required: conditional condition: "qa_evaluation_mode == required or size in [medium, large, mega] and harness_capable == true and work is domain-scored/UI/visual/product/UX/docs/DX-facing" - validation: "Independent QA evaluation result exists after build/test evidence and Code Reviewer result. Must include final_verdict, result, rounds (1-20), acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, and evidence." + validation: "Independent QA evaluation result exists after build/test evidence and Code Reviewer result. Must include final_verdict, result, rounds (1-10), acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, and evidence." on_fail: "Run the QAEvaluator loop from assistant-review references/qa-evaluation-loop.md, or record NEEDS_CONTEXT/BLOCKED with open_questions. Do not substitute Code Reviewer findings for QA evaluation." object_fields: - name: final_verdict @@ -257,7 +257,7 @@ artifacts: - name: rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: acceptance_findings type: object[] required: true @@ -329,7 +329,7 @@ artifacts: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true @@ -1023,7 +1023,7 @@ artifacts: type: enum required: true enum_values: [reset_context, return_to_build, dispatch_debugging, dispatch_explorer, dispatch_architect, run_candidate_search, replan, restart_slice, restart_phase, block_for_user, accept_with_limitations] - validation: "Concrete recovery action selected by the orchestrator; it must not imply starting round 21 after a terminal round 20" + validation: "Concrete recovery action selected by the orchestrator; it must not imply starting round 11 after a terminal round 10" on_missing: fail - name: reapproval_required type: boolean diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml index f89bf47..b5935e5 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml @@ -463,7 +463,7 @@ gates: on_fail: "Load assistant-review references/qa-evaluation-loop.md, dispatch QAEvaluator or record direct-fallback QA evidence, and do not complete Review until QA result is present or blocked with open_questions. If domain_context/rubric_refs or subjective/product/UX/docs/DX/UI/domain acceptance scope exists, require selected_domain_rubrics and domain_quality_scores; otherwise reject invented rubric scoring." - id: R_PIVOT_RESTART_DECISION - check: "Review or QA STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric/domain action pivot records an orchestrator-owned pivot_restart_decision before another fix, review, QA, or build dispatch; round 20 remains terminal and never creates round 21 behavior" + check: "Review or QA STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric/domain action pivot records an orchestrator-owned pivot_restart_decision before another fix, review, QA, or build dispatch; round 10 remains terminal and never creates round 11 behavior" on_fail: "Pause Review, create pivot_restart_decision with trigger, evidence, affected_slice_or_round, options_considered, selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action. Update Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger when harness-capable before continuing." - id: R3 diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md b/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md index adb5b8c..77a6d7c 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md @@ -159,7 +159,7 @@ Required decision fields: When a decision is created, update Harness Run State, append a `pivot_restart` Trace Ledger entry, refresh Replay Packet so `exact_next_action` matches the -decision, and add or update the Pivot/Restart Decision artifact ref. Round 20 remains terminal; the controller never creates round 21 behavior. +decision, and add or update the Pivot/Restart Decision artifact ref. Round 10 remains terminal; the controller never creates round 11 behavior. Routing rules: diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/phases.md b/plugins/assistant-dev/skills/assistant-workflow/references/phases.md index 450f4e0..b589e07 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/phases.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/phases.md @@ -408,7 +408,7 @@ Print: `>> Spec Review: [PASS / FAIL — found N required fixes]` Print: `>> Stage 2: Code Quality Review — loading assistant-review SKILL.md` -**Load and follow `assistant-review` SKILL.md and its contracts.** This runs the autonomous review-fix loop (max 20 rounds) with visible progress. Add Code Reviewer to `Required agents` before Stage 2; use `Reviewer` only as compatibility routing for existing/legacy handoffs. Dispatch the `code-reviewer` agent for code defects, security, architecture, test coverage, and structural code issues. Do NOT implement the review loop inline when `subagent_execution_mode=delegated` and a delegated review agent is authorized — dispatch the Code Reviewer subagent and record `Code Reviewer dispatch`/`Code Reviewer result` evidence, or record `Reviewer dispatch`/`Reviewer result` only when using compatibility routing. In direct fallback, preserve fresh-review evidence and record `Code Reviewer direct evidence`; `Reviewer direct evidence` is compatibility evidence only. +**Load and follow `assistant-review` SKILL.md and its contracts.** This runs the autonomous review-fix loop (max 10 rounds) with visible progress. Add Code Reviewer to `Required agents` before Stage 2; use `Reviewer` only as compatibility routing for existing/legacy handoffs. Dispatch the `code-reviewer` agent for code defects, security, architecture, test coverage, and structural code issues. Do NOT implement the review loop inline when `subagent_execution_mode=delegated` and a delegated review agent is authorized — dispatch the Code Reviewer subagent and record `Code Reviewer dispatch`/`Code Reviewer result` evidence, or record `Reviewer dispatch`/`Reviewer result` only when using compatibility routing. In direct fallback, preserve fresh-review evidence and record `Code Reviewer direct evidence`; `Reviewer direct evidence` is compatibility evidence only. The `assistant-review` skill will: - Dispatch Code Reviewer subagents in delegated mode, using Reviewer compatibility only for existing handoffs, or preserve fresh-review evidence in direct fallback mode @@ -426,15 +426,15 @@ Print: `>> Stage 3: QA Evaluation — loading assistant-review references/qa-eva Run QA Evaluation for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. -The QA Evaluator receives Done Contract when present, acceptance criteria, verification evidence, code review result, domain_context/rubric_refs when applicable, round 1-20, previously_failed_acceptance_items, and qa_filter_policy. It loads assistant-review `references/domain-rubrics.md` only when acceptance criteria, Done Contract, domain_context, or explicit rubric_refs require subjective/product/UX/docs/DX/UI/domain scoring. It returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression or score_entry, evidence, and open_questions when blocked. +The QA Evaluator receives Done Contract when present, acceptance criteria, verification evidence, code review result, domain_context/rubric_refs when applicable, round 1-10, previously_failed_acceptance_items, and qa_filter_policy. It loads assistant-review `references/domain-rubrics.md` only when acceptance criteria, Done Contract, domain_context, or explicit rubric_refs require subjective/product/UX/docs/DX/UI/domain scoring. It returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression or score_entry, evidence, and open_questions when blocked. QA Evaluation focuses on acceptance and final result. It does not replace Code Reviewer, and it must not report general code defects, security issues, architecture concerns, or test coverage gaps unless they directly block an acceptance criterion or Done Contract item. If QA score progression reports STAGNATION, repeated DRIFT, repeated REGRESSION, or any scoped domain rubric returns action `pivot`, pause the QA loop and create an orchestrator-owned `pivot_restart_decision` before another QA/build dispatch. -Round 20 remains terminal: return the final QA verdict and remaining failed -acceptance items instead of starting round 21. +Round 10 remains terminal: return the final QA verdict and remaining failed +acceptance items instead of starting round 11. ### Status gate diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/prompts/pr-review.md b/plugins/assistant-dev/skills/assistant-workflow/references/prompts/pr-review.md index f64fce9..249506a 100755 --- a/plugins/assistant-dev/skills/assistant-workflow/references/prompts/pr-review.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/prompts/pr-review.md @@ -4,7 +4,7 @@ Load this during Stage 2 of the review cycle. This is the code quality gate — ## When to use -Run this review as part of the autonomous review-fix loop: after all build steps complete and tests pass. The loop runs up to 20 rounds, fixing must-fix and should-fix items each round until clean. You do NOT need to wait for user input between rounds. +Run this review as part of the autonomous review-fix loop: after all build steps complete and tests pass. The loop runs up to 10 rounds, fixing must-fix and should-fix items each round until clean. You do NOT need to wait for user input between rounds. For small tasks, do a quick pass through the "must-check" items only. For medium+ tasks, work through the full checklist. @@ -13,7 +13,7 @@ For small tasks, do a quick pass through the "must-check" items only. For medium When dispatching the Reviewer subagent for round N > 1, prepend this context to the prompt: ``` -## Previously Fixed (Round N of 20) +## Previously Fixed (Round N of 10) This is review round [N]. The following [count] items were already found and fixed in prior rounds — do NOT re-report them: @@ -23,8 +23,8 @@ and fixed in prior rounds — do NOT re-report them: Focus ONLY on NEW high-confidence findings not in the above list. Raise your confidence threshold each round: - Round 1-3: report findings at 80%+ confidence -- Round 4-15: report findings at 85%+ confidence -- Round 16-20: report findings at 90%+ confidence +- Round 4-7: report findings at 85%+ confidence +- Round 8-10: report findings at 90%+ confidence If you find no new issues above your confidence threshold, say "No new findings." ``` diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/subagent-roles.md b/plugins/assistant-dev/skills/assistant-workflow/references/subagent-roles.md index fae490f..40a87a6 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/subagent-roles.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/subagent-roles.md @@ -96,7 +96,7 @@ Use the code-reviewer agent. This is review round {N}. The following {count} items were already found and fixed — do NOT re-report them: {previously_fixed list} Focus ONLY on NEW high-confidence findings. -Confidence threshold: Round 1-3: 80%+ | Round 4-15: 85%+ | Round 16-20: 90%+ +Confidence threshold: Round 1-3: 80%+ | Round 4-7: 85%+ | Round 8-10: 90%+ ``` ## QA Evaluator dispatch (QA rounds) @@ -117,12 +117,12 @@ Do not replace Code Reviewer. Report only acceptance findings backed by evidence **QA Round N > 1:** ``` -Use the qa-evaluator agent. This is QA round {N} of 20. +Use the qa-evaluator agent. This is QA round {N} of 10. Previously failed acceptance items: {previously_failed_acceptance_items} -Do not re-report resolved items. In rounds 16-20, only unresolved acceptance +Do not re-report resolved items. In rounds 8-10, only unresolved acceptance blockers or high-confidence acceptance risks keep the QA loop open. -Round 20 is terminal; return the final verdict instead of implying round 21. +Round 10 is terminal; return the final verdict instead of implying round 11. ``` ## Fallback: agents not installed diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md index ded6a25..5d8eac2 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md @@ -254,7 +254,7 @@ Plan approval: [yes/no + date] - Required fixes: [none, or ordered fix list] ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Previously fixed: 0 items from prior rounds - Found this round: [count] must-fix, [count] should-fix, [count] nits (all fixed below) - Rubric: correctness=[score] quality=[score] architecture=[score] security=[score] coverage=[score] @@ -271,7 +271,7 @@ Plan approval: [yes/no + date] - Re-test: PASS ### Quality Review #2 (autonomous re-review) -- Round: 2 of 20 +- Round: 2 of 10 - Previously fixed: [count] items from prior rounds - Found this round: [count] must-fix, [count] should-fix, [count] nits (all fixed below) - Rubric: correctness=[score] quality=[score] architecture=[score] security=[score] coverage=[score] @@ -289,7 +289,7 @@ Plan approval: [yes/no + date] [Note: On test failure, skip this entry — write only "- Result: HAS_REMAINING_ITEMS" to Final result] ### QA Evaluation #1 -- Round: 1 of 20 +- Round: 1 of 10 - Mode: required | optional | not_required - Done Contract: [ref or N/A with reason] - Acceptance criteria checked: [count] @@ -306,7 +306,7 @@ Plan approval: [yes/no + date] - Acceptance findings: - [blocker|concern|observation] [criterion] — [evidence] → [fix/defer/open question] -[...repeat QA Evaluation until accepted, accepted_with_concerns, blocked, or max round 20 reached...] +[...repeat QA Evaluation until accepted, accepted_with_concerns, blocked, or max round 10 reached...] ### Final result - Result: CLEAN | ISSUES_FIXED | HAS_REMAINING_ITEMS diff --git a/skills/assistant-review/SKILL.md b/skills/assistant-review/SKILL.md index a773561..884eb43 100644 --- a/skills/assistant-review/SKILL.md +++ b/skills/assistant-review/SKILL.md @@ -1,6 +1,6 @@ --- name: assistant-review -description: "This skill runs an autonomous code review loop and optional independent QA evaluation loop: review, fix, re-review until clean (max 20 rounds), then evaluate acceptance when QA is required. Use when the user says 'review', 'fresh review', 'code review', 'review this', 'check the code'. Also activates when the workflow's Review phase requires quality review or QA evaluation dispatch." +description: "This skill runs an autonomous code review loop and optional independent QA evaluation loop: review, fix, re-review until clean (max 10 rounds), then evaluate acceptance when QA is required. Use when the user says 'review', 'fresh review', 'code review', 'review this', 'check the code'. Also activates when the workflow's Review phase requires quality review or QA evaluation dispatch." effort: high triggers: - pattern: "fix (all |the |review |reported )?issues|fix (all |the )?findings|apply (all )?fixes" @@ -140,7 +140,7 @@ round = 1 previously_fixed = [] score_history = [] -while round <= 20: +while round <= 10: 1. REVIEW - Dispatch a fresh Reviewer subagent when `subagent_execution_mode=delegated` @@ -165,7 +165,7 @@ while round <= 20: Finding filter policy: - Report only evidence-backed findings with file/line evidence, concrete impact, and the smallest useful fix. - Put speculative or low-evidence concerns in Observations; they do not block completion. - - In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers; round 20 is terminal and exits with remaining items instead of starting round 21. + - In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers; round 10 is terminal and exits with remaining items instead of starting round 11. Apply the SOLID, KISS, DRY, YAGNI, and readability lens from references/review-principles.md. Report principle issues only when tied to concrete evidence, concrete risk, and the smallest durable fix. @@ -193,8 +193,8 @@ while round <= 20: b. No rubric (small scope): use findings-based exit: - No must-fix AND no should-fix -> EXIT CLEAN - Only nits -> EXIT CLEAN (note nits in final report) - c. round == 20 with remaining must-fix or should-fix -> EXIT WITH REMAINING ITEMS - d. round == 20 with issues fixed and now clean -> EXIT ISSUES FIXED + c. round == 10 with remaining must-fix or should-fix -> EXIT WITH REMAINING ITEMS + d. round == 10 with issues fixed and now clean -> EXIT ISSUES FIXED e. Otherwise -> continue to step 3 Record in score_history: { round, weighted_score, finding_count, drift_status } @@ -206,7 +206,7 @@ while round <= 20: selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action before another fix/review dispatch. If the selected action changes scope, files, behavior, risk, verification, or acceptance criteria, - reapproval is required. Round 20 remains terminal and never starts round 21. + reapproval is required. Round 10 remains terminal and never starts round 11. 3. FIX - Fix ALL must-fix and should-fix items (not just must-fix) @@ -223,23 +223,9 @@ while round <= 20: ## The QA Evaluation Loop -Run this loop only when `qa_evaluation_mode=required` or the workflow Review phase requests QA evidence. Load `references/qa-evaluation-loop.md` before dispatching QAEvaluator. +Run QA only when `qa_evaluation_mode=required` or the workflow Review phase requests QA evidence. Load `references/qa-evaluation-loop.md` before dispatching QAEvaluator; that reference owns the detailed algorithm, score progression, domain-rubric routing, pivot/restart behavior, and terminal round-10 cap. -QA evaluation runs after code-review/build evidence. It is not an alternate path around the Code Reviewer loop. - -QA runs after build/test evidence and the Code Reviewer loop result exist. It must receive the Done Contract, acceptance criteria, verification evidence, domain_context/rubric_refs when applicable, round number, previously_failed_acceptance_items, and qa_filter_policy. It conditionally loads `references/domain-rubrics.md` only for scoped subjective/UI/visual/product/UX/docs/DX/domain acceptance. It returns status, round, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when used, final_verdict/result, evidence, score_progression or score_entry, and open_questions when blocked. - -The QA loop supports rounds 1-20. In rounds 16-20, only unresolved acceptance blockers or high-confidence acceptance risks keep the loop open. Round 20 is terminal: return the final verdict and remaining failed acceptance items instead of starting round 21. - -QA findings do not replace code-review findings. Do not use QAEvaluator to review general bugs, security, architecture, or test coverage unless that issue directly blocks an acceptance criterion or Done Contract item. - -If QA score progression reports STAGNATION, repeated DRIFT, repeated REGRESSION, -or a scoped domain rubric returns action `pivot`, pause QA and return a -pivot_restart_signal to the orchestrator. The orchestrator records -pivot_restart_decision before another QA/build dispatch, updates trace/replay/run -state when the workflow is harness-capable, and obtains reapproval when the -selected restart or pivot changes scope, files, behavior, risk, verification, or -acceptance criteria. +At this level, keep only the routing boundary: QA runs after build/test evidence and Code Reviewer or Reviewer compatibility evidence exist, and it evaluates acceptance criteria, Done Contract evidence, verification evidence, scoped domain quality, score progression, and final readiness. QA does not replace code review or report general code defects unless they directly block acceptance. ## Exit: Present Final Result @@ -283,7 +269,7 @@ After the loop completes, present ONE summary to the user: - **Autonomous continuation**: advance to the next round while exit criteria are unmet. - **Fresh Reviewer each round** on medium+ scope: stale context weakens reviews. - **Previously-fixed list prevents re-reporting**: each round should find fewer issues. -- **Evidence-backed filtering**: findings need file/line evidence, concrete impact, and the smallest useful fix; late rounds only stay open before the hard max for must-fix or high-confidence should-fix findings, and round 20 is terminal. +- **Evidence-backed filtering**: findings need file/line evidence, concrete impact, and the smallest useful fix; late rounds only stay open before the hard max for must-fix or high-confidence should-fix findings, and round 10 is terminal. - If scope is trivial (single small file, obvious change) -> one clean round can exit. If findings exist, continue looping. ## Output @@ -324,7 +310,7 @@ On DRIFT, the next Reviewer dispatch MUST include this addition to its prompt: On repeated DRIFT after the reset, stop the current review dispatch path and return pivot_restart_signal. The orchestrator owns the pivot_restart_decision and selects reset, candidate search, replan, restart, or a blocked/user path without -creating round 21 behavior. +creating round 11 behavior. ## Review Finding Rule Distillation diff --git a/skills/assistant-review/contracts/handoffs.yaml b/skills/assistant-review/contracts/handoffs.yaml index a4b41d5..6d483ee 100644 --- a/skills/assistant-review/contracts/handoffs.yaml +++ b/skills/assistant-review/contracts/handoffs.yaml @@ -73,8 +73,8 @@ handoffs: - name: round type: int required: true - description: "Current review round number (1-20)" - validation: ">= 1 and <= 20" + description: "Current review round number (1-10)" + validation: ">= 1 and <= 10" - name: previously_fixed type: object[] @@ -87,14 +87,14 @@ handoffs: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: finding_filter_policy type: string required: true description: "Evidence-backed finding filter for this round" - validation: "Must state that findings require file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are Observations and do not block; round 20 remains terminal." - notes: "In rounds 16-20, only must-fix or high-confidence should-fix findings count as blockers. The loop never starts round 21." + validation: "Must state that findings require file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are Observations and do not block; round 10 remains terminal." + notes: "In rounds 8-10, only must-fix or high-confidence should-fix findings count as blockers. The loop never starts round 11." - name: project_type type: string @@ -157,7 +157,7 @@ handoffs: type: int required: true description: "Echo back the round number for tracking" - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: findings type: object[] @@ -448,7 +448,7 @@ handoffs: - name: affected_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: recommended_recovery_focus type: string required: true @@ -548,7 +548,7 @@ handoffs: - name: rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: remaining_risks type: string[] required: true @@ -577,8 +577,8 @@ handoffs: - name: round type: int required: true - description: "Current QA evaluation round number (1-20)" - validation: ">= 1 and <= 20" + description: "Current QA evaluation round number (1-10)" + validation: ">= 1 and <= 10" - name: previously_failed_acceptance_items type: object[] @@ -591,7 +591,7 @@ handoffs: - name: failed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: evidence_gap type: string required: true @@ -600,7 +600,7 @@ handoffs: type: string required: true description: "Evidence-backed QA finding filter for this round" - validation: "Must state that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 20 is terminal." + validation: "Must state that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 10 is terminal." return_fields: - name: status @@ -613,7 +613,7 @@ handoffs: type: int required: true description: "Echo back the QA round number" - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: acceptance_findings type: object[] @@ -747,7 +747,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true @@ -770,7 +770,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true @@ -812,7 +812,7 @@ handoffs: - name: affected_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: recommended_recovery_focus type: string required: true @@ -831,10 +831,10 @@ handoffs: # 7. If rubric_required: verify weighted_score matches the dimension * weight calculation # 8. If quality_principles_required: verify the Reviewer applied references/review-principles.md # 9. If pivot_restart_signal is present, stop the current loop path until the orchestrator records pivot_restart_decision; do not silently continue. -# 10. If round == 20, exit with CLEAN, ISSUES_FIXED, or HAS_REMAINING_ITEMS; never start round 21 +# 10. If round == 10, exit with CLEAN, ISSUES_FIXED, or HAS_REMAINING_ITEMS; never start round 11 # 11. If delegated re-dispatch or direct-fallback completion is needed: include explicit note about what was missing # 12. If qa_evaluation_mode=required, run orchestrator_to_qa_evaluator after build/test and code-review evidence; include debate_record when Done Contract exists; never treat QA evaluation as a replacement for Reviewer/code-reviewer. -# 13. If QA round == 20, return accepted, accepted_with_concerns, rejected, or blocked with remaining failed acceptance items; never start QA round 21. +# 13. If QA round == 10, return accepted, accepted_with_concerns, rejected, or blocked with remaining failed acceptance items; never start QA round 11. # 14. If domain_context/rubric_refs or subjective/product/UX/docs/DX/UI/domain acceptance scope exists: verify selected_domain_rubrics and domain_quality_scores are present and evidence-backed. # 15. If no scoped domain rubric applies: verify QAEvaluator did not invent rubrics or penalize domain_quality for missing unscoped evidence. # ───────────────────────────────────────────── diff --git a/skills/assistant-review/contracts/input.yaml b/skills/assistant-review/contracts/input.yaml index 455f98a..67467bb 100644 --- a/skills/assistant-review/contracts/input.yaml +++ b/skills/assistant-review/contracts/input.yaml @@ -206,7 +206,7 @@ fields: required: conditional condition: "qa_evaluation_mode == required" description: "Evidence filter QAEvaluator must apply to acceptance findings" - validation: "States that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 20 is terminal" + validation: "States that QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns are non-blocking; round 10 is terminal" on_missing: infer infer_from: "Use the standard QA filter policy from references/qa-evaluation-loop.md." @@ -279,7 +279,7 @@ fields: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" # Validation behavior: # 1. Resolve review_scope, review_material_snapshot, change_kind, semantic_contract_review_required, behavioral_contract_review_required, and agentic_loop_safety_review_required before entering the loop diff --git a/skills/assistant-review/contracts/output.yaml b/skills/assistant-review/contracts/output.yaml index 2b8ca43..1d6aafb 100644 --- a/skills/assistant-review/contracts/output.yaml +++ b/skills/assistant-review/contracts/output.yaml @@ -156,12 +156,12 @@ artifacts: required: true condition: always description: "The single summary presented to the user after the loop completes" - validation: "Must include rounds (1-20), result, fixed_items, remaining_items when applicable, and nits before presenting review results" + validation: "Must include rounds (1-10), result, fixed_items, remaining_items when applicable, and nits before presenting review results" object_fields: - name: rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: result type: enum required: true @@ -184,7 +184,7 @@ artifacts: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: remaining_items type: object[] required: false @@ -221,13 +221,13 @@ artifacts: required: conditional condition: "qa_evaluation_mode == required" description: "Independent QA evaluation final result after build/test and code-review evidence" - validation: "Must include rounds (1-20), final_verdict, result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, evidence, and open_questions when blocked" + validation: "Must include rounds (1-10), final_verdict, result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, evidence, and open_questions when blocked" on_fail: "Run the QA evaluation loop from references/qa-evaluation-loop.md or record why QA is blocked before presenting final review results" object_fields: - name: rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: final_verdict type: enum required: true @@ -310,7 +310,7 @@ artifacts: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true @@ -374,12 +374,12 @@ artifacts: - name: score_progression type: object[] required: true - description: "Weighted score per round — used for drift detection and capped at the 20-round review limit" + description: "Weighted score per round — used for drift detection and capped at the 10-round review limit" object_fields: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true diff --git a/skills/assistant-review/contracts/phase-gates.yaml b/skills/assistant-review/contracts/phase-gates.yaml index c1c6083..55ad66a 100644 --- a/skills/assistant-review/contracts/phase-gates.yaml +++ b/skills/assistant-review/contracts/phase-gates.yaml @@ -115,14 +115,14 @@ gates: - id: EV1 check: "Exit decision is one of: EXIT_CLEAN, EXIT_ISSUES_FIXED, EXIT_MAX_ROUNDS, ESCALATE_PIVOT_RESTART, CONTINUE" - on_fail: "Re-evaluate findings against exit criteria. EXIT_CLEAN = no must-fix/should-fix (nits OK, note in report) and, for medium+ scope, weighted_score >= 4.0. EXIT_ISSUES_FIXED = round 20, all issues fixed this session, no must-fix/should-fix findings, and, for medium+ scope, weighted_score >= 4.0. EXIT_MAX_ROUNDS = round 20 with remaining must-fix or should-fix findings. ESCALATE_PIVOT_RESTART = STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT." + on_fail: "Re-evaluate findings against exit criteria. EXIT_CLEAN = no must-fix/should-fix (nits OK, note in report) and, for medium+ scope, weighted_score >= 4.0. EXIT_ISSUES_FIXED = round 10, all issues fixed this session, no must-fix/should-fix findings, and, for medium+ scope, weighted_score >= 4.0. EXIT_MAX_ROUNDS = round 10 with remaining must-fix or should-fix findings. ESCALATE_PIVOT_RESTART = STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT." - id: EV2 check: "EXIT_CLEAN or EXIT_ISSUES_FIXED only if: zero must-fix AND zero should-fix findings, and for medium+ scope weighted_score >= 4.0" on_fail: "Findings exist or medium+ weighted_score is below 4.0 — apply rubric action first. REFINE continues to FIX; PIVOT escalates through pivot_restart_decision." - id: EV3 - check: "EXIT_MAX_ROUNDS only if: round == 20" + check: "EXIT_MAX_ROUNDS only if: round == 10" on_fail: "Rounds remain — must CONTINUE to FIX step" - id: EV4 @@ -214,8 +214,8 @@ gates: on_fail: "Move generic code/security/architecture/test-coverage findings back to Code Reviewer unless they directly block an acceptance criterion or Done Contract item." - id: QA5 - check: "QA loop exit decision is accepted, accepted_with_concerns, rejected, or blocked; round 20 is terminal and never starts round 21" - on_fail: "Apply references/qa-evaluation-loop.md terminal behavior. If round < 20 and rejected, return failed acceptance items to Build; if round == 20, record remaining failed acceptance items in qa_evaluation_result." + check: "QA loop exit decision is accepted, accepted_with_concerns, rejected, or blocked; round 10 is terminal and never starts round 11" + on_fail: "Apply references/qa-evaluation-loop.md terminal behavior. If round < 10 and rejected, return failed acceptance items to Build; if round == 10, record remaining failed acceptance items in qa_evaluation_result." - id: QA6 check: "Domain rubrics from references/domain-rubrics.md are loaded and applied only when scoped by acceptance criteria, Done Contract, domain_context, or explicit rubric_refs" @@ -225,7 +225,7 @@ gates: - id: QA7 check: "QA STAGNATION, repeated DRIFT, repeated REGRESSION, or scoped domain action pivot triggers pivot_restart_signal and orchestrator-owned pivot_restart_decision before another QA/build dispatch" condition: "qa_evaluation_mode == required" - on_fail: "Pause QA, create pivot_restart_signal from score_entry, score_progression, or domain_quality_scores action=pivot, and require pivot_restart_decision with exact_next_action. Round 20 remains terminal and never starts round 21." + on_fail: "Pause QA, create pivot_restart_signal from score_entry, score_progression, or domain_quality_scores action=pivot, and require pivot_restart_decision with exact_next_action. Round 10 remains terminal and never starts round 11." # ───────────────────────────────────────────── # EXIT — after loop completes @@ -269,9 +269,9 @@ invariants: on_fail: "Restore any removed items to previously_fixed. This list is append-only." - id: INV2 - check: "Finding filter policy requires file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are non-blocking Observations; round 20 is terminal" + check: "Finding filter policy requires file/line evidence, concrete impact, and the smallest useful fix; speculative concerns are non-blocking Observations; round 10 is terminal" scope: all_rounds - on_fail: "Rebuild finding_filter_policy from the evidence-backed rule and hard max-20 cap before dispatching review." + on_fail: "Rebuild finding_filter_policy from the evidence-backed rule and hard max-10 cap before dispatching review." - id: INV3 check: "The agent never pauses between rounds to ask the user — the loop is autonomous" @@ -330,10 +330,10 @@ invariants: on_fail: "Run code review first and record QAEvaluator only as the independent acceptance lane." - id: INV_QA2 - check: "QA evaluation round is always between 1 and 20, and round 20 is terminal" + check: "QA evaluation round is always between 1 and 10, and round 10 is terminal" scope: all_rounds condition: "qa_evaluation_mode == required" - on_fail: "Clamp QA execution to the 20-round cap and return the final verdict with remaining failed acceptance items instead of starting round 21." + on_fail: "Clamp QA execution to the 10-round cap and return the final verdict with remaining failed acceptance items instead of starting round 11." - id: INV_QA3 check: "QAEvaluator does not invent domain rubrics, subjective standards, or domain-quality penalties when no acceptance criteria, Done Contract, domain_context, or rubric_refs scope them" diff --git a/skills/assistant-review/evals/cases.json b/skills/assistant-review/evals/cases.json index 1d9ec63..354c5f9 100644 --- a/skills/assistant-review/evals/cases.json +++ b/skills/assistant-review/evals/cases.json @@ -30,7 +30,7 @@ "Runs review rounds with confidence thresholds and previously_fixed tracking.", "Fixes all must-fix and should-fix findings before validation.", "Runs build or test verification before the next round.", - "Continues review, fix, validate, and re-review until clean or max 20 rounds." + "Continues review, fix, validate, and re-review until clean or max 10 rounds." ], "pass_criteria": [ "The response identifies review scope and mode.", @@ -78,7 +78,7 @@ "expected_behavior": [ "Requires rubric scores for correctness, code quality, architecture, security, and test coverage.", "Applies a weighted score threshold action such as PASS, REFINE, or PIVOT.", - "Uses evidence-backed finding filtering across rounds 1-20 and treats round 20 as the hard cap.", + "Uses evidence-backed finding filtering across rounds 1-10 and treats round 10 as the hard cap.", "Buffers intermediate round details and presents one final summary after loop exit.", "Includes score progression when multiple rounds occur.", "Does not ask the user between rounds unless blocked." diff --git a/skills/assistant-review/references/qa-evaluation-loop.md b/skills/assistant-review/references/qa-evaluation-loop.md index f51f9b0..496dd0c 100644 --- a/skills/assistant-review/references/qa-evaluation-loop.md +++ b/skills/assistant-review/references/qa-evaluation-loop.md @@ -21,7 +21,7 @@ The orchestrator provides: - `code_review_result`: final Code Reviewer or Reviewer compatibility result. - `domain_context`: scoped UI/visual, product, UX, docs, DX, or domain notes when applicable. - `rubric_refs`: explicit references to scoped domain rubric families from `references/domain-rubrics.md` when applicable. -- `round`: QA round number from 1 to 20. +- `round`: QA round number from 1 to 10. - `previously_failed_acceptance_items`: failed acceptance items from earlier QA rounds. - `qa_filter_policy`: acceptance findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists; speculative concerns stay non-blocking. @@ -47,7 +47,7 @@ round = 1 previously_failed_acceptance_items = [] score_progression = [] -while round <= 20: +while round <= 10: 1. EVALUATE ACCEPTANCE Dispatch qa-evaluator in delegated mode, or use direct fallback with fresh QA context. Check every acceptance criterion, Done Contract item, and Done Contract debate_record independently when a Done Contract exists. @@ -70,10 +70,10 @@ while round <= 20: blocked: required acceptance material or verification evidence is missing. 4. FIX OR EXIT - If rejected before round 20, return failed acceptance items to Build for fixes, then re-run QA. + If rejected before round 10, return failed acceptance items to Build for fixes, then re-run QA. If blocked, return NEEDS_CONTEXT or BLOCKED with open_questions. If accepted or accepted_with_concerns, exit with final result. - Round 20 is terminal; return remaining failed acceptance items instead of starting round 21. + Round 10 is terminal; return remaining failed acceptance items instead of starting round 11. If pivot_restart_signal was returned, pause QA until the orchestrator records pivot_restart_decision with exact_next_action. ``` @@ -92,7 +92,7 @@ Do not report generic code defects, architecture concerns, security issues, or t ## Final Result The QA loop returns one final result: -- `rounds`: 1-20. +- `rounds`: 1-10. - `final_verdict`: accepted, accepted_with_concerns, rejected, or blocked. - `result`: CLEAN, ISSUES_FIXED, HAS_REMAINING_ITEMS, or BLOCKED. - `acceptance_findings`: remaining or resolved acceptance findings with evidence. diff --git a/skills/assistant-review/references/review-rubric.md b/skills/assistant-review/references/review-rubric.md index 36c864b..5a41070 100644 --- a/skills/assistant-review/references/review-rubric.md +++ b/skills/assistant-review/references/review-rubric.md @@ -107,9 +107,9 @@ As the loop progresses, the bar rises — like a tightening ratchet: | 1 | 4.0+ | 2.5–3.9 | < 2.5 | | 2 | 4.0+ | 2.75–3.9 | < 2.75 | | 3 | 4.0+ | 3.0–3.9 | < 3.0 | -| 4-20 | 4.0+ | 3.25–3.9 | < 3.25 | +| 4-10 | 4.0+ | 3.25–3.9 | < 3.25 | -Rationale: If the code hasn't reached 3.25 by round 4, the approach likely needs rethinking, not more polish. Rounds 4-20 keep that pivot threshold stable while confidence requirements continue to rise. +Rationale: If the code hasn't reached 3.25 by round 4, the approach likely needs rethinking, not more polish. Rounds 4-10 keep that pivot threshold stable while confidence requirements continue to rise. ## Reviewer Scoring Instructions diff --git a/skills/assistant-review/references/score-tracking.md b/skills/assistant-review/references/score-tracking.md index a90d1c8..3d6ca57 100644 --- a/skills/assistant-review/references/score-tracking.md +++ b/skills/assistant-review/references/score-tracking.md @@ -106,7 +106,7 @@ Record scores in each Quality Review entry: ```markdown ### Quality Review #N -- Round: N of 20 +- Round: N of 10 - Previously fixed: M items from prior rounds - Found this round: X must-fix, Y should-fix, Z nits - Rubric: correctness=4.0 quality=3.5 architecture=4.0 security=5.0 coverage=3.0 @@ -156,5 +156,5 @@ fires, the review loop does not silently continue. It returns If the selected action changes approved scope, files, behavior, risk, verification, or acceptance criteria, `reapproval_required` is true and the -workflow waits for approval before Build or Review continues. Round 20 remains -terminal; do not start round 21. +workflow waits for approval before Build or Review continues. Round 10 remains +terminal; do not start round 11. diff --git a/skills/assistant-skill-creator/evals/cases.json b/skills/assistant-skill-creator/evals/cases.json index 0481666..bc52cdb 100644 --- a/skills/assistant-skill-creator/evals/cases.json +++ b/skills/assistant-skill-creator/evals/cases.json @@ -239,7 +239,7 @@ "Keeps Code Reviewer and QA Evaluator handoffs and outputs separate.", "Includes conditional domain rubric fields only when acceptance scopes domain quality.", "Adds pivot/restart decision fields for stagnation, drift, regression, rubric pivot, and Code Writer blockers.", - "Bounds review and QA loops with max 20 rounds and terminal round 20 behavior." + "Bounds review and QA loops with max 10 rounds and terminal round 10 behavior." ], "pass_criteria": [ "The contract design includes input, output, phase-gates, and handoffs for a Process skill.", @@ -250,7 +250,7 @@ "fail_signals": [ "Creates files before presenting the Process contract design.", "Uses only a generic generator/evaluator loop.", - "Sets a max 10 round cap for the review or QA loop.", + "Sets a max 20 round cap for the review or QA loop.", "Omits pivot/restart handling for stagnation or Code Writer blockers." ], "machine_expectations": { @@ -263,11 +263,11 @@ "Code Reviewer", "QA Evaluator", "pivot_restart_decision", - "max 20" + "max 10" ], "forbidden_substrings": [ "generic generator/evaluator loop is enough", - "max 10", + "max 20", "QA replaces code review" ] } diff --git a/skills/assistant-skill-creator/references/harness-patterns.md b/skills/assistant-skill-creator/references/harness-patterns.md index 436af52..87fac42 100644 --- a/skills/assistant-skill-creator/references/harness-patterns.md +++ b/skills/assistant-skill-creator/references/harness-patterns.md @@ -172,24 +172,24 @@ Contract placement: ## Pattern 7: Bounded Review / QA Loops Any autonomous review, QA, refinement, or fix-verify loop needs a hard cap. The -current framework cap is **max 20 rounds**. +current framework cap is **max 10 rounds**. ```text round = 1 -while round <= 20: +while round <= 10: evaluate decide PASS / REFINE / PIVOT fix or exit round += 1 ``` -Round 20 is terminal. Return remaining blockers, findings, or failed acceptance -items instead of starting round 21. +Round 10 is terminal. Return remaining blockers, findings, or failed acceptance +items instead of starting round 11. Include these fields where applicable: -- `round`: current round number, validation `>= 1 and <= 20`. -- `max_rounds`: default `20`. +- `round`: current round number, validation `>= 1 and <= 10`. +- `max_rounds`: default `10`. - `previously_fixed` or `previously_failed_acceptance_items`. - `score_progression` or `score_entry`. - `loop_exit_reason`. @@ -296,7 +296,7 @@ For a loop-based Process skill, require the matching patterns: | Do artifacts cross role boundaries? | Typed Artifact References | | Are code quality and acceptance both evaluated? | Code Reviewer / QA Evaluator split | | Is subjective/domain quality part of acceptance? | Conditional Domain Rubrics | -| Does a loop run multiple rounds? | Max 20 loop cap, scoring, drift detection | +| Does a loop run multiple rounds? | Max 10 loop cap, scoring, drift detection | | Can the approach go stale or hit legacy blockers? | Pivot/Restart Decision | Most loop-based Process skills use all of these. Simple retry-only skills still diff --git a/skills/assistant-workflow/contracts/handoffs.yaml b/skills/assistant-workflow/contracts/handoffs.yaml index 5d9bb4d..baf5ba2 100644 --- a/skills/assistant-workflow/contracts/handoffs.yaml +++ b/skills/assistant-workflow/contracts/handoffs.yaml @@ -1380,8 +1380,8 @@ handoffs: - name: round type: int required: true - description: "Current review round number (1-20)" - validation: ">= 1 and <= 20" + description: "Current review round number (1-10)" + validation: ">= 1 and <= 10" - name: previously_fixed type: object[] required: true @@ -1393,13 +1393,13 @@ handoffs: - name: fixed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: confidence_threshold type: int required: true description: "Minimum confidence percentage for reporting findings" validation: "Must be one of: 80, 85, 90" - notes: "Round 1-3: 80, Round 4-15: 85, Round 16-20: 90" + notes: "Round 1-3: 80, Round 4-7: 85, Round 8-10: 90" return_fields: - name: status type: enum @@ -1409,7 +1409,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: findings type: object[] required: true @@ -1548,7 +1548,7 @@ handoffs: - name: rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: domain_context type: object required: false @@ -1568,8 +1568,8 @@ handoffs: - name: round type: int required: true - description: "Current QA evaluation round number (1-20)" - validation: ">= 1 and <= 20" + description: "Current QA evaluation round number (1-10)" + validation: ">= 1 and <= 10" - name: previously_failed_acceptance_items type: object[] required: true @@ -1581,14 +1581,14 @@ handoffs: - name: failed_in_round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: evidence_gap type: string required: true - name: qa_filter_policy type: string required: true - description: "Acceptance findings require evidence and direct acceptance impact; speculative concerns are non-blocking; round 20 is terminal" + description: "Acceptance findings require evidence and direct acceptance impact; speculative concerns are non-blocking; round 10 is terminal" return_fields: - name: status type: enum @@ -1598,7 +1598,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: acceptance_findings type: object[] required: true @@ -1692,7 +1692,7 @@ handoffs: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true diff --git a/skills/assistant-workflow/contracts/output.yaml b/skills/assistant-workflow/contracts/output.yaml index f913222..fe50c33 100644 --- a/skills/assistant-workflow/contracts/output.yaml +++ b/skills/assistant-workflow/contracts/output.yaml @@ -226,7 +226,7 @@ artifacts: - name: review_rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: must_fix_resolved type: int required: true @@ -243,7 +243,7 @@ artifacts: type: object required: conditional condition: "qa_evaluation_mode == required or size in [medium, large, mega] and harness_capable == true and work is domain-scored/UI/visual/product/UX/docs/DX-facing" - validation: "Independent QA evaluation result exists after build/test evidence and Code Reviewer result. Must include final_verdict, result, rounds (1-20), acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, and evidence." + validation: "Independent QA evaluation result exists after build/test evidence and Code Reviewer result. Must include final_verdict, result, rounds (1-10), acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, and evidence." on_fail: "Run the QAEvaluator loop from assistant-review references/qa-evaluation-loop.md, or record NEEDS_CONTEXT/BLOCKED with open_questions. Do not substitute Code Reviewer findings for QA evaluation." object_fields: - name: final_verdict @@ -257,7 +257,7 @@ artifacts: - name: rounds type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: acceptance_findings type: object[] required: true @@ -329,7 +329,7 @@ artifacts: - name: round type: int required: true - validation: ">= 1 and <= 20" + validation: ">= 1 and <= 10" - name: weighted_score type: float required: true @@ -1023,7 +1023,7 @@ artifacts: type: enum required: true enum_values: [reset_context, return_to_build, dispatch_debugging, dispatch_explorer, dispatch_architect, run_candidate_search, replan, restart_slice, restart_phase, block_for_user, accept_with_limitations] - validation: "Concrete recovery action selected by the orchestrator; it must not imply starting round 21 after a terminal round 20" + validation: "Concrete recovery action selected by the orchestrator; it must not imply starting round 11 after a terminal round 10" on_missing: fail - name: reapproval_required type: boolean diff --git a/skills/assistant-workflow/contracts/phase-gates.yaml b/skills/assistant-workflow/contracts/phase-gates.yaml index f89bf47..b5935e5 100644 --- a/skills/assistant-workflow/contracts/phase-gates.yaml +++ b/skills/assistant-workflow/contracts/phase-gates.yaml @@ -463,7 +463,7 @@ gates: on_fail: "Load assistant-review references/qa-evaluation-loop.md, dispatch QAEvaluator or record direct-fallback QA evidence, and do not complete Review until QA result is present or blocked with open_questions. If domain_context/rubric_refs or subjective/product/UX/docs/DX/UI/domain acceptance scope exists, require selected_domain_rubrics and domain_quality_scores; otherwise reject invented rubric scoring." - id: R_PIVOT_RESTART_DECISION - check: "Review or QA STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric/domain action pivot records an orchestrator-owned pivot_restart_decision before another fix, review, QA, or build dispatch; round 20 remains terminal and never creates round 21 behavior" + check: "Review or QA STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric/domain action pivot records an orchestrator-owned pivot_restart_decision before another fix, review, QA, or build dispatch; round 10 remains terminal and never creates round 11 behavior" on_fail: "Pause Review, create pivot_restart_decision with trigger, evidence, affected_slice_or_round, options_considered, selected_action, reapproval_required, next_agent, recovery_pointer, and exact_next_action. Update Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger when harness-capable before continuing." - id: R3 diff --git a/skills/assistant-workflow/references/harness-controller.md b/skills/assistant-workflow/references/harness-controller.md index adb5b8c..77a6d7c 100644 --- a/skills/assistant-workflow/references/harness-controller.md +++ b/skills/assistant-workflow/references/harness-controller.md @@ -159,7 +159,7 @@ Required decision fields: When a decision is created, update Harness Run State, append a `pivot_restart` Trace Ledger entry, refresh Replay Packet so `exact_next_action` matches the -decision, and add or update the Pivot/Restart Decision artifact ref. Round 20 remains terminal; the controller never creates round 21 behavior. +decision, and add or update the Pivot/Restart Decision artifact ref. Round 10 remains terminal; the controller never creates round 11 behavior. Routing rules: diff --git a/skills/assistant-workflow/references/phases.md b/skills/assistant-workflow/references/phases.md index 450f4e0..b589e07 100644 --- a/skills/assistant-workflow/references/phases.md +++ b/skills/assistant-workflow/references/phases.md @@ -408,7 +408,7 @@ Print: `>> Spec Review: [PASS / FAIL — found N required fixes]` Print: `>> Stage 2: Code Quality Review — loading assistant-review SKILL.md` -**Load and follow `assistant-review` SKILL.md and its contracts.** This runs the autonomous review-fix loop (max 20 rounds) with visible progress. Add Code Reviewer to `Required agents` before Stage 2; use `Reviewer` only as compatibility routing for existing/legacy handoffs. Dispatch the `code-reviewer` agent for code defects, security, architecture, test coverage, and structural code issues. Do NOT implement the review loop inline when `subagent_execution_mode=delegated` and a delegated review agent is authorized — dispatch the Code Reviewer subagent and record `Code Reviewer dispatch`/`Code Reviewer result` evidence, or record `Reviewer dispatch`/`Reviewer result` only when using compatibility routing. In direct fallback, preserve fresh-review evidence and record `Code Reviewer direct evidence`; `Reviewer direct evidence` is compatibility evidence only. +**Load and follow `assistant-review` SKILL.md and its contracts.** This runs the autonomous review-fix loop (max 10 rounds) with visible progress. Add Code Reviewer to `Required agents` before Stage 2; use `Reviewer` only as compatibility routing for existing/legacy handoffs. Dispatch the `code-reviewer` agent for code defects, security, architecture, test coverage, and structural code issues. Do NOT implement the review loop inline when `subagent_execution_mode=delegated` and a delegated review agent is authorized — dispatch the Code Reviewer subagent and record `Code Reviewer dispatch`/`Code Reviewer result` evidence, or record `Reviewer dispatch`/`Reviewer result` only when using compatibility routing. In direct fallback, preserve fresh-review evidence and record `Code Reviewer direct evidence`; `Reviewer direct evidence` is compatibility evidence only. The `assistant-review` skill will: - Dispatch Code Reviewer subagents in delegated mode, using Reviewer compatibility only for existing handoffs, or preserve fresh-review evidence in direct fallback mode @@ -426,15 +426,15 @@ Print: `>> Stage 3: QA Evaluation — loading assistant-review references/qa-eva Run QA Evaluation for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. -The QA Evaluator receives Done Contract when present, acceptance criteria, verification evidence, code review result, domain_context/rubric_refs when applicable, round 1-20, previously_failed_acceptance_items, and qa_filter_policy. It loads assistant-review `references/domain-rubrics.md` only when acceptance criteria, Done Contract, domain_context, or explicit rubric_refs require subjective/product/UX/docs/DX/UI/domain scoring. It returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression or score_entry, evidence, and open_questions when blocked. +The QA Evaluator receives Done Contract when present, acceptance criteria, verification evidence, code review result, domain_context/rubric_refs when applicable, round 1-10, previously_failed_acceptance_items, and qa_filter_policy. It loads assistant-review `references/domain-rubrics.md` only when acceptance criteria, Done Contract, domain_context, or explicit rubric_refs require subjective/product/UX/docs/DX/UI/domain scoring. It returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression or score_entry, evidence, and open_questions when blocked. QA Evaluation focuses on acceptance and final result. It does not replace Code Reviewer, and it must not report general code defects, security issues, architecture concerns, or test coverage gaps unless they directly block an acceptance criterion or Done Contract item. If QA score progression reports STAGNATION, repeated DRIFT, repeated REGRESSION, or any scoped domain rubric returns action `pivot`, pause the QA loop and create an orchestrator-owned `pivot_restart_decision` before another QA/build dispatch. -Round 20 remains terminal: return the final QA verdict and remaining failed -acceptance items instead of starting round 21. +Round 10 remains terminal: return the final QA verdict and remaining failed +acceptance items instead of starting round 11. ### Status gate diff --git a/skills/assistant-workflow/references/prompts/pr-review.md b/skills/assistant-workflow/references/prompts/pr-review.md index f64fce9..249506a 100755 --- a/skills/assistant-workflow/references/prompts/pr-review.md +++ b/skills/assistant-workflow/references/prompts/pr-review.md @@ -4,7 +4,7 @@ Load this during Stage 2 of the review cycle. This is the code quality gate — ## When to use -Run this review as part of the autonomous review-fix loop: after all build steps complete and tests pass. The loop runs up to 20 rounds, fixing must-fix and should-fix items each round until clean. You do NOT need to wait for user input between rounds. +Run this review as part of the autonomous review-fix loop: after all build steps complete and tests pass. The loop runs up to 10 rounds, fixing must-fix and should-fix items each round until clean. You do NOT need to wait for user input between rounds. For small tasks, do a quick pass through the "must-check" items only. For medium+ tasks, work through the full checklist. @@ -13,7 +13,7 @@ For small tasks, do a quick pass through the "must-check" items only. For medium When dispatching the Reviewer subagent for round N > 1, prepend this context to the prompt: ``` -## Previously Fixed (Round N of 20) +## Previously Fixed (Round N of 10) This is review round [N]. The following [count] items were already found and fixed in prior rounds — do NOT re-report them: @@ -23,8 +23,8 @@ and fixed in prior rounds — do NOT re-report them: Focus ONLY on NEW high-confidence findings not in the above list. Raise your confidence threshold each round: - Round 1-3: report findings at 80%+ confidence -- Round 4-15: report findings at 85%+ confidence -- Round 16-20: report findings at 90%+ confidence +- Round 4-7: report findings at 85%+ confidence +- Round 8-10: report findings at 90%+ confidence If you find no new issues above your confidence threshold, say "No new findings." ``` diff --git a/skills/assistant-workflow/references/subagent-roles.md b/skills/assistant-workflow/references/subagent-roles.md index fae490f..40a87a6 100644 --- a/skills/assistant-workflow/references/subagent-roles.md +++ b/skills/assistant-workflow/references/subagent-roles.md @@ -96,7 +96,7 @@ Use the code-reviewer agent. This is review round {N}. The following {count} items were already found and fixed — do NOT re-report them: {previously_fixed list} Focus ONLY on NEW high-confidence findings. -Confidence threshold: Round 1-3: 80%+ | Round 4-15: 85%+ | Round 16-20: 90%+ +Confidence threshold: Round 1-3: 80%+ | Round 4-7: 85%+ | Round 8-10: 90%+ ``` ## QA Evaluator dispatch (QA rounds) @@ -117,12 +117,12 @@ Do not replace Code Reviewer. Report only acceptance findings backed by evidence **QA Round N > 1:** ``` -Use the qa-evaluator agent. This is QA round {N} of 20. +Use the qa-evaluator agent. This is QA round {N} of 10. Previously failed acceptance items: {previously_failed_acceptance_items} -Do not re-report resolved items. In rounds 16-20, only unresolved acceptance +Do not re-report resolved items. In rounds 8-10, only unresolved acceptance blockers or high-confidence acceptance risks keep the QA loop open. -Round 20 is terminal; return the final verdict instead of implying round 21. +Round 10 is terminal; return the final verdict instead of implying round 11. ``` ## Fallback: agents not installed diff --git a/skills/assistant-workflow/references/task-journal-template.md b/skills/assistant-workflow/references/task-journal-template.md index ded6a25..5d8eac2 100644 --- a/skills/assistant-workflow/references/task-journal-template.md +++ b/skills/assistant-workflow/references/task-journal-template.md @@ -254,7 +254,7 @@ Plan approval: [yes/no + date] - Required fixes: [none, or ordered fix list] ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Previously fixed: 0 items from prior rounds - Found this round: [count] must-fix, [count] should-fix, [count] nits (all fixed below) - Rubric: correctness=[score] quality=[score] architecture=[score] security=[score] coverage=[score] @@ -271,7 +271,7 @@ Plan approval: [yes/no + date] - Re-test: PASS ### Quality Review #2 (autonomous re-review) -- Round: 2 of 20 +- Round: 2 of 10 - Previously fixed: [count] items from prior rounds - Found this round: [count] must-fix, [count] should-fix, [count] nits (all fixed below) - Rubric: correctness=[score] quality=[score] architecture=[score] security=[score] coverage=[score] @@ -289,7 +289,7 @@ Plan approval: [yes/no + date] [Note: On test failure, skip this entry — write only "- Result: HAS_REMAINING_ITEMS" to Final result] ### QA Evaluation #1 -- Round: 1 of 20 +- Round: 1 of 10 - Mode: required | optional | not_required - Done Contract: [ref or N/A with reason] - Acceptance criteria checked: [count] @@ -306,7 +306,7 @@ Plan approval: [yes/no + date] - Acceptance findings: - [blocker|concern|observation] [criterion] — [evidence] → [fix/defer/open question] -[...repeat QA Evaluation until accepted, accepted_with_concerns, blocked, or max round 20 reached...] +[...repeat QA Evaluation until accepted, accepted_with_concerns, blocked, or max round 10 reached...] ### Final result - Result: CLEAN | ISSUES_FIXED | HAS_REMAINING_ITEMS diff --git a/tests/p0-p4/assistant-review-reference-contracts.sh b/tests/p0-p4/assistant-review-reference-contracts.sh index f5db41f..8647873 100644 --- a/tests/p0-p4/assistant-review-reference-contracts.sh +++ b/tests/p0-p4/assistant-review-reference-contracts.sh @@ -113,7 +113,7 @@ for file_and_term in \ "$review_skill::REFINE (weighted below 4.0 but not PIVOT), including zero findings -> continue to step 3" \ "$review_skill::Medium+ CLEAN and ISSUES_FIXED require weighted >= 4.0" \ "$review_phase_gates::EXIT_CLEAN or EXIT_ISSUES_FIXED only if: zero must-fix AND zero should-fix findings, and for medium+ scope weighted_score >= 4.0" \ - "$review_rubric::| 4-20 | 4.0+ | 3.25"; do + "$review_rubric::| 4-10 | 4.0+ | 3.25"; do file="${file_and_term%%::*}" term="${file_and_term#*::}" if [[ ! -f "$file" ]] || ! grep -Fq "$term" "$file"; then diff --git a/tests/p0-p4/domain-rubrics-contracts.sh b/tests/p0-p4/domain-rubrics-contracts.sh index 1fd92c2..80aaeb2 100644 --- a/tests/p0-p4/domain-rubrics-contracts.sh +++ b/tests/p0-p4/domain-rubrics-contracts.sh @@ -57,7 +57,7 @@ for file_and_term in \ "$review_dir/references/qa-evaluation-loop.md::domain_quality_scores" \ "$review_dir/references/qa-evaluation-loop.md::Do not invent domain rubrics" \ "$review_dir/SKILL.md::references/domain-rubrics.md\` only when" \ - "$review_dir/SKILL.md::selected_domain_rubrics/domain_quality_scores when used" \ + "$review_dir/SKILL.md::selected_domain_rubrics/domain_quality_scores when scoped" \ "$review_dir/SKILL.md::Code Reviewer still owns code defects"; do file="${file_and_term%%::*}" term="${file_and_term#*::}" diff --git a/tests/p0-p4/harness-docs-evals-contracts.sh b/tests/p0-p4/harness-docs-evals-contracts.sh index 7b1474b..2aba69e 100644 --- a/tests/p0-p4/harness-docs-evals-contracts.sh +++ b/tests/p0-p4/harness-docs-evals-contracts.sh @@ -49,13 +49,13 @@ require_terms "harness design guide" "$harness_doc" \ "Pivot/Restart Controller" \ "legacy_code_bug" \ "assistant-debugging" \ - "max 20 rounds" \ - "Round 20 is terminal" + "max 10 rounds" \ + "Round 10 is terminal" -test_start "harness design guide avoids stale three-agent and 10-round wording" +test_start "harness design guide avoids stale three-agent and 20-round wording" stale_harness_output="$(mktemp)" p0p4_register_cleanup "$stale_harness_output" -if rg -n -e 'three-agent' -e '3-agent' -e 'max 10' -e 'round <= 10' "$harness_doc" >"$stale_harness_output"; then +if rg -n -e 'three-agent' -e '3-agent' -e 'max 20' -e 'round <= 20' "$harness_doc" >"$stale_harness_output"; then fail "harness design guide contains stale harness wording; see $stale_harness_output" else pass @@ -68,7 +68,7 @@ require_terms "docs eval README" "$eval_readme" \ "separate Code Reviewer and QA Evaluator evidence" \ "QA loop behavior with conditional domain rubrics" \ "pivot/restart decisions for stagnation and Code Writer blockers" \ - "terminal max 20 review/QA round behavior" + "terminal max 10 review/QA round behavior" test_start "skill contract guide describes process harness controller contracts" require_terms "skill contract guide" "$contract_guide" \ @@ -78,7 +78,7 @@ require_terms "skill contract guide" "$contract_guide" \ "artifacts when the controller requires them" \ "Code Reviewer and QA Evaluator" \ "handoffs stay separate" \ - "max 20" \ + "max 10" \ "pivot_restart_decision" test_start "skill-creator harness patterns cover controller artifacts and loop safety" @@ -91,7 +91,7 @@ require_terms "skill creator harness patterns" "$skill_creator_patterns" \ "Code Review And QA Separation" \ "Conditional Domain Rubrics" \ "Bounded Review / QA Loops" \ - "max 20 rounds" \ + "max 10 rounds" \ "Pivot / Restart Decisions" \ "legacy_code_bug" \ "Agentic Loop Safety" @@ -103,12 +103,12 @@ if jq -e ' case_category("harness-done-contract-recipe-before-build"; "harness_controller") and case_category("code-reviewer-and-qa-evaluator-evidence-split"; "review_qa_split") and case_category("pivot-restart-on-stagnation-or-code-writer-blocker"; "pivot_restart") - and case_category("review-and-qa-terminal-cap-20"; "review_loop_cap") + and case_category("review-and-qa-terminal-cap-10"; "review_loop_cap") and all(.cases[] | select(.id as $id | [ "harness-done-contract-recipe-before-build", "code-reviewer-and-qa-evaluator-evidence-split", "pivot-restart-on-stagnation-or-code-writer-blocker", - "review-and-qa-terminal-cap-20" + "review-and-qa-terminal-cap-10" ] | index($id)); (.machine_expectations.required_substrings | type == "array" and length > 0) and (.machine_expectations.forbidden_substrings | type == "array" and length > 0) @@ -130,8 +130,8 @@ for case_and_term in \ "code-reviewer-and-qa-evaluator-evidence-split::rubric_refs" \ "pivot-restart-on-stagnation-or-code-writer-blocker::pivot_restart_decision" \ "pivot-restart-on-stagnation-or-code-writer-blocker::legacy_code_bug" \ - "review-and-qa-terminal-cap-20::max 20" \ - "review-and-qa-terminal-cap-20::round 21"; do + "review-and-qa-terminal-cap-10::max 10" \ + "review-and-qa-terminal-cap-10::round 11"; do case_id="${case_and_term%%::*}" term="${case_and_term#*::}" if ! jq -e --arg case_id "$case_id" --arg term "$term" ' @@ -160,7 +160,7 @@ if jq -e ' "Code Reviewer", "QA Evaluator", "pivot_restart_decision", - "max 20" + "max 10" ])) and (.machine_expectations.forbidden_substrings | length > 0) ) diff --git a/tests/p0-p4/qa-evaluator-contracts.sh b/tests/p0-p4/qa-evaluator-contracts.sh index 28d23d3..4a7535e 100644 --- a/tests/p0-p4/qa-evaluator-contracts.sh +++ b/tests/p0-p4/qa-evaluator-contracts.sh @@ -102,16 +102,16 @@ else fail "Codex install failed; see /tmp/p0p4-install-qa-evaluator.err" fi -test_start "assistant-review declares independent QA evaluation loop and 20-round cap" +test_start "assistant-review routes independent QA evaluation to reference with 10-round cap" review_dir="$FRAMEWORK_DIR/skills/assistant-review" missing_review_terms=() for file_and_term in \ "$review_dir/SKILL.md::optional independent QA evaluation loop" \ "$review_dir/SKILL.md::QA evaluation runs after code-review/build evidence" \ - "$review_dir/SKILL.md::The QA loop supports rounds 1-20" \ - "$review_dir/SKILL.md::Round 20 is terminal" \ - "$review_dir/references/qa-evaluation-loop.md::while round <= 20" \ - "$review_dir/references/qa-evaluation-loop.md::Round 20 is terminal" \ + "$review_dir/SKILL.md::Load \`references/qa-evaluation-loop.md\` before dispatching QAEvaluator" \ + "$review_dir/SKILL.md::that reference owns the detailed algorithm" \ + "$review_dir/references/qa-evaluation-loop.md::while round <= 10" \ + "$review_dir/references/qa-evaluation-loop.md::Round 10 is terminal" \ "$review_dir/references/qa-evaluation-loop.md::does not replace code-reviewer" \ "$review_dir/contracts/handoffs.yaml::to: QAEvaluator" \ "$review_dir/contracts/handoffs.yaml::- name: debate_record" \ @@ -121,7 +121,7 @@ for file_and_term in \ "$review_dir/contracts/handoffs.yaml::debate_record when Done Contract exists" \ "$review_dir/contracts/handoffs.yaml::qa_scorecard" \ "$review_dir/contracts/handoffs.yaml::score_entry" \ - "$review_dir/contracts/handoffs.yaml::The loop never starts round 21." \ + "$review_dir/contracts/handoffs.yaml::The loop never starts round 11." \ "$review_dir/contracts/input.yaml::- name: qa_evaluation_mode" \ "$review_dir/contracts/input.yaml::- name: debate_record" \ "$review_dir/contracts/input.yaml::pre-build debate/subagent-perspective evidence" \ @@ -135,7 +135,7 @@ for file_and_term in \ "$review_dir/contracts/phase-gates.yaml::QA evaluation starts only after build/test verification evidence and Code Reviewer or Reviewer compatibility result are available" \ "$review_dir/contracts/phase-gates.yaml::INV_QA4" \ "$review_dir/contracts/phase-gates.yaml::QA findings require acceptance criteria, Done Contract, verification evidence, scoped domain-context support, and debate_record when Done Contract exists" \ - "$review_dir/contracts/phase-gates.yaml::round 20 is terminal"; do + "$review_dir/contracts/phase-gates.yaml::round 10 is terminal"; do file="${file_and_term%%::*}" term="${file_and_term#*::}" if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then @@ -238,8 +238,8 @@ for file_and_term in \ "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml::Stay in the QA lane" \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/subagent-dispatch.md::Code Reviewer" \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/subagent-dispatch.md::QA Evaluator" \ - "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::Code Reviewer continues to own code defects, security, architecture, and test-coverage review" \ - "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::QA findings do not replace code-review findings"; do + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::Keep QA evaluation separate from code review" \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::Code Reviewer continues to own code defects, security, architecture, and test-coverage review"; do file="${file_and_term%%::*}" term="${file_and_term#*::}" if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then diff --git a/tests/p0-p4/review-loop-cap-contracts.sh b/tests/p0-p4/review-loop-cap-contracts.sh index 5d2311e..0f1a28e 100755 --- a/tests/p0-p4/review-loop-cap-contracts.sh +++ b/tests/p0-p4/review-loop-cap-contracts.sh @@ -11,6 +11,7 @@ active_review_cap_files=( "$FRAMEWORK_DIR/skills/assistant-review/contracts/output.yaml" "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml" "$FRAMEWORK_DIR/skills/assistant-review/contracts/phase-gates.yaml" + "$FRAMEWORK_DIR/skills/assistant-review/references/qa-evaluation-loop.md" "$FRAMEWORK_DIR/skills/assistant-review/references/score-tracking.md" "$FRAMEWORK_DIR/skills/assistant-review/references/review-rubric.md" "$FRAMEWORK_DIR/skills/assistant-review/evals/cases.json" @@ -26,48 +27,51 @@ active_review_cap_files=( "$FRAMEWORK_DIR/install.sh" "$FRAMEWORK_DIR/agents/codex/code-reviewer.toml" "$FRAMEWORK_DIR/agents/codex/reviewer.toml" + "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml" "$FRAMEWORK_DIR/agents/claude/code-reviewer.md" "$FRAMEWORK_DIR/agents/claude/reviewer.md" + "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md" "$FRAMEWORK_DIR/docs/harness-design-guide.md" ) # Historical/generated presentation artifacts are intentionally excluded from # this runtime cap guard; S9 owns deeper docs/presentation refresh. -test_start "active review-loop surfaces reject stale 10-round limits" +test_start "active review-loop surfaces reject accidental 20-round limits" stale_cap_output="$(mktemp)" p0p4_register_cleanup "$stale_cap_output" -if rg -n -e 'max 10' \ - -e '<= ?10' \ - -e 'round <= 10' \ - -e 'Round: N of 10' \ - -e 'Round [0-9]+ of 10' \ - -e 'Round 10 is terminal' \ - -e 'round 10 is terminal' \ - -e 'round 11' \ - -e 'N between 1 and 10' \ +if rg -n -e 'max 20' \ + -e 'max-20' \ + -e '<= ?20' \ + -e 'round <= 20' \ + -e 'Round: N of 20' \ + -e 'Round [0-9]+ of 20' \ + -e 'Round 20 is terminal' \ + -e 'round 20 is terminal' \ + -e 'round 21' \ + -e 'N between 1 and 20' \ "${active_review_cap_files[@]}" >"$stale_cap_output"; then - fail "found stale review-loop cap text in active surfaces; see $stale_cap_output" + fail "found accidental 20-round review-loop cap text in active surfaces; see $stale_cap_output" else pass fi -test_start "assistant-review contracts and loop use 20-round cap" +test_start "assistant-review contracts and loop use 10-round cap" missing_review_cap_terms=() for file_and_term in \ - "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::max 20 rounds" \ - "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::while round <= 20:" \ - "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::round 20 is terminal" \ - "$FRAMEWORK_DIR/skills/assistant-review/contracts/input.yaml::validation: \">= 1 and <= 20\"" \ - "$FRAMEWORK_DIR/skills/assistant-review/contracts/output.yaml::rounds (1-20)" \ - "$FRAMEWORK_DIR/skills/assistant-review/contracts/output.yaml::capped at the 20-round review limit" \ - "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml::Current review round number (1-20)" \ - "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml::round 20 remains terminal" \ - "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml::The loop never starts round 21." \ - "$FRAMEWORK_DIR/skills/assistant-review/contracts/phase-gates.yaml::EXIT_MAX_ROUNDS = round 20" \ - "$FRAMEWORK_DIR/skills/assistant-review/references/score-tracking.md::Round: N of 20" \ - "$FRAMEWORK_DIR/skills/assistant-review/references/review-rubric.md::| 4-20 | 4.0+ | 3.25" \ - "$FRAMEWORK_DIR/skills/assistant-review/evals/cases.json::max 20 rounds" \ - "$FRAMEWORK_DIR/skills/assistant-review/evals/cases.json::rounds 1-20"; do + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::max 10 rounds" \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::while round <= 10:" \ + "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md::round 10 is terminal" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/input.yaml::validation: \">= 1 and <= 10\"" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/output.yaml::rounds (1-10)" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/output.yaml::capped at the 10-round review limit" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml::Current review round number (1-10)" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml::round 10 remains terminal" \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/handoffs.yaml::The loop never starts round 11." \ + "$FRAMEWORK_DIR/skills/assistant-review/contracts/phase-gates.yaml::EXIT_MAX_ROUNDS = round 10" \ + "$FRAMEWORK_DIR/skills/assistant-review/references/score-tracking.md::Round: N of 10" \ + "$FRAMEWORK_DIR/skills/assistant-review/references/review-rubric.md::| 4-10 | 4.0+ | 3.25" \ + "$FRAMEWORK_DIR/skills/assistant-review/evals/cases.json::max 10 rounds" \ + "$FRAMEWORK_DIR/skills/assistant-review/evals/cases.json::rounds 1-10"; do file="${file_and_term%%::*}" term="${file_and_term#*::}" if [[ ! -f "$file" ]] || ! grep -Fq "$term" "$file"; then @@ -77,30 +81,30 @@ done if [[ "${#missing_review_cap_terms[@]}" -eq 0 ]]; then pass else - fail "assistant-review 20-round cap terms missing: ${missing_review_cap_terms[*]}" + fail "assistant-review 10-round cap terms missing: ${missing_review_cap_terms[*]}" fi -test_start "workflow hooks installer and prompts use 20-round review cap" +test_start "workflow hooks installer and prompts use 10-round review cap" missing_runtime_cap_terms=() for file_and_term in \ - "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::validation: \">= 1 and <= 20\"" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::Current review round number (1-20)" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::Round 16-20: 90" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md::max 20 rounds" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::Round: 1 of 20" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::Round: 2 of 20" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/references/prompts/pr-review.md::up to 20 rounds" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/references/prompts/pr-review.md::Round N of 20" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/references/subagent-roles.md::Round 16-20: 90%+" \ - "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh::Round: N of 20" \ - "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh::N between 1 and 20" \ - "$FRAMEWORK_DIR/hooks/scripts/workflow-enforcer.sh::max 20 rounds" \ - "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh::max_round\" -ne 20" \ - "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh::round\" -gt 20" \ - "$FRAMEWORK_DIR/install.sh::Autonomous review-fix loop (max 20 rounds)" \ - "$FRAMEWORK_DIR/install.sh::clean (max 20 rounds)" \ - "$FRAMEWORK_DIR/docs/harness-design-guide.md::while round <= 20:" \ - "$FRAMEWORK_DIR/docs/harness-design-guide.md::| 20 | Terminal report"; do + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml::validation: \">= 1 and <= 10\"" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::Current review round number (1-10)" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/handoffs.yaml::Round 8-10: 90" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md::max 10 rounds" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::Round: 1 of 10" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::Round: 2 of 10" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/prompts/pr-review.md::up to 10 rounds" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/prompts/pr-review.md::Round N of 10" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/subagent-roles.md::Round 8-10: 90%+" \ + "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh::Round: N of 10" \ + "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh::N between 1 and 10" \ + "$FRAMEWORK_DIR/hooks/scripts/workflow-enforcer.sh::max 10 rounds" \ + "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh::max_round\" -ne 10" \ + "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh::round\" -gt 10" \ + "$FRAMEWORK_DIR/install.sh::Autonomous review-fix loop (max 10 rounds)" \ + "$FRAMEWORK_DIR/install.sh::clean (max 10 rounds)" \ + "$FRAMEWORK_DIR/docs/harness-design-guide.md::while round <= 10:" \ + "$FRAMEWORK_DIR/docs/harness-design-guide.md::| 10 | Terminal report"; do file="${file_and_term%%::*}" term="${file_and_term#*::}" if [[ ! -f "$file" ]] || ! grep -Fq "$term" "$file"; then @@ -110,10 +114,10 @@ done if [[ "${#missing_runtime_cap_terms[@]}" -eq 0 ]]; then pass else - fail "workflow/runtime 20-round cap terms missing: ${missing_runtime_cap_terms[*]}" + fail "workflow/runtime 10-round cap terms missing: ${missing_runtime_cap_terms[*]}" fi -test_start "code-reviewer and reviewer compatibility prompts preserve role split with 20-round terminal guidance" +test_start "code-reviewer and reviewer compatibility prompts preserve role split with 10-round terminal guidance" missing_reviewer_prompt_terms=() for file in \ "$FRAMEWORK_DIR/agents/codex/code-reviewer.toml" \ @@ -121,9 +125,9 @@ for file in \ "$FRAMEWORK_DIR/agents/claude/code-reviewer.md" \ "$FRAMEWORK_DIR/agents/claude/reviewer.md"; do for term in \ - "In rounds 16-20" \ - "Round 20 is terminal" \ - "round 21"; do + "In rounds 8-10" \ + "Round 10 is terminal" \ + "round 11"; do if ! grep -Fq "$term" "$file"; then missing_reviewer_prompt_terms+=("${file#$FRAMEWORK_DIR/}: $term") fi @@ -132,7 +136,7 @@ done if [[ "${#missing_reviewer_prompt_terms[@]}" -eq 0 ]]; then pass else - fail "reviewer prompt 20-round compatibility terms missing: ${missing_reviewer_prompt_terms[*]}" + fail "reviewer prompt 10-round compatibility terms missing: ${missing_reviewer_prompt_terms[*]}" fi p0p4_finish_suite "${BASH_SOURCE[0]}" diff --git a/tests/p0-p4/task-packet-contracts.sh b/tests/p0-p4/task-packet-contracts.sh index bbb22c6..8365bf5 100644 --- a/tests/p0-p4/task-packet-contracts.sh +++ b/tests/p0-p4/task-packet-contracts.sh @@ -2061,7 +2061,7 @@ for file_and_term in \ "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/phase-gates.yaml::R_PIVOT_RESTART_DECISION" \ "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/phase-gates.yaml::INV_PIVOT_RESTART_REAPPROVAL" \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/harness-controller.md::## Pivot/Restart Controller" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/references/harness-controller.md::Round 20 remains terminal" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/harness-controller.md::Round 10 remains terminal" \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md::Code Writer unexpected blockers" \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md::STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT" \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::## Pivot/Restart Log" \ diff --git a/tests/test-hooks.sh b/tests/test-hooks.sh index e241cad..5ea52a4 100755 --- a/tests/test-hooks.sh +++ b/tests/test-hooks.sh @@ -181,7 +181,7 @@ qa_evaluation_mode: required - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -253,7 +253,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -397,7 +397,7 @@ qa_evaluation_mode: not_required - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -497,7 +497,7 @@ qa_evaluation_mode: required - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -717,6 +717,108 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi +if test_start "workflow-phase-gates: Codex reviewer lifecycle with matching QA-labeled agent_id satisfies QA Evaluator"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Reviewer dispatch: event cr-1 +- Code Reviewer result: PASS event cr-1 +- QA Evaluator dispatch: reviewer compatibility id=qa-reviewer-1 +- QA Evaluator result: accepted final_verdict accepted score_progression 4.00 +TASK + cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' +{"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} +{"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} +{"event":"SubagentStart","agent_type":"reviewer","agent_name":"reviewer","agent_id":"qa-reviewer-1"} +{"event":"SubagentStop","agent_type":"reviewer","agent_name":"reviewer","agent_id":"qa-reviewer-1"} +JSONL + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") + if [[ "$helper_reason" == "complete" ]]; then + pass + else + fail "expected complete, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + +if test_start "workflow-phase-gates: Codex reviewer lifecycle with QA-labeled agent_id prefix does not satisfy QA Evaluator"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Reviewer dispatch: event cr-1 +- Code Reviewer result: PASS event cr-1 +- QA Evaluator dispatch: reviewer compatibility id=qa-reviewer-1 +- QA Evaluator result: accepted final_verdict accepted score_progression 4.00 +TASK + cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' +{"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} +{"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} +{"event":"SubagentStart","agent_type":"reviewer","agent_name":"reviewer","agent_id":"qa-reviewer"} +{"event":"SubagentStop","agent_type":"reviewer","agent_name":"reviewer","agent_id":"qa-reviewer"} +JSONL + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") + if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then + pass + else + fail "expected delegated_missing_qa_evaluator, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + +if test_start "workflow-phase-gates: Codex stale reviewer lifecycle without matching QA-labeled agent_id blocks QA Evaluator"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Reviewer dispatch: event cr-1 +- Code Reviewer result: PASS event cr-1 +- QA Evaluator dispatch: reviewer compatibility id=qa-reviewer-1 +- QA Evaluator result: accepted final_verdict accepted score_progression 4.00 +TASK + cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' +{"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} +{"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} +{"event":"SubagentStart","agent_type":"reviewer","agent_name":"reviewer","agent_id":"old-qa-reviewer"} +{"event":"SubagentStop","agent_type":"reviewer","agent_name":"reviewer","agent_id":"old-qa-reviewer"} +JSONL + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") + if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then + pass + else + fail "expected delegated_missing_qa_evaluator, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + if test_start "workflow-phase-gates: default review phase missing Code Reviewer evidence blocks"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' @@ -1409,7 +1511,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1510,7 +1612,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1558,7 +1660,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1609,7 +1711,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1658,7 +1760,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1712,7 +1814,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -1772,7 +1874,7 @@ Required agents: - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 @@ -3053,12 +3155,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Weighted: 4.00 - Delta from previous: +0.20 @@ -3096,7 +3198,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3136,7 +3238,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3188,7 +3290,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3240,7 +3342,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.5, code_quality 3.5, architecture 3.5, security 3.5, test_coverage 3.5 - Weighted: 3.50 @@ -3277,7 +3379,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 1.0, code_quality 1.0, architecture 1.0, security 1.0, test_coverage 1.0 - Weighted: 4.00 @@ -3314,12 +3416,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3371,12 +3473,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3430,7 +3532,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3493,7 +3595,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3533,7 +3635,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -3900,7 +4002,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 999.00 @@ -3934,7 +4036,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 1.0, code_quality 1.0, architecture 1.0, security 1.0, test_coverage 1.0 - Weighted: 4.00 @@ -3968,7 +4070,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.5, code_quality 3.5, architecture 3.5, security 3.5, test_coverage 3.5 - Weighted: 3.50 @@ -4002,12 +4104,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Weighted: 4.00 - Delta from previous: +0.20 @@ -4042,12 +4144,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.1, code_quality 4.1, architecture 4.1, security 4.1, test_coverage 4.1 - Weighted: 4.10 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4083,7 +4185,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4116,7 +4218,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4150,7 +4252,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4184,7 +4286,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4220,7 +4322,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4256,7 +4358,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4292,7 +4394,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4328,12 +4430,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4369,7 +4471,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4404,7 +4506,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4440,12 +4542,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.5, code_quality 4.5, architecture 4.5, security 4.5, test_coverage 4.5 - Weighted: 4.50 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4481,7 +4583,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4517,7 +4619,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4553,12 +4655,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4595,12 +4697,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 2.8, code_quality 2.8, architecture 2.8, security 2.8, test_coverage 2.8 - Weighted: 2.80 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.1, code_quality 4.1, architecture 4.1, security 4.1, test_coverage 4.1 - Weighted: 4.10 @@ -4636,12 +4738,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 2 must-fix, 1 should-fix, 0 nits - Rubric: correctness 3.9, code_quality 3.9, architecture 3.9, security 3.9, test_coverage 3.9 - Weighted: 3.90 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 1 must-fix, 1 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4677,12 +4779,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 2 must-fix, 1 should-fix, 0 nits - Rubric: correctness 3.9, code_quality 3.9, architecture 3.9, security 3.9, test_coverage 3.9 - Weighted: 3.90 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 1 must-fix, 1 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4719,12 +4821,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 2 must-fix, 1 should-fix, 0 nits - Rubric: correctness 3.9, code_quality 3.9, architecture 3.9, security 3.9, test_coverage 3.9 - Weighted: 3.90 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 1 must-fix, 1 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4761,12 +4863,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 2 must-fix, 1 should-fix, 0 nits - Rubric: correctness 3.9, code_quality 3.9, architecture 3.9, security 3.9, test_coverage 3.9 - Weighted: 3.90 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 1 must-fix, 1 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4786,7 +4888,7 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi -if test_start "workflow-phase-gates: HAS_REMAINING_ITEMS at round 20 without rationale blocks"; then +if test_start "workflow-phase-gates: HAS_REMAINING_ITEMS at round 10 without rationale blocks"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' # Task @@ -4803,112 +4905,60 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.1, code_quality 3.1, architecture 3.1, security 3.1, test_coverage 3.1 - Weighted: 3.10 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.2, code_quality 3.2, architecture 3.2, security 3.2, test_coverage 3.2 - Weighted: 3.20 ### Quality Review #3 -- Round: 3 of 20 +- Round: 3 of 10 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.3, code_quality 3.3, architecture 3.3, security 3.3, test_coverage 3.3 - Weighted: 3.30 ### Quality Review #4 -- Round: 4 of 20 +- Round: 4 of 10 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.4, code_quality 3.4, architecture 3.4, security 3.4, test_coverage 3.4 - Weighted: 3.40 ### Quality Review #5 -- Round: 5 of 20 +- Round: 5 of 10 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.5, code_quality 3.5, architecture 3.5, security 3.5, test_coverage 3.5 - Weighted: 3.50 ### Quality Review #6 -- Round: 6 of 20 +- Round: 6 of 10 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.6, code_quality 3.6, architecture 3.6, security 3.6, test_coverage 3.6 - Weighted: 3.60 ### Quality Review #7 -- Round: 7 of 20 +- Round: 7 of 10 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.7, code_quality 3.7, architecture 3.7, security 3.7, test_coverage 3.7 - Weighted: 3.70 ### Quality Review #8 -- Round: 8 of 20 +- Round: 8 of 10 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #9 -- Round: 9 of 20 +- Round: 9 of 10 - Found this round: 2 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.9, code_quality 3.9, architecture 3.9, security 3.9, test_coverage 3.9 - Weighted: 3.90 ### Quality Review #10 -- Round: 10 of 20 +- Round: 10 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 - Delta from previous: +0.10 - Drift check: GENUINE -### Quality Review #11 -- Round: 11 of 20 -- Found this round: 1 must-fix, 0 should-fix, 0 nits -- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 -- Weighted: 4.00 -### Quality Review #12 -- Round: 12 of 20 -- Found this round: 1 must-fix, 0 should-fix, 0 nits -- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 -- Weighted: 4.00 -### Quality Review #13 -- Round: 13 of 20 -- Found this round: 1 must-fix, 0 should-fix, 0 nits -- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 -- Weighted: 4.00 -### Quality Review #14 -- Round: 14 of 20 -- Found this round: 1 must-fix, 0 should-fix, 0 nits -- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 -- Weighted: 4.00 -### Quality Review #15 -- Round: 15 of 20 -- Found this round: 1 must-fix, 0 should-fix, 0 nits -- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 -- Weighted: 4.00 -### Quality Review #16 -- Round: 16 of 20 -- Found this round: 1 must-fix, 0 should-fix, 0 nits -- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 -- Weighted: 4.00 -### Quality Review #17 -- Round: 17 of 20 -- Found this round: 1 must-fix, 0 should-fix, 0 nits -- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 -- Weighted: 4.00 -### Quality Review #18 -- Round: 18 of 20 -- Found this round: 1 must-fix, 0 should-fix, 0 nits -- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 -- Weighted: 4.00 -### Quality Review #19 -- Round: 19 of 20 -- Found this round: 1 must-fix, 0 should-fix, 0 nits -- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 -- Weighted: 4.00 -### Quality Review #20 -- Round: 20 of 20 -- Found this round: 1 must-fix, 0 should-fix, 0 nits -- Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 -- Weighted: 4.00 -- Delta from previous: +0.00 -- Drift check: NEUTRAL ### Final result - Result: HAS_REMAINING_ITEMS -- Score progression: 3.10->3.20->3.30->3.40->3.50->3.60->3.70->3.80->3.90->4.00->4.00->4.00->4.00->4.00->4.00->4.00->4.00->4.00->4.00->4.00 +- Score progression: 3.10->3.20->3.30->3.40->3.50->3.60->3.70->3.80->3.90->4.00 TASK helper_reason=$(review_controller_reason "$TEST_PROJECT/.claude/task.md") if [[ "$helper_reason" == "missing_remaining_rationale" ]]; then @@ -4919,7 +4969,7 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi -if test_start "workflow-phase-gates: round 21 of 20 blocks"; then +if test_start "workflow-phase-gates: round 11 of 10 blocks"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' # Task @@ -4935,8 +4985,8 @@ Plan approval: yes - Changed files mismatch: none - Verification evidence mismatch: none - Required fixes: none -### Quality Review #21 -- Round: 21 of 20 +### Quality Review #11 +- Round: 11 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4972,7 +5022,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #2 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -4989,7 +5039,7 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi -if test_start "workflow-phase-gates: valid medium completion with round 2 of 20, drift check, score progression, weighted 4.00 allows"; then +if test_start "workflow-phase-gates: valid medium completion with round 2 of 10, drift check, score progression, weighted 4.00 allows"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' # Task @@ -5006,12 +5056,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.0, code_quality 4.0, architecture 4.0, security 4.0, test_coverage 4.0 - Weighted: 4.00 @@ -5047,7 +5097,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness: 4.1; quality 4.2, architecture=4.3; security: 4.4; coverage 4.5 - Weighted: 4.27 @@ -5081,19 +5131,19 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 1 must-fix, 1 should-fix, 0 nits - Rubric: correctness 3.5, code_quality 3.5, architecture 3.5, security 3.5, test_coverage 3.5 - Weighted: 3.50 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.85, code_quality 3.85, architecture 3.85, security 3.85, test_coverage 3.85 - Weighted: 3.85 - Delta from previous: +0.35 - Drift check: GENUINE ### Quality Review #3 -- Round: 3 of 20 +- Round: 3 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness 4.1, code_quality 4.1, architecture 4.1, security 4.1, test_coverage 4.1 - Weighted: 4.10 @@ -5186,12 +5236,12 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 1 must-fix, 0 should-fix, 0 nits - Rubric: correctness 3.8, code_quality 3.8, architecture 3.8, security 3.8, test_coverage 3.8 - Weighted: 3.80 ### Quality Review #2 -- Round: 2 of 20 +- Round: 2 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Weighted: 4.00 - Delta from previous: +0.20 @@ -7287,7 +7337,7 @@ Plan approval: yes - Verification evidence mismatch: none - Required fixes: none ### Quality Review #1 -- Round: 1 of 20 +- Round: 1 of 10 - Found this round: 0 must-fix, 0 should-fix, 0 nits - Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 - Weighted: 4.00 From faa76ac6befac311fbe7287e3a9161b388b40b1b Mon Sep 17 00:00:00 2001 From: Laimis Date: Thu, 2 Jul 2026 14:58:11 +0300 Subject: [PATCH 3/9] Treat complete journals as inactive --- hooks/scripts/task-journal-resolver.sh | 8 +- tests/test-hooks.sh | 112 +++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 3 deletions(-) diff --git a/hooks/scripts/task-journal-resolver.sh b/hooks/scripts/task-journal-resolver.sh index 2f4524e..eed414e 100644 --- a/hooks/scripts/task-journal-resolver.sh +++ b/hooks/scripts/task-journal-resolver.sh @@ -178,9 +178,11 @@ assistant_task_journal_completed() { ' "$task_file" 2>/dev/null ) - if [[ "$status_token" == "DONE" ]]; then - return 0 - fi + case "$status_token" in + DONE|COMPLETE|COMPLETED) + return 0 + ;; + esac if grep -qF 'WORKFLOW COMPLETE' "$task_file" 2>/dev/null; then return 0 diff --git a/tests/test-hooks.sh b/tests/test-hooks.sh index 5ea52a4..586f274 100755 --- a/tests/test-hooks.sh +++ b/tests/test-hooks.sh @@ -1092,6 +1092,32 @@ EOF rm -rf "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" fi +if test_start "session-start: Codex, Status COMPLETE task journal → no active task journal"; then + mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' +# Task +Status: COMPLETE +Step: old complete work +EOF + + local_tmp_out=$(mktemp) + HOOK_EXIT=0 + env HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/session-start.sh" \ + > "$local_tmp_out" 2>/dev/null <<< '{"session_id":"test"}' || HOOK_EXIT=$? + HOOK_STDOUT=$(cat "$local_tmp_out") + rm -f "$local_tmp_out" + + additional_context=$(echo "$HOOK_STDOUT" | jq -r '.hookSpecificOutput.additionalContext // empty' 2>/dev/null || true) + if [[ $HOOK_EXIT -eq 0 ]] && is_valid_json "$HOOK_STDOUT" \ + && [[ "$additional_context" != *"ACTIVE TASK JOURNAL"* ]] \ + && [[ "$additional_context" != *"old complete work"* ]]; then + pass + else + fail "exit=$HOOK_EXIT, COMPLETE task journal was injected" + fi + rm -rf "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" +fi + if test_start "session-start: Codex, WORKFLOW COMPLETE task journal → no active task journal"; then mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' @@ -6694,6 +6720,36 @@ EOF rm -f "$TEST_PROJECT/.codex/task.md" fi +if test_start "workflow-enforcer: Codex, Status COMPLETE task journal → lightweight rules reminder"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' +Task: Complete codex task +Status: COMPLETE +Triaged as: small +EOF + echo '{"prompt": "new task", "hook_event_name": "UserPromptSubmit"}' | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-enforcer.sh" \ + > /tmp/_wf_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wf_out) + rm -f /tmp/_wf_out + if [[ $HOOK_EXIT -eq 0 ]] && echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.additionalContext' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | grep -q "WORKFLOW RULES" \ + && echo "$HOOK_STDOUT" | grep -q "STATE BOOTSTRAP" \ + && echo "$HOOK_STDOUT" | grep -q ".codex/task.md" \ + && echo "$HOOK_STDOUT" | grep -q ".codex/context-map.md" \ + && echo "$HOOK_STDOUT" | grep -q "resolve clarification readiness before PLAN" \ + && echo "$HOOK_STDOUT" | grep -q "Do not enter PLAN by silently assuming answers" \ + && ! echo "$HOOK_STDOUT" | grep -q "WORKFLOW STATE" \ + && ! echo "$HOOK_STDOUT" | grep -q "Complete codex task"; then + pass + else + fail "exit=$HOOK_EXIT, stdout='$HOOK_STDOUT'" + fi + rm -f "$TEST_PROJECT/.codex/task.md" +fi + if test_start "workflow-enforcer: Codex, WORKFLOW COMPLETE task journal → lightweight rules reminder"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" mkdir -p "$TEST_PROJECT/.codex" @@ -7697,6 +7753,62 @@ EOF rm -rf "$TEST_PROJECT/.codex" fi +if test_start "workflow-enforcer: COMPLETE observed after active cache → no stale cache restore after delete"; then + clear_workflow_cache + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' +Task: Cached complete task +Status: BUILDING +Triaged as: medium +Plan approval: yes +EOF + + echo '{"prompt": "prime cache", "hook_event_name": "UserPromptSubmit"}' | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-enforcer.sh" \ + > /tmp/_wf_out 2>/dev/null + rm -f /tmp/_wf_out + + cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' +Task: Cached complete task +Status: COMPLETE +Triaged as: medium +Plan approval: yes +EOF + + echo '{"prompt": "observe completed", "hook_event_name": "UserPromptSubmit"}' | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-enforcer.sh" \ + > /tmp/_wf_out 2>/dev/null + rm -f /tmp/_wf_out + + cache_entries_after_complete=$(find "$TEST_AGENT_HOME" -path '*/cache/workflow-state/*' -print 2>/dev/null || true) + rm -f "$TEST_PROJECT/.codex/task.md" + + FORK_ROOT=$(mktemp -d)/"$(basename "$TEST_PROJECT")" + mkdir -p "$FORK_ROOT/subagent/worktree" + ( + cd "$FORK_ROOT/subagent/worktree" + echo '{"prompt": "continue", "hook_event_name": "UserPromptSubmit"}' | \ + HOME="$TEST_AGENT_HOME" bash "$HOOKS_DIR/workflow-enforcer.sh" \ + > /tmp/_wf_out 2>/dev/null + ) + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wf_out) + rm -f /tmp/_wf_out + rm -rf "$(dirname "$FORK_ROOT")" + + if [[ $HOOK_EXIT -eq 0 ]] && echo "$HOOK_STDOUT" | grep -q "WORKFLOW RULES" \ + && ! echo "$HOOK_STDOUT" | grep -q "WORKFLOW STATE" \ + && ! echo "$HOOK_STDOUT" | grep -q "Cached complete task" \ + && [[ -z "$cache_entries_after_complete" ]]; then + pass + else + fail "exit=$HOOK_EXIT, stdout='$HOOK_STDOUT', cache_entries_after_complete='$cache_entries_after_complete'" + fi + clear_workflow_cache + rm -rf "$TEST_PROJECT/.codex" +fi + if test_start "workflow-enforcer: legacy metadata-less cache entry → ignored and removed"; then clear_workflow_cache rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" From c7143ca584ad7f817a494d258b5ac5398546f606 Mon Sep 17 00:00:00 2001 From: Laimis Date: Thu, 2 Jul 2026 16:53:10 +0300 Subject: [PATCH 4/9] Compact active task journal context --- docs/hook-output-benchmarks.md | 6 +- hooks/scripts/post-compact.sh | 26 +- hooks/scripts/session-start.sh | 10 +- hooks/scripts/task-journal-resolver.sh | 10 +- hooks/scripts/workflow-guard.sh | 2 +- hooks/scripts/workflow-phase-gates.sh | 18 +- .../skills/assistant-workflow/SKILL.md | 2 +- skills/assistant-workflow/SKILL.md | 2 +- tests/p0-p4/installed-hook-smoke.sh | 26 +- tests/test-hooks.sh | 290 ++++++++++++++++-- 10 files changed, 338 insertions(+), 54 deletions(-) diff --git a/docs/hook-output-benchmarks.md b/docs/hook-output-benchmarks.md index 9ecf086..b46ecd8 100644 --- a/docs/hook-output-benchmarks.md +++ b/docs/hook-output-benchmarks.md @@ -12,9 +12,9 @@ Current rows: 7 | hook_name | scenario | stdout_bytes | stdout_words | stderr_bytes | exit_code | first_blocker_or_action | |---|---|---:|---:|---:|---:|---| -| session-start | codex active task journal | 1766 | 183 | 0 | 0 | ACTIVE TASK JOURNAL (read this first — it has full task state): | -| workflow-enforcer | codex building phase gates | 1625 | 184 | 0 | 0 | WORKFLOW STATE (auto-injected every prompt): | -| post-compact | codex restored active task | 1526 | 154 | 0 | 0 | RESTORED AFTER COMPACTION — Task journal: | +| session-start | codex active task journal | 980 | 102 | 0 | 0 | ACTIVE TASK JOURNAL AVAILABLE | +| workflow-enforcer | codex building phase gates | 1626 | 184 | 0 | 0 | WORKFLOW STATE (auto-injected every prompt): | +| post-compact | codex restored active task | 749 | 75 | 0 | 0 | RESTORED AFTER COMPACTION — Active task journal available | | skill-router | codex assistant-workflow route | 498 | 47 | 0 | 0 | SKILL MATCH (1/1): This request matches assistant-workflow. You MUST load and follow this SKILL.md and its contracts before acting. At minimum: triage the task size, then build with tests included in the Build phase. Skipping workflow for speed is explicitly prohibited. Required inputs for this skill: [task_description,task_type,clarification_status,unresolved_clarification_topics,clarification_defaults_applied] | | stop-review | codex missing spec review blocker | 371 | 54 | 0 | 0 | Task journal shows an active workflow but no Spec Review was run. You MUST run Stage 1 first: load references/prompts/spec-review.md, compare each approved plan step/task packet/component against actual changes, append a structured Spec Review entry with Result: PASS or FAIL, fix any FAIL items, then continue to quality review. | | subagent-monitor | codex code-writer start | 200 | 20 | 0 | 0 | SUBAGENT CONSTRAINT: You are a code writer. Write code only. Do NOT run builds or tests — builder-tester handles that. | diff --git a/hooks/scripts/post-compact.sh b/hooks/scripts/post-compact.sh index 9c7d214..0ca8611 100755 --- a/hooks/scripts/post-compact.sh +++ b/hooks/scripts/post-compact.sh @@ -16,16 +16,18 @@ # CLAUDE_PROJECT_DIR / CODEX_PROJECT_DIR — project root # # Behavior: -# Re-injects task journal, session state, working buffer, depth profile, and memory-graph instructions after compaction. +# Re-injects compact task journal pointer, session state, working buffer, depth profile, and memory-graph instructions after compaction. # No output if nothing found (exit 0). # Not installed for Gemini — session-start.sh handles re-injection on resume. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$SCRIPT_DIR/task-journal-resolver.sh" + INPUT=$(cat) -PROJECT_DIR="${CLAUDE_PROJECT_DIR:-${GEMINI_PROJECT_DIR:-${CODEX_PROJECT_DIR:-$(pwd)}}}" +PROJECT_DIR="$(assistant_resolve_project_dir "$(pwd)")" AGENT_HOME="$HOME/.claude" IS_CODEX=false if [[ -n "${GEMINI_PROJECT_DIR:-}" ]]; then @@ -40,17 +42,15 @@ fi context_parts=() -# Re-inject task journal -for dir in .claude .gemini .codex; do - TASK_FILE="$PROJECT_DIR/$dir/task.md" - if [[ -f "$TASK_FILE" ]]; then - task_content=$(cat "$TASK_FILE") - context_parts+=("RESTORED AFTER COMPACTION — Task journal:") - context_parts+=("$task_content") - context_parts+=("---") - break - fi -done +# Re-inject compact active task journal pointer +TASK_FILE="$(assistant_find_task_journal "$PROJECT_DIR" "$(pwd)" || true)" +if [[ -f "$TASK_FILE" ]] && ! assistant_task_journal_completed "$TASK_FILE"; then + assistant_cache_task_journal "$TASK_FILE" "$PROJECT_DIR" + context_parts+=("RESTORED AFTER COMPACTION — Active task journal available") + context_parts+=("Task journal path: $TASK_FILE") + context_parts+=("Read this file when task recovery details are needed.") + context_parts+=("---") +fi # Re-inject Telos context (purpose/strategic priorities — lightweight, always relevant) TELOS_FILE="$AGENT_HOME/telos.md" diff --git a/hooks/scripts/session-start.sh b/hooks/scripts/session-start.sh index a7e4d4f..93f4c34 100755 --- a/hooks/scripts/session-start.sh +++ b/hooks/scripts/session-start.sh @@ -16,7 +16,7 @@ # CLAUDE_PROJECT_DIR / GEMINI_PROJECT_DIR / CODEX_PROJECT_DIR — project root # # Behavior: -# 1. Active task journal ({state_dir}/task.md) — injects full state +# 1. Active task journal ({state_dir}/task.md) — emits compact recovery pointer # 2. Telos context (~/{agent}/telos.md) — purpose/strategic priorities # 3. Compact instruction to call memory_context / memory_search via memory-graph MCP # 4. No output if nothing found (exit 0) @@ -61,10 +61,10 @@ context_parts=() TASK_FILE="$(assistant_find_task_journal "$PROJECT_DIR" "$(pwd)" || true)" if [[ -f "$TASK_FILE" ]] && ! assistant_task_journal_completed "$TASK_FILE"; then - task_content=$(cat "$TASK_FILE") assistant_cache_task_journal "$TASK_FILE" "$PROJECT_DIR" - context_parts+=("ACTIVE TASK JOURNAL (read this first — it has full task state):") - context_parts+=("$task_content") + context_parts+=("ACTIVE TASK JOURNAL AVAILABLE") + context_parts+=("Task journal path: $TASK_FILE") + context_parts+=("Read this file when task recovery details are needed.") context_parts+=("---") fi @@ -95,7 +95,7 @@ fi # 5. Instruction to load project context via memory-graph MCP if $IS_CODEX; then context_parts+=("SESSION START — Codex Protocol:") - context_parts+=("1. Read active task journal above first when present.") + context_parts+=("1. If an active task journal is available above, read it when task recovery details are needed.") context_parts+=("2. Run memory_context before planning/implementation; use memory_search as needed.") context_parts+=("3. Consult AGENTS.md for the full role, workflow, memory, and review protocol.") context_parts+=("---") diff --git a/hooks/scripts/task-journal-resolver.sh b/hooks/scripts/task-journal-resolver.sh index eed414e..e88fb8c 100644 --- a/hooks/scripts/task-journal-resolver.sh +++ b/hooks/scripts/task-journal-resolver.sh @@ -168,8 +168,14 @@ assistant_task_journal_completed() { status_token=$( awk ' - $0 ~ /^(#+[[:space:]]*)?Status:/ { - sub(/^(#+[[:space:]]*)?Status:[[:space:]]*/, "", $0) + /^##[[:space:]]+Task[[:space:]]*:/ { + next + } + /^##([[:space:]]|$)/ { + exit + } + $0 ~ /^#?[[:space:]]*Status:/ { + sub(/^#?[[:space:]]*Status:[[:space:]]*/, "", $0) gsub(/^[[:space:]]+|[[:space:]]+$/, "", $0) split($0, parts, /[[:space:]]+/) print toupper(parts[1]) diff --git a/hooks/scripts/workflow-guard.sh b/hooks/scripts/workflow-guard.sh index 36db838..9ecffae 100755 --- a/hooks/scripts/workflow-guard.sh +++ b/hooks/scripts/workflow-guard.sh @@ -138,7 +138,7 @@ TASK_FILE="$(assistant_find_task_journal "$PROJECT_DIR" "$(pwd)" || true)" assistant_cache_task_journal "$TASK_FILE" "$PROJECT_DIR" # Check if we're in an active build phase -status=$(grep -m1 "^Status:" "$TASK_FILE" 2>/dev/null || echo "") +status="$(assistant_phase_status "$TASK_FILE" || true)" if [[ "$status" != *"BUILDING"* && "$status" != *"VERIFYING"* && "$status" != *"REVIEWING"* ]]; then exit 0 fi diff --git a/hooks/scripts/workflow-phase-gates.sh b/hooks/scripts/workflow-phase-gates.sh index 34bf496..393ebb5 100644 --- a/hooks/scripts/workflow-phase-gates.sh +++ b/hooks/scripts/workflow-phase-gates.sh @@ -19,7 +19,23 @@ assistant_phase_scalar_field() { } assistant_phase_status() { - assistant_phase_scalar_field "$1" "Status" + local file="$1" + awk ' + function is_task_heading(line) { + return line ~ /^##+[[:space:]]*Task([[:space:]]*:|[[:space:]]*$)/ + } + function is_nested_section(line) { + return line ~ /^##+[[:space:]]+/ && !is_task_heading(line) + } + is_nested_section($0) { + exit + } + $0 ~ "^(#[[:space:]]*)?Status:" { + sub("^(#[[:space:]]*)?Status:[[:space:]]*", "", $0) + print + exit + } + ' "$file" 2>/dev/null } assistant_phase_is_medium_plus() { diff --git a/plugins/assistant-dev/skills/assistant-workflow/SKILL.md b/plugins/assistant-dev/skills/assistant-workflow/SKILL.md index 732deaa..4ea7c36 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/SKILL.md +++ b/plugins/assistant-dev/skills/assistant-workflow/SKILL.md @@ -251,7 +251,7 @@ Review findings must cite evidence and concrete risk. Avoid generic style feedba ## Context Management -- **On continuation**: read the active project task journal FIRST; it has the full task state +- **On continuation**: use compact pointers first; read the active project task journal when recovery details are needed. The journal has the full task state - Small: read only target files. Medium: read touched files + plan template. - Large: read interfaces/contracts + plan template + playbook. - Mega: each slice gets its own strict slice brief and context. diff --git a/skills/assistant-workflow/SKILL.md b/skills/assistant-workflow/SKILL.md index 732deaa..4ea7c36 100644 --- a/skills/assistant-workflow/SKILL.md +++ b/skills/assistant-workflow/SKILL.md @@ -251,7 +251,7 @@ Review findings must cite evidence and concrete risk. Avoid generic style feedba ## Context Management -- **On continuation**: read the active project task journal FIRST; it has the full task state +- **On continuation**: use compact pointers first; read the active project task journal when recovery details are needed. The journal has the full task state - Small: read only target files. Medium: read touched files + plan template. - Large: read interfaces/contracts + plan template + playbook. - Mega: each slice gets its own strict slice brief and context. diff --git a/tests/p0-p4/installed-hook-smoke.sh b/tests/p0-p4/installed-hook-smoke.sh index d463449..7c5279d 100644 --- a/tests/p0-p4/installed-hook-smoke.sh +++ b/tests/p0-p4/installed-hook-smoke.sh @@ -129,9 +129,10 @@ TASK workflow_guard="$installed_hooks_dir/workflow-guard.sh" stop_review="$installed_hooks_dir/stop-review.sh" subagent_monitor="$installed_hooks_dir/subagent-monitor.sh" + post_compact="$installed_hooks_dir/post-compact.sh" if [[ -z "$smoke_failed" ]]; then - for required_path in "$session_start" "$skill_router" "$workflow_enforcer" "$workflow_guard" "$stop_review" "$subagent_monitor"; do + for required_path in "$session_start" "$skill_router" "$workflow_enforcer" "$workflow_guard" "$stop_review" "$subagent_monitor" "$post_compact"; do if [[ ! -x "$required_path" ]]; then smoke_failed="required Codex workflow hook is not executable: $required_path" break @@ -156,6 +157,29 @@ TASK fi fi + if [[ -z "$smoke_failed" ]]; then + post_compact_out="$(mktemp)" + post_compact_err="$(mktemp)" + post_compact_exit=0 + env HOME="$HOOK_SMOKE_HOME" CODEX_PROJECT_DIR="$HOOK_SMOKE_PROJECT" bash "$post_compact" \ + >"$post_compact_out" 2>"$post_compact_err" <<< '{"hook_event_name":"PostCompact","trigger":"manual"}' || post_compact_exit=$? + post_compact_stdout="$(cat "$post_compact_out")" + rm -f "$post_compact_out" "$post_compact_err" + post_compact_context="$(echo "$post_compact_stdout" | jq -r '.systemMessage // empty' 2>/dev/null || true)" + if [[ "$post_compact_exit" -ne 0 ]]; then + smoke_failed="installed post-compact.sh failed with exit $post_compact_exit" + elif [[ -n "$post_compact_stdout" ]] && ! echo "$post_compact_stdout" | jq -e . >/dev/null 2>&1; then + smoke_failed="installed post-compact.sh emitted non-JSON stdout" + elif [[ -n "$post_compact_stdout" ]] && ! echo "$post_compact_stdout" | jq -e '(has("hookSpecificOutput") | not) and (.systemMessage | type == "string")' >/dev/null 2>&1; then + smoke_failed="installed post-compact.sh emitted unsupported JSON shape: stdout='$post_compact_stdout'" + elif [[ "$post_compact_context" == *"RESTORED AFTER COMPACTION — Active task journal available"* \ + || "$post_compact_context" == *"ACTIVE TASK JOURNAL AVAILABLE"* \ + || "$post_compact_context" == *"stale completed smoke task"* \ + || "$post_compact_context" == *"This completed task text must not be injected"* ]]; then + smoke_failed="installed post-compact.sh injected completed task journal state" + fi + fi + if [[ -z "$smoke_failed" ]]; then router_out="$(mktemp)" router_err="$(mktemp)" diff --git a/tests/test-hooks.sh b/tests/test-hooks.sh index 586f274..c6b7e6e 100755 --- a/tests/test-hooks.sh +++ b/tests/test-hooks.sh @@ -899,14 +899,18 @@ if test_start "session-start: Claude, no task journal, no memory → no output"; fi fi -if test_start "session-start: Claude, with task journal → outputs journal"; then +if test_start "session-start: Claude, with task journal → compact reminder only"; then mkdir -p "$TEST_PROJECT/.claude" "$TEST_AGENT_HOME/.claude" - echo -e "# Task\nStatus: BUILDING\nStep: implement auth" > "$TEST_PROJECT/.claude/task.md" + echo -e "# Task\nStatus: BUILDING\nStep: unique claude session body" > "$TEST_PROJECT/.claude/task.md" HOME="$TEST_AGENT_HOME" run_hook session-start.sh claude - if [[ $HOOK_EXIT -eq 0 && "$HOOK_STDOUT" == *"ACTIVE TASK JOURNAL"* ]]; then + if [[ $HOOK_EXIT -eq 0 \ + && "$HOOK_STDOUT" == *"ACTIVE TASK JOURNAL AVAILABLE"* \ + && "$HOOK_STDOUT" == *"Task journal path:"* \ + && "$HOOK_STDOUT" == *".claude/task.md"* \ + && "$HOOK_STDOUT" != *"unique claude session body"* ]]; then pass else - fail "exit=$HOOK_EXIT, stdout missing ACTIVE TASK JOURNAL" + fail "exit=$HOOK_EXIT, expected compact active journal reminder without body" fi rm -rf "$TEST_PROJECT/.claude" "$TEST_AGENT_HOME/.claude" fi @@ -1036,9 +1040,9 @@ if test_start "session-start: Codex, with task journal → hookSpecificOutput JS rm -rf "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" fi -if test_start "session-start: Codex, with graph rules → compact MCP retrieval instruction and journal"; then +if test_start "session-start: Codex, with graph rules → compact MCP retrieval instruction and journal reminder"; then mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex/memory" - echo -e "# Task\nStatus: BUILDING\nStep: keep task visible" > "$TEST_PROJECT/.codex/task.md" + echo -e "# Task\nStatus: BUILDING\nStep: unique codex session body" > "$TEST_PROJECT/.codex/task.md" echo '{"kind":"entity","name":"always-use-tabs","type":"rule","observations":["Always use tabs for indentation"],"sourceFile":null,"createdAt":"2025-01-01T00:00:00Z","updatedAt":"2025-01-01T00:00:00Z"}' > "$TEST_AGENT_HOME/.codex/memory/graph.jsonl" local_tmp_out=$(mktemp) @@ -1054,8 +1058,10 @@ if test_start "session-start: Codex, with graph rules → compact MCP retrieval && [[ "$additional_context" == *"memory_context"* ]] \ && [[ "$additional_context" == *"memory_search"* ]] \ && [[ "$additional_context" == *"MCP"* ]] \ - && [[ "$additional_context" == *"ACTIVE TASK JOURNAL"* ]] \ - && [[ "$additional_context" == *"keep task visible"* ]] \ + && [[ "$additional_context" == *"ACTIVE TASK JOURNAL AVAILABLE"* ]] \ + && [[ "$additional_context" == *"Task journal path:"* ]] \ + && [[ "$additional_context" == *".codex/task.md"* ]] \ + && [[ "$additional_context" != *"unique codex session body"* ]] \ && [[ "$additional_context" != *"always-use-tabs"* ]] \ && [[ "$additional_context" != *"Always use tabs for indentation"* ]] \ && [[ "$additional_context" != *"graph.jsonl"* ]]; then @@ -1066,10 +1072,10 @@ if test_start "session-start: Codex, with graph rules → compact MCP retrieval rm -rf "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" fi -if test_start "session-start: Codex, Status DONE task journal → no active task journal"; then +if test_start "session-start: Codex, canonical Status DONE task journal → no active task journal"; then mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' -# Task +## Task: completed session journal Status: DONE Step: old completed work EOF @@ -1145,6 +1151,66 @@ EOF rm -rf "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" fi +if test_start "session-start: Codex, Slice Status DONE with root BUILDING → active task journal"; then + mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' +# Task +Status: BUILDING +Step: unique root building body + +## Slice Status: DONE +EOF + + local_tmp_out=$(mktemp) + HOOK_EXIT=0 + env HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/session-start.sh" \ + > "$local_tmp_out" 2>/dev/null <<< '{"session_id":"test"}' || HOOK_EXIT=$? + HOOK_STDOUT=$(cat "$local_tmp_out") + rm -f "$local_tmp_out" + + additional_context=$(echo "$HOOK_STDOUT" | jq -r '.hookSpecificOutput.additionalContext // empty' 2>/dev/null || true) + if [[ $HOOK_EXIT -eq 0 ]] && is_valid_json "$HOOK_STDOUT" \ + && [[ "$additional_context" == *"ACTIVE TASK JOURNAL AVAILABLE"* ]] \ + && [[ "$additional_context" == *"Task journal path:"* ]] \ + && [[ "$additional_context" == *".codex/task.md"* ]] \ + && [[ "$additional_context" != *"unique root building body"* ]]; then + pass + else + fail "exit=$HOOK_EXIT, Slice Status DONE completed a root BUILDING journal" + fi + rm -rf "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" +fi + +if test_start "session-start: Codex, nested bare Status DONE without root status → active task journal"; then + mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' +# Task +Task: unique nested status body + +## Slice +Status: DONE +EOF + + local_tmp_out=$(mktemp) + HOOK_EXIT=0 + env HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/session-start.sh" \ + > "$local_tmp_out" 2>/dev/null <<< '{"session_id":"test"}' || HOOK_EXIT=$? + HOOK_STDOUT=$(cat "$local_tmp_out") + rm -f "$local_tmp_out" + + additional_context=$(echo "$HOOK_STDOUT" | jq -r '.hookSpecificOutput.additionalContext // empty' 2>/dev/null || true) + if [[ $HOOK_EXIT -eq 0 ]] && is_valid_json "$HOOK_STDOUT" \ + && [[ "$additional_context" == *"ACTIVE TASK JOURNAL AVAILABLE"* ]] \ + && [[ "$additional_context" == *"Task journal path:"* ]] \ + && [[ "$additional_context" == *".codex/task.md"* ]] \ + && [[ "$additional_context" != *"unique nested status body"* ]]; then + pass + else + fail "exit=$HOOK_EXIT, nested bare Status DONE completed a journal without root status" + fi + rm -rf "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" +fi + if test_start "session-start: Codex, completed state dir with active state dir → active journal wins"; then mkdir -p "$TEST_PROJECT/.claude" "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" cat > "$TEST_PROJECT/.claude/task.md" <<'EOF' @@ -1155,7 +1221,7 @@ EOF cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' # Task Status: BUILDING -Step: active codex work +Step: active codex unique body EOF local_tmp_out=$(mktemp) @@ -1167,8 +1233,10 @@ EOF additional_context=$(echo "$HOOK_STDOUT" | jq -r '.hookSpecificOutput.additionalContext // empty' 2>/dev/null || true) if [[ $HOOK_EXIT -eq 0 ]] && is_valid_json "$HOOK_STDOUT" \ - && [[ "$additional_context" == *"ACTIVE TASK JOURNAL"* ]] \ - && [[ "$additional_context" == *"active codex work"* ]] \ + && [[ "$additional_context" == *"ACTIVE TASK JOURNAL AVAILABLE"* ]] \ + && [[ "$additional_context" == *"Task journal path:"* ]] \ + && [[ "$additional_context" == *".codex/task.md"* ]] \ + && [[ "$additional_context" != *"active codex unique body"* ]] \ && [[ "$additional_context" != *"old completed work"* ]]; then pass else @@ -1182,7 +1250,7 @@ if test_start "session-start: Codex, Status NOT DONE task journal → active tas cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' # Task Status: NOT DONE -Step: still active work +Step: unique not done session body EOF local_tmp_out=$(mktemp) @@ -1194,8 +1262,10 @@ EOF additional_context=$(echo "$HOOK_STDOUT" | jq -r '.hookSpecificOutput.additionalContext // empty' 2>/dev/null || true) if [[ $HOOK_EXIT -eq 0 ]] && is_valid_json "$HOOK_STDOUT" \ - && [[ "$additional_context" == *"ACTIVE TASK JOURNAL"* ]] \ - && [[ "$additional_context" == *"still active work"* ]]; then + && [[ "$additional_context" == *"ACTIVE TASK JOURNAL AVAILABLE"* ]] \ + && [[ "$additional_context" == *"Task journal path:"* ]] \ + && [[ "$additional_context" == *".codex/task.md"* ]] \ + && [[ "$additional_context" != *"unique not done session body"* ]]; then pass else fail "exit=$HOOK_EXIT, Status: NOT DONE was treated as completed" @@ -1279,14 +1349,18 @@ if test_start "post-compact: Claude, no task journal → memory protocol reminde rm -rf "$TEST_AGENT_HOME/.claude" fi -if test_start "post-compact: Claude, with task journal → re-injects content"; then +if test_start "post-compact: Claude, with task journal → compact reminder only"; then mkdir -p "$TEST_PROJECT/.claude" "$TEST_AGENT_HOME/.claude" - echo -e "# Task\nStatus: BUILDING" > "$TEST_PROJECT/.claude/task.md" + echo -e "# Task\nStatus: BUILDING\nStep: unique claude compact body" > "$TEST_PROJECT/.claude/task.md" HOME="$TEST_AGENT_HOME" run_hook post-compact.sh claude - if [[ $HOOK_EXIT -eq 0 && "$HOOK_STDOUT" == *"RESTORED AFTER COMPACTION"* ]]; then + if [[ $HOOK_EXIT -eq 0 \ + && "$HOOK_STDOUT" == *"RESTORED AFTER COMPACTION — Active task journal available"* \ + && "$HOOK_STDOUT" == *"Task journal path:"* \ + && "$HOOK_STDOUT" == *".claude/task.md"* \ + && "$HOOK_STDOUT" != *"unique claude compact body"* ]]; then pass else - fail "exit=$HOOK_EXIT, stdout missing RESTORED AFTER COMPACTION" + fail "exit=$HOOK_EXIT, expected compact PostCompact reminder without body" fi rm -rf "$TEST_PROJECT/.claude" "$TEST_AGENT_HOME/.claude" fi @@ -1311,7 +1385,7 @@ fi if test_start "post-compact: Codex, with task journal → universal PostCompact JSON"; then mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" - echo -e "# Task\nStatus: BUILDING\nStep: codex post compact" > "$TEST_PROJECT/.codex/task.md" + echo -e "# Task\nStatus: BUILDING\nStep: unique codex compact body" > "$TEST_PROJECT/.codex/task.md" local_tmp_out=$(mktemp) HOOK_EXIT=0 @@ -1324,8 +1398,10 @@ if test_start "post-compact: Codex, with task journal → universal PostCompact if [[ $HOOK_EXIT -eq 0 ]] \ && is_valid_json "$HOOK_STDOUT" \ && echo "$HOOK_STDOUT" | jq -e 'has("hookSpecificOutput") | not' >/dev/null 2>&1 \ - && [[ "$additional_context" == *"RESTORED AFTER COMPACTION"* ]] \ - && [[ "$additional_context" == *"codex post compact"* ]] \ + && [[ "$additional_context" == *"RESTORED AFTER COMPACTION — Active task journal available"* ]] \ + && [[ "$additional_context" == *"Task journal path:"* ]] \ + && [[ "$additional_context" == *".codex/task.md"* ]] \ + && [[ "$additional_context" != *"unique codex compact body"* ]] \ && [[ "$additional_context" == *"memory_context"* ]] \ && [[ "$additional_context" == *"Preserve response phase separation after compaction"* ]] \ && [[ "$additional_context" == *"progress/commentary updates are not final answers"* ]]; then @@ -1336,6 +1412,36 @@ if test_start "post-compact: Codex, with task journal → universal PostCompact rm -rf "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" fi +if test_start "post-compact: Codex, canonical Status DONE task journal → no active task journal reminder"; then + mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' +## Task: completed compact journal +Status: DONE +Step: unique completed compact body +EOF + + local_tmp_out=$(mktemp) + HOOK_EXIT=0 + env HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/post-compact.sh" \ + > "$local_tmp_out" 2>/dev/null <<< '{"hook_event_name":"PostCompact","turn_id":"test"}' || HOOK_EXIT=$? + HOOK_STDOUT=$(cat "$local_tmp_out") + rm -f "$local_tmp_out" + + additional_context=$(echo "$HOOK_STDOUT" | jq -r '.systemMessage // empty' 2>/dev/null || true) + if [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e 'has("hookSpecificOutput") | not' >/dev/null 2>&1 \ + && [[ "$additional_context" != *"RESTORED AFTER COMPACTION — Active task journal available"* ]] \ + && [[ "$additional_context" != *"ACTIVE TASK JOURNAL AVAILABLE"* ]] \ + && [[ "$additional_context" != *"unique completed compact body"* ]] \ + && [[ "$additional_context" == *"memory_context"* ]]; then + pass + else + fail "exit=$HOOK_EXIT, completed Codex PostCompact journal was injected" + fi + rm -rf "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex" +fi + echo "" # ── stop-review.sh tests ───────────────────────────────────────────────────── @@ -5725,6 +5831,32 @@ if test_start "workflow-guard: Edit with DONE status → no output"; then fi fi +if test_start "workflow-guard: Codex nested slice BUILDING without root status → no active Build block"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +## Task: no root status journal + +## Slice +Status: BUILDING +TASK + echo '{"tool_name": "apply_patch", "tool_input": {"command": "*** Begin Patch\n*** Update File: src/App.cs\n*** End Patch"}}' | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if [[ $HOOK_EXIT -eq 0 && -z "$HOOK_STDOUT" ]]; then + pass + elif [[ $HOOK_EXIT -eq 0 ]] && ! echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ + && ! echo "$HOOK_STDOUT" | grep -q "active Build/Verify"; then + pass + else + fail "exit=$HOOK_EXIT, nested slice status triggered active Build/Verify block; stdout='$HOOK_STDOUT'" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + if test_start "workflow-guard: Edit with BUILDING status → outputs warning"; then mkdir -p "$TEST_PROJECT/.claude" echo -e "Status: BUILDING [step 2/3]" > "$TEST_PROJECT/.claude/task.md" @@ -6118,6 +6250,80 @@ echo "" echo "task-journal-resolver.sh" +if test_start "task-journal-resolver: canonical task status tokens complete journal"; then + RESOLVER_TEST_PROJECT=$(mktemp -d) + mkdir -p "$RESOLVER_TEST_PROJECT/.codex" + resolver_failed_status="" + + for resolver_status in DONE COMPLETE COMPLETED; do + cat > "$RESOLVER_TEST_PROJECT/.codex/task.md" < "$RESOLVER_TEST_PROJECT/.codex/task.md" <<'EOF' +## Task: active root journal +Status: BUILDING + +## Slice Status: DONE +EOF + if bash -c ' + set -euo pipefail + . "$1" + assistant_task_journal_completed "$2" + ' bash "$HOOKS_DIR/task-journal-resolver.sh" "$RESOLVER_TEST_PROJECT/.codex/task.md"; then + resolver_nested_ok=false + resolver_failed_case="root BUILDING plus slice status" + fi + + cat > "$RESOLVER_TEST_PROJECT/.codex/task.md" <<'EOF' +## Task: no root status journal + +## Slice +Status: DONE +EOF + if $resolver_nested_ok && bash -c ' + set -euo pipefail + . "$1" + assistant_task_journal_completed "$2" + ' bash "$HOOKS_DIR/task-journal-resolver.sh" "$RESOLVER_TEST_PROJECT/.codex/task.md"; then + resolver_nested_ok=false + resolver_failed_case="nested bare slice status without root status" + fi + + if $resolver_nested_ok; then + pass + else + fail "$resolver_failed_case was treated as completed" + fi + + rm -rf "$RESOLVER_TEST_PROJECT" +fi + if test_start "task-journal-resolver: cache writes stay silent on permission failure"; then RESOLVER_TEST_HOME=$(mktemp -d) RESOLVER_TEST_PROJECT=$(mktemp -d) @@ -6350,7 +6556,7 @@ if test_start "codex default install: workflow hooks install and compaction hook fail "Codex default hooks.json did not register workflow/delegation hooks" else mkdir -p "$TEST_PROJECT/.codex" "$INSTALL_TEST_HOME/.codex" - echo -e "# Task\nStatus: BUILDING\nStep: installed codex compaction" > "$TEST_PROJECT/.codex/task.md" + echo -e "# Task\nStatus: BUILDING\nStep: unique installed codex compaction body" > "$TEST_PROJECT/.codex/task.md" local_tmp_out=$(mktemp) HOOK_EXIT=0 @@ -6366,8 +6572,11 @@ if test_start "codex default install: workflow hooks install and compaction hook additional_context=$(echo "$HOOK_STDOUT" | jq -r '.systemMessage // empty' 2>/dev/null || true) if [[ $HOOK_EXIT -eq 0 ]] \ && is_valid_json "$HOOK_STDOUT" \ - && echo "$HOOK_STDOUT" | jq -e 'has("hookSpecificOutput") | not' >/dev/null 2>&1 \ - && [[ "$additional_context" == *"installed codex compaction"* ]]; then + && echo "$HOOK_STDOUT" | jq -e 'has("systemMessage") and (has("hookSpecificOutput") | not)' >/dev/null 2>&1 \ + && [[ "$additional_context" == *"RESTORED AFTER COMPACTION — Active task journal available"* ]] \ + && [[ "$additional_context" == *"Task journal path:"* ]] \ + && [[ "$additional_context" == *".codex/task.md"* ]] \ + && [[ "$additional_context" != *"unique installed codex compaction body"* ]]; then pass else fail "installed Codex post-compact hook did not emit universal JSON, stdout='$HOOK_STDOUT', install_stdout='$INSTALL_STDOUT'" @@ -6833,6 +7042,35 @@ EOF rm -rf "$TEST_PROJECT/.codex" fi +if test_start "workflow-enforcer: Codex, nested Status DONE without root status → UNKNOWN phase"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'EOF' +## Task: nested status phase regression +Triaged as: small +Plan approval: yes + +## Slice +Status: DONE +EOF + echo '{"prompt": "continue", "hook_event_name": "UserPromptSubmit"}' | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-enforcer.sh" \ + > /tmp/_wf_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wf_out) + rm -f /tmp/_wf_out + if [[ $HOOK_EXIT -eq 0 ]] && echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.additionalContext' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | grep -q "Task: nested status phase regression" \ + && echo "$HOOK_STDOUT" | grep -q "Phase: UNKNOWN" \ + && ! echo "$HOOK_STDOUT" | grep -q "Phase: DONE" \ + && ! echo "$HOOK_STDOUT" | grep -q "WORKFLOW RULES"; then + pass + else + fail "exit=$HOOK_EXIT, nested slice Status DONE became task phase; stdout='$HOOK_STDOUT'" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + if test_start "workflow-enforcer: pending clarification on medium → includes clarification gate warning"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'EOF' From 75fbb9f5757fb3a697ca1b1f39308c56b4d58a03 Mon Sep 17 00:00:00 2001 From: Laimis Date: Fri, 3 Jul 2026 06:38:47 +0300 Subject: [PATCH 5/9] Reduce workflow harness overhead --- agents/claude/qa-evaluator.md | 2 +- agents/codex/qa-evaluator.toml | 2 +- docs/hook-output-benchmarks.md | 6 +- docs/instruction-overload-reduction.md | 6 + hooks/scripts/stop-review.sh | 141 ++---- hooks/scripts/workflow-enforcer.sh | 21 +- hooks/scripts/workflow-phase-gates.sh | 434 +++++++++++++++-- .../skills/assistant-review/SKILL.md | 2 +- .../assistant-review/contracts/input.yaml | 24 +- .../references/qa-evaluation-loop.md | 8 +- .../references/harness-patterns.md | 2 +- .../skills/assistant-workflow/SKILL.md | 7 +- .../assistant-workflow/contracts/input.yaml | 24 +- .../assistant-workflow/contracts/output.yaml | 2 +- .../contracts/phase-gates.yaml | 3 +- .../references/harness-controller.md | 20 +- .../assistant-workflow/references/phases.md | 12 +- .../references/plan-harness-appendix.md | 99 ++++ .../references/plan-template.md | 190 ++------ .../task-journal-harness-appendix.md | 145 ++++++ .../references/task-journal-template.md | 181 ++----- skills/assistant-review/SKILL.md | 2 +- skills/assistant-review/contracts/input.yaml | 24 +- .../references/qa-evaluation-loop.md | 8 +- .../references/harness-patterns.md | 2 +- skills/assistant-workflow/SKILL.md | 7 +- .../assistant-workflow/contracts/input.yaml | 24 +- .../assistant-workflow/contracts/output.yaml | 2 +- .../contracts/phase-gates.yaml | 3 +- .../references/harness-controller.md | 20 +- .../assistant-workflow/references/phases.md | 12 +- .../references/plan-harness-appendix.md | 99 ++++ .../references/plan-template.md | 190 ++------ .../task-journal-harness-appendix.md | 145 ++++++ .../references/task-journal-template.md | 181 ++----- tests/p0-p4/domain-rubrics-contracts.sh | 0 tests/p0-p4/harness-controller-contracts.sh | 82 +++- tests/p0-p4/instruction-overload-contracts.sh | 43 +- tests/p0-p4/qa-evaluator-contracts.sh | 40 +- tests/p0-p4/review-loop-cap-contracts.sh | 5 +- tests/p0-p4/runtime-phase-gate-contracts.sh | 33 +- tests/p0-p4/task-packet-contracts.sh | 6 +- tests/p0-p4/worker-status-contracts.sh | 27 +- tests/test-hooks.sh | 457 +++++++++++++++++- tools/hooks/benchmark-hook-output.sh | 18 + 45 files changed, 1921 insertions(+), 840 deletions(-) create mode 100644 plugins/assistant-dev/skills/assistant-workflow/references/plan-harness-appendix.md create mode 100644 plugins/assistant-dev/skills/assistant-workflow/references/task-journal-harness-appendix.md create mode 100644 skills/assistant-workflow/references/plan-harness-appendix.md create mode 100644 skills/assistant-workflow/references/task-journal-harness-appendix.md mode change 100644 => 100755 tests/p0-p4/domain-rubrics-contracts.sh diff --git a/agents/claude/qa-evaluator.md b/agents/claude/qa-evaluator.md index faca825..2c3cf8d 100644 --- a/agents/claude/qa-evaluator.md +++ b/agents/claude/qa-evaluator.md @@ -40,7 +40,7 @@ Then return: ## Status meanings - `DONE`: QA evaluation complete and final_verdict is accepted -- `DONE_WITH_CONCERNS`: QA evaluation complete with accepted_with_concerns or non-blocking risks +- `DONE_WITH_CONCERNS`: QA evaluation complete with accepted_with_concerns, non-blocking risks, or final_verdict=rejected and result=HAS_REMAINING_ITEMS when failed acceptance items should return to Build before round 10 or be reported at terminal round 10 - `NEEDS_CONTEXT`: missing Done Contract, acceptance criteria, verification evidence, or domain/rubric context prevents evaluation - `BLOCKED`: environment, permission, or unavailable evidence prevents evaluation diff --git a/agents/codex/qa-evaluator.toml b/agents/codex/qa-evaluator.toml index 022827c..7863afd 100644 --- a/agents/codex/qa-evaluator.toml +++ b/agents/codex/qa-evaluator.toml @@ -38,7 +38,7 @@ Then return: ## Status meanings - `DONE`: QA evaluation complete and final_verdict is accepted -- `DONE_WITH_CONCERNS`: QA evaluation complete with accepted_with_concerns or non-blocking risks +- `DONE_WITH_CONCERNS`: QA evaluation complete with accepted_with_concerns, non-blocking risks, or final_verdict=rejected and result=HAS_REMAINING_ITEMS when failed acceptance items should return to Build before round 10 or be reported at terminal round 10 - `NEEDS_CONTEXT`: missing Done Contract, acceptance criteria, verification evidence, or domain/rubric context prevents evaluation - `BLOCKED`: environment, permission, or unavailable evidence prevents evaluation diff --git a/docs/hook-output-benchmarks.md b/docs/hook-output-benchmarks.md index b46ecd8..e97955b 100644 --- a/docs/hook-output-benchmarks.md +++ b/docs/hook-output-benchmarks.md @@ -13,10 +13,10 @@ Current rows: 7 | hook_name | scenario | stdout_bytes | stdout_words | stderr_bytes | exit_code | first_blocker_or_action | |---|---|---:|---:|---:|---:|---| | session-start | codex active task journal | 980 | 102 | 0 | 0 | ACTIVE TASK JOURNAL AVAILABLE | -| workflow-enforcer | codex building phase gates | 1626 | 184 | 0 | 0 | WORKFLOW STATE (auto-injected every prompt): | +| workflow-enforcer | codex building phase gates | 1511 | 164 | 0 | 0 | WORKFLOW STATE (auto-injected every prompt): | | post-compact | codex restored active task | 749 | 75 | 0 | 0 | RESTORED AFTER COMPACTION — Active task journal available | | skill-router | codex assistant-workflow route | 498 | 47 | 0 | 0 | SKILL MATCH (1/1): This request matches assistant-workflow. You MUST load and follow this SKILL.md and its contracts before acting. At minimum: triage the task size, then build with tests included in the Build phase. Skipping workflow for speed is explicitly prohibited. Required inputs for this skill: [task_description,task_type,clarification_status,unresolved_clarification_topics,clarification_defaults_applied] | -| stop-review | codex missing spec review blocker | 371 | 54 | 0 | 0 | Task journal shows an active workflow but no Spec Review was run. You MUST run Stage 1 first: load references/prompts/spec-review.md, compare each approved plan step/task packet/component against actual changes, append a structured Spec Review entry with Result: PASS or FAIL, fix any FAIL items, then continue to quality review. | +| stop-review | codex missing spec review blocker | 207 | 26 | 0 | 0 | review_gate:no_spec_review missing=no Spec Review; ### Spec Review #N action=run Spec Review first and record PASS/FAIL plus scope, mismatch, and required-fix fields | | subagent-monitor | codex code-writer start | 200 | 20 | 0 | 0 | SUBAGENT CONSTRAINT: You are a code writer. Write code only. Do NOT run builds or tests — builder-tester handles that. | | subagent-monitor | codex code-writer stop | 0 | 0 | 0 | 0 | recorded SubagentStop lifecycle evidence | @@ -29,6 +29,8 @@ The C slice trimmed repeated explanatory prose from `session-start`, `workflow-e | c-hook-output-trim | session-start | 2122 | 225 | 1766 | 183 | | c-hook-output-trim | workflow-enforcer | 1892 | 232 | 1625 | 184 | | c-hook-output-trim | post-compact | 2131 | 218 | 1526 | 154 | +| runtime-gate-output-trim | workflow-enforcer | 1626 | 184 | 1511 | 164 | +| runtime-gate-output-trim | stop-review | 371 | 54 | 207 | 26 | ## Signal Anchors Checked diff --git a/docs/instruction-overload-reduction.md b/docs/instruction-overload-reduction.md index 2255af4..659f180 100644 --- a/docs/instruction-overload-reduction.md +++ b/docs/instruction-overload-reduction.md @@ -99,6 +99,12 @@ Strict hook templates now register one stop gate: `stop-review.sh`. It owns the `harness-gate.sh` remains as a legacy script with tests, but it is no longer installed by strict hook templates as a second competing stop hook. +### Phase-scoped hook warnings + +`workflow-phase-gates.sh` now owns runtime reason keys plus exact missing field/action text. `workflow-enforcer.sh` formats only current-phase warnings: BUILDING surfaces missing plan and build-role subagent evidence, REVIEWING surfaces review evidence, and DOCUMENTING surfaces metrics. Future-phase subagent gaps remain visible in the gate state line without adding recovery prose too early. + +`stop-review.sh` uses compact blockers in the form `gate:key missing=... action=...`, so stop output keeps the blocker reason, exact missing field, and recovery action without restating the full workflow policy. + ### Generated plugin-local skill mirrors Root `skills/assistant-*` directories remain the source of truth. Plugin-local skill copies under `plugins/*/skills/` are treated as generated release artifacts and are checked/regenerated by: diff --git a/hooks/scripts/stop-review.sh b/hooks/scripts/stop-review.sh index 4ef2326..6d3aff8 100755 --- a/hooks/scripts/stop-review.sh +++ b/hooks/scripts/stop-review.sh @@ -79,36 +79,41 @@ if ! assistant_phase_status_is_lifecycle_active "$status"; then exit 0 fi -# Medium+ strict harness checks are consolidated here instead of registering a -# second Stop/AfterAgent hook. Emit only the first actionable blocker. -if assistant_phase_is_medium_plus "$TASK_FILE"; then - has_plan=$(grep -m1 -E "^(Plan approval:|## Plan)" "$TASK_FILE" 2>/dev/null || true) - if [[ -z "$has_plan" ]]; then - HARNESS_REASON="No plan found in task journal. Medium+ strict workflows require an approved plan before building. Run the Plan phase first." - elif ! assistant_phase_has_plan_approval "$TASK_FILE"; then - HARNESS_REASON="Plan exists but is not approved. Present the plan, wait for user approval, and record Plan approval: yes before continuing." - fi -fi +emit_stop_reason() { + local reason="$1" -if [[ -n "${HARNESS_REASON:-}" ]]; then if $IS_GEMINI; then touch "${RETRY_FLAG}" - jq -n --arg reason "$HARNESS_REASON" '{decision: "retry", reason: $reason}' + jq -n --arg reason "$reason" '{decision: "retry", reason: $reason}' else - jq -n --arg reason "$HARNESS_REASON" '{decision: "block", reason: $reason}' + jq -n --arg reason "$reason" '{decision: "block", reason: $reason}' fi +} + +compact_stop_reason() { + local gate="$1" + local key="$2" + local field action + + field="$(assistant_phase_reason_missing_field "$gate" "$key")" + action="$(assistant_phase_reason_action "$gate" "$key")" + printf '%s:%s missing=%s action=%s\n' "$gate" "$key" "$field" "$action" +} + +# Medium+ strict harness checks are consolidated here instead of registering a +# second Stop/AfterAgent hook. Emit only the first actionable blocker. +if assistant_phase_is_medium_plus "$TASK_FILE"; then + plan_missing_key="$(assistant_phase_plan_missing_reason_key "$TASK_FILE")" +fi + +if [[ "${plan_missing_key:-complete}" != "complete" ]]; then + emit_stop_reason "$(compact_stop_reason "plan_gate" "$plan_missing_key")" exit 0 fi subagent_missing_key="$(assistant_phase_subagent_evidence_missing_reason_key "$TASK_FILE")" if [[ "$subagent_missing_key" != "complete" ]]; then - SUBAGENT_REASON="Task journal subagent evidence gate failed ($subagent_missing_key). If authorization is required, ask once for the needed subagent delegation scope and wait before continuing phases that require subagents. For delegated mode, record Agent Dispatch Log evidence with dispatch/result entries for every required workflow role (Code Mapper/Explorer/Architect during discovery/decomposition/planning when required, Code Writer and Builder/Tester during Build when required, and Code Reviewer during Review; legacy Reviewer labels are compatibility routing only). Medium+ implementation work also needs per-slice dispatch evidence when implementation slices exist. For direct_fallback, record an explicit reason (authorization_denied, subagents_unavailable, or policy_disallowed) plus role-equivalent direct evidence for every required workflow role, including Code Reviewer direct evidence during Review. Silent fallback, unresolved authorization, inline review in delegated mode, or not_applicable with required roles cannot complete." - if $IS_GEMINI; then - touch "${RETRY_FLAG}" - jq -n --arg reason "$SUBAGENT_REASON" '{decision: "retry", reason: $reason}' - else - jq -n --arg reason "$SUBAGENT_REASON" '{decision: "block", reason: $reason}' - fi + emit_stop_reason "$(compact_stop_reason "subagent_evidence_gate" "$subagent_missing_key")" exit 0 fi @@ -119,62 +124,8 @@ fi # 3. Final result must have a "- Result:" line after that Quality Review review_missing_key="$(assistant_phase_review_missing_reason_key "$TASK_FILE")" -if [[ "$review_missing_key" == "no_spec_review" ]]; then - REVIEW_REASON="Task journal shows an active workflow but no Spec Review was run. You MUST run Stage 1 first: load references/prompts/spec-review.md, compare each approved plan step/task packet/component against actual changes, append a structured Spec Review entry with Result: PASS or FAIL, fix any FAIL items, then continue to quality review." -elif [[ "$review_missing_key" == "spec_not_pass" ]]; then - REVIEW_REASON="Task journal has a Spec Review entry, but the latest structured spec compliance result is not PASS. Fix required spec issues, re-test, and re-run Spec Review until it records '- Result: PASS' before quality review can satisfy the review cycle." -elif [[ "$review_missing_key" == "no_quality_review" ]]; then - REVIEW_REASON="Task journal has Spec Review PASS but no Quality Review. You MUST run Stage 2 separately: load assistant-review SKILL.md and contracts, run the autonomous quality review loop, and append a Quality Review entry. Quality review cannot substitute for Spec Review, and Spec Review cannot substitute for quality review." -elif [[ "$review_missing_key" == "no_final_result" ]]; then - REVIEW_REASON="Task journal has review entries but the review cycle is not complete — no Final Result found. You must finish the review cycle: fix remaining must-fix issues, re-test, re-review, and write the Final Result summary in the Review Log section of the task journal." -elif [[ "$review_missing_key" == "qa_rejected" ]]; then - REVIEW_REASON="QA required and the latest QA Evaluation was rejected (qa_rejected). Fix the QA findings, rerun QA, and record QA Evaluation evidence; the QA final verdict/result must be accepted or accepted_with_concerns before stopping." -elif [[ "$review_missing_key" == "qa_blocked" ]]; then - REVIEW_REASON="QA required and the latest QA Evaluation is blocked (qa_blocked). Resolve the QA blocker, rerun QA, and record QA Evaluation evidence; the QA final verdict/result must be accepted or accepted_with_concerns before stopping." -elif [[ "$review_missing_key" == "qa_final_result_missing" ]]; then - REVIEW_REASON="QA required but the QA final verdict/result is missing (qa_final_result_missing). Run or rerun QA and record QA Evaluation evidence; the QA final verdict/result must be accepted or accepted_with_concerns before stopping." -elif [[ "$review_missing_key" == "qa_not_accepted" ]]; then - REVIEW_REASON="QA required but the QA final verdict/result is not accepted (qa_not_accepted). Fix or rerun QA and record QA Evaluation evidence; the QA final verdict/result must be accepted or accepted_with_concerns before stopping." -elif [[ "$review_missing_key" == "missing_review_round" ]]; then - REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_review_round). Add '- Round: N of 10' to the latest Quality Review entry after the latest Spec Review PASS." -elif [[ "$review_missing_key" == "round_overflow" ]]; then - REVIEW_REASON="Medium+ Quality Review has invalid controller evidence (round_overflow). Use '- Round: N of 10' with N between 1 and 10, then rerun or repair the review loop." -elif [[ "$review_missing_key" == "missing_findings_summary" ]]; then - REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_findings_summary). Add a findings summary such as '- Found this round: 0 must-fix, 0 should-fix, 0 nits' to the latest Quality Review." -elif [[ "$review_missing_key" == "missing_rubric_scores" ]]; then - REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_rubric_scores). Add '- Rubric:' to the latest Quality Review with numeric 0..5 scores for correctness, code_quality or quality, architecture, security, and test_coverage or coverage." -elif [[ "$review_missing_key" == "missing_weighted_score" ]]; then - REVIEW_REASON="Medium+ Quality Review has a missing, invalid, or mismatched weighted score (missing_weighted_score). Add the latest '- Weighted: N.NN' score to the Quality Review entry and ensure it matches the rubric formula from references/review-rubric.md." -elif [[ "$review_missing_key" == "missing_delta_from_previous" ]]; then - REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_delta_from_previous). For review rounds after round 1, record the actual previous Quality Review block after the latest Spec Review PASS with valid '- Found this round:' and '- Weighted:' lines, then add '- Delta from previous: ...' to the latest round." -elif [[ "$review_missing_key" == "missing_drift_check" ]]; then - REVIEW_REASON="Medium+ Quality Review is missing controller evidence (missing_drift_check). For review rounds after round 1, add '- Drift check: GENUINE' or an explicit regression/drift classification." -elif [[ "$review_missing_key" == "missing_score_progression" ]]; then - REVIEW_REASON="Medium+ Final Result is missing controller evidence (missing_score_progression). Add '- Score progression: ...' after the final review result so score movement is explicit." -elif [[ "$review_missing_key" == "weighted_score_below_pass" ]]; then - REVIEW_REASON="Medium+ review cannot finish as CLEAN or ISSUES_FIXED because the latest weighted score is below the 4.00 pass threshold (weighted_score_below_pass). Improve the lowest-scoring dimensions and rerun Quality Review." -elif [[ "$review_missing_key" == "unresolved_findings" ]]; then - REVIEW_REASON="Medium+ review cannot finish as CLEAN or ISSUES_FIXED while the latest Quality Review still lists must-fix or should-fix findings (unresolved_findings). Fix or explicitly carry remaining items through the review loop." -elif [[ "$review_missing_key" == "missing_remaining_rationale" ]]; then - REVIEW_REASON="Medium+ Final Result reports HAS_REMAINING_ITEMS without a concrete remaining-item or blocker rationale (missing_remaining_rationale). Add '- Remaining items:' or '- Blocker:' with specific unresolved work, evidence, and owner." -fi - if [[ "$review_missing_key" != "complete" ]]; then - if $IS_GEMINI; then - # Gemini AfterAgent: "retry" forces another agent loop - # Mark retry flag so next invocation exits (prevents infinite loop) - touch "${RETRY_FLAG}" - jq -n --arg reason "$REVIEW_REASON" '{ - decision: "retry", - reason: $reason - }' - else - # Claude Stop: "block" prevents the stop - jq -n --arg reason "$REVIEW_REASON" '{ - decision: "block", - reason: $reason - }' - fi + emit_stop_reason "$(compact_stop_reason "review_gate" "$review_missing_key")" exit 0 fi @@ -183,50 +134,20 @@ METRICS_FILE="$(assistant_phase_metrics_file)" TODAY=$(date +%Y-%m-%d) if ! assistant_phase_has_metrics_today; then - METRICS_REASON="Review is complete but no metrics entry was recorded for today ($TODAY). Append a JSONL entry to $METRICS_FILE with task details (date, project, task, size, review_rounds, etc.) before stopping. Format: {\"date\":\"$TODAY\",\"project\":\"[name]\",\"task\":\"[description]\",\"size\":\"[size]\",\"retriage\":false,\"review_rounds\":N,\"plan_deviations\":N,\"build_failures\":N,\"criteria_defined\":N,\"criteria_skipped\":[],\"agent_readiness_score\":null,\"components_count\":null,\"components_verified\":null}" + metrics_field="$(assistant_phase_reason_missing_field "metrics_gate" "missing_metrics_today")" + metrics_action="$(assistant_phase_reason_action "metrics_gate" "missing_metrics_today")" + METRICS_REASON="metrics_gate:missing_metrics_today missing=$metrics_field at $METRICS_FILE date=$TODAY action=$metrics_action" fi if [[ -n "${METRICS_REASON:-}" ]]; then - if $IS_GEMINI; then - touch "${RETRY_FLAG}" - jq -n --arg reason "$METRICS_REASON" '{decision: "retry", reason: $reason}' - else - jq -n --arg reason "$METRICS_REASON" '{decision: "block", reason: $reason}' - fi + emit_stop_reason "$METRICS_REASON" exit 0 fi learning_missing_key="$(assistant_phase_learning_missing_reason_key "$TASK_FILE")" -if [[ "$learning_missing_key" == "no_learning_controller" ]]; then - LEARNING_REASON="Medium+ DOCUMENTING task is missing the canonical Learning Controller block (no_learning_controller). Add '### Learning Controller' with Memory trend checked, Learning evidence reviewed, Review findings considered, Build/test failures considered, User corrections considered, Durable lesson decision, Persistence evidence, and No-save rationale when no durable write occurred." -elif [[ "$learning_missing_key" == "missing_memory_trend_checked" ]]; then - LEARNING_REASON="Learning Controller is missing valid memory trend evidence (missing_memory_trend_checked). Add '- Memory trend checked: checked', 'backend_unavailable', 'policy_disallowed', or 'not_configured'." -elif [[ "$learning_missing_key" == "missing_learning_evidence_reviewed" ]]; then - LEARNING_REASON="Learning Controller is missing reviewed evidence (missing_learning_evidence_reviewed). Under '- Learning evidence reviewed:', add at least one evidence item such as review_finding, build_test_failure, user_correction, memory_trend, or none with a reason." -elif [[ "$learning_missing_key" == "missing_review_findings_considered" ]]; then - LEARNING_REASON="Learning Controller is missing review finding consideration (missing_review_findings_considered). Under '- Review findings considered:', add the finding lesson decision or an explicit none-with-reason item." -elif [[ "$learning_missing_key" == "missing_build_test_failures_considered" ]]; then - LEARNING_REASON="Learning Controller is missing build/test failure consideration (missing_build_test_failures_considered). Under '- Build/test failures considered:', add the failure lesson decision or an explicit none-with-reason item." -elif [[ "$learning_missing_key" == "missing_user_corrections_considered" ]]; then - LEARNING_REASON="Learning Controller is missing user correction consideration (missing_user_corrections_considered). Under '- User corrections considered:', add the correction lesson decision or an explicit none-with-reason item." -elif [[ "$learning_missing_key" == "missing_durable_lesson_decision" ]]; then - LEARNING_REASON="Learning Controller is missing a valid durable lesson decision (missing_durable_lesson_decision). Add '- Durable lesson decision: durable_saved', 'durable_updated', 'skipped_not_durable', 'backend_unavailable', 'policy_disallowed', or 'refused_sensitive'." -elif [[ "$learning_missing_key" == "missing_persistence_evidence" ]]; then - LEARNING_REASON="Learning Controller is missing persistence evidence (missing_persistence_evidence). For durable_saved or durable_updated, add non-empty memory_reflect, memory_add_insight, or backend evidence; N/A, none, and TBD do not satisfy saved/updated decisions." -elif [[ "$learning_missing_key" == "missing_no_save_rationale" ]]; then - LEARNING_REASON="Learning Controller is missing a no-save rationale (missing_no_save_rationale). When no durable write occurred, add '- No-save rationale:' with a concrete reason; N/A, none, and TBD do not satisfy this gate." -else - LEARNING_REASON="Learning Controller gate failed ($learning_missing_key). Complete the canonical ### Learning Controller block before stopping." -fi - if [[ "$learning_missing_key" != "complete" ]]; then - if $IS_GEMINI; then - touch "${RETRY_FLAG}" - jq -n --arg reason "$LEARNING_REASON" '{decision: "retry", reason: $reason}' - else - jq -n --arg reason "$LEARNING_REASON" '{decision: "block", reason: $reason}' - fi + emit_stop_reason "$(compact_stop_reason "learning_gate" "$learning_missing_key")" exit 0 fi diff --git a/hooks/scripts/workflow-enforcer.sh b/hooks/scripts/workflow-enforcer.sh index c905bf7..8729619 100755 --- a/hooks/scripts/workflow-enforcer.sh +++ b/hooks/scripts/workflow-enforcer.sh @@ -475,24 +475,33 @@ SUBAGENT AUTHORIZATION GATE: - Do not switch to direct_fallback unless the user denies authorization, policy disallows spawning, or a real spawn attempt proves subagents unavailable." fi -if [[ "$subagent_gate_status" != "complete" ]]; then +if [[ "$is_building" == "yes" && "$has_plan_approval" == "no" && "$size" != "small" && "$size" != "trivial" ]]; then + plan_missing_key="$(assistant_phase_plan_missing_reason_key "$TASK_FILE")" + plan_missing_field="$(assistant_phase_reason_missing_field "plan_gate" "$plan_missing_key")" + plan_action="$(assistant_phase_reason_action "plan_gate" "$plan_missing_key")" context+=" -WARNING: Subagent evidence gate incomplete ($subagent_gate_status). If execution mode is delegated, dispatch and record every required workflow role before moving on; if using direct_fallback, record a valid fallback reason plus role-equivalent evidence. Do not silently complete Discovery or Review inline when delegated." +WARNING: You are BUILDING without an approved plan ($plan_missing_key). missing=$plan_missing_field action=$plan_action" fi -if [[ "$is_building" == "yes" && "$has_plan_approval" == "no" && "$size" != "small" && "$size" != "trivial" ]]; then +subagent_warning_key="$(assistant_phase_subagent_warning_reason_key "$TASK_FILE" "$subagent_gate_status" "$status" || true)" +subagent_warning_action="$(assistant_phase_subagent_warning_action "$subagent_warning_key" "$status" || true)" +if [[ -n "$subagent_warning_action" ]]; then context+=" -WARNING: You are BUILDING without an approved plan. Medium+ tasks require plan approval first. STOP and get plan approved." +WARNING: Subagent evidence gate incomplete ($subagent_warning_key). $subagent_warning_action" fi if [[ ( "$is_reviewing" == "yes" || "$is_documenting" == "yes" ) && "$has_review_completion" == "no" ]]; then + review_missing_field="$(assistant_phase_reason_missing_field "review_gate" "$review_gate_status")" + review_action="$(assistant_phase_reason_action "review_gate" "$review_gate_status")" context+=" -WARNING: Review gate incomplete ($review_gate_status). Complete structured Spec Review PASS, Quality Review, and Final Result before leaving REVIEW/DOCUMENT." +WARNING: Review gate incomplete ($review_gate_status). missing=$review_missing_field action=$review_action" fi if [[ "$is_documenting" == "yes" && "$has_metrics_today" == "no" ]]; then + metrics_missing_field="$(assistant_phase_reason_missing_field "metrics_gate" "missing_metrics_today")" + metrics_action="$(assistant_phase_reason_action "metrics_gate" "missing_metrics_today")" context+=" -WARNING: Metrics gate incomplete. Record today's workflow metrics before finishing DOCUMENT." +WARNING: Metrics gate incomplete. missing=$metrics_missing_field action=$metrics_action" fi if [[ "$clarification_gate_active" == "yes" && "$requires_saved_clarification_state" == "yes" ]]; then diff --git a/hooks/scripts/workflow-phase-gates.sh b/hooks/scripts/workflow-phase-gates.sh index 393ebb5..347e452 100644 --- a/hooks/scripts/workflow-phase-gates.sh +++ b/hooks/scripts/workflow-phase-gates.sh @@ -53,6 +53,20 @@ assistant_phase_has_plan_approval() { grep -qE "(^Plan approval:.*yes|PLAN COMPLETE \(approved\))" "$file" 2>/dev/null } +assistant_phase_plan_missing_reason_key() { + local file="$1" + local has_plan + + has_plan="$(grep -m1 -E "^(Plan approval:|## Plan)" "$file" 2>/dev/null || true)" + if [[ -z "$has_plan" ]]; then + printf 'no_plan\n' + elif ! assistant_phase_has_plan_approval "$file"; then + printf 'plan_not_approved\n' + else + printf 'complete\n' + fi +} + assistant_phase_has_spec_review_entry() { local file="$1" grep -qE "^### Spec Review #[0-9]+" "$file" 2>/dev/null @@ -1345,7 +1359,7 @@ assistant_phase_subagent_policy_state() { assistant_phase_scalar_field "$1" "Subagent policy state" } -assistant_phase_labeled_evidence_value() { +assistant_phase_exact_labeled_evidence_value() { local file="$1" local label="$2" awk -v label="$label" ' @@ -1364,6 +1378,35 @@ assistant_phase_labeled_evidence_value() { ' "$file" 2>/dev/null } +assistant_phase_compact_labeled_evidence_value() { + local file="$1" + local label="$2" + local role + + case "$label" in + *" dispatch") role="${label% dispatch}" ;; + *" result") role="${label% result}" ;; + *" direct evidence") role="${label% direct evidence}" ;; + *) return 1 ;; + esac + + assistant_phase_exact_labeled_evidence_value "$file" "$role dispatch/result/direct evidence" +} + +assistant_phase_labeled_evidence_value() { + local file="$1" + local label="$2" + local value + + value="$(assistant_phase_exact_labeled_evidence_value "$file" "$label")" + if [[ -n "$value" ]]; then + printf '%s\n' "$value" + return 0 + fi + + assistant_phase_compact_labeled_evidence_value "$file" "$label" +} + assistant_phase_has_labeled_evidence() { local file="$1" local label="$2" @@ -1493,52 +1536,6 @@ assistant_phase_text_mentions_agent_id_token() { ' } -assistant_phase_event_is_reviewer_compat() { - local json_line="$1" - local agent_type agent_name - agent_type="$(assistant_phase_json_field_value "$json_line" "agent_type" | tr '[:upper:]' '[:lower:]')" - agent_name="$(assistant_phase_json_field_value "$json_line" "agent_name" | tr '[:upper:]' '[:lower:]')" - [[ "$agent_type" == "reviewer" || "$agent_name" == "reviewer" ]] -} - -assistant_phase_codex_reviewer_event_ids() { - local file="$1" - local event="$2" - local events_file line id - events_file="$(assistant_phase_subagent_events_file "$file")" - [[ -f "$events_file" ]] || return 1 - while IFS= read -r line; do - [[ "$line" == *"\"event\":\"$event\""* ]] || continue - assistant_phase_event_is_reviewer_compat "$line" || continue - id="$(assistant_phase_json_field_value "$line" "agent_id")" - [[ -n "$id" ]] || continue - printf '%s\n' "$id" - done < "$events_file" -} - -assistant_phase_has_qa_reviewer_compat_event_pair_evidence() { - local file="$1" - local start_id - while IFS= read -r start_id; do - [[ -n "$start_id" ]] || continue - assistant_phase_journal_mentions_agent_id "$file" "QA Evaluator" "$start_id" || continue - if assistant_phase_codex_reviewer_event_ids "$file" "SubagentStop" | grep -Fxq -- "$start_id"; then - return 0 - fi - done < <(assistant_phase_codex_reviewer_event_ids "$file" "SubagentStart" || true) - return 1 -} - -assistant_phase_has_qa_reviewer_compat_start_event_evidence() { - local file="$1" - local start_id - while IFS= read -r start_id; do - [[ -n "$start_id" ]] || continue - assistant_phase_journal_mentions_agent_id "$file" "QA Evaluator" "$start_id" && return 0 - done < <(assistant_phase_codex_reviewer_event_ids "$file" "SubagentStart" || true) - return 1 -} - assistant_phase_has_role_event_pair_evidence() { local file="$1" local role="$2" @@ -1550,9 +1547,6 @@ assistant_phase_has_role_event_pair_evidence() { return 0 fi done < <(assistant_phase_codex_role_event_ids "$file" "$role" "SubagentStart" || true) - if [[ "$role" == "QA Evaluator" ]]; then - assistant_phase_has_qa_reviewer_compat_event_pair_evidence "$file" && return 0 - fi return 1 } @@ -1564,9 +1558,6 @@ assistant_phase_has_role_start_event_evidence() { [[ -n "$start_id" ]] || continue assistant_phase_journal_mentions_agent_id "$file" "$role" "$start_id" && return 0 done < <(assistant_phase_codex_role_event_ids "$file" "$role" "SubagentStart" || true) - if [[ "$role" == "QA Evaluator" ]]; then - assistant_phase_has_qa_reviewer_compat_start_event_evidence "$file" && return 0 - fi return 1 } @@ -1679,6 +1670,38 @@ assistant_phase_requires_qa_evaluator() { return low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]*[:=][[:space:]]*(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ || low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]+(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ } + function qa_section_mode_required_line(line, value, low) { + line = trim(line) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", line) + low = tolower(line) + if (low !~ /^mode[[:space:]]*:/) { + return 0 + } + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + value = trim(value) + if (value ~ /\|/ || bracket_placeholder(value)) { + return 0 + } + low = tolower(value) + return low ~ /^required([[:space:][:punct:]]|$)/ + } + function qa_section_mode_not_required_line(line, value, low) { + line = trim(line) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", line) + low = tolower(line) + if (low !~ /^mode[[:space:]]*:/) { + return 0 + } + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + value = trim(value) + if (value ~ /\|/ || bracket_placeholder(value)) { + return 0 + } + low = tolower(value) + return low ~ /^(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ + } function line_marks_not_required(line, low) { low = tolower(line) return low ~ /(^|[[:space:][:punct:]])(n\/a|not_required|not required|not-required|not-applicable|not_applicable|not applicable)([[:space:][:punct:]]|$)/ @@ -1701,11 +1724,37 @@ assistant_phase_requires_qa_evaluator() { if (!bracket_placeholder(value) && !explicit_not_required(value)) { found = 1 } + } else if (low ~ /^qa evaluator dispatch\/result\/direct evidence:/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + if (!bracket_placeholder(value) && !explicit_not_required(value)) { + found = 1 + } } } { line = $0 low = tolower(line) + if (low ~ /^###[[:space:]]*qa evaluation([[:space:]#]|$)/) { + in_qa_evaluation = 1 + in_required_agents = 0 + in_required_gates = 0 + next + } + if (in_qa_evaluation && low ~ /^#+[[:space:]]+/) { + in_qa_evaluation = 0 + } + if (in_qa_evaluation) { + if (qa_section_mode_required_line(line)) { + required_mode = 1 + found = 1 + next + } + if (qa_section_mode_not_required_line(line)) { + not_required_mode = 1 + next + } + } if (qa_mode_required_line(line)) { required_mode = 1 found = 1 @@ -1806,10 +1855,283 @@ assistant_phase_requires_subagent_roles() { [[ -n "$(assistant_phase_required_subagent_roles "$1")" ]] } +assistant_phase_qa_evaluator_subagent_evidence_is_actionable() { + [[ "$(assistant_phase_review_missing_reason_key "$1")" == "complete" ]] +} + assistant_phase_role_reason_slug() { printf '%s' "$1" | tr '[:upper:]/ ' '[:lower:]__' } +assistant_phase_reason_missing_field() { + local gate="$1" + local key="$2" + + case "$gate:$key" in + plan_gate:no_plan) printf 'No plan found; Plan approval: yes\n' ;; + plan_gate:plan_not_approved) printf 'Plan exists but is not approved; Plan approval: yes\n' ;; + + subagent_evidence_gate:authorization_required_unresolved) printf 'Subagent policy state / Subagent execution mode\n' ;; + subagent_evidence_gate:delegated_missing_code_mapper) printf 'Code Mapper dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_explorer) printf 'Explorer dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_architect) printf 'Architect dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_code_writer) printf 'Code Writer dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_builder_tester) printf 'Builder/Tester dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_code_reviewer) printf 'Code Reviewer dispatch/result evidence during Review\n' ;; + subagent_evidence_gate:delegated_missing_qa_evaluator) printf 'QA Evaluator dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_per_slice) printf 'Per-slice dispatch evidence\n' ;; + subagent_evidence_gate:direct_fallback_invalid_policy_state) printf 'Subagent policy state for direct_fallback\n' ;; + subagent_evidence_gate:direct_fallback_missing_reason) printf 'Direct fallback reason\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_mapper) printf 'Code Mapper direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_explorer) printf 'Explorer direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_architect) printf 'Architect direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_writer) printf 'Code Writer direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_builder_tester) printf 'Builder/Tester direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_reviewer) printf 'Code Reviewer direct evidence during Review\n' ;; + subagent_evidence_gate:direct_fallback_missing_qa_evaluator) printf 'QA Evaluator direct evidence\n' ;; + subagent_evidence_gate:not_applicable_with_required_roles) printf 'Subagent execution mode\n' ;; + subagent_evidence_gate:unknown_execution_mode) printf 'Subagent execution mode\n' ;; + + review_gate:no_spec_review) printf 'no Spec Review; ### Spec Review #N\n' ;; + review_gate:spec_not_pass) printf 'latest Spec Review result not PASS; - Result: PASS\n' ;; + review_gate:no_quality_review) printf 'no Quality Review after latest Spec Review PASS; ### Quality Review #N\n' ;; + review_gate:no_final_result) printf 'no Final Result; ### Final result with - Result: CLEAN|ISSUES_FIXED|HAS_REMAINING_ITEMS\n' ;; + review_gate:qa_rejected) printf 'QA Evaluation final verdict/result accepted or accepted_with_concerns\n' ;; + review_gate:qa_blocked) printf 'QA Evaluation final verdict/result accepted or accepted_with_concerns\n' ;; + review_gate:qa_final_result_missing) printf 'QA Evaluation final verdict/result\n' ;; + review_gate:qa_not_accepted) printf 'QA Evaluation final verdict/result accepted or accepted_with_concerns\n' ;; + review_gate:missing_review_round) printf '%s\n' '- Round: N of 10' ;; + review_gate:round_overflow) printf '%s\n' '- Round: N of 10 with N between 1 and 10' ;; + review_gate:missing_findings_summary) printf '%s\n' '- Found this round: X must-fix, Y should-fix, Z nits' ;; + review_gate:missing_rubric_scores) printf '%s\n' '- Rubric: correctness, code_quality/quality, architecture, security, test_coverage/coverage scores' ;; + review_gate:missing_weighted_score) printf 'missing or mismatched weighted score; - Weighted: N.NN\n' ;; + review_gate:missing_delta_from_previous) printf '%s\n' '- Delta from previous: +/-N.NN' ;; + review_gate:missing_drift_check) printf '%s\n' '- Drift check: GENUINE or REGRESSION' ;; + review_gate:missing_score_progression) printf '%s\n' '- Score progression: ...' ;; + review_gate:weighted_score_below_pass) printf '%s\n' '- Weighted: score >= 4.00 for CLEAN/ISSUES_FIXED' ;; + review_gate:unresolved_findings) printf '0 must-fix and 0 should-fix findings for CLEAN/ISSUES_FIXED\n' ;; + review_gate:missing_remaining_rationale) printf '%s\n' '- Remaining items: or - Blocker: with concrete evidence and owner' ;; + + metrics_gate:missing_metrics_today) printf 'workflow metrics JSONL entry for today\n' ;; + + learning_gate:no_learning_controller) printf '### Learning Controller\n' ;; + learning_gate:missing_memory_trend_checked) printf '%s\n' '- Memory trend checked: checked|backend_unavailable|policy_disallowed|not_configured' ;; + learning_gate:missing_learning_evidence_reviewed) printf '%s\n' '- Learning evidence reviewed: list item' ;; + learning_gate:missing_review_findings_considered) printf '%s\n' '- Review findings considered: list item' ;; + learning_gate:missing_build_test_failures_considered) printf '%s\n' '- Build/test failures considered: list item' ;; + learning_gate:missing_user_corrections_considered) printf '%s\n' '- User corrections considered: list item' ;; + learning_gate:missing_durable_lesson_decision) printf '%s\n' '- Durable lesson decision: durable_saved|durable_updated|skipped_not_durable|backend_unavailable|policy_disallowed|refused_sensitive' ;; + learning_gate:missing_persistence_evidence) printf '%s\n' '- Persistence evidence: concrete saved/updated memory evidence' ;; + learning_gate:missing_no_save_rationale) printf '%s\n' '- No-save rationale: concrete reason' ;; + + *) printf 'required gate field\n' ;; + esac +} + +assistant_phase_reason_action() { + local gate="$1" + local key="$2" + + case "$gate:$key" in + plan_gate:no_plan) printf 'run PLAN, get user approval, then record Plan approval: yes\n' ;; + plan_gate:plan_not_approved) printf 'present the plan, wait for approval, then record Plan approval: yes\n' ;; + + subagent_evidence_gate:authorization_required_unresolved) printf 'ask once for subagent authorization or denial before continuing delegated workflow work\n' ;; + subagent_evidence_gate:delegated_missing_code_mapper) printf 'record Code Mapper dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; + subagent_evidence_gate:delegated_missing_explorer) printf 'record Explorer dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; + subagent_evidence_gate:delegated_missing_architect) printf 'record Architect dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; + subagent_evidence_gate:delegated_missing_code_writer) printf 'record Code Writer dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; + subagent_evidence_gate:delegated_missing_builder_tester) printf 'record Builder/Tester dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; + subagent_evidence_gate:delegated_missing_code_reviewer) printf 'record Code Reviewer during Review; legacy Reviewer labels are compatibility routing only\n' ;; + subagent_evidence_gate:delegated_missing_qa_evaluator) printf 'run QA Evaluator after build/test and code-review evidence, then record dispatch/result\n' ;; + subagent_evidence_gate:delegated_missing_per_slice) printf 'record Per-slice dispatch evidence for the implementation slice\n' ;; + subagent_evidence_gate:direct_fallback_invalid_policy_state) printf 'set policy state to authorization_denied, subagents_unavailable, or policy_disallowed\n' ;; + subagent_evidence_gate:direct_fallback_missing_reason) printf 'record Direct fallback reason: authorization_denied|subagents_unavailable|policy_disallowed\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_mapper) printf 'record Code Mapper direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_explorer) printf 'record Explorer direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_architect) printf 'record Architect direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_writer) printf 'record Code Writer direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_builder_tester) printf 'record Builder/Tester direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_reviewer) printf 'record Code Reviewer direct evidence during Review\n' ;; + subagent_evidence_gate:direct_fallback_missing_qa_evaluator) printf 'record QA Evaluator direct evidence after build/test and code-review evidence\n' ;; + subagent_evidence_gate:not_applicable_with_required_roles) printf 'set execution mode to delegated or direct_fallback and record matching evidence\n' ;; + subagent_evidence_gate:unknown_execution_mode) printf 'set Subagent execution mode to delegated, direct_fallback, or not_applicable with valid evidence\n' ;; + + review_gate:no_spec_review) printf 'run Spec Review first and record PASS/FAIL plus scope, mismatch, and required-fix fields\n' ;; + review_gate:spec_not_pass) printf 'fix spec gaps and rerun Spec Review until the latest structured result is PASS\n' ;; + review_gate:no_quality_review) printf 'run assistant-review Stage 2 and append Quality Review after latest Spec Review PASS\n' ;; + review_gate:no_final_result) printf 'finish the review loop and write the Final Result summary after Quality Review\n' ;; + review_gate:qa_rejected) printf 'fix QA findings, rerun QA Evaluation evidence, and record accepted or accepted_with_concerns\n' ;; + review_gate:qa_blocked) printf 'resolve the QA blocker, rerun QA Evaluation evidence, and record accepted or accepted_with_concerns\n' ;; + review_gate:qa_final_result_missing) printf 'run or rerun QA Evaluation evidence and record accepted or accepted_with_concerns\n' ;; + review_gate:qa_not_accepted) printf 'fix or rerun QA Evaluation evidence until accepted or accepted_with_concerns\n' ;; + review_gate:missing_review_round) printf 'add - Round: N of 10 to the latest Quality Review\n' ;; + review_gate:round_overflow) printf 'repair the Quality Review heading/round so N is 1..10 and max is 10\n' ;; + review_gate:missing_findings_summary) printf 'add the latest must-fix/should-fix/nits finding counts\n' ;; + review_gate:missing_rubric_scores) printf 'add numeric 0..5 rubric scores to the latest Quality Review\n' ;; + review_gate:missing_weighted_score) printf 'add a numeric - Weighted: N.NN matching the rubric formula; fix any mismatched weighted score\n' ;; + review_gate:missing_delta_from_previous) printf 'add a valid delta from the previous Quality Review round\n' ;; + review_gate:missing_drift_check) printf 'add a valid drift classification for review rounds after round 1\n' ;; + review_gate:missing_score_progression) printf 'add Score progression matching the observed Quality Review weighted sequence\n' ;; + review_gate:weighted_score_below_pass) printf 'improve low-scoring dimensions and rerun Quality Review before CLEAN/ISSUES_FIXED\n' ;; + review_gate:unresolved_findings) printf 'fix or explicitly carry remaining must-fix/should-fix findings through the loop\n' ;; + review_gate:missing_remaining_rationale) printf 'add concrete remaining-item or blocker rationale with evidence and owner\n' ;; + + metrics_gate:missing_metrics_today) printf 'append today'\''s workflow metrics JSONL entry before stopping\n' ;; + + learning_gate:no_learning_controller) printf 'add the canonical Learning Controller block after Final Result\n' ;; + learning_gate:missing_memory_trend_checked) printf 'record one valid Memory trend checked value\n' ;; + learning_gate:missing_learning_evidence_reviewed) printf 'add reviewed learning evidence or none with a reason\n' ;; + learning_gate:missing_review_findings_considered) printf 'add review finding consideration or none with a reason\n' ;; + learning_gate:missing_build_test_failures_considered) printf 'add build/test failure consideration or none with a reason\n' ;; + learning_gate:missing_user_corrections_considered) printf 'add user correction consideration or none with a reason\n' ;; + learning_gate:missing_durable_lesson_decision) printf 'record one valid durable lesson decision\n' ;; + learning_gate:missing_persistence_evidence) printf 'add concrete memory_reflect, memory_add_insight, or backend evidence for saved/updated lessons\n' ;; + learning_gate:missing_no_save_rationale) printf 'add a concrete no-save rationale when no durable write occurred\n' ;; + + *) printf 'complete the required gate evidence before stopping\n' ;; + esac +} + +assistant_phase_subagent_reason_role() { + local key="$1" + + case "$key" in + *_code_mapper) printf 'Code Mapper\n' ;; + *_explorer) printf 'Explorer\n' ;; + *_architect) printf 'Architect\n' ;; + *_code_writer) printf 'Code Writer\n' ;; + *_builder_tester) printf 'Builder/Tester\n' ;; + *_code_reviewer) printf 'Code Reviewer\n' ;; + *_qa_evaluator) printf 'QA Evaluator\n' ;; + delegated_missing_per_slice) printf 'Per-slice\n' ;; + *) printf '\n' ;; + esac +} + +assistant_phase_status_has_any() { + local status="$1" + shift + local phase + + for phase in "$@"; do + [[ "$status" == *"$phase"* ]] && return 0 + done + return 1 +} + +assistant_phase_subagent_reason_is_current_phase_actionable() { + local key="$1" + local status="$2" + local role + + case "$key" in + complete) + return 1 + ;; + authorization_required_unresolved|direct_fallback_invalid_policy_state|direct_fallback_missing_reason|not_applicable_with_required_roles|unknown_execution_mode) + assistant_phase_status_has_any "$status" DISCOVERING DECOMPOSING PLANNING BUILDING VERIFYING REVIEWING DOCUMENTING + return $? + ;; + esac + + role="$(assistant_phase_subagent_reason_role "$key")" + case "$role" in + "Code Mapper"|"Explorer"|"Architect") + assistant_phase_status_has_any "$status" DISCOVERING DECOMPOSING PLANNING + ;; + "Code Writer"|"Builder/Tester"|"Per-slice") + assistant_phase_status_has_any "$status" BUILDING VERIFYING + ;; + "Code Reviewer"|"QA Evaluator") + assistant_phase_status_has_any "$status" REVIEWING + ;; + *) + return 1 + ;; + esac +} + +assistant_phase_subagent_warning_action() { + local key="$1" + local status="$2" + local field action + + assistant_phase_subagent_reason_is_current_phase_actionable "$key" "$status" || return 1 + field="$(assistant_phase_reason_missing_field "subagent_evidence_gate" "$key")" + action="$(assistant_phase_reason_action "subagent_evidence_gate" "$key")" + printf 'missing=%s action=%s\n' "$field" "$action" +} + +assistant_phase_subagent_warning_reason_key() { + local file="$1" + local key="$2" + local status="$3" + local mode policy_state roles role candidate + + if [[ "$key" == *_qa_evaluator ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then + key="complete" + fi + + if assistant_phase_subagent_reason_is_current_phase_actionable "$key" "$status"; then + printf '%s\n' "$key" + return 0 + fi + + mode="$(assistant_phase_subagent_mode "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" + policy_state="$(assistant_phase_subagent_policy_state "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" + roles="$(assistant_phase_required_subagent_roles "$file")" + + case "$mode" in + delegated) + while IFS= read -r role; do + [[ -n "$role" ]] || continue + if ! assistant_phase_has_role_dispatch_result_evidence "$file" "$role"; then + candidate="delegated_missing_$(assistant_phase_role_reason_slug "$role")" + if [[ "$candidate" == *_qa_evaluator ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then + continue + fi + if assistant_phase_subagent_reason_is_current_phase_actionable "$candidate" "$status"; then + printf '%s\n' "$candidate" + return 0 + fi + fi + done <<< "$roles" + if printf '%s\n' "$roles" | grep -Eq '^(Code Writer|Builder/Tester)$' \ + && ! assistant_phase_has_per_slice_dispatch_evidence "$file"; then + candidate="delegated_missing_per_slice" + if assistant_phase_subagent_reason_is_current_phase_actionable "$candidate" "$status"; then + printf '%s\n' "$candidate" + return 0 + fi + fi + ;; + direct_fallback) + if [[ "$policy_state" != "authorization_denied" && "$policy_state" != "subagents_unavailable" && "$policy_state" != "policy_disallowed" ]]; then + return 1 + fi + if ! assistant_phase_direct_fallback_reason_valid "$file"; then + return 1 + fi + while IFS= read -r role; do + [[ -n "$role" ]] || continue + if ! assistant_phase_has_role_equivalent_evidence "$file" "$role"; then + candidate="direct_fallback_missing_$(assistant_phase_role_reason_slug "$role")" + if [[ "$candidate" == *_qa_evaluator ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then + continue + fi + if assistant_phase_subagent_reason_is_current_phase_actionable "$candidate" "$status"; then + printf '%s\n' "$candidate" + return 0 + fi + fi + done <<< "$roles" + ;; + esac + + return 1 +} + assistant_phase_subagent_evidence_missing_reason_key() { local file="$1" local mode policy_state role roles @@ -1838,6 +2160,9 @@ assistant_phase_subagent_evidence_missing_reason_key() { delegated) while IFS= read -r role; do [[ -n "$role" ]] || continue + if [[ "$role" == "QA Evaluator" ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then + continue + fi if ! assistant_phase_has_role_dispatch_result_evidence "$file" "$role"; then printf 'delegated_missing_%s\n' "$(assistant_phase_role_reason_slug "$role")" return 0 @@ -1860,6 +2185,9 @@ assistant_phase_subagent_evidence_missing_reason_key() { fi while IFS= read -r role; do [[ -n "$role" ]] || continue + if [[ "$role" == "QA Evaluator" ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then + continue + fi if ! assistant_phase_has_role_equivalent_evidence "$file" "$role"; then printf 'direct_fallback_missing_%s\n' "$(assistant_phase_role_reason_slug "$role")" return 0 diff --git a/plugins/assistant-dev/skills/assistant-review/SKILL.md b/plugins/assistant-dev/skills/assistant-review/SKILL.md index 884eb43..7765577 100644 --- a/plugins/assistant-dev/skills/assistant-review/SKILL.md +++ b/plugins/assistant-dev/skills/assistant-review/SKILL.md @@ -78,7 +78,7 @@ Use the smallest mode that answers the request; combine modes when reviewing imp - **Semantic contract review**: for skill/workflow/framework changes, check contract inheritance, method-template alignment, eval coverage, method-signature fidelity, and high-stakes recommendation guards before judging the change clean. - **Maintainability review**: apply SOLID/KISS/DRY/YAGNI/readability only when a concrete risk exists. - **Security handoff**: invoke `assistant-security` when the reviewed surface touches auth, permissions, secrets, input handling, persistence, shell commands, dependency/config changes, network calls, or external integrations. -- **QA evaluation**: after code-review/build evidence exists, dispatch QAEvaluator when `qa_evaluation_mode=required` for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work. Load `references/qa-evaluation-loop.md`. +- **QA evaluation**: after code-review/build evidence exists, dispatch QAEvaluator when `qa_evaluation_mode=required` for explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. Load `references/qa-evaluation-loop.md`. - **Domain rubric QA**: within QA evaluation, load `references/domain-rubrics.md` only when scoped by acceptance criteria, Done Contract, `domain_context`, or explicit `rubric_refs`. QAEvaluator selects rubric families and returns domain-quality scores; Code Reviewer still owns code defects, security, architecture, and test coverage. Finding format: diff --git a/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml b/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml index 67467bb..55137e2 100644 --- a/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml +++ b/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml @@ -93,19 +93,35 @@ fields: default: [] on_missing: skip + - name: harness_capable + type: boolean + required: false + description: "Whether the reviewed task is scoped into the workflow harness controller; ordinary medium+ review scopes default to false." + validation: "False by default. True only for explicitly requested harness/QA acceptance work, long-running work, trace/replay-ready multi-slice work, high-risk harness work, domain-scored work, UI/visual/product/UX/docs/DX-facing work, or accepted Done Contract/Harness Recipe presence." + default: false + on_missing: infer + infer_from: > + Default false. Set true only when the review material, task packet, or user request explicitly + scopes harness work; when an accepted Done Contract or Harness Recipe exists; or when the work is + long-running, trace/replay-ready multi-slice, high-risk harness, domain-scored, or + UI/visual/product/UX/docs/DX-facing. Do not infer true from scope_size=medium/large, + Reviewer delegation, or ordinary source-changing review scope. + - name: qa_evaluation_mode type: enum required: false enum_values: [not_required, optional, required] description: "Whether the independent QA evaluation loop should run after code-review/build evidence" - validation: "required for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work; not_required must have a reason in final output when skipped for medium+ harness work" + validation: "required only for explicit QA/acceptance evaluation, harness_capable == true with acceptance evaluation in scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing scope; not_required must have a reason when scoped acceptance QA is skipped" default: not_required on_missing: infer infer_from: > - Use required when the task is medium+ harness-capable, has a Done Contract, is domain-scored, - affects UI/visual/product/UX/docs/DX quality, or the user explicitly requests QA/acceptance evaluation. + Use required when the user explicitly requests QA/acceptance evaluation, an accepted Done Contract + exists, harness_capable == true and acceptance evaluation is in scope, the task is domain-scored, + or it affects UI/visual/product/UX/docs/DX quality. Use optional when acceptance material exists but the orchestrator has not marked QA as required. - Use not_required for trivial/small code-review-only work with no separate acceptance evidence need. + Use not_required for ordinary medium+ code-review-only/source-changing work with no separate + acceptance evidence need. - name: done_contract type: object diff --git a/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md b/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md index 496dd0c..5219919 100644 --- a/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md +++ b/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md @@ -5,11 +5,13 @@ Use this reference after build/test evidence and the Code Reviewer loop exist. Q ## When QA Runs Run QA evaluation when any of these are true: -- Medium+ harness-capable work has a Done Contract. -- The task is subjective, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly asks for QA. +- The user explicitly asks for QA or acceptance evaluation. +- The task has an accepted Done Contract. +- The task is harness-capable and acceptance evaluation is in scope. +- The task is subjective, domain-scored, or UI/visual/product/UX/docs/DX-facing. - Workflow Review needs a final acceptance verdict beyond code/security/architecture/test-coverage review. -Skip QA evaluation only when the task is trivial/small, has no acceptance material beyond a direct code fix, and the workflow output records `qa_evaluation_mode=not_required` with a reason. +Skip QA evaluation when the task has no explicit QA request, Done Contract, harness-capable acceptance scope, domain-scored criteria, or UI/visual/product/UX/docs/DX scope, and the workflow output records `qa_evaluation_mode=not_required` with a reason. ## Inputs diff --git a/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md b/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md index 87fac42..db66e89 100644 --- a/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md +++ b/plugins/assistant-dev/skills/assistant-skill-creator/references/harness-patterns.md @@ -81,7 +81,7 @@ Long-running Process skills need first-class recovery artifacts. | Artifact | Required When | Fields | |---|---|---| -| Harness Run State | Medium+ harness-capable or delegated loops | task id/name, phase, slice, status, blockers, last verification, next action, recovery pointer | +| Harness Run State | Medium+ harness-capable loops; delegated loops only when separately harness-capable | task id/name, phase, slice, status, blockers, last verification, next action, recovery pointer | | Trace Ledger | Any trace/replay-ready loop | agent events, decisions, verification results, deviations, blockers, artifact refs | | Replay Packet | Compaction, restart, handoff, or failure recovery is likely | pinned context, artifact refs, validation state, exact next action, run-state/trace refs | diff --git a/plugins/assistant-dev/skills/assistant-workflow/SKILL.md b/plugins/assistant-dev/skills/assistant-workflow/SKILL.md index 4ea7c36..64e16c9 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/SKILL.md +++ b/plugins/assistant-dev/skills/assistant-workflow/SKILL.md @@ -25,7 +25,8 @@ This skill is intentionally agent-agnostic: it must work in restricted company e - Medium+ work has an approved plan before implementation; small work has an inline plan and proceeds without ceremony unless risk requires approval. - Medium+ harness-capable work has an accepted Done Contract and Harness Recipe before Build. - Trace/replay-ready harness work maintains Harness Run State, Trace Ledger, and Replay Packet artifacts. -- Medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work records independent QA Evaluator evidence after build/test and code-review evidence. +- Ordinary medium+ workflow tasks default to `harness_capable=false` and do not inherit Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless explicit harness/QA criteria apply. +- Harness-capable acceptance, Done Contract, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work records independent QA Evaluator evidence after build/test and code-review evidence. - Behavior changes have tests or explicit validation attached to the implementation step they protect. - Final output reports changed files, verification evidence, residual risks, and next steps. - Candidate Search is used for explicit alternatives, open-ended architecture/design, optimization, high uncertainty, repeated failed attempts, unclear/flaky bugs, or reviewer-requested pivots — not as default ceremony. @@ -211,7 +212,7 @@ Load `references/phases.md` and execute the phase matching your current stage. U For subagent roles and dispatch rules, load `references/subagent-dispatch.md` and resolve `subagent_policy_state`, `subagent_execution_mode`, and `subagent_authorization_scope` before any subagent spawn. Assistant Framework policy requires explicit user authorization before spawning subagents for development/code-work roles unless the current user prompt already explicitly authorizes them for this task. Ask once for the needed delegation scope and wait before continuing phases that require subagents. A sufficient prompt is: `This workflow expects Code Mapper, Architect, Code Writer, Builder/Tester, Code Reviewer, and QA Evaluator when QA is required for [scope]. May I use subagents for this task?` After authorization, use `delegated` mode and spawn the configured role agents. Use `direct_fallback` only when authorization is denied, policy disallows spawning, or a real spawn attempt fails because subagents/custom agents are unavailable; do not infer unavailability merely because no visible tool is named `Task`, `delegate`, or `subagent`. For BES-style option exploration, load `references/candidate-search.md` only when `search_mode: candidate_search` is selected. For mega tasks and anti-patterns, load `references/mega-and-patterns.md`. -Load `references/harness-controller.md` only when medium+ work is harness-capable: long-running, multi-slice, delegated, high-risk, subjective/domain-scored, trace/replay-ready, or explicitly requested as harness work. +Load `references/harness-controller.md` only when medium+ work is harness-capable: long-running, trace/replay-ready multi-slice, high-risk harness, subjective/domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested as harness work. ## Planning Checklist @@ -244,7 +245,7 @@ Before final output for medium+ or high-risk work: - Review the diff against the acceptance criteria. - For bugfixes, verify the review material includes reproduction/root-cause evidence from `assistant-debugging` or a clear reason it was not applicable. - Use `assistant-review` for code quality/spec review when the change is non-trivial. -- Use QA Evaluator through `assistant-review` for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested acceptance QA after build/test and code-review evidence. +- Use QA Evaluator through `assistant-review` when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work after build/test and code-review evidence. - Use `assistant-security` when touching auth, user input, secrets, persistence, network calls, shell commands, dependency/config changes, or external integrations. Review findings must cite evidence and concrete risk. Avoid generic style feedback unless it affects correctness, security, maintainability, or test reliability. diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml index f107b5e..9568c6d 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml @@ -135,19 +135,35 @@ fields: on_missing: skip notes: "Populated by Idea-to-Action pipeline when input is a vague idea rather than a concrete task" + - name: harness_capable + type: boolean + required: false + description: "Whether the task is scoped into the harness controller; ordinary medium+ workflow tasks default to false." + validation: "False by default. True only for explicitly requested harness work, long-running work, trace/replay-ready multi-slice work, high-risk harness work, domain-scored work, UI/visual/product/UX/docs/DX-facing work, or accepted Done Contract/Harness Recipe presence." + default: false + on_missing: infer + infer_from: > + Default false. Set true only when the user or approved plan explicitly requests harness work, + the work is long-running, trace/replay-ready multi-slice work, high-risk harness work, + domain-scored, UI/visual/product/UX/docs/DX-facing, or already has an accepted Done Contract + or Harness Recipe. Do not infer true from size=medium+, delegation alone, or ordinary + source-changing workflow tasks. + - name: qa_evaluation_mode type: enum required: false enum_values: [not_required, optional, required] description: "Whether Review phase should run the independent QA Evaluator acceptance loop after build/test and code-review evidence" - validation: "required for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work; not_required must be justified when a medium+ harness task skips QA" + validation: "required only for explicit QA/acceptance evaluation, harness_capable == true with acceptance evaluation in scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing scope; not_required is the default for ordinary medium+ source-changing work." default: not_required on_missing: infer infer_from: > - Use required when the task is medium+ harness-capable, has a Done Contract, is domain-scored, - affects UI/visual/product/UX/docs/DX quality, or explicitly requests QA/acceptance evaluation. + Use required when the user explicitly requests QA/acceptance evaluation, the task has an accepted + Done Contract, harness_capable == true and acceptance evaluation is in scope, the task is + domain-scored, or it affects UI/visual/product/UX/docs/DX quality. Use optional when acceptance material exists but QA is not required. - Use not_required for trivial/small code-review-only work with no separate acceptance evidence need. + Use not_required for ordinary medium+ code-review-only/source-changing work with no separate + acceptance evidence need. - name: search_mode type: enum diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml index fe50c33..31e1243 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml @@ -242,7 +242,7 @@ artifacts: - name: qa_evaluation_result type: object required: conditional - condition: "qa_evaluation_mode == required or size in [medium, large, mega] and harness_capable == true and work is domain-scored/UI/visual/product/UX/docs/DX-facing" + condition: "qa_evaluation_mode == required" validation: "Independent QA evaluation result exists after build/test evidence and Code Reviewer result. Must include final_verdict, result, rounds (1-10), acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, and evidence." on_fail: "Run the QAEvaluator loop from assistant-review references/qa-evaluation-loop.md, or record NEEDS_CONTEXT/BLOCKED with open_questions. Do not substitute Code Reviewer findings for QA evaluation." object_fields: diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml index b5935e5..5e657a3 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml @@ -59,7 +59,8 @@ gates: on_fail: "Select required workflow agents when subagent_execution_mode=delegated, or record direct fallback roles and responsibilities from references/subagent-dispatch.md and risk-specific gate packs. not_applicable is invalid for source-changing Build tasks." - id: T_QA_EVALUATOR - check: "For medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work: required_agents includes QA Evaluator in addition to Code Writer, Builder/Tester, and Code Reviewer or Reviewer compatibility" + check: "When qa_evaluation_mode == required: required_agents includes QA Evaluator in addition to Code Writer, Builder/Tester, and Code Reviewer or Reviewer compatibility" + condition: "qa_evaluation_mode == required" on_fail: "Add QA Evaluator as a Review-phase role, or record qa_evaluation_mode=not_required with a concrete reason when QA is not applicable." - id: T9 diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md b/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md index 77a6d7c..93f196c 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/harness-controller.md @@ -1,12 +1,24 @@ # Harness Controller -Use this reference only for medium+ work that is harness-capable: long-running, -multi-slice, delegated, high-risk, subjective/domain-scored, trace/replay-ready, -or explicitly requested as harness work. Do not load it for small local fixes. +Use this reference only for medium+ work that is explicitly harness-capable: +long-running, trace/replay-ready multi-slice, high-risk harness, +subjective/domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly +requested as harness work. Do not load it for small local fixes, ordinary +medium+ source changes, or delegation alone. + +`harness_capable` defaults to false. Set it to true only when one of the +explicit criteria above is present in the request, approved plan, task packet, +or accepted Done Contract/Harness Recipe evidence. + +Base plan and task-journal templates keep compact refs and `N/A: [reason]` +fields. Load `references/plan-harness-appendix.md` and +`references/task-journal-harness-appendix.md` only when the full harness, +typed artifact, pivot/restart, or QA schemas are needed. ## Pre-Build Gate -Before Build starts, the approved plan must contain both: +When `harness_capable=true`, before Build starts the approved plan must contain +both: - an accepted Done Contract - a selected Harness Recipe diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/phases.md b/plugins/assistant-dev/skills/assistant-workflow/references/phases.md index b589e07..2521c94 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/phases.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/phases.md @@ -173,7 +173,7 @@ Print: `>> Direct fallback Architect responsibility` (when `subagent_execution_m **Entry rule:** Do not enter Plan while the saved clarification state is pending. Resume Plan only after Discover records `Clarification status: ready` and all implementation-shaping fields are explicit or explicitly defaulted. -Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. For medium+ harness-capable work, load `references/harness-controller.md` and add the Done Contract, Harness Recipe, Harness Run State ref, Trace Ledger ref, Replay Packet ref, and Artifact Reference Ledger before task packets. Then read `references/plan-template.md` and use the correct tier: +Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. Treat `harness_capable` as false unless the task is long-running, trace/replay-ready multi-slice, high-risk harness, domain-scored, UI/visual/product/UX/docs/DX-facing, explicitly requested as harness/QA work, or already has an accepted Done Contract/Harness Recipe. For medium+ harness-capable work, load `references/harness-controller.md` plus `references/plan-harness-appendix.md`, then add compact Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger refs before task packets. Then read `references/plan-template.md` and use the correct tier: - Small: inline plan (goal, files, risks, tests). Do not wait for approval unless risk, ambiguity, user instruction, or a scope-changing decision makes approval necessary. - Medium: standard plan (drop Security/Operability unless the task touches auth, PII, payments, or infra) - Large/Mega: full plan (all sections including Security and Operability) @@ -183,7 +183,7 @@ Before writing the plan, load `references/artifact-first-output-contract.md` and 3. Analyze 1-3 options with tradeoffs, pick one 4. Identify risks and edge cases 5. Put the Artifact Contract before task packets and map every medium+ task packet to at least one required artifact or acceptance criterion -6. For medium+ harness-capable work, put the Done Contract, Harness Recipe, run-state/trace/replay artifact refs, and Artifact Reference Ledger before Build. The Done Contract must include done_when, not_done_when, verification, owner/consumer, acceptance criteria, and at least two debate perspectives; the Harness Recipe must be selected from task/model/risk/context profile. The ledger must type each artifact with artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, and summary. +6. For medium+ harness-capable work, put compact refs for Done Contract, Harness Recipe, run-state/trace/replay artifacts, and Artifact Reference Ledger before Build; load `references/plan-harness-appendix.md` for the full schemas. The base plan keeps `N/A: [reason]` for non-harness work. 7. For medium+ tasks: consume the Decompose slice manifest directly in the plan and align each task packet to exactly one slice_id without rediscovering boundaries 8. Write ordered implementation steps with file paths 9. For large/mega: fill in Security and Operability sections. For medium: only if the task touches auth, PII, payments, or infra (promote to Full tier per plan-template.md) @@ -239,7 +239,7 @@ For medium+ tasks, create a task journal using `references/task-journal-template Capture **constraints** from Discovery/Plan (e.g. "don't touch ProjectA", "stay on .NET 8"). Check constraints before each step. -For medium+ harness-capable work, confirm the task journal or carried-forward plan has an accepted Done Contract, selected Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger before dispatching Code Writer or Builder/Tester. If any are missing, stop Build and return to Plan or repair state for the corrective action in `references/harness-controller.md`. +For medium+ harness-capable work, confirm the task journal or carried-forward plan has compact refs for an accepted Done Contract, selected Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger before dispatching Code Writer or Builder/Tester. Load `references/task-journal-harness-appendix.md` for the full schema. If any required ref is missing, stop Build and return to Plan or repair state for the corrective action in `references/harness-controller.md`. Keep the runtime artifacts current as execution progresses: - Update Harness Run State after each slice/step, blocker, verification result, phase transition, or next-action change. @@ -253,9 +253,9 @@ Keep the runtime artifacts current as execution progresses: - **Code Writer** (`code-writer`): implements code following the plan - **Builder/Tester** (`builder-tester`): builds, writes tests, runs tests - **Code Reviewer** (`code-reviewer`, or `reviewer` compatibility through assistant-review delegated review): independent code/security/architecture/test coverage review evidence for Review -- **QA Evaluator** (`qa-evaluator`): independent acceptance, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result evaluation when QA is required +- **QA Evaluator** (`qa-evaluator`): independent acceptance, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result evaluation when `qa_evaluation_mode=required` -Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Code Reviewer; `reviewer` remains compatibility routing for existing/legacy handoffs. Medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work also adds QA Evaluator. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. +Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Code Reviewer; `reviewer` remains compatibility routing for existing/legacy handoffs. Add QA Evaluator only when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. **Direct fallback mode (`subagent_execution_mode=direct_fallback`):** when authorization is denied, subagents are unavailable, or policy disallows spawning, the active agent may implement, test, and review directly, but must preserve the same phases, contracts, evidence requirements, and review/security gates. Do not pretend delegation happened; record `subagent_policy_state`, `subagent_execution_mode`, the explicit `Direct fallback reason: authorization_denied | subagents_unavailable | policy_disallowed`, and the direct-execution evidence. @@ -424,7 +424,7 @@ For medium+ tasks: full spec review plus autonomous code quality loop via `assis Print: `>> Stage 3: QA Evaluation — loading assistant-review references/qa-evaluation-loop.md` when QA is required. -Run QA Evaluation for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. +Run QA Evaluation only when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. The QA Evaluator receives Done Contract when present, acceptance criteria, verification evidence, code review result, domain_context/rubric_refs when applicable, round 1-10, previously_failed_acceptance_items, and qa_filter_policy. It loads assistant-review `references/domain-rubrics.md` only when acceptance criteria, Done Contract, domain_context, or explicit rubric_refs require subjective/product/UX/docs/DX/UI/domain scoring. It returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression or score_entry, evidence, and open_questions when blocked. diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/plan-harness-appendix.md b/plugins/assistant-dev/skills/assistant-workflow/references/plan-harness-appendix.md new file mode 100644 index 0000000..b280454 --- /dev/null +++ b/plugins/assistant-dev/skills/assistant-workflow/references/plan-harness-appendix.md @@ -0,0 +1,99 @@ +# Plan Harness Appendix + +Load this appendix only for medium+ harness-capable work: explicit harness work, +long-running or trace/replay-ready multi-slice work, high-risk harness work, +accepted Done Contract/Harness Recipe work, domain-scored work, or +UI/visual/product/UX/docs/DX-facing work. For ordinary medium+ source changes, +keep the base plan's harness fields as `N/A: [reason]`. + +## Task Packet Harness Fields + +Use these fields inside an executable task packet only when the task is +harness-capable. + +```markdown +- Harness refs: + - done_contract_ref: [Done Contract section/ref, or N/A: reason] + - harness_recipe_ref: [Harness Recipe section/ref, or N/A: reason] + - harness_run_state_ref: [Harness Run State section/ref, or N/A: reason] + - trace_ledger_ref: [Trace Ledger section/ref, or N/A: reason] + - replay_packet_ref: [Replay Packet section/ref, or N/A: reason] + - artifact_reference_ledger_ref: [Artifact Reference Ledger section/ref, or N/A: reason] +- Typed artifact refs: + - artifact_id: [stable task-local id] + artifact_type: [done_contract | harness_recipe | harness_run_state | trace_ledger | replay_packet | pivot_restart_decision | changed_files | verification_evidence | plan_deviation | task_packet | context_map | test_result | review_result | qa_evaluation_result] + producer: [role/subagent/hook/task packet] + consumer: [role/subagent/hook/phase] + location_ref: [typed location/ref pointer] + schema_or_contract: [contract/template/required fields] + validation_status: [pending | valid | invalid | stale | not_applicable] + summary: [concise state] +``` + +## Done Contract + +Required for medium+ harness-capable work before Build. + +- done_when: + - [binary outcome that proves done] +- not_done_when: + - [failure state that blocks done] +- verification: + - [command, inspection, review, or manual check] +- owner_consumer: [owner and downstream consumer] +- acceptance_criteria: + - [explicit binary criterion] +- debate_record: + - perspective: [role/subagent/direct perspective 1] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] + - perspective: [role/subagent/direct perspective 2] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] +- accepted_by: [user/orchestrator/approved plan reference] + +## Harness Recipe + +Required for medium+ harness-capable work before Build; selected from +task/model/risk/context profile per `references/harness-controller.md`. + +- task_profile: [task type, size, slice count, TDD/debugging applicability] +- model_profile: [agent/model constraints, delegation mode, tool limits] +- risk_profile: [risk tier, safety gates, review depth, rollback needs] +- context_profile: [exact/summarized/omitted context and trace/replay needs] +- selected_recipe: [concise recipe label] +- recipe_rationale: [why this profile selects the recipe] +- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] +- corrective_action: [what to do if missing or stale] + +## Runtime Harness Artifacts + +Required for medium+ harness-capable work when the recipe needs recovery, +handoff, or trace/replay evidence. + +- harness_run_state_ref: [where task_id/task_name/phase/slice/status/blockers/last_verification/next_action/recovery_pointer will be maintained] +- trace_ledger_ref: [where ordered agent events, decisions, verification results, plan deviations, and artifact refs will be appended] +- replay_packet_ref: [where pinned context, artifact refs, validation state, and exact next action will be refreshed] +- corrective_action: [what to do if run-state/trace/replay evidence is missing or stale] + +## Artifact Reference Ledger + +Required for medium+ harness-capable work when artifacts pass between agents. +Each row is a typed producer/consumer record, not an ad hoc string reference. + +| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | +|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| +| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/pivot_restart_decision/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role] | [role/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | + +## QA Routing + +QA Evaluator remains separate from Code Reviewer. Use this routing only when +`qa_evaluation_mode=required`: explicit QA/acceptance evaluation, +harness-capable acceptance scope, accepted Done Contract, domain-scored work, or +UI/visual/product/UX/docs/DX-facing work. + +- qa_evaluation_mode: [required | optional | not_required] +- qa_trigger_reason: [explicit QA request, Done Contract, harness acceptance scope, domain-scored, UI/visual/product/UX/docs/DX, or N/A: reason] +- code_review_result_ref: [Code Reviewer/Reviewer compatibility result ref] +- qa_evaluation_result_ref: [QA Evaluator result ref, or N/A: reason] +- domain_context_ref: [domain/rubric context ref, or N/A: reason] diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md index 10814cd..321c7ca 100755 --- a/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md @@ -2,6 +2,11 @@ Three tiers — match ceremony to task size (don't get fancy when N is small). +Harness details are optional appendices. For medium+ harness-capable work, keep +compact refs in the base plan and load `references/plan-harness-appendix.md` +for the full schema. For non-harness work, record `N/A: [reason]` instead of +copying harness sections into the base plan. + ## Small Tasks — Inline Plan No separate plan document needed. Include directly in your response: @@ -55,21 +60,15 @@ For Medium and Large/Mega plans, write implementation work as executable task pa - Expected success signal: [exit code 0, passing test name, output marker, etc.] - Evidence to record: - [test result, eval fixture, changed file, review note, or artifact proof] -- Harness refs: - - done_contract_ref: [Done Contract section/ref, or N/A] - - harness_recipe_ref: [Harness Recipe section/ref, or N/A] - - harness_run_state_ref: [Harness Run State section/ref, or N/A] - - trace_ledger_ref: [Trace Ledger section/ref, or N/A] - - replay_packet_ref: [Replay Packet section/ref, or N/A] -- Typed artifact refs: - - artifact_id: [stable task-local id] - artifact_type: [done_contract | harness_recipe | harness_run_state | trace_ledger | replay_packet | changed_files | verification_evidence | plan_deviation | task_packet | context_map | test_result | review_result | qa_evaluation_result] - producer: [role/subagent/hook/task packet] - consumer: [role/subagent/hook/phase] - location_ref: [typed location/ref pointer] - schema_or_contract: [contract/template/required fields] - validation_status: [pending | valid | invalid | stale | not_applicable] - summary: [concise state] +- Harness routing: + - applies: [required for medium+ harness-capable work; otherwise N/A: reason] + - appendix: [references/plan-harness-appendix.md, or N/A: reason] + - done_contract_ref: [section/ref, or N/A: reason] + - harness_recipe_ref: [section/ref, or N/A: reason] + - harness_run_state_ref: [section/ref, or N/A: reason] + - trace_ledger_ref: [section/ref, or N/A: reason] + - replay_packet_ref: [section/ref, or N/A: reason] + - artifact_reference_ledger_ref: [section/ref, or N/A: reason] - Deviation / rollback rule: - [what to do if required files/behavior differ from plan; include rollback/revert boundary] - Worker status / evidence: @@ -186,61 +185,21 @@ Covers the essentials without Security/Operability overhead. Fill this in during - Owner/consumer: [user, reviewer, downstream tool, runtime] - Non-goals/exclusions: [what must not be produced] -## Done Contract - -Required for medium+ harness-capable work before Build; omit with a brief N/A -rationale for non-harness work. - -- done_when: - - [binary outcome that proves done] -- not_done_when: - - [failure state that blocks done] -- verification: - - [command, inspection, review, or manual check] -- owner_consumer: [owner and downstream consumer] -- acceptance_criteria: - - [explicit binary criterion] -- debate_record: - - perspective: [role/subagent/direct perspective 1] - concern_or_support: [concise point] - resolution: [accepted, rejected, or changed] - - perspective: [role/subagent/direct perspective 2] - concern_or_support: [concise point] - resolution: [accepted, rejected, or changed] -- accepted_by: [user/orchestrator/approved plan reference] - -## Harness Recipe - -Required for medium+ harness-capable work before Build; selected from -task/model/risk/context profile per `references/harness-controller.md`. - -- task_profile: [task type, size, slice count, TDD/debugging applicability] -- model_profile: [agent/model constraints, delegation mode, tool limits] -- risk_profile: [risk tier, safety gates, review depth, rollback needs] -- context_profile: [exact/summarized/omitted context and trace/replay needs] -- selected_recipe: [concise recipe label] -- recipe_rationale: [why this profile selects the recipe] -- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] -- corrective_action: [what to do if missing or stale] - -## Runtime Harness Artifacts - -Required for medium+ harness-capable work; omit with a brief N/A rationale for -non-harness work. - -- harness_run_state_ref: [where task_id/task_name/phase/slice/status/blockers/last_verification/next_action/recovery_pointer will be maintained] -- trace_ledger_ref: [where ordered agent events, decisions, verification results, plan deviations, and artifact refs will be appended] -- replay_packet_ref: [where pinned context, artifact refs, validation state, and exact next action will be refreshed] -- corrective_action: [what to do if run-state/trace/replay evidence is missing or stale] +## Harness Appendix Routing -## Artifact Reference Ledger +Required only for medium+ harness-capable work; otherwise keep every field as +`N/A: [reason]`. Load `references/plan-harness-appendix.md` for the full Done +Contract, Harness Recipe, runtime artifact, typed artifact ref, and QA routing +schemas. -Required for medium+ harness-capable work when artifacts pass between agents. -Each row is a typed producer/consumer record, not an ad hoc string reference. - -| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | -|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| -| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role] | [role/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | +- appendix_status: [required | N/A: reason] +- done_contract_ref: [section/ref, or N/A: reason] +- harness_recipe_ref: [section/ref, or N/A: reason] +- harness_run_state_ref: [section/ref, or N/A: reason] +- trace_ledger_ref: [section/ref, or N/A: reason] +- replay_packet_ref: [section/ref, or N/A: reason] +- artifact_reference_ledger_ref: [section/ref, or N/A: reason] +- qa_evaluation_mode: [required | optional | not_required + reason] ## Slice manifest from Decompose @@ -258,45 +217,7 @@ Use the Executable Task Packet structure above for each approved slice. Order pa Everything from Medium, plus Security and Operability sections. Use when the task touches auth, external inputs, infrastructure, or multi-module boundaries. ```markdown -## Goal -- [1-3 sentence restated requirement from Discovery] - -## Triage result -- Task type: [feature | bugfix | refactor | migration | rewrite | config | infra | security | docs | spike] -- Risk tier: [low | moderate | high | critical] -- Required gates: [common gates + task-category gate packs from references/triage-rubric.md] -- Required agents: [roles/skills selected from size, task type, and risk] -- Search mode: [none | lightweight | candidate_search] - -## Constraints & decisions (from Discovery) -- [Q&A question]: [chosen option and why] -- [Q&A question]: [chosen option and why] -- Assumed (not explicitly asked): [assumption and reasoning] -- Non-goals: [what's explicitly out of scope] - -## Research (current state) -- Modules/subprojects: ... -- Key files/paths: ... -- Entrypoints: ... -- Configs/flags: ... -- Data models: ... -- Existing patterns: ... - -## Architecture -- Current architecture: [identified or "new project"] -- Architecture for this change: [Clean/MVVM/Hexagonal/etc.] -- Layer rules: - - [e.g., Domain has no external dependencies] - - [e.g., ViewModels don't reference Views] -- Dependency direction: [A → B → C] -- New files placement: - - [file → layer/folder rationale] -- SOLID design notes: - - SRP: [which classes own which responsibility — flag any class with >1 reason to change] - - OCP: [will new variants require modifying existing classes? If yes, plan extension points] - - LSP: [any inheritance hierarchies? Do subtypes preserve base type contracts?] - - ISP: [any interfaces? Are they minimal or do implementers need to stub methods?] - - DIP: [which high-level modules depend on abstractions vs concrete implementations?] +Start with the Medium template, then add the sections below. ## Security considerations - Data classification: [does this touch PII, auth, payments, external inputs?] @@ -316,41 +237,26 @@ Everything from Medium, plus Security and Operability sections. Use when the tas - Revert commit sufficient: [yes/no] - Runbook updates: [new on-call procedures needed?] -## Analysis -### Candidate search summary -- Candidate search summary: [N/A unless search_mode=candidate_search; otherwise selected candidate and why] -- Candidate archive: [{agent_state_dir}/candidate-search.md when local state is allowed, or inline plan section] -- Goal tree source: [acceptance criteria/slice criteria used] - -### Options -1. [approach] — [tradeoff] -2. [approach] — [tradeoff] - -### Decision -- Chosen: [#] because [reason] - -### Risks / edge cases -- [risk]: [mitigation] - -## Decomposition Plan Review - -- Scope understanding: [pass/fix needed + evidence] -- Slice/subagent count: [count + sanity rationale] -- Step/cost budget: [budget or direct-fallback rationale] -- Dependency order: [summary] -- Output-plan match: [artifact/verification alignment] -- Fallback path: [subagent path or direct equivalent] -- Broad-split rejection: [required proof that layer-only, module-only, folder-only, feature-only, setup-only, contract-only, and broad component-style splits were rejected unless verified deliverable artifact slices] -- Decision: proceed | revise_decomposition | return_to_discover - -## Artifact Reference Ledger - -Required for medium+ harness-capable work when artifacts pass between agents. -Each row is a typed producer/consumer record, not an ad hoc string reference. - -| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | -|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| -| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role] | [role/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | +## Large/Mega addenda +- Triage result: include `Subagent policy state`, `Subagent execution mode`, `Subagent authorization scope`, and `Search mode` as in Medium. +- Architecture: include LSP and ISP in addition to SRP/OCP/DIP. +- Decomposition Plan Review: reuse Medium fields and keep `- Broad-split rejection:` proof for large/mega slice plans. +- Harness Appendix Routing: reuse the Medium compact refs and load `references/plan-harness-appendix.md` only when harness-capable. + +## Harness Appendix Routing + +Required only for medium+ harness-capable work; otherwise record +`N/A: [reason]`. Load `references/plan-harness-appendix.md` for full harness, +typed artifact reference, pivot/restart, and QA routing schemas. + +- appendix_status: [required | N/A: reason] +- done_contract_ref: [section/ref, or N/A: reason] +- harness_recipe_ref: [section/ref, or N/A: reason] +- harness_run_state_ref: [section/ref, or N/A: reason] +- trace_ledger_ref: [section/ref, or N/A: reason] +- replay_packet_ref: [section/ref, or N/A: reason] +- artifact_reference_ledger_ref: [section/ref, or N/A: reason] +- qa_evaluation_mode: [required | optional | not_required + reason] ## Slice manifest from Decompose diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-harness-appendix.md b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-harness-appendix.md new file mode 100644 index 0000000..d311d37 --- /dev/null +++ b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-harness-appendix.md @@ -0,0 +1,145 @@ +# Task Journal Harness Appendix + +Load this appendix only when the task journal is tracking medium+ +harness-capable work, QA evaluation, typed artifact references, or a +Pivot/Restart Decision. The base task journal keeps compact refs and +`N/A: [reason]` fields so ordinary medium+ work does not inherit this detail. + +## Done Contract + +[required before Build for medium+ harness-capable work] + +- done_when: + - [binary outcome that proves done] +- not_done_when: + - [failure state that blocks done] +- verification: + - [command, inspection, review, or manual check] +- owner_consumer: [owner and downstream consumer] +- acceptance_criteria: + - [explicit binary criterion] +- debate_record: + - perspective: [role/subagent/direct perspective 1] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] + - perspective: [role/subagent/direct perspective 2] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] +- accepted_by: [user/orchestrator/approved plan reference] + +## Harness Recipe + +[required before Build for medium+ harness-capable work] + +- task_profile: [task type, size, slice count, TDD/debugging applicability] +- model_profile: [agent/model constraints, delegation mode, tool limits] +- risk_profile: [risk tier, safety gates, review depth, rollback needs] +- context_profile: [exact/summarized/omitted context and trace/replay needs] +- selected_recipe: [concise recipe label] +- recipe_rationale: [why this task/model/risk/context profile selects the recipe] +- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] +- corrective_action: [what to do if missing or stale] + +## Harness Run State + +[required for medium+ harness-capable work] + +- task_id: [stable task/run id] +- task_name: [human-readable task name] +- phase: [TRIAGE | DISCOVER | DECOMPOSE | PLAN | DESIGN | BUILD | REVIEW | DOCUMENT | COMPLETE] +- slice: [current slice id/name, next pending slice, or N/A] +- status: [not_started | in_progress | blocked | verifying | reviewing | restarting | documenting | complete] +- blockers: + - [current blocker, or none] +- last_verification: + - command_or_check: [command/check, or pending/N/A] + - result: [passed | failed | skipped | not_applicable | pending] + - evidence: [concise evidence or reason] +- next_action: [exact next action] +- recovery_pointer: [task packet, trace entry, ledger row, file section, or artifact ref] +- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] + +## Trace Ledger + +[required for medium+ harness-capable work; append-only ordered execution evidence] + +| Seq | Timestamp/Order | Event Type | Actor | Summary | Artifact Refs | +|-----|-----------------|------------|-------|---------|---------------| +| 1 | [timestamp or ordered marker] | [agent_event/decision/verification/plan_deviation/pivot_restart/artifact_ref/blocker/recovery] | [role/subagent/user/hook] | [event, decision, command/result, deviation, blocker, pivot/restart, or recovery] | [file/section/dispatch id/command ref] | + +## Replay Packet + +[required for medium+ harness-capable work before compaction, failure handoff, +phase handoff, or end-of-turn] + +- pinned_context: + - [stable requirement, constraint, approved plan/slice id, or non-goal] +- artifact_refs: + - [task journal/context map/plan/run state/trace/validation/changed file ref] +- validation_state: + - completed_checks: [checks already completed] + - pending_checks: [checks still pending] + - last_result: [latest verification/review result] +- exact_next_action: [single concrete next action after replay] +- run_state_ref: [Harness Run State section/ref] +- trace_ledger_ref: [Trace Ledger section/ref] +- recovery_pointer: [where to resume] +- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] + +## Pivot/Restart Log + +[append when review/QA stagnation, repeated drift/regression, rubric/domain +pivot, Code Writer blocker, verification blocker, plan deviation, or scope +change triggers recovery] + +### Pivot/Restart Decision #N + +- trigger: [STAGNATION | repeated_DRIFT | repeated_REGRESSION | pivot | code_writer_blocker | verification_blocker | plan_deviation | scope_change] +- evidence: + - source: [review score entry, QA score entry, Code Writer blocker, verification failure, trace row, or task packet] + detail: [concise evidence] +- affected_slice_or_round: [slice id/name, review round, QA round, or phase] +- options_considered: + - option: [reset context / dispatch debugging / dispatch explorer / dispatch architect / candidate search / replan / restart slice / restart phase / block for user / accept with limitations] + tradeoff: [why this option helps or risks the task] + disposition: [selected | rejected | deferred] +- selected_action: [reset_context | return_to_build | dispatch_debugging | dispatch_explorer | dispatch_architect | run_candidate_search | replan | restart_slice | restart_phase | block_for_user | accept_with_limitations] +- reapproval_required: [true when scope/files/behavior/risk/verification/acceptance changes; otherwise false] +- next_agent: [Builder/Tester | Code Writer | Explorer | Architect | Reviewer | QAEvaluator | assistant-debugging | candidate-search | user | none] +- recovery_pointer: [task packet, trace row, replay packet, plan section, or file path] +- exact_next_action: [single concrete action after the decision] + +## Artifact Reference Ledger + +[required for medium+ harness-capable work when artifacts pass between agents; +typed producer/consumer records, not ad hoc strings] + +| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | +|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| +| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/pivot_restart_decision/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role/subagent/hook] | [role/subagent/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | + +## QA Evaluation Log + +QA Evaluation is required only when `qa_evaluation_mode=required`. It runs after +build/test evidence and Code Reviewer or Reviewer compatibility evidence. + +### QA Evaluation #1 + +- Round: 1 of 10 +- Mode: required | optional | not_required +- Done Contract: [ref or N/A with reason] +- Acceptance criteria checked: [count] +- Verification evidence checked: [commands/checks/manual/review refs] +- Code Review evidence: [Code Reviewer/Reviewer result ref] +- Domain/rubric refs: [product/UX/UI/docs/DX/domain refs, or N/A when not scoped] +- Selected domain rubrics: [ui_visual_design | ux_product_acceptance | documentation_quality | developer_experience | domain_specific_craft | N/A] +- Domain quality scores: [rubric.dimension=score/action/evidence, or N/A when not scoped] +- QA scorecard: acceptance_coverage=[score] evidence_strength=[score] domain_quality=[score] final_readiness=[score] +- Weighted: [score] +- Score progression: [round1->...] +- Pivot/Restart decision: [ref if STAGNATION, repeated DRIFT, repeated REGRESSION, or domain action pivot occurred; otherwise N/A] +- Final verdict: accepted | accepted_with_concerns | rejected | blocked +- Acceptance findings: + - [blocker|concern|observation] [criterion] - [evidence] -> [fix/defer/open question] + +[...repeat QA Evaluation until accepted, accepted_with_concerns, blocked, or max round 10 reached...] diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md index 5d8eac2..7944d4d 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md @@ -1,6 +1,6 @@ # Task Journal Template -Write to `{agent_state_dir}/task.md` in the project root when a local agent state directory is configured and policy allows local state artifacts. If no safe state directory is available, keep the same content in the response/plan packet. This state is the single source of truth for the current task — it survives context compression and session continuations when persisted. It is a framework-owned ignored state artifact, so the orchestrator may create and update it directly when allowed. +Write to `{agent_state_dir}/task.md` in the project root when a local state directory is configured and policy allows. If none is safe, keep the same content in the response/plan packet. This framework-owned ignored artifact is the task source of truth, survives compression/continuation when persisted, and may be updated by the orchestrator. ## When to create - Any task that enters clarification wait: during Discover, before printing clarification questions or the wait message @@ -10,9 +10,8 @@ Write to `{agent_state_dir}/task.md` in the project root when a local agent stat ## When to update - When clarification questions are asked, answered, or resolved via explicit `defaults` - After each Build step completes (update Progress, Artifact Registry, and check off Milestones) -- After each medium+ harness event that changes phase, slice, status, blockers, verification, next action, decisions, deviations, or artifact refs -- After any Pivot/Restart Decision, including review/QA stagnation, repeated drift/regression, Code Writer blockers, or selected restart/replan action -- After any producer creates/updates a typed artifact reference or any consumer changes `validation_status` +- After each medium+ harness-capable event listed in `references/task-journal-harness-appendix.md` +- After any Pivot/Restart Decision, typed artifact reference update, or QA evaluation result when applicable - When key decisions are made - When constraints are added by the user - After each review cycle pass (append to Review Log — never overwrite) @@ -54,35 +53,20 @@ Plan approval: [yes/no + date] ## Agent Dispatch Log [strict subagent evidence inspected by stop-review/phase gates] -- Required roles: Code Writer, Builder/Tester, Code Reviewer [Reviewer remains compatibility routing; plus QA Evaluator when QA is required; plus Code Mapper/Explorer/Architect when required by size/risk] +- Required roles: Code Writer, Builder/Tester, Code Reviewer; QA Evaluator when `qa_evaluation_mode=required`; Code Mapper/Explorer/Architect by size/risk; Reviewer for legacy compatibility. - Execution mode: delegated | direct_fallback | not_applicable -- Codex lifecycle evidence: delegated Codex roles must have matching `.codex/subagent-events.jsonl` `SubagentStart` + `SubagentStop` records; dispatch/result entries must cite the matching `agent_id`; journal text alone is not proof. +- Codex lifecycle evidence: delegated Codex roles need matching `.codex/subagent-events.jsonl` `SubagentStart`/`SubagentStop` records and dispatch/result refs with the same `agent_id`; journal text alone is insufficient. - Direct fallback reason: [authorization_denied | subagents_unavailable | policy_disallowed | N/A] -- Code Mapper dispatch: [subagent/tool/run id, mapping packet, timestamp, or N/A only in direct_fallback/not required] -- Code Mapper result: [context map summary/evidence, or N/A only in direct_fallback/not required] -- Code Mapper direct evidence: [role-equivalent context map evidence when direct_fallback, else N/A] -- Explorer dispatch: [subagent/tool/run id, exploration packet, timestamp, or N/A only in direct_fallback/not required] -- Explorer result: [execution path/dependency evidence, or N/A only in direct_fallback/not required] -- Explorer direct evidence: [role-equivalent exploration evidence when direct_fallback, else N/A] -- Architect dispatch: [subagent/tool/run id, architecture/decomposition packet, timestamp, or N/A only in direct_fallback/not required] -- Architect result: [slice/blueprint/design evidence, or N/A only in direct_fallback/not required] -- Architect direct evidence: [role-equivalent decomposition/architecture evidence when direct_fallback, else N/A] -- Code Writer dispatch: [subagent/tool/run id, prompt/packet, timestamp, or N/A only in direct_fallback/not required] -- Code Writer result: [returned files/summary/evidence, or N/A only in direct_fallback/not required] -- Code Writer direct evidence: [role-equivalent implementation evidence when direct_fallback, else N/A] -- Builder/Tester dispatch: [subagent/tool/run id, verification packet, timestamp, or N/A only in direct_fallback/not required] -- Builder/Tester result: [commands/results/success signal returned, or N/A only in direct_fallback/not required] -- Builder/Tester direct evidence: [role-equivalent build/test/validation evidence when direct_fallback, else N/A] -- Code Reviewer dispatch: [code-reviewer subagent/tool/run id or assistant-review dispatch, or N/A only when Reviewer compatibility/direct_fallback/not required] -- Code Reviewer result: [spec/quality review result evidence returned, or N/A only when Reviewer compatibility/direct_fallback/not required] -- Code Reviewer direct evidence: [fresh review/spec+quality evidence when direct_fallback, else N/A] -- Reviewer dispatch: [compatibility reviewer subagent/tool/run id or assistant-review dispatch, or N/A only in direct_fallback/not required] -- Reviewer result: [compatibility spec/quality review result evidence returned, or N/A only in direct_fallback/not required] -- Reviewer direct evidence: [compatibility fresh review/spec+quality evidence when direct_fallback, else N/A] -- QA Evaluator dispatch: [qa-evaluator subagent/tool/run id or assistant-review QA dispatch, or N/A only when qa_evaluation_mode=not_required/direct_fallback] -- QA Evaluator result: [acceptance final_verdict/result, score_progression, and evidence returned, or N/A only when qa_evaluation_mode=not_required/direct_fallback] -- QA Evaluator direct evidence: [fresh acceptance/Done Contract/verification QA evidence when direct_fallback, else N/A] -- Per-slice dispatch evidence: [medium+ delegated only: slice_id -> Code Writer dispatch/result + Builder/Tester dispatch/result; direct_fallback records sequential role-equivalent slice evidence] +- Evidence shorthand: delegated = dispatch/result refs; direct_fallback = role-equivalent direct evidence; N/A only when role not required. +- Code Mapper dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] +- Explorer dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] +- Architect dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] +- Code Writer dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] +- Builder/Tester dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] +- Code Reviewer dispatch/result/direct evidence: [delegated refs | Code Reviewer direct evidence | Reviewer legacy compatibility | N/A] +- Reviewer dispatch/result/direct evidence: [compatibility refs/direct evidence when used | N/A] +- QA Evaluator dispatch/result/direct evidence: [delegated QA refs | direct evidence | N/A when not required] +- Per-slice dispatch evidence: [medium+ delegated slice_id -> Code Writer + Builder/Tester refs; otherwise N/A: reason] ## Constraints - [user-stated boundaries, e.g. "Do not modify ProjectA"] @@ -101,100 +85,17 @@ Plan approval: [yes/no + date] |------|---------|-----------| | [path] | [what and why] | Step N | -## Done Contract -[required before Build for medium+ harness-capable work; otherwise record N/A with reason] -- done_when: - - [binary outcome that proves done] -- not_done_when: - - [failure state that blocks done] -- verification: - - [command, inspection, review, or manual check] -- owner_consumer: [owner and downstream consumer] -- acceptance_criteria: - - [explicit binary criterion] -- debate_record: - - perspective: [role/subagent/direct perspective 1] - concern_or_support: [concise point] - resolution: [accepted, rejected, or changed] - - perspective: [role/subagent/direct perspective 2] - concern_or_support: [concise point] - resolution: [accepted, rejected, or changed] -- accepted_by: [user/orchestrator/approved plan reference] - -## Harness Recipe -[required before Build for medium+ harness-capable work; otherwise record N/A with reason] -- task_profile: [task type, size, slice count, TDD/debugging applicability] -- model_profile: [agent/model constraints, delegation mode, tool limits] -- risk_profile: [risk tier, safety gates, review depth, rollback needs] -- context_profile: [exact/summarized/omitted context and trace/replay needs] -- selected_recipe: [concise recipe label] -- recipe_rationale: [why this task/model/risk/context profile selects the recipe] -- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] -- corrective_action: [what to do if missing or stale] - -## Harness Run State -[required for medium+ harness-capable work; otherwise record N/A with reason] -- task_id: [stable task/run id] -- task_name: [human-readable task name] -- phase: [TRIAGE | DISCOVER | DECOMPOSE | PLAN | DESIGN | BUILD | REVIEW | DOCUMENT | COMPLETE] -- slice: [current slice id/name, next pending slice, or N/A] -- status: [not_started | in_progress | blocked | verifying | reviewing | restarting | documenting | complete] -- blockers: - - [current blocker, or none] -- last_verification: - - command_or_check: [command/check, or pending/N/A] - - result: [passed | failed | skipped | not_applicable | pending] - - evidence: [concise evidence or reason] -- next_action: [exact next action] -- recovery_pointer: [task packet, trace entry, ledger row, file section, or artifact ref] -- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] - -## Trace Ledger -[required for medium+ harness-capable work; append-only ordered execution evidence] -| Seq | Timestamp/Order | Event Type | Actor | Summary | Artifact Refs | -|-----|-----------------|------------|-------|---------|---------------| -| 1 | [timestamp or ordered marker] | [agent_event/decision/verification/plan_deviation/pivot_restart/artifact_ref/blocker/recovery] | [role/subagent/user/hook] | [event, decision, command/result, deviation, blocker, pivot/restart, or recovery] | [file/section/dispatch id/command ref] | - -## Replay Packet -[required for medium+ harness-capable work before compaction, failure handoff, phase handoff, or end-of-turn] -- pinned_context: - - [stable requirement, constraint, approved plan/slice id, or non-goal] -- artifact_refs: - - [task journal/context map/plan/run state/trace/validation/changed file ref] -- validation_state: - - completed_checks: [checks already completed] - - pending_checks: [checks still pending] - - last_result: [latest verification/review result] -- exact_next_action: [single concrete next action after replay] -- run_state_ref: [Harness Run State section/ref] -- trace_ledger_ref: [Trace Ledger section/ref] -- recovery_pointer: [where to resume] -- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] - -## Pivot/Restart Log -[append when review/QA stagnation, repeated drift/regression, rubric/domain pivot, Code Writer blocker, verification blocker, plan deviation, or scope change triggers recovery; otherwise record N/A with reason] - -### Pivot/Restart Decision #N -- trigger: [STAGNATION | repeated_DRIFT | repeated_REGRESSION | pivot | code_writer_blocker | verification_blocker | plan_deviation | scope_change] -- evidence: - - source: [review score entry, QA score entry, Code Writer blocker, verification failure, trace row, or task packet] - detail: [concise evidence] -- affected_slice_or_round: [slice id/name, review round, QA round, or phase] -- options_considered: - - option: [reset context / dispatch debugging / dispatch explorer / dispatch architect / candidate search / replan / restart slice / restart phase / block for user / accept with limitations] - tradeoff: [why this option helps or risks the task] - disposition: [selected | rejected | deferred] -- selected_action: [reset_context | return_to_build | dispatch_debugging | dispatch_explorer | dispatch_architect | run_candidate_search | replan | restart_slice | restart_phase | block_for_user | accept_with_limitations] -- reapproval_required: [true when scope/files/behavior/risk/verification/acceptance changes; otherwise false] -- next_agent: [Builder/Tester | Code Writer | Explorer | Architect | Reviewer | QAEvaluator | assistant-debugging | candidate-search | user | none] -- recovery_pointer: [task packet, trace row, replay packet, plan section, or file path] -- exact_next_action: [single concrete action after the decision] - -## Artifact Reference Ledger -[required for medium+ harness-capable work when artifacts pass between agents; typed producer/consumer records, not ad hoc strings] -| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | -|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| -| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/pivot_restart_decision/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role/subagent/hook] | [role/subagent/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | +## Harness Appendix Routing +[required only for medium+ harness-capable work; otherwise record `N/A: [reason]`] +- Appendix: `references/task-journal-harness-appendix.md` +- Done Contract: [section/ref, or N/A: reason] +- Harness Recipe: [section/ref, or N/A: reason] +- Harness Run State: [section/ref, or N/A: reason] +- Trace Ledger: [section/ref, or N/A: reason] +- Replay Packet: [section/ref, or N/A: reason] +- Pivot/Restart Log: [section/ref when triggered, or N/A: reason] +- Artifact Reference Ledger: [section/ref when artifacts cross agents, or N/A: reason] +- QA Evaluation Log: [section/ref when qa_evaluation_mode=required, or N/A: reason] ## Milestones [compression-safe boundaries — each marks a point where context can be safely truncated] @@ -288,25 +189,13 @@ Plan approval: [yes/no + date] [...repeat until clean or max rounds reached...] [Note: On test failure, skip this entry — write only "- Result: HAS_REMAINING_ITEMS" to Final result] -### QA Evaluation #1 -- Round: 1 of 10 +### QA Evaluation - Mode: required | optional | not_required -- Done Contract: [ref or N/A with reason] -- Acceptance criteria checked: [count] -- Verification evidence checked: [commands/checks/manual/review refs] -- Code Review evidence: [Code Reviewer/Reviewer result ref] -- Domain/rubric refs: [product/UX/UI/docs/DX/domain refs, or N/A when not scoped] -- Selected domain rubrics: [ui_visual_design | ux_product_acceptance | documentation_quality | developer_experience | domain_specific_craft | N/A] -- Domain quality scores: [rubric.dimension=score/action/evidence, or N/A when not scoped] -- QA scorecard: acceptance_coverage=[score] evidence_strength=[score] domain_quality=[score] final_readiness=[score] -- Weighted: [score] -- Score progression: [round1->...] -- Pivot/Restart decision: [ref if STAGNATION, repeated DRIFT, repeated REGRESSION, or domain action pivot occurred; otherwise N/A] -- Final verdict: accepted | accepted_with_concerns | rejected | blocked -- Acceptance findings: - - [blocker|concern|observation] [criterion] — [evidence] → [fix/defer/open question] - -[...repeat QA Evaluation until accepted, accepted_with_concerns, blocked, or max round 10 reached...] +- QA Evaluator result: [final_verdict/result ref, or N/A: reason] +- Selected domain rubrics: [families used, or N/A] +- Domain quality scores: [compact scores, or N/A] +- Code Review evidence: [Code Reviewer/Reviewer result ref, or N/A: reason] +- Full schema: `references/task-journal-harness-appendix.md#qa-evaluation-log` when required ### Final result - Result: CLEAN | ISSUES_FIXED | HAS_REMAINING_ITEMS @@ -352,7 +241,7 @@ Plan approval: [yes/no + date] 3. **Clarification** updates — question caps are maximums, not quotas. Clear medium+ tasks may record `Clarification questions asked: 0` with `Clarification confidence: high`. While waiting, keep `Status: DISCOVERING`, set `Clarification status: needs_clarification`, set `Clarification defaults applied: false`, set confidence/cap/admissibility fields, and list every unresolved implementation-shaping topic. On explicit answers, clear unresolved topics, keep `Clarification defaults applied: false`, and set `Clarification status: ready`. On explicit `defaults`, print the applied defaults, clear unresolved topics, set `Clarification defaults applied: true`, and set `Clarification status: ready`. 4. **Decompose** — medium+ tasks set `Status: DECOMPOSING` after Discover is ready, then persist the slice manifest before moving on to planning. Small tasks skip this state. 5. **Plan approval** — once ready to plan, set `Status: PLANNING`, include the slice manifest in the plan for medium+ tasks, capture the approved plan, and update `Plan approval`. -6. **Build** each step — update Progress, Artifact Registry, Artifact Reference Ledger, Key Decisions, Status, Harness Run State, Trace Ledger, Replay Packet, and Pivot/Restart Log when triggered after each step. For medium+ tasks, update the Slice Verification Ledger after each slice and do not start the next slice until the current one is `VERIFIED`. Check off Milestones when reached. +6. **Build** each step — update Progress, Artifact Registry, Key Decisions, Status, and the harness appendix refs when triggered. For medium+ tasks, update the Slice Verification Ledger after each slice and do not start the next slice until the current one is `VERIFIED`. Check off Milestones when reached. 7. **Review cycle** when all steps done — Spec Review first (structured PASS/FAIL from `references/prompts/spec-review.md`), then Quality Review (assistant-review quality loop), fix must-fix → re-test → re-review until clean or a Pivot/Restart Decision routes recovery, fill Final Result 8. **Document** after review cycle passes — fill Verification Summary, add the Learning Controller block, Status: DOCUMENTING 9. **Handoff** to user — they test manually and add Review Notes @@ -364,8 +253,8 @@ Plan approval: [yes/no + date] - Keep entries concise — this is a log, not documentation - Resume from clarification waits only on explicit numbered answers or explicit `defaults` - Constraints are checked before each Build step -- Producer roles update Artifact Reference Ledger entries when they create or move artifacts; Consumer roles validate `schema_or_contract` and update `validation_status` before using them +- Producer roles update Artifact Reference Ledger entries in `references/task-journal-harness-appendix.md` when they create or move artifacts; Consumer roles validate `schema_or_contract` and update `validation_status` before using them - Pivot/Restart Decisions are append-only recovery records. If the selected action changes scope, files, behavior, risk, verification, or acceptance criteria, record `reapproval_required: true` and wait for approval before continuing. - On context continuation: read the configured task journal FIRST when it exists, before any other action - Never delete constraints unless the user explicitly removes them -- The Replay Packet replaces the need for context-handoff templates during active harness work +- The Replay Packet in `references/task-journal-harness-appendix.md` replaces context-handoff templates during active harness work diff --git a/skills/assistant-review/SKILL.md b/skills/assistant-review/SKILL.md index 884eb43..7765577 100644 --- a/skills/assistant-review/SKILL.md +++ b/skills/assistant-review/SKILL.md @@ -78,7 +78,7 @@ Use the smallest mode that answers the request; combine modes when reviewing imp - **Semantic contract review**: for skill/workflow/framework changes, check contract inheritance, method-template alignment, eval coverage, method-signature fidelity, and high-stakes recommendation guards before judging the change clean. - **Maintainability review**: apply SOLID/KISS/DRY/YAGNI/readability only when a concrete risk exists. - **Security handoff**: invoke `assistant-security` when the reviewed surface touches auth, permissions, secrets, input handling, persistence, shell commands, dependency/config changes, network calls, or external integrations. -- **QA evaluation**: after code-review/build evidence exists, dispatch QAEvaluator when `qa_evaluation_mode=required` for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work. Load `references/qa-evaluation-loop.md`. +- **QA evaluation**: after code-review/build evidence exists, dispatch QAEvaluator when `qa_evaluation_mode=required` for explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. Load `references/qa-evaluation-loop.md`. - **Domain rubric QA**: within QA evaluation, load `references/domain-rubrics.md` only when scoped by acceptance criteria, Done Contract, `domain_context`, or explicit `rubric_refs`. QAEvaluator selects rubric families and returns domain-quality scores; Code Reviewer still owns code defects, security, architecture, and test coverage. Finding format: diff --git a/skills/assistant-review/contracts/input.yaml b/skills/assistant-review/contracts/input.yaml index 67467bb..55137e2 100644 --- a/skills/assistant-review/contracts/input.yaml +++ b/skills/assistant-review/contracts/input.yaml @@ -93,19 +93,35 @@ fields: default: [] on_missing: skip + - name: harness_capable + type: boolean + required: false + description: "Whether the reviewed task is scoped into the workflow harness controller; ordinary medium+ review scopes default to false." + validation: "False by default. True only for explicitly requested harness/QA acceptance work, long-running work, trace/replay-ready multi-slice work, high-risk harness work, domain-scored work, UI/visual/product/UX/docs/DX-facing work, or accepted Done Contract/Harness Recipe presence." + default: false + on_missing: infer + infer_from: > + Default false. Set true only when the review material, task packet, or user request explicitly + scopes harness work; when an accepted Done Contract or Harness Recipe exists; or when the work is + long-running, trace/replay-ready multi-slice, high-risk harness, domain-scored, or + UI/visual/product/UX/docs/DX-facing. Do not infer true from scope_size=medium/large, + Reviewer delegation, or ordinary source-changing review scope. + - name: qa_evaluation_mode type: enum required: false enum_values: [not_required, optional, required] description: "Whether the independent QA evaluation loop should run after code-review/build evidence" - validation: "required for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work; not_required must have a reason in final output when skipped for medium+ harness work" + validation: "required only for explicit QA/acceptance evaluation, harness_capable == true with acceptance evaluation in scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing scope; not_required must have a reason when scoped acceptance QA is skipped" default: not_required on_missing: infer infer_from: > - Use required when the task is medium+ harness-capable, has a Done Contract, is domain-scored, - affects UI/visual/product/UX/docs/DX quality, or the user explicitly requests QA/acceptance evaluation. + Use required when the user explicitly requests QA/acceptance evaluation, an accepted Done Contract + exists, harness_capable == true and acceptance evaluation is in scope, the task is domain-scored, + or it affects UI/visual/product/UX/docs/DX quality. Use optional when acceptance material exists but the orchestrator has not marked QA as required. - Use not_required for trivial/small code-review-only work with no separate acceptance evidence need. + Use not_required for ordinary medium+ code-review-only/source-changing work with no separate + acceptance evidence need. - name: done_contract type: object diff --git a/skills/assistant-review/references/qa-evaluation-loop.md b/skills/assistant-review/references/qa-evaluation-loop.md index 496dd0c..5219919 100644 --- a/skills/assistant-review/references/qa-evaluation-loop.md +++ b/skills/assistant-review/references/qa-evaluation-loop.md @@ -5,11 +5,13 @@ Use this reference after build/test evidence and the Code Reviewer loop exist. Q ## When QA Runs Run QA evaluation when any of these are true: -- Medium+ harness-capable work has a Done Contract. -- The task is subjective, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly asks for QA. +- The user explicitly asks for QA or acceptance evaluation. +- The task has an accepted Done Contract. +- The task is harness-capable and acceptance evaluation is in scope. +- The task is subjective, domain-scored, or UI/visual/product/UX/docs/DX-facing. - Workflow Review needs a final acceptance verdict beyond code/security/architecture/test-coverage review. -Skip QA evaluation only when the task is trivial/small, has no acceptance material beyond a direct code fix, and the workflow output records `qa_evaluation_mode=not_required` with a reason. +Skip QA evaluation when the task has no explicit QA request, Done Contract, harness-capable acceptance scope, domain-scored criteria, or UI/visual/product/UX/docs/DX scope, and the workflow output records `qa_evaluation_mode=not_required` with a reason. ## Inputs diff --git a/skills/assistant-skill-creator/references/harness-patterns.md b/skills/assistant-skill-creator/references/harness-patterns.md index 87fac42..db66e89 100644 --- a/skills/assistant-skill-creator/references/harness-patterns.md +++ b/skills/assistant-skill-creator/references/harness-patterns.md @@ -81,7 +81,7 @@ Long-running Process skills need first-class recovery artifacts. | Artifact | Required When | Fields | |---|---|---| -| Harness Run State | Medium+ harness-capable or delegated loops | task id/name, phase, slice, status, blockers, last verification, next action, recovery pointer | +| Harness Run State | Medium+ harness-capable loops; delegated loops only when separately harness-capable | task id/name, phase, slice, status, blockers, last verification, next action, recovery pointer | | Trace Ledger | Any trace/replay-ready loop | agent events, decisions, verification results, deviations, blockers, artifact refs | | Replay Packet | Compaction, restart, handoff, or failure recovery is likely | pinned context, artifact refs, validation state, exact next action, run-state/trace refs | diff --git a/skills/assistant-workflow/SKILL.md b/skills/assistant-workflow/SKILL.md index 4ea7c36..64e16c9 100644 --- a/skills/assistant-workflow/SKILL.md +++ b/skills/assistant-workflow/SKILL.md @@ -25,7 +25,8 @@ This skill is intentionally agent-agnostic: it must work in restricted company e - Medium+ work has an approved plan before implementation; small work has an inline plan and proceeds without ceremony unless risk requires approval. - Medium+ harness-capable work has an accepted Done Contract and Harness Recipe before Build. - Trace/replay-ready harness work maintains Harness Run State, Trace Ledger, and Replay Packet artifacts. -- Medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work records independent QA Evaluator evidence after build/test and code-review evidence. +- Ordinary medium+ workflow tasks default to `harness_capable=false` and do not inherit Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless explicit harness/QA criteria apply. +- Harness-capable acceptance, Done Contract, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work records independent QA Evaluator evidence after build/test and code-review evidence. - Behavior changes have tests or explicit validation attached to the implementation step they protect. - Final output reports changed files, verification evidence, residual risks, and next steps. - Candidate Search is used for explicit alternatives, open-ended architecture/design, optimization, high uncertainty, repeated failed attempts, unclear/flaky bugs, or reviewer-requested pivots — not as default ceremony. @@ -211,7 +212,7 @@ Load `references/phases.md` and execute the phase matching your current stage. U For subagent roles and dispatch rules, load `references/subagent-dispatch.md` and resolve `subagent_policy_state`, `subagent_execution_mode`, and `subagent_authorization_scope` before any subagent spawn. Assistant Framework policy requires explicit user authorization before spawning subagents for development/code-work roles unless the current user prompt already explicitly authorizes them for this task. Ask once for the needed delegation scope and wait before continuing phases that require subagents. A sufficient prompt is: `This workflow expects Code Mapper, Architect, Code Writer, Builder/Tester, Code Reviewer, and QA Evaluator when QA is required for [scope]. May I use subagents for this task?` After authorization, use `delegated` mode and spawn the configured role agents. Use `direct_fallback` only when authorization is denied, policy disallows spawning, or a real spawn attempt fails because subagents/custom agents are unavailable; do not infer unavailability merely because no visible tool is named `Task`, `delegate`, or `subagent`. For BES-style option exploration, load `references/candidate-search.md` only when `search_mode: candidate_search` is selected. For mega tasks and anti-patterns, load `references/mega-and-patterns.md`. -Load `references/harness-controller.md` only when medium+ work is harness-capable: long-running, multi-slice, delegated, high-risk, subjective/domain-scored, trace/replay-ready, or explicitly requested as harness work. +Load `references/harness-controller.md` only when medium+ work is harness-capable: long-running, trace/replay-ready multi-slice, high-risk harness, subjective/domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested as harness work. ## Planning Checklist @@ -244,7 +245,7 @@ Before final output for medium+ or high-risk work: - Review the diff against the acceptance criteria. - For bugfixes, verify the review material includes reproduction/root-cause evidence from `assistant-debugging` or a clear reason it was not applicable. - Use `assistant-review` for code quality/spec review when the change is non-trivial. -- Use QA Evaluator through `assistant-review` for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested acceptance QA after build/test and code-review evidence. +- Use QA Evaluator through `assistant-review` when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work after build/test and code-review evidence. - Use `assistant-security` when touching auth, user input, secrets, persistence, network calls, shell commands, dependency/config changes, or external integrations. Review findings must cite evidence and concrete risk. Avoid generic style feedback unless it affects correctness, security, maintainability, or test reliability. diff --git a/skills/assistant-workflow/contracts/input.yaml b/skills/assistant-workflow/contracts/input.yaml index f107b5e..9568c6d 100644 --- a/skills/assistant-workflow/contracts/input.yaml +++ b/skills/assistant-workflow/contracts/input.yaml @@ -135,19 +135,35 @@ fields: on_missing: skip notes: "Populated by Idea-to-Action pipeline when input is a vague idea rather than a concrete task" + - name: harness_capable + type: boolean + required: false + description: "Whether the task is scoped into the harness controller; ordinary medium+ workflow tasks default to false." + validation: "False by default. True only for explicitly requested harness work, long-running work, trace/replay-ready multi-slice work, high-risk harness work, domain-scored work, UI/visual/product/UX/docs/DX-facing work, or accepted Done Contract/Harness Recipe presence." + default: false + on_missing: infer + infer_from: > + Default false. Set true only when the user or approved plan explicitly requests harness work, + the work is long-running, trace/replay-ready multi-slice work, high-risk harness work, + domain-scored, UI/visual/product/UX/docs/DX-facing, or already has an accepted Done Contract + or Harness Recipe. Do not infer true from size=medium+, delegation alone, or ordinary + source-changing workflow tasks. + - name: qa_evaluation_mode type: enum required: false enum_values: [not_required, optional, required] description: "Whether Review phase should run the independent QA Evaluator acceptance loop after build/test and code-review evidence" - validation: "required for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work; not_required must be justified when a medium+ harness task skips QA" + validation: "required only for explicit QA/acceptance evaluation, harness_capable == true with acceptance evaluation in scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing scope; not_required is the default for ordinary medium+ source-changing work." default: not_required on_missing: infer infer_from: > - Use required when the task is medium+ harness-capable, has a Done Contract, is domain-scored, - affects UI/visual/product/UX/docs/DX quality, or explicitly requests QA/acceptance evaluation. + Use required when the user explicitly requests QA/acceptance evaluation, the task has an accepted + Done Contract, harness_capable == true and acceptance evaluation is in scope, the task is + domain-scored, or it affects UI/visual/product/UX/docs/DX quality. Use optional when acceptance material exists but QA is not required. - Use not_required for trivial/small code-review-only work with no separate acceptance evidence need. + Use not_required for ordinary medium+ code-review-only/source-changing work with no separate + acceptance evidence need. - name: search_mode type: enum diff --git a/skills/assistant-workflow/contracts/output.yaml b/skills/assistant-workflow/contracts/output.yaml index fe50c33..31e1243 100644 --- a/skills/assistant-workflow/contracts/output.yaml +++ b/skills/assistant-workflow/contracts/output.yaml @@ -242,7 +242,7 @@ artifacts: - name: qa_evaluation_result type: object required: conditional - condition: "qa_evaluation_mode == required or size in [medium, large, mega] and harness_capable == true and work is domain-scored/UI/visual/product/UX/docs/DX-facing" + condition: "qa_evaluation_mode == required" validation: "Independent QA evaluation result exists after build/test evidence and Code Reviewer result. Must include final_verdict, result, rounds (1-10), acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped domain rubrics were used, score_progression, and evidence." on_fail: "Run the QAEvaluator loop from assistant-review references/qa-evaluation-loop.md, or record NEEDS_CONTEXT/BLOCKED with open_questions. Do not substitute Code Reviewer findings for QA evaluation." object_fields: diff --git a/skills/assistant-workflow/contracts/phase-gates.yaml b/skills/assistant-workflow/contracts/phase-gates.yaml index b5935e5..5e657a3 100644 --- a/skills/assistant-workflow/contracts/phase-gates.yaml +++ b/skills/assistant-workflow/contracts/phase-gates.yaml @@ -59,7 +59,8 @@ gates: on_fail: "Select required workflow agents when subagent_execution_mode=delegated, or record direct fallback roles and responsibilities from references/subagent-dispatch.md and risk-specific gate packs. not_applicable is invalid for source-changing Build tasks." - id: T_QA_EVALUATOR - check: "For medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work: required_agents includes QA Evaluator in addition to Code Writer, Builder/Tester, and Code Reviewer or Reviewer compatibility" + check: "When qa_evaluation_mode == required: required_agents includes QA Evaluator in addition to Code Writer, Builder/Tester, and Code Reviewer or Reviewer compatibility" + condition: "qa_evaluation_mode == required" on_fail: "Add QA Evaluator as a Review-phase role, or record qa_evaluation_mode=not_required with a concrete reason when QA is not applicable." - id: T9 diff --git a/skills/assistant-workflow/references/harness-controller.md b/skills/assistant-workflow/references/harness-controller.md index 77a6d7c..93f196c 100644 --- a/skills/assistant-workflow/references/harness-controller.md +++ b/skills/assistant-workflow/references/harness-controller.md @@ -1,12 +1,24 @@ # Harness Controller -Use this reference only for medium+ work that is harness-capable: long-running, -multi-slice, delegated, high-risk, subjective/domain-scored, trace/replay-ready, -or explicitly requested as harness work. Do not load it for small local fixes. +Use this reference only for medium+ work that is explicitly harness-capable: +long-running, trace/replay-ready multi-slice, high-risk harness, +subjective/domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly +requested as harness work. Do not load it for small local fixes, ordinary +medium+ source changes, or delegation alone. + +`harness_capable` defaults to false. Set it to true only when one of the +explicit criteria above is present in the request, approved plan, task packet, +or accepted Done Contract/Harness Recipe evidence. + +Base plan and task-journal templates keep compact refs and `N/A: [reason]` +fields. Load `references/plan-harness-appendix.md` and +`references/task-journal-harness-appendix.md` only when the full harness, +typed artifact, pivot/restart, or QA schemas are needed. ## Pre-Build Gate -Before Build starts, the approved plan must contain both: +When `harness_capable=true`, before Build starts the approved plan must contain +both: - an accepted Done Contract - a selected Harness Recipe diff --git a/skills/assistant-workflow/references/phases.md b/skills/assistant-workflow/references/phases.md index b589e07..2521c94 100644 --- a/skills/assistant-workflow/references/phases.md +++ b/skills/assistant-workflow/references/phases.md @@ -173,7 +173,7 @@ Print: `>> Direct fallback Architect responsibility` (when `subagent_execution_m **Entry rule:** Do not enter Plan while the saved clarification state is pending. Resume Plan only after Discover records `Clarification status: ready` and all implementation-shaping fields are explicit or explicitly defaulted. -Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. For medium+ harness-capable work, load `references/harness-controller.md` and add the Done Contract, Harness Recipe, Harness Run State ref, Trace Ledger ref, Replay Packet ref, and Artifact Reference Ledger before task packets. Then read `references/plan-template.md` and use the correct tier: +Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. Treat `harness_capable` as false unless the task is long-running, trace/replay-ready multi-slice, high-risk harness, domain-scored, UI/visual/product/UX/docs/DX-facing, explicitly requested as harness/QA work, or already has an accepted Done Contract/Harness Recipe. For medium+ harness-capable work, load `references/harness-controller.md` plus `references/plan-harness-appendix.md`, then add compact Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger refs before task packets. Then read `references/plan-template.md` and use the correct tier: - Small: inline plan (goal, files, risks, tests). Do not wait for approval unless risk, ambiguity, user instruction, or a scope-changing decision makes approval necessary. - Medium: standard plan (drop Security/Operability unless the task touches auth, PII, payments, or infra) - Large/Mega: full plan (all sections including Security and Operability) @@ -183,7 +183,7 @@ Before writing the plan, load `references/artifact-first-output-contract.md` and 3. Analyze 1-3 options with tradeoffs, pick one 4. Identify risks and edge cases 5. Put the Artifact Contract before task packets and map every medium+ task packet to at least one required artifact or acceptance criterion -6. For medium+ harness-capable work, put the Done Contract, Harness Recipe, run-state/trace/replay artifact refs, and Artifact Reference Ledger before Build. The Done Contract must include done_when, not_done_when, verification, owner/consumer, acceptance criteria, and at least two debate perspectives; the Harness Recipe must be selected from task/model/risk/context profile. The ledger must type each artifact with artifact_id, artifact_type, producer, consumer, location_ref, schema_or_contract, validation_status, and summary. +6. For medium+ harness-capable work, put compact refs for Done Contract, Harness Recipe, run-state/trace/replay artifacts, and Artifact Reference Ledger before Build; load `references/plan-harness-appendix.md` for the full schemas. The base plan keeps `N/A: [reason]` for non-harness work. 7. For medium+ tasks: consume the Decompose slice manifest directly in the plan and align each task packet to exactly one slice_id without rediscovering boundaries 8. Write ordered implementation steps with file paths 9. For large/mega: fill in Security and Operability sections. For medium: only if the task touches auth, PII, payments, or infra (promote to Full tier per plan-template.md) @@ -239,7 +239,7 @@ For medium+ tasks, create a task journal using `references/task-journal-template Capture **constraints** from Discovery/Plan (e.g. "don't touch ProjectA", "stay on .NET 8"). Check constraints before each step. -For medium+ harness-capable work, confirm the task journal or carried-forward plan has an accepted Done Contract, selected Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger before dispatching Code Writer or Builder/Tester. If any are missing, stop Build and return to Plan or repair state for the corrective action in `references/harness-controller.md`. +For medium+ harness-capable work, confirm the task journal or carried-forward plan has compact refs for an accepted Done Contract, selected Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger before dispatching Code Writer or Builder/Tester. Load `references/task-journal-harness-appendix.md` for the full schema. If any required ref is missing, stop Build and return to Plan or repair state for the corrective action in `references/harness-controller.md`. Keep the runtime artifacts current as execution progresses: - Update Harness Run State after each slice/step, blocker, verification result, phase transition, or next-action change. @@ -253,9 +253,9 @@ Keep the runtime artifacts current as execution progresses: - **Code Writer** (`code-writer`): implements code following the plan - **Builder/Tester** (`builder-tester`): builds, writes tests, runs tests - **Code Reviewer** (`code-reviewer`, or `reviewer` compatibility through assistant-review delegated review): independent code/security/architecture/test coverage review evidence for Review -- **QA Evaluator** (`qa-evaluator`): independent acceptance, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result evaluation when QA is required +- **QA Evaluator** (`qa-evaluator`): independent acceptance, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result evaluation when `qa_evaluation_mode=required` -Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Code Reviewer; `reviewer` remains compatibility routing for existing/legacy handoffs. Medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work also adds QA Evaluator. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. +Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Code Reviewer; `reviewer` remains compatibility routing for existing/legacy handoffs. Add QA Evaluator only when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. **Direct fallback mode (`subagent_execution_mode=direct_fallback`):** when authorization is denied, subagents are unavailable, or policy disallows spawning, the active agent may implement, test, and review directly, but must preserve the same phases, contracts, evidence requirements, and review/security gates. Do not pretend delegation happened; record `subagent_policy_state`, `subagent_execution_mode`, the explicit `Direct fallback reason: authorization_denied | subagents_unavailable | policy_disallowed`, and the direct-execution evidence. @@ -424,7 +424,7 @@ For medium+ tasks: full spec review plus autonomous code quality loop via `assis Print: `>> Stage 3: QA Evaluation — loading assistant-review references/qa-evaluation-loop.md` when QA is required. -Run QA Evaluation for medium+ harness-capable, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. +Run QA Evaluation only when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. The QA Evaluator receives Done Contract when present, acceptance criteria, verification evidence, code review result, domain_context/rubric_refs when applicable, round 1-10, previously_failed_acceptance_items, and qa_filter_policy. It loads assistant-review `references/domain-rubrics.md` only when acceptance criteria, Done Contract, domain_context, or explicit rubric_refs require subjective/product/UX/docs/DX/UI/domain scoring. It returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression or score_entry, evidence, and open_questions when blocked. diff --git a/skills/assistant-workflow/references/plan-harness-appendix.md b/skills/assistant-workflow/references/plan-harness-appendix.md new file mode 100644 index 0000000..b280454 --- /dev/null +++ b/skills/assistant-workflow/references/plan-harness-appendix.md @@ -0,0 +1,99 @@ +# Plan Harness Appendix + +Load this appendix only for medium+ harness-capable work: explicit harness work, +long-running or trace/replay-ready multi-slice work, high-risk harness work, +accepted Done Contract/Harness Recipe work, domain-scored work, or +UI/visual/product/UX/docs/DX-facing work. For ordinary medium+ source changes, +keep the base plan's harness fields as `N/A: [reason]`. + +## Task Packet Harness Fields + +Use these fields inside an executable task packet only when the task is +harness-capable. + +```markdown +- Harness refs: + - done_contract_ref: [Done Contract section/ref, or N/A: reason] + - harness_recipe_ref: [Harness Recipe section/ref, or N/A: reason] + - harness_run_state_ref: [Harness Run State section/ref, or N/A: reason] + - trace_ledger_ref: [Trace Ledger section/ref, or N/A: reason] + - replay_packet_ref: [Replay Packet section/ref, or N/A: reason] + - artifact_reference_ledger_ref: [Artifact Reference Ledger section/ref, or N/A: reason] +- Typed artifact refs: + - artifact_id: [stable task-local id] + artifact_type: [done_contract | harness_recipe | harness_run_state | trace_ledger | replay_packet | pivot_restart_decision | changed_files | verification_evidence | plan_deviation | task_packet | context_map | test_result | review_result | qa_evaluation_result] + producer: [role/subagent/hook/task packet] + consumer: [role/subagent/hook/phase] + location_ref: [typed location/ref pointer] + schema_or_contract: [contract/template/required fields] + validation_status: [pending | valid | invalid | stale | not_applicable] + summary: [concise state] +``` + +## Done Contract + +Required for medium+ harness-capable work before Build. + +- done_when: + - [binary outcome that proves done] +- not_done_when: + - [failure state that blocks done] +- verification: + - [command, inspection, review, or manual check] +- owner_consumer: [owner and downstream consumer] +- acceptance_criteria: + - [explicit binary criterion] +- debate_record: + - perspective: [role/subagent/direct perspective 1] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] + - perspective: [role/subagent/direct perspective 2] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] +- accepted_by: [user/orchestrator/approved plan reference] + +## Harness Recipe + +Required for medium+ harness-capable work before Build; selected from +task/model/risk/context profile per `references/harness-controller.md`. + +- task_profile: [task type, size, slice count, TDD/debugging applicability] +- model_profile: [agent/model constraints, delegation mode, tool limits] +- risk_profile: [risk tier, safety gates, review depth, rollback needs] +- context_profile: [exact/summarized/omitted context and trace/replay needs] +- selected_recipe: [concise recipe label] +- recipe_rationale: [why this profile selects the recipe] +- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] +- corrective_action: [what to do if missing or stale] + +## Runtime Harness Artifacts + +Required for medium+ harness-capable work when the recipe needs recovery, +handoff, or trace/replay evidence. + +- harness_run_state_ref: [where task_id/task_name/phase/slice/status/blockers/last_verification/next_action/recovery_pointer will be maintained] +- trace_ledger_ref: [where ordered agent events, decisions, verification results, plan deviations, and artifact refs will be appended] +- replay_packet_ref: [where pinned context, artifact refs, validation state, and exact next action will be refreshed] +- corrective_action: [what to do if run-state/trace/replay evidence is missing or stale] + +## Artifact Reference Ledger + +Required for medium+ harness-capable work when artifacts pass between agents. +Each row is a typed producer/consumer record, not an ad hoc string reference. + +| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | +|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| +| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/pivot_restart_decision/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role] | [role/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | + +## QA Routing + +QA Evaluator remains separate from Code Reviewer. Use this routing only when +`qa_evaluation_mode=required`: explicit QA/acceptance evaluation, +harness-capable acceptance scope, accepted Done Contract, domain-scored work, or +UI/visual/product/UX/docs/DX-facing work. + +- qa_evaluation_mode: [required | optional | not_required] +- qa_trigger_reason: [explicit QA request, Done Contract, harness acceptance scope, domain-scored, UI/visual/product/UX/docs/DX, or N/A: reason] +- code_review_result_ref: [Code Reviewer/Reviewer compatibility result ref] +- qa_evaluation_result_ref: [QA Evaluator result ref, or N/A: reason] +- domain_context_ref: [domain/rubric context ref, or N/A: reason] diff --git a/skills/assistant-workflow/references/plan-template.md b/skills/assistant-workflow/references/plan-template.md index 10814cd..321c7ca 100755 --- a/skills/assistant-workflow/references/plan-template.md +++ b/skills/assistant-workflow/references/plan-template.md @@ -2,6 +2,11 @@ Three tiers — match ceremony to task size (don't get fancy when N is small). +Harness details are optional appendices. For medium+ harness-capable work, keep +compact refs in the base plan and load `references/plan-harness-appendix.md` +for the full schema. For non-harness work, record `N/A: [reason]` instead of +copying harness sections into the base plan. + ## Small Tasks — Inline Plan No separate plan document needed. Include directly in your response: @@ -55,21 +60,15 @@ For Medium and Large/Mega plans, write implementation work as executable task pa - Expected success signal: [exit code 0, passing test name, output marker, etc.] - Evidence to record: - [test result, eval fixture, changed file, review note, or artifact proof] -- Harness refs: - - done_contract_ref: [Done Contract section/ref, or N/A] - - harness_recipe_ref: [Harness Recipe section/ref, or N/A] - - harness_run_state_ref: [Harness Run State section/ref, or N/A] - - trace_ledger_ref: [Trace Ledger section/ref, or N/A] - - replay_packet_ref: [Replay Packet section/ref, or N/A] -- Typed artifact refs: - - artifact_id: [stable task-local id] - artifact_type: [done_contract | harness_recipe | harness_run_state | trace_ledger | replay_packet | changed_files | verification_evidence | plan_deviation | task_packet | context_map | test_result | review_result | qa_evaluation_result] - producer: [role/subagent/hook/task packet] - consumer: [role/subagent/hook/phase] - location_ref: [typed location/ref pointer] - schema_or_contract: [contract/template/required fields] - validation_status: [pending | valid | invalid | stale | not_applicable] - summary: [concise state] +- Harness routing: + - applies: [required for medium+ harness-capable work; otherwise N/A: reason] + - appendix: [references/plan-harness-appendix.md, or N/A: reason] + - done_contract_ref: [section/ref, or N/A: reason] + - harness_recipe_ref: [section/ref, or N/A: reason] + - harness_run_state_ref: [section/ref, or N/A: reason] + - trace_ledger_ref: [section/ref, or N/A: reason] + - replay_packet_ref: [section/ref, or N/A: reason] + - artifact_reference_ledger_ref: [section/ref, or N/A: reason] - Deviation / rollback rule: - [what to do if required files/behavior differ from plan; include rollback/revert boundary] - Worker status / evidence: @@ -186,61 +185,21 @@ Covers the essentials without Security/Operability overhead. Fill this in during - Owner/consumer: [user, reviewer, downstream tool, runtime] - Non-goals/exclusions: [what must not be produced] -## Done Contract - -Required for medium+ harness-capable work before Build; omit with a brief N/A -rationale for non-harness work. - -- done_when: - - [binary outcome that proves done] -- not_done_when: - - [failure state that blocks done] -- verification: - - [command, inspection, review, or manual check] -- owner_consumer: [owner and downstream consumer] -- acceptance_criteria: - - [explicit binary criterion] -- debate_record: - - perspective: [role/subagent/direct perspective 1] - concern_or_support: [concise point] - resolution: [accepted, rejected, or changed] - - perspective: [role/subagent/direct perspective 2] - concern_or_support: [concise point] - resolution: [accepted, rejected, or changed] -- accepted_by: [user/orchestrator/approved plan reference] - -## Harness Recipe - -Required for medium+ harness-capable work before Build; selected from -task/model/risk/context profile per `references/harness-controller.md`. - -- task_profile: [task type, size, slice count, TDD/debugging applicability] -- model_profile: [agent/model constraints, delegation mode, tool limits] -- risk_profile: [risk tier, safety gates, review depth, rollback needs] -- context_profile: [exact/summarized/omitted context and trace/replay needs] -- selected_recipe: [concise recipe label] -- recipe_rationale: [why this profile selects the recipe] -- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] -- corrective_action: [what to do if missing or stale] - -## Runtime Harness Artifacts - -Required for medium+ harness-capable work; omit with a brief N/A rationale for -non-harness work. - -- harness_run_state_ref: [where task_id/task_name/phase/slice/status/blockers/last_verification/next_action/recovery_pointer will be maintained] -- trace_ledger_ref: [where ordered agent events, decisions, verification results, plan deviations, and artifact refs will be appended] -- replay_packet_ref: [where pinned context, artifact refs, validation state, and exact next action will be refreshed] -- corrective_action: [what to do if run-state/trace/replay evidence is missing or stale] +## Harness Appendix Routing -## Artifact Reference Ledger +Required only for medium+ harness-capable work; otherwise keep every field as +`N/A: [reason]`. Load `references/plan-harness-appendix.md` for the full Done +Contract, Harness Recipe, runtime artifact, typed artifact ref, and QA routing +schemas. -Required for medium+ harness-capable work when artifacts pass between agents. -Each row is a typed producer/consumer record, not an ad hoc string reference. - -| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | -|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| -| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role] | [role/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | +- appendix_status: [required | N/A: reason] +- done_contract_ref: [section/ref, or N/A: reason] +- harness_recipe_ref: [section/ref, or N/A: reason] +- harness_run_state_ref: [section/ref, or N/A: reason] +- trace_ledger_ref: [section/ref, or N/A: reason] +- replay_packet_ref: [section/ref, or N/A: reason] +- artifact_reference_ledger_ref: [section/ref, or N/A: reason] +- qa_evaluation_mode: [required | optional | not_required + reason] ## Slice manifest from Decompose @@ -258,45 +217,7 @@ Use the Executable Task Packet structure above for each approved slice. Order pa Everything from Medium, plus Security and Operability sections. Use when the task touches auth, external inputs, infrastructure, or multi-module boundaries. ```markdown -## Goal -- [1-3 sentence restated requirement from Discovery] - -## Triage result -- Task type: [feature | bugfix | refactor | migration | rewrite | config | infra | security | docs | spike] -- Risk tier: [low | moderate | high | critical] -- Required gates: [common gates + task-category gate packs from references/triage-rubric.md] -- Required agents: [roles/skills selected from size, task type, and risk] -- Search mode: [none | lightweight | candidate_search] - -## Constraints & decisions (from Discovery) -- [Q&A question]: [chosen option and why] -- [Q&A question]: [chosen option and why] -- Assumed (not explicitly asked): [assumption and reasoning] -- Non-goals: [what's explicitly out of scope] - -## Research (current state) -- Modules/subprojects: ... -- Key files/paths: ... -- Entrypoints: ... -- Configs/flags: ... -- Data models: ... -- Existing patterns: ... - -## Architecture -- Current architecture: [identified or "new project"] -- Architecture for this change: [Clean/MVVM/Hexagonal/etc.] -- Layer rules: - - [e.g., Domain has no external dependencies] - - [e.g., ViewModels don't reference Views] -- Dependency direction: [A → B → C] -- New files placement: - - [file → layer/folder rationale] -- SOLID design notes: - - SRP: [which classes own which responsibility — flag any class with >1 reason to change] - - OCP: [will new variants require modifying existing classes? If yes, plan extension points] - - LSP: [any inheritance hierarchies? Do subtypes preserve base type contracts?] - - ISP: [any interfaces? Are they minimal or do implementers need to stub methods?] - - DIP: [which high-level modules depend on abstractions vs concrete implementations?] +Start with the Medium template, then add the sections below. ## Security considerations - Data classification: [does this touch PII, auth, payments, external inputs?] @@ -316,41 +237,26 @@ Everything from Medium, plus Security and Operability sections. Use when the tas - Revert commit sufficient: [yes/no] - Runbook updates: [new on-call procedures needed?] -## Analysis -### Candidate search summary -- Candidate search summary: [N/A unless search_mode=candidate_search; otherwise selected candidate and why] -- Candidate archive: [{agent_state_dir}/candidate-search.md when local state is allowed, or inline plan section] -- Goal tree source: [acceptance criteria/slice criteria used] - -### Options -1. [approach] — [tradeoff] -2. [approach] — [tradeoff] - -### Decision -- Chosen: [#] because [reason] - -### Risks / edge cases -- [risk]: [mitigation] - -## Decomposition Plan Review - -- Scope understanding: [pass/fix needed + evidence] -- Slice/subagent count: [count + sanity rationale] -- Step/cost budget: [budget or direct-fallback rationale] -- Dependency order: [summary] -- Output-plan match: [artifact/verification alignment] -- Fallback path: [subagent path or direct equivalent] -- Broad-split rejection: [required proof that layer-only, module-only, folder-only, feature-only, setup-only, contract-only, and broad component-style splits were rejected unless verified deliverable artifact slices] -- Decision: proceed | revise_decomposition | return_to_discover - -## Artifact Reference Ledger - -Required for medium+ harness-capable work when artifacts pass between agents. -Each row is a typed producer/consumer record, not an ad hoc string reference. - -| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | -|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| -| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role] | [role/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | +## Large/Mega addenda +- Triage result: include `Subagent policy state`, `Subagent execution mode`, `Subagent authorization scope`, and `Search mode` as in Medium. +- Architecture: include LSP and ISP in addition to SRP/OCP/DIP. +- Decomposition Plan Review: reuse Medium fields and keep `- Broad-split rejection:` proof for large/mega slice plans. +- Harness Appendix Routing: reuse the Medium compact refs and load `references/plan-harness-appendix.md` only when harness-capable. + +## Harness Appendix Routing + +Required only for medium+ harness-capable work; otherwise record +`N/A: [reason]`. Load `references/plan-harness-appendix.md` for full harness, +typed artifact reference, pivot/restart, and QA routing schemas. + +- appendix_status: [required | N/A: reason] +- done_contract_ref: [section/ref, or N/A: reason] +- harness_recipe_ref: [section/ref, or N/A: reason] +- harness_run_state_ref: [section/ref, or N/A: reason] +- trace_ledger_ref: [section/ref, or N/A: reason] +- replay_packet_ref: [section/ref, or N/A: reason] +- artifact_reference_ledger_ref: [section/ref, or N/A: reason] +- qa_evaluation_mode: [required | optional | not_required + reason] ## Slice manifest from Decompose diff --git a/skills/assistant-workflow/references/task-journal-harness-appendix.md b/skills/assistant-workflow/references/task-journal-harness-appendix.md new file mode 100644 index 0000000..d311d37 --- /dev/null +++ b/skills/assistant-workflow/references/task-journal-harness-appendix.md @@ -0,0 +1,145 @@ +# Task Journal Harness Appendix + +Load this appendix only when the task journal is tracking medium+ +harness-capable work, QA evaluation, typed artifact references, or a +Pivot/Restart Decision. The base task journal keeps compact refs and +`N/A: [reason]` fields so ordinary medium+ work does not inherit this detail. + +## Done Contract + +[required before Build for medium+ harness-capable work] + +- done_when: + - [binary outcome that proves done] +- not_done_when: + - [failure state that blocks done] +- verification: + - [command, inspection, review, or manual check] +- owner_consumer: [owner and downstream consumer] +- acceptance_criteria: + - [explicit binary criterion] +- debate_record: + - perspective: [role/subagent/direct perspective 1] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] + - perspective: [role/subagent/direct perspective 2] + concern_or_support: [concise point] + resolution: [accepted, rejected, or changed] +- accepted_by: [user/orchestrator/approved plan reference] + +## Harness Recipe + +[required before Build for medium+ harness-capable work] + +- task_profile: [task type, size, slice count, TDD/debugging applicability] +- model_profile: [agent/model constraints, delegation mode, tool limits] +- risk_profile: [risk tier, safety gates, review depth, rollback needs] +- context_profile: [exact/summarized/omitted context and trace/replay needs] +- selected_recipe: [concise recipe label] +- recipe_rationale: [why this task/model/risk/context profile selects the recipe] +- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] +- corrective_action: [what to do if missing or stale] + +## Harness Run State + +[required for medium+ harness-capable work] + +- task_id: [stable task/run id] +- task_name: [human-readable task name] +- phase: [TRIAGE | DISCOVER | DECOMPOSE | PLAN | DESIGN | BUILD | REVIEW | DOCUMENT | COMPLETE] +- slice: [current slice id/name, next pending slice, or N/A] +- status: [not_started | in_progress | blocked | verifying | reviewing | restarting | documenting | complete] +- blockers: + - [current blocker, or none] +- last_verification: + - command_or_check: [command/check, or pending/N/A] + - result: [passed | failed | skipped | not_applicable | pending] + - evidence: [concise evidence or reason] +- next_action: [exact next action] +- recovery_pointer: [task packet, trace entry, ledger row, file section, or artifact ref] +- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] + +## Trace Ledger + +[required for medium+ harness-capable work; append-only ordered execution evidence] + +| Seq | Timestamp/Order | Event Type | Actor | Summary | Artifact Refs | +|-----|-----------------|------------|-------|---------|---------------| +| 1 | [timestamp or ordered marker] | [agent_event/decision/verification/plan_deviation/pivot_restart/artifact_ref/blocker/recovery] | [role/subagent/user/hook] | [event, decision, command/result, deviation, blocker, pivot/restart, or recovery] | [file/section/dispatch id/command ref] | + +## Replay Packet + +[required for medium+ harness-capable work before compaction, failure handoff, +phase handoff, or end-of-turn] + +- pinned_context: + - [stable requirement, constraint, approved plan/slice id, or non-goal] +- artifact_refs: + - [task journal/context map/plan/run state/trace/validation/changed file ref] +- validation_state: + - completed_checks: [checks already completed] + - pending_checks: [checks still pending] + - last_result: [latest verification/review result] +- exact_next_action: [single concrete next action after replay] +- run_state_ref: [Harness Run State section/ref] +- trace_ledger_ref: [Trace Ledger section/ref] +- recovery_pointer: [where to resume] +- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] + +## Pivot/Restart Log + +[append when review/QA stagnation, repeated drift/regression, rubric/domain +pivot, Code Writer blocker, verification blocker, plan deviation, or scope +change triggers recovery] + +### Pivot/Restart Decision #N + +- trigger: [STAGNATION | repeated_DRIFT | repeated_REGRESSION | pivot | code_writer_blocker | verification_blocker | plan_deviation | scope_change] +- evidence: + - source: [review score entry, QA score entry, Code Writer blocker, verification failure, trace row, or task packet] + detail: [concise evidence] +- affected_slice_or_round: [slice id/name, review round, QA round, or phase] +- options_considered: + - option: [reset context / dispatch debugging / dispatch explorer / dispatch architect / candidate search / replan / restart slice / restart phase / block for user / accept with limitations] + tradeoff: [why this option helps or risks the task] + disposition: [selected | rejected | deferred] +- selected_action: [reset_context | return_to_build | dispatch_debugging | dispatch_explorer | dispatch_architect | run_candidate_search | replan | restart_slice | restart_phase | block_for_user | accept_with_limitations] +- reapproval_required: [true when scope/files/behavior/risk/verification/acceptance changes; otherwise false] +- next_agent: [Builder/Tester | Code Writer | Explorer | Architect | Reviewer | QAEvaluator | assistant-debugging | candidate-search | user | none] +- recovery_pointer: [task packet, trace row, replay packet, plan section, or file path] +- exact_next_action: [single concrete action after the decision] + +## Artifact Reference Ledger + +[required for medium+ harness-capable work when artifacts pass between agents; +typed producer/consumer records, not ad hoc strings] + +| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | +|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| +| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/pivot_restart_decision/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role/subagent/hook] | [role/subagent/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | + +## QA Evaluation Log + +QA Evaluation is required only when `qa_evaluation_mode=required`. It runs after +build/test evidence and Code Reviewer or Reviewer compatibility evidence. + +### QA Evaluation #1 + +- Round: 1 of 10 +- Mode: required | optional | not_required +- Done Contract: [ref or N/A with reason] +- Acceptance criteria checked: [count] +- Verification evidence checked: [commands/checks/manual/review refs] +- Code Review evidence: [Code Reviewer/Reviewer result ref] +- Domain/rubric refs: [product/UX/UI/docs/DX/domain refs, or N/A when not scoped] +- Selected domain rubrics: [ui_visual_design | ux_product_acceptance | documentation_quality | developer_experience | domain_specific_craft | N/A] +- Domain quality scores: [rubric.dimension=score/action/evidence, or N/A when not scoped] +- QA scorecard: acceptance_coverage=[score] evidence_strength=[score] domain_quality=[score] final_readiness=[score] +- Weighted: [score] +- Score progression: [round1->...] +- Pivot/Restart decision: [ref if STAGNATION, repeated DRIFT, repeated REGRESSION, or domain action pivot occurred; otherwise N/A] +- Final verdict: accepted | accepted_with_concerns | rejected | blocked +- Acceptance findings: + - [blocker|concern|observation] [criterion] - [evidence] -> [fix/defer/open question] + +[...repeat QA Evaluation until accepted, accepted_with_concerns, blocked, or max round 10 reached...] diff --git a/skills/assistant-workflow/references/task-journal-template.md b/skills/assistant-workflow/references/task-journal-template.md index 5d8eac2..7944d4d 100644 --- a/skills/assistant-workflow/references/task-journal-template.md +++ b/skills/assistant-workflow/references/task-journal-template.md @@ -1,6 +1,6 @@ # Task Journal Template -Write to `{agent_state_dir}/task.md` in the project root when a local agent state directory is configured and policy allows local state artifacts. If no safe state directory is available, keep the same content in the response/plan packet. This state is the single source of truth for the current task — it survives context compression and session continuations when persisted. It is a framework-owned ignored state artifact, so the orchestrator may create and update it directly when allowed. +Write to `{agent_state_dir}/task.md` in the project root when a local state directory is configured and policy allows. If none is safe, keep the same content in the response/plan packet. This framework-owned ignored artifact is the task source of truth, survives compression/continuation when persisted, and may be updated by the orchestrator. ## When to create - Any task that enters clarification wait: during Discover, before printing clarification questions or the wait message @@ -10,9 +10,8 @@ Write to `{agent_state_dir}/task.md` in the project root when a local agent stat ## When to update - When clarification questions are asked, answered, or resolved via explicit `defaults` - After each Build step completes (update Progress, Artifact Registry, and check off Milestones) -- After each medium+ harness event that changes phase, slice, status, blockers, verification, next action, decisions, deviations, or artifact refs -- After any Pivot/Restart Decision, including review/QA stagnation, repeated drift/regression, Code Writer blockers, or selected restart/replan action -- After any producer creates/updates a typed artifact reference or any consumer changes `validation_status` +- After each medium+ harness-capable event listed in `references/task-journal-harness-appendix.md` +- After any Pivot/Restart Decision, typed artifact reference update, or QA evaluation result when applicable - When key decisions are made - When constraints are added by the user - After each review cycle pass (append to Review Log — never overwrite) @@ -54,35 +53,20 @@ Plan approval: [yes/no + date] ## Agent Dispatch Log [strict subagent evidence inspected by stop-review/phase gates] -- Required roles: Code Writer, Builder/Tester, Code Reviewer [Reviewer remains compatibility routing; plus QA Evaluator when QA is required; plus Code Mapper/Explorer/Architect when required by size/risk] +- Required roles: Code Writer, Builder/Tester, Code Reviewer; QA Evaluator when `qa_evaluation_mode=required`; Code Mapper/Explorer/Architect by size/risk; Reviewer for legacy compatibility. - Execution mode: delegated | direct_fallback | not_applicable -- Codex lifecycle evidence: delegated Codex roles must have matching `.codex/subagent-events.jsonl` `SubagentStart` + `SubagentStop` records; dispatch/result entries must cite the matching `agent_id`; journal text alone is not proof. +- Codex lifecycle evidence: delegated Codex roles need matching `.codex/subagent-events.jsonl` `SubagentStart`/`SubagentStop` records and dispatch/result refs with the same `agent_id`; journal text alone is insufficient. - Direct fallback reason: [authorization_denied | subagents_unavailable | policy_disallowed | N/A] -- Code Mapper dispatch: [subagent/tool/run id, mapping packet, timestamp, or N/A only in direct_fallback/not required] -- Code Mapper result: [context map summary/evidence, or N/A only in direct_fallback/not required] -- Code Mapper direct evidence: [role-equivalent context map evidence when direct_fallback, else N/A] -- Explorer dispatch: [subagent/tool/run id, exploration packet, timestamp, or N/A only in direct_fallback/not required] -- Explorer result: [execution path/dependency evidence, or N/A only in direct_fallback/not required] -- Explorer direct evidence: [role-equivalent exploration evidence when direct_fallback, else N/A] -- Architect dispatch: [subagent/tool/run id, architecture/decomposition packet, timestamp, or N/A only in direct_fallback/not required] -- Architect result: [slice/blueprint/design evidence, or N/A only in direct_fallback/not required] -- Architect direct evidence: [role-equivalent decomposition/architecture evidence when direct_fallback, else N/A] -- Code Writer dispatch: [subagent/tool/run id, prompt/packet, timestamp, or N/A only in direct_fallback/not required] -- Code Writer result: [returned files/summary/evidence, or N/A only in direct_fallback/not required] -- Code Writer direct evidence: [role-equivalent implementation evidence when direct_fallback, else N/A] -- Builder/Tester dispatch: [subagent/tool/run id, verification packet, timestamp, or N/A only in direct_fallback/not required] -- Builder/Tester result: [commands/results/success signal returned, or N/A only in direct_fallback/not required] -- Builder/Tester direct evidence: [role-equivalent build/test/validation evidence when direct_fallback, else N/A] -- Code Reviewer dispatch: [code-reviewer subagent/tool/run id or assistant-review dispatch, or N/A only when Reviewer compatibility/direct_fallback/not required] -- Code Reviewer result: [spec/quality review result evidence returned, or N/A only when Reviewer compatibility/direct_fallback/not required] -- Code Reviewer direct evidence: [fresh review/spec+quality evidence when direct_fallback, else N/A] -- Reviewer dispatch: [compatibility reviewer subagent/tool/run id or assistant-review dispatch, or N/A only in direct_fallback/not required] -- Reviewer result: [compatibility spec/quality review result evidence returned, or N/A only in direct_fallback/not required] -- Reviewer direct evidence: [compatibility fresh review/spec+quality evidence when direct_fallback, else N/A] -- QA Evaluator dispatch: [qa-evaluator subagent/tool/run id or assistant-review QA dispatch, or N/A only when qa_evaluation_mode=not_required/direct_fallback] -- QA Evaluator result: [acceptance final_verdict/result, score_progression, and evidence returned, or N/A only when qa_evaluation_mode=not_required/direct_fallback] -- QA Evaluator direct evidence: [fresh acceptance/Done Contract/verification QA evidence when direct_fallback, else N/A] -- Per-slice dispatch evidence: [medium+ delegated only: slice_id -> Code Writer dispatch/result + Builder/Tester dispatch/result; direct_fallback records sequential role-equivalent slice evidence] +- Evidence shorthand: delegated = dispatch/result refs; direct_fallback = role-equivalent direct evidence; N/A only when role not required. +- Code Mapper dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] +- Explorer dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] +- Architect dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] +- Code Writer dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] +- Builder/Tester dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] +- Code Reviewer dispatch/result/direct evidence: [delegated refs | Code Reviewer direct evidence | Reviewer legacy compatibility | N/A] +- Reviewer dispatch/result/direct evidence: [compatibility refs/direct evidence when used | N/A] +- QA Evaluator dispatch/result/direct evidence: [delegated QA refs | direct evidence | N/A when not required] +- Per-slice dispatch evidence: [medium+ delegated slice_id -> Code Writer + Builder/Tester refs; otherwise N/A: reason] ## Constraints - [user-stated boundaries, e.g. "Do not modify ProjectA"] @@ -101,100 +85,17 @@ Plan approval: [yes/no + date] |------|---------|-----------| | [path] | [what and why] | Step N | -## Done Contract -[required before Build for medium+ harness-capable work; otherwise record N/A with reason] -- done_when: - - [binary outcome that proves done] -- not_done_when: - - [failure state that blocks done] -- verification: - - [command, inspection, review, or manual check] -- owner_consumer: [owner and downstream consumer] -- acceptance_criteria: - - [explicit binary criterion] -- debate_record: - - perspective: [role/subagent/direct perspective 1] - concern_or_support: [concise point] - resolution: [accepted, rejected, or changed] - - perspective: [role/subagent/direct perspective 2] - concern_or_support: [concise point] - resolution: [accepted, rejected, or changed] -- accepted_by: [user/orchestrator/approved plan reference] - -## Harness Recipe -[required before Build for medium+ harness-capable work; otherwise record N/A with reason] -- task_profile: [task type, size, slice count, TDD/debugging applicability] -- model_profile: [agent/model constraints, delegation mode, tool limits] -- risk_profile: [risk tier, safety gates, review depth, rollback needs] -- context_profile: [exact/summarized/omitted context and trace/replay needs] -- selected_recipe: [concise recipe label] -- recipe_rationale: [why this task/model/risk/context profile selects the recipe] -- required_artifacts: [Done Contract, task packet, verification, trace/handoff artifacts] -- corrective_action: [what to do if missing or stale] - -## Harness Run State -[required for medium+ harness-capable work; otherwise record N/A with reason] -- task_id: [stable task/run id] -- task_name: [human-readable task name] -- phase: [TRIAGE | DISCOVER | DECOMPOSE | PLAN | DESIGN | BUILD | REVIEW | DOCUMENT | COMPLETE] -- slice: [current slice id/name, next pending slice, or N/A] -- status: [not_started | in_progress | blocked | verifying | reviewing | restarting | documenting | complete] -- blockers: - - [current blocker, or none] -- last_verification: - - command_or_check: [command/check, or pending/N/A] - - result: [passed | failed | skipped | not_applicable | pending] - - evidence: [concise evidence or reason] -- next_action: [exact next action] -- recovery_pointer: [task packet, trace entry, ledger row, file section, or artifact ref] -- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] - -## Trace Ledger -[required for medium+ harness-capable work; append-only ordered execution evidence] -| Seq | Timestamp/Order | Event Type | Actor | Summary | Artifact Refs | -|-----|-----------------|------------|-------|---------|---------------| -| 1 | [timestamp or ordered marker] | [agent_event/decision/verification/plan_deviation/pivot_restart/artifact_ref/blocker/recovery] | [role/subagent/user/hook] | [event, decision, command/result, deviation, blocker, pivot/restart, or recovery] | [file/section/dispatch id/command ref] | - -## Replay Packet -[required for medium+ harness-capable work before compaction, failure handoff, phase handoff, or end-of-turn] -- pinned_context: - - [stable requirement, constraint, approved plan/slice id, or non-goal] -- artifact_refs: - - [task journal/context map/plan/run state/trace/validation/changed file ref] -- validation_state: - - completed_checks: [checks already completed] - - pending_checks: [checks still pending] - - last_result: [latest verification/review result] -- exact_next_action: [single concrete next action after replay] -- run_state_ref: [Harness Run State section/ref] -- trace_ledger_ref: [Trace Ledger section/ref] -- recovery_pointer: [where to resume] -- pivot_restart_decision_ref: [Pivot/Restart Decision ref when active, or N/A] - -## Pivot/Restart Log -[append when review/QA stagnation, repeated drift/regression, rubric/domain pivot, Code Writer blocker, verification blocker, plan deviation, or scope change triggers recovery; otherwise record N/A with reason] - -### Pivot/Restart Decision #N -- trigger: [STAGNATION | repeated_DRIFT | repeated_REGRESSION | pivot | code_writer_blocker | verification_blocker | plan_deviation | scope_change] -- evidence: - - source: [review score entry, QA score entry, Code Writer blocker, verification failure, trace row, or task packet] - detail: [concise evidence] -- affected_slice_or_round: [slice id/name, review round, QA round, or phase] -- options_considered: - - option: [reset context / dispatch debugging / dispatch explorer / dispatch architect / candidate search / replan / restart slice / restart phase / block for user / accept with limitations] - tradeoff: [why this option helps or risks the task] - disposition: [selected | rejected | deferred] -- selected_action: [reset_context | return_to_build | dispatch_debugging | dispatch_explorer | dispatch_architect | run_candidate_search | replan | restart_slice | restart_phase | block_for_user | accept_with_limitations] -- reapproval_required: [true when scope/files/behavior/risk/verification/acceptance changes; otherwise false] -- next_agent: [Builder/Tester | Code Writer | Explorer | Architect | Reviewer | QAEvaluator | assistant-debugging | candidate-search | user | none] -- recovery_pointer: [task packet, trace row, replay packet, plan section, or file path] -- exact_next_action: [single concrete action after the decision] - -## Artifact Reference Ledger -[required for medium+ harness-capable work when artifacts pass between agents; typed producer/consumer records, not ad hoc strings] -| Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary | -|-------------|---------------|----------|----------|--------------|--------------------|-------------------|---------| -| [id] | [done_contract/harness_recipe/harness_run_state/trace_ledger/replay_packet/pivot_restart_decision/changed_files/verification_evidence/plan_deviation/task_packet/context_map/test_result/review_result/qa_evaluation_result] | [role/subagent/hook] | [role/subagent/phase] | [file/section/dispatch/command ref] | [contract/template/fields] | [pending/valid/invalid/stale/not_applicable] | [concise state] | +## Harness Appendix Routing +[required only for medium+ harness-capable work; otherwise record `N/A: [reason]`] +- Appendix: `references/task-journal-harness-appendix.md` +- Done Contract: [section/ref, or N/A: reason] +- Harness Recipe: [section/ref, or N/A: reason] +- Harness Run State: [section/ref, or N/A: reason] +- Trace Ledger: [section/ref, or N/A: reason] +- Replay Packet: [section/ref, or N/A: reason] +- Pivot/Restart Log: [section/ref when triggered, or N/A: reason] +- Artifact Reference Ledger: [section/ref when artifacts cross agents, or N/A: reason] +- QA Evaluation Log: [section/ref when qa_evaluation_mode=required, or N/A: reason] ## Milestones [compression-safe boundaries — each marks a point where context can be safely truncated] @@ -288,25 +189,13 @@ Plan approval: [yes/no + date] [...repeat until clean or max rounds reached...] [Note: On test failure, skip this entry — write only "- Result: HAS_REMAINING_ITEMS" to Final result] -### QA Evaluation #1 -- Round: 1 of 10 +### QA Evaluation - Mode: required | optional | not_required -- Done Contract: [ref or N/A with reason] -- Acceptance criteria checked: [count] -- Verification evidence checked: [commands/checks/manual/review refs] -- Code Review evidence: [Code Reviewer/Reviewer result ref] -- Domain/rubric refs: [product/UX/UI/docs/DX/domain refs, or N/A when not scoped] -- Selected domain rubrics: [ui_visual_design | ux_product_acceptance | documentation_quality | developer_experience | domain_specific_craft | N/A] -- Domain quality scores: [rubric.dimension=score/action/evidence, or N/A when not scoped] -- QA scorecard: acceptance_coverage=[score] evidence_strength=[score] domain_quality=[score] final_readiness=[score] -- Weighted: [score] -- Score progression: [round1->...] -- Pivot/Restart decision: [ref if STAGNATION, repeated DRIFT, repeated REGRESSION, or domain action pivot occurred; otherwise N/A] -- Final verdict: accepted | accepted_with_concerns | rejected | blocked -- Acceptance findings: - - [blocker|concern|observation] [criterion] — [evidence] → [fix/defer/open question] - -[...repeat QA Evaluation until accepted, accepted_with_concerns, blocked, or max round 10 reached...] +- QA Evaluator result: [final_verdict/result ref, or N/A: reason] +- Selected domain rubrics: [families used, or N/A] +- Domain quality scores: [compact scores, or N/A] +- Code Review evidence: [Code Reviewer/Reviewer result ref, or N/A: reason] +- Full schema: `references/task-journal-harness-appendix.md#qa-evaluation-log` when required ### Final result - Result: CLEAN | ISSUES_FIXED | HAS_REMAINING_ITEMS @@ -352,7 +241,7 @@ Plan approval: [yes/no + date] 3. **Clarification** updates — question caps are maximums, not quotas. Clear medium+ tasks may record `Clarification questions asked: 0` with `Clarification confidence: high`. While waiting, keep `Status: DISCOVERING`, set `Clarification status: needs_clarification`, set `Clarification defaults applied: false`, set confidence/cap/admissibility fields, and list every unresolved implementation-shaping topic. On explicit answers, clear unresolved topics, keep `Clarification defaults applied: false`, and set `Clarification status: ready`. On explicit `defaults`, print the applied defaults, clear unresolved topics, set `Clarification defaults applied: true`, and set `Clarification status: ready`. 4. **Decompose** — medium+ tasks set `Status: DECOMPOSING` after Discover is ready, then persist the slice manifest before moving on to planning. Small tasks skip this state. 5. **Plan approval** — once ready to plan, set `Status: PLANNING`, include the slice manifest in the plan for medium+ tasks, capture the approved plan, and update `Plan approval`. -6. **Build** each step — update Progress, Artifact Registry, Artifact Reference Ledger, Key Decisions, Status, Harness Run State, Trace Ledger, Replay Packet, and Pivot/Restart Log when triggered after each step. For medium+ tasks, update the Slice Verification Ledger after each slice and do not start the next slice until the current one is `VERIFIED`. Check off Milestones when reached. +6. **Build** each step — update Progress, Artifact Registry, Key Decisions, Status, and the harness appendix refs when triggered. For medium+ tasks, update the Slice Verification Ledger after each slice and do not start the next slice until the current one is `VERIFIED`. Check off Milestones when reached. 7. **Review cycle** when all steps done — Spec Review first (structured PASS/FAIL from `references/prompts/spec-review.md`), then Quality Review (assistant-review quality loop), fix must-fix → re-test → re-review until clean or a Pivot/Restart Decision routes recovery, fill Final Result 8. **Document** after review cycle passes — fill Verification Summary, add the Learning Controller block, Status: DOCUMENTING 9. **Handoff** to user — they test manually and add Review Notes @@ -364,8 +253,8 @@ Plan approval: [yes/no + date] - Keep entries concise — this is a log, not documentation - Resume from clarification waits only on explicit numbered answers or explicit `defaults` - Constraints are checked before each Build step -- Producer roles update Artifact Reference Ledger entries when they create or move artifacts; Consumer roles validate `schema_or_contract` and update `validation_status` before using them +- Producer roles update Artifact Reference Ledger entries in `references/task-journal-harness-appendix.md` when they create or move artifacts; Consumer roles validate `schema_or_contract` and update `validation_status` before using them - Pivot/Restart Decisions are append-only recovery records. If the selected action changes scope, files, behavior, risk, verification, or acceptance criteria, record `reapproval_required: true` and wait for approval before continuing. - On context continuation: read the configured task journal FIRST when it exists, before any other action - Never delete constraints unless the user explicitly removes them -- The Replay Packet replaces the need for context-handoff templates during active harness work +- The Replay Packet in `references/task-journal-harness-appendix.md` replaces context-handoff templates during active harness work diff --git a/tests/p0-p4/domain-rubrics-contracts.sh b/tests/p0-p4/domain-rubrics-contracts.sh old mode 100644 new mode 100755 diff --git a/tests/p0-p4/harness-controller-contracts.sh b/tests/p0-p4/harness-controller-contracts.sh index 734acf3..a7e59f3 100644 --- a/tests/p0-p4/harness-controller-contracts.sh +++ b/tests/p0-p4/harness-controller-contracts.sh @@ -25,14 +25,16 @@ require_terms() { workflow_dir="$FRAMEWORK_DIR/skills/assistant-workflow" harness_ref="$workflow_dir/references/harness-controller.md" +plan_appendix="$workflow_dir/references/plan-harness-appendix.md" +task_journal_appendix="$workflow_dir/references/task-journal-harness-appendix.md" test_start "harness controller reference defines Done Contract and recipe selection" if [[ ! -f "$harness_ref" ]]; then fail "missing skills/assistant-workflow/references/harness-controller.md" else require_terms "harness reference" "$harness_ref" \ - "Use this reference only for medium+ work that is harness-capable" \ - "Before Build starts" \ + "Use this reference only for medium+ work that is explicitly harness-capable" \ + "before Build starts" \ "Done Contract" \ "Harness Recipe" \ 'done_when' \ @@ -54,14 +56,17 @@ test_start "workflow loads harness reference only for relevant medium+ work" missing_load_terms=() for term in \ "Medium+ harness-capable work has an accepted Done Contract and Harness Recipe before Build." \ + 'Ordinary medium+ workflow tasks default to `harness_capable=false`' \ "Load \`references/harness-controller.md\` only when medium+ work is harness-capable"; do if ! grep -Fq -- "$term" "$workflow_dir/SKILL.md"; then missing_load_terms+=("SKILL.md: $term") fi done for term in \ + "Treat \`harness_capable\` as false unless" \ "For medium+ harness-capable work, load \`references/harness-controller.md\`" \ - "confirm the task journal or carried-forward plan has an accepted Done Contract, selected Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger before dispatching Code Writer or Builder/Tester" \ + "has compact refs for an accepted Done Contract, selected Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger before dispatching Code Writer or Builder/Tester" \ + "references/task-journal-harness-appendix.md" \ 'done_contract_ref' \ 'harness_recipe_ref'; do if ! grep -Fq -- "$term" "$workflow_dir/references/phases.md"; then @@ -74,6 +79,33 @@ else fail "workflow harness load guards missing terms: ${missing_load_terms[*]}" fi +test_start "workflow defaults harness_capable false unless explicitly scoped" +require_terms "workflow input contract" "$workflow_dir/contracts/input.yaml" \ + "- name: harness_capable" \ + "default: false" \ + "Default false." \ + "explicitly requested harness work" \ + "long-running" \ + "trace/replay-ready multi-slice" \ + "high-risk harness" \ + "domain-scored" \ + "UI/visual/product/UX/docs/DX-facing" \ + "Do not infer true from size=medium+" \ + "delegation alone" \ + "ordinary medium+ workflow tasks default to false" \ + "source-changing workflow tasks" + +test_start "harness artifacts are not unconditional medium+ requirements" +unconditional_harness_terms_output="$(mktemp)" +p0p4_register_cleanup "$unconditional_harness_terms_output" +if rg -n 'medium\+ tasks[^.\n]*(Done Contract|Harness Recipe|Trace Ledger|Replay Packet|Artifact Reference Ledger)' \ + "$workflow_dir" \ + "$FRAMEWORK_DIR/skills/assistant-review" >"$unconditional_harness_terms_output"; then + fail "found unconditional medium+ harness artifact requirement; see $unconditional_harness_terms_output" +else + pass +fi + test_start "phase gates require Done Contract and Harness Recipe before Build" phase_gates="$workflow_dir/contracts/phase-gates.yaml" require_terms "phase gates" "$phase_gates" \ @@ -114,21 +146,55 @@ require_terms "output contract" "$output_contract" \ test_start "plan, journal, and handoffs carry harness artifacts without mirror edits" missing_surface_terms=() for term in \ - "## Done Contract" \ - "## Harness Recipe" \ + "## Harness Appendix Routing" \ + "references/plan-harness-appendix.md" \ + "N/A: [reason]" \ "done_contract_ref" \ - "harness_recipe_ref"; do + "harness_recipe_ref" \ + "artifact_reference_ledger_ref"; do if ! grep -Fq -- "$term" "$workflow_dir/references/plan-template.md"; then missing_surface_terms+=("plan-template.md: $term") fi done +for term in \ + "## Harness Appendix Routing" \ + "references/task-journal-harness-appendix.md" \ + "N/A: [reason]" \ + "Done Contract:" \ + "Harness Recipe:" \ + "QA Evaluator dispatch"; do + if ! grep -Fq -- "$term" "$workflow_dir/references/task-journal-template.md"; then + missing_surface_terms+=("task-journal-template.md: $term") + fi +done for term in \ "## Done Contract" \ "## Harness Recipe" \ + "## Runtime Harness Artifacts" \ + "harness_run_state_ref" \ + "trace_ledger_ref" \ + "replay_packet_ref" \ + "## Artifact Reference Ledger" \ + "## QA Routing" \ "done_when" \ "corrective_action"; do - if ! grep -Fq -- "$term" "$workflow_dir/references/task-journal-template.md"; then - missing_surface_terms+=("task-journal-template.md: $term") + if [[ ! -f "$plan_appendix" ]] || ! grep -Fq -- "$term" "$plan_appendix"; then + missing_surface_terms+=("plan-harness-appendix.md: $term") + fi +done +for term in \ + "## Done Contract" \ + "## Harness Recipe" \ + "## Harness Run State" \ + "## Trace Ledger" \ + "## Replay Packet" \ + "## Pivot/Restart Log" \ + "## Artifact Reference Ledger" \ + "## QA Evaluation Log" \ + "### QA Evaluation #1" \ + "pivot_restart_decision"; do + if [[ ! -f "$task_journal_appendix" ]] || ! grep -Fq -- "$term" "$task_journal_appendix"; then + missing_surface_terms+=("task-journal-harness-appendix.md: $term") fi done for term in \ diff --git a/tests/p0-p4/instruction-overload-contracts.sh b/tests/p0-p4/instruction-overload-contracts.sh index fa0f41e..5e89d2f 100755 --- a/tests/p0-p4/instruction-overload-contracts.sh +++ b/tests/p0-p4/instruction-overload-contracts.sh @@ -8,6 +8,10 @@ p0p4_bootstrap_suite "${BASH_SOURCE[0]}" workflow_output="$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml" workflow_gates="$FRAMEWORK_DIR/skills/assistant-workflow/contracts/phase-gates.yaml" +plan_template="$FRAMEWORK_DIR/skills/assistant-workflow/references/plan-template.md" +task_journal_template="$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md" +plan_harness_appendix="$FRAMEWORK_DIR/skills/assistant-workflow/references/plan-harness-appendix.md" +task_journal_harness_appendix="$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-harness-appendix.md" reduction_doc="$FRAMEWORK_DIR/docs/instruction-overload-reduction.md" benchmark_script="$FRAMEWORK_DIR/tools/hooks/benchmark-hook-output.sh" benchmark_doc="$FRAMEWORK_DIR/docs/hook-output-benchmarks.md" @@ -88,6 +92,12 @@ if [[ -f "$benchmark_doc" ]]; then benchmark_failures+=("benchmark doc missing $hook row") fi done + if ! grep -Fq "runtime-gate-output-trim" "$benchmark_doc"; then + benchmark_failures+=("benchmark doc missing runtime-gate-output-trim history row") + fi +fi +if [[ -f "$benchmark_script" ]] && ! grep -Fq "runtime-gate-output-trim" "$benchmark_script"; then + benchmark_failures+=("benchmark script does not render runtime-gate-output-trim history") fi if [[ "${#benchmark_failures[@]}" -eq 0 ]]; then pass @@ -127,10 +137,39 @@ if awk ' ' "$workflow_output"; then bloat_failures+=("review_result is still unconditionally required") fi -if [[ "${#bloat_failures[@]}" -eq 0 ]] && grep -Fq "Prompt bloat linting" "$reduction_doc"; then +if [[ "${#bloat_failures[@]}" -eq 0 ]] \ + && grep -Fq "Prompt bloat linting" "$reduction_doc" \ + && grep -Fq "Phase-scoped hook warnings" "$reduction_doc" \ + && grep -Fq 'gate:key missing=... action=...' "$reduction_doc"; then + pass +else + fail "prompt bloat lint failed: ${bloat_failures[*]:-reduction doc missing phase-scoped warning/prompt bloat sections}" +fi + +test_start "base workflow templates keep harness detail in optional appendices" +template_bloat_failures=() +combined_template_words=$(( $(wc -w < "$plan_template") + $(wc -w < "$task_journal_template") )) +if [[ "$combined_template_words" -gt 3984 ]]; then + template_bloat_failures+=("base plan+journal word count $combined_template_words exceeds 3984") +fi +for file in "$plan_harness_appendix" "$task_journal_harness_appendix"; do + if [[ ! -f "$file" ]]; then + template_bloat_failures+=("missing ${file#$FRAMEWORK_DIR/}") + fi +done +if ! grep -Fq "references/plan-harness-appendix.md" "$plan_template"; then + template_bloat_failures+=("plan-template.md missing plan harness appendix pointer") +fi +if ! grep -Fq "references/task-journal-harness-appendix.md" "$task_journal_template"; then + template_bloat_failures+=("task-journal-template.md missing task journal harness appendix pointer") +fi +if ! grep -Fq "N/A: [reason]" "$plan_template" || ! grep -Fq "N/A: [reason]" "$task_journal_template"; then + template_bloat_failures+=("base templates missing N/A-with-reason fields") +fi +if [[ "${#template_bloat_failures[@]}" -eq 0 ]]; then pass else - fail "prompt bloat lint failed: ${bloat_failures[*]:-reduction doc missing Prompt bloat linting section}" + fail "optional harness appendix split failed: ${template_bloat_failures[*]}" fi p0p4_finish_suite "${BASH_SOURCE[0]}" diff --git a/tests/p0-p4/qa-evaluator-contracts.sh b/tests/p0-p4/qa-evaluator-contracts.sh index 4a7535e..91db67d 100644 --- a/tests/p0-p4/qa-evaluator-contracts.sh +++ b/tests/p0-p4/qa-evaluator-contracts.sh @@ -43,7 +43,9 @@ if [[ -f "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml" ]]; then "score progression" \ "final acceptance result" \ "Do NOT replace code-reviewer" \ - "Do NOT edit any files"; do + "Do NOT edit any files" \ + "final_verdict=rejected and result=HAS_REMAINING_ITEMS" \ + "failed acceptance items should return to Build before round 10"; do if ! grep -Fq -- "$term" "$FRAMEWORK_DIR/agents/codex/qa-evaluator.toml"; then missing_agent_terms+=("agents/codex/qa-evaluator.toml: $term") fi @@ -58,7 +60,9 @@ if [[ -f "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md" ]]; then "score progression" \ "final acceptance result" \ "Do NOT replace code-reviewer" \ - "Do NOT edit any files"; do + "Do NOT edit any files" \ + "final_verdict=rejected and result=HAS_REMAINING_ITEMS" \ + "failed acceptance items should return to Build before round 10"; do if ! grep -Fq -- "$term" "$FRAMEWORK_DIR/agents/claude/qa-evaluator.md"; then missing_agent_terms+=("agents/claude/qa-evaluator.md: $term") fi @@ -186,13 +190,14 @@ for file_and_term in \ "$workflow_dir/contracts/output.yaml::qa_evaluator_evidence" \ "$workflow_dir/contracts/phase-gates.yaml::R_QA_EVALUATION" \ "$workflow_dir/contracts/phase-gates.yaml::Code Reviewer or Reviewer compatibility evidence is recorded separately from QA Evaluator evidence" \ + "$workflow_dir/contracts/phase-gates.yaml::When qa_evaluation_mode == required: required_agents includes QA Evaluator" \ "$workflow_dir/references/phases.md::Stage 3: QA Evaluation" \ "$workflow_dir/references/phases.md::QA Evaluation result must exist when qa_evaluation_mode=required." \ "$workflow_dir/references/subagent-dispatch.md::QA Evaluator" \ "$workflow_dir/references/subagent-dispatch.md::QA evidence gate" \ "$workflow_dir/references/subagent-roles.md::QA Evaluator dispatch" \ "$workflow_dir/references/task-journal-template.md::QA Evaluator dispatch" \ - "$workflow_dir/references/task-journal-template.md::### QA Evaluation #1" \ + "$workflow_dir/references/task-journal-harness-appendix.md::### QA Evaluation #1" \ "$workflow_dir/references/sub-task-brief-template.md::qa-evaluator"; do file="${file_and_term%%::*}" term="${file_and_term#*::}" @@ -206,6 +211,35 @@ else fail "workflow QA evaluator evidence surfaces missing: ${missing_workflow_terms[*]}" fi +test_start "QA evaluation required mode is explicitly scoped" +review_dir="$FRAMEWORK_DIR/skills/assistant-review" +require_terms "workflow QA mode" "$workflow_dir/contracts/input.yaml" \ + "- name: harness_capable" \ + "default: false" \ + "Use required when the user explicitly requests QA/acceptance evaluation" \ + "accepted Done Contract" \ + "harness_capable == true and acceptance evaluation is in scope" \ + "domain-scored" \ + "UI/visual/product/UX/docs/DX" \ + "Use not_required for ordinary medium+ code-review-only/source-changing work" +require_terms "review QA mode" "$review_dir/contracts/input.yaml" \ + "- name: harness_capable" \ + "default: false" \ + "Use required when the user explicitly requests QA/acceptance evaluation" \ + "accepted Done Contract" \ + "harness_capable == true and acceptance evaluation is in scope" \ + "domain-scored" \ + "UI/visual/product/UX/docs/DX" \ + "Use not_required for ordinary medium+ code-review-only/source-changing work" +require_terms "QA loop routing" "$review_dir/references/qa-evaluation-loop.md" \ + "The user explicitly asks for QA or acceptance evaluation." \ + "The task has an accepted Done Contract." \ + "The task is harness-capable and acceptance evaluation is in scope." \ + "Skip QA evaluation when the task has no explicit QA request, Done Contract, harness-capable acceptance scope, domain-scored criteria, or UI/visual/product/UX/docs/DX scope" +require_terms "workflow QA output mode" "$workflow_dir/contracts/output.yaml" \ + 'condition: "qa_evaluation_mode == required"' \ + "QA evaluation does not replace review_result" + test_start "workflow QA Done Contract debate_record contract is mirrored" workflow_debate_failures=() for workflow_qa_dir in \ diff --git a/tests/p0-p4/review-loop-cap-contracts.sh b/tests/p0-p4/review-loop-cap-contracts.sh index 0f1a28e..c8b1ad8 100755 --- a/tests/p0-p4/review-loop-cap-contracts.sh +++ b/tests/p0-p4/review-loop-cap-contracts.sh @@ -96,8 +96,9 @@ for file_and_term in \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/prompts/pr-review.md::up to 10 rounds" \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/prompts/pr-review.md::Round N of 10" \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/subagent-roles.md::Round 8-10: 90%+" \ - "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh::Round: N of 10" \ - "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh::N between 1 and 10" \ + "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh::compact_stop_reason()" \ + "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh::review_gate:missing_review_round) printf '%s\n' '- Round: N of 10'" \ + "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh::review_gate:round_overflow) printf '%s\n' '- Round: N of 10 with N between 1 and 10'" \ "$FRAMEWORK_DIR/hooks/scripts/workflow-enforcer.sh::max 10 rounds" \ "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh::max_round\" -ne 10" \ "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh::round\" -gt 10" \ diff --git a/tests/p0-p4/runtime-phase-gate-contracts.sh b/tests/p0-p4/runtime-phase-gate-contracts.sh index ec66e21..f38aa61 100644 --- a/tests/p0-p4/runtime-phase-gate-contracts.sh +++ b/tests/p0-p4/runtime-phase-gate-contracts.sh @@ -11,10 +11,15 @@ if [[ ! -f "$helper_file" ]]; then else for term in \ "assistant_phase_has_plan_approval" \ + "assistant_phase_plan_missing_reason_key" \ "assistant_phase_review_missing_reason_key" \ "assistant_phase_review_controller_missing_reason_key" \ "assistant_phase_learning_missing_reason_key" \ - "assistant_phase_has_metrics_today"; do + "assistant_phase_has_metrics_today" \ + "assistant_phase_reason_missing_field" \ + "assistant_phase_reason_action" \ + "assistant_phase_subagent_warning_reason_key" \ + "assistant_phase_subagent_warning_action"; do if ! grep -Fq "$term" "$helper_file"; then missing_runtime_helper_terms+=("$term") fi @@ -48,8 +53,18 @@ for term in \ "missing_durable_lesson_decision" \ "missing_persistence_evidence" \ "missing_no_save_rationale"; do + if ! grep -Fq "$term" "$helper_file"; then + missing_runtime_helper_terms+=("workflow-phase-gates.sh owns $term") + fi +done +for term in \ + "compact_stop_reason" \ + "subagent_evidence_gate" \ + "review_gate" \ + "learning_gate" \ + "metrics_gate"; do if ! grep -Fq "$term" "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh"; then - missing_runtime_helper_terms+=("stop-review.sh handles $term") + missing_runtime_helper_terms+=("stop-review.sh formats $term") fi done if [[ "${#missing_runtime_helper_terms[@]}" -eq 0 ]]; then @@ -69,6 +84,9 @@ for term in \ "cap is maximum, not quota" \ "Question admissibility" \ "WARNING: You are BUILDING without an approved plan" \ + "WARNING: Subagent evidence gate incomplete" \ + "assistant_phase_subagent_warning_reason_key" \ + "assistant_phase_subagent_warning_action" \ "WARNING: Review gate incomplete" \ "WARNING: Metrics gate incomplete"; do if ! grep -Fq "$term" "$FRAMEWORK_DIR/hooks/scripts/workflow-enforcer.sh"; then @@ -90,7 +108,9 @@ phase_gates="$workflow_dir/contracts/phase-gates.yaml" harness_ref="$workflow_dir/references/harness-controller.md" phases_ref="$workflow_dir/references/phases.md" task_journal_template="$workflow_dir/references/task-journal-template.md" +task_journal_harness_appendix="$workflow_dir/references/task-journal-harness-appendix.md" plan_template="$workflow_dir/references/plan-template.md" +plan_harness_appendix="$workflow_dir/references/plan-harness-appendix.md" for term in \ "- name: harness_run_state" \ "task_id" \ @@ -144,19 +164,22 @@ for term in \ "task_name" \ "last_verification" \ "exact_next_action"; do - if ! grep -Fq -- "$term" "$task_journal_template"; then - missing_runtime_artifact_terms+=("task-journal-template.md: $term") + if [[ ! -f "$task_journal_harness_appendix" ]] || ! grep -Fq -- "$term" "$task_journal_harness_appendix"; then + missing_runtime_artifact_terms+=("task-journal-harness-appendix.md: $term") fi done for term in \ "harness_run_state_ref" \ "trace_ledger_ref" \ "replay_packet_ref" \ - "Runtime Harness Artifacts"; do + "references/plan-harness-appendix.md"; do if ! grep -Fq -- "$term" "$plan_template"; then missing_runtime_artifact_terms+=("plan-template.md: $term") fi done +if [[ ! -f "$plan_harness_appendix" ]] || ! grep -Fq -- "Runtime Harness Artifacts" "$plan_harness_appendix"; then + missing_runtime_artifact_terms+=("plan-harness-appendix.md: Runtime Harness Artifacts") +fi for term in \ "Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger" \ "Update Harness Run State after each slice/step" \ diff --git a/tests/p0-p4/task-packet-contracts.sh b/tests/p0-p4/task-packet-contracts.sh index 8365bf5..f7add41 100644 --- a/tests/p0-p4/task-packet-contracts.sh +++ b/tests/p0-p4/task-packet-contracts.sh @@ -724,7 +724,7 @@ workflow_subagent_gate_terms=( "skills/assistant-workflow/references/task-journal-template.md|Reviewer dispatch" "skills/assistant-workflow/references/task-journal-template.md|not required" "skills/assistant-workflow/references/task-journal-template.md|Code Writer dispatch" - "skills/assistant-workflow/references/task-journal-template.md|Builder/Tester result" + "skills/assistant-workflow/references/task-journal-template.md|Builder/Tester dispatch/result/direct evidence" "skills/assistant-workflow/references/task-journal-template.md|Direct fallback reason" "skills/assistant-workflow/references/phases.md|Resolve \`subagent_policy_state\`, \`subagent_execution_mode\`, and \`subagent_authorization_scope\` before spawning any subagent" "skills/assistant-workflow/references/phases.md|add Code Mapper to \`Required agents\`" @@ -2064,8 +2064,8 @@ for file_and_term in \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/harness-controller.md::Round 10 remains terminal" \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md::Code Writer unexpected blockers" \ "$FRAMEWORK_DIR/skills/assistant-workflow/references/phases.md::STAGNATION, repeated DRIFT, repeated REGRESSION, or rubric action PIVOT" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::## Pivot/Restart Log" \ - "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md::pivot_restart_decision"; do + "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-harness-appendix.md::## Pivot/Restart Log" \ + "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-harness-appendix.md::pivot_restart_decision"; do file="${file_and_term%%::*}" term="${file_and_term#*::}" if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then diff --git a/tests/p0-p4/worker-status-contracts.sh b/tests/p0-p4/worker-status-contracts.sh index f9b2b4d..567dfc6 100644 --- a/tests/p0-p4/worker-status-contracts.sh +++ b/tests/p0-p4/worker-status-contracts.sh @@ -44,7 +44,9 @@ test_start "workflow handoffs define typed artifact references for worker packet workflow_dir="$FRAMEWORK_DIR/skills/assistant-workflow" output_contract="$workflow_dir/contracts/output.yaml" plan_template="$workflow_dir/references/plan-template.md" +plan_harness_appendix="$workflow_dir/references/plan-harness-appendix.md" task_journal_template="$workflow_dir/references/task-journal-template.md" +task_journal_harness_appendix="$workflow_dir/references/task-journal-harness-appendix.md" phases_ref="$workflow_dir/references/phases.md" harness_ref="$workflow_dir/references/harness-controller.md" missing_typed_artifact_terms=() @@ -81,24 +83,39 @@ for term in \ missing_typed_artifact_terms+=("output.yaml: $term") fi done +for term in \ + "## Harness Appendix Routing" \ + "references/plan-harness-appendix.md" \ + "artifact_reference_ledger_ref"; do + if ! grep -Fq -- "$term" "$plan_template"; then + missing_typed_artifact_terms+=("plan-template.md: $term") + fi +done for term in \ "## Artifact Reference Ledger" \ "Typed artifact refs:" \ "Artifact ID | Artifact Type | Producer | Consumer | Location Ref | Schema or Contract | Validation Status | Summary"; do - if ! grep -Fq -- "$term" "$plan_template"; then - missing_typed_artifact_terms+=("plan-template.md: $term") + if [[ ! -f "$plan_harness_appendix" ]] || ! grep -Fq -- "$term" "$plan_harness_appendix"; then + missing_typed_artifact_terms+=("plan-harness-appendix.md: $term") + fi +done +for term in \ + "## Harness Appendix Routing" \ + "references/task-journal-harness-appendix.md"; do + if ! grep -Fq -- "$term" "$task_journal_template"; then + missing_typed_artifact_terms+=("task-journal-template.md: $term") fi done for term in \ "## Artifact Reference Ledger" \ "Producer roles update Artifact Reference Ledger entries" \ 'Consumer roles validate `schema_or_contract` and update `validation_status`'; do - if ! grep -Fq -- "$term" "$task_journal_template"; then - missing_typed_artifact_terms+=("task-journal-template.md: $term") + if [[ ! -f "$task_journal_harness_appendix" ]] || ! grep -Fq -- "$term" "$task_journal_harness_appendix" "$task_journal_template"; then + missing_typed_artifact_terms+=("task-journal-harness-appendix.md/task-journal-template.md: $term") fi done for term in \ - "Artifact Reference Ledger before task packets" \ + "Artifact Reference Ledger refs before task packets" \ 'typed `artifact_refs`' \ "changed_files, verification_evidence, pivot_restart_decision, and plan_deviation refs"; do if ! grep -Fq -- "$term" "$phases_ref"; then diff --git a/tests/test-hooks.sh b/tests/test-hooks.sh index c6b7e6e..f3925c3 100755 --- a/tests/test-hooks.sh +++ b/tests/test-hooks.sh @@ -192,6 +192,32 @@ $qa_evidence TASK } +append_required_qa_review_complete_log() { + local task_file="$1" + cat >> "$task_file" <<'TASK' +## Review Log +### Spec Review #1 +- Result: PASS +- Scope reviewed: approved plan and changed files +- Missing acceptance criteria: none +- Extra scope: none +- Changed files mismatch: none +- Verification evidence mismatch: none +- Required fixes: none +### Quality Review #1 +- Round: 1 of 10 +- Found this round: 0 must-fix, 0 should-fix, 0 nits +- Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 +- Weighted: 4.00 +### Final result +- Result: CLEAN +- Score progression: 4.00 +### QA Evaluation #1 +- Final verdict: accepted +- QA result: accepted +TASK +} + make_codex_version_stub() { local version="$1" local stub_dir @@ -609,6 +635,45 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi +if test_start "workflow-phase-gates: incomplete review gate suppresses missing QA Evaluator evidence"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: medium +Plan approval: yes +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Mapper +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Mapper dispatch: multi_agent id=cm-1 +- Code Mapper result: context map returned +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: PASS clean +## Review Log +### Spec Review #1 +- Result: PASS +- Scope reviewed: approved plan and changed files +- Missing acceptance criteria: none +- Extra scope: none +- Changed files mismatch: none +- Verification evidence mismatch: none +- Required fixes: none +TASK + review_reason=$(review_gate_reason "$TEST_PROJECT/.claude/task.md") + subagent_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$review_reason" == "no_quality_review" && "$subagent_reason" == "complete" ]]; then + pass + else + fail "expected review no_quality_review and complete subagent gate, got review='$review_reason' subagent='$subagent_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + if test_start "workflow-phase-gates: pending QA Evaluator dispatch blocks delegated_missing_qa_evaluator"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' @@ -627,6 +692,7 @@ qa_evaluation_mode: required - QA Evaluator dispatch: waiting for QA evaluator - QA Evaluator result: PASS final_verdict accepted with score_progression 4.00 TASK + append_required_qa_review_complete_log "$TEST_PROJECT/.claude/task.md" helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then pass @@ -654,6 +720,7 @@ qa_evaluation_mode: required - QA Evaluator dispatch: not_required - QA Evaluator result: PASS final_verdict accepted with score_progression 4.00 TASK + append_required_qa_review_complete_log "$TEST_PROJECT/.claude/task.md" helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then pass @@ -681,6 +748,7 @@ qa_evaluation_mode: required - QA Evaluator dispatch: multi_agent id=qa-1 - QA Evaluator result: in_progress TASK + append_required_qa_review_complete_log "$TEST_PROJECT/.claude/task.md" helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then pass @@ -707,6 +775,51 @@ qa_evaluation_mode: required - Code Reviewer result: PASS clean - QA Evaluator dispatch: multi_agent id=qa-1 - QA Evaluator result: not required +TASK + append_required_qa_review_complete_log "$TEST_PROJECT/.claude/task.md" + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then + pass + else + fail "expected delegated_missing_qa_evaluator, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: QA Evaluation Mode required makes QA Evaluator required"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +## Agent Dispatch Log +- Code Reviewer dispatch: multi_agent id=cr-1 +- Code Reviewer result: PASS clean +## Review Log +### Spec Review #1 +- Result: PASS +- Scope reviewed: approved plan and changed files +- Missing acceptance criteria: none +- Extra scope: none +- Changed files mismatch: none +- Verification evidence mismatch: none +- Required fixes: none +### Quality Review #1 +- Round: 1 of 10 +- Found this round: 0 must-fix, 0 should-fix, 0 nits +- Rubric: correctness=4 quality=4 architecture=4 security=4 coverage=4 +- Weighted: 4.00 +### Final result +- Result: CLEAN +- Score progression: 4.00 +### QA Evaluation #1 +- Mode: required +- Final verdict: accepted +- QA result: accepted TASK helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then @@ -717,7 +830,135 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi -if test_start "workflow-phase-gates: Codex reviewer lifecycle with matching QA-labeled agent_id satisfies QA Evaluator"; then +if test_start "workflow-phase-gates: compact template Code Writer and Builder Tester evidence satisfies delegated roles"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: BUILDING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Writer +- Builder/Tester +## Agent Dispatch Log +- Code Writer dispatch/result/direct evidence: dispatch cw-1; result DONE changed src/App.cs +- Builder/Tester dispatch/result/direct evidence: dispatch bt-1; result DONE tests passed +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "complete" ]]; then + pass + else + fail "expected complete, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: Codex compact template role evidence matches lifecycle events"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +# Task +Status: BUILDING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Writer +- Builder/Tester +## Agent Dispatch Log +- Code Writer dispatch/result/direct evidence: dispatch cw-1; result DONE changed src/App.cs +- Builder/Tester dispatch/result/direct evidence: dispatch bt-1; result DONE tests passed +TASK + cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' +{"event":"SubagentStart","agent_type":"code-writer","agent_name":"code-writer","agent_id":"cw-1"} +{"event":"SubagentStop","agent_type":"code-writer","agent_name":"code-writer","agent_id":"cw-1"} +{"event":"SubagentStart","agent_type":"builder-tester","agent_name":"builder-tester","agent_id":"bt-1"} +{"event":"SubagentStop","agent_type":"builder-tester","agent_name":"builder-tester","agent_id":"bt-1"} +JSONL + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") + if [[ "$helper_reason" == "complete" ]]; then + pass + else + fail "expected complete, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + +if test_start "workflow-phase-gates: compact template Code Writer placeholder evidence does not satisfy role"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: BUILDING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Writer +- Builder/Tester +## Agent Dispatch Log +- Code Writer dispatch/result/direct evidence: [dispatch + result refs when delegated; role-equivalent direct evidence when direct_fallback; N/A only when role not required] +- Builder/Tester dispatch/result/direct evidence: dispatch bt-1; result DONE tests passed +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_code_writer" ]]; then + pass + else + fail "expected delegated_missing_code_writer, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: compact template Builder Tester placeholder evidence does not satisfy role"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: BUILDING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Writer +- Builder/Tester +## Agent Dispatch Log +- Code Writer dispatch/result/direct evidence: dispatch cw-1; result DONE changed src/App.cs +- Builder/Tester dispatch/result/direct evidence: [dispatch + result refs when delegated; role-equivalent direct evidence when direct_fallback; N/A only when role not required] +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "delegated_missing_builder_tester" ]]; then + pass + else + fail "expected delegated_missing_builder_tester, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: compact direct fallback N/A does not satisfy required role"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: BUILDING +Triaged as: small +Subagent policy state: subagents_unavailable +Subagent execution mode: direct_fallback +Required agents: +- Code Writer +- Builder/Tester +## Agent Dispatch Log +- Direct fallback reason: subagents_unavailable +- Code Writer dispatch/result/direct evidence: N/A: direct_fallback +- Builder/Tester direct evidence: ran focused verification directly +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "direct_fallback_missing_code_writer" ]]; then + pass + else + fail "expected direct_fallback_missing_code_writer, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: Codex reviewer lifecycle with matching QA-labeled agent_id does not satisfy QA Evaluator"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" mkdir -p "$TEST_PROJECT/.codex" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' @@ -736,11 +977,47 @@ qa_evaluation_mode: required - QA Evaluator dispatch: reviewer compatibility id=qa-reviewer-1 - QA Evaluator result: accepted final_verdict accepted score_progression 4.00 TASK + append_required_qa_review_complete_log "$TEST_PROJECT/.codex/task.md" cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' {"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} {"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} {"event":"SubagentStart","agent_type":"reviewer","agent_name":"reviewer","agent_id":"qa-reviewer-1"} {"event":"SubagentStop","agent_type":"reviewer","agent_name":"reviewer","agent_id":"qa-reviewer-1"} +JSONL + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") + if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then + pass + else + fail "expected delegated_missing_qa_evaluator, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + +if test_start "workflow-phase-gates: Codex qa-evaluator lifecycle with matching QA-labeled agent_id satisfies QA Evaluator"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Reviewer dispatch: event cr-1 +- Code Reviewer result: PASS event cr-1 +- QA Evaluator dispatch: qa-evaluator id=qa-1 +- QA Evaluator result: accepted final_verdict accepted score_progression 4.00 id=qa-1 +TASK + append_required_qa_review_complete_log "$TEST_PROJECT/.codex/task.md" + cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' +{"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} +{"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} +{"event":"SubagentStart","agent_type":"qa-evaluator","agent_name":"QAEvaluator","agent_id":"qa-1"} +{"event":"SubagentStop","agent_type":"qa-evaluator","agent_name":"QAEvaluator","agent_id":"qa-1"} JSONL helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") if [[ "$helper_reason" == "complete" ]]; then @@ -770,6 +1047,7 @@ qa_evaluation_mode: required - QA Evaluator dispatch: reviewer compatibility id=qa-reviewer-1 - QA Evaluator result: accepted final_verdict accepted score_progression 4.00 TASK + append_required_qa_review_complete_log "$TEST_PROJECT/.codex/task.md" cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' {"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} {"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} @@ -804,6 +1082,7 @@ qa_evaluation_mode: required - QA Evaluator dispatch: reviewer compatibility id=qa-reviewer-1 - QA Evaluator result: accepted final_verdict accepted score_progression 4.00 TASK + append_required_qa_review_complete_log "$TEST_PROJECT/.codex/task.md" cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' {"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} {"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} @@ -854,6 +1133,7 @@ Required agents: - Code Reviewer dispatch: code-reviewer round 1 - Code Reviewer result: PASS clean TASK + append_required_qa_review_complete_log "$TEST_PROJECT/.claude/task.md" helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then pass @@ -1488,7 +1768,9 @@ if test_start "stop-review: Claude, DOCUMENTING no review → blocks with JSON"; if [[ $HOOK_EXIT -eq 0 ]] \ && is_valid_json "$HOOK_STDOUT" \ && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ - && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "no Spec Review"; then + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "review_gate:no_spec_review" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "missing=no Spec Review" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "action="; then pass else fail "exit=$HOOK_EXIT, expected DOCUMENTING to block without review" @@ -1704,7 +1986,9 @@ TASK if [[ $HOOK_EXIT -eq 0 ]] \ && is_valid_json "$HOOK_STDOUT" \ && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ - && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "subagent evidence gate failed"; then + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "subagent_evidence_gate:" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "missing=" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "action="; then pass else fail "exit=$HOOK_EXIT, expected subagent evidence gate block" @@ -2075,6 +2359,51 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi +if test_start "stop-review: QA-required review missing Quality Review blocks review gate before QA evidence"; then + mkdir -p "$TEST_PROJECT/.claude" "$TEST_AGENT_HOME/.claude/memory/metrics" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: REVIEWING +Triaged as: medium +Plan approval: yes +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Mapper +- Code Reviewer +- QA Evaluator +qa_evaluation_mode: required +## Agent Dispatch Log +- Code Mapper dispatch: mapper-1 +- Code Mapper result: context map returned +- Code Reviewer dispatch: code-reviewer round 1 +- Code Reviewer result: PASS clean +## Review Log +### Spec Review #1 +- Result: PASS +- Scope reviewed: approved plan and changed files +- Missing acceptance criteria: none +- Extra scope: none +- Changed files mismatch: none +- Verification evidence mismatch: none +- Required fixes: none +TASK + _today=$(date +%Y-%m-%d) + echo "{\"date\":\"$_today\",\"project\":\"test\",\"task\":\"test\",\"size\":\"medium\"}" > "$TEST_AGENT_HOME/.claude/memory/metrics/workflow-metrics.jsonl" + + HOME="$TEST_AGENT_HOME" run_hook stop-review.sh claude + if [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "review_gate:no_quality_review" \ + && ! echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "delegated_missing_qa_evaluator"; then + pass + else + fail "exit=$HOOK_EXIT, expected review_gate:no_quality_review before QA evidence; stdout='$HOOK_STDOUT'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + if test_start "stop-review: direct fallback explicit reason and evidence passes subagent gate"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' @@ -2152,7 +2481,7 @@ TASK if [[ $HOOK_EXIT -eq 0 ]] \ && is_valid_json "$HOOK_STDOUT" \ && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ - && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "subagent evidence"; then + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "subagent_evidence_gate:"; then pass else fail "exit=$HOOK_EXIT, expected missing subagent evidence block, stdout='$HOOK_STDOUT'" @@ -2240,7 +2569,7 @@ TASK if [[ $HOOK_EXIT -eq 0 ]] \ && is_valid_json "$HOOK_STDOUT" \ && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ - && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "subagent evidence"; then + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "subagent_evidence_gate:"; then pass else fail "exit=$HOOK_EXIT, expected direct fallback evidence block, stdout='$HOOK_STDOUT'" @@ -3208,7 +3537,9 @@ TASK if [[ $HOOK_EXIT -eq 0 ]] \ && is_valid_json "$HOOK_STDOUT" \ && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ - && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "No plan found"; then + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "plan_gate:no_plan" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "No plan found" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "action="; then pass else fail "exit=$HOOK_EXIT, expected medium task to block without plan" @@ -3229,7 +3560,9 @@ TASK if [[ $HOOK_EXIT -eq 0 ]] \ && is_valid_json "$HOOK_STDOUT" \ && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ - && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "Plan exists but is not approved"; then + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "plan_gate:plan_not_approved" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "Plan exists but is not approved" \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "action="; then pass else fail "exit=$HOOK_EXIT, expected medium task to block without plan approval" @@ -7581,6 +7914,116 @@ EOF fi fi +if test_start "workflow-enforcer: BUILDING medium missing current subagent evidence → includes subagent warning"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'EOF' +Task: Runtime current subagent gate +Status: BUILDING +Triaged as: medium +Clarification status: ready +Clarification defaults applied: false +Unresolved clarification topics: +Plan approval: yes +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Mapper +- Code Writer +## Agent Dispatch Log +- Code Mapper dispatch: mapper-1 +- Code Mapper result: context map returned +EOF + echo '{"prompt": "continue build", "hook_event_name": "UserPromptSubmit"}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-enforcer.sh" \ + > /tmp/_wf_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wf_out) + rm -f /tmp/_wf_out + if [[ $HOOK_EXIT -eq 0 ]] \ + && echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.additionalContext' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | grep -q "Subagent evidence gate: delegated_missing_code_writer" \ + && echo "$HOOK_STDOUT" | grep -q "WARNING: Subagent evidence gate incomplete (delegated_missing_code_writer)" \ + && echo "$HOOK_STDOUT" | grep -q "missing=Code Writer dispatch/result evidence" \ + && echo "$HOOK_STDOUT" | grep -q "action="; then + pass + else + fail "exit=$HOOK_EXIT, stdout='$HOOK_STDOUT'" + fi + rm -f "$TEST_PROJECT/.claude/task.md" +fi + +if test_start "workflow-enforcer: BUILDING missing mapper and writer warns for current Code Writer"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'EOF' +Task: Runtime current subagent gate +Status: BUILDING +Triaged as: medium +Clarification status: ready +Clarification defaults applied: false +Unresolved clarification topics: +Plan approval: yes +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Mapper +- Code Writer +## Agent Dispatch Log +EOF + echo '{"prompt": "continue build", "hook_event_name": "UserPromptSubmit"}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-enforcer.sh" \ + > /tmp/_wf_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wf_out) + rm -f /tmp/_wf_out + if [[ $HOOK_EXIT -eq 0 ]] \ + && echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.additionalContext' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | grep -q "Subagent evidence gate: delegated_missing_code_mapper" \ + && echo "$HOOK_STDOUT" | grep -q "WARNING: Subagent evidence gate incomplete (delegated_missing_code_writer)" \ + && echo "$HOOK_STDOUT" | grep -q "missing=Code Writer dispatch/result evidence" \ + && echo "$HOOK_STDOUT" | grep -q "action=record Code Writer"; then + pass + else + fail "exit=$HOOK_EXIT, stdout='$HOOK_STDOUT'" + fi + rm -f "$TEST_PROJECT/.claude/task.md" +fi + +if test_start "workflow-enforcer: BUILDING medium missing future QA evidence → state only"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'EOF' +Task: Runtime future QA gate +Status: BUILDING +Triaged as: medium +Clarification status: ready +Clarification defaults applied: false +Unresolved clarification topics: +Plan approval: yes +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Mapper +- QA Evaluator +## Agent Dispatch Log +- Code Mapper dispatch: mapper-1 +- Code Mapper result: context map returned +EOF + echo '{"prompt": "continue build", "hook_event_name": "UserPromptSubmit"}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-enforcer.sh" \ + > /tmp/_wf_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wf_out) + rm -f /tmp/_wf_out + if [[ $HOOK_EXIT -eq 0 ]] \ + && echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.additionalContext' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | grep -q "Subagent evidence gate: complete" \ + && ! echo "$HOOK_STDOUT" | grep -q "WARNING: Subagent evidence gate incomplete"; then + pass + else + fail "exit=$HOOK_EXIT, stdout='$HOOK_STDOUT'" + fi + rm -f "$TEST_PROJECT/.claude/task.md" +fi + if test_start "workflow-enforcer: REVIEWING with incomplete review → includes review gate warning"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'EOF' diff --git a/tools/hooks/benchmark-hook-output.sh b/tools/hooks/benchmark-hook-output.sh index 5f2a8a5..50be1ce 100755 --- a/tools/hooks/benchmark-hook-output.sh +++ b/tools/hooks/benchmark-hook-output.sh @@ -36,6 +36,10 @@ ANCHOR_FAILURES=0 CURRENT_ROOT="" PROJECT_DIR="" AGENT_HOME="" +WORKFLOW_ENFORCER_PHASE_TRIM_BYTES="" +WORKFLOW_ENFORCER_PHASE_TRIM_WORDS="" +STOP_REVIEW_PHASE_TRIM_BYTES="" +STOP_REVIEW_PHASE_TRIM_WORDS="" markdown_escape() { local value="$1" @@ -105,6 +109,14 @@ run_hook_scenario() { stderr_bytes="$(wc -c < "$stderr_file" | tr -d ' ')" first_action="$(extract_first_action "$stdout_file" "$fallback_action")" + if [[ "$hook_name" == "workflow-enforcer" && "$scenario" == "codex building phase gates" ]]; then + WORKFLOW_ENFORCER_PHASE_TRIM_BYTES="$stdout_bytes" + WORKFLOW_ENFORCER_PHASE_TRIM_WORDS="$stdout_words" + elif [[ "$hook_name" == "stop-review" && "$scenario" == "codex missing spec review blocker" ]]; then + STOP_REVIEW_PHASE_TRIM_BYTES="$stdout_bytes" + STOP_REVIEW_PHASE_TRIM_WORDS="$stdout_words" + fi + cat "$stdout_file" "$stderr_file" > "$combined_file" if [[ -n "$extra_anchor_file" && -f "$extra_anchor_file" ]]; then cat "$extra_anchor_file" >> "$combined_file" @@ -298,6 +310,12 @@ The C slice trimmed repeated explanatory prose from `session-start`, `workflow-e | c-hook-output-trim | session-start | 2122 | 225 | 1766 | 183 | | c-hook-output-trim | workflow-enforcer | 1892 | 232 | 1625 | 184 | | c-hook-output-trim | post-compact | 2131 | 218 | 1526 | 154 | +EOF + cat < Date: Sat, 4 Jul 2026 16:53:16 +0300 Subject: [PATCH 6/9] Harden workflow guard and phase gates --- agents/claude/code-writer.md | 2 +- .../agent-guidance-clean-review-2026-07-04.md | 61 + ...ent-guidance-violation-audit-2026-07-03.md | 244 ++ docs/cleanup-audit-2026-06-25.md | 2 +- docs/hook-output-benchmarks.md | 2 +- hooks/scripts/hook-runtime.sh | 210 ++ hooks/scripts/stop-review.sh | 14 +- hooks/scripts/subagent-monitor.sh | 37 +- hooks/scripts/workflow-enforcer.sh | 106 +- hooks/scripts/workflow-guard.d/path-policy.sh | 182 ++ .../workflow-guard.d/shell-write-parser.sh | 2270 +++++++++++++++++ .../workflow-state-artifacts.sh | 70 + hooks/scripts/workflow-guard.sh | 116 +- .../learning-controller.sh | 264 ++ .../scripts/workflow-phase-gates.d/metrics.sh | 26 + .../workflow-phase-gates.d/qa-controller.sh | 323 +++ .../review-controller.sh | 150 ++ .../subagent-evidence.sh | 397 +++ .../subagent-orchestration.sh | 350 +++ hooks/scripts/workflow-phase-gates.sh | 1380 +--------- install.sh | 14 +- .../references/subagent-dispatch.md | 2 +- .../references/task-journal-template.md | 3 +- .../references/subagent-dispatch.md | 2 +- .../references/task-journal-template.md | 3 +- tests/p0-p4/review-loop-cap-contracts.sh | 29 +- tests/p0-p4/runtime-phase-gate-contracts.sh | 19 +- tests/p0-p4/task-packet-contracts.sh | 39 + tests/test-hooks.sh | 1546 ++++++++++- tools/hooks/benchmark-hook-output.sh | 11 +- 30 files changed, 6320 insertions(+), 1554 deletions(-) create mode 100644 docs/agent-guidance-clean-review-2026-07-04.md create mode 100644 docs/agent-guidance-violation-audit-2026-07-03.md create mode 100644 hooks/scripts/hook-runtime.sh create mode 100644 hooks/scripts/workflow-guard.d/path-policy.sh create mode 100644 hooks/scripts/workflow-guard.d/shell-write-parser.sh create mode 100644 hooks/scripts/workflow-guard.d/workflow-state-artifacts.sh create mode 100644 hooks/scripts/workflow-phase-gates.d/learning-controller.sh create mode 100644 hooks/scripts/workflow-phase-gates.d/metrics.sh create mode 100644 hooks/scripts/workflow-phase-gates.d/qa-controller.sh create mode 100644 hooks/scripts/workflow-phase-gates.d/review-controller.sh create mode 100644 hooks/scripts/workflow-phase-gates.d/subagent-evidence.sh create mode 100644 hooks/scripts/workflow-phase-gates.d/subagent-orchestration.sh diff --git a/agents/claude/code-writer.md b/agents/claude/code-writer.md index fa1a440..b6346c7 100644 --- a/agents/claude/code-writer.md +++ b/agents/claude/code-writer.md @@ -1,7 +1,7 @@ --- name: code-writer description: Focused code implementer that writes production code following a plan. Does not run builds or tests — that's the builder-tester's job. Does not review — that's code-reviewer's job; reviewer remains compatibility routing. Use during build phase. -tools: Read, Grep, Glob, LS, Edit, Write, Bash +tools: Read, Grep, Glob, LS, Edit, Write model: opus --- diff --git a/docs/agent-guidance-clean-review-2026-07-04.md b/docs/agent-guidance-clean-review-2026-07-04.md new file mode 100644 index 0000000..d53969f --- /dev/null +++ b/docs/agent-guidance-clean-review-2026-07-04.md @@ -0,0 +1,61 @@ +# Agent Guidance Clean Review - 2026-07-04 + +## Result + +The original pre-fix audit found 6 counted violations, all should-fix, documented in `docs/agent-guidance-violation-audit-2026-07-03.md`. + +Terminal clean-review result: remaining official-source violation count = 0. + +## Remediation Matrix + +| Violation | Major fix surfaces | +|---|---| +| V1 source-changing role inference | `workflow-phase-gates` subagent modules and tests. | +| V2 missing `jq` fail-closed | `hook-runtime` plus enforcement hooks and tests. | +| V3 bounded delegation approval | `workflow-enforcer` and tests. | +| V4 protected task-bound lifecycle evidence | `hook-runtime`, `subagent-monitor`, phase gates, task template, and tests. | +| V5 monolithic phase gates and later workflow guard split | `workflow-phase-gates.d` and `workflow-guard.d` modules, install support, and tests. | +| V6 least-privilege Claude roles and Builder/Tester guards | `agents/claude/code-writer.md`, workflow-guard modules, and tests. | + +## Review Loop + +Initial terminal Round 10 review found one remaining shell wrapper blocker (`nohup`, `timeout`, `stdbuf`, `flock`). Follow-up Slice S2 fixed those wrappers with a clean independent review. Previous fixes covered Builder/Tester shell mutator and wrapper parsing, quoted wrapper redirection false positive handling, and the `workflow-guard` module split. + +## Validation Evidence + +- `workflow-guard: builder-tester` 38/38 +- `workflow-guard` 61/61 +- S2 common executor wrapper deny probes 4/4 +- S2 preserved allow probes 4/4 +- S2 previous deny spot checks 3/3 +- Codex install/default/strict install filters passed +- `bash tests/test-hooks.sh` 309/309 +- `bash tests/test-p0-p4-contracts.sh` 227/227 +- Broad bash syntax checks passed, including `workflow-guard.d` modules +- `validate-skills` 16 skills +- Plugin sync check all mirrors checked +- `git diff --check` passed +- .NET builds succeeded; MemoryGraph NU1900 warning because NuGet vulnerability data was unreachable. + +## Official Sources Checked + +- +- +- +- +- +- +- +- +- + +## Residual Risks + +- Shell write-target parsing remains heuristic for simple proven commands, but the reviewed unsupported, wrapper, and indirect shell grammar paths now fail closed as an unproven shell write target. +- The NuGet vulnerability data warning is environmental. +- No blocking residual official-source violations remain. + +## Evidence Sources + +- `docs/agent-guidance-violation-audit-2026-07-03.md` +- `.codex/task.md` diff --git a/docs/agent-guidance-violation-audit-2026-07-03.md b/docs/agent-guidance-violation-audit-2026-07-03.md new file mode 100644 index 0000000..cc16bff --- /dev/null +++ b/docs/agent-guidance-violation-audit-2026-07-03.md @@ -0,0 +1,244 @@ +# Agent Guidance Violation Audit - 2026-07-03 + +Note: This is the original pre-fix audit snapshot. It may describe violations that were later remediated during the follow-up workflow, and should not be read as the final clean report. + +## Scope + +Audit-only review of Assistant Framework guidance and enforcement surfaces against current OpenAI and Anthropic/Claude recommendations for agents, skills, loops, approvals, guardrails, hooks, and evals. + +Reviewed local surfaces: + +- Root instructions: `AGENTS.md`, `CLAUDE.md` +- Skills: 16 root `skills/*/SKILL.md` entrypoints +- Skill contracts: 46 root `skills/*/contracts/*` files +- Agents: 8 Claude agents and 8 Codex agents +- Hooks/settings: 16 hook scripts plus agent settings JSON +- Tests/evals: P0/P4 contract suites, hook tests, skill eval fixtures, framework instruction evals +- Mirrors: generated plugin skill mirrors checked with `tools/plugins/sync-plugin-skills.sh --check` + +## Official Guidance Baseline + +Sources used: + +- OpenAI Codex AGENTS.md: +- OpenAI Codex skills: +- OpenAI Codex subagents: +- OpenAI Codex best practices: +- OpenAI agents orchestration: +- OpenAI guardrails and human approvals: +- OpenAI agent workflow evals: +- Claude Code subagents: +- Claude Code skills: +- Claude Code hooks: +- Claude Code memory: +- Anthropic Building Effective Agents: +- Claude evaluation docs: + +Key expectations distilled from those sources: + +- Instructions should be durable, concise, discoverable, and scoped to the repository or skill that needs them. +- Skills should be focused on one repeatable job, with clear descriptions, inputs, outputs, and progressive disclosure of references/scripts. +- Specialist agents should have narrow jobs, clear routing descriptions, and tool access matched to their role. +- Multi-agent orchestration should make ownership explicit: who performs work, who reviews, who gives the final answer, and when handoffs happen. +- Guardrails and approvals should be explicit, stateful, and fail safely for sensitive actions. +- Hook code runs with user permissions, so hook inputs and paths should be validated/sanitized and high-risk behavior should be tested. +- Loops should have measurable criteria, environment evidence, stopping conditions, and trace/eval coverage. + +## Count Summary + +| Severity | Count | Meaning | +|---|---:|---| +| Must-fix | 0 | No finding showed an immediate always-on breakage or confirmed data loss/security exploit in the reviewed state. | +| Should-fix | 6 | Clear mismatch with official guidance or the framework's own contracts; likely to cause workflow drift, false completion, or weak enforcement. | +| Nit | 0 | No cosmetic-only items counted as violations. | + +Total counted violations: 6. + +Theme count: + +| Theme | Count | +|---|---:| +| Runtime role/evidence enforcement | 1 | +| Guardrail dependency fail-open behavior | 1 | +| Human approval parsing | 1 | +| Lifecycle evidence integrity and path safety | 1 | +| Maintainability of enforcement loop code | 1 | +| Agent least-privilege tool surface | 1 | + +## Findings + +### V1 - Source-changing role requirements can be bypassed when `Required agents` is malformed or omitted + +Severity: should-fix + +Evidence: + +- `skills/assistant-workflow/references/phases.md:258` says source-changing Build tasks must infer at least Code Writer, Builder/Tester, and Code Reviewer. +- `skills/assistant-workflow/references/phases.md:262` says source-changing development/code-work must keep those roles in `Required agents`. +- `skills/assistant-workflow/contracts/phase-gates.yaml:58` requires source-changing development/code-work to include Code Writer, Builder/Tester, and Code Reviewer. +- `hooks/scripts/workflow-phase-gates.sh:1803` starts role inference. +- `hooks/scripts/workflow-phase-gates.sh:1831` adds Code Mapper for medium+ tasks, and `hooks/scripts/workflow-phase-gates.sh:1838` adds Code Reviewer only for Review/Document. +- `hooks/scripts/workflow-phase-gates.sh:1841` then scans the `Required agents` text. +- `hooks/scripts/workflow-phase-gates.sh:2154` returns `complete` when no roles were inferred. + +Why this violates guidance: + +OpenAI orchestration guidance emphasizes explicit ownership and handoffs for specialists. Claude subagent guidance emphasizes specialized agents with clear responsibilities. The framework's own contract also requires Code Writer, Builder/Tester, and Code Reviewer for source-changing Build work. Runtime enforcement currently trusts the journal too much; it can miss the required Build roles instead of deriving them from source-changing state. + +Impact: + +A malformed or incomplete task journal can allow Build/Review gates to pass without Code Writer, Builder/Tester, or Code Reviewer evidence, especially for small source-changing work where medium+ Code Mapper inference does not fire. + +Recommendation: + +Teach `assistant_phase_required_subagent_roles` to infer Code Writer, Builder/Tester, and Code Reviewer whenever the journal indicates source/test/docs/config/hooks/contracts/generated-artifact changes or Build/Verify status for development/code-work. Add regression tests for omitted `Required agents`, malformed `Required agents`, and small source-changing Build tasks. + +### V2 - Critical guardrail hooks fail open when `jq` is unavailable + +Severity: should-fix + +Evidence: + +- `hooks/scripts/workflow-enforcer.sh:24` exits 0 if `jq` is missing. +- `hooks/scripts/stop-review.sh:31` exits 0 if `jq` is missing. +- `hooks/scripts/workflow-guard.sh:24` exits 0 if `jq` is missing. +- `hooks/scripts/subagent-monitor.sh:14` exits 0 if `jq` is missing. + +Why this violates guidance: + +OpenAI guardrail/approval guidance treats guardrails as validation points in the workflow. Claude hook guidance says hooks should be tested carefully because they run with user permissions. A missing parser should not silently disable plan, review, delegation, or evidence gates. + +Impact: + +If `jq` is missing or not on PATH, core workflow enforcement becomes advisory or disappears entirely. The task can proceed without plan/review/delegation gates even though the hooks appear installed. + +Recommendation: + +For enforcement hooks, fail closed with a concise blocking message when `jq` is unavailable. For non-critical advisory hooks, keep fail-open behavior but label it intentionally. Add a startup/install check that verifies `jq` before enabling hooks. + +### V3 - Delegation approval detection accepts broad substrings as explicit consent + +Severity: should-fix + +Evidence: + +- `hooks/scripts/workflow-enforcer.sh:115` through `hooks/scripts/workflow-enforcer.sh:127` treats phrases such as `use subagents`, `use delegation`, `use agents`, and `delegated agents` as approval anywhere in the prompt. +- `hooks/scripts/workflow-enforcer.sh:463` through `hooks/scripts/workflow-enforcer.sh:468` converts that match into "Current prompt explicitly authorized subagents/delegation." + +Why this violates guidance: + +OpenAI human approval guidance distinguishes approved/rejected actions and pauses until the human approves. Broad substring matching can turn discussion, quoted text, or a question into authorization. That weakens the explicit-consent contract the framework is trying to enforce. + +Impact: + +A prompt such as "should I use agents here?" or "do not assume use agents is approval" can be misread as delegation approval depending on surrounding text. That creates false-positive authorization for subagent spawning. + +Recommendation: + +Use anchored approval intents rather than raw substrings. Require imperative or affirmative forms such as "yes, use subagents", "I approve subagents", or "authorize delegation for this task". Add negative and question-form tests. + +### V4 - Codex lifecycle evidence is project-local and can be forged by ordinary workspace writes + +Severity: should-fix + +Evidence: + +- `hooks/scripts/subagent-monitor.sh:37` derives `PROJECT_DIR` from environment or hook input. +- `hooks/scripts/subagent-monitor.sh:59` through `hooks/scripts/subagent-monitor.sh:71` appends lifecycle events to `$PROJECT_DIR/.codex/subagent-events.jsonl`. +- `hooks/scripts/workflow-guard.sh:124` through `hooks/scripts/workflow-guard.sh:130` only guards file-editing tools for orchestrator warnings; Bash writes are not covered by this integrity boundary. + +Why this violates guidance: + +OpenAI eval/tracing guidance relies on trustworthy traces of tool calls, handoffs, guardrails, and workflow events. Claude hook guidance says hook inputs and paths should be validated and sanitized because hooks run with full user permissions. Here, evidence used to prove subagent spawning lives in the same workspace the model can edit. + +Impact: + +Workflow gates can treat `.codex/subagent-events.jsonl` as proof of real lifecycle activity even though the file is writable from ordinary workspace operations. A compromised or careless flow could forge evidence, and path-derived writes also need stronger project-boundary validation. + +Recommendation: + +Persist lifecycle evidence in a protected state location outside project source, or sign/hash hook-written entries with data unavailable to normal model writes. At minimum, validate that `PROJECT_DIR` resolves inside the expected workspace root, reject traversal/symlink escapes, and add tests proving Bash cannot satisfy lifecycle evidence by writing the file directly. + +### V5 - `workflow-phase-gates.sh` is a monolithic 2,208-line enforcement surface + +Severity: should-fix + +Evidence: + +- `hooks/scripts/workflow-phase-gates.sh` is 2,208 lines. +- The same file contains scalar parsing, root status parsing, review-loop checks, metrics gates, learning gates, QA gates, subagent role inference, lifecycle evidence checks, and corrective message formatting. +- Role/evidence responsibilities alone span `hooks/scripts/workflow-phase-gates.sh:1453`, `hooks/scripts/workflow-phase-gates.sh:1803`, `hooks/scripts/workflow-phase-gates.sh:1878`, `hooks/scripts/workflow-phase-gates.sh:1943`, `hooks/scripts/workflow-phase-gates.sh:2003`, and `hooks/scripts/workflow-phase-gates.sh:2154`. + +Why this violates guidance: + +OpenAI and Claude guidance both favor narrow, clearly scoped components: skills for repeatable workflows, agents for narrow jobs, and explicit guardrails. The enforcement surface has grown into a dense multi-domain file, which makes it easier for runtime behavior to drift away from contracts. + +Impact: + +The file is hard to audit and test by responsibility. Findings V1 and V4 are symptoms of this concentration: small changes in one helper can affect unrelated gates. + +Recommendation: + +Split by responsibility while preserving the public helper API: parsing/status helpers, plan/build gates, review-loop gates, subagent evidence gates, QA gates, and metrics/learning gates. Add shellcheck-friendly tests around each extracted module. + +### V6 - Claude write-capable agents have broader tool access than their role constraints require + +Severity: should-fix + +Evidence: + +- `agents/claude/code-writer.md:4` grants `Bash`, while `agents/claude/code-writer.md:38` says the agent must not run builds or tests. +- `agents/claude/builder-tester.md:4` grants `Edit`, `Write`, and `Bash`, while `agents/claude/builder-tester.md:41` and `agents/claude/builder-tester.md:42` say it must not modify production code and may only create or edit test files/build configuration. +- Read-only Claude agents correctly use read-only tools, for example `agents/claude/code-reviewer.md:4` and `agents/claude/qa-evaluator.md:4`. + +Why this violates guidance: + +Claude subagent guidance explicitly supports tool allowlists/denylists and recommends limiting tools to enforce role constraints. OpenAI subagent guidance also says custom agents work best when the job and tool surface are clear. The current boundaries for Code Writer and Builder/Tester are mostly prompt-only. + +Impact: + +The Code Writer can run commands despite being told not to run builds/tests. The Builder/Tester can edit production files despite being told not to. This weakens the intended ownership split and makes violations harder to catch until review. + +Recommendation: + +Remove `Bash` from Code Writer unless a narrow implementation command use case is explicitly needed. For Builder/Tester, either split test-writing from command-running or add a PreToolUse hook keyed by agent name that blocks production-code writes and unsafe Bash commands. Keep read-only agents as they are. + +## Positive Compliance Evidence + +- `AGENTS.md` is 7,472 bytes and 129 lines, below Codex's default project-doc budget and within Claude memory guidance to keep repository instructions concise. +- `CLAUDE.md` exists, so Claude Code has a native memory file rather than relying only on `AGENTS.md`. +- Skill validation passes for all 16 skills. +- Contract checks are strong around review loops, QA evaluator separation, harness/eval docs, instruction overload, and task packets. +- The review loop has a terminal cap and contract tests. +- Code Reviewer and QA Evaluator are distinct roles and have read-only Claude tool surfaces. +- Skill eval fixtures and framework instruction eval fixtures validate. +- Generated plugin skill mirrors are in sync. + +## Observations Not Counted As Violations + +- `CLAUDE.md` duplicates much of `AGENTS.md` rather than importing it. Claude docs support importing an AGENTS.md-style file from CLAUDE.md, but duplication is not automatically a violation because this repo deliberately emits agent-specific guidance. Consider a sync check if drift becomes a recurring issue. +- `assistant-review/SKILL.md` remains a large entrypoint, but current instruction-overload contract checks pass and detailed QA algorithm content has been moved into `references/qa-evaluation-loop.md`. +- Several non-critical hooks also exit 0 when `jq` is missing (`skill-router.sh`, `learning-signals.sh`, `post-tool-context.sh`, `tool-failure-advisor.sh`). These were not counted because they are advisory/routing helpers rather than terminal workflow gates, but install-time dependency validation would still improve reliability. + +## Validation Run + +| Check | Result | +|---|---| +| `bash tools/skills/validate-skills.sh` | Passed: 16 skills validated | +| `bash tests/p0-p4/instruction-overload-contracts.sh` | Passed: 7 passed, 0 failed | +| `bash tests/p0-p4/review-loop-cap-contracts.sh` | Passed: 4 passed, 0 failed | +| `bash tests/p0-p4/qa-evaluator-contracts.sh` | Passed: 12 passed, 0 failed | +| `bash tools/evals/run-skill-evals.sh --validate-fixture` | Passed: 16 fixtures valid | +| `bash tools/evals/run-framework-instruction-evals.sh --validate-fixture` | Passed | +| `bash tests/test-hooks.sh --filter stop-review` | Passed: 67 passed, 0 failed, 202 skipped | +| `bash tools/plugins/sync-plugin-skills.sh --check` | Passed | +| `bash tests/p0-p4/harness-docs-evals-contracts.sh` | Passed: 8 passed, 0 failed | +| `bash -n hooks/scripts/*.sh` | Passed | + +Not run: full `tests/test-hooks.sh`, full `tests/test-p0-p4-contracts.sh`, .NET build/test suites, install dry-run, or install hook test mode. + +## Overall Result + +Result: `DONE_WITH_CONCERNS` + +The framework is broadly aligned with the OpenAI/Claude direction: specialist roles, concise repo instructions, contracts, review loops, QA separation, eval fixtures, and hook-based runtime enforcement are all present. The remaining violations are concentrated in enforcement fidelity: fail-open dependencies, over-broad approval parsing, evidence integrity, a missing role-inference edge, a monolithic gate file, and one Claude least-privilege mismatch. diff --git a/docs/cleanup-audit-2026-06-25.md b/docs/cleanup-audit-2026-06-25.md index 15fe4b3..9e3fbef 100644 --- a/docs/cleanup-audit-2026-06-25.md +++ b/docs/cleanup-audit-2026-06-25.md @@ -151,7 +151,7 @@ These can be removed without changing tracked project behavior. Completed items 3. Stale `.codex/` workflow state after the active task is complete. - Status: NOT CLEARED by this execution. - Current `.codex/**` remains ignored live workflow state. - - Clear `.codex/task.md`, `.codex/context-map.md`, `.codex/working-buffer.md`, `.codex/session.md`, or `.codex/subagent-events.jsonl` only after no active task needs resume context. + - Clear `.codex/task.md`, `.codex/context-map.md`, `.codex/working-buffer.md`, `.codex/session.md`, or any project-local diagnostic `.codex/subagent-events.jsonl` only after no active task needs resume context. Recommended verification after clearing ignored files: diff --git a/docs/hook-output-benchmarks.md b/docs/hook-output-benchmarks.md index e97955b..fbd7c05 100644 --- a/docs/hook-output-benchmarks.md +++ b/docs/hook-output-benchmarks.md @@ -42,4 +42,4 @@ The C slice trimmed repeated explanatory prose from `session-start`, `workflow-e - `subagent-monitor` / `codex code-writer start`: `SUBAGENT CONSTRAINT`, `SubagentStart`, `code-writer` - `subagent-monitor` / `codex code-writer stop`: `SubagentStart`, `SubagentStop`, `cw-bench-1` -The subagent stop scenario emits no user-facing stdout in the current hook behavior. The benchmark records the locally feasible lifecycle evidence by checking the project-local `.codex/subagent-events.jsonl` file written by `subagent-monitor.sh`. +The subagent stop scenario emits no user-facing stdout in the current hook behavior. The benchmark records the locally feasible lifecycle evidence by checking the protected agent-owned workflow-state event file written by `subagent-monitor.sh`; project-local `.codex/subagent-events.jsonl` files are diagnostic only. diff --git a/hooks/scripts/hook-runtime.sh b/hooks/scripts/hook-runtime.sh new file mode 100644 index 0000000..a88c636 --- /dev/null +++ b/hooks/scripts/hook-runtime.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# hook-runtime.sh -- Shared runtime helpers for Assistant Framework hooks. + +assistant_hook_fail_closed_missing_jq() { + local hook_name="$1" + local hook_event="${2:-}" + local reason + + if [[ -z "$hook_event" ]]; then + hook_event="$(assistant_hook_event_for_hook_name "$hook_name")" + fi + + reason="Assistant Framework critical hook $hook_name requires jq; install jq or disable workflow hooks intentionally. jq is required for workflow lifecycle enforcement." + if [[ "$hook_event" == "SubagentStart" ]] && ! assistant_hook_runtime_is_codex; then + reason+=" SubagentStart cannot block in Claude, so subagent lifecycle enforcement is degraded until jq is installed." + fi + + assistant_hook_emit_block "$hook_event" "$reason" +} + +assistant_hook_require_jq() { + local hook_name="$1" + local hook_event="${2:-}" + + if ! command -v jq >/dev/null 2>&1; then + assistant_hook_fail_closed_missing_jq "$hook_name" "$hook_event" + exit 0 + fi +} + +assistant_hook_json_string() { + local value="${1:-}" + + value=${value//\\/\\\\} + value=${value//\"/\\\"} + value=${value//$'\n'/\\n} + value=${value//$'\r'/\\r} + value=${value//$'\t'/\\t} + printf '"%s"' "$value" +} + +assistant_hook_runtime_is_codex() { + local codex_home="${CODEX_HOME:-$HOME/.codex}" + local caller="${BASH_SOURCE[1]:-}" + + [[ -n "${CODEX_PROJECT_DIR:-}" ]] && return 0 + [[ -n "${SCRIPT_DIR:-}" && "$SCRIPT_DIR" == "$codex_home/"* ]] && return 0 + [[ -n "$caller" && "$caller" == "$codex_home/"* ]] && return 0 + return 1 +} + +assistant_hook_event_for_hook_name() { + case "${1:-}" in + workflow-guard.sh) printf 'PreToolUse\n' ;; + workflow-enforcer.sh) printf 'UserPromptSubmit\n' ;; + stop-review.sh) + if [[ -n "${GEMINI_PROJECT_DIR:-}" ]]; then + printf 'AfterAgent\n' + else + printf 'Stop\n' + fi + ;; + subagent-monitor.sh) printf 'SubagentStart\n' ;; + pre-compress.sh) printf 'PreCompact\n' ;; + post-compact.sh) printf 'PostCompact\n' ;; + session-start.sh) printf 'SessionStart\n' ;; + *) printf '%s\n' "${2:-}" ;; + esac +} + +assistant_hook_emit_block() { + local hook_event="${1:-}" + local reason="${2:-Assistant Framework hook blocked this action.}" + local reason_json + + if assistant_hook_runtime_is_codex; then + if command -v jq >/dev/null 2>&1; then + jq -cn --arg reason "$reason" '{decision: "block", reason: $reason}' + else + reason_json="$(assistant_hook_json_string "$reason")" + printf '{"decision":"block","reason":%s}\n' "$reason_json" + fi + return + fi + + case "$hook_event" in + PreToolUse) + if command -v jq >/dev/null 2>&1; then + jq -cn --arg reason "$reason" '{ + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: $reason + } + }' + else + reason_json="$(assistant_hook_json_string "$reason")" + printf '{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"deny","permissionDecisionReason":%s}}\n' "$reason_json" + fi + ;; + AfterAgent) + if command -v jq >/dev/null 2>&1; then + jq -cn --arg reason "$reason" '{decision: "retry", reason: $reason}' + else + reason_json="$(assistant_hook_json_string "$reason")" + printf '{"decision":"retry","reason":%s}\n' "$reason_json" + fi + ;; + SubagentStart) + if command -v jq >/dev/null 2>&1; then + jq -cn --arg reason "$reason" '{ + hookSpecificOutput: { + hookEventName: "SubagentStart", + additionalContext: $reason + } + }' + else + reason_json="$(assistant_hook_json_string "$reason")" + printf '{"hookSpecificOutput":{"hookEventName":"SubagentStart","additionalContext":%s}}\n' "$reason_json" + fi + ;; + *) + if command -v jq >/dev/null 2>&1; then + jq -cn --arg reason "$reason" '{decision: "block", reason: $reason}' + else + reason_json="$(assistant_hook_json_string "$reason")" + printf '{"decision":"block","reason":%s}\n' "$reason_json" + fi + ;; + esac +} + +assistant_hook_canonical_existing_dir() { + local dir="${1:-}" + local canonical + + [[ -n "$dir" && -d "$dir" ]] || return 1 + canonical="$( + cd "$dir" >/dev/null 2>&1 && + pwd -P + )" || return 1 + [[ "$canonical" == /* ]] || return 1 + printf '%s\n' "$canonical" +} + +assistant_hook_path_hash() { + local path="$1" + printf '%s' "$path" | cksum | awk '{print $1}' +} + +assistant_hook_codex_home() { + printf '%s\n' "${CODEX_HOME:-$HOME/.codex}" +} + +assistant_hook_task_identity_from_file() { + local file="$1" + local identity + + [[ -n "$file" && -f "$file" ]] || return 1 + identity="$( + awk ' + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + function unusable(value, low) { + value = trim(value) + low = tolower(value) + return value == "" || + value ~ /^\[[^]]*\]$/ || + value ~ /\|/ || + low ~ /^(unknown|missing|pending|todo|tbd|none|null|n\/a|na|not[ _-]?(available|applicable))([[:space:][:punct:]]|$)/ + } + /^##[[:space:]]+Task([[:space:]:]|$)/ { + next + } + /^##+[[:space:]]+/ { + exit + } + $0 ~ "^(#[[:space:]]*)?Created:" { + sub("^(#[[:space:]]*)?Created:[[:space:]]*", "", $0) + value = trim($0) + if (!unusable(value)) { + print value + exit + } + } + ' "$file" 2>/dev/null + )" + + [[ -n "$identity" ]] || return 1 + printf '%s\n' "$identity" +} + +assistant_hook_codex_subagent_events_file_for_project() { + local project_dir="$1" + local task_identity="${2:-}" + local canonical hash task_hash + + canonical="$(assistant_hook_canonical_existing_dir "$project_dir")" || return 1 + hash="$(assistant_hook_path_hash "$canonical")" + if [[ -n "$task_identity" ]]; then + task_hash="$(assistant_hook_path_hash "$task_identity")" + printf '%s/cache/workflow-state/subagent-events/path-%s-task-%s.jsonl\n' "$(assistant_hook_codex_home)" "$hash" "$task_hash" + return 0 + fi + + printf '%s/cache/workflow-state/subagent-events/path-%s.jsonl\n' "$(assistant_hook_codex_home)" "$hash" +} diff --git a/hooks/scripts/stop-review.sh b/hooks/scripts/stop-review.sh index 6d3aff8..f819115 100755 --- a/hooks/scripts/stop-review.sh +++ b/hooks/scripts/stop-review.sh @@ -27,10 +27,14 @@ set -euo pipefail -# jq is required for both JSON parsing and output -command -v jq >/dev/null 2>&1 || { exit 0; } - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$SCRIPT_DIR/hook-runtime.sh" +STOP_REVIEW_HOOK_EVENT="Stop" +if [[ -n "${GEMINI_PROJECT_DIR:-}" ]]; then + STOP_REVIEW_HOOK_EVENT="AfterAgent" +fi +assistant_hook_require_jq "stop-review.sh" "$STOP_REVIEW_HOOK_EVENT" + . "$SCRIPT_DIR/task-journal-resolver.sh" . "$SCRIPT_DIR/workflow-phase-gates.sh" @@ -84,9 +88,9 @@ emit_stop_reason() { if $IS_GEMINI; then touch "${RETRY_FLAG}" - jq -n --arg reason "$reason" '{decision: "retry", reason: $reason}' + assistant_hook_emit_block "AfterAgent" "$reason" else - jq -n --arg reason "$reason" '{decision: "block", reason: $reason}' + assistant_hook_emit_block "Stop" "$reason" fi } diff --git a/hooks/scripts/subagent-monitor.sh b/hooks/scripts/subagent-monitor.sh index f286e21..1b4085a 100755 --- a/hooks/scripts/subagent-monitor.sh +++ b/hooks/scripts/subagent-monitor.sh @@ -6,14 +6,16 @@ # Codex SubagentStart / SubagentStop # # Codex evidence: -# Appends real lifecycle events to /.codex/subagent-events.jsonl so -# phase gates can distinguish actual spawned agents from task-journal text. +# Appends real lifecycle events to agent-owned workflow state outside the +# workspace so phase gates can distinguish actual spawned agents from +# task-journal text and project-local diagnostic files. set -euo pipefail -command -v jq >/dev/null 2>&1 || exit 0 - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$SCRIPT_DIR/hook-runtime.sh" +assistant_hook_require_jq "subagent-monitor.sh" "SubagentStart" + if [[ -f "$SCRIPT_DIR/task-journal-resolver.sh" ]]; then . "$SCRIPT_DIR/task-journal-resolver.sh" fi @@ -30,7 +32,7 @@ CWD_INPUT=$(printf '%s' "$INPUT" | jq -r '.cwd // ""' 2>/dev/null || true) [[ -n "$AGENT_NAME" ]] || exit 0 IS_CODEX=false -if [[ -n "${CODEX_PROJECT_DIR:-}" || "$SCRIPT_DIR" == "$HOME/.codex/"* || "$EVENT" == Subagent* ]]; then +if assistant_hook_runtime_is_codex || [[ "$EVENT" == Subagent* && -z "${CLAUDE_PROJECT_DIR:-}" && -z "${GEMINI_PROJECT_DIR:-}" ]]; then IS_CODEX=true fi @@ -53,11 +55,22 @@ esac if $IS_CODEX; then # Codex SubagentStart/SubagentStop hook input uses agent_type and agent_id. - # Persist machine-readable evidence in the project state directory. The file - # is deliberately project-local so tests and reviewers can verify actual - # lifecycle events without trusting model-written journal text. + # Persist machine-readable evidence in protected agent-owned state. A + # project-local .codex/subagent-events.jsonl file is diagnostic only and is + # never authoritative for phase-gate enforcement. if [[ "$EVENT" == "SubagentStart" || "$EVENT" == "SubagentStop" ]]; then - mkdir -p "$PROJECT_DIR/.codex" + TASK_FILE="" + TASK_IDENTITY="" + if ! PROJECT_DIR="$(assistant_hook_canonical_existing_dir "$PROJECT_DIR")"; then + assistant_hook_emit_block "$EVENT" "Subagent lifecycle evidence refused: project directory must be an existing absolute canonical directory." + exit 0 + fi + TASK_FILE="$PROJECT_DIR/.codex/task.md" + if [[ -f "$TASK_FILE" ]]; then + TASK_IDENTITY="$(assistant_hook_task_identity_from_file "$TASK_FILE" || true)" + fi + EVENTS_FILE="$(assistant_hook_codex_subagent_events_file_for_project "$PROJECT_DIR" "$TASK_IDENTITY")" + mkdir -p "$(dirname "$EVENTS_FILE")" jq -cn \ --arg event "$EVENT" \ --arg agent_type "$AGENT_NAME" \ @@ -66,9 +79,11 @@ if $IS_CODEX; then --arg turn_id "$TURN_ID" \ --arg session_id "$SESSION_ID" \ --arg transcript_path "$TRANSCRIPT_PATH" \ + --arg project_dir "$PROJECT_DIR" \ + --arg task_identity "$TASK_IDENTITY" \ --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{event:$event,agent_type:$agent_type,agent_name:$agent_name,agent_id:$agent_id,turn_id:$turn_id,session_id:$session_id,transcript_path:$transcript_path,timestamp:$timestamp}' \ - >> "$PROJECT_DIR/.codex/subagent-events.jsonl" + '{event:$event,agent_type:$agent_type,agent_name:$agent_name,agent_id:$agent_id,turn_id:$turn_id,session_id:$session_id,transcript_path:$transcript_path,project_dir:$project_dir,task_identity:$task_identity,timestamp:$timestamp}' \ + >> "$EVENTS_FILE" fi if [[ "$EVENT" == "SubagentStart" && -n "$role_constraint" ]]; then diff --git a/hooks/scripts/workflow-enforcer.sh b/hooks/scripts/workflow-enforcer.sh index 8729619..ef07873 100755 --- a/hooks/scripts/workflow-enforcer.sh +++ b/hooks/scripts/workflow-enforcer.sh @@ -21,9 +21,10 @@ set -euo pipefail -command -v jq >/dev/null 2>&1 || exit 0 - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$SCRIPT_DIR/hook-runtime.sh" +assistant_hook_require_jq "workflow-enforcer.sh" "UserPromptSubmit" + # Shared resolver handles nested cwd and sub-agent cache fallback. . "$SCRIPT_DIR/task-journal-resolver.sh" . "$SCRIPT_DIR/workflow-phase-gates.sh" @@ -37,7 +38,7 @@ TASK_FILE="$(assistant_find_task_journal "$PROJECT_DIR" "$(pwd)" || true)" STATE_DIR=".claude" if [[ -n "${GEMINI_PROJECT_DIR:-}" ]]; then STATE_DIR=".gemini" -elif [[ -n "${CODEX_PROJECT_DIR:-}" || "$SCRIPT_DIR" == "$HOME/.codex/"* ]]; then +elif assistant_hook_runtime_is_codex; then STATE_DIR=".codex" fi @@ -82,49 +83,68 @@ $extra_context" emit_additional_context "$context" } -assistant_codex_prompt_subagent_decision() { - local prompt_lc - prompt_lc="$(printf '%s' "$PROMPT" | tr '[:upper:]' '[:lower:]')" +assistant_codex_prompt_has_subagent_denial() { + printf '%s\n' "$PROMPT" | awk ' + { + line = tolower($0) + if (line ~ /(^|[^[:alnum:]_-])(deny|decline)[[:space:]]+(subagents|agents|delegation)([^[:alnum:]_-]|$)/ || + line ~ /(^|[^[:alnum:]_-])(no|without)[[:space:]]+(subagents|agents|delegation)([^[:alnum:]_-]|$)/ || + line ~ /(^|[^[:alnum:]_-])(don'\''t|dont|do not)[[:space:]]+(delegate|use[[:space:]]+(subagents|agents|delegation))([^[:alnum:]_-]|$)/ || + line ~ /(^|[^[:alnum:]_-])(don'\''t|dont|do not|not)[[:space:]]+(approve|authorize)[[:space:]]+(use[[:space:]]+)?(subagents|agents|delegation)([^[:alnum:]_-]|$)/ || + line ~ /(^|[^[:alnum:]_-])direct[[:space:]_-]+fallback([^[:alnum:]_-]|$)/) { + found = 1 + } + } + END { + exit found ? 0 : 1 + } + ' +} - if [[ "$prompt_lc" == *"deny subagents"* \ - || "$prompt_lc" == *"deny delegation"* \ - || "$prompt_lc" == *"decline subagents"* \ - || "$prompt_lc" == *"decline delegation"* \ - || "$prompt_lc" == *"no subagents"* \ - || "$prompt_lc" == *"no delegation"* \ - || "$prompt_lc" == *"without subagents"* \ - || "$prompt_lc" == *"without delegation"* \ - || "$prompt_lc" == *"don't delegate"* \ - || "$prompt_lc" == *"dont delegate"* \ - || "$prompt_lc" == *"do not delegate"* \ - || "$prompt_lc" == *"don't use subagents"* \ - || "$prompt_lc" == *"dont use subagents"* \ - || "$prompt_lc" == *"do not use subagents"* \ - || "$prompt_lc" == *"don't use delegation"* \ - || "$prompt_lc" == *"dont use delegation"* \ - || "$prompt_lc" == *"do not use delegation"* \ - || "$prompt_lc" == *"no agents"* \ - || "$prompt_lc" == *"don't use agents"* \ - || "$prompt_lc" == *"dont use agents"* \ - || "$prompt_lc" == *"do not use agents"* \ - || "$prompt_lc" == *"direct fallback"* ]]; then +assistant_codex_prompt_has_bounded_subagent_approval() { + printf '%s\n' "$PROMPT" | awk ' + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + function looks_meta(line) { + return line ~ /(^|[^[:alnum:]_-])(assume|example|phrase|quote|quoted|question|should|whether)([^[:alnum:]_-]|$)/ + } + function approve_line(line, low) { + line = trim(line) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", line) + low = tolower(line) + if (low ~ /\?/ || low ~ /[`"'\'']/ || looks_meta(low)) { + return 0 + } + return low ~ /^(yes|y|yep|yeah),?[[:space:]]+(please[[:space:]]+)?(use|spawn)[[:space:]]+(subagents|agents|delegation)([[:space:].!]|$)/ || + low ~ /^(approve|approved|authorize|authorized)[[:space:]]+(use[[:space:]]+)?(subagents|agents|delegation)([[:space:].!]|$)/ || + low ~ /^i[[:space:]]+(approve|authorize)[[:space:]]+(use[[:space:]]+)?(subagents|agents|delegation)([[:space:].!]|$)/ || + low ~ /^(please[[:space:]]+)?(use|spawn)[[:space:]]+(subagents|delegation)([[:space:].!]|$)/ || + low ~ /^(please[[:space:]]+)?delegate[[:space:]]+this[[:space:]]+work[[:space:]]+in[[:space:]]+parallel([[:space:].!]|$)/ || + low ~ /^(please[[:space:]]+)?spawn[[:space:]]+two[[:space:]]+agents([[:space:].!]|$)/ || + low ~ /^(please[[:space:]]+)?use[[:space:]]+one[[:space:]]+agent[[:space:]]+per[[:space:]]+point([[:space:].!]|$)/ || + low ~ /^(please[[:space:]]+)?delegate[[:space:]]+(the[[:space:]]+)?work([[:space:].!]|$)/ + } + { + if (approve_line($0)) { + found = 1 + } + } + END { + exit found ? 0 : 1 + } + ' +} + +assistant_codex_prompt_subagent_decision() { + if assistant_codex_prompt_has_subagent_denial; then printf 'denied\n' return fi - if [[ "$prompt_lc" == *"approve subagents"* \ - || "$prompt_lc" == *"authorize subagents"* \ - || "$prompt_lc" == *"use subagents"* \ - || "$prompt_lc" == *"spawn subagents"* \ - || "$prompt_lc" == *"approve delegation"* \ - || "$prompt_lc" == *"authorize delegation"* \ - || "$prompt_lc" == *"use delegation"* \ - || "$prompt_lc" == *"delegate work"* \ - || "$prompt_lc" == *"delegate the work"* \ - || "$prompt_lc" == *"please delegate"* \ - || "$prompt_lc" == *"use agents"* \ - || "$prompt_lc" == *"spawn agents"* \ - || "$prompt_lc" == *"delegated agents"* ]]; then + if assistant_codex_prompt_has_bounded_subagent_approval; then printf 'approved\n' return fi @@ -140,7 +160,7 @@ assistant_codex_prompt_looks_like_dev_work() { assistant_block_subagent_authorization() { local reason="$1" - jq -cn --arg reason "$reason" '{decision: "block", reason: $reason}' + assistant_hook_emit_block "UserPromptSubmit" "$reason" } read_scalar_field() { @@ -190,7 +210,7 @@ if [[ -z "$TASK_FILE" ]] || assistant_task_journal_completed "$TASK_FILE"; then if [[ "$codex_subagent_decision" == "none" ]]; then emit_workflow_rules_context "CODEX SUBAGENT AUTHORIZATION (ask-once): - Ask once for the needed delegation scope and WAIT before responsibilities that require Code Mapper, Explorer, Architect, Code Writer, Builder/Tester, Code Reviewer, or QA Evaluator (legacy Reviewer labels are compatibility routing only). -- Authorization examples: 'Use delegation', 'use delegation when possible', 'delegate work', 'use agents', 'spawn agents', 'approve subagents for this task'. +- Authorization examples: 'Use delegation.', 'yes, use subagents', 'delegate this work in parallel', 'spawn two agents', 'use one agent per point'. - Denial examples: 'no delegation', 'do not delegate', 'no agents', 'do not use agents', 'deny subagents and use direct fallback'. - Do not hard block this first prompt. Ask, then wait before delegated responsibilities." exit 0 diff --git a/hooks/scripts/workflow-guard.d/path-policy.sh b/hooks/scripts/workflow-guard.d/path-policy.sh new file mode 100644 index 0000000..a4c74ec --- /dev/null +++ b/hooks/scripts/workflow-guard.d/path-policy.sh @@ -0,0 +1,182 @@ +# Path and actor policy helpers for workflow-guard.sh. + +assistant_tool_actor_name() { + echo "$INPUT" | jq -r ' + .agent_name // + .agent_type // + .subagent_name // + .subagent_type // + .actor_name // + .actor_type // + .metadata.agent_name // + .metadata.agent_type // + .tool_input.agent_name // + .tool_input.agent_type // + (if (.actor? | type) == "string" then .actor elif (.actor? | type) == "object" then .actor.name // .actor.type else empty end) // + empty + ' 2>/dev/null | head -n 1 +} + +assistant_actor_slug() { + printf '%s' "$1" | tr '[:upper:]_' '[:lower:]-' | sed 's/[[:space:]]\+/-/g' +} + +assistant_actor_is() { + local actor="$1" + local expected="$2" + actor="$(assistant_actor_slug "$actor")" + case "$expected:$actor" in + code-writer:code-writer|code-writer:codewriter) return 0 ;; + builder-tester:builder-tester|builder-tester:builder/tester|builder-tester:buildertester) return 0 ;; + *) return 1 ;; + esac +} + +assistant_tool_target_paths() { + local direct_paths patch_text + + direct_paths=$(echo "$INPUT" | jq -r '[.tool_input.file_path?, .tool_input.path?, .tool_input.filename?] | .[]? // empty' 2>/dev/null || true) + if [[ -n "$direct_paths" ]]; then + printf '%s\n' "$direct_paths" + fi + + patch_text=$(echo "$INPUT" | jq -r '[.tool_input.patch?, .tool_input.input?, .tool_input.command?] | .[]? // empty' 2>/dev/null || true) + if [[ -n "$patch_text" ]]; then + printf '%s\n' "$patch_text" | awk ' + /^\*\*\* (Add|Update|Delete) File: / { + sub(/^\*\*\* (Add|Update|Delete) File: /, "", $0) + print + } + /^\*\*\* Move to: / { + sub(/^\*\*\* Move to: /, "", $0) + print + } + ' + fi +} + +assistant_path_has_traversal() { + local candidate="${1:-}" + + candidate="${candidate#./}" + case "$candidate" in + ..|../*|*/..|*/../*) + return 0 + ;; + esac + + return 1 +} + +assistant_canonical_existing_path() { + local target="${1:-}" + local canonical + + [[ -n "$target" && -e "$target" ]] || return 1 + if command -v realpath >/dev/null 2>&1; then + canonical="$(realpath "$target" 2>/dev/null || true)" + if [[ -n "$canonical" && "$canonical" == /* ]]; then + printf '%s\n' "$canonical" + return 0 + fi + fi + + if [[ -d "$target" ]]; then + assistant_canonical_dir "$target" + return 0 + fi + + printf '%s/%s\n' "$(assistant_canonical_dir "$(dirname "$target")")" "$(basename "$target")" +} + +assistant_canonical_project_target_path() { + local candidate="${1:-}" + local target existing_dir suffix canonical_dir canonical_path + + [[ -n "$candidate" ]] || return 1 + [[ -n "${PROJECT_DIR:-}" && -d "$PROJECT_DIR" ]] || return 1 + if assistant_path_has_traversal "$candidate"; then + return 1 + fi + + if [[ "$candidate" == /* ]]; then + target="$candidate" + else + candidate="${candidate#./}" + target="$PROJECT_DIR/$candidate" + fi + + if [[ -e "$target" ]]; then + canonical_path="$(assistant_canonical_existing_path "$target")" || return 1 + else + existing_dir="$(dirname "$target")" + suffix="$(basename "$target")" + while [[ ! -d "$existing_dir" ]]; do + case "$existing_dir" in + ""|.|/) + return 1 + ;; + esac + suffix="$(basename "$existing_dir")/$suffix" + existing_dir="$(dirname "$existing_dir")" + done + + canonical_dir="$(assistant_canonical_dir "$existing_dir")" + [[ "$canonical_dir" == /* ]] || return 1 + canonical_path="$canonical_dir/$suffix" + fi + + [[ "$canonical_path" == "$PROJECT_DIR" || "$canonical_path" == "$PROJECT_DIR/"* ]] || return 1 + printf '%s\n' "$canonical_path" +} + +assistant_project_relative_path() { + local canonical_path="${1:-}" + + [[ -n "$canonical_path" ]] || return 1 + if [[ "$canonical_path" == "$PROJECT_DIR" ]]; then + printf '%s\n' "." + return 0 + fi + [[ "$canonical_path" == "$PROJECT_DIR/"* ]] || return 1 + printf '%s\n' "${canonical_path#"$PROJECT_DIR/"}" +} + +assistant_builder_tester_path_allowed() { + local candidate="${1:-}" + local canonical_path relative_path low + + [[ -n "$candidate" ]] || return 1 + canonical_path="$(assistant_canonical_project_target_path "$candidate")" || return 1 + relative_path="$(assistant_project_relative_path "$canonical_path")" || return 1 + low="$(printf '%s' "$relative_path" | tr '[:upper:]' '[:lower:]')" + + case "$low" in + test/*|tests/*|*/test/*|*/tests/*|*.test/*|*.tests/*|\ + *.csproj|*.sln|directory.build.props|directory.build.targets|\ + global.json|nuget.config|package.json|package-lock.json|pnpm-lock.yaml|yarn.lock|bun.lockb|makefile|\ + .github/workflows/*|*/.github/workflows/*) + return 0 + ;; + esac + + return 1 +} + +assistant_builder_tester_targets_disallowed_path() { + local path + while IFS= read -r path; do + [[ -n "$path" ]] || continue + if ! assistant_builder_tester_path_allowed "$path"; then + printf '%s\n' "$path" + return 0 + fi + done < <(assistant_tool_target_paths) + + return 1 +} + +assistant_bash_command_targets_lifecycle_evidence() { + local command="${1:-}" + [[ "$command" == *"subagent-events.jsonl"* || "$command" == *"workflow-state/subagent-events"* ]] +} diff --git a/hooks/scripts/workflow-guard.d/shell-write-parser.sh b/hooks/scripts/workflow-guard.d/shell-write-parser.sh new file mode 100644 index 0000000..8a51531 --- /dev/null +++ b/hooks/scripts/workflow-guard.d/shell-write-parser.sh @@ -0,0 +1,2270 @@ +# Shell write-target parser helpers for workflow-guard.sh. + +assistant_shell_clean_write_target_path() { + local candidate="${1:-}" + local quote + + candidate="$(printf '%s' "$candidate" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + [[ -n "$candidate" ]] || return 1 + + quote="${candidate:0:1}" + if [[ "$quote" == "\"" && "${candidate: -1}" == "\"" ]]; then + candidate="${candidate#\"}" + candidate="${candidate%\"}" + elif [[ "$quote" == "'" && "${candidate: -1}" == "'" ]]; then + candidate="${candidate#\'}" + candidate="${candidate%\'}" + elif [[ "$quote" == "\"" || "$quote" == "'" ]]; then + printf '%s\n' "__assistant_unproven_shell_write_target__" + return 0 + fi + + case "$candidate" in + ""|/dev/null) + return 1 + ;; + *'$'*|*'`'*|*'('*|*')'*|*'{'*|*'}'*|*'['*|*']'*|*'*'*|*'?'*|*'<'*|*'>'*) + printf '%s\n' "__assistant_unproven_shell_write_target__" + return 0 + ;; + esac + + printf '%s\n' "$candidate" +} + +assistant_shell_emit_write_target_path() { + assistant_shell_clean_write_target_path "$1" || true +} + +assistant_shell_emit_unproven_write_target() { + printf '%s\n' "__assistant_unproven_shell_write_target__" +} + +assistant_shell_pop_word() { + local text="${1:-}" + local word_var="${2:-}" + local rest_var="${3:-}" + local parsed_word="" + local parsed_rest + local quote="" + local c + local i=0 + local length + + text="$(printf '%s' "$text" | sed 's/^[[:space:]]*//')" + [[ -n "$text" ]] || return 1 + + length="${#text}" + while (( i < length )); do + c="${text:$i:1}" + if [[ -z "$quote" ]]; then + case "$c" in + [[:space:]]) + break + ;; + "'") + quote="'" + ((i += 1)) + continue + ;; + '"') + quote='"' + ((i += 1)) + continue + ;; + "\\") + if (( i + 1 < length )); then + ((i += 1)) + c="${text:$i:1}" + fi + ;; + esac + parsed_word+="$c" + elif [[ "$quote" == "'" ]]; then + if [[ "$c" == "'" ]]; then + quote="" + else + parsed_word+="$c" + fi + else + if [[ "$c" == '"' ]]; then + quote="" + elif [[ "$c" == "\\" ]]; then + if (( i + 1 < length )); then + ((i += 1)) + parsed_word+="${text:$i:1}" + else + parsed_word+="$c" + fi + else + parsed_word+="$c" + fi + fi + ((i += 1)) + done + + if [[ -n "$quote" ]]; then + return 2 + fi + + parsed_rest="${text:$i}" + parsed_rest="$(printf '%s' "$parsed_rest" | sed 's/^[[:space:]]*//')" + printf -v "$word_var" '%s' "$parsed_word" + printf -v "$rest_var" '%s' "$parsed_rest" +} + +assistant_shell_word_is_dynamic() { + local word="${1:-}" + + [[ "$word" == *'$'* || "$word" == *'`'* ]] +} + +assistant_shell_word_is_assignment_prefix() { + local word="${1:-}" + + [[ "$word" =~ ^[A-Za-z_][A-Za-z0-9_]*(\+)?= ]] +} + +assistant_shell_command_word_basename() { + local word="${1:-}" + local basename + + [[ -n "$word" ]] || return 1 + basename="${word##*/}" + [[ -n "$basename" ]] || return 1 + + printf '%s\n' "$basename" + case "$word" in + */*) + case "$word" in + *'$'*|*'`'*|*'*'*|*'?'*|*'['*|*']'*|*'{'*|*'}'*|*'('*|*')'*) + return 2 + ;; + esac + ;; + esac + + return 0 +} + +assistant_shell_command_word_matches() { + local word="${1:-}" + local expected="${2:-}" + local basename basename_status + + [[ -n "$expected" ]] || return 1 + + basename_status=0 + basename="$(assistant_shell_command_word_basename "$word")" || basename_status=$? + if [[ "$basename" != "$expected" ]]; then + return 1 + fi + if (( basename_status == 2 )); then + return 2 + fi + return 0 +} + +assistant_shell_command_position_tail() { + local text="${1:-}" + local rest word after_word status + + rest="$(printf '%s' "$text" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + while true; do + status=0 + assistant_shell_pop_word "$rest" word after_word || status=$? + if (( status == 1 )); then + return 1 + fi + if (( status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + if assistant_shell_word_is_assignment_prefix "$word"; then + rest="$after_word" + continue + fi + if [[ -n "$after_word" ]]; then + printf '%s %s\n' "$word" "$after_word" + else + printf '%s\n' "$word" + fi + return 0 + done +} + +assistant_shell_has_command_substitution() { + local command="${1:-}" + local quote="" + local c next + local i=0 + local length + + length="${#command}" + while (( i < length )); do + c="${command:$i:1}" + if [[ "$quote" == "'" ]]; then + if [[ "$c" == "'" ]]; then + quote="" + fi + else + if [[ "$c" == "'" && -z "$quote" ]]; then + quote="'" + elif [[ "$c" == '"' ]]; then + if [[ -z "$quote" ]]; then + quote='"' + else + quote="" + fi + elif [[ "$c" == "\\" ]]; then + if (( i + 1 < length )); then + ((i += 1)) + fi + elif [[ "$c" == '`' ]]; then + return 0 + elif [[ "$c" == '$' ]]; then + next="${command:$((i + 1)):1}" + if [[ "$next" == "(" ]]; then + return 0 + fi + fi + fi + ((i += 1)) + done + + return 1 +} + +assistant_shell_segment_is_unproven() { + [[ "${1:-}" == "__assistant_unproven_shell_write_target__" ]] +} + +assistant_shell_is_apply_patch_heredoc_command() { + local command="${1:-}" + local word rest status + local header body delimiter_part delimiter after_delimiter + local strip_tabs=false + local remaining line compare + + status=0 + assistant_shell_pop_word "$command" word rest || status=$? + [[ "$status" -eq 0 && "$word" == "apply_patch" ]] || return 1 + [[ "$rest" == *$'\n'* ]] || return 1 + + header="${rest%%$'\n'*}" + body="${rest#*$'\n'}" + header="$(printf '%s' "$header" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + + case "$header" in + '<<<'*) + return 1 + ;; + '<<-'*) + strip_tabs=true + delimiter_part="${header#<<-}" + ;; + '<<'*) + delimiter_part="${header#<<}" + ;; + *) + return 1 + ;; + esac + + status=0 + assistant_shell_pop_word "$delimiter_part" delimiter after_delimiter || status=$? + [[ "$status" -eq 0 && -n "$delimiter" && -z "$after_delimiter" ]] || return 1 + + remaining="$body" + while [[ "$remaining" == *$'\n'* ]]; do + line="${remaining%%$'\n'*}" + remaining="${remaining#*$'\n'}" + compare="$line" + if [[ "$strip_tabs" == "true" ]]; then + while [[ "${compare:0:1}" == $'\t' ]]; do + compare="${compare:1}" + done + fi + if [[ "$compare" == "$delimiter" ]]; then + [[ "$remaining" =~ ^[[:space:]]*$ ]] + return $? + fi + done + + compare="$remaining" + if [[ "$strip_tabs" == "true" ]]; then + while [[ "${compare:0:1}" == $'\t' ]]; do + compare="${compare:1}" + done + fi + [[ "$compare" == "$delimiter" ]] +} + +assistant_shell_token_is_unsupported_syntax() { + local token="${1:-}" + local expect_command="${2:-false}" + local in_find_command="${3:-false}" + local command_name + + if [[ "$in_find_command" == "true" ]]; then + case "$token" in + -exec|-execdir|-delete|-ok|-okdir|-fprint|-fprint0|-fprintf|-fls) + return 0 + ;; + esac + fi + + if [[ "$expect_command" == "true" ]]; then + command_name="$(assistant_shell_command_word_basename "$token" || true)" + case "$command_name" in + !|coproc|function|for|while|until|case|select|if|alias|eval|source|xargs|.) + return 0 + ;; + command|exec|builtin|time|nice|nohup|timeout|stdbuf|flock) + return 0 + ;; + esac + fi + + return 1 +} + +assistant_shell_has_unsupported_syntax() { + local command="${1:-}" + local quote="" + local token="" + local expect_command=true + local in_find_command=false + local token_is_find=false + local c next + local i=0 + local length + + length="${#command}" + while (( i < length )); do + c="${command:$i:1}" + if [[ -z "$quote" ]]; then + case "$c" in + "'") + quote="'" + ;; + '"') + quote='"' + ;; + "\\") + if (( i + 1 < length )); then + ((i += 1)) + token+="${command:$i:1}" + fi + ;; + '$') + next="${command:$((i + 1)):1}" + if [[ "$next" == "(" ]]; then + return 0 + fi + token+="$c" + ;; + '`') + return 0 + ;; + $'\n') + if [[ -n "$token" ]]; then + if assistant_shell_token_is_unsupported_syntax "$token" "$expect_command" "$in_find_command"; then + return 0 + fi + token_is_find=false + if assistant_shell_command_word_matches "$token" "find"; then + token_is_find=true + fi + if [[ "$token_is_find" == "true" && "$expect_command" == "true" ]]; then + in_find_command=true + fi + token="" + fi + expect_command=true + in_find_command=false + ;; + [[:space:]]) + if [[ -n "$token" ]]; then + if assistant_shell_token_is_unsupported_syntax "$token" "$expect_command" "$in_find_command"; then + return 0 + fi + token_is_find=false + if assistant_shell_command_word_matches "$token" "find"; then + token_is_find=true + fi + if [[ "$token_is_find" == "true" && "$expect_command" == "true" ]]; then + in_find_command=true + fi + if [[ "$expect_command" != "true" ]] || ! assistant_shell_word_is_assignment_prefix "$token"; then + expect_command=false + fi + token="" + fi + ;; + ';'|'|') + if [[ -n "$token" ]]; then + if assistant_shell_token_is_unsupported_syntax "$token" "$expect_command" "$in_find_command"; then + return 0 + fi + token_is_find=false + if assistant_shell_command_word_matches "$token" "find"; then + token_is_find=true + fi + if [[ "$token_is_find" == "true" && "$expect_command" == "true" ]]; then + in_find_command=true + fi + token="" + fi + expect_command=true + in_find_command=false + next="${command:$((i + 1)):1}" + if [[ "$c" == "|" && "$next" == "|" ]]; then + ((i += 1)) + fi + ;; + '&') + next="${command:$((i + 1)):1}" + if [[ "$next" == ">" ]]; then + if [[ -n "$token" ]]; then + if assistant_shell_token_is_unsupported_syntax "$token" "$expect_command" "$in_find_command"; then + return 0 + fi + token_is_find=false + if assistant_shell_command_word_matches "$token" "find"; then + token_is_find=true + fi + if [[ "$token_is_find" == "true" && "$expect_command" == "true" ]]; then + in_find_command=true + fi + expect_command=false + token="" + fi + else + if [[ -n "$token" ]]; then + if assistant_shell_token_is_unsupported_syntax "$token" "$expect_command" "$in_find_command"; then + return 0 + fi + token="" + fi + expect_command=true + in_find_command=false + if [[ "$next" == "&" ]]; then + ((i += 1)) + fi + fi + ;; + '<') + next="${command:$((i + 1)):1}" + if [[ "$next" == "(" || "$next" == "<" || "$next" == ">" ]]; then + return 0 + fi + if [[ -n "$token" ]]; then + if assistant_shell_token_is_unsupported_syntax "$token" "$expect_command" "$in_find_command"; then + return 0 + fi + token_is_find=false + if assistant_shell_command_word_matches "$token" "find"; then + token_is_find=true + fi + if [[ "$token_is_find" == "true" && "$expect_command" == "true" ]]; then + in_find_command=true + fi + expect_command=false + token="" + fi + ;; + '>') + next="${command:$((i + 1)):1}" + if [[ "$next" == "(" ]]; then + return 0 + fi + if [[ -n "$token" ]]; then + if assistant_shell_token_is_unsupported_syntax "$token" "$expect_command" "$in_find_command"; then + return 0 + fi + token_is_find=false + if assistant_shell_command_word_matches "$token" "find"; then + token_is_find=true + fi + if [[ "$token_is_find" == "true" && "$expect_command" == "true" ]]; then + in_find_command=true + fi + expect_command=false + token="" + fi + ;; + '('|')'|'{'|'}') + return 0 + ;; + *) + token+="$c" + ;; + esac + elif [[ "$quote" == "'" ]]; then + if [[ "$c" == "'" ]]; then + quote="" + fi + else + if [[ "$c" == '"' ]]; then + quote="" + elif [[ "$c" == "\\" ]] && (( i + 1 < length )); then + ((i += 1)) + elif [[ "$c" == '$' ]]; then + next="${command:$((i + 1)):1}" + if [[ "$next" == "(" ]]; then + return 0 + fi + elif [[ "$c" == '`' ]]; then + return 0 + fi + fi + ((i += 1)) + done + + if [[ -n "$quote" ]]; then + return 0 + fi + + if [[ -n "$token" ]]; then + if assistant_shell_token_is_unsupported_syntax "$token" "$expect_command" "$in_find_command"; then + return 0 + fi + fi + + return 1 +} + +assistant_shell_emit_unproven_if_unsupported_syntax() { + local command="${1:-}" + + if assistant_shell_has_unsupported_syntax "$command"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + + return 1 +} + +assistant_shell_emit_segment_if_command() { + local segment="${1:-}" + local command_name="${2:-}" + local trimmed command_tail word rest status matches_status + + trimmed="$(printf '%s' "$segment" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + [[ -n "$trimmed" ]] || return 0 + + command_tail="$(assistant_shell_command_position_tail "$trimmed" || true)" + [[ -n "$command_tail" ]] || return 0 + if assistant_shell_segment_is_unproven "$command_tail"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + + status=0 + assistant_shell_pop_word "$command_tail" word rest || status=$? + if (( status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + if (( status == 0 )); then + matches_status=0 + assistant_shell_command_word_matches "$word" "$command_name" || matches_status=$? + if (( matches_status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + if (( matches_status == 0 )); then + printf '%s\n' "$command_tail" + fi + fi +} + +assistant_shell_python_command_word_matches() { + local word="${1:-}" + local basename basename_status + + basename_status=0 + basename="$(assistant_shell_command_word_basename "$word")" || basename_status=$? + case "$basename" in + python|python[0-9]*) + if (( basename_status == 2 )); then + return 2 + fi + return 0 + ;; + esac + + return 1 +} + +assistant_shell_node_command_is_present() { + local command="${1:-}" + local segment command_tail word rest status matches_status + + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + command_tail="$(assistant_shell_command_position_tail "$segment" || true)" + [[ -n "$command_tail" ]] || continue + if assistant_shell_segment_is_unproven "$command_tail"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + status=0 + assistant_shell_pop_word "$command_tail" word rest || status=$? + if (( status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + if (( status == 0 )); then + matches_status=0 + assistant_shell_command_word_matches "$word" "node" || matches_status=$? + if (( matches_status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + if (( matches_status == 0 )); then + return 0 + fi + fi + done < <(assistant_shell_command_segments "$command" "node") + + return 1 +} + +assistant_shell_python_command_is_present() { + local command="${1:-}" + local segment + + while IFS= read -r segment; do + [[ -n "$segment" ]] || continue + return 0 + done < <(assistant_shell_python_command_segments "$command") + + return 1 +} + +assistant_shell_emit_python_segment_if_command() { + local segment="${1:-}" + local trimmed command_tail word rest status matches_status + + trimmed="$(printf '%s' "$segment" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + [[ -n "$trimmed" ]] || return 0 + + command_tail="$(assistant_shell_command_position_tail "$trimmed" || true)" + [[ -n "$command_tail" ]] || return 0 + if assistant_shell_segment_is_unproven "$command_tail"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + + status=0 + assistant_shell_pop_word "$command_tail" word rest || status=$? + if (( status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + if (( status == 0 )); then + matches_status=0 + assistant_shell_python_command_word_matches "$word" || matches_status=$? + if (( matches_status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + if (( matches_status == 0 )); then + printf '%s\n' "$command_tail" + fi + fi +} + +assistant_shell_command_segments() { + local command="${1:-}" + local command_name="${2:-}" + local segment="" + local quote="" + local c next + local i=0 + local length + + [[ -n "$command_name" ]] || return 0 + command="${command//$'\n'/;}" + length="${#command}" + + while (( i < length )); do + c="${command:$i:1}" + if [[ -z "$quote" ]]; then + case "$c" in + "'") + quote="'" + segment+="$c" + ;; + '"') + quote='"' + segment+="$c" + ;; + "\\") + segment+="$c" + if (( i + 1 < length )); then + ((i += 1)) + segment+="${command:$i:1}" + fi + ;; + ';'|'|'|'&') + assistant_shell_emit_segment_if_command "$segment" "$command_name" + segment="" + next="${command:$((i + 1)):1}" + if [[ ( "$c" == "|" || "$c" == "&" ) && "$next" == "$c" ]]; then + ((i += 1)) + fi + ;; + *) + segment+="$c" + ;; + esac + elif [[ "$quote" == "'" ]]; then + segment+="$c" + if [[ "$c" == "'" ]]; then + quote="" + fi + else + segment+="$c" + if [[ "$c" == '"' ]]; then + quote="" + elif [[ "$c" == "\\" ]] && (( i + 1 < length )); then + ((i += 1)) + segment+="${command:$i:1}" + fi + fi + ((i += 1)) + done + + if [[ -n "$quote" ]]; then + assistant_shell_emit_unproven_write_target + return 0 + fi + + assistant_shell_emit_segment_if_command "$segment" "$command_name" +} + +assistant_shell_tokens_from_segment() { + local segment="${1:-}" + local rest word status + + rest="$segment" + while true; do + status=0 + assistant_shell_pop_word "$rest" word rest || status=$? + if (( status == 1 )); then + return 0 + fi + if (( status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + printf '%s\n' "$word" + done +} + +assistant_shell_sed_target_paths_from_segment() { + local segment="${1:-}" + local tokens=() + local token next_token + local target_operands=() + local i count + local inplace=false + local script_seen=false + local saw_target=false + + while IFS= read -r token; do + if assistant_shell_segment_is_unproven "$token"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + tokens+=("$token") + done < <(assistant_shell_tokens_from_segment "$segment") + + count="${#tokens[@]}" + (( count > 1 )) || return 0 + + i=1 + while (( i < count )); do + token="${tokens[$i]}" + case "$token" in + -i|--in-place) + inplace=true + next_token="${tokens[$((i + 1))]:-}" + if [[ -z "$next_token" || "$next_token" == "''" || "$next_token" == '""' ]]; then + ((i += 2)) + else + ((i += 1)) + fi + continue + ;; + -i?*|--in-place=*) + inplace=true + ((i += 1)) + continue + ;; + -e|--expression|-f|--file) + script_seen=true + ((i += 2)) + continue + ;; + --expression=*|--file=*) + script_seen=true + ((i += 1)) + continue + ;; + -*) + ((i += 1)) + continue + ;; + esac + + if [[ "$script_seen" != "true" ]]; then + script_seen=true + else + target_operands+=("$token") + saw_target=true + fi + ((i += 1)) + done + + if [[ "$inplace" == "true" ]]; then + if [[ "$saw_target" == "true" ]]; then + for token in "${target_operands[@]}"; do + assistant_shell_emit_write_target_path "$token" + done + else + assistant_shell_emit_unproven_write_target + fi + fi +} + +assistant_shell_sed_target_paths() { + local command="${1:-}" + local segment + + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + continue + fi + assistant_shell_sed_target_paths_from_segment "$segment" + done < <(assistant_shell_command_segments "$command" "sed") +} + +assistant_shell_perl_target_paths_from_segment() { + local segment="${1:-}" + local tokens=() + local token + local target_operands=() + local i count + local inplace=false + local saw_target=false + + while IFS= read -r token; do + if assistant_shell_segment_is_unproven "$token"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + tokens+=("$token") + done < <(assistant_shell_tokens_from_segment "$segment") + + count="${#tokens[@]}" + (( count > 1 )) || return 0 + + i=1 + while (( i < count )); do + token="${tokens[$i]}" + case "$token" in + --) + ((i += 1)) + break + ;; + -e|-E) + ((i += 2)) + continue + ;; + -i|-i?*) + inplace=true + ((i += 1)) + continue + ;; + -?*i*) + inplace=true + ((i += 1)) + continue + ;; + -*) + ((i += 1)) + continue + ;; + esac + + target_operands+=("$token") + saw_target=true + ((i += 1)) + done + + while (( i < count )); do + target_operands+=("${tokens[$i]}") + saw_target=true + ((i += 1)) + done + + if [[ "$inplace" == "true" ]]; then + if [[ "$saw_target" == "true" ]]; then + for token in "${target_operands[@]}"; do + assistant_shell_emit_write_target_path "$token" + done + else + assistant_shell_emit_unproven_write_target + fi + fi +} + +assistant_shell_perl_target_paths() { + local command="${1:-}" + local segment + + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + continue + fi + assistant_shell_perl_target_paths_from_segment "$segment" + done < <(assistant_shell_command_segments "$command" "perl") +} + +assistant_shell_last_operand_target_paths_from_segment() { + local segment="${1:-}" + local command_name="${2:-}" + local tokens=() + local token last_operand option_arg + local i count + local saw_operand=false + local install_directory_mode=false + local target_directory_mode=false + + while IFS= read -r token; do + if assistant_shell_segment_is_unproven "$token"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + tokens+=("$token") + done < <(assistant_shell_tokens_from_segment "$segment") + + count="${#tokens[@]}" + if (( count <= 1 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + + i=1 + while (( i < count )); do + token="${tokens[$i]}" + if [[ -n "${option_arg:-}" ]]; then + if [[ "$option_arg" == "target-directory" ]]; then + assistant_shell_emit_write_target_path "$token" + saw_operand=true + target_directory_mode=true + fi + option_arg="" + ((i += 1)) + continue + fi + + case "$token" in + --) + ((i += 1)) + break + ;; + -t|--target-directory) + option_arg="target-directory" + ((i += 1)) + continue + ;; + --target-directory=*) + assistant_shell_emit_write_target_path "${token#--target-directory=}" + saw_operand=true + target_directory_mode=true + ((i += 1)) + continue + ;; + esac + + if [[ "$command_name" == "install" ]]; then + case "$token" in + -d|--directory) + install_directory_mode=true + ((i += 1)) + continue + ;; + -m|-o|-g|-S|--mode|--owner|--group|--suffix) + option_arg="skip" + ((i += 1)) + continue + ;; + --mode=*|--owner=*|--group=*|--suffix=*) + ((i += 1)) + continue + ;; + esac + fi + + case "$token" in + -*) + ((i += 1)) + continue + ;; + esac + + last_operand="$token" + saw_operand=true + if [[ "$command_name" == "mv" ]]; then + assistant_shell_emit_write_target_path "$token" + fi + if [[ "$install_directory_mode" == "true" && "$target_directory_mode" != "true" ]]; then + assistant_shell_emit_write_target_path "$token" + fi + ((i += 1)) + done + + while (( i < count )); do + last_operand="${tokens[$i]}" + saw_operand=true + if [[ "$command_name" == "mv" ]]; then + assistant_shell_emit_write_target_path "${tokens[$i]}" + fi + if [[ "$install_directory_mode" == "true" && "$target_directory_mode" != "true" ]]; then + assistant_shell_emit_write_target_path "${tokens[$i]}" + fi + ((i += 1)) + done + + if [[ "$command_name" == "mv" ]]; then + [[ "$saw_operand" == "true" ]] || assistant_shell_emit_unproven_write_target + return 0 + fi + + if [[ "$target_directory_mode" == "true" || "$install_directory_mode" == "true" ]]; then + [[ "$saw_operand" == "true" ]] || assistant_shell_emit_unproven_write_target + return 0 + fi + + if [[ -n "${last_operand:-}" ]]; then + assistant_shell_emit_write_target_path "$last_operand" + else + assistant_shell_emit_unproven_write_target + fi +} + +assistant_shell_last_operand_target_paths() { + local command="${1:-}" + local command_name + local segment + + for command_name in cp mv install; do + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + continue + fi + assistant_shell_last_operand_target_paths_from_segment "$segment" "$command_name" + done < <(assistant_shell_command_segments "$command" "$command_name") + done +} + +assistant_shell_operand_target_paths_from_segment() { + local segment="${1:-}" + local command_name="${2:-}" + local tokens=() + local token option_arg + local i count + local saw_target=false + + while IFS= read -r token; do + if assistant_shell_segment_is_unproven "$token"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + tokens+=("$token") + done < <(assistant_shell_tokens_from_segment "$segment") + + count="${#tokens[@]}" + if (( count <= 1 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + + i=1 + while (( i < count )); do + token="${tokens[$i]}" + if [[ -n "${option_arg:-}" ]]; then + option_arg="" + ((i += 1)) + continue + fi + + case "$token" in + --) + ((i += 1)) + break + ;; + esac + + if [[ "$command_name" == "touch" ]]; then + case "$token" in + -d|-r|-t|--date|--reference|--time) + option_arg="skip" + ((i += 1)) + continue + ;; + --date=*|--reference=*|--time=*) + ((i += 1)) + continue + ;; + esac + fi + + if [[ "$command_name" == "truncate" ]]; then + case "$token" in + -s|-r|--size|--reference) + option_arg="skip" + ((i += 1)) + continue + ;; + --size=*|--reference=*) + ((i += 1)) + continue + ;; + esac + fi + + case "$token" in + -*) + ((i += 1)) + continue + ;; + esac + + assistant_shell_emit_write_target_path "$token" + saw_target=true + ((i += 1)) + done + + while (( i < count )); do + assistant_shell_emit_write_target_path "${tokens[$i]}" + saw_target=true + ((i += 1)) + done + + if [[ "$saw_target" != "true" ]]; then + assistant_shell_emit_unproven_write_target + fi +} + +assistant_shell_operand_target_paths() { + local command="${1:-}" + local command_name + local segment + + for command_name in rm touch truncate; do + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + continue + fi + assistant_shell_operand_target_paths_from_segment "$segment" "$command_name" + done < <(assistant_shell_command_segments "$command" "$command_name") + done +} + +assistant_shell_dd_target_paths() { + local command="${1:-}" + local segment token + local saw_of + + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + continue + fi + saw_of=false + while IFS= read -r token; do + if assistant_shell_segment_is_unproven "$token"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + case "$token" in + of=*) + saw_of=true + if [[ -n "${token#of=}" ]]; then + assistant_shell_emit_write_target_path "${token#of=}" + else + assistant_shell_emit_unproven_write_target + fi + ;; + esac + done < <(assistant_shell_tokens_from_segment "$segment") + done < <(assistant_shell_command_segments "$command" "dd") +} + +assistant_shell_node_write_target_paths() { + local command="${1:-}" + local primitive rest after path + local saw_path=false + local single_quote="'" + + [[ "$command" == *"writeFileSync("* || "$command" == *"appendFileSync("* || "$command" == *"createWriteStream("* ]] || return 0 + assistant_shell_node_command_is_present "$command" || return 0 + + for primitive in "writeFileSync(" "appendFileSync(" "createWriteStream("; do + rest="$command" + while [[ "$rest" == *"$primitive"* ]]; do + after="${rest#*"$primitive"}" + after="$(printf '%s' "$after" | sed 's/^[[:space:]]*//')" + if [[ "$after" == \"* && "${after#\"}" == *\"* ]]; then + after="${after#\"}" + path="${after%%\"*}" + assistant_shell_emit_write_target_path "$path" + saw_path=true + elif [[ "$after" == "$single_quote"* && "${after#"$single_quote"}" == *"$single_quote"* ]]; then + after="${after#"$single_quote"}" + path="${after%%"$single_quote"*}" + assistant_shell_emit_write_target_path "$path" + saw_path=true + else + assistant_shell_emit_unproven_write_target + saw_path=true + fi + rest="$after" + done + done + + if [[ "$saw_path" != "true" ]]; then + assistant_shell_emit_unproven_write_target + fi +} + +assistant_shell_python_command_segments() { + local command="${1:-}" + local segment="" + local quote="" + local c next + local i=0 + local length + + command="${command//$'\n'/;}" + length="${#command}" + + while (( i < length )); do + c="${command:$i:1}" + if [[ -z "$quote" ]]; then + case "$c" in + "'") + quote="'" + segment+="$c" + ;; + '"') + quote='"' + segment+="$c" + ;; + "\\") + segment+="$c" + if (( i + 1 < length )); then + ((i += 1)) + segment+="${command:$i:1}" + fi + ;; + ';'|'|'|'&') + assistant_shell_emit_python_segment_if_command "$segment" + segment="" + next="${command:$((i + 1)):1}" + if [[ ( "$c" == "|" || "$c" == "&" ) && "$next" == "$c" ]]; then + ((i += 1)) + fi + ;; + *) + segment+="$c" + ;; + esac + elif [[ "$quote" == "'" ]]; then + segment+="$c" + if [[ "$c" == "'" ]]; then + quote="" + fi + else + segment+="$c" + if [[ "$c" == '"' ]]; then + quote="" + elif [[ "$c" == "\\" ]] && (( i + 1 < length )); then + ((i += 1)) + segment+="${command:$i:1}" + fi + fi + ((i += 1)) + done + + if [[ -n "$quote" ]]; then + assistant_shell_emit_unproven_write_target + return 0 + fi + + assistant_shell_emit_python_segment_if_command "$segment" +} + +assistant_shell_python_inline_code_payloads_from_segment() { + local segment="${1:-}" + local tokens=() + local token + local i count + + while IFS= read -r token; do + if assistant_shell_segment_is_unproven "$token"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + tokens+=("$token") + done < <(assistant_shell_tokens_from_segment "$segment") + + count="${#tokens[@]}" + (( count > 1 )) || return 0 + + i=1 + while (( i < count )); do + token="${tokens[$i]}" + case "$token" in + --) + return 0 + ;; + esac + + if [[ "$token" == -* ]]; then + if [[ "$token" == "-c" ]] || [[ "$token" == -[!-]* && "$token" == *c* ]]; then + if (( i + 1 < count )); then + printf '%s\n' "${tokens[$((i + 1))]}" + else + assistant_shell_emit_unproven_write_target + fi + return 0 + fi + ((i += 1)) + continue + fi + + return 0 + done +} + +assistant_shell_python_code_has_unqualified_call() { + local code="${1:-}" + local function_name="${2:-}" + local call_regex + + [[ -n "$function_name" ]] || return 1 + + call_regex="(^|[^[:alnum:]_.])${function_name}[[:space:]]*\\(" + [[ "$code" =~ $call_regex ]] +} + +assistant_shell_python_code_has_qualified_call() { + local code="${1:-}" + local qualifier="${2:-}" + local function_name="${3:-}" + local call_regex + + [[ -n "$qualifier" && -n "$function_name" ]] || return 1 + + call_regex="(^|[^[:alnum:]_])${qualifier}[[:space:]]*\\.[[:space:]]*${function_name}[[:space:]]*\\(" + [[ "$code" =~ $call_regex ]] +} + +assistant_shell_python_code_has_method_call() { + local code="${1:-}" + local method_name="${2:-}" + local method_regex + + [[ -n "$method_name" ]] || return 1 + + method_regex="\\.[[:space:]]*${method_name}[[:space:]]*\\(" + [[ "$code" =~ $method_regex ]] +} + +assistant_shell_python_code_has_path_constructor_mutation_call() { + local code="${1:-}" + local method_name path_regex + + for method_name in touch mkdir rmdir unlink rename replace symlink_to hardlink_to chmod; do + path_regex="(^|[^[:alnum:]_.])(Path|pathlib[[:space:]]*\\.[[:space:]]*Path)[[:space:]]*\\([^)]*\\)[[:space:]]*\\.[[:space:]]*${method_name}[[:space:]]*\\(" + if [[ "$code" =~ $path_regex ]]; then + return 0 + fi + done + + return 1 +} + +assistant_shell_python_code_has_pathlib_mutation_call() { + local code="${1:-}" + local method_name + + if assistant_shell_python_code_has_path_constructor_mutation_call "$code"; then + return 0 + fi + + for method_name in touch mkdir rmdir unlink symlink_to hardlink_to chmod; do + if assistant_shell_python_code_has_method_call "$code" "$method_name"; then + return 0 + fi + done + + case "$code" in + *"from pathlib import"*|*"import pathlib"*) + for method_name in rename replace; do + if assistant_shell_python_code_has_method_call "$code" "$method_name"; then + return 0 + fi + done + ;; + esac + + return 1 +} + +assistant_shell_python_code_has_module_alias_mutation_call() { + local code="${1:-}" + local module_name="${2:-}" + local alias_regex alias function_name + shift 2 || return 1 + + alias_regex="(^|[;[:space:]])import[[:space:]]+([^;]*[,[:space:]])?${module_name}[[:space:]]+as[[:space:]]+([A-Za-z_][A-Za-z0-9_]*)" + if [[ "$code" =~ $alias_regex ]]; then + alias="${BASH_REMATCH[3]}" + for function_name in "$@"; do + if assistant_shell_python_code_has_qualified_call "$code" "$alias" "$function_name"; then + return 0 + fi + done + fi + + return 1 +} + +assistant_shell_python_code_has_imported_module_mutation_call() { + local code="${1:-}" + local module_name="${2:-}" + local import_regex alias_regex function_name alias + shift 2 || return 1 + + import_regex="(^|[;[:space:]])from[[:space:]]+${module_name}[[:space:]]+import[[:space:]]+" + [[ "$code" =~ $import_regex ]] || return 1 + + for function_name in "$@"; do + if assistant_shell_python_code_has_unqualified_call "$code" "$function_name"; then + return 0 + fi + alias_regex="(^|[;[:space:]])from[[:space:]]+${module_name}[[:space:]]+import[[:space:]]+([^;]*[,[:space:]])?${function_name}[[:space:]]+as[[:space:]]+([A-Za-z_][A-Za-z0-9_]*)" + if [[ "$code" =~ $alias_regex ]]; then + alias="${BASH_REMATCH[3]}" + if assistant_shell_python_code_has_unqualified_call "$code" "$alias"; then + return 0 + fi + fi + done + + return 1 +} + +assistant_shell_python_code_has_module_mutation_call() { + local code="${1:-}" + local module_name="${2:-}" + local function_name + shift 2 || return 1 + + for function_name in "$@"; do + if assistant_shell_python_code_has_qualified_call "$code" "$module_name" "$function_name"; then + return 0 + fi + done + + if assistant_shell_python_code_has_module_alias_mutation_call "$code" "$module_name" "$@"; then + return 0 + fi + + assistant_shell_python_code_has_imported_module_mutation_call "$code" "$module_name" "$@" +} + +assistant_shell_python_code_has_opaque_mutation_primitive() { + local code="${1:-}" + + if assistant_shell_python_code_has_pathlib_mutation_call "$code"; then + return 0 + fi + + if assistant_shell_python_code_has_module_mutation_call "$code" "os" \ + remove unlink rmdir rename replace mkdir makedirs; then + return 0 + fi + + if assistant_shell_python_code_has_module_mutation_call "$code" "shutil" \ + copyfile copy copy2 copytree move rmtree; then + return 0 + fi + + return 1 +} + +assistant_shell_python_opaque_mutation_target_paths() { + local command="${1:-}" + local segment payload + + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + continue + fi + while IFS= read -r payload; do + if assistant_shell_segment_is_unproven "$payload"; then + assistant_shell_emit_unproven_write_target + elif assistant_shell_python_code_has_opaque_mutation_primitive "$payload"; then + assistant_shell_emit_unproven_write_target + fi + done < <(assistant_shell_python_inline_code_payloads_from_segment "$segment") + done < <(assistant_shell_python_command_segments "$command") +} + +assistant_shell_inline_eval_target_paths_from_segment() { + local segment="${1:-}" + local command_name="${2:-}" + local tokens=() + local token + local i count + local saw_inline_eval=false + local inplace=false + + while IFS= read -r token; do + if assistant_shell_segment_is_unproven "$token"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + tokens+=("$token") + done < <(assistant_shell_tokens_from_segment "$segment") + + count="${#tokens[@]}" + (( count > 1 )) || return 0 + + i=1 + while (( i < count )); do + token="${tokens[$i]}" + case "$token" in + --) + break + ;; + esac + + case "$command_name" in + ruby) + case "$token" in + -e|-e?*|-[!-]*e*) + assistant_shell_emit_unproven_write_target + return 0 + ;; + esac + ;; + php) + case "$token" in + -r|-r?*) + assistant_shell_emit_unproven_write_target + return 0 + ;; + esac + ;; + perl) + case "$token" in + -i|-i?*|-?*i*) + inplace=true + ;; + -e|-E|-e?*|-E?*|-[!-]*e*|-[!-]*E*) + saw_inline_eval=true + ((i += 2)) + continue + ;; + esac + ;; + esac + + ((i += 1)) + done + + if [[ "$command_name" == "perl" && "$saw_inline_eval" == "true" && "$inplace" != "true" ]]; then + assistant_shell_emit_unproven_write_target + fi +} + +assistant_shell_awk_inline_eval_target_paths_from_segment() { + local segment="${1:-}" + local tokens=() + local token option_arg + local i count + + while IFS= read -r token; do + if assistant_shell_segment_is_unproven "$token"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + tokens+=("$token") + done < <(assistant_shell_tokens_from_segment "$segment") + + count="${#tokens[@]}" + (( count > 1 )) || return 0 + + i=1 + while (( i < count )); do + token="${tokens[$i]}" + if [[ -n "${option_arg:-}" ]]; then + option_arg="" + ((i += 1)) + continue + fi + + case "$token" in + --) + ((i += 1)) + break + ;; + -f|--file|-F|-v|-W) + option_arg="skip" + ((i += 1)) + continue + ;; + -f?*|--file=*|-F?*|-v?*|-W?*) + ((i += 1)) + continue + ;; + -*) + ((i += 1)) + continue + ;; + esac + + assistant_shell_emit_unproven_write_target + return 0 + done +} + +assistant_shell_inline_eval_target_paths() { + local command="${1:-}" + local segment command_name + + for command_name in ruby php perl; do + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + continue + fi + assistant_shell_inline_eval_target_paths_from_segment "$segment" "$command_name" + done < <(assistant_shell_command_segments "$command" "$command_name") + done + + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + continue + fi + assistant_shell_awk_inline_eval_target_paths_from_segment "$segment" + done < <(assistant_shell_command_segments "$command" "awk") +} + +assistant_shell_emit_redirection_target_from_text() { + local text="${1:-}" + local target rest status + + text="$(printf '%s' "$text" | sed 's/^[[:space:]]*//')" + [[ -n "$text" ]] || { + assistant_shell_emit_unproven_write_target + return 0 + } + + case "$text" in + '&'[0-9]*|'&-') + return 0 + ;; + esac + + status=0 + assistant_shell_pop_word "$text" target rest || status=$? + if (( status != 0 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + + case "$target" in + '&'[0-9]*|'&-') + return 0 + ;; + esac + + assistant_shell_emit_write_target_path "$target" +} + +assistant_shell_redirection_target_paths() { + local command="${1:-}" + local quote="" + local c next rest + local i=0 + local length + local target_offset + + command="${command//$'\n'/ }" + length="${#command}" + + while (( i < length )); do + c="${command:$i:1}" + if [[ -z "$quote" ]]; then + case "$c" in + "'") + quote="'" + ;; + '"') + quote='"' + ;; + "\\") + if (( i + 1 < length )); then + ((i += 1)) + fi + ;; + '&') + next="${command:$((i + 1)):1}" + if [[ "$next" == ">" ]]; then + target_offset=$((i + 2)) + if [[ "${command:$target_offset:1}" == ">" ]]; then + target_offset=$((target_offset + 1)) + fi + rest="${command:$target_offset}" + assistant_shell_emit_redirection_target_from_text "$rest" + fi + ;; + '>') + target_offset=$((i + 1)) + next="${command:$target_offset:1}" + if [[ "$next" == ">" || "$next" == "|" ]]; then + target_offset=$((target_offset + 1)) + fi + rest="${command:$target_offset}" + assistant_shell_emit_redirection_target_from_text "$rest" + ;; + esac + elif [[ "$quote" == "'" ]]; then + if [[ "$c" == "'" ]]; then + quote="" + fi + else + if [[ "$c" == '"' ]]; then + quote="" + elif [[ "$c" == "\\" ]] && (( i + 1 < length )); then + ((i += 1)) + fi + fi + ((i += 1)) + done + + if [[ -n "$quote" ]]; then + assistant_shell_emit_unproven_write_target + fi +} + +assistant_shell_tee_target_paths_from_segment() { + local segment="${1:-}" + local tokens=() + local token option_arg + local i count + + while IFS= read -r token; do + if assistant_shell_segment_is_unproven "$token"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + tokens+=("$token") + done < <(assistant_shell_tokens_from_segment "$segment") + + count="${#tokens[@]}" + (( count > 1 )) || { + assistant_shell_emit_unproven_write_target + return 0 + } + + i=1 + while (( i < count )); do + token="${tokens[$i]}" + if [[ -n "${option_arg:-}" ]]; then + option_arg="" + ((i += 1)) + continue + fi + case "$token" in + --) + ((i += 1)) + break + ;; + -a|--append|-i|--ignore-interrupts|-p|--output-error) + ((i += 1)) + continue + ;; + --output-error=*) + ((i += 1)) + continue + ;; + -*) + ((i += 1)) + continue + ;; + esac + if [[ "$token" =~ ^[0-9]+$ ]]; then + ((i += 1)) + continue + fi + assistant_shell_emit_write_target_path "$token" + ((i += 1)) + done + + while (( i < count )); do + assistant_shell_emit_write_target_path "${tokens[$i]}" + ((i += 1)) + done +} + +assistant_shell_tee_target_paths() { + local command="${1:-}" + local segment + + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + continue + fi + assistant_shell_tee_target_paths_from_segment "$segment" + done < <(assistant_shell_command_segments "$command" "tee") +} + +assistant_shell_python_has_write_primitive() { + local command="${1:-}" + local open_positional_write_double_regex open_positional_write_single_regex + local open_method_write_double_regex open_method_write_single_regex + local mode_write_double_regex mode_write_single_regex + + assistant_shell_python_command_is_present "$command" || return 1 + [[ "$command" == *".write_text("* || "$command" == *".write_bytes("* ]] && return 0 + [[ "$command" == *"open("* && "$command" == *".write("* ]] && return 0 + + open_positional_write_double_regex='(^|[^[:alnum:]_.])open[[:space:]]*\([^)]*,[[:space:]]*"[^"]*[wax+][^"]*"' + open_positional_write_single_regex="(^|[^[:alnum:]_.])open[[:space:]]*\\([^)]*,[[:space:]]*'[^']*[wax+][^']*'" + open_method_write_double_regex='\.open[[:space:]]*\([[:space:]]*"[^"]*[wax+][^"]*"' + open_method_write_single_regex="\\.open[[:space:]]*\\([[:space:]]*'[^']*[wax+][^']*'" + mode_write_double_regex='mode[[:space:]]*=[[:space:]]*"[^"]*[wax+][^"]*"' + mode_write_single_regex="mode[[:space:]]*=[[:space:]]*'[^']*[wax+][^']*'" + + [[ "$command" =~ $open_positional_write_double_regex \ + || "$command" =~ $open_positional_write_single_regex \ + || "$command" =~ $open_method_write_double_regex \ + || "$command" =~ $open_method_write_single_regex \ + || ( "$command" == *"open("* && "$command" =~ $mode_write_double_regex ) \ + || ( "$command" == *"open("* && "$command" =~ $mode_write_single_regex ) ]] +} + +assistant_shell_python_mode_is_write_mode() { + local mode="${1:-}" + + [[ "$mode" == *w* || "$mode" == *a* || "$mode" == *x* || "$mode" == *+* ]] +} + +assistant_shell_python_open_args_have_write_mode() { + local args="${1:-}" + local call_args positional mode + local single_quote="'" + local keyword_double_regex='(^|,)[[:space:]]*mode[[:space:]]*=[[:space:]]*"([^"]*)"' + local keyword_single_regex="(^|,)[[:space:]]*mode[[:space:]]*=[[:space:]]*'([^']*)'" + + call_args="${args%%)*}" + + if [[ "$call_args" =~ $keyword_double_regex ]]; then + mode="${BASH_REMATCH[2]}" + assistant_shell_python_mode_is_write_mode "$mode" + return $? + fi + if [[ "$call_args" =~ $keyword_single_regex ]]; then + mode="${BASH_REMATCH[2]}" + assistant_shell_python_mode_is_write_mode "$mode" + return $? + fi + + positional="$(printf '%s' "$call_args" | sed 's/^[[:space:]]*//')" + if [[ "$positional" == ","* ]]; then + positional="${positional#,}" + positional="$(printf '%s' "$positional" | sed 's/^[[:space:]]*//')" + fi + + if [[ "$positional" == \"* && "${positional#\"}" == *\"* ]]; then + positional="${positional#\"}" + mode="${positional%%\"*}" + assistant_shell_python_mode_is_write_mode "$mode" + return $? + fi + if [[ "$positional" == "$single_quote"* && "${positional#"$single_quote"}" == *"$single_quote"* ]]; then + positional="${positional#"$single_quote"}" + mode="${positional%%"$single_quote"*}" + assistant_shell_python_mode_is_write_mode "$mode" + return $? + fi + + return 1 +} + +assistant_shell_python_write_target_paths() { + local command="${1:-}" + local rest before after path after_quote open_after saw_path=false + local single_quote="'" + + assistant_shell_python_has_write_primitive "$command" || return 0 + + rest="$command" + while [[ "$rest" == *"Path("* ]]; do + after="${rest#*"Path("}" + if [[ "$after" == \"* && "${after#\"}" == *\"* ]]; then + after="${after#\"}" + path="${after%%\"*}" + after_quote="${after#*\"}" + if [[ "$after_quote" == *").write_text("* || "$after_quote" == *").write_bytes("* ]]; then + assistant_shell_emit_write_target_path "$path" + saw_path=true + elif [[ "$after_quote" == *").open("* ]]; then + open_after="${after_quote#*").open("}" + if assistant_shell_python_open_args_have_write_mode "$open_after"; then + assistant_shell_emit_write_target_path "$path" + saw_path=true + elif [[ "$open_after" == *".write("* ]]; then + assistant_shell_emit_unproven_write_target + saw_path=true + fi + fi + elif [[ "$after" == "$single_quote"* && "${after#"$single_quote"}" == *"$single_quote"* ]]; then + after="${after#"$single_quote"}" + path="${after%%"$single_quote"*}" + after_quote="${after#*"$single_quote"}" + if [[ "$after_quote" == *").write_text("* || "$after_quote" == *").write_bytes("* ]]; then + assistant_shell_emit_write_target_path "$path" + saw_path=true + elif [[ "$after_quote" == *").open("* ]]; then + open_after="${after_quote#*").open("}" + if assistant_shell_python_open_args_have_write_mode "$open_after"; then + assistant_shell_emit_write_target_path "$path" + saw_path=true + elif [[ "$open_after" == *".write("* ]]; then + assistant_shell_emit_unproven_write_target + saw_path=true + fi + fi + fi + rest="$after" + done + + rest="$command" + while [[ "$rest" == *"open("* ]]; do + before="${rest%%"open("*}" + after="${rest#*"open("}" + if [[ "${before: -1}" == "." ]]; then + rest="$after" + continue + fi + if [[ "$after" == \"* && "${after#\"}" == *\"* ]]; then + after="${after#\"}" + path="${after%%\"*}" + after_quote="${after#*\"}" + if assistant_shell_python_open_args_have_write_mode "$after_quote"; then + assistant_shell_emit_write_target_path "$path" + saw_path=true + elif [[ "$after_quote" == *".write("* ]]; then + assistant_shell_emit_unproven_write_target + saw_path=true + fi + elif [[ "$after" == "$single_quote"* && "${after#"$single_quote"}" == *"$single_quote"* ]]; then + after="${after#"$single_quote"}" + path="${after%%"$single_quote"*}" + after_quote="${after#*"$single_quote"}" + if assistant_shell_python_open_args_have_write_mode "$after_quote"; then + assistant_shell_emit_write_target_path "$path" + saw_path=true + elif [[ "$after_quote" == *".write("* ]]; then + assistant_shell_emit_unproven_write_target + saw_path=true + fi + else + if assistant_shell_python_open_args_have_write_mode "$after" || [[ "$after" == *".write("* ]]; then + assistant_shell_emit_unproven_write_target + saw_path=true + fi + fi + rest="$after" + done + + if [[ "$saw_path" != "true" ]]; then + printf '%s\n' "__assistant_unproven_shell_write_target__" + fi +} + +assistant_shell_direct_write_target_paths() { + local command="${1:-}" + + assistant_shell_redirection_target_paths "$command" + assistant_shell_tee_target_paths "$command" + assistant_shell_python_opaque_mutation_target_paths "$command" + assistant_shell_python_write_target_paths "$command" + assistant_shell_sed_target_paths "$command" + assistant_shell_perl_target_paths "$command" + assistant_shell_last_operand_target_paths "$command" + assistant_shell_operand_target_paths "$command" + assistant_shell_dd_target_paths "$command" + assistant_shell_node_write_target_paths "$command" + assistant_shell_inline_eval_target_paths "$command" +} + +assistant_shell_env_unwrapped_commands() { + local command="${1:-}" + local trimmed rest env_word word after_word status command_name_status command_name + + trimmed="$(assistant_shell_command_position_tail "$command" || true)" + [[ -n "$trimmed" ]] || return 0 + if assistant_shell_segment_is_unproven "$trimmed"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + status=0 + assistant_shell_pop_word "$trimmed" env_word rest || status=$? + if (( status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + command_name_status=0 + command_name="$(assistant_shell_command_word_basename "$env_word")" || command_name_status=$? + [[ "$command_name" == "env" ]] || return 0 + if (( command_name_status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + + while true; do + status=0 + assistant_shell_pop_word "$rest" word after_word || status=$? + if (( status == 1 )); then + return 0 + elif (( status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + + case "$word" in + -i|--ignore-environment|-0|--null) + rest="$after_word" + continue + ;; + -u|--unset|-C|--chdir) + status=0 + assistant_shell_pop_word "$after_word" word rest || status=$? + if (( status != 0 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + continue + ;; + --unset=*|--chdir=*) + rest="$after_word" + continue + ;; + -*) + assistant_shell_emit_unproven_write_target + return 0 + ;; + esac + + if [[ "$word" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]]; then + rest="$after_word" + continue + fi + + if assistant_shell_token_is_unsupported_syntax "$word" "true" "false"; then + assistant_shell_emit_unproven_write_target + return 0 + fi + + command_name_status=0 + command_name="$(assistant_shell_command_word_basename "$word")" || command_name_status=$? + case "$command_name" in + bash|sh|zsh|node|python|python[0-9]*|sed|perl|cp|mv|rm|touch|install|truncate|dd|tee) + if (( command_name_status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + if [[ -n "$after_word" ]]; then + printf '%s %s\n' "$word" "$after_word" + else + printf '%s\n' "$word" + fi + ;; + *) + assistant_shell_emit_unproven_write_target + ;; + esac + return 0 + done +} + +assistant_shell_c_payloads_from_segment() { + local segment="${1:-}" + local shell_name="${2:-}" + local rest shell_word word payload status matches_status + + status=0 + assistant_shell_pop_word "$segment" shell_word rest || status=$? + if (( status != 0 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + + matches_status=0 + assistant_shell_command_word_matches "$shell_word" "$shell_name" || matches_status=$? + if (( matches_status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + (( matches_status == 0 )) || return 0 + + while true; do + status=0 + assistant_shell_pop_word "$rest" word rest || status=$? + if (( status == 1 )); then + return 0 + elif (( status == 2 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + + if [[ "$word" == "--" ]]; then + return 0 + fi + + if [[ "$word" == -* ]]; then + if [[ "$word" == "-c" ]] || [[ "$word" == -[!-]* && "$word" == *c* ]]; then + status=0 + assistant_shell_pop_word "$rest" payload rest || status=$? + if (( status != 0 )); then + assistant_shell_emit_unproven_write_target + elif assistant_shell_word_is_dynamic "$payload"; then + assistant_shell_emit_unproven_write_target + else + printf '%s\n' "$payload" + fi + return 0 + fi + + case "$word" in + -o|-O|--init-file|--rcfile) + status=0 + assistant_shell_pop_word "$rest" word rest || status=$? + if (( status != 0 )); then + assistant_shell_emit_unproven_write_target + return 0 + fi + ;; + esac + continue + fi + + return 0 + done +} + +assistant_shell_c_payload_commands() { + local command="${1:-}" + local shell_name segment + + for shell_name in bash sh zsh; do + while IFS= read -r segment; do + if assistant_shell_segment_is_unproven "$segment"; then + assistant_shell_emit_unproven_write_target + continue + fi + assistant_shell_c_payloads_from_segment "$segment" "$shell_name" + done < <(assistant_shell_command_segments "$command" "$shell_name") + done +} + +assistant_shell_indirect_command_payloads() { + local command="${1:-}" + local unwrapped + + if assistant_shell_has_command_substitution "$command"; then + assistant_shell_emit_unproven_write_target + fi + + assistant_shell_c_payload_commands "$command" + + while IFS= read -r unwrapped; do + [[ -n "$unwrapped" ]] || continue + if [[ "$unwrapped" == "__assistant_unproven_shell_write_target__" ]]; then + assistant_shell_emit_unproven_write_target + continue + fi + printf '%s\n' "$unwrapped" + assistant_shell_c_payload_commands "$unwrapped" + done < <(assistant_shell_env_unwrapped_commands "$command") +} + +assistant_shell_write_target_paths() { + local command="${1:-}" + local payload + + if assistant_shell_is_apply_patch_heredoc_command "$command"; then + return 0 + fi + if assistant_shell_emit_unproven_if_unsupported_syntax "$command"; then + return 0 + fi + + assistant_shell_direct_write_target_paths "$command" + + while IFS= read -r payload; do + [[ -n "$payload" ]] || continue + if [[ "$payload" == "__assistant_unproven_shell_write_target__" ]]; then + assistant_shell_emit_unproven_write_target + elif assistant_shell_is_apply_patch_heredoc_command "$payload"; then + continue + elif assistant_shell_emit_unproven_if_unsupported_syntax "$payload"; then + continue + else + assistant_shell_direct_write_target_paths "$payload" + fi + done < <(assistant_shell_indirect_command_payloads "$command") +} + +assistant_builder_tester_bash_disallowed_write_target() { + local command="${1:-}" + local path + + while IFS= read -r path; do + [[ -n "$path" ]] || continue + if [[ "$path" == "__assistant_unproven_shell_write_target__" ]]; then + printf '%s\n' "unproven shell write target" + return 0 + fi + if ! assistant_builder_tester_path_allowed "$path"; then + printf '%s\n' "$path" + return 0 + fi + done < <(assistant_shell_write_target_paths "$command") + + return 1 +} diff --git a/hooks/scripts/workflow-guard.d/workflow-state-artifacts.sh b/hooks/scripts/workflow-guard.d/workflow-state-artifacts.sh new file mode 100644 index 0000000..d1b77c8 --- /dev/null +++ b/hooks/scripts/workflow-guard.d/workflow-state-artifacts.sh @@ -0,0 +1,70 @@ +# Workflow state artifact helpers for workflow-guard.sh. + +assistant_is_workflow_state_artifact_path() { + local candidate="${1:-}" + local canonical_path relative_path state_dir artifact_path configured_state_dir + + [[ -n "$candidate" ]] || return 1 + canonical_path="$(assistant_canonical_project_target_path "$candidate")" || return 1 + relative_path="$(assistant_project_relative_path "$canonical_path")" || return 1 + + state_dir="${relative_path%%/*}" + artifact_path="${relative_path#*/}" + [[ "$artifact_path" != "$relative_path" ]] || return 1 + + case "$artifact_path" in + task.md|context-map.md|session.md|working-buffer.md) + ;; + *) + return 1 + ;; + esac + + while IFS= read -r configured_state_dir; do + if [[ "$state_dir" == "$configured_state_dir" ]]; then + return 0 + fi + done < <(assistant_task_state_dirs) + + return 1 +} + +assistant_patch_targets_only_workflow_state_artifacts() { + local patch_text="${1:-}" + local path + local saw_path=false + + [[ -n "$patch_text" ]] || return 1 + + while IFS= read -r path; do + saw_path=true + if ! assistant_is_workflow_state_artifact_path "$path"; then + return 1 + fi + done < <( + printf '%s\n' "$patch_text" | awk ' + /^\*\*\* (Add|Update|Delete) File: / { + sub(/^\*\*\* (Add|Update|Delete) File: /, "", $0) + print + } + ' + ) + + [[ "$saw_path" == "true" ]] +} + +assistant_tool_targets_only_workflow_state_artifacts() { + local direct_path patch_text + + direct_path=$(echo "$INPUT" | jq -r '.tool_input.file_path // .tool_input.path // .tool_input.filename // empty' 2>/dev/null || true) + if [[ -n "$direct_path" ]] && assistant_is_workflow_state_artifact_path "$direct_path"; then + return 0 + fi + + patch_text=$(echo "$INPUT" | jq -r '.tool_input.patch // .tool_input.input // empty' 2>/dev/null || true) + if assistant_patch_targets_only_workflow_state_artifacts "$patch_text"; then + return 0 + fi + + return 1 +} diff --git a/hooks/scripts/workflow-guard.sh b/hooks/scripts/workflow-guard.sh index 9ecffae..f6661cd 100755 --- a/hooks/scripts/workflow-guard.sh +++ b/hooks/scripts/workflow-guard.sh @@ -21,85 +21,76 @@ set -euo pipefail -command -v jq >/dev/null 2>&1 || exit 0 - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$SCRIPT_DIR/hook-runtime.sh" +assistant_hook_require_jq "workflow-guard.sh" "PreToolUse" + . "$SCRIPT_DIR/task-journal-resolver.sh" . "$SCRIPT_DIR/workflow-phase-gates.sh" INPUT=$(cat) TOOL_NAME=$(echo "$INPUT" | jq -r '.tool_name // ""') IS_CODEX=false -if [[ -n "${CODEX_PROJECT_DIR:-}" || "$SCRIPT_DIR" == "$HOME/.codex/"* ]]; then +if assistant_hook_runtime_is_codex; then IS_CODEX=true fi +PROJECT_DIR="$(assistant_resolve_project_dir "$(pwd)")" +if [[ -d "$PROJECT_DIR" ]]; then + PROJECT_DIR="$(assistant_canonical_dir "$PROJECT_DIR")" +fi -assistant_is_workflow_state_artifact_path() { - local candidate="${1:-}" - - [[ -n "$candidate" ]] || return 1 - candidate="${candidate#./}" - - case "$candidate" in - .claude/task.md|.claude/context-map.md|.claude/session.md|.claude/working-buffer.md|\ - .codex/task.md|.codex/context-map.md|.codex/session.md|.codex/working-buffer.md|\ - .gemini/task.md|.gemini/context-map.md|.gemini/session.md|.gemini/working-buffer.md|\ - */.claude/task.md|*/.claude/context-map.md|*/.claude/session.md|*/.claude/working-buffer.md|\ - */.codex/task.md|*/.codex/context-map.md|*/.codex/session.md|*/.codex/working-buffer.md|\ - */.gemini/task.md|*/.gemini/context-map.md|*/.gemini/session.md|*/.gemini/working-buffer.md) - return 0 - ;; - esac - - return 1 -} - -assistant_patch_targets_only_workflow_state_artifacts() { - local patch_text="${1:-}" - local path - local saw_path=false - - [[ -n "$patch_text" ]] || return 1 - - while IFS= read -r path; do - saw_path=true - if ! assistant_is_workflow_state_artifact_path "$path"; then - return 1 - fi - done < <( - printf '%s\n' "$patch_text" | awk ' - /^\*\*\* (Add|Update|Delete) File: / { - sub(/^\*\*\* (Add|Update|Delete) File: /, "", $0) - print - } - ' - ) - - [[ "$saw_path" == "true" ]] +assistant_workflow_guard_block() { + local reason="$1" + assistant_hook_emit_block "PreToolUse" "$reason" } -assistant_tool_targets_only_workflow_state_artifacts() { - local direct_path patch_text +assistant_workflow_guard_source_module() { + local module_name="$1" + local module_path="$SCRIPT_DIR/workflow-guard.d/$module_name" - direct_path=$(echo "$INPUT" | jq -r '.tool_input.file_path // .tool_input.path // .tool_input.filename // empty' 2>/dev/null || true) - if [[ -n "$direct_path" ]] && assistant_is_workflow_state_artifact_path "$direct_path"; then - return 0 + if [[ ! -f "$module_path" ]]; then + assistant_workflow_guard_block "Required workflow-guard module is missing: workflow-guard.d/$module_name" + exit 0 fi - patch_text=$(echo "$INPUT" | jq -r '.tool_input.patch // .tool_input.input // empty' 2>/dev/null || true) - if assistant_patch_targets_only_workflow_state_artifacts "$patch_text"; then - return 0 + if ! . "$module_path"; then + assistant_workflow_guard_block "Required workflow-guard module failed to load: workflow-guard.d/$module_name" + exit 0 fi - - return 1 } +assistant_workflow_guard_source_module "path-policy.sh" +assistant_workflow_guard_source_module "shell-write-parser.sh" +assistant_workflow_guard_source_module "workflow-state-artifacts.sh" + # Auto-add --tl:on to dotnet build/test commands (Terminal Logger for cleaner output). # Codex currently rejects PreToolUse updatedInput payloads, so keep this optimization # on agents that support input mutation and no-op on Codex. if [[ "$TOOL_NAME" == "Bash" ]]; then COMMAND=$(echo "$INPUT" | jq -r '.tool_input.command // empty' 2>/dev/null) if [[ -n "$COMMAND" ]]; then + ACTOR_NAME="$(assistant_tool_actor_name)" + if assistant_actor_is "$ACTOR_NAME" "code-writer"; then + assistant_workflow_guard_block "Code Writer is not allowed to run Bash. Builder/Tester owns builds, tests, and command execution." + exit 0 + fi + if assistant_bash_command_targets_lifecycle_evidence "$COMMAND"; then + assistant_workflow_guard_block "Lifecycle evidence files are hook-owned. Do not read or write subagent-events evidence through Bash; rely on SubagentStart/SubagentStop hooks." + exit 0 + fi + if assistant_actor_is "$ACTOR_NAME" "builder-tester"; then + disallowed_path="$(assistant_builder_tester_targets_disallowed_path || true)" + if [[ -n "$disallowed_path" ]]; then + assistant_workflow_guard_block "Builder/Tester may only edit test files and build configuration. Disallowed path: $disallowed_path" + exit 0 + fi + disallowed_path="$(assistant_builder_tester_bash_disallowed_write_target "$COMMAND" || true)" + if [[ -n "$disallowed_path" ]]; then + assistant_workflow_guard_block "Builder/Tester may only edit test files and build configuration. Disallowed path: $disallowed_path" + exit 0 + fi + fi + NEEDS_UPDATE=false UPDATED_COMMAND="$COMMAND" @@ -130,7 +121,15 @@ case "$TOOL_NAME" in *) exit 0 ;; esac -PROJECT_DIR="$(assistant_resolve_project_dir "$(pwd)")" +ACTOR_NAME="$(assistant_tool_actor_name)" +if assistant_actor_is "$ACTOR_NAME" "builder-tester"; then + disallowed_path="$(assistant_builder_tester_targets_disallowed_path || true)" + if [[ -n "$disallowed_path" ]]; then + assistant_workflow_guard_block "Builder/Tester may only edit test files and build configuration. Disallowed path: $disallowed_path" + exit 0 + fi +fi + TASK_FILE="$(assistant_find_task_journal "$PROJECT_DIR" "$(pwd)" || true)" # No active task = no enforcement (ad-hoc edits are fine) @@ -149,11 +148,6 @@ if assistant_tool_targets_only_workflow_state_artifacts; then exit 0 fi -assistant_workflow_guard_block() { - local reason="$1" - jq -cn --arg reason "$reason" '{decision: "block", reason: $reason}' -} - subagent_missing_key="$(assistant_phase_subagent_evidence_missing_reason_key "$TASK_FILE" || true)" subagent_policy_state="$(assistant_phase_subagent_policy_state "$TASK_FILE" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" subagent_execution_mode="$(assistant_phase_subagent_mode "$TASK_FILE" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" diff --git a/hooks/scripts/workflow-phase-gates.d/learning-controller.sh b/hooks/scripts/workflow-phase-gates.d/learning-controller.sh new file mode 100644 index 0000000..ecebbd4 --- /dev/null +++ b/hooks/scripts/workflow-phase-gates.d/learning-controller.sh @@ -0,0 +1,264 @@ +#!/usr/bin/env bash +# learning-controller.sh -- Learning controller gate helpers. + +assistant_phase_has_learning_controller() { + local file="$1" + grep -qE "^### Learning Controller[[:space:]]*$" "$file" 2>/dev/null +} + +assistant_phase_learning_controller_block() { + local file="$1" + awk ' + /^### Learning Controller[[:space:]]*$/ { + found = 1 + in_block = 1 + next + } + in_block && /^### / { exit } + in_block { print } + END { exit found ? 0 : 1 } + ' "$file" 2>/dev/null +} + +assistant_phase_has_learning_controller_after_line() { + local file="$1" + local minimum_line="$2" + [[ -n "$minimum_line" ]] || return 1 + awk -v minimum_line="$minimum_line" ' + BEGIN { minimum_line += 0 } + NR <= minimum_line { next } + /^### Learning Controller[[:space:]]*$/ { + found = 1 + exit + } + END { exit found ? 0 : 1 } + ' "$file" 2>/dev/null +} + +assistant_phase_learning_controller_block_after_line() { + local file="$1" + local minimum_line="$2" + [[ -n "$minimum_line" ]] || return 1 + awk -v minimum_line="$minimum_line" ' + BEGIN { minimum_line += 0 } + NR <= minimum_line { next } + /^### Learning Controller[[:space:]]*$/ { + found = 1 + in_block = 1 + next + } + in_block && /^### / { exit } + in_block { print } + END { exit found ? 0 : 1 } + ' "$file" 2>/dev/null +} + +assistant_phase_learning_field_value() { + local block="$1" + local label="$2" + printf '%s\n' "$block" | awk -v label="$label" ' + BEGIN { wanted = tolower(label) ":" } + { + line = $0 + sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", line) + low = tolower(line) + if (index(low, wanted) == 1) { + sub(/^[^:]*:[[:space:]]*/, "", line) + sub(/[[:space:]]*$/, "", line) + print line + exit + } + } + ' +} + +assistant_phase_learning_evidence_item_is_valid() { + local value="$1" + local label item_value + + value="$(assistant_phase_trim_value "$value")" + if [[ ! "$value" =~ ^([^:]+):[[:space:]]*(.*)$ ]]; then + return 1 + fi + + label="$(assistant_phase_trim_value "${BASH_REMATCH[1]}")" + label="$(printf '%s' "$label" | tr '[:upper:]' '[:lower:]' | sed -E 's/[[:space:]-]+/_/g')" + case "$label" in + none|review_finding|build_test_failure|user_correction|memory_trend) ;; + *) + return 1 + ;; + esac + + item_value="$(assistant_phase_trim_value "${BASH_REMATCH[2]}")" + ! assistant_phase_value_is_noneish "$item_value" && ! assistant_phase_value_is_bracket_placeholder "$item_value" +} + +assistant_phase_learning_considered_item_is_valid() { + local value="$1" + local item_value + + value="$(assistant_phase_trim_value "$value")" + if assistant_phase_value_is_noneish "$value" || assistant_phase_value_is_bracket_placeholder "$value"; then + return 1 + fi + + if [[ "$value" =~ ^[^:]+:[[:space:]]*(.*)$ ]]; then + item_value="$(assistant_phase_trim_value "${BASH_REMATCH[1]}")" + ! assistant_phase_value_is_noneish "$item_value" && ! assistant_phase_value_is_bracket_placeholder "$item_value" + return $? + fi + + return 0 +} + +assistant_phase_learning_section_has_item() { + local block="$1" + local label="$2" + local validator="${3:-learning_evidence}" + local item + local found=0 + local invalid=0 + + while IFS= read -r item; do + found=1 + case "$validator" in + learning_evidence) + assistant_phase_learning_evidence_item_is_valid "$item" || invalid=1 + ;; + considered) + assistant_phase_learning_considered_item_is_valid "$item" || invalid=1 + ;; + esac + done < <( + printf '%s\n' "$block" | awk -v label="$label" ' + function is_learning_field(line, clean) { + clean = line + sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", clean) + return clean ~ /^(Memory trend checked|Learning evidence reviewed|Review findings considered|Build\/test failures considered|User corrections considered|Durable lesson decision|Persistence evidence|No-save rationale):/ + } + BEGIN { wanted = tolower(label) ":" } + { + line = $0 + clean = line + sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", clean) + if (index(tolower(clean), wanted) == 1) { + in_section = 1 + next + } + if (in_section && is_learning_field($0)) { + exit + } + if (in_section && $0 ~ /^[[:space:]]+[-*][[:space:]]+/) { + item = $0 + sub(/^[[:space:]]*[-*][[:space:]]+/, "", item) + sub(/[[:space:]]*$/, "", item) + print item + } + } + ' + ) + + [[ "$found" -eq 1 && "$invalid" -eq 0 ]] +} + +assistant_phase_learning_missing_reason_key() { + local file="$1" + local status block trend decision persistence no_save_rationale + local spec_pass_line quality_review_line final_result_line + + if ! assistant_phase_is_medium_plus "$file"; then + printf 'complete\n' + return 0 + fi + + status="$(assistant_phase_status "$file" || true)" + if [[ "$status" != *"DOCUMENTING"* ]]; then + printf 'complete\n' + return 0 + fi + + spec_pass_line="$(assistant_phase_latest_spec_review_pass_line "$file" || true)" + if [[ -n "$spec_pass_line" ]]; then + quality_review_line="$(assistant_phase_quality_review_after_line "$file" "$spec_pass_line" || true)" + if [[ -n "$quality_review_line" ]]; then + final_result_line="$(assistant_phase_final_result_heading_line_after_line "$file" "$quality_review_line" || true)" + fi + fi + + if [[ -n "$final_result_line" ]]; then + if ! assistant_phase_has_learning_controller_after_line "$file" "$final_result_line"; then + printf 'no_learning_controller\n' + return 0 + fi + block="$(assistant_phase_learning_controller_block_after_line "$file" "$final_result_line" || true)" + else + if ! assistant_phase_has_learning_controller "$file"; then + printf 'no_learning_controller\n' + return 0 + fi + block="$(assistant_phase_learning_controller_block "$file" || true)" + fi + + trend="$(assistant_phase_trim_value "$(assistant_phase_learning_field_value "$block" "Memory trend checked")")" + case "$trend" in + checked|backend_unavailable|policy_disallowed|not_configured) ;; + *) + printf 'missing_memory_trend_checked\n' + return 0 + ;; + esac + + if ! assistant_phase_learning_section_has_item "$block" "Learning evidence reviewed" "learning_evidence"; then + printf 'missing_learning_evidence_reviewed\n' + return 0 + fi + + if ! assistant_phase_learning_section_has_item "$block" "Review findings considered" "considered"; then + printf 'missing_review_findings_considered\n' + return 0 + fi + + if ! assistant_phase_learning_section_has_item "$block" "Build/test failures considered" "considered"; then + printf 'missing_build_test_failures_considered\n' + return 0 + fi + + if ! assistant_phase_learning_section_has_item "$block" "User corrections considered" "considered"; then + printf 'missing_user_corrections_considered\n' + return 0 + fi + + decision="$(assistant_phase_trim_value "$(assistant_phase_learning_field_value "$block" "Durable lesson decision")")" + case "$decision" in + durable_saved|durable_updated|skipped_not_durable|backend_unavailable|policy_disallowed|refused_sensitive) ;; + *) + printf 'missing_durable_lesson_decision\n' + return 0 + ;; + esac + + persistence="$(assistant_phase_trim_value "$(assistant_phase_learning_field_value "$block" "Persistence evidence")")" + if [[ -z "$persistence" ]]; then + printf 'missing_persistence_evidence\n' + return 0 + fi + + case "$decision" in + durable_saved|durable_updated) + if assistant_phase_value_is_noneish "$persistence" || assistant_phase_value_is_bracket_placeholder "$persistence"; then + printf 'missing_persistence_evidence\n' + return 0 + fi + ;; + skipped_not_durable|backend_unavailable|policy_disallowed|refused_sensitive) + no_save_rationale="$(assistant_phase_trim_value "$(assistant_phase_learning_field_value "$block" "No-save rationale")")" + if assistant_phase_value_is_noneish "$no_save_rationale" || assistant_phase_value_is_bracket_placeholder "$no_save_rationale"; then + printf 'missing_no_save_rationale\n' + return 0 + fi + ;; + esac + + printf 'complete\n' +} diff --git a/hooks/scripts/workflow-phase-gates.d/metrics.sh b/hooks/scripts/workflow-phase-gates.d/metrics.sh new file mode 100644 index 0000000..f63cdc0 --- /dev/null +++ b/hooks/scripts/workflow-phase-gates.d/metrics.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# metrics.sh -- Workflow metrics gate helpers. + +assistant_phase_agent_home() { + if [[ -n "${CODEX_PROJECT_DIR:-}" ]]; then + printf '%s/.codex\n' "$HOME" + elif [[ -n "${GEMINI_PROJECT_DIR:-}" ]]; then + printf '%s/.gemini\n' "$HOME" + else + printf '%s/.claude\n' "$HOME" + fi +} + +assistant_phase_metrics_file() { + printf '%s/memory/metrics/workflow-metrics.jsonl\n' "$(assistant_phase_agent_home)" +} + +assistant_phase_has_metrics_today() { + local metrics_file + local today + metrics_file="$(assistant_phase_metrics_file)" + today="$(date +%Y-%m-%d)" + + [[ -f "$metrics_file" ]] || return 1 + grep -q "\"date\":\"$today\"" "$metrics_file" 2>/dev/null +} diff --git a/hooks/scripts/workflow-phase-gates.d/qa-controller.sh b/hooks/scripts/workflow-phase-gates.d/qa-controller.sh new file mode 100644 index 0000000..345a129 --- /dev/null +++ b/hooks/scripts/workflow-phase-gates.d/qa-controller.sh @@ -0,0 +1,323 @@ +#!/usr/bin/env bash +# qa-controller.sh -- QA and review completion gate helpers. + +assistant_phase_requires_qa_evaluator() { + local file="$1" + awk ' + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + function explicit_not_required(value, low) { + low = tolower(trim(value)) + return low ~ /^(n\/a|na|not_required|not required|not-required|not-applicable|not_applicable|not applicable)([[:space:][:punct:]]|$)/ + } + function bracket_placeholder(value) { + value = trim(value) + return value ~ /^\[[^]]*\]$/ + } + function qa_mode_required_line(line, low) { + low = tolower(trim(line)) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", low) + return low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]*[:=][[:space:]]*required([[:space:][:punct:]]|$)/ || + low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]+required([[:space:][:punct:]]|$)/ + } + function qa_mode_not_required_line(line, low) { + low = tolower(trim(line)) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", low) + return low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]*[:=][[:space:]]*(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ || + low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]+(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ + } + function qa_section_mode_required_line(line, value, low) { + line = trim(line) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", line) + low = tolower(line) + if (low !~ /^mode[[:space:]]*:/) { + return 0 + } + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + value = trim(value) + if (value ~ /\|/ || bracket_placeholder(value)) { + return 0 + } + low = tolower(value) + return low ~ /^required([[:space:][:punct:]]|$)/ + } + function qa_section_mode_not_required_line(line, value, low) { + line = trim(line) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", line) + low = tolower(line) + if (low !~ /^mode[[:space:]]*:/) { + return 0 + } + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + value = trim(value) + if (value ~ /\|/ || bracket_placeholder(value)) { + return 0 + } + low = tolower(value) + return low ~ /^(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ + } + function line_marks_not_required(line, low) { + low = tolower(line) + return low ~ /(^|[[:space:][:punct:]])(n\/a|not_required|not required|not-required|not-applicable|not_applicable|not applicable)([[:space:][:punct:]]|$)/ + } + function qa_marker(line, low) { + low = tolower(line) + return low ~ /(qa[ _-]?evaluator|qaevaluator|qa[ _-]?evaluation|qa_evaluation_mode|qa[ _-]?loop)/ + } + function scan_required_line(line) { + if (qa_marker(line) && !line_marks_not_required(line)) { + found = 1 + } + } + function scan_qa_label(line, value, low) { + sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", line) + low = tolower(line) + if (low ~ /^(qa evaluator dispatch|qa evaluator result|qa evaluator direct evidence):/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + if (!bracket_placeholder(value) && !explicit_not_required(value)) { + found = 1 + } + } else if (low ~ /^qa evaluator dispatch\/result\/direct evidence:/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + if (!bracket_placeholder(value) && !explicit_not_required(value)) { + found = 1 + } + } + } + { + line = $0 + low = tolower(line) + if (low ~ /^###[[:space:]]*qa evaluation([[:space:]#]|$)/) { + in_qa_evaluation = 1 + in_required_agents = 0 + in_required_gates = 0 + next + } + if (in_qa_evaluation && low ~ /^#+[[:space:]]+/) { + in_qa_evaluation = 0 + } + if (in_qa_evaluation) { + if (qa_section_mode_required_line(line)) { + required_mode = 1 + found = 1 + next + } + if (qa_section_mode_not_required_line(line)) { + not_required_mode = 1 + next + } + } + if (qa_mode_required_line(line)) { + required_mode = 1 + found = 1 + next + } + if (qa_mode_not_required_line(line)) { + not_required_mode = 1 + next + } + + scan_qa_label(line) + + if (low ~ /^required agents:[[:space:]]*$/) { + in_required_agents = 1 + in_required_gates = 0 + next + } + if (low ~ /^required gates:[[:space:]]*$/) { + in_required_gates = 1 + in_required_agents = 0 + next + } + if (low ~ /^required agents:[[:space:]]*.+$/ || + low ~ /^required gates:[[:space:]]*.+$/) { + scan_required_line(line) + next + } + if ((in_required_agents || in_required_gates) && line ~ /^[[:space:]]*-[[:space:]]+/) { + scan_required_line(line) + next + } + if ((in_required_agents || in_required_gates) && line ~ /^[^[:space:]-]/) { + in_required_agents = 0 + in_required_gates = 0 + } + } + END { + if (not_required_mode && !required_mode) { + exit 1 + } + exit found ? 0 : 1 + } + ' "$file" 2>/dev/null +} + +assistant_phase_latest_qa_final_result() { + local file="$1" + local minimum_line="${2:-0}" + awk -v minimum_line="$minimum_line" ' + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + function template_placeholder(value, low) { + low = tolower(trim(value)) + gsub(/`/, "", low) + return low ~ /^\[[^]]*\]$/ || low ~ /\|/ + } + function normalize_result(value, low) { + if (template_placeholder(value)) return "" + low = tolower(trim(value)) + gsub(/`/, "", low) + if (low ~ /(^|[^[:alnum:]_])not[ _-]?accepted([^[:alnum:]_]|$)/) return "not_accepted" + if (low ~ /(^|[^[:alnum:]_])accepted[ _-]?with[ _-]?concerns([^[:alnum:]_]|$)/) return "accepted_with_concerns" + if (low ~ /(^|[^[:alnum:]_])accepted([^[:alnum:]_]|$)/) return "accepted" + if (low ~ /(^|[^[:alnum:]_])rejected([^[:alnum:]_]|$)/) return "rejected" + if (low ~ /(^|[^[:alnum:]_])blocked([^[:alnum:]_]|$)/) return "blocked" + if (low ~ /(^|[^[:alnum:]_])not[ _-]?required([^[:alnum:]_]|$)/) return "not_required" + return "" + } + function record_result(value) { + latest = normalize_result(value) + } + function compact_final_marker(value, low) { + low = tolower(value) + return low ~ /(final[ _-]?verdict|qa[ _-]?result)/ + } + BEGIN { + minimum_line += 0 + } + NR <= minimum_line { + next + } + /^### QA Evaluation #[0-9]+/ { + latest = "" + next + } + { + line = $0 + sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", line) + low = tolower(line) + if (low ~ /^final verdict[[:space:]]*:/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + record_result(value) + next + } + if (low ~ /^qa result[[:space:]]*:/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + record_result(value) + next + } + if (low ~ /^qa evaluator result[[:space:]]*:/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + if (compact_final_marker(value) || normalize_result(value) != "") { + record_result(value) + } + } + } + END { + if (latest != "") { + print latest + exit 0 + } + exit 1 + } + ' "$file" 2>/dev/null +} + +assistant_phase_qa_final_result_missing_reason_key() { + local file="$1" + local minimum_line="${2:-0}" + local qa_result + + if ! assistant_phase_requires_qa_evaluator "$file"; then + printf 'complete\n' + return 0 + fi + + qa_result="$(assistant_phase_latest_qa_final_result "$file" "$minimum_line" || true)" + case "$qa_result" in + accepted|accepted_with_concerns) + printf 'complete\n' + ;; + rejected) + printf 'qa_rejected\n' + ;; + blocked) + printf 'qa_blocked\n' + ;; + "") + printf 'qa_final_result_missing\n' + ;; + not_accepted) + printf 'qa_not_accepted\n' + ;; + *) + printf 'qa_not_accepted\n' + ;; + esac +} + +assistant_phase_review_missing_reason_key() { + local file="$1" + local spec_pass_line + local quality_review_line + local review_controller_reason + local qa_reason + + if ! assistant_phase_has_spec_review_entry "$file"; then + printf 'no_spec_review\n' + return 0 + fi + + spec_pass_line="$(assistant_phase_latest_spec_review_pass_line "$file" || true)" + if [[ -z "$spec_pass_line" ]]; then + printf 'spec_not_pass\n' + return 0 + fi + + quality_review_line="$(assistant_phase_quality_review_after_line "$file" "$spec_pass_line" || true)" + if [[ -z "$quality_review_line" ]]; then + printf 'no_quality_review\n' + return 0 + fi + + if assistant_phase_is_medium_plus "$file"; then + review_controller_reason="$(assistant_phase_review_controller_missing_reason_key "$file" "$quality_review_line" "$spec_pass_line")" + if [[ "$review_controller_reason" != "complete" ]]; then + printf '%s\n' "$review_controller_reason" + return 0 + fi + assistant_phase_qa_final_result_missing_reason_key "$file" "$quality_review_line" + return 0 + fi + + if ! assistant_phase_final_result_after_line "$file" "$quality_review_line" >/dev/null; then + printf 'no_final_result\n' + return 0 + fi + + qa_reason="$(assistant_phase_qa_final_result_missing_reason_key "$file" "$quality_review_line")" + if [[ "$qa_reason" != "complete" ]]; then + printf '%s\n' "$qa_reason" + return 0 + fi + + printf 'complete\n' +} + +assistant_phase_review_complete() { + local file="$1" + [[ "$(assistant_phase_review_missing_reason_key "$file")" == "complete" ]] +} diff --git a/hooks/scripts/workflow-phase-gates.d/review-controller.sh b/hooks/scripts/workflow-phase-gates.d/review-controller.sh new file mode 100644 index 0000000..6ec0976 --- /dev/null +++ b/hooks/scripts/workflow-phase-gates.d/review-controller.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# review-controller.sh -- Quality review controller gate helpers. + +assistant_phase_review_controller_missing_reason_key() { + local file="$1" + local quality_review_line="$2" + local minimum_review_line="${3:-0}" + local quality_block final_block round_line findings_line rubric_line weighted_line delta_line drift_line score_progression_line + local round max_round heading_round must_fix should_fix current_findings weighted delta drift_value score_progression final_result_line final_result score_x100 + local previous_quality_review_line previous_quality_block previous_weighted previous_findings previous_score observed_weighted_sequence + + quality_block="$(assistant_phase_review_block_after_line "$file" "$quality_review_line" || true)" + final_block="$(assistant_phase_final_result_block_after_line "$file" "$quality_review_line" || true)" + + round_line="$(printf '%s\n' "$quality_block" | grep -m1 -E "^[[:space:]]*-[[:space:]]Round:" || true)" + if [[ ! "$round_line" =~ Round:[[:space:]]*([0-9]+)[[:space:]]+of[[:space:]]+([0-9]+) ]]; then + printf 'missing_review_round\n' + return 0 + fi + round="${BASH_REMATCH[1]}" + max_round="${BASH_REMATCH[2]}" + if [[ "$max_round" -ne 10 || "$round" -lt 1 || "$round" -gt 10 ]]; then + printf 'round_overflow\n' + return 0 + fi + heading_round="$(assistant_phase_quality_review_heading_round "$file" "$quality_review_line" || true)" + if [[ -z "$heading_round" || "$heading_round" -ne "$round" ]]; then + printf 'round_overflow\n' + return 0 + fi + + findings_line="$(printf '%s\n' "$quality_block" | grep -m1 -Ei "^[[:space:]]*-[[:space:]]Found([[:space:]]this[[:space:]]round)?:" || true)" + if [[ -z "$findings_line" ]]; then + printf 'missing_findings_summary\n' + return 0 + fi + if [[ "$findings_line" =~ ([0-9]+)[[:space:]]+must-fix ]]; then + must_fix="${BASH_REMATCH[1]}" + else + printf 'missing_findings_summary\n' + return 0 + fi + if [[ "$findings_line" =~ ([0-9]+)[[:space:]]+should-fix ]]; then + should_fix="${BASH_REMATCH[1]}" + else + printf 'missing_findings_summary\n' + return 0 + fi + current_findings="$((must_fix + should_fix))" + + rubric_line="$(printf '%s\n' "$quality_block" | grep -E "^[[:space:]]*-[[:space:]]Rubric:" | tail -1 || true)" + if [[ -z "$rubric_line" ]] || ! assistant_phase_rubric_scores_are_valid "$rubric_line"; then + printf 'missing_rubric_scores\n' + return 0 + fi + + weighted_line="$(printf '%s\n' "$quality_block" | grep -E "^[[:space:]]*-[[:space:]]Weighted:" | tail -1 || true)" + weighted="$(assistant_phase_value_after_colon "$weighted_line")" + if [[ -z "$weighted_line" ]] || ! assistant_phase_decimal_in_range "$weighted" "0" "5" \ + || ! assistant_phase_rubric_weighted_score_matches "$rubric_line" "$weighted"; then + printf 'missing_weighted_score\n' + return 0 + fi + + if [[ "$round" -gt 1 ]]; then + delta_line="$(printf '%s\n' "$quality_block" | grep -m1 -E "^[[:space:]]*-[[:space:]]Delta from previous:" || true)" + delta="$(assistant_phase_value_after_colon "$delta_line")" + if [[ -z "$delta_line" ]] || ! assistant_phase_signed_decimal_is_valid "$delta"; then + printf 'missing_delta_from_previous\n' + return 0 + fi + drift_line="$(printf '%s\n' "$quality_block" | grep -m1 -E "^[[:space:]]*-[[:space:]]Drift check:" || true)" + drift_value="$(assistant_phase_value_after_colon "$drift_line")" + if [[ -z "$drift_line" ]] || ! assistant_phase_drift_check_is_valid "$drift_value"; then + printf 'missing_drift_check\n' + return 0 + fi + fi + + final_result_line="$(printf '%s\n' "$final_block" | grep -m1 -E "^[[:space:]]*-[[:space:]]Result:[[:space:]]*(CLEAN|ISSUES_FIXED|HAS_REMAINING_ITEMS)[[:space:]]*$" || true)" + if [[ ! "$final_result_line" =~ Result:[[:space:]]*(CLEAN|ISSUES_FIXED|HAS_REMAINING_ITEMS)[[:space:]]*$ ]]; then + printf 'no_final_result\n' + return 0 + fi + final_result="${BASH_REMATCH[1]}" + + score_progression_line="$(printf '%s\n' "$final_block" | grep -m1 -E "^[[:space:]]*-[[:space:]]Score progression:" || true)" + score_progression="$(assistant_phase_value_after_colon "$score_progression_line")" + if [[ -z "$score_progression_line" ]] || ! assistant_phase_score_progression_is_valid "$score_progression" "$weighted" "$round"; then + printf 'missing_score_progression\n' + return 0 + fi + + if ! observed_weighted_sequence="$(assistant_phase_quality_review_observed_weighted_sequence "$file" "$minimum_review_line" "$quality_review_line" "$round")"; then + if [[ "$round" -gt 1 ]]; then + printf 'missing_delta_from_previous\n' + else + printf 'missing_score_progression\n' + fi + return 0 + fi + + if ! assistant_phase_score_progression_matches_observed_sequence "$score_progression" "$observed_weighted_sequence"; then + printf 'missing_score_progression\n' + return 0 + fi + + if [[ "$round" -gt 1 ]]; then + previous_quality_review_line="$(assistant_phase_previous_quality_review_line_before_line "$file" "$quality_review_line" "$minimum_review_line" || true)" + if [[ -z "$previous_quality_review_line" ]]; then + printf 'missing_delta_from_previous\n' + return 0 + fi + previous_quality_block="$(assistant_phase_review_block_after_line "$file" "$previous_quality_review_line" || true)" + previous_weighted="$(assistant_phase_review_weighted_from_block "$previous_quality_block" || true)" + previous_findings="$(assistant_phase_review_findings_count_from_block "$previous_quality_block" || true)" + if [[ -z "$previous_weighted" || -z "$previous_findings" ]]; then + printf 'missing_delta_from_previous\n' + return 0 + fi + previous_score="$previous_weighted" + if [[ -z "$previous_score" ]] || ! assistant_phase_delta_matches_scores "$delta" "$previous_score" "$weighted"; then + printf 'missing_delta_from_previous\n' + return 0 + fi + if ! assistant_phase_drift_check_matches_movement "$drift_value" "$delta" "$previous_findings" "$current_findings"; then + printf 'missing_drift_check\n' + return 0 + fi + fi + + if [[ "$final_result" == "CLEAN" || "$final_result" == "ISSUES_FIXED" ]]; then + score_x100="$(awk -v score="$weighted" 'BEGIN { printf "%d", score * 100 }')" + if [[ "$score_x100" -lt 400 ]]; then + printf 'weighted_score_below_pass\n' + return 0 + fi + if [[ "$must_fix" -ne 0 || "$should_fix" -ne 0 ]]; then + printf 'unresolved_findings\n' + return 0 + fi + elif [[ "$final_result" == "HAS_REMAINING_ITEMS" ]]; then + if ! assistant_phase_final_result_has_remaining_rationale "$final_block"; then + printf 'missing_remaining_rationale\n' + return 0 + fi + fi + + printf 'complete\n' +} diff --git a/hooks/scripts/workflow-phase-gates.d/subagent-evidence.sh b/hooks/scripts/workflow-phase-gates.d/subagent-evidence.sh new file mode 100644 index 0000000..07e75d6 --- /dev/null +++ b/hooks/scripts/workflow-phase-gates.d/subagent-evidence.sh @@ -0,0 +1,397 @@ +#!/usr/bin/env bash +# subagent-evidence.sh -- Subagent role inference and lifecycle evidence helpers. + +assistant_phase_is_codex_task() { + local file="$1" + [[ "$file" == */.codex/task.md || "$file" == .codex/task.md ]] +} + +assistant_phase_subagent_project_dir() { + local file="$1" + local project_dir + + project_dir="$(dirname "$(dirname "$file")")" + assistant_hook_canonical_existing_dir "$project_dir" +} + +assistant_phase_subagent_events_file() { + local file="$1" + local project_dir task_identity + + if assistant_phase_is_codex_task "$file"; then + project_dir="$(assistant_phase_subagent_project_dir "$file")" || return 1 + task_identity="$(assistant_hook_task_identity_from_file "$file")" || return 1 + assistant_hook_codex_subagent_events_file_for_project "$project_dir" "$task_identity" + return $? + fi + + printf '%s/subagent-events.jsonl\n' "$(dirname "$file")" +} + +assistant_phase_role_agent_pattern() { + case "$1" in + "Code Mapper") printf 'code-mapper|codemapper|Code Mapper' ;; + "Explorer") printf 'explorer|Explorer' ;; + "Architect") printf 'architect|Architect' ;; + "Code Writer") printf 'code-writer|codewriter|Code Writer' ;; + "Builder/Tester") printf 'builder-tester|builder/tester|Builder/Tester' ;; + "Code Reviewer") printf 'code-reviewer|codereviewer|Code Reviewer|reviewer|Reviewer' ;; + "QA Evaluator") printf 'qa-evaluator|qaevaluator|QA Evaluator' ;; + "Reviewer") printf 'reviewer|Reviewer|code-reviewer|codereviewer|Code Reviewer' ;; + *) printf '%s' "$1" ;; + esac +} + +assistant_phase_json_field_value() { + local json_line="$1" + local field="$2" + printf '%s\n' "$json_line" | sed -n 's/.*"'"$field"'":"\([^"]*\)".*/\1/p' +} + +assistant_phase_event_role_matches() { + local json_line="$1" + local role="$2" + local pattern agent_type agent_name name + pattern="$(assistant_phase_role_agent_pattern "$role")" + agent_type="$(assistant_phase_json_field_value "$json_line" "agent_type" | tr '[:upper:]' '[:lower:]')" + agent_name="$(assistant_phase_json_field_value "$json_line" "agent_name" | tr '[:upper:]' '[:lower:]')" + IFS='|' read -r -a names <<< "$pattern" + for name in "${names[@]}"; do + name="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]')" + [[ "$agent_type" == "$name" || "$agent_name" == "$name" ]] && return 0 + done + return 1 +} + +assistant_phase_codex_role_event_ids() { + local file="$1" + local role="$2" + local event="$3" + local events_file task_identity line id event_task_identity + task_identity="$(assistant_hook_task_identity_from_file "$file")" || return 1 + events_file="$(assistant_phase_subagent_events_file "$file")" || return 1 + [[ -f "$events_file" ]] || return 1 + while IFS= read -r line; do + [[ "$line" == *"\"event\":\"$event\""* ]] || continue + event_task_identity="$(assistant_phase_json_field_value "$line" "task_identity")" + [[ "$event_task_identity" == "$task_identity" ]] || continue + assistant_phase_event_role_matches "$line" "$role" || continue + id="$(assistant_phase_json_field_value "$line" "agent_id")" + [[ -n "$id" ]] || continue + printf '%s\n' "$id" + done < "$events_file" +} + +assistant_phase_journal_mentions_agent_id() { + local file="$1" + local role="$2" + local agent_id="$3" + local dispatch result combined + [[ -n "$agent_id" ]] || return 1 + dispatch="$(assistant_phase_labeled_evidence_value "$file" "$role dispatch")" + result="$(assistant_phase_labeled_evidence_value "$file" "$role result")" + combined="$dispatch +$result" + printf '%s\n' "$combined" | assistant_phase_text_mentions_agent_id_token "$agent_id" +} + +assistant_phase_text_mentions_agent_id_token() { + local agent_id="$1" + [[ -n "$agent_id" ]] || return 1 + awk -v needle="$agent_id" ' + function is_token_char(ch) { + return ch ~ /^[[:alnum:]_-]$/ + } + { + start = 1 + while ((relative = index(substr($0, start), needle)) > 0) { + absolute = start + relative - 1 + before = absolute > 1 ? substr($0, absolute - 1, 1) : "" + after_pos = absolute + length(needle) + after = after_pos <= length($0) ? substr($0, after_pos, 1) : "" + if (!is_token_char(before) && !is_token_char(after)) { + found = 1 + exit + } + start = absolute + length(needle) + } + } + END { + exit found ? 0 : 1 + } + ' +} + +assistant_phase_has_role_event_pair_evidence() { + local file="$1" + local role="$2" + local start_id + while IFS= read -r start_id; do + [[ -n "$start_id" ]] || continue + assistant_phase_journal_mentions_agent_id "$file" "$role" "$start_id" || continue + if assistant_phase_codex_role_event_ids "$file" "$role" "SubagentStop" | grep -Fxq -- "$start_id"; then + return 0 + fi + done < <(assistant_phase_codex_role_event_ids "$file" "$role" "SubagentStart" || true) + return 1 +} + +assistant_phase_has_role_start_event_evidence() { + local file="$1" + local role="$2" + local start_id + while IFS= read -r start_id; do + [[ -n "$start_id" ]] || continue + assistant_phase_journal_mentions_agent_id "$file" "$role" "$start_id" && return 0 + done < <(assistant_phase_codex_role_event_ids "$file" "$role" "SubagentStart" || true) + return 1 +} + +assistant_phase_has_role_stop_event_evidence() { + assistant_phase_has_role_event_pair_evidence "$1" "$2" +} + +assistant_phase_role_evidence_labels() { + case "$1" in + "Code Reviewer") + printf 'Code Reviewer\nReviewer\n' + ;; + "Reviewer") + printf 'Code Reviewer\nReviewer\n' + ;; + *) + printf '%s\n' "$1" + ;; + esac +} + +assistant_phase_has_single_role_dispatch_result_evidence() { + local file="$1" + local role="$2" + + assistant_phase_has_labeled_evidence "$file" "$role dispatch" \ + && assistant_phase_has_labeled_evidence "$file" "$role result" \ + || return 1 + + # Codex task journals are not sufficient proof by themselves: Codex can write + # fake dispatch text after doing work inline. Require protected lifecycle + # evidence captured by SubagentStart/SubagentStop hooks for delegated roles. + if assistant_phase_is_codex_task "$file"; then + assistant_phase_has_role_start_event_evidence "$file" "$role" \ + && assistant_phase_has_role_stop_event_evidence "$file" "$role" + return $? + fi + + return 0 +} + +assistant_phase_has_role_dispatch_result_evidence() { + local file="$1" + local role="$2" + local evidence_role + + while IFS= read -r evidence_role; do + [[ -n "$evidence_role" ]] || continue + assistant_phase_has_single_role_dispatch_result_evidence "$file" "$evidence_role" && return 0 + done < <(assistant_phase_role_evidence_labels "$role") + + return 1 +} + +assistant_phase_direct_fallback_reason_valid() { + local file="$1" + grep -qiE "^[[:space:]]*[-*]?[[:space:]]*Direct fallback reason:[[:space:]]*(authorization_denied|subagents_unavailable|policy_disallowed)([[:space:]]|$)" "$file" 2>/dev/null +} + +assistant_phase_has_single_role_equivalent_evidence() { + local file="$1" + local role="$2" + assistant_phase_has_labeled_evidence "$file" "$role direct evidence" +} + +assistant_phase_has_role_equivalent_evidence() { + local file="$1" + local role="$2" + local evidence_role + + while IFS= read -r evidence_role; do + [[ -n "$evidence_role" ]] || continue + assistant_phase_has_single_role_equivalent_evidence "$file" "$evidence_role" && return 0 + done < <(assistant_phase_role_evidence_labels "$role") + + return 1 +} + +assistant_phase_has_per_slice_dispatch_evidence() { + local file="$1" + ! assistant_phase_is_medium_plus "$file" && return 0 + assistant_phase_has_labeled_evidence "$file" "Per-slice dispatch evidence" +} + +assistant_phase_has_explicit_no_source_changes() { + local file="$1" + awk ' + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + function value_after_colon(line, value) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + return trim(value) + } + { + line = $0 + sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", line) + low = tolower(line) + if (low ~ /^(source changes|source change|source-changing|project source changes|production source changes|source files changed|source-changing work)[[:space:]]*:/) { + value = tolower(value_after_colon(line)) + if (value ~ /^(no|none)([[:space:][:punct:]]|$)/) { + found = 1 + } + } + if (low ~ /^no source changes[[:space:]]*:/) { + value = tolower(value_after_colon(line)) + if (value ~ /^(yes|true)([[:space:][:punct:]]|$)/) { + found = 1 + } + } + } + END { + exit found ? 0 : 1 + } + ' "$file" 2>/dev/null +} + +assistant_phase_task_type_is_development_work() { + local file="$1" + local task_type + task_type="$(assistant_phase_scalar_field "$file" "Task type" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" + [[ "$task_type" =~ ^(feature|bugfix|refactor|migration|rewrite|config|infra|security|docs|test|tests|tooling|dev|development|code)$ ]] +} + +assistant_phase_journal_mentions_source_change_path() { + local file="$1" + awk ' + function source_path(line, low) { + low = tolower(line) + if (low ~ /\.(codex|claude|gemini)\//) { + return 0 + } + return low ~ /(^|[[:space:]`"'\''(|])(\.?\/)?(agents|codex-rules|docs|hooks|skills|tests|tools)\// || + low ~ /(^|[[:space:]`"'\''(|])(agents\.md|claude\.md|readme\.md|install\.sh)([[:space:]`"'\'')]|$)/ || + low ~ /\.(bash|cs|csproj|json|md|props|sh|sln|targets|toml|yaml|yml)([[:space:]`"'\'')]|$)/ + } + { + line = $0 + low = tolower(line) + if (low ~ /(likely touched paths|changed files|modified files|files changed|artifact registry)/) { + in_paths = 1 + if (source_path(line)) { + found = 1 + } + next + } + if (in_paths && line ~ /^[[:space:]]*([-*]|\|)/) { + if (source_path(line)) { + found = 1 + } + next + } + if (in_paths && line ~ /^[^[:space:]-|]/) { + in_paths = 0 + } + } + END { + exit found ? 0 : 1 + } + ' "$file" 2>/dev/null +} + +assistant_phase_journal_declares_source_changes() { + local file="$1" + grep -qiE "^[[:space:]]*[-*]?[[:space:]]*(Source changes|Source change|Source-changing|Project source changes|Production source changes|Source files changed|Source-changing work):[[:space:]]*(yes|true|required|will|planned|expected|source-changing)([[:space:][:punct:]]|$)" "$file" 2>/dev/null +} + +assistant_phase_source_changing_work() { + local file="$1" + local status="$2" + local status_lc + + assistant_phase_has_explicit_no_source_changes "$file" && return 1 + assistant_phase_journal_declares_source_changes "$file" && return 0 + assistant_phase_journal_mentions_source_change_path "$file" && return 0 + + status_lc="$(printf '%s' "$status" | tr '[:upper:]' '[:lower:]')" + if [[ "$status_lc" =~ (building|verifying) ]] && assistant_phase_task_type_is_development_work "$file"; then + return 0 + fi + + return 1 +} + +assistant_phase_required_subagent_roles() { + local file="$1" + local status mode qa_required source_changing + status="$(assistant_phase_status "$file" | tr '[:upper:]' '[:lower:]' || true)" + mode="$(assistant_phase_subagent_mode "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" + qa_required="$(assistant_phase_requires_qa_evaluator "$file" && printf yes || printf no)" + source_changing="$(assistant_phase_source_changing_work "$file" "$status" && printf yes || printf no)" + awk -v is_medium_plus="$(assistant_phase_is_medium_plus "$file" && printf yes || printf no)" -v status="$status" -v mode="$mode" -v qa_required="$qa_required" -v source_changing="$source_changing" ' + function emit(role) { + if (!seen[role]) { + seen[role] = 1 + print role + } + } + function scan(line, low) { + low = tolower(line) + if (low ~ /code mapper|code-mapper/) emit("Code Mapper") + if (low ~ /explorer/) emit("Explorer") + if (low ~ /architect/) emit("Architect") + if (low ~ /code writer|code-writer/) emit("Code Writer") + if (low ~ /builder\/tester|builder-tester/) emit("Builder/Tester") + if (low ~ /qa[ _-]?evaluator|qaevaluator/) emit("QA Evaluator") + if (low ~ /code[ _-]?reviewer|codereviewer/) { + emit("Code Reviewer") + } else if (low ~ /(^|[^[:alnum:]_-])reviewer([^[:alnum:]_-]|$)/) { + emit("Code Reviewer") + } + } + BEGIN { + if (source_changing == "yes") { + emit("Code Writer") + emit("Builder/Tester") + emit("Code Reviewer") + } + # Medium+ discovery always requires a Code Mapper context map once + # subagent execution mode has been resolved. Add the role from task + # size even if the journal forgot to list it. + if (mode ~ /^(delegated|direct_fallback)$/ && is_medium_plus == "yes") emit("Code Mapper") + # Once a delegated/fallback task is in Review/Document, the review + # role is required even for no-op/no-code-change outcomes; otherwise + # "review phase" can be satisfied inline while claiming delegated mode. + if (mode ~ /^(delegated|direct_fallback)$/ && status ~ /(reviewing|documenting)/) emit("Code Reviewer") + if (qa_required == "yes") emit("QA Evaluator") + } + /^Required agents:[[:space:]]*$/ { in_required = 1; next } + /^Required agents:[[:space:]]*(.+)$/ { + scan($0) + next + } + in_required && /^[[:space:]]*-[[:space:]]+/ { + scan($0) + next + } + in_required && /^[^[:space:]-]/ { in_required = 0 } + ' "$file" 2>/dev/null +} + +assistant_phase_requires_subagent_roles() { + [[ -n "$(assistant_phase_required_subagent_roles "$1")" ]] +} + +assistant_phase_qa_evaluator_subagent_evidence_is_actionable() { + [[ "$(assistant_phase_review_missing_reason_key "$1")" == "complete" ]] +} diff --git a/hooks/scripts/workflow-phase-gates.d/subagent-orchestration.sh b/hooks/scripts/workflow-phase-gates.d/subagent-orchestration.sh new file mode 100644 index 0000000..c57a743 --- /dev/null +++ b/hooks/scripts/workflow-phase-gates.d/subagent-orchestration.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash +# subagent-orchestration.sh -- Subagent gate warning and reason orchestration. + +assistant_phase_role_reason_slug() { + printf '%s' "$1" | tr '[:upper:]/ ' '[:lower:]__' +} + +assistant_phase_reason_missing_field() { + local gate="$1" + local key="$2" + + case "$gate:$key" in + plan_gate:no_plan) printf 'No plan found; Plan approval: yes\n' ;; + plan_gate:plan_not_approved) printf 'Plan exists but is not approved; Plan approval: yes\n' ;; + + subagent_evidence_gate:authorization_required_unresolved) printf 'Subagent policy state / Subagent execution mode\n' ;; + subagent_evidence_gate:delegated_missing_code_mapper) printf 'Code Mapper dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_explorer) printf 'Explorer dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_architect) printf 'Architect dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_code_writer) printf 'Code Writer dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_builder_tester) printf 'Builder/Tester dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_code_reviewer) printf 'Code Reviewer dispatch/result evidence during Review\n' ;; + subagent_evidence_gate:delegated_missing_qa_evaluator) printf 'QA Evaluator dispatch/result evidence\n' ;; + subagent_evidence_gate:delegated_missing_per_slice) printf 'Per-slice dispatch evidence\n' ;; + subagent_evidence_gate:direct_fallback_invalid_policy_state) printf 'Subagent policy state for direct_fallback\n' ;; + subagent_evidence_gate:direct_fallback_missing_reason) printf 'Direct fallback reason\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_mapper) printf 'Code Mapper direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_explorer) printf 'Explorer direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_architect) printf 'Architect direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_writer) printf 'Code Writer direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_builder_tester) printf 'Builder/Tester direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_reviewer) printf 'Code Reviewer direct evidence during Review\n' ;; + subagent_evidence_gate:direct_fallback_missing_qa_evaluator) printf 'QA Evaluator direct evidence\n' ;; + subagent_evidence_gate:not_applicable_with_required_roles) printf 'Subagent execution mode\n' ;; + subagent_evidence_gate:unknown_execution_mode) printf 'Subagent execution mode\n' ;; + + review_gate:no_spec_review) printf 'no Spec Review; ### Spec Review #N\n' ;; + review_gate:spec_not_pass) printf 'latest Spec Review result not PASS; - Result: PASS\n' ;; + review_gate:no_quality_review) printf 'no Quality Review after latest Spec Review PASS; ### Quality Review #N\n' ;; + review_gate:no_final_result) printf 'no Final Result; ### Final result with - Result: CLEAN|ISSUES_FIXED|HAS_REMAINING_ITEMS\n' ;; + review_gate:qa_rejected) printf 'QA Evaluation final verdict/result accepted or accepted_with_concerns\n' ;; + review_gate:qa_blocked) printf 'QA Evaluation final verdict/result accepted or accepted_with_concerns\n' ;; + review_gate:qa_final_result_missing) printf 'QA Evaluation final verdict/result\n' ;; + review_gate:qa_not_accepted) printf 'QA Evaluation final verdict/result accepted or accepted_with_concerns\n' ;; + review_gate:missing_review_round) printf '%s\n' '- Round: N of 10' ;; + review_gate:round_overflow) printf '%s\n' '- Round: N of 10 with N between 1 and 10' ;; + review_gate:missing_findings_summary) printf '%s\n' '- Found this round: X must-fix, Y should-fix, Z nits' ;; + review_gate:missing_rubric_scores) printf '%s\n' '- Rubric: correctness, code_quality/quality, architecture, security, test_coverage/coverage scores' ;; + review_gate:missing_weighted_score) printf 'missing or mismatched weighted score; - Weighted: N.NN\n' ;; + review_gate:missing_delta_from_previous) printf '%s\n' '- Delta from previous: +/-N.NN' ;; + review_gate:missing_drift_check) printf '%s\n' '- Drift check: GENUINE or REGRESSION' ;; + review_gate:missing_score_progression) printf '%s\n' '- Score progression: ...' ;; + review_gate:weighted_score_below_pass) printf '%s\n' '- Weighted: score >= 4.00 for CLEAN/ISSUES_FIXED' ;; + review_gate:unresolved_findings) printf '0 must-fix and 0 should-fix findings for CLEAN/ISSUES_FIXED\n' ;; + review_gate:missing_remaining_rationale) printf '%s\n' '- Remaining items: or - Blocker: with concrete evidence and owner' ;; + + metrics_gate:missing_metrics_today) printf 'workflow metrics JSONL entry for today\n' ;; + + learning_gate:no_learning_controller) printf '### Learning Controller\n' ;; + learning_gate:missing_memory_trend_checked) printf '%s\n' '- Memory trend checked: checked|backend_unavailable|policy_disallowed|not_configured' ;; + learning_gate:missing_learning_evidence_reviewed) printf '%s\n' '- Learning evidence reviewed: list item' ;; + learning_gate:missing_review_findings_considered) printf '%s\n' '- Review findings considered: list item' ;; + learning_gate:missing_build_test_failures_considered) printf '%s\n' '- Build/test failures considered: list item' ;; + learning_gate:missing_user_corrections_considered) printf '%s\n' '- User corrections considered: list item' ;; + learning_gate:missing_durable_lesson_decision) printf '%s\n' '- Durable lesson decision: durable_saved|durable_updated|skipped_not_durable|backend_unavailable|policy_disallowed|refused_sensitive' ;; + learning_gate:missing_persistence_evidence) printf '%s\n' '- Persistence evidence: concrete saved/updated memory evidence' ;; + learning_gate:missing_no_save_rationale) printf '%s\n' '- No-save rationale: concrete reason' ;; + + *) printf 'required gate field\n' ;; + esac +} + +assistant_phase_reason_action() { + local gate="$1" + local key="$2" + + case "$gate:$key" in + plan_gate:no_plan) printf 'run PLAN, get user approval, then record Plan approval: yes\n' ;; + plan_gate:plan_not_approved) printf 'present the plan, wait for approval, then record Plan approval: yes\n' ;; + + subagent_evidence_gate:authorization_required_unresolved) printf 'ask once for subagent authorization or denial before continuing delegated workflow work\n' ;; + subagent_evidence_gate:delegated_missing_code_mapper) printf 'record Code Mapper dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; + subagent_evidence_gate:delegated_missing_explorer) printf 'record Explorer dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; + subagent_evidence_gate:delegated_missing_architect) printf 'record Architect dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; + subagent_evidence_gate:delegated_missing_code_writer) printf 'record Code Writer dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; + subagent_evidence_gate:delegated_missing_builder_tester) printf 'record Builder/Tester dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; + subagent_evidence_gate:delegated_missing_code_reviewer) printf 'record Code Reviewer during Review; legacy Reviewer labels are compatibility routing only\n' ;; + subagent_evidence_gate:delegated_missing_qa_evaluator) printf 'run QA Evaluator after build/test and code-review evidence, then record dispatch/result\n' ;; + subagent_evidence_gate:delegated_missing_per_slice) printf 'record Per-slice dispatch evidence for the implementation slice\n' ;; + subagent_evidence_gate:direct_fallback_invalid_policy_state) printf 'set policy state to authorization_denied, subagents_unavailable, or policy_disallowed\n' ;; + subagent_evidence_gate:direct_fallback_missing_reason) printf 'record Direct fallback reason: authorization_denied|subagents_unavailable|policy_disallowed\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_mapper) printf 'record Code Mapper direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_explorer) printf 'record Explorer direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_architect) printf 'record Architect direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_writer) printf 'record Code Writer direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_builder_tester) printf 'record Builder/Tester direct evidence\n' ;; + subagent_evidence_gate:direct_fallback_missing_code_reviewer) printf 'record Code Reviewer direct evidence during Review\n' ;; + subagent_evidence_gate:direct_fallback_missing_qa_evaluator) printf 'record QA Evaluator direct evidence after build/test and code-review evidence\n' ;; + subagent_evidence_gate:not_applicable_with_required_roles) printf 'set execution mode to delegated or direct_fallback and record matching evidence\n' ;; + subagent_evidence_gate:unknown_execution_mode) printf 'set Subagent execution mode to delegated, direct_fallback, or not_applicable with valid evidence\n' ;; + + review_gate:no_spec_review) printf 'run Spec Review first and record PASS/FAIL plus scope, mismatch, and required-fix fields\n' ;; + review_gate:spec_not_pass) printf 'fix spec gaps and rerun Spec Review until the latest structured result is PASS\n' ;; + review_gate:no_quality_review) printf 'run assistant-review Stage 2 and append Quality Review after latest Spec Review PASS\n' ;; + review_gate:no_final_result) printf 'finish the review loop and write the Final Result summary after Quality Review\n' ;; + review_gate:qa_rejected) printf 'fix QA findings, rerun QA Evaluation evidence, and record accepted or accepted_with_concerns\n' ;; + review_gate:qa_blocked) printf 'resolve the QA blocker, rerun QA Evaluation evidence, and record accepted or accepted_with_concerns\n' ;; + review_gate:qa_final_result_missing) printf 'run or rerun QA Evaluation evidence and record accepted or accepted_with_concerns\n' ;; + review_gate:qa_not_accepted) printf 'fix or rerun QA Evaluation evidence until accepted or accepted_with_concerns\n' ;; + review_gate:missing_review_round) printf 'add - Round: N of 10 to the latest Quality Review\n' ;; + review_gate:round_overflow) printf 'repair the Quality Review heading/round so N is 1..10 and max is 10\n' ;; + review_gate:missing_findings_summary) printf 'add the latest must-fix/should-fix/nits finding counts\n' ;; + review_gate:missing_rubric_scores) printf 'add numeric 0..5 rubric scores to the latest Quality Review\n' ;; + review_gate:missing_weighted_score) printf 'add a numeric - Weighted: N.NN matching the rubric formula; fix any mismatched weighted score\n' ;; + review_gate:missing_delta_from_previous) printf 'add a valid delta from the previous Quality Review round\n' ;; + review_gate:missing_drift_check) printf 'add a valid drift classification for review rounds after round 1\n' ;; + review_gate:missing_score_progression) printf 'add Score progression matching the observed Quality Review weighted sequence\n' ;; + review_gate:weighted_score_below_pass) printf 'improve low-scoring dimensions and rerun Quality Review before CLEAN/ISSUES_FIXED\n' ;; + review_gate:unresolved_findings) printf 'fix or explicitly carry remaining must-fix/should-fix findings through the loop\n' ;; + review_gate:missing_remaining_rationale) printf 'add concrete remaining-item or blocker rationale with evidence and owner\n' ;; + + metrics_gate:missing_metrics_today) printf 'append today'\''s workflow metrics JSONL entry before stopping\n' ;; + + learning_gate:no_learning_controller) printf 'add the canonical Learning Controller block after Final Result\n' ;; + learning_gate:missing_memory_trend_checked) printf 'record one valid Memory trend checked value\n' ;; + learning_gate:missing_learning_evidence_reviewed) printf 'add reviewed learning evidence or none with a reason\n' ;; + learning_gate:missing_review_findings_considered) printf 'add review finding consideration or none with a reason\n' ;; + learning_gate:missing_build_test_failures_considered) printf 'add build/test failure consideration or none with a reason\n' ;; + learning_gate:missing_user_corrections_considered) printf 'add user correction consideration or none with a reason\n' ;; + learning_gate:missing_durable_lesson_decision) printf 'record one valid durable lesson decision\n' ;; + learning_gate:missing_persistence_evidence) printf 'add concrete memory_reflect, memory_add_insight, or backend evidence for saved/updated lessons\n' ;; + learning_gate:missing_no_save_rationale) printf 'add a concrete no-save rationale when no durable write occurred\n' ;; + + *) printf 'complete the required gate evidence before stopping\n' ;; + esac +} + +assistant_phase_subagent_reason_role() { + local key="$1" + + case "$key" in + *_code_mapper) printf 'Code Mapper\n' ;; + *_explorer) printf 'Explorer\n' ;; + *_architect) printf 'Architect\n' ;; + *_code_writer) printf 'Code Writer\n' ;; + *_builder_tester) printf 'Builder/Tester\n' ;; + *_code_reviewer) printf 'Code Reviewer\n' ;; + *_qa_evaluator) printf 'QA Evaluator\n' ;; + delegated_missing_per_slice) printf 'Per-slice\n' ;; + *) printf '\n' ;; + esac +} + +assistant_phase_status_has_any() { + local status="$1" + shift + local phase + + for phase in "$@"; do + [[ "$status" == *"$phase"* ]] && return 0 + done + return 1 +} + +assistant_phase_subagent_reason_is_current_phase_actionable() { + local key="$1" + local status="$2" + local role + + case "$key" in + complete) + return 1 + ;; + authorization_required_unresolved|direct_fallback_invalid_policy_state|direct_fallback_missing_reason|not_applicable_with_required_roles|unknown_execution_mode) + assistant_phase_status_has_any "$status" DISCOVERING DECOMPOSING PLANNING BUILDING VERIFYING REVIEWING DOCUMENTING + return $? + ;; + esac + + role="$(assistant_phase_subagent_reason_role "$key")" + case "$role" in + "Code Mapper"|"Explorer"|"Architect") + assistant_phase_status_has_any "$status" DISCOVERING DECOMPOSING PLANNING + ;; + "Code Writer"|"Builder/Tester"|"Per-slice") + assistant_phase_status_has_any "$status" BUILDING VERIFYING + ;; + "Code Reviewer"|"QA Evaluator") + assistant_phase_status_has_any "$status" REVIEWING + ;; + *) + return 1 + ;; + esac +} + +assistant_phase_subagent_warning_action() { + local key="$1" + local status="$2" + local field action + + assistant_phase_subagent_reason_is_current_phase_actionable "$key" "$status" || return 1 + field="$(assistant_phase_reason_missing_field "subagent_evidence_gate" "$key")" + action="$(assistant_phase_reason_action "subagent_evidence_gate" "$key")" + printf 'missing=%s action=%s\n' "$field" "$action" +} + +assistant_phase_subagent_warning_reason_key() { + local file="$1" + local key="$2" + local status="$3" + local mode policy_state roles role candidate + + if [[ "$key" == *_qa_evaluator ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then + key="complete" + fi + + if assistant_phase_subagent_reason_is_current_phase_actionable "$key" "$status"; then + printf '%s\n' "$key" + return 0 + fi + + mode="$(assistant_phase_subagent_mode "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" + policy_state="$(assistant_phase_subagent_policy_state "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" + roles="$(assistant_phase_required_subagent_roles "$file")" + + case "$mode" in + delegated) + while IFS= read -r role; do + [[ -n "$role" ]] || continue + if ! assistant_phase_has_role_dispatch_result_evidence "$file" "$role"; then + candidate="delegated_missing_$(assistant_phase_role_reason_slug "$role")" + if [[ "$candidate" == *_qa_evaluator ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then + continue + fi + if assistant_phase_subagent_reason_is_current_phase_actionable "$candidate" "$status"; then + printf '%s\n' "$candidate" + return 0 + fi + fi + done <<< "$roles" + if printf '%s\n' "$roles" | grep -Eq '^(Code Writer|Builder/Tester)$' \ + && ! assistant_phase_has_per_slice_dispatch_evidence "$file"; then + candidate="delegated_missing_per_slice" + if assistant_phase_subagent_reason_is_current_phase_actionable "$candidate" "$status"; then + printf '%s\n' "$candidate" + return 0 + fi + fi + ;; + direct_fallback) + if [[ "$policy_state" != "authorization_denied" && "$policy_state" != "subagents_unavailable" && "$policy_state" != "policy_disallowed" ]]; then + return 1 + fi + if ! assistant_phase_direct_fallback_reason_valid "$file"; then + return 1 + fi + while IFS= read -r role; do + [[ -n "$role" ]] || continue + if ! assistant_phase_has_role_equivalent_evidence "$file" "$role"; then + candidate="direct_fallback_missing_$(assistant_phase_role_reason_slug "$role")" + if [[ "$candidate" == *_qa_evaluator ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then + continue + fi + if assistant_phase_subagent_reason_is_current_phase_actionable "$candidate" "$status"; then + printf '%s\n' "$candidate" + return 0 + fi + fi + done <<< "$roles" + ;; + esac + + return 1 +} + +assistant_phase_subagent_evidence_missing_reason_key() { + local file="$1" + local mode policy_state role roles + mode="$(assistant_phase_subagent_mode "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" + policy_state="$(assistant_phase_subagent_policy_state "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" + roles="$(assistant_phase_required_subagent_roles "$file")" + + # Authorization-required is a wait state, not an execution mode. If a task + # has reached an active workflow phase while authorization is unresolved, + # block before it can silently complete work inline. + if [[ "$policy_state" == "authorization_required" ]]; then + printf 'authorization_required_unresolved\n' + return 0 + fi + + # Strict subagent evidence applies whenever workflow subagent roles are + # declared, not only when source code changed. Discovery/review-only work can + # legitimately skip Code Writer and Builder/Tester, but delegated Code Mapper, + # Explorer, Architect, or Code Reviewer responsibilities still need evidence. + if [[ -z "$roles" ]]; then + printf 'complete\n' + return 0 + fi + + case "$mode" in + delegated) + while IFS= read -r role; do + [[ -n "$role" ]] || continue + if [[ "$role" == "QA Evaluator" ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then + continue + fi + if ! assistant_phase_has_role_dispatch_result_evidence "$file" "$role"; then + printf 'delegated_missing_%s\n' "$(assistant_phase_role_reason_slug "$role")" + return 0 + fi + done <<< "$roles" + if printf '%s\n' "$roles" | grep -Eq '^(Code Writer|Builder/Tester)$' \ + && ! assistant_phase_has_per_slice_dispatch_evidence "$file"; then + printf 'delegated_missing_per_slice\n' + return 0 + fi + ;; + direct_fallback) + if [[ "$policy_state" != "authorization_denied" && "$policy_state" != "subagents_unavailable" && "$policy_state" != "policy_disallowed" ]]; then + printf 'direct_fallback_invalid_policy_state\n' + return 0 + fi + if ! assistant_phase_direct_fallback_reason_valid "$file"; then + printf 'direct_fallback_missing_reason\n' + return 0 + fi + while IFS= read -r role; do + [[ -n "$role" ]] || continue + if [[ "$role" == "QA Evaluator" ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then + continue + fi + if ! assistant_phase_has_role_equivalent_evidence "$file" "$role"; then + printf 'direct_fallback_missing_%s\n' "$(assistant_phase_role_reason_slug "$role")" + return 0 + fi + done <<< "$roles" + ;; + not_applicable|"") + printf 'not_applicable_with_required_roles\n' + return 0 + ;; + *) + printf 'unknown_execution_mode\n' + return 0 + ;; + esac + + printf 'complete\n' +} diff --git a/hooks/scripts/workflow-phase-gates.sh b/hooks/scripts/workflow-phase-gates.sh index 347e452..10933da 100644 --- a/hooks/scripts/workflow-phase-gates.sh +++ b/hooks/scripts/workflow-phase-gates.sh @@ -6,6 +6,11 @@ # task-journal-resolver.sh so these helpers do not accidentally revive stale # workflow state. +ASSISTANT_PHASE_GATES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -f "$ASSISTANT_PHASE_GATES_DIR/hook-runtime.sh" ]]; then + . "$ASSISTANT_PHASE_GATES_DIR/hook-runtime.sh" +fi + assistant_phase_scalar_field() { local file="$1" local label="$2" @@ -754,603 +759,6 @@ assistant_phase_final_result_has_remaining_rationale() { return 1 } -assistant_phase_review_controller_missing_reason_key() { - local file="$1" - local quality_review_line="$2" - local minimum_review_line="${3:-0}" - local quality_block final_block round_line findings_line rubric_line weighted_line delta_line drift_line score_progression_line - local round max_round heading_round must_fix should_fix current_findings weighted delta drift_value score_progression final_result_line final_result score_x100 - local previous_quality_review_line previous_quality_block previous_weighted previous_findings previous_score observed_weighted_sequence - - quality_block="$(assistant_phase_review_block_after_line "$file" "$quality_review_line" || true)" - final_block="$(assistant_phase_final_result_block_after_line "$file" "$quality_review_line" || true)" - - round_line="$(printf '%s\n' "$quality_block" | grep -m1 -E "^[[:space:]]*-[[:space:]]Round:" || true)" - if [[ ! "$round_line" =~ Round:[[:space:]]*([0-9]+)[[:space:]]+of[[:space:]]+([0-9]+) ]]; then - printf 'missing_review_round\n' - return 0 - fi - round="${BASH_REMATCH[1]}" - max_round="${BASH_REMATCH[2]}" - if [[ "$max_round" -ne 10 || "$round" -lt 1 || "$round" -gt 10 ]]; then - printf 'round_overflow\n' - return 0 - fi - heading_round="$(assistant_phase_quality_review_heading_round "$file" "$quality_review_line" || true)" - if [[ -z "$heading_round" || "$heading_round" -ne "$round" ]]; then - printf 'round_overflow\n' - return 0 - fi - - findings_line="$(printf '%s\n' "$quality_block" | grep -m1 -Ei "^[[:space:]]*-[[:space:]]Found([[:space:]]this[[:space:]]round)?:" || true)" - if [[ -z "$findings_line" ]]; then - printf 'missing_findings_summary\n' - return 0 - fi - if [[ "$findings_line" =~ ([0-9]+)[[:space:]]+must-fix ]]; then - must_fix="${BASH_REMATCH[1]}" - else - printf 'missing_findings_summary\n' - return 0 - fi - if [[ "$findings_line" =~ ([0-9]+)[[:space:]]+should-fix ]]; then - should_fix="${BASH_REMATCH[1]}" - else - printf 'missing_findings_summary\n' - return 0 - fi - current_findings="$((must_fix + should_fix))" - - rubric_line="$(printf '%s\n' "$quality_block" | grep -E "^[[:space:]]*-[[:space:]]Rubric:" | tail -1 || true)" - if [[ -z "$rubric_line" ]] || ! assistant_phase_rubric_scores_are_valid "$rubric_line"; then - printf 'missing_rubric_scores\n' - return 0 - fi - - weighted_line="$(printf '%s\n' "$quality_block" | grep -E "^[[:space:]]*-[[:space:]]Weighted:" | tail -1 || true)" - weighted="$(assistant_phase_value_after_colon "$weighted_line")" - if [[ -z "$weighted_line" ]] || ! assistant_phase_decimal_in_range "$weighted" "0" "5" \ - || ! assistant_phase_rubric_weighted_score_matches "$rubric_line" "$weighted"; then - printf 'missing_weighted_score\n' - return 0 - fi - - if [[ "$round" -gt 1 ]]; then - delta_line="$(printf '%s\n' "$quality_block" | grep -m1 -E "^[[:space:]]*-[[:space:]]Delta from previous:" || true)" - delta="$(assistant_phase_value_after_colon "$delta_line")" - if [[ -z "$delta_line" ]] || ! assistant_phase_signed_decimal_is_valid "$delta"; then - printf 'missing_delta_from_previous\n' - return 0 - fi - drift_line="$(printf '%s\n' "$quality_block" | grep -m1 -E "^[[:space:]]*-[[:space:]]Drift check:" || true)" - drift_value="$(assistant_phase_value_after_colon "$drift_line")" - if [[ -z "$drift_line" ]] || ! assistant_phase_drift_check_is_valid "$drift_value"; then - printf 'missing_drift_check\n' - return 0 - fi - fi - - final_result_line="$(printf '%s\n' "$final_block" | grep -m1 -E "^[[:space:]]*-[[:space:]]Result:[[:space:]]*(CLEAN|ISSUES_FIXED|HAS_REMAINING_ITEMS)[[:space:]]*$" || true)" - if [[ ! "$final_result_line" =~ Result:[[:space:]]*(CLEAN|ISSUES_FIXED|HAS_REMAINING_ITEMS)[[:space:]]*$ ]]; then - printf 'no_final_result\n' - return 0 - fi - final_result="${BASH_REMATCH[1]}" - - score_progression_line="$(printf '%s\n' "$final_block" | grep -m1 -E "^[[:space:]]*-[[:space:]]Score progression:" || true)" - score_progression="$(assistant_phase_value_after_colon "$score_progression_line")" - if [[ -z "$score_progression_line" ]] || ! assistant_phase_score_progression_is_valid "$score_progression" "$weighted" "$round"; then - printf 'missing_score_progression\n' - return 0 - fi - - if ! observed_weighted_sequence="$(assistant_phase_quality_review_observed_weighted_sequence "$file" "$minimum_review_line" "$quality_review_line" "$round")"; then - if [[ "$round" -gt 1 ]]; then - printf 'missing_delta_from_previous\n' - else - printf 'missing_score_progression\n' - fi - return 0 - fi - - if ! assistant_phase_score_progression_matches_observed_sequence "$score_progression" "$observed_weighted_sequence"; then - printf 'missing_score_progression\n' - return 0 - fi - - if [[ "$round" -gt 1 ]]; then - previous_quality_review_line="$(assistant_phase_previous_quality_review_line_before_line "$file" "$quality_review_line" "$minimum_review_line" || true)" - if [[ -z "$previous_quality_review_line" ]]; then - printf 'missing_delta_from_previous\n' - return 0 - fi - previous_quality_block="$(assistant_phase_review_block_after_line "$file" "$previous_quality_review_line" || true)" - previous_weighted="$(assistant_phase_review_weighted_from_block "$previous_quality_block" || true)" - previous_findings="$(assistant_phase_review_findings_count_from_block "$previous_quality_block" || true)" - if [[ -z "$previous_weighted" || -z "$previous_findings" ]]; then - printf 'missing_delta_from_previous\n' - return 0 - fi - previous_score="$previous_weighted" - if [[ -z "$previous_score" ]] || ! assistant_phase_delta_matches_scores "$delta" "$previous_score" "$weighted"; then - printf 'missing_delta_from_previous\n' - return 0 - fi - if ! assistant_phase_drift_check_matches_movement "$drift_value" "$delta" "$previous_findings" "$current_findings"; then - printf 'missing_drift_check\n' - return 0 - fi - fi - - if [[ "$final_result" == "CLEAN" || "$final_result" == "ISSUES_FIXED" ]]; then - score_x100="$(awk -v score="$weighted" 'BEGIN { printf "%d", score * 100 }')" - if [[ "$score_x100" -lt 400 ]]; then - printf 'weighted_score_below_pass\n' - return 0 - fi - if [[ "$must_fix" -ne 0 || "$should_fix" -ne 0 ]]; then - printf 'unresolved_findings\n' - return 0 - fi - elif [[ "$final_result" == "HAS_REMAINING_ITEMS" ]]; then - if ! assistant_phase_final_result_has_remaining_rationale "$final_block"; then - printf 'missing_remaining_rationale\n' - return 0 - fi - fi - - printf 'complete\n' -} - -assistant_phase_has_learning_controller() { - local file="$1" - grep -qE "^### Learning Controller[[:space:]]*$" "$file" 2>/dev/null -} - -assistant_phase_learning_controller_block() { - local file="$1" - awk ' - /^### Learning Controller[[:space:]]*$/ { - found = 1 - in_block = 1 - next - } - in_block && /^### / { exit } - in_block { print } - END { exit found ? 0 : 1 } - ' "$file" 2>/dev/null -} - -assistant_phase_has_learning_controller_after_line() { - local file="$1" - local minimum_line="$2" - [[ -n "$minimum_line" ]] || return 1 - awk -v minimum_line="$minimum_line" ' - BEGIN { minimum_line += 0 } - NR <= minimum_line { next } - /^### Learning Controller[[:space:]]*$/ { - found = 1 - exit - } - END { exit found ? 0 : 1 } - ' "$file" 2>/dev/null -} - -assistant_phase_learning_controller_block_after_line() { - local file="$1" - local minimum_line="$2" - [[ -n "$minimum_line" ]] || return 1 - awk -v minimum_line="$minimum_line" ' - BEGIN { minimum_line += 0 } - NR <= minimum_line { next } - /^### Learning Controller[[:space:]]*$/ { - found = 1 - in_block = 1 - next - } - in_block && /^### / { exit } - in_block { print } - END { exit found ? 0 : 1 } - ' "$file" 2>/dev/null -} - -assistant_phase_learning_field_value() { - local block="$1" - local label="$2" - printf '%s\n' "$block" | awk -v label="$label" ' - BEGIN { wanted = tolower(label) ":" } - { - line = $0 - sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", line) - low = tolower(line) - if (index(low, wanted) == 1) { - sub(/^[^:]*:[[:space:]]*/, "", line) - sub(/[[:space:]]*$/, "", line) - print line - exit - } - } - ' -} - -assistant_phase_learning_evidence_item_is_valid() { - local value="$1" - local label item_value - - value="$(assistant_phase_trim_value "$value")" - if [[ ! "$value" =~ ^([^:]+):[[:space:]]*(.*)$ ]]; then - return 1 - fi - - label="$(assistant_phase_trim_value "${BASH_REMATCH[1]}")" - label="$(printf '%s' "$label" | tr '[:upper:]' '[:lower:]' | sed -E 's/[[:space:]-]+/_/g')" - case "$label" in - none|review_finding|build_test_failure|user_correction|memory_trend) ;; - *) - return 1 - ;; - esac - - item_value="$(assistant_phase_trim_value "${BASH_REMATCH[2]}")" - ! assistant_phase_value_is_noneish "$item_value" && ! assistant_phase_value_is_bracket_placeholder "$item_value" -} - -assistant_phase_learning_considered_item_is_valid() { - local value="$1" - local item_value - - value="$(assistant_phase_trim_value "$value")" - if assistant_phase_value_is_noneish "$value" || assistant_phase_value_is_bracket_placeholder "$value"; then - return 1 - fi - - if [[ "$value" =~ ^[^:]+:[[:space:]]*(.*)$ ]]; then - item_value="$(assistant_phase_trim_value "${BASH_REMATCH[1]}")" - ! assistant_phase_value_is_noneish "$item_value" && ! assistant_phase_value_is_bracket_placeholder "$item_value" - return $? - fi - - return 0 -} - -assistant_phase_learning_section_has_item() { - local block="$1" - local label="$2" - local validator="${3:-learning_evidence}" - local item - local found=0 - local invalid=0 - - while IFS= read -r item; do - found=1 - case "$validator" in - learning_evidence) - assistant_phase_learning_evidence_item_is_valid "$item" || invalid=1 - ;; - considered) - assistant_phase_learning_considered_item_is_valid "$item" || invalid=1 - ;; - esac - done < <( - printf '%s\n' "$block" | awk -v label="$label" ' - function is_learning_field(line, clean) { - clean = line - sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", clean) - return clean ~ /^(Memory trend checked|Learning evidence reviewed|Review findings considered|Build\/test failures considered|User corrections considered|Durable lesson decision|Persistence evidence|No-save rationale):/ - } - BEGIN { wanted = tolower(label) ":" } - { - line = $0 - clean = line - sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", clean) - if (index(tolower(clean), wanted) == 1) { - in_section = 1 - next - } - if (in_section && is_learning_field($0)) { - exit - } - if (in_section && $0 ~ /^[[:space:]]+[-*][[:space:]]+/) { - item = $0 - sub(/^[[:space:]]*[-*][[:space:]]+/, "", item) - sub(/[[:space:]]*$/, "", item) - print item - } - } - ' - ) - - [[ "$found" -eq 1 && "$invalid" -eq 0 ]] -} - -assistant_phase_learning_missing_reason_key() { - local file="$1" - local status block trend decision persistence no_save_rationale - local spec_pass_line quality_review_line final_result_line - - if ! assistant_phase_is_medium_plus "$file"; then - printf 'complete\n' - return 0 - fi - - status="$(assistant_phase_status "$file" || true)" - if [[ "$status" != *"DOCUMENTING"* ]]; then - printf 'complete\n' - return 0 - fi - - spec_pass_line="$(assistant_phase_latest_spec_review_pass_line "$file" || true)" - if [[ -n "$spec_pass_line" ]]; then - quality_review_line="$(assistant_phase_quality_review_after_line "$file" "$spec_pass_line" || true)" - if [[ -n "$quality_review_line" ]]; then - final_result_line="$(assistant_phase_final_result_heading_line_after_line "$file" "$quality_review_line" || true)" - fi - fi - - if [[ -n "$final_result_line" ]]; then - if ! assistant_phase_has_learning_controller_after_line "$file" "$final_result_line"; then - printf 'no_learning_controller\n' - return 0 - fi - block="$(assistant_phase_learning_controller_block_after_line "$file" "$final_result_line" || true)" - else - if ! assistant_phase_has_learning_controller "$file"; then - printf 'no_learning_controller\n' - return 0 - fi - block="$(assistant_phase_learning_controller_block "$file" || true)" - fi - - trend="$(assistant_phase_trim_value "$(assistant_phase_learning_field_value "$block" "Memory trend checked")")" - case "$trend" in - checked|backend_unavailable|policy_disallowed|not_configured) ;; - *) - printf 'missing_memory_trend_checked\n' - return 0 - ;; - esac - - if ! assistant_phase_learning_section_has_item "$block" "Learning evidence reviewed" "learning_evidence"; then - printf 'missing_learning_evidence_reviewed\n' - return 0 - fi - - if ! assistant_phase_learning_section_has_item "$block" "Review findings considered" "considered"; then - printf 'missing_review_findings_considered\n' - return 0 - fi - - if ! assistant_phase_learning_section_has_item "$block" "Build/test failures considered" "considered"; then - printf 'missing_build_test_failures_considered\n' - return 0 - fi - - if ! assistant_phase_learning_section_has_item "$block" "User corrections considered" "considered"; then - printf 'missing_user_corrections_considered\n' - return 0 - fi - - decision="$(assistant_phase_trim_value "$(assistant_phase_learning_field_value "$block" "Durable lesson decision")")" - case "$decision" in - durable_saved|durable_updated|skipped_not_durable|backend_unavailable|policy_disallowed|refused_sensitive) ;; - *) - printf 'missing_durable_lesson_decision\n' - return 0 - ;; - esac - - persistence="$(assistant_phase_trim_value "$(assistant_phase_learning_field_value "$block" "Persistence evidence")")" - if [[ -z "$persistence" ]]; then - printf 'missing_persistence_evidence\n' - return 0 - fi - - case "$decision" in - durable_saved|durable_updated) - if assistant_phase_value_is_noneish "$persistence" || assistant_phase_value_is_bracket_placeholder "$persistence"; then - printf 'missing_persistence_evidence\n' - return 0 - fi - ;; - skipped_not_durable|backend_unavailable|policy_disallowed|refused_sensitive) - no_save_rationale="$(assistant_phase_trim_value "$(assistant_phase_learning_field_value "$block" "No-save rationale")")" - if assistant_phase_value_is_noneish "$no_save_rationale" || assistant_phase_value_is_bracket_placeholder "$no_save_rationale"; then - printf 'missing_no_save_rationale\n' - return 0 - fi - ;; - esac - - printf 'complete\n' -} - -assistant_phase_review_complete() { - local file="$1" - [[ "$(assistant_phase_review_missing_reason_key "$file")" == "complete" ]] -} - -assistant_phase_latest_qa_final_result() { - local file="$1" - local minimum_line="${2:-0}" - awk -v minimum_line="$minimum_line" ' - function trim(value) { - sub(/^[[:space:]]+/, "", value) - sub(/[[:space:]]+$/, "", value) - return value - } - function template_placeholder(value, low) { - low = tolower(trim(value)) - gsub(/`/, "", low) - return low ~ /^\[[^]]*\]$/ || low ~ /\|/ - } - function normalize_result(value, low) { - if (template_placeholder(value)) return "" - low = tolower(trim(value)) - gsub(/`/, "", low) - if (low ~ /(^|[^[:alnum:]_])not[ _-]?accepted([^[:alnum:]_]|$)/) return "not_accepted" - if (low ~ /(^|[^[:alnum:]_])accepted[ _-]?with[ _-]?concerns([^[:alnum:]_]|$)/) return "accepted_with_concerns" - if (low ~ /(^|[^[:alnum:]_])accepted([^[:alnum:]_]|$)/) return "accepted" - if (low ~ /(^|[^[:alnum:]_])rejected([^[:alnum:]_]|$)/) return "rejected" - if (low ~ /(^|[^[:alnum:]_])blocked([^[:alnum:]_]|$)/) return "blocked" - if (low ~ /(^|[^[:alnum:]_])not[ _-]?required([^[:alnum:]_]|$)/) return "not_required" - return "" - } - function record_result(value) { - latest = normalize_result(value) - } - function compact_final_marker(value, low) { - low = tolower(value) - return low ~ /(final[ _-]?verdict|qa[ _-]?result)/ - } - BEGIN { - minimum_line += 0 - } - NR <= minimum_line { - next - } - /^### QA Evaluation #[0-9]+/ { - latest = "" - next - } - { - line = $0 - sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", line) - low = tolower(line) - if (low ~ /^final verdict[[:space:]]*:/) { - value = line - sub(/^[^:]*:[[:space:]]*/, "", value) - record_result(value) - next - } - if (low ~ /^qa result[[:space:]]*:/) { - value = line - sub(/^[^:]*:[[:space:]]*/, "", value) - record_result(value) - next - } - if (low ~ /^qa evaluator result[[:space:]]*:/) { - value = line - sub(/^[^:]*:[[:space:]]*/, "", value) - if (compact_final_marker(value) || normalize_result(value) != "") { - record_result(value) - } - } - } - END { - if (latest != "") { - print latest - exit 0 - } - exit 1 - } - ' "$file" 2>/dev/null -} - -assistant_phase_qa_final_result_missing_reason_key() { - local file="$1" - local minimum_line="${2:-0}" - local qa_result - - if ! assistant_phase_requires_qa_evaluator "$file"; then - printf 'complete\n' - return 0 - fi - - qa_result="$(assistant_phase_latest_qa_final_result "$file" "$minimum_line" || true)" - case "$qa_result" in - accepted|accepted_with_concerns) - printf 'complete\n' - ;; - rejected) - printf 'qa_rejected\n' - ;; - blocked) - printf 'qa_blocked\n' - ;; - "") - printf 'qa_final_result_missing\n' - ;; - not_accepted) - printf 'qa_not_accepted\n' - ;; - *) - printf 'qa_not_accepted\n' - ;; - esac -} - -assistant_phase_review_missing_reason_key() { - local file="$1" - local spec_pass_line - local quality_review_line - local review_controller_reason - local qa_reason - - if ! assistant_phase_has_spec_review_entry "$file"; then - printf 'no_spec_review\n' - return 0 - fi - - spec_pass_line="$(assistant_phase_latest_spec_review_pass_line "$file" || true)" - if [[ -z "$spec_pass_line" ]]; then - printf 'spec_not_pass\n' - return 0 - fi - - quality_review_line="$(assistant_phase_quality_review_after_line "$file" "$spec_pass_line" || true)" - if [[ -z "$quality_review_line" ]]; then - printf 'no_quality_review\n' - return 0 - fi - - if assistant_phase_is_medium_plus "$file"; then - review_controller_reason="$(assistant_phase_review_controller_missing_reason_key "$file" "$quality_review_line" "$spec_pass_line")" - if [[ "$review_controller_reason" != "complete" ]]; then - printf '%s\n' "$review_controller_reason" - return 0 - fi - assistant_phase_qa_final_result_missing_reason_key "$file" "$quality_review_line" - return 0 - fi - - if ! assistant_phase_final_result_after_line "$file" "$quality_review_line" >/dev/null; then - printf 'no_final_result\n' - return 0 - fi - - qa_reason="$(assistant_phase_qa_final_result_missing_reason_key "$file" "$quality_review_line")" - if [[ "$qa_reason" != "complete" ]]; then - printf '%s\n' "$qa_reason" - return 0 - fi - - printf 'complete\n' -} - -assistant_phase_agent_home() { - if [[ -n "${CODEX_PROJECT_DIR:-}" ]]; then - printf '%s/.codex\n' "$HOME" - elif [[ -n "${GEMINI_PROJECT_DIR:-}" ]]; then - printf '%s/.gemini\n' "$HOME" - else - printf '%s/.claude\n' "$HOME" - fi -} - -assistant_phase_metrics_file() { - printf '%s/memory/metrics/workflow-metrics.jsonl\n' "$(assistant_phase_agent_home)" -} - -assistant_phase_has_metrics_today() { - local metrics_file - local today - metrics_file="$(assistant_phase_metrics_file)" - today="$(date +%Y-%m-%d)" - - [[ -f "$metrics_file" ]] || return 1 - grep -q "\"date\":\"$today\"" "$metrics_file" 2>/dev/null -} - assistant_phase_subagent_mode() { assistant_phase_scalar_field "$1" "Subagent execution mode" } @@ -1435,774 +843,16 @@ assistant_phase_labeled_evidence_value_is_placeholder() { return 1 } -assistant_phase_is_codex_task() { - local file="$1" - [[ "$file" == */.codex/task.md || "$file" == .codex/task.md ]] -} - -assistant_phase_subagent_events_file() { - local file="$1" - printf '%s/subagent-events.jsonl\n' "$(dirname "$file")" -} - -assistant_phase_role_agent_pattern() { - case "$1" in - "Code Mapper") printf 'code-mapper|codemapper|Code Mapper' ;; - "Explorer") printf 'explorer|Explorer' ;; - "Architect") printf 'architect|Architect' ;; - "Code Writer") printf 'code-writer|codewriter|Code Writer' ;; - "Builder/Tester") printf 'builder-tester|builder/tester|Builder/Tester' ;; - "Code Reviewer") printf 'code-reviewer|codereviewer|Code Reviewer|reviewer|Reviewer' ;; - "QA Evaluator") printf 'qa-evaluator|qaevaluator|QA Evaluator' ;; - "Reviewer") printf 'reviewer|Reviewer|code-reviewer|codereviewer|Code Reviewer' ;; - *) printf '%s' "$1" ;; - esac -} - -assistant_phase_json_field_value() { - local json_line="$1" - local field="$2" - printf '%s\n' "$json_line" | sed -n 's/.*"'"$field"'":"\([^"]*\)".*/\1/p' -} - -assistant_phase_event_role_matches() { - local json_line="$1" - local role="$2" - local pattern name agent_type agent_name - pattern="$(assistant_phase_role_agent_pattern "$role")" - agent_type="$(assistant_phase_json_field_value "$json_line" "agent_type" | tr '[:upper:]' '[:lower:]')" - agent_name="$(assistant_phase_json_field_value "$json_line" "agent_name" | tr '[:upper:]' '[:lower:]')" - IFS='|' read -r -a names <<< "$pattern" - for name in "${names[@]}"; do - name="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]')" - [[ "$agent_type" == "$name" || "$agent_name" == "$name" ]] && return 0 - done - return 1 -} - -assistant_phase_codex_role_event_ids() { - local file="$1" - local role="$2" - local event="$3" - local events_file line id - events_file="$(assistant_phase_subagent_events_file "$file")" - [[ -f "$events_file" ]] || return 1 - while IFS= read -r line; do - [[ "$line" == *"\"event\":\"$event\""* ]] || continue - assistant_phase_event_role_matches "$line" "$role" || continue - id="$(assistant_phase_json_field_value "$line" "agent_id")" - [[ -n "$id" ]] || continue - printf '%s\n' "$id" - done < "$events_file" -} - -assistant_phase_journal_mentions_agent_id() { - local file="$1" - local role="$2" - local agent_id="$3" - local dispatch result combined - [[ -n "$agent_id" ]] || return 1 - dispatch="$(assistant_phase_labeled_evidence_value "$file" "$role dispatch")" - result="$(assistant_phase_labeled_evidence_value "$file" "$role result")" - combined="$dispatch -$result" - printf '%s\n' "$combined" | assistant_phase_text_mentions_agent_id_token "$agent_id" -} - -assistant_phase_text_mentions_agent_id_token() { - local agent_id="$1" - [[ -n "$agent_id" ]] || return 1 - awk -v needle="$agent_id" ' - function is_token_char(ch) { - return ch ~ /^[[:alnum:]_-]$/ - } - { - start = 1 - while ((relative = index(substr($0, start), needle)) > 0) { - absolute = start + relative - 1 - before = absolute > 1 ? substr($0, absolute - 1, 1) : "" - after_pos = absolute + length(needle) - after = after_pos <= length($0) ? substr($0, after_pos, 1) : "" - if (!is_token_char(before) && !is_token_char(after)) { - found = 1 - exit - } - start = absolute + length(needle) - } - } - END { - exit found ? 0 : 1 - } - ' -} - -assistant_phase_has_role_event_pair_evidence() { - local file="$1" - local role="$2" - local start_id - while IFS= read -r start_id; do - [[ -n "$start_id" ]] || continue - assistant_phase_journal_mentions_agent_id "$file" "$role" "$start_id" || continue - if assistant_phase_codex_role_event_ids "$file" "$role" "SubagentStop" | grep -Fxq -- "$start_id"; then - return 0 - fi - done < <(assistant_phase_codex_role_event_ids "$file" "$role" "SubagentStart" || true) - return 1 -} - -assistant_phase_has_role_start_event_evidence() { - local file="$1" - local role="$2" - local start_id - while IFS= read -r start_id; do - [[ -n "$start_id" ]] || continue - assistant_phase_journal_mentions_agent_id "$file" "$role" "$start_id" && return 0 - done < <(assistant_phase_codex_role_event_ids "$file" "$role" "SubagentStart" || true) - return 1 -} - -assistant_phase_has_role_stop_event_evidence() { - assistant_phase_has_role_event_pair_evidence "$1" "$2" -} - -assistant_phase_role_evidence_labels() { - case "$1" in - "Code Reviewer") - printf 'Code Reviewer\nReviewer\n' - ;; - "Reviewer") - printf 'Code Reviewer\nReviewer\n' - ;; - *) - printf '%s\n' "$1" - ;; - esac -} - -assistant_phase_has_single_role_dispatch_result_evidence() { - local file="$1" - local role="$2" - - assistant_phase_has_labeled_evidence "$file" "$role dispatch" \ - && assistant_phase_has_labeled_evidence "$file" "$role result" \ - || return 1 - - # Codex task journals are not sufficient proof by themselves: Codex can write - # fake dispatch text after doing work inline. Require real lifecycle evidence - # captured by SubagentStart/SubagentStop hooks for delegated Codex roles. - if assistant_phase_is_codex_task "$file"; then - assistant_phase_has_role_start_event_evidence "$file" "$role" \ - && assistant_phase_has_role_stop_event_evidence "$file" "$role" - return $? +assistant_phase_source_gate_module() { + local module="$ASSISTANT_PHASE_GATES_DIR/workflow-phase-gates.d/$1" + if [[ -f "$module" ]]; then + . "$module" fi - - return 0 -} - -assistant_phase_has_role_dispatch_result_evidence() { - local file="$1" - local role="$2" - local evidence_role - - while IFS= read -r evidence_role; do - [[ -n "$evidence_role" ]] || continue - assistant_phase_has_single_role_dispatch_result_evidence "$file" "$evidence_role" && return 0 - done < <(assistant_phase_role_evidence_labels "$role") - - return 1 -} - -assistant_phase_direct_fallback_reason_valid() { - local file="$1" - grep -qiE "^[[:space:]]*[-*]?[[:space:]]*Direct fallback reason:[[:space:]]*(authorization_denied|subagents_unavailable|policy_disallowed)([[:space:]]|$)" "$file" 2>/dev/null -} - -assistant_phase_has_single_role_equivalent_evidence() { - local file="$1" - local role="$2" - assistant_phase_has_labeled_evidence "$file" "$role direct evidence" -} - -assistant_phase_has_role_equivalent_evidence() { - local file="$1" - local role="$2" - local evidence_role - - while IFS= read -r evidence_role; do - [[ -n "$evidence_role" ]] || continue - assistant_phase_has_single_role_equivalent_evidence "$file" "$evidence_role" && return 0 - done < <(assistant_phase_role_evidence_labels "$role") - - return 1 -} - -assistant_phase_has_per_slice_dispatch_evidence() { - local file="$1" - ! assistant_phase_is_medium_plus "$file" && return 0 - assistant_phase_has_labeled_evidence "$file" "Per-slice dispatch evidence" } -assistant_phase_requires_qa_evaluator() { - local file="$1" - awk ' - function trim(value) { - sub(/^[[:space:]]+/, "", value) - sub(/[[:space:]]+$/, "", value) - return value - } - function explicit_not_required(value, low) { - low = tolower(trim(value)) - return low ~ /^(n\/a|na|not_required|not required|not-required|not-applicable|not_applicable|not applicable)([[:space:][:punct:]]|$)/ - } - function bracket_placeholder(value) { - value = trim(value) - return value ~ /^\[[^]]*\]$/ - } - function qa_mode_required_line(line, low) { - low = tolower(trim(line)) - sub(/^[[:space:]]*[-*][[:space:]]*/, "", low) - return low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]*[:=][[:space:]]*required([[:space:][:punct:]]|$)/ || - low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]+required([[:space:][:punct:]]|$)/ - } - function qa_mode_not_required_line(line, low) { - low = tolower(trim(line)) - sub(/^[[:space:]]*[-*][[:space:]]*/, "", low) - return low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]*[:=][[:space:]]*(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ || - low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]+(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ - } - function qa_section_mode_required_line(line, value, low) { - line = trim(line) - sub(/^[[:space:]]*[-*][[:space:]]*/, "", line) - low = tolower(line) - if (low !~ /^mode[[:space:]]*:/) { - return 0 - } - value = line - sub(/^[^:]*:[[:space:]]*/, "", value) - value = trim(value) - if (value ~ /\|/ || bracket_placeholder(value)) { - return 0 - } - low = tolower(value) - return low ~ /^required([[:space:][:punct:]]|$)/ - } - function qa_section_mode_not_required_line(line, value, low) { - line = trim(line) - sub(/^[[:space:]]*[-*][[:space:]]*/, "", line) - low = tolower(line) - if (low !~ /^mode[[:space:]]*:/) { - return 0 - } - value = line - sub(/^[^:]*:[[:space:]]*/, "", value) - value = trim(value) - if (value ~ /\|/ || bracket_placeholder(value)) { - return 0 - } - low = tolower(value) - return low ~ /^(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ - } - function line_marks_not_required(line, low) { - low = tolower(line) - return low ~ /(^|[[:space:][:punct:]])(n\/a|not_required|not required|not-required|not-applicable|not_applicable|not applicable)([[:space:][:punct:]]|$)/ - } - function qa_marker(line, low) { - low = tolower(line) - return low ~ /(qa[ _-]?evaluator|qaevaluator|qa[ _-]?evaluation|qa_evaluation_mode|qa[ _-]?loop)/ - } - function scan_required_line(line) { - if (qa_marker(line) && !line_marks_not_required(line)) { - found = 1 - } - } - function scan_qa_label(line, value, low) { - sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", line) - low = tolower(line) - if (low ~ /^(qa evaluator dispatch|qa evaluator result|qa evaluator direct evidence):/) { - value = line - sub(/^[^:]*:[[:space:]]*/, "", value) - if (!bracket_placeholder(value) && !explicit_not_required(value)) { - found = 1 - } - } else if (low ~ /^qa evaluator dispatch\/result\/direct evidence:/) { - value = line - sub(/^[^:]*:[[:space:]]*/, "", value) - if (!bracket_placeholder(value) && !explicit_not_required(value)) { - found = 1 - } - } - } - { - line = $0 - low = tolower(line) - if (low ~ /^###[[:space:]]*qa evaluation([[:space:]#]|$)/) { - in_qa_evaluation = 1 - in_required_agents = 0 - in_required_gates = 0 - next - } - if (in_qa_evaluation && low ~ /^#+[[:space:]]+/) { - in_qa_evaluation = 0 - } - if (in_qa_evaluation) { - if (qa_section_mode_required_line(line)) { - required_mode = 1 - found = 1 - next - } - if (qa_section_mode_not_required_line(line)) { - not_required_mode = 1 - next - } - } - if (qa_mode_required_line(line)) { - required_mode = 1 - found = 1 - next - } - if (qa_mode_not_required_line(line)) { - not_required_mode = 1 - next - } - - scan_qa_label(line) - - if (low ~ /^required agents:[[:space:]]*$/) { - in_required_agents = 1 - in_required_gates = 0 - next - } - if (low ~ /^required gates:[[:space:]]*$/) { - in_required_gates = 1 - in_required_agents = 0 - next - } - if (low ~ /^required agents:[[:space:]]*.+$/ || - low ~ /^required gates:[[:space:]]*.+$/) { - scan_required_line(line) - next - } - if ((in_required_agents || in_required_gates) && line ~ /^[[:space:]]*-[[:space:]]+/) { - scan_required_line(line) - next - } - if ((in_required_agents || in_required_gates) && line ~ /^[^[:space:]-]/) { - in_required_agents = 0 - in_required_gates = 0 - } - } - END { - if (not_required_mode && !required_mode) { - exit 1 - } - exit found ? 0 : 1 - } - ' "$file" 2>/dev/null -} - -assistant_phase_required_subagent_roles() { - local file="$1" - local status mode qa_required - status="$(assistant_phase_status "$file" | tr '[:upper:]' '[:lower:]' || true)" - mode="$(assistant_phase_subagent_mode "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" - qa_required="$(assistant_phase_requires_qa_evaluator "$file" && printf yes || printf no)" - awk -v is_medium_plus="$(assistant_phase_is_medium_plus "$file" && printf yes || printf no)" -v status="$status" -v mode="$mode" -v qa_required="$qa_required" ' - function emit(role) { - if (!seen[role]) { - seen[role] = 1 - print role - } - } - function scan(line, low) { - low = tolower(line) - if (low ~ /code mapper|code-mapper/) emit("Code Mapper") - if (low ~ /explorer/) emit("Explorer") - if (low ~ /architect/) emit("Architect") - if (low ~ /code writer|code-writer/) emit("Code Writer") - if (low ~ /builder\/tester|builder-tester/) emit("Builder/Tester") - if (low ~ /qa[ _-]?evaluator|qaevaluator/) emit("QA Evaluator") - if (low ~ /code[ _-]?reviewer|codereviewer/) { - emit("Code Reviewer") - } else if (low ~ /(^|[^[:alnum:]_-])reviewer([^[:alnum:]_-]|$)/) { - emit("Code Reviewer") - } - } - BEGIN { - # Medium+ discovery always requires a Code Mapper context map once - # subagent execution mode has been resolved. Add the role from task - # size even if the journal forgot to list it. - if (mode ~ /^(delegated|direct_fallback)$/ && is_medium_plus == "yes") emit("Code Mapper") - # Once a delegated/fallback task is in Review/Document, the review - # role is required even for no-op/no-code-change outcomes; otherwise - # "review phase" can be satisfied inline while claiming delegated mode. - if (mode ~ /^(delegated|direct_fallback)$/ && status ~ /(reviewing|documenting)/) emit("Code Reviewer") - if (qa_required == "yes") emit("QA Evaluator") - } - /^Required agents:[[:space:]]*$/ { in_required = 1; next } - /^Required agents:[[:space:]]*(.+)$/ { - scan($0) - next - } - in_required && /^[[:space:]]*-[[:space:]]+/ { - scan($0) - next - } - in_required && /^[^[:space:]-]/ { in_required = 0 } - ' "$file" 2>/dev/null -} - -assistant_phase_requires_subagent_roles() { - [[ -n "$(assistant_phase_required_subagent_roles "$1")" ]] -} - -assistant_phase_qa_evaluator_subagent_evidence_is_actionable() { - [[ "$(assistant_phase_review_missing_reason_key "$1")" == "complete" ]] -} - -assistant_phase_role_reason_slug() { - printf '%s' "$1" | tr '[:upper:]/ ' '[:lower:]__' -} - -assistant_phase_reason_missing_field() { - local gate="$1" - local key="$2" - - case "$gate:$key" in - plan_gate:no_plan) printf 'No plan found; Plan approval: yes\n' ;; - plan_gate:plan_not_approved) printf 'Plan exists but is not approved; Plan approval: yes\n' ;; - - subagent_evidence_gate:authorization_required_unresolved) printf 'Subagent policy state / Subagent execution mode\n' ;; - subagent_evidence_gate:delegated_missing_code_mapper) printf 'Code Mapper dispatch/result evidence\n' ;; - subagent_evidence_gate:delegated_missing_explorer) printf 'Explorer dispatch/result evidence\n' ;; - subagent_evidence_gate:delegated_missing_architect) printf 'Architect dispatch/result evidence\n' ;; - subagent_evidence_gate:delegated_missing_code_writer) printf 'Code Writer dispatch/result evidence\n' ;; - subagent_evidence_gate:delegated_missing_builder_tester) printf 'Builder/Tester dispatch/result evidence\n' ;; - subagent_evidence_gate:delegated_missing_code_reviewer) printf 'Code Reviewer dispatch/result evidence during Review\n' ;; - subagent_evidence_gate:delegated_missing_qa_evaluator) printf 'QA Evaluator dispatch/result evidence\n' ;; - subagent_evidence_gate:delegated_missing_per_slice) printf 'Per-slice dispatch evidence\n' ;; - subagent_evidence_gate:direct_fallback_invalid_policy_state) printf 'Subagent policy state for direct_fallback\n' ;; - subagent_evidence_gate:direct_fallback_missing_reason) printf 'Direct fallback reason\n' ;; - subagent_evidence_gate:direct_fallback_missing_code_mapper) printf 'Code Mapper direct evidence\n' ;; - subagent_evidence_gate:direct_fallback_missing_explorer) printf 'Explorer direct evidence\n' ;; - subagent_evidence_gate:direct_fallback_missing_architect) printf 'Architect direct evidence\n' ;; - subagent_evidence_gate:direct_fallback_missing_code_writer) printf 'Code Writer direct evidence\n' ;; - subagent_evidence_gate:direct_fallback_missing_builder_tester) printf 'Builder/Tester direct evidence\n' ;; - subagent_evidence_gate:direct_fallback_missing_code_reviewer) printf 'Code Reviewer direct evidence during Review\n' ;; - subagent_evidence_gate:direct_fallback_missing_qa_evaluator) printf 'QA Evaluator direct evidence\n' ;; - subagent_evidence_gate:not_applicable_with_required_roles) printf 'Subagent execution mode\n' ;; - subagent_evidence_gate:unknown_execution_mode) printf 'Subagent execution mode\n' ;; - - review_gate:no_spec_review) printf 'no Spec Review; ### Spec Review #N\n' ;; - review_gate:spec_not_pass) printf 'latest Spec Review result not PASS; - Result: PASS\n' ;; - review_gate:no_quality_review) printf 'no Quality Review after latest Spec Review PASS; ### Quality Review #N\n' ;; - review_gate:no_final_result) printf 'no Final Result; ### Final result with - Result: CLEAN|ISSUES_FIXED|HAS_REMAINING_ITEMS\n' ;; - review_gate:qa_rejected) printf 'QA Evaluation final verdict/result accepted or accepted_with_concerns\n' ;; - review_gate:qa_blocked) printf 'QA Evaluation final verdict/result accepted or accepted_with_concerns\n' ;; - review_gate:qa_final_result_missing) printf 'QA Evaluation final verdict/result\n' ;; - review_gate:qa_not_accepted) printf 'QA Evaluation final verdict/result accepted or accepted_with_concerns\n' ;; - review_gate:missing_review_round) printf '%s\n' '- Round: N of 10' ;; - review_gate:round_overflow) printf '%s\n' '- Round: N of 10 with N between 1 and 10' ;; - review_gate:missing_findings_summary) printf '%s\n' '- Found this round: X must-fix, Y should-fix, Z nits' ;; - review_gate:missing_rubric_scores) printf '%s\n' '- Rubric: correctness, code_quality/quality, architecture, security, test_coverage/coverage scores' ;; - review_gate:missing_weighted_score) printf 'missing or mismatched weighted score; - Weighted: N.NN\n' ;; - review_gate:missing_delta_from_previous) printf '%s\n' '- Delta from previous: +/-N.NN' ;; - review_gate:missing_drift_check) printf '%s\n' '- Drift check: GENUINE or REGRESSION' ;; - review_gate:missing_score_progression) printf '%s\n' '- Score progression: ...' ;; - review_gate:weighted_score_below_pass) printf '%s\n' '- Weighted: score >= 4.00 for CLEAN/ISSUES_FIXED' ;; - review_gate:unresolved_findings) printf '0 must-fix and 0 should-fix findings for CLEAN/ISSUES_FIXED\n' ;; - review_gate:missing_remaining_rationale) printf '%s\n' '- Remaining items: or - Blocker: with concrete evidence and owner' ;; - - metrics_gate:missing_metrics_today) printf 'workflow metrics JSONL entry for today\n' ;; - - learning_gate:no_learning_controller) printf '### Learning Controller\n' ;; - learning_gate:missing_memory_trend_checked) printf '%s\n' '- Memory trend checked: checked|backend_unavailable|policy_disallowed|not_configured' ;; - learning_gate:missing_learning_evidence_reviewed) printf '%s\n' '- Learning evidence reviewed: list item' ;; - learning_gate:missing_review_findings_considered) printf '%s\n' '- Review findings considered: list item' ;; - learning_gate:missing_build_test_failures_considered) printf '%s\n' '- Build/test failures considered: list item' ;; - learning_gate:missing_user_corrections_considered) printf '%s\n' '- User corrections considered: list item' ;; - learning_gate:missing_durable_lesson_decision) printf '%s\n' '- Durable lesson decision: durable_saved|durable_updated|skipped_not_durable|backend_unavailable|policy_disallowed|refused_sensitive' ;; - learning_gate:missing_persistence_evidence) printf '%s\n' '- Persistence evidence: concrete saved/updated memory evidence' ;; - learning_gate:missing_no_save_rationale) printf '%s\n' '- No-save rationale: concrete reason' ;; - - *) printf 'required gate field\n' ;; - esac -} - -assistant_phase_reason_action() { - local gate="$1" - local key="$2" - - case "$gate:$key" in - plan_gate:no_plan) printf 'run PLAN, get user approval, then record Plan approval: yes\n' ;; - plan_gate:plan_not_approved) printf 'present the plan, wait for approval, then record Plan approval: yes\n' ;; - - subagent_evidence_gate:authorization_required_unresolved) printf 'ask once for subagent authorization or denial before continuing delegated workflow work\n' ;; - subagent_evidence_gate:delegated_missing_code_mapper) printf 'record Code Mapper dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; - subagent_evidence_gate:delegated_missing_explorer) printf 'record Explorer dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; - subagent_evidence_gate:delegated_missing_architect) printf 'record Architect dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; - subagent_evidence_gate:delegated_missing_code_writer) printf 'record Code Writer dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; - subagent_evidence_gate:delegated_missing_builder_tester) printf 'record Builder/Tester dispatch/result with real lifecycle evidence when Codex is delegated\n' ;; - subagent_evidence_gate:delegated_missing_code_reviewer) printf 'record Code Reviewer during Review; legacy Reviewer labels are compatibility routing only\n' ;; - subagent_evidence_gate:delegated_missing_qa_evaluator) printf 'run QA Evaluator after build/test and code-review evidence, then record dispatch/result\n' ;; - subagent_evidence_gate:delegated_missing_per_slice) printf 'record Per-slice dispatch evidence for the implementation slice\n' ;; - subagent_evidence_gate:direct_fallback_invalid_policy_state) printf 'set policy state to authorization_denied, subagents_unavailable, or policy_disallowed\n' ;; - subagent_evidence_gate:direct_fallback_missing_reason) printf 'record Direct fallback reason: authorization_denied|subagents_unavailable|policy_disallowed\n' ;; - subagent_evidence_gate:direct_fallback_missing_code_mapper) printf 'record Code Mapper direct evidence\n' ;; - subagent_evidence_gate:direct_fallback_missing_explorer) printf 'record Explorer direct evidence\n' ;; - subagent_evidence_gate:direct_fallback_missing_architect) printf 'record Architect direct evidence\n' ;; - subagent_evidence_gate:direct_fallback_missing_code_writer) printf 'record Code Writer direct evidence\n' ;; - subagent_evidence_gate:direct_fallback_missing_builder_tester) printf 'record Builder/Tester direct evidence\n' ;; - subagent_evidence_gate:direct_fallback_missing_code_reviewer) printf 'record Code Reviewer direct evidence during Review\n' ;; - subagent_evidence_gate:direct_fallback_missing_qa_evaluator) printf 'record QA Evaluator direct evidence after build/test and code-review evidence\n' ;; - subagent_evidence_gate:not_applicable_with_required_roles) printf 'set execution mode to delegated or direct_fallback and record matching evidence\n' ;; - subagent_evidence_gate:unknown_execution_mode) printf 'set Subagent execution mode to delegated, direct_fallback, or not_applicable with valid evidence\n' ;; - - review_gate:no_spec_review) printf 'run Spec Review first and record PASS/FAIL plus scope, mismatch, and required-fix fields\n' ;; - review_gate:spec_not_pass) printf 'fix spec gaps and rerun Spec Review until the latest structured result is PASS\n' ;; - review_gate:no_quality_review) printf 'run assistant-review Stage 2 and append Quality Review after latest Spec Review PASS\n' ;; - review_gate:no_final_result) printf 'finish the review loop and write the Final Result summary after Quality Review\n' ;; - review_gate:qa_rejected) printf 'fix QA findings, rerun QA Evaluation evidence, and record accepted or accepted_with_concerns\n' ;; - review_gate:qa_blocked) printf 'resolve the QA blocker, rerun QA Evaluation evidence, and record accepted or accepted_with_concerns\n' ;; - review_gate:qa_final_result_missing) printf 'run or rerun QA Evaluation evidence and record accepted or accepted_with_concerns\n' ;; - review_gate:qa_not_accepted) printf 'fix or rerun QA Evaluation evidence until accepted or accepted_with_concerns\n' ;; - review_gate:missing_review_round) printf 'add - Round: N of 10 to the latest Quality Review\n' ;; - review_gate:round_overflow) printf 'repair the Quality Review heading/round so N is 1..10 and max is 10\n' ;; - review_gate:missing_findings_summary) printf 'add the latest must-fix/should-fix/nits finding counts\n' ;; - review_gate:missing_rubric_scores) printf 'add numeric 0..5 rubric scores to the latest Quality Review\n' ;; - review_gate:missing_weighted_score) printf 'add a numeric - Weighted: N.NN matching the rubric formula; fix any mismatched weighted score\n' ;; - review_gate:missing_delta_from_previous) printf 'add a valid delta from the previous Quality Review round\n' ;; - review_gate:missing_drift_check) printf 'add a valid drift classification for review rounds after round 1\n' ;; - review_gate:missing_score_progression) printf 'add Score progression matching the observed Quality Review weighted sequence\n' ;; - review_gate:weighted_score_below_pass) printf 'improve low-scoring dimensions and rerun Quality Review before CLEAN/ISSUES_FIXED\n' ;; - review_gate:unresolved_findings) printf 'fix or explicitly carry remaining must-fix/should-fix findings through the loop\n' ;; - review_gate:missing_remaining_rationale) printf 'add concrete remaining-item or blocker rationale with evidence and owner\n' ;; - - metrics_gate:missing_metrics_today) printf 'append today'\''s workflow metrics JSONL entry before stopping\n' ;; - - learning_gate:no_learning_controller) printf 'add the canonical Learning Controller block after Final Result\n' ;; - learning_gate:missing_memory_trend_checked) printf 'record one valid Memory trend checked value\n' ;; - learning_gate:missing_learning_evidence_reviewed) printf 'add reviewed learning evidence or none with a reason\n' ;; - learning_gate:missing_review_findings_considered) printf 'add review finding consideration or none with a reason\n' ;; - learning_gate:missing_build_test_failures_considered) printf 'add build/test failure consideration or none with a reason\n' ;; - learning_gate:missing_user_corrections_considered) printf 'add user correction consideration or none with a reason\n' ;; - learning_gate:missing_durable_lesson_decision) printf 'record one valid durable lesson decision\n' ;; - learning_gate:missing_persistence_evidence) printf 'add concrete memory_reflect, memory_add_insight, or backend evidence for saved/updated lessons\n' ;; - learning_gate:missing_no_save_rationale) printf 'add a concrete no-save rationale when no durable write occurred\n' ;; - - *) printf 'complete the required gate evidence before stopping\n' ;; - esac -} - -assistant_phase_subagent_reason_role() { - local key="$1" - - case "$key" in - *_code_mapper) printf 'Code Mapper\n' ;; - *_explorer) printf 'Explorer\n' ;; - *_architect) printf 'Architect\n' ;; - *_code_writer) printf 'Code Writer\n' ;; - *_builder_tester) printf 'Builder/Tester\n' ;; - *_code_reviewer) printf 'Code Reviewer\n' ;; - *_qa_evaluator) printf 'QA Evaluator\n' ;; - delegated_missing_per_slice) printf 'Per-slice\n' ;; - *) printf '\n' ;; - esac -} - -assistant_phase_status_has_any() { - local status="$1" - shift - local phase - - for phase in "$@"; do - [[ "$status" == *"$phase"* ]] && return 0 - done - return 1 -} - -assistant_phase_subagent_reason_is_current_phase_actionable() { - local key="$1" - local status="$2" - local role - - case "$key" in - complete) - return 1 - ;; - authorization_required_unresolved|direct_fallback_invalid_policy_state|direct_fallback_missing_reason|not_applicable_with_required_roles|unknown_execution_mode) - assistant_phase_status_has_any "$status" DISCOVERING DECOMPOSING PLANNING BUILDING VERIFYING REVIEWING DOCUMENTING - return $? - ;; - esac - - role="$(assistant_phase_subagent_reason_role "$key")" - case "$role" in - "Code Mapper"|"Explorer"|"Architect") - assistant_phase_status_has_any "$status" DISCOVERING DECOMPOSING PLANNING - ;; - "Code Writer"|"Builder/Tester"|"Per-slice") - assistant_phase_status_has_any "$status" BUILDING VERIFYING - ;; - "Code Reviewer"|"QA Evaluator") - assistant_phase_status_has_any "$status" REVIEWING - ;; - *) - return 1 - ;; - esac -} - -assistant_phase_subagent_warning_action() { - local key="$1" - local status="$2" - local field action - - assistant_phase_subagent_reason_is_current_phase_actionable "$key" "$status" || return 1 - field="$(assistant_phase_reason_missing_field "subagent_evidence_gate" "$key")" - action="$(assistant_phase_reason_action "subagent_evidence_gate" "$key")" - printf 'missing=%s action=%s\n' "$field" "$action" -} - -assistant_phase_subagent_warning_reason_key() { - local file="$1" - local key="$2" - local status="$3" - local mode policy_state roles role candidate - - if [[ "$key" == *_qa_evaluator ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then - key="complete" - fi - - if assistant_phase_subagent_reason_is_current_phase_actionable "$key" "$status"; then - printf '%s\n' "$key" - return 0 - fi - - mode="$(assistant_phase_subagent_mode "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" - policy_state="$(assistant_phase_subagent_policy_state "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" - roles="$(assistant_phase_required_subagent_roles "$file")" - - case "$mode" in - delegated) - while IFS= read -r role; do - [[ -n "$role" ]] || continue - if ! assistant_phase_has_role_dispatch_result_evidence "$file" "$role"; then - candidate="delegated_missing_$(assistant_phase_role_reason_slug "$role")" - if [[ "$candidate" == *_qa_evaluator ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then - continue - fi - if assistant_phase_subagent_reason_is_current_phase_actionable "$candidate" "$status"; then - printf '%s\n' "$candidate" - return 0 - fi - fi - done <<< "$roles" - if printf '%s\n' "$roles" | grep -Eq '^(Code Writer|Builder/Tester)$' \ - && ! assistant_phase_has_per_slice_dispatch_evidence "$file"; then - candidate="delegated_missing_per_slice" - if assistant_phase_subagent_reason_is_current_phase_actionable "$candidate" "$status"; then - printf '%s\n' "$candidate" - return 0 - fi - fi - ;; - direct_fallback) - if [[ "$policy_state" != "authorization_denied" && "$policy_state" != "subagents_unavailable" && "$policy_state" != "policy_disallowed" ]]; then - return 1 - fi - if ! assistant_phase_direct_fallback_reason_valid "$file"; then - return 1 - fi - while IFS= read -r role; do - [[ -n "$role" ]] || continue - if ! assistant_phase_has_role_equivalent_evidence "$file" "$role"; then - candidate="direct_fallback_missing_$(assistant_phase_role_reason_slug "$role")" - if [[ "$candidate" == *_qa_evaluator ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then - continue - fi - if assistant_phase_subagent_reason_is_current_phase_actionable "$candidate" "$status"; then - printf '%s\n' "$candidate" - return 0 - fi - fi - done <<< "$roles" - ;; - esac - - return 1 -} - -assistant_phase_subagent_evidence_missing_reason_key() { - local file="$1" - local mode policy_state role roles - mode="$(assistant_phase_subagent_mode "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" - policy_state="$(assistant_phase_subagent_policy_state "$file" | tr '[:upper:]' '[:lower:]' | xargs 2>/dev/null || true)" - roles="$(assistant_phase_required_subagent_roles "$file")" - - # Authorization-required is a wait state, not an execution mode. If a task - # has reached an active workflow phase while authorization is unresolved, - # block before it can silently complete work inline. - if [[ "$policy_state" == "authorization_required" ]]; then - printf 'authorization_required_unresolved\n' - return 0 - fi - - # Strict subagent evidence applies whenever workflow subagent roles are - # declared, not only when source code changed. Discovery/review-only work can - # legitimately skip Code Writer and Builder/Tester, but delegated Code Mapper, - # Explorer, Architect, or Code Reviewer responsibilities still need evidence. - if [[ -z "$roles" ]]; then - printf 'complete\n' - return 0 - fi - - case "$mode" in - delegated) - while IFS= read -r role; do - [[ -n "$role" ]] || continue - if [[ "$role" == "QA Evaluator" ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then - continue - fi - if ! assistant_phase_has_role_dispatch_result_evidence "$file" "$role"; then - printf 'delegated_missing_%s\n' "$(assistant_phase_role_reason_slug "$role")" - return 0 - fi - done <<< "$roles" - if printf '%s\n' "$roles" | grep -Eq '^(Code Writer|Builder/Tester)$' \ - && ! assistant_phase_has_per_slice_dispatch_evidence "$file"; then - printf 'delegated_missing_per_slice\n' - return 0 - fi - ;; - direct_fallback) - if [[ "$policy_state" != "authorization_denied" && "$policy_state" != "subagents_unavailable" && "$policy_state" != "policy_disallowed" ]]; then - printf 'direct_fallback_invalid_policy_state\n' - return 0 - fi - if ! assistant_phase_direct_fallback_reason_valid "$file"; then - printf 'direct_fallback_missing_reason\n' - return 0 - fi - while IFS= read -r role; do - [[ -n "$role" ]] || continue - if [[ "$role" == "QA Evaluator" ]] && ! assistant_phase_qa_evaluator_subagent_evidence_is_actionable "$file"; then - continue - fi - if ! assistant_phase_has_role_equivalent_evidence "$file" "$role"; then - printf 'direct_fallback_missing_%s\n' "$(assistant_phase_role_reason_slug "$role")" - return 0 - fi - done <<< "$roles" - ;; - not_applicable|"") - printf 'not_applicable_with_required_roles\n' - return 0 - ;; - *) - printf 'unknown_execution_mode\n' - return 0 - ;; - esac - - printf 'complete\n' -} +assistant_phase_source_gate_module "review-controller.sh" +assistant_phase_source_gate_module "qa-controller.sh" +assistant_phase_source_gate_module "learning-controller.sh" +assistant_phase_source_gate_module "metrics.sh" +assistant_phase_source_gate_module "subagent-evidence.sh" +assistant_phase_source_gate_module "subagent-orchestration.sh" diff --git a/install.sh b/install.sh index c8b9ece..3ed14d2 100755 --- a/install.sh +++ b/install.sh @@ -131,7 +131,7 @@ hook_profile_allowed_json() { printf '%s\n' '["skill-router.sh","session-start.sh","pre-compress.sh","post-compact.sh","task-journal-resolver.sh"]' ;; workflow) - printf '%s\n' '["session-start.sh","skill-router.sh","learning-signals.sh","workflow-enforcer.sh","workflow-guard.sh","stop-review.sh","subagent-monitor.sh","pre-compress.sh","post-compact.sh","task-journal-resolver.sh","workflow-phase-gates.sh"]' + printf '%s\n' '["session-start.sh","skill-router.sh","learning-signals.sh","workflow-enforcer.sh","workflow-guard.sh","stop-review.sh","subagent-monitor.sh","pre-compress.sh","post-compact.sh","task-journal-resolver.sh","workflow-phase-gates.sh","hook-runtime.sh"]' ;; strict) printf '%s\n' 'null' @@ -154,7 +154,7 @@ hook_selected_for_profile() { ;; workflow) case "$hook_name" in - session-start.sh|skill-router.sh|learning-signals.sh|workflow-enforcer.sh|workflow-guard.sh|stop-review.sh|subagent-monitor.sh|pre-compress.sh|post-compact.sh|task-journal-resolver.sh|workflow-phase-gates.sh) return 0 ;; + session-start.sh|skill-router.sh|learning-signals.sh|workflow-enforcer.sh|workflow-guard.sh|stop-review.sh|subagent-monitor.sh|pre-compress.sh|post-compact.sh|task-journal-resolver.sh|workflow-phase-gates.sh|hook-runtime.sh) return 0 ;; *) return 1 ;; esac ;; @@ -1551,7 +1551,7 @@ if $INSTALL_HOOKS; then # SubagentStart/SubagentStop, PreToolUse, PreCompact, and PostCompact. if [[ "$AGENT" == "codex" ]]; then case "$hook_name" in - session-start.sh|skill-router.sh|stop-review.sh|harness-gate.sh|learning-signals.sh|workflow-enforcer.sh|workflow-guard.sh|subagent-monitor.sh|pre-compress.sh|post-compact.sh|task-journal-resolver.sh|workflow-phase-gates.sh) ;; # supported + shared helper dependencies + session-start.sh|skill-router.sh|stop-review.sh|harness-gate.sh|learning-signals.sh|workflow-enforcer.sh|workflow-guard.sh|subagent-monitor.sh|pre-compress.sh|post-compact.sh|task-journal-resolver.sh|workflow-phase-gates.sh|hook-runtime.sh) ;; # supported + shared helper dependencies *) continue ;; # skip unsupported hooks esac if [[ "$CODEX_SUPPORTS_COMPACTION_HOOKS" != "true" ]]; then @@ -1575,6 +1575,14 @@ if $INSTALL_HOOKS; then fi cp "$hook_script" "$HOOKS_TARGET/" done + if hook_selected_for_profile "workflow-phase-gates.sh" && [[ -d "$HOOKS_SOURCE/scripts/workflow-phase-gates.d" ]]; then + mkdir -p "$HOOKS_TARGET/workflow-phase-gates.d" + cp "$HOOKS_SOURCE/scripts/workflow-phase-gates.d/"*.sh "$HOOKS_TARGET/workflow-phase-gates.d/" + fi + if hook_selected_for_profile "workflow-guard.sh" && [[ -d "$HOOKS_SOURCE/scripts/workflow-guard.d" ]]; then + mkdir -p "$HOOKS_TARGET/workflow-guard.d" + cp "$HOOKS_SOURCE/scripts/workflow-guard.d/"*.sh "$HOOKS_TARGET/workflow-guard.d/" + fi # chmod only if files were actually copied if compgen -G "$HOOKS_TARGET/*.sh" >/dev/null; then chmod +x "$HOOKS_TARGET/"*.sh diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md b/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md index 6e30f6d..27cdd92 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md @@ -72,7 +72,7 @@ Every phase has a declared role responsibility. Phases without a subagent role a ## Dispatch guidelines - **Every task gets at minimum**: Code Writer → Builder/Tester → Code Reviewer responsibilities. `reviewer` remains valid compatibility routing for existing handoffs, but new dispatches should use `code-reviewer` for code defects, security, architecture, test coverage, and structural code issues. In delegated mode these are subagents; in direct fallback they are explicitly recorded role-equivalent steps. No code ships without build + test + review evidence. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts MUST infer required_agents with at least Code Writer, Builder/Tester, and Code Reviewer (or Reviewer compatibility); `not_applicable` is invalid for source-changing Build tasks. -- **Strict evidence gate**: delegated mode is not complete until the task journal Agent Dispatch Log records Code Writer dispatch/result, Builder/Tester dispatch/result, and Code Reviewer dispatch/result evidence, or Reviewer dispatch/result evidence when compatibility routing is used. Medium+ delegated slice work also records per-slice dispatch evidence before each slice is marked verified. **Codex adds a stronger check:** delegated Codex evidence must correspond to real `SubagentStart`/`SubagentStop` lifecycle records in `.codex/subagent-events.jsonl`; each dispatch/result entry must reference the matching lifecycle `agent_id`, and task-journal text alone is treated as insufficient because Codex can otherwise run phases inline and write claimed dispatch entries. Direct fallback is allowed only for explicit `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` reasons, and must record equivalent role, phase, verification, and review evidence; silent fallback fails the stop-review/phase gates. +- **Strict evidence gate**: delegated mode is not complete until the task journal Agent Dispatch Log records Code Writer dispatch/result, Builder/Tester dispatch/result, and Code Reviewer dispatch/result evidence, or Reviewer dispatch/result evidence when compatibility routing is used. Medium+ delegated slice work also records per-slice dispatch evidence before each slice is marked verified. **Codex adds a stronger check:** delegated Codex evidence must correspond to real hook-written `SubagentStart`/`SubagentStop` lifecycle records in protected agent-owned workflow state; each dispatch/result entry must reference the matching lifecycle `agent_id`, and task-journal text or project-local `.codex/subagent-events.jsonl` diagnostics are insufficient because Codex can otherwise run phases inline and write claimed dispatch entries. Direct fallback is allowed only for explicit `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` reasons, and must record equivalent role, phase, verification, and review evidence; silent fallback fails the stop-review/phase gates. - **QA evidence gate**: when QA is required, delegated mode is not complete until the task journal Agent Dispatch Log records QA Evaluator dispatch/result evidence after Builder/Tester and Code Reviewer evidence. Direct fallback must record fresh QA Evaluator direct evidence separately from Code Reviewer direct evidence. - **Every medium+ task gets Architect decomposition responsibility**: In delegated mode the Architect proposes smallest iterable slice boundaries; in direct fallback the same criteria and evidence are recorded directly. - **Launch in parallel** when agents are independent (e.g., Code Mapper + Explorer on different modules) diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md index 7944d4d..94b6bae 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md @@ -23,6 +23,7 @@ Write to `{agent_state_dir}/task.md` in the project root when a local state dire ```markdown ## Task: [1-sentence description] +Created: [stable task identity, e.g. ISO timestamp created once] Status: DISCOVERING | DECOMPOSING | PLANNING | BUILDING [step N/M] | REVIEWING | DOCUMENTING | DONE Triaged as: [small | medium | large | mega] Task type: [feature | bugfix | refactor | migration | rewrite | config | infra | security | docs | spike] @@ -55,7 +56,7 @@ Plan approval: [yes/no + date] [strict subagent evidence inspected by stop-review/phase gates] - Required roles: Code Writer, Builder/Tester, Code Reviewer; QA Evaluator when `qa_evaluation_mode=required`; Code Mapper/Explorer/Architect by size/risk; Reviewer for legacy compatibility. - Execution mode: delegated | direct_fallback | not_applicable -- Codex lifecycle evidence: delegated Codex roles need matching `.codex/subagent-events.jsonl` `SubagentStart`/`SubagentStop` records and dispatch/result refs with the same `agent_id`; journal text alone is insufficient. +- Codex lifecycle evidence: delegated Codex roles need hook-written protected workflow-state `SubagentStart`/`SubagentStop` records task-bound to this journal's `Created:` identity, plus dispatch/result refs with the same `agent_id`; project-local `.codex/subagent-events.jsonl` is diagnostic only and journal text alone is insufficient. - Direct fallback reason: [authorization_denied | subagents_unavailable | policy_disallowed | N/A] - Evidence shorthand: delegated = dispatch/result refs; direct_fallback = role-equivalent direct evidence; N/A only when role not required. - Code Mapper dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] diff --git a/skills/assistant-workflow/references/subagent-dispatch.md b/skills/assistant-workflow/references/subagent-dispatch.md index 6e30f6d..27cdd92 100644 --- a/skills/assistant-workflow/references/subagent-dispatch.md +++ b/skills/assistant-workflow/references/subagent-dispatch.md @@ -72,7 +72,7 @@ Every phase has a declared role responsibility. Phases without a subagent role a ## Dispatch guidelines - **Every task gets at minimum**: Code Writer → Builder/Tester → Code Reviewer responsibilities. `reviewer` remains valid compatibility routing for existing handoffs, but new dispatches should use `code-reviewer` for code defects, security, architecture, test coverage, and structural code issues. In delegated mode these are subagents; in direct fallback they are explicitly recorded role-equivalent steps. No code ships without build + test + review evidence. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts MUST infer required_agents with at least Code Writer, Builder/Tester, and Code Reviewer (or Reviewer compatibility); `not_applicable` is invalid for source-changing Build tasks. -- **Strict evidence gate**: delegated mode is not complete until the task journal Agent Dispatch Log records Code Writer dispatch/result, Builder/Tester dispatch/result, and Code Reviewer dispatch/result evidence, or Reviewer dispatch/result evidence when compatibility routing is used. Medium+ delegated slice work also records per-slice dispatch evidence before each slice is marked verified. **Codex adds a stronger check:** delegated Codex evidence must correspond to real `SubagentStart`/`SubagentStop` lifecycle records in `.codex/subagent-events.jsonl`; each dispatch/result entry must reference the matching lifecycle `agent_id`, and task-journal text alone is treated as insufficient because Codex can otherwise run phases inline and write claimed dispatch entries. Direct fallback is allowed only for explicit `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` reasons, and must record equivalent role, phase, verification, and review evidence; silent fallback fails the stop-review/phase gates. +- **Strict evidence gate**: delegated mode is not complete until the task journal Agent Dispatch Log records Code Writer dispatch/result, Builder/Tester dispatch/result, and Code Reviewer dispatch/result evidence, or Reviewer dispatch/result evidence when compatibility routing is used. Medium+ delegated slice work also records per-slice dispatch evidence before each slice is marked verified. **Codex adds a stronger check:** delegated Codex evidence must correspond to real hook-written `SubagentStart`/`SubagentStop` lifecycle records in protected agent-owned workflow state; each dispatch/result entry must reference the matching lifecycle `agent_id`, and task-journal text or project-local `.codex/subagent-events.jsonl` diagnostics are insufficient because Codex can otherwise run phases inline and write claimed dispatch entries. Direct fallback is allowed only for explicit `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` reasons, and must record equivalent role, phase, verification, and review evidence; silent fallback fails the stop-review/phase gates. - **QA evidence gate**: when QA is required, delegated mode is not complete until the task journal Agent Dispatch Log records QA Evaluator dispatch/result evidence after Builder/Tester and Code Reviewer evidence. Direct fallback must record fresh QA Evaluator direct evidence separately from Code Reviewer direct evidence. - **Every medium+ task gets Architect decomposition responsibility**: In delegated mode the Architect proposes smallest iterable slice boundaries; in direct fallback the same criteria and evidence are recorded directly. - **Launch in parallel** when agents are independent (e.g., Code Mapper + Explorer on different modules) diff --git a/skills/assistant-workflow/references/task-journal-template.md b/skills/assistant-workflow/references/task-journal-template.md index 7944d4d..94b6bae 100644 --- a/skills/assistant-workflow/references/task-journal-template.md +++ b/skills/assistant-workflow/references/task-journal-template.md @@ -23,6 +23,7 @@ Write to `{agent_state_dir}/task.md` in the project root when a local state dire ```markdown ## Task: [1-sentence description] +Created: [stable task identity, e.g. ISO timestamp created once] Status: DISCOVERING | DECOMPOSING | PLANNING | BUILDING [step N/M] | REVIEWING | DOCUMENTING | DONE Triaged as: [small | medium | large | mega] Task type: [feature | bugfix | refactor | migration | rewrite | config | infra | security | docs | spike] @@ -55,7 +56,7 @@ Plan approval: [yes/no + date] [strict subagent evidence inspected by stop-review/phase gates] - Required roles: Code Writer, Builder/Tester, Code Reviewer; QA Evaluator when `qa_evaluation_mode=required`; Code Mapper/Explorer/Architect by size/risk; Reviewer for legacy compatibility. - Execution mode: delegated | direct_fallback | not_applicable -- Codex lifecycle evidence: delegated Codex roles need matching `.codex/subagent-events.jsonl` `SubagentStart`/`SubagentStop` records and dispatch/result refs with the same `agent_id`; journal text alone is insufficient. +- Codex lifecycle evidence: delegated Codex roles need hook-written protected workflow-state `SubagentStart`/`SubagentStop` records task-bound to this journal's `Created:` identity, plus dispatch/result refs with the same `agent_id`; project-local `.codex/subagent-events.jsonl` is diagnostic only and journal text alone is insufficient. - Direct fallback reason: [authorization_denied | subagents_unavailable | policy_disallowed | N/A] - Evidence shorthand: delegated = dispatch/result refs; direct_fallback = role-equivalent direct evidence; N/A only when role not required. - Code Mapper dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] diff --git a/tests/p0-p4/review-loop-cap-contracts.sh b/tests/p0-p4/review-loop-cap-contracts.sh index c8b1ad8..b168b39 100755 --- a/tests/p0-p4/review-loop-cap-contracts.sh +++ b/tests/p0-p4/review-loop-cap-contracts.sh @@ -5,6 +5,25 @@ if [[ -z "${P0P4_HARNESS_LOADED:-}" ]]; then fi p0p4_bootstrap_suite "${BASH_SOURCE[0]}" +workflow_phase_gate_runtime_files=("$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh") +for module_file in "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.d/"*.sh; do + [[ -e "$module_file" ]] && workflow_phase_gate_runtime_files+=("$module_file") +done +review_cap_runtime_term_present() { + local file="$1" + local term="$2" + local runtime_file + if [[ "$file" == "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh" ]]; then + for runtime_file in "${workflow_phase_gate_runtime_files[@]}"; do + [[ -f "$runtime_file" ]] || continue + grep -Fq "$term" "$runtime_file" && return 0 + done + return 1 + fi + + [[ -f "$file" ]] && grep -Fq "$term" "$file" +} + active_review_cap_files=( "$FRAMEWORK_DIR/skills/assistant-review/SKILL.md" "$FRAMEWORK_DIR/skills/assistant-review/contracts/input.yaml" @@ -23,7 +42,7 @@ active_review_cap_files=( "$FRAMEWORK_DIR/skills/assistant-workflow/references/subagent-roles.md" "$FRAMEWORK_DIR/hooks/scripts/stop-review.sh" "$FRAMEWORK_DIR/hooks/scripts/workflow-enforcer.sh" - "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh" + "${workflow_phase_gate_runtime_files[@]}" "$FRAMEWORK_DIR/install.sh" "$FRAMEWORK_DIR/agents/codex/code-reviewer.toml" "$FRAMEWORK_DIR/agents/codex/reviewer.toml" @@ -108,8 +127,12 @@ for file_and_term in \ "$FRAMEWORK_DIR/docs/harness-design-guide.md::| 10 | Terminal report"; do file="${file_and_term%%::*}" term="${file_and_term#*::}" - if [[ ! -f "$file" ]] || ! grep -Fq "$term" "$file"; then - missing_runtime_cap_terms+=("${file#$FRAMEWORK_DIR/}: $term") + if ! review_cap_runtime_term_present "$file" "$term"; then + if [[ "$file" == "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh" ]]; then + missing_runtime_cap_terms+=("hooks/scripts/workflow-phase-gates.sh or workflow-phase-gates.d/*.sh: $term") + else + missing_runtime_cap_terms+=("${file#$FRAMEWORK_DIR/}: $term") + fi fi done if [[ "${#missing_runtime_cap_terms[@]}" -eq 0 ]]; then diff --git a/tests/p0-p4/runtime-phase-gate-contracts.sh b/tests/p0-p4/runtime-phase-gate-contracts.sh index f38aa61..9bdf227 100644 --- a/tests/p0-p4/runtime-phase-gate-contracts.sh +++ b/tests/p0-p4/runtime-phase-gate-contracts.sh @@ -6,6 +6,19 @@ p0p4_bootstrap_suite "${BASH_SOURCE[0]}" test_start "runtime phase-gate helper exists and is wired into runtime hooks" missing_runtime_helper_terms=() helper_file="$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.sh" +helper_surfaces=("$helper_file") +for module_file in "$FRAMEWORK_DIR/hooks/scripts/workflow-phase-gates.d/"*.sh; do + [[ -e "$module_file" ]] && helper_surfaces+=("$module_file") +done +runtime_helper_term_present() { + local term="$1" + local surface + for surface in "${helper_surfaces[@]}"; do + [[ -f "$surface" ]] || continue + grep -Fq "$term" "$surface" && return 0 + done + return 1 +} if [[ ! -f "$helper_file" ]]; then missing_runtime_helper_terms+=("workflow-phase-gates.sh exists") else @@ -20,7 +33,7 @@ else "assistant_phase_reason_action" \ "assistant_phase_subagent_warning_reason_key" \ "assistant_phase_subagent_warning_action"; do - if ! grep -Fq "$term" "$helper_file"; then + if ! runtime_helper_term_present "$term"; then missing_runtime_helper_terms+=("$term") fi done @@ -53,8 +66,8 @@ for term in \ "missing_durable_lesson_decision" \ "missing_persistence_evidence" \ "missing_no_save_rationale"; do - if ! grep -Fq "$term" "$helper_file"; then - missing_runtime_helper_terms+=("workflow-phase-gates.sh owns $term") + if ! runtime_helper_term_present "$term"; then + missing_runtime_helper_terms+=("workflow-phase-gates helpers own $term") fi done for term in \ diff --git a/tests/p0-p4/task-packet-contracts.sh b/tests/p0-p4/task-packet-contracts.sh index f7add41..1fef5a3 100644 --- a/tests/p0-p4/task-packet-contracts.sh +++ b/tests/p0-p4/task-packet-contracts.sh @@ -653,6 +653,45 @@ else fail "assistant-workflow root/plugin mirrors differ: ${workflow_unsynced_pairs[*]}" fi +test_start "workflow task journal template anchors Codex lifecycle evidence to Created identity" +missing_task_identity_terms=() +task_journal_template_surfaces=( + "skills/assistant-workflow/references/task-journal-template.md" + "plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md" +) +for surface in "${task_journal_template_surfaces[@]}"; do + file="$FRAMEWORK_DIR/$surface" + if [[ ! -f "$file" ]]; then + missing_task_identity_terms+=("$surface: file missing") + continue + fi + if ! awk ' + BEGIN { found_task = 0; ok = 0 } + /^## Task:/ { + found_task = 1 + getline + if ($0 ~ /^Created: / && NR <= 35) ok = 1 + exit + } + END { exit (found_task && ok) ? 0 : 1 } + ' "$file"; then + missing_task_identity_terms+=("$surface: Created must immediately follow ## Task near top") + fi + for term in \ + "hook-written protected workflow-state \`SubagentStart\`/\`SubagentStop\` records" \ + "task-bound to this journal's \`Created:\` identity" \ + "project-local \`.codex/subagent-events.jsonl\` is diagnostic only"; do + if ! grep -Fq -- "$term" "$file"; then + missing_task_identity_terms+=("$surface: $term") + fi + done +done +if [[ "${#missing_task_identity_terms[@]}" -eq 0 ]]; then + pass +else + fail "task-journal-template.md must anchor protected lifecycle evidence to Created identity: ${missing_task_identity_terms[*]}" +fi + test_start "workflow subagent policy state gates delegation before fallback" missing_workflow_subagent_gate=() workflow_subagent_gate_surfaces=( diff --git a/tests/test-hooks.sh b/tests/test-hooks.sh index f3925c3..6b86f7b 100755 --- a/tests/test-hooks.sh +++ b/tests/test-hooks.sh @@ -110,6 +110,49 @@ is_valid_json() { echo "$1" | jq empty 2>/dev/null } +assert_claude_pretooluse_deny_contains() { + local expected="$1" + [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e --arg expected "$expected" ' + .hookSpecificOutput.hookEventName == "PreToolUse" + and .hookSpecificOutput.permissionDecision == "deny" + and (.hookSpecificOutput.permissionDecisionReason | contains($expected)) + ' >/dev/null 2>&1 +} + +assert_top_level_block_contains() { + local expected="$1" + [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e --arg expected "$expected" ' + .decision == "block" + and (.reason | contains($expected)) + ' >/dev/null 2>&1 +} + +assert_top_level_retry_contains() { + local expected="$1" + [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e --arg expected "$expected" ' + .decision == "retry" + and (.reason | contains($expected)) + ' >/dev/null 2>&1 +} + +run_workflow_guard_builder_tester_bash() { + local command="$1" + + jq -n --arg command "$command" \ + '{tool_name: "Bash", agent_type: "builder-tester", tool_input: {command: $command}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out +} + clear_workflow_cache() { rm -rf \ "$TEST_AGENT_HOME/.codex/cache/workflow-state" \ @@ -117,6 +160,80 @@ clear_workflow_cache() { "$TEST_AGENT_HOME/.gemini/cache/workflow-state" } +run_hook_without_jq() { + local script="$1" + local input="${2:-{}}" + local agent="${3:-codex}" + local no_jq_path dirname_path tmp_out tmp_err + local env_args=() + + dirname_path="$(command -v dirname)" + no_jq_path="$(mktemp -d)" + ln -s "$dirname_path" "$no_jq_path/dirname" + tmp_out=$(mktemp) + tmp_err=$(mktemp) + + case "$agent" in + claude) env_args+=(CLAUDE_PROJECT_DIR="$TEST_PROJECT") ;; + gemini) env_args+=(GEMINI_PROJECT_DIR="$TEST_PROJECT") ;; + codex) env_args+=(CODEX_PROJECT_DIR="$TEST_PROJECT") ;; + *) echo "Unknown missing-jq agent: $agent" >&2; exit 1 ;; + esac + + HOOK_EXIT=0 + env PATH="$no_jq_path" HOME="$TEST_AGENT_HOME" "${env_args[@]}" \ + "$BASH" "$HOOKS_DIR/$script" > "$tmp_out" 2> "$tmp_err" <<< "$input" || HOOK_EXIT=$? + + HOOK_STDOUT=$(cat "$tmp_out") + HOOK_STDERR=$(cat "$tmp_err") + rm -f "$tmp_out" "$tmp_err" + rm -rf "$no_jq_path" +} + +codex_protected_events_file() { + local project_dir="${1:-$TEST_PROJECT}" + local task_file="${2:-}" + local task_identity="${3:-}" + + if [[ -z "$task_identity" && -n "$task_file" && -f "$task_file" ]]; then + task_identity="$( + HOME="$TEST_AGENT_HOME" bash -c ' + . "$1" + assistant_hook_task_identity_from_file "$2" || true + ' _ "$HOOKS_DIR/hook-runtime.sh" "$task_file" + )" + fi + + HOME="$TEST_AGENT_HOME" bash -c ' + . "$1" + assistant_hook_codex_subagent_events_file_for_project "$2" "$3" + ' _ "$HOOKS_DIR/hook-runtime.sh" "$project_dir" "$task_identity" +} + +record_codex_subagent_event() { + local event="$1" + local agent_type="$2" + local agent_id="$3" + local project_dir="${4:-$TEST_PROJECT}" + local tmp_out + + tmp_out=$(mktemp) + printf '{"hook_event_name":"%s","agent_type":"%s","agent_id":"%s","cwd":"%s"}\n' \ + "$event" "$agent_type" "$agent_id" "$project_dir" | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$project_dir" bash "$HOOKS_DIR/subagent-monitor.sh" \ + > "$tmp_out" 2>/dev/null + rm -f "$tmp_out" +} + +record_codex_subagent_event_pair() { + local agent_type="$1" + local agent_id="$2" + local project_dir="${3:-$TEST_PROJECT}" + + record_codex_subagent_event "SubagentStart" "$agent_type" "$agent_id" "$project_dir" + record_codex_subagent_event "SubagentStop" "$agent_type" "$agent_id" "$project_dir" +} + review_controller_reason() { local task_file="$1" bash -c ' @@ -146,12 +263,20 @@ review_gate_reason() { subagent_evidence_reason() { local task_file="$1" - bash -c ' + HOME="$TEST_AGENT_HOME" bash -c ' . "$1" assistant_phase_subagent_evidence_missing_reason_key "$2" ' _ "$HOOKS_DIR/workflow-phase-gates.sh" "$task_file" } +required_subagent_roles() { + local task_file="$1" + HOME="$TEST_AGENT_HOME" bash -c ' + . "$1" + assistant_phase_required_subagent_roles "$2" + ' _ "$HOOKS_DIR/workflow-phase-gates.sh" "$task_file" +} + write_required_qa_review_task() { local task_file="$1" local qa_evidence="$2" @@ -257,10 +382,198 @@ if ! command -v jq >/dev/null 2>&1; then echo "" fi +# ── critical hook dependency tests ──────────────────────────────────────────── + +echo "critical hook dependencies" + +if test_start "critical hooks: workflow-guard missing jq under Claude PreToolUse denies"; then + run_hook_without_jq "workflow-guard.sh" '{"hook_event_name":"PreToolUse","tool_name":"Edit","tool_input":{"file_path":"src/App.cs"}}' "claude" + if assert_claude_pretooluse_deny_contains "requires jq"; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude PreToolUse deny mentioning jq; stdout='$HOOK_STDOUT' stderr='$HOOK_STDERR'" + fi +fi + +if test_start "critical hooks: workflow-enforcer missing jq blocks UserPromptSubmit"; then + run_hook_without_jq "workflow-enforcer.sh" '{"hook_event_name":"UserPromptSubmit","prompt":"fix the hook"}' "claude" + if assert_top_level_block_contains "requires jq"; then + pass + else + fail "exit=$HOOK_EXIT, expected UserPromptSubmit block mentioning jq; stdout='$HOOK_STDOUT' stderr='$HOOK_STDERR'" + fi +fi + +if test_start "critical hooks: stop-review missing jq blocks Claude Stop"; then + run_hook_without_jq "stop-review.sh" '{"hook_event_name":"Stop","stop_hook_active":false}' "claude" + if assert_top_level_block_contains "requires jq"; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude Stop block mentioning jq; stdout='$HOOK_STDOUT' stderr='$HOOK_STDERR'" + fi +fi + +if test_start "critical hooks: stop-review missing jq retries Gemini AfterAgent"; then + run_hook_without_jq "stop-review.sh" '{"hook_event_name":"AfterAgent","agent_output":"done"}' "gemini" + if assert_top_level_retry_contains "requires jq"; then + pass + else + fail "exit=$HOOK_EXIT, expected Gemini AfterAgent retry mentioning jq; stdout='$HOOK_STDOUT' stderr='$HOOK_STDERR'" + fi +fi + +if test_start "critical hooks: subagent-monitor missing jq under Claude SubagentStart adds degraded context"; then + run_hook_without_jq "subagent-monitor.sh" '{"hook_event_name":"SubagentStart","agent_type":"code-writer","agent_id":"cw-1"}' "claude" + if [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e ' + .hookSpecificOutput.hookEventName == "SubagentStart" + and (.hookSpecificOutput.additionalContext | contains("requires jq")) + and ( + (.hookSpecificOutput.additionalContext | contains("cannot block")) + or (.hookSpecificOutput.additionalContext | contains("degraded")) + ) + ' >/dev/null 2>&1 \ + && ! echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude SubagentStart additionalContext degraded jq warning; stdout='$HOOK_STDOUT' stderr='$HOOK_STDERR'" + fi +fi + +if test_start "critical hooks: subagent-monitor missing jq under Codex SubagentStart blocks"; then + run_hook_without_jq "subagent-monitor.sh" '{"hook_event_name":"SubagentStart","agent_type":"code-writer","agent_id":"cw-1"}' "codex" + if assert_top_level_block_contains "requires jq"; then + pass + else + fail "exit=$HOOK_EXIT, expected Codex SubagentStart block mentioning jq; stdout='$HOOK_STDOUT' stderr='$HOOK_STDERR'" + fi +fi + # ── workflow-phase-gates.sh tests ──────────────────────────────────────────── echo "workflow-phase-gates.sh" +if test_start "workflow-phase-gates: focused modules source and public helpers load"; then + gate_modules=( + review-controller.sh + qa-controller.sh + learning-controller.sh + metrics.sh + subagent-evidence.sh + subagent-orchestration.sh + ) + gate_helpers=( + assistant_phase_review_controller_missing_reason_key + assistant_phase_requires_qa_evaluator + assistant_phase_review_missing_reason_key + assistant_phase_learning_missing_reason_key + assistant_phase_has_metrics_today + assistant_phase_required_subagent_roles + assistant_phase_has_role_dispatch_result_evidence + assistant_phase_subagent_evidence_missing_reason_key + assistant_phase_reason_missing_field + assistant_phase_subagent_warning_action + ) + missing_gate_module="" + for gate_module in "${gate_modules[@]}"; do + if [[ ! -f "$HOOKS_DIR/workflow-phase-gates.d/$gate_module" ]] \ + || ! grep -Fq "\"$gate_module\"" "$HOOKS_DIR/workflow-phase-gates.sh"; then + missing_gate_module="$gate_module" + break + fi + done + if [[ -z "$missing_gate_module" ]] \ + && bash -c ' + . "$1" + shift + for helper in "$@"; do + declare -F "$helper" >/dev/null || exit 1 + done + ' _ "$HOOKS_DIR/workflow-phase-gates.sh" "${gate_helpers[@]}"; then + pass + else + fail "workflow-phase-gates.sh did not load focused module/helpers; missing_module='$missing_gate_module'" + fi +fi + +if test_start "workflow-phase-gates: QA evaluator requirement helper has single definition"; then + definition_count=$( + grep -R -h -E '^[[:space:]]*(function[[:space:]]+)?assistant_phase_requires_qa_evaluator[[:space:]]*\(\)' \ + "$HOOKS_DIR/workflow-phase-gates.sh" "$HOOKS_DIR/workflow-phase-gates.d"/*.sh | wc -l | tr -d '[:space:]' + ) + if [[ "$definition_count" == "1" ]]; then + pass + else + fail "expected exactly one assistant_phase_requires_qa_evaluator definition, found $definition_count" + fi +fi + +if test_start "workflow-phase-gates: source-changing BUILDING without Required agents infers required roles"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: BUILDING +Triaged as: small +Task type: feature +Subagent execution mode: not_applicable +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + roles=$(required_subagent_roles "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "not_applicable_with_required_roles" ]] \ + && printf '%s\n' "$roles" | grep -qx "Code Writer" \ + && printf '%s\n' "$roles" | grep -qx "Builder/Tester" \ + && printf '%s\n' "$roles" | grep -qx "Code Reviewer"; then + pass + else + fail "expected inferred Code Writer/Builder/Tester/Code Reviewer and not_applicable block; reason='$helper_reason' roles='$roles'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: source-changing VERIFYING with malformed Required agents infers required roles"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: VERIFYING +Triaged as: small +Task type: bugfix +Subagent execution mode: not_applicable +Required agents: ??? +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + roles=$(required_subagent_roles "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "not_applicable_with_required_roles" ]] \ + && printf '%s\n' "$roles" | grep -qx "Code Writer" \ + && printf '%s\n' "$roles" | grep -qx "Builder/Tester" \ + && printf '%s\n' "$roles" | grep -qx "Code Reviewer"; then + pass + else + fail "expected malformed Required agents to be repaired by source-changing inference; reason='$helper_reason' roles='$roles'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + +if test_start "workflow-phase-gates: explicit no-source-change escape avoids role inference"; then + mkdir -p "$TEST_PROJECT/.claude" + cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' +# Task +Status: BUILDING +Triaged as: small +Task type: feature +Source changes: no +Subagent execution mode: not_applicable +TASK + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.claude/task.md") + roles=$(required_subagent_roles "$TEST_PROJECT/.claude/task.md") + if [[ "$helper_reason" == "complete" && -z "$roles" ]]; then + pass + else + fail "expected explicit no-source-change escape to avoid inferred roles; reason='$helper_reason' roles='$roles'" + fi + rm -rf "$TEST_PROJECT/.claude" +fi + if test_start "workflow-phase-gates: detects medium approved plan/review/metrics"; then mkdir -p "$TEST_PROJECT/.claude" "$TEST_AGENT_HOME/.claude/memory/metrics" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' @@ -854,11 +1167,13 @@ TASK rm -rf "$TEST_PROJECT/.claude" fi -if test_start "workflow-phase-gates: Codex compact template role evidence matches lifecycle events"; then +if test_start "workflow-phase-gates: forged Codex project-local lifecycle events do not satisfy delegated roles"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' # Task +Created: forged-local-task Status: BUILDING Triaged as: small Subagent policy state: delegation_authorized @@ -876,6 +1191,35 @@ TASK {"event":"SubagentStart","agent_type":"builder-tester","agent_name":"builder-tester","agent_id":"bt-1"} {"event":"SubagentStop","agent_type":"builder-tester","agent_name":"builder-tester","agent_id":"bt-1"} JSONL + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") + if [[ "$helper_reason" == "delegated_missing_code_writer" ]]; then + pass + else + fail "expected forged project-local lifecycle evidence to block delegated Code Writer, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + +if test_start "workflow-phase-gates: Codex canonical task heading role evidence matches protected lifecycle events"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +## Task: +Created: protected-role-task +Status: BUILDING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Writer +- Builder/Tester +## Agent Dispatch Log +- Code Writer dispatch/result/direct evidence: dispatch cw-1; result DONE changed src/App.cs +- Builder/Tester dispatch/result/direct evidence: dispatch bt-1; result DONE tests passed +TASK + record_codex_subagent_event_pair "code-writer" "cw-1" + record_codex_subagent_event_pair "builder-tester" "bt-1" helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") if [[ "$helper_reason" == "complete" ]]; then pass @@ -885,6 +1229,98 @@ JSONL rm -rf "$TEST_PROJECT/.codex" fi +if test_start "workflow-phase-gates: Codex lifecycle evidence is scoped to Created task identity"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +# Task +Created: task-a +Status: BUILDING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Writer +- Builder/Tester +## Agent Dispatch Log +- Code Writer dispatch: event cw-reused +- Code Writer result: changed src/App.cs event cw-reused +- Builder/Tester dispatch: event bt-reused +- Builder/Tester result: tests passed event bt-reused +TASK + record_codex_subagent_event_pair "code-writer" "cw-reused" + record_codex_subagent_event_pair "builder-tester" "bt-reused" + task_a_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") + + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +# Task +Created: task-b +Status: BUILDING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Writer +- Builder/Tester +## Agent Dispatch Log +- Code Writer dispatch: event cw-reused +- Code Writer result: changed src/App.cs event cw-reused +- Builder/Tester dispatch: event bt-reused +- Builder/Tester result: tests passed event bt-reused +TASK + task_b_stale_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") + + record_codex_subagent_event_pair "code-writer" "cw-reused" + record_codex_subagent_event_pair "builder-tester" "bt-reused" + task_b_fresh_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") + + if [[ "$task_a_reason" == "complete" ]] \ + && [[ "$task_b_stale_reason" == "delegated_missing_code_writer" ]] \ + && [[ "$task_b_fresh_reason" == "complete" ]]; then + pass + else + fail "expected task A complete, task B stale block, task B fresh complete; got A='$task_a_reason' B-stale='$task_b_stale_reason' B-fresh='$task_b_fresh_reason'" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + +if test_start "workflow-phase-gates: Codex task without Created fails closed despite lifecycle evidence"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache + mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +# Task +Status: BUILDING +Triaged as: small +Subagent policy state: delegation_authorized +Subagent execution mode: delegated +Required agents: +- Code Writer +- Builder/Tester +## Agent Dispatch Log +- Code Writer dispatch: event cw-1 +- Code Writer result: changed src/App.cs event cw-1 +- Builder/Tester dispatch: event bt-1 +- Builder/Tester result: tests passed event bt-1 +TASK + record_codex_subagent_event_pair "code-writer" "cw-1" + record_codex_subagent_event_pair "builder-tester" "bt-1" + cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' +{"event":"SubagentStart","agent_type":"code-writer","agent_name":"code-writer","agent_id":"cw-1"} +{"event":"SubagentStop","agent_type":"code-writer","agent_name":"code-writer","agent_id":"cw-1"} +{"event":"SubagentStart","agent_type":"builder-tester","agent_name":"builder-tester","agent_id":"bt-1"} +{"event":"SubagentStop","agent_type":"builder-tester","agent_name":"builder-tester","agent_id":"bt-1"} +JSONL + helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") + if [[ "$helper_reason" == "delegated_missing_code_writer" ]]; then + pass + else + fail "expected missing Created to fail closed with delegated_missing_code_writer, got '$helper_reason'" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + if test_start "workflow-phase-gates: compact template Code Writer placeholder evidence does not satisfy role"; then mkdir -p "$TEST_PROJECT/.claude" cat > "$TEST_PROJECT/.claude/task.md" <<'TASK' @@ -960,9 +1396,11 @@ fi if test_start "workflow-phase-gates: Codex reviewer lifecycle with matching QA-labeled agent_id does not satisfy QA Evaluator"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' # Task +Created: reviewer-qa-compatibility-task Status: REVIEWING Triaged as: small Subagent policy state: delegation_authorized @@ -978,12 +1416,8 @@ qa_evaluation_mode: required - QA Evaluator result: accepted final_verdict accepted score_progression 4.00 TASK append_required_qa_review_complete_log "$TEST_PROJECT/.codex/task.md" - cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' -{"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} -{"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} -{"event":"SubagentStart","agent_type":"reviewer","agent_name":"reviewer","agent_id":"qa-reviewer-1"} -{"event":"SubagentStop","agent_type":"reviewer","agent_name":"reviewer","agent_id":"qa-reviewer-1"} -JSONL + record_codex_subagent_event_pair "code-reviewer" "cr-1" + record_codex_subagent_event_pair "reviewer" "qa-reviewer-1" helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then pass @@ -995,9 +1429,11 @@ fi if test_start "workflow-phase-gates: Codex qa-evaluator lifecycle with matching QA-labeled agent_id satisfies QA Evaluator"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' # Task +Created: qa-evaluator-task Status: REVIEWING Triaged as: small Subagent policy state: delegation_authorized @@ -1013,12 +1449,8 @@ qa_evaluation_mode: required - QA Evaluator result: accepted final_verdict accepted score_progression 4.00 id=qa-1 TASK append_required_qa_review_complete_log "$TEST_PROJECT/.codex/task.md" - cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' -{"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} -{"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} -{"event":"SubagentStart","agent_type":"qa-evaluator","agent_name":"QAEvaluator","agent_id":"qa-1"} -{"event":"SubagentStop","agent_type":"qa-evaluator","agent_name":"QAEvaluator","agent_id":"qa-1"} -JSONL + record_codex_subagent_event_pair "code-reviewer" "cr-1" + record_codex_subagent_event_pair "qa-evaluator" "qa-1" helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") if [[ "$helper_reason" == "complete" ]]; then pass @@ -1030,9 +1462,11 @@ fi if test_start "workflow-phase-gates: Codex reviewer lifecycle with QA-labeled agent_id prefix does not satisfy QA Evaluator"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' # Task +Created: reviewer-qa-prefix-task Status: REVIEWING Triaged as: small Subagent policy state: delegation_authorized @@ -1048,12 +1482,8 @@ qa_evaluation_mode: required - QA Evaluator result: accepted final_verdict accepted score_progression 4.00 TASK append_required_qa_review_complete_log "$TEST_PROJECT/.codex/task.md" - cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' -{"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} -{"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} -{"event":"SubagentStart","agent_type":"reviewer","agent_name":"reviewer","agent_id":"qa-reviewer"} -{"event":"SubagentStop","agent_type":"reviewer","agent_name":"reviewer","agent_id":"qa-reviewer"} -JSONL + record_codex_subagent_event_pair "code-reviewer" "cr-1" + record_codex_subagent_event_pair "reviewer" "qa-reviewer" helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then pass @@ -1065,9 +1495,11 @@ fi if test_start "workflow-phase-gates: Codex stale reviewer lifecycle without matching QA-labeled agent_id blocks QA Evaluator"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' # Task +Created: stale-qa-reviewer-task Status: REVIEWING Triaged as: small Subagent policy state: delegation_authorized @@ -1083,12 +1515,8 @@ qa_evaluation_mode: required - QA Evaluator result: accepted final_verdict accepted score_progression 4.00 TASK append_required_qa_review_complete_log "$TEST_PROJECT/.codex/task.md" - cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' -{"event":"SubagentStart","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} -{"event":"SubagentStop","agent_type":"code-reviewer","agent_name":"code-reviewer","agent_id":"cr-1"} -{"event":"SubagentStart","agent_type":"reviewer","agent_name":"reviewer","agent_id":"old-qa-reviewer"} -{"event":"SubagentStop","agent_type":"reviewer","agent_name":"reviewer","agent_id":"old-qa-reviewer"} -JSONL + record_codex_subagent_event_pair "code-reviewer" "cr-1" + record_codex_subagent_event_pair "reviewer" "old-qa-reviewer" helper_reason=$(subagent_evidence_reason "$TEST_PROJECT/.codex/task.md") if [[ "$helper_reason" == "delegated_missing_qa_evaluator" ]]; then pass @@ -2150,9 +2578,11 @@ fi if test_start "stop-review: Codex delegated journal evidence without lifecycle events blocks"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex/memory/metrics" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' # Task +Created: codex-no-lifecycle-stop-task Status: REVIEWING Triaged as: medium Plan approval: yes @@ -2204,9 +2634,11 @@ fi if test_start "stop-review: Codex stale lifecycle events without referenced agent_id block"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex/memory/metrics" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' # Task +Created: codex-unreferenced-lifecycle-task Status: REVIEWING Triaged as: medium Plan approval: yes @@ -2238,12 +2670,8 @@ Required agents: - Result: CLEAN - Score progression: 4.00 TASK - cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' -{"event":"SubagentStart","agent_type":"code-mapper","agent_name":"code-mapper","agent_id":"old-cm"} -{"event":"SubagentStop","agent_type":"code-mapper","agent_name":"code-mapper","agent_id":"old-cm"} -{"event":"SubagentStart","agent_type":"reviewer","agent_name":"reviewer","agent_id":"old-rv"} -{"event":"SubagentStop","agent_type":"reviewer","agent_name":"reviewer","agent_id":"old-rv"} -JSONL + record_codex_subagent_event_pair "code-mapper" "old-cm" + record_codex_subagent_event_pair "reviewer" "old-rv" _today=$(date +%Y-%m-%d) echo "{\"date\":\"$_today\",\"project\":\"test\",\"task\":\"test\",\"size\":\"medium\"}" > "$TEST_AGENT_HOME/.codex/memory/metrics/workflow-metrics.jsonl" @@ -2264,9 +2692,11 @@ fi if test_start "stop-review: Codex delegated lifecycle events plus journal evidence pass"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" "$TEST_AGENT_HOME/.codex/memory/metrics" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' # Task +Created: codex-stop-review-pass-task Status: REVIEWING Triaged as: medium Plan approval: yes @@ -2298,12 +2728,8 @@ Required agents: - Result: CLEAN - Score progression: 4.00 TASK - cat > "$TEST_PROJECT/.codex/subagent-events.jsonl" <<'JSONL' -{"event":"SubagentStart","agent_type":"code-mapper","agent_name":"code-mapper","agent_id":"cm-1"} -{"event":"SubagentStop","agent_type":"code-mapper","agent_name":"code-mapper","agent_id":"cm-1"} -{"event":"SubagentStart","agent_type":"reviewer","agent_name":"reviewer","agent_id":"rv-1"} -{"event":"SubagentStop","agent_type":"reviewer","agent_name":"reviewer","agent_id":"rv-1"} -JSONL + record_codex_subagent_event_pair "code-mapper" "cm-1" + record_codex_subagent_event_pair "reviewer" "rv-1" _today=$(date +%Y-%m-%d) echo "{\"date\":\"$_today\",\"project\":\"test\",\"task\":\"test\",\"size\":\"medium\"}" > "$TEST_AGENT_HOME/.codex/memory/metrics/workflow-metrics.jsonl" @@ -6072,7 +6498,14 @@ echo "subagent-monitor.sh" if test_start "subagent-monitor: Codex SubagentStart records lifecycle event and context"; then rm -rf "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +# Task +Created: monitor-start-task +Status: BUILDING +TASK + EVENTS_FILE="$(codex_protected_events_file "$TEST_PROJECT" "$TEST_PROJECT/.codex/task.md")" echo "{\"hook_event_name\":\"SubagentStart\",\"agent_type\":\"code-writer\",\"agent_id\":\"cw-1\",\"turn_id\":\"turn-1\",\"session_id\":\"sess-1\",\"cwd\":\"$TEST_PROJECT\"}" | \ HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/subagent-monitor.sh" \ > /tmp/_subagent_out 2>/dev/null @@ -6082,21 +6515,30 @@ if test_start "subagent-monitor: Codex SubagentStart records lifecycle event and if [[ $HOOK_EXIT -eq 0 ]] \ && is_valid_json "$HOOK_STDOUT" \ && echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.hookEventName == "SubagentStart"' >/dev/null 2>&1 \ - && [[ -f "$TEST_PROJECT/.codex/subagent-events.jsonl" ]] \ - && grep -q '"event":"SubagentStart"' "$TEST_PROJECT/.codex/subagent-events.jsonl" \ - && grep -q '"agent_type":"code-writer"' "$TEST_PROJECT/.codex/subagent-events.jsonl" \ - && grep -q '"turn_id":"turn-1"' "$TEST_PROJECT/.codex/subagent-events.jsonl" \ - && grep -q '"session_id":"sess-1"' "$TEST_PROJECT/.codex/subagent-events.jsonl"; then + && [[ -f "$EVENTS_FILE" ]] \ + && grep -q '"event":"SubagentStart"' "$EVENTS_FILE" \ + && grep -q '"agent_type":"code-writer"' "$EVENTS_FILE" \ + && grep -q '"turn_id":"turn-1"' "$EVENTS_FILE" \ + && grep -q '"session_id":"sess-1"' "$EVENTS_FILE" \ + && grep -q '"task_identity":"monitor-start-task"' "$EVENTS_FILE" \ + && [[ ! -f "$TEST_PROJECT/.codex/subagent-events.jsonl" ]]; then pass else - fail "exit=$HOOK_EXIT, expected Codex lifecycle event; stdout='$HOOK_STDOUT'; events='$(cat "$TEST_PROJECT/.codex/subagent-events.jsonl" 2>/dev/null || true)'" + fail "exit=$HOOK_EXIT, expected protected Codex lifecycle event; stdout='$HOOK_STDOUT'; events='$(cat "$EVENTS_FILE" 2>/dev/null || true)'" fi rm -rf "$TEST_PROJECT/.codex" fi if test_start "subagent-monitor: Codex SubagentStop records lifecycle event without plain-text output"; then rm -rf "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" + cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +# Task +Created: monitor-stop-task +Status: REVIEWING +TASK + EVENTS_FILE="$(codex_protected_events_file "$TEST_PROJECT" "$TEST_PROJECT/.codex/task.md")" echo "{\"hook_event_name\":\"SubagentStop\",\"agent_type\":\"reviewer\",\"agent_id\":\"rv-1\",\"agent_transcript_path\":\"/tmp/rv.jsonl\",\"cwd\":\"$TEST_PROJECT\"}" | \ HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/subagent-monitor.sh" \ > /tmp/_subagent_out 2>/dev/null @@ -6104,16 +6546,36 @@ if test_start "subagent-monitor: Codex SubagentStop records lifecycle event with HOOK_STDOUT=$(cat /tmp/_subagent_out) rm -f /tmp/_subagent_out if [[ $HOOK_EXIT -eq 0 && -z "$HOOK_STDOUT" ]] \ - && [[ -f "$TEST_PROJECT/.codex/subagent-events.jsonl" ]] \ - && grep -q '"event":"SubagentStop"' "$TEST_PROJECT/.codex/subagent-events.jsonl" \ - && grep -q '"agent_type":"reviewer"' "$TEST_PROJECT/.codex/subagent-events.jsonl"; then + && [[ -f "$EVENTS_FILE" ]] \ + && grep -q '"event":"SubagentStop"' "$EVENTS_FILE" \ + && grep -q '"agent_type":"reviewer"' "$EVENTS_FILE" \ + && grep -q '"task_identity":"monitor-stop-task"' "$EVENTS_FILE"; then pass else - fail "exit=$HOOK_EXIT, expected Codex stop event without stdout; stdout='$HOOK_STDOUT'; events='$(cat "$TEST_PROJECT/.codex/subagent-events.jsonl" 2>/dev/null || true)'" + fail "exit=$HOOK_EXIT, expected protected Codex stop event without stdout; stdout='$HOOK_STDOUT'; events='$(cat "$EVENTS_FILE" 2>/dev/null || true)'" fi rm -rf "$TEST_PROJECT/.codex" fi +if test_start "subagent-monitor: Codex invalid traversal cwd is rejected"; then + clear_workflow_cache + invalid_cwd="$TEST_PROJECT/../missing-subagent-dir" + echo "{\"hook_event_name\":\"SubagentStart\",\"agent_type\":\"code-writer\",\"agent_id\":\"cw-invalid\",\"cwd\":\"$invalid_cwd\"}" | \ + HOME="$TEST_AGENT_HOME" CODEX_HOME="$FRAMEWORK_DIR" bash "$HOOKS_DIR/subagent-monitor.sh" \ + > /tmp/_subagent_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_subagent_out) + rm -f /tmp/_subagent_out + if [[ $HOOK_EXIT -eq 0 ]] \ + && is_valid_json "$HOOK_STDOUT" \ + && echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ + && echo "$HOOK_STDOUT" | jq -r '.reason' | grep -q "existing absolute canonical directory"; then + pass + else + fail "exit=$HOOK_EXIT, expected invalid cwd block; stdout='$HOOK_STDOUT'" + fi +fi + # ── workflow-guard.sh tests ────────────────────────────────────────────────── echo "workflow-guard.sh" @@ -6133,6 +6595,851 @@ if test_start "workflow-guard: non-Edit tool → no output"; then fi fi +if test_start "workflow-guard: Claude code-writer frontmatter does not grant Bash"; then + if grep -Fq 'tools: Read, Grep, Glob, LS, Edit, Write' "$FRAMEWORK_DIR/agents/claude/code-writer.md" \ + && ! grep -Eq '^tools: .*Bash' "$FRAMEWORK_DIR/agents/claude/code-writer.md"; then + pass + else + fail "agents/claude/code-writer.md should not include Bash in frontmatter tools" + fi +fi + +if test_start "workflow-guard: code-writer Bash is blocked"; then + echo '{"tool_name":"Bash","agent_type":"code-writer","tool_input":{"command":"dotnet test"}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if assert_claude_pretooluse_deny_contains "Code Writer is not allowed to run Bash"; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude PreToolUse deny for code-writer Bash; stdout='$HOOK_STDOUT'" + fi +fi + +if test_start "workflow-guard: lifecycle evidence Bash is blocked"; then + echo '{"tool_name":"Bash","tool_input":{"command":"cat .codex/subagent-events.jsonl"}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if assert_claude_pretooluse_deny_contains "Lifecycle evidence files are hook-owned"; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude PreToolUse deny for lifecycle evidence Bash; stdout='$HOOK_STDOUT'" + fi +fi + +if test_start "workflow-guard: builder-tester production edit is blocked"; then + echo '{"tool_name":"Edit","agent_type":"builder-tester","tool_input":{"file_path":"hooks/scripts/workflow-guard.sh"}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude PreToolUse deny for builder-tester production edit; stdout='$HOOK_STDOUT'" + fi +fi + +if test_start "workflow-guard: builder-tester production C# files with test suffix substrings are blocked"; then + builder_tester_block_miss="" + for target_path in "src/Contest.cs" "src/Latest.cs"; do + printf '{"tool_name":"Edit","agent_type":"builder-tester","tool_input":{"file_path":"%s"}}\n' "$target_path" | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if ! assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + builder_tester_block_miss="$target_path exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_block_miss" ]]; then + pass + else + fail "$builder_tester_block_miss" + fi +fi + +if test_start "workflow-guard: builder-tester traversal into production edit is blocked"; then + echo '{"tool_name":"Edit","agent_type":"builder-tester","tool_input":{"file_path":"tests/../hooks/scripts/workflow-guard.sh"}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude PreToolUse deny for builder-tester traversal edit; stdout='$HOOK_STDOUT'" + fi +fi + +if test_start "workflow-guard: builder-tester outside absolute tests path writes are blocked"; then + outside_tests_path="$(dirname "$TEST_PROJECT")/outside-project/tests/test-hooks.sh" + builder_tester_outside_miss="" + jq -n --arg path "$outside_tests_path" \ + '{tool_name: "Edit", agent_type: "builder-tester", tool_input: {file_path: $path}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if ! assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + builder_tester_outside_miss="Edit path=$outside_tests_path exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + fi + + if [[ -z "$builder_tester_outside_miss" ]]; then + jq -n --arg patch "$(printf "*** Begin Patch\n*** Update File: %s\n@@\n-old\n+new\n*** End Patch" "$outside_tests_path")" \ + '{tool_name: "apply_patch", agent_type: "builder-tester", tool_input: {patch: $patch}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if ! assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + builder_tester_outside_miss="apply_patch path=$outside_tests_path exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + fi + fi + + if [[ -z "$builder_tester_outside_miss" ]]; then + for command in \ + "printf x | tee $outside_tests_path" \ + "printf x > $outside_tests_path" \ + "python3 -c 'open(\"$outside_tests_path\",\"w\").write(\"x\")'"; do + jq -n --arg command "$command" \ + '{tool_name: "Bash", agent_type: "builder-tester", tool_input: {command: $command}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if ! assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + builder_tester_outside_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + fi + + if [[ -z "$builder_tester_outside_miss" ]]; then + pass + else + fail "$builder_tester_outside_miss" + fi +fi + +if test_start "workflow-guard: builder-tester apply_patch command production edit is blocked"; then + jq -n --arg command $'*** Begin Patch\n*** Update File: hooks/scripts/workflow-guard.sh\n@@\n-old\n+new\n*** End Patch' \ + '{tool_name: "apply_patch", agent_type: "builder-tester", tool_input: {command: $command}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude PreToolUse deny for builder-tester apply_patch command production edit; stdout='$HOOK_STDOUT'" + fi +fi + +if test_start "workflow-guard: builder-tester apply_patch move-to production edit is blocked"; then + jq -n --arg patch $'*** Begin Patch\n*** Update File: tests/test-hooks.sh\n*** Move to: hooks/scripts/workflow-guard.sh\n@@\n-old\n+new\n*** End Patch' \ + '{tool_name: "apply_patch", agent_type: "builder-tester", tool_input: {patch: $patch}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude PreToolUse deny for builder-tester apply_patch move-to production edit; stdout='$HOOK_STDOUT'" + fi +fi + +if test_start "workflow-guard: builder-tester Bash apply_patch production edit is blocked"; then + bash_patch_command=$(printf "apply_patch <<'PATCH'\n*** Begin Patch\n*** Update File: hooks/scripts/workflow-guard.sh\n@@\n-old\n+new\n*** End Patch\nPATCH") + jq -n --arg command "$bash_patch_command" \ + '{tool_name: "Bash", agent_type: "builder-tester", tool_input: {command: $command}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude PreToolUse deny for builder-tester Bash apply_patch production edit; stdout='$HOOK_STDOUT'" + fi +fi + +if test_start "workflow-guard: builder-tester Bash apply_patch move-to production edit is blocked"; then + bash_patch_command=$(printf "apply_patch <<'PATCH'\n*** Begin Patch\n*** Update File: tests/test-hooks.sh\n*** Move to: hooks/scripts/workflow-guard.sh\n@@\n-old\n+new\n*** End Patch\nPATCH") + jq -n --arg command "$bash_patch_command" \ + '{tool_name: "Bash", agent_type: "builder-tester", tool_input: {command: $command}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + pass + else + fail "exit=$HOOK_EXIT, expected Claude PreToolUse deny for builder-tester Bash apply_patch move-to production edit; stdout='$HOOK_STDOUT'" + fi +fi + +if test_start "workflow-guard: builder-tester Bash production writes are blocked"; then + builder_tester_write_miss="" + for command in \ + "printf x | tee hooks/scripts/workflow-guard.sh" \ + "printf x > hooks/scripts/workflow-guard.sh" \ + "python3 -c 'open(\"hooks/scripts/workflow-guard.sh\",\"w\").write(\"x\")'" \ + "python3 -c 'open(\"hooks/scripts/workflow-guard.sh\", mode=\"w\").write(\"x\")'" \ + "python3 -c 'from pathlib import Path; Path(\"hooks/scripts/workflow-guard.sh\").open(\"w\").write(\"x\")'" \ + "python3 -c 'from pathlib import Path; Path(\"hooks/scripts/workflow-guard.sh\").open(mode=\"w\").write(\"x\")'"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + builder_tester_write_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_write_miss" ]]; then + pass + else + fail "$builder_tester_write_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash wrapper production redirections are blocked"; then + builder_tester_wrapper_redirection_miss="" + for command in \ + "bash -lc 'echo hi > hooks/scripts/workflow-guard.sh'" \ + "sh -c 'printf hi > hooks/scripts/workflow-guard.sh'" \ + "zsh -c 'echo hi > hooks/scripts/workflow-guard.sh'" \ + "env bash -lc 'echo hi > hooks/scripts/workflow-guard.sh'" \ + "bash -lc 'echo hi' > hooks/scripts/workflow-guard.sh"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + builder_tester_wrapper_redirection_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_wrapper_redirection_miss" ]]; then + pass + else + fail "$builder_tester_wrapper_redirection_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash common mutating production targets are blocked"; then + builder_tester_mutator_miss="" + for command in \ + "sed -i 's/old/new/' hooks/scripts/workflow-guard.sh" \ + "perl -pi -e 's/old/new/' hooks/scripts/workflow-guard.sh" \ + "cp tests/test-hooks.sh hooks/scripts/workflow-guard.sh" \ + "mv tests/generated-test.sh hooks/scripts/workflow-guard.sh" \ + "rm hooks/scripts/workflow-guard.sh" \ + "touch hooks/scripts/workflow-guard.sh" \ + "install tests/test-hooks.sh hooks/scripts/workflow-guard.sh" \ + "truncate -s 0 hooks/scripts/workflow-guard.sh" \ + "dd if=/dev/zero of=hooks/scripts/workflow-guard.sh" \ + "node -e 'const fs=require(\"fs\"); fs.writeFileSync(\"hooks/scripts/workflow-guard.sh\",\"x\")'" \ + "node -e 'const fs=require(\"fs\"); fs.appendFileSync(\"hooks/scripts/workflow-guard.sh\",\"x\")'" \ + "node -e 'const fs=require(\"fs\"); fs.createWriteStream(\"hooks/scripts/workflow-guard.sh\")'"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + builder_tester_mutator_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_mutator_miss" ]]; then + pass + else + fail "$builder_tester_mutator_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash absolute executable production writes are blocked"; then + builder_tester_absolute_executable_miss="" + for command in \ + "/usr/bin/sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "/bin/rm hooks/scripts/workflow-guard.sh" \ + "/bin/bash -c \"touch hooks/scripts/workflow-guard.sh\"" \ + "/usr/bin/python3 -c 'open(\"hooks/scripts/workflow-guard.sh\",\"w\").write(\"x\")'" \ + "/usr/bin/perl -pi -e 's/old/new/' hooks/scripts/workflow-guard.sh" \ + "/usr/bin/env /usr/bin/sed -i s/old/new/ hooks/scripts/workflow-guard.sh"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "hooks/scripts/workflow-guard.sh" \ + && ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_absolute_executable_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_absolute_executable_miss" ]]; then + pass + else + fail "$builder_tester_absolute_executable_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash assignment-prefixed production writes are blocked"; then + builder_tester_assignment_miss="" + for command in \ + "FOO=1 sed -i 's/old/new/' hooks/scripts/workflow-guard.sh" \ + "FOO=1 touch hooks/scripts/workflow-guard.sh" \ + "FOO=1 bash -c \"touch hooks/scripts/workflow-guard.sh\"" \ + "FOO=1 bash -c \"sed -i s/old/new/ hooks/scripts/workflow-guard.sh\""; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + builder_tester_assignment_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_assignment_miss" ]]; then + pass + else + fail "$builder_tester_assignment_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash nested production wrappers are blocked"; then + builder_tester_wrapper_miss="" + for command in \ + "bash -c \"sed -i s/old/new/ hooks/scripts/workflow-guard.sh\"" \ + "sh -c \"rm hooks/scripts/workflow-guard.sh\"" \ + "zsh -c \"touch hooks/scripts/workflow-guard.sh\"" \ + "env node -e 'const fs=require(\"fs\"); fs.writeFileSync(\"hooks/scripts/workflow-guard.sh\",\"x\")'" \ + "env bash -c \"sed -i s/old/new/ hooks/scripts/workflow-guard.sh\""; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "Builder/Tester may only edit test files and build configuration"; then + builder_tester_wrapper_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_wrapper_miss" ]]; then + pass + else + fail "$builder_tester_wrapper_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash shell command wrappers fail closed"; then + builder_tester_shell_wrapper_miss="" + for command in \ + "command sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "exec sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "time sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "builtin eval 'touch hooks/scripts/workflow-guard.sh'" \ + "FOO=1 command sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "FOO=1 exec sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "FOO=1 time sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "FOO=1 builtin eval 'touch hooks/scripts/workflow-guard.sh'"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_shell_wrapper_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_shell_wrapper_miss" ]]; then + pass + else + fail "$builder_tester_shell_wrapper_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash env, nice, and common executor wrappers fail closed"; then + builder_tester_env_executor_miss="" + for command in \ + "env xargs sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "env -i xargs sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "env time sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "nice sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "nohup sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "timeout 5 sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "stdbuf -o0 sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "flock /tmp/lock sed -i s/old/new/ hooks/scripts/workflow-guard.sh"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_env_executor_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_env_executor_miss" ]]; then + pass + else + fail "$builder_tester_env_executor_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash reserved precommands fail closed"; then + builder_tester_reserved_precommand_miss="" + for command in \ + "! sed -i s/old/new/ hooks/scripts/workflow-guard.sh" \ + "! touch hooks/scripts/workflow-guard.sh" \ + "! bash -c \"touch hooks/scripts/workflow-guard.sh\"" \ + "coproc sed -i s/old/new/ hooks/scripts/workflow-guard.sh"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_reserved_precommand_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_reserved_precommand_miss" ]]; then + pass + else + fail "$builder_tester_reserved_precommand_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash opaque wrapper writes fail closed"; then + builder_tester_opaque_miss="" + for command in \ + "bash -c \"\$MUTATING_COMMAND\"" \ + "sh -c 'sed -i s/old/new/ \$TARGET_FILE'" \ + "zsh -c \"touch \$TARGET_FILE\"" \ + "env bash -c \"\$MUTATING_COMMAND\"" \ + "env node -e 'const fs=require(\"fs\"); fs.writeFileSync(process.env.TARGET_FILE,\"x\")'"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_opaque_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_opaque_miss" ]]; then + pass + else + fail "$builder_tester_opaque_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash command substitutions fail closed"; then + builder_tester_substitution_miss="" + for command in \ + "\$(sed -i s/old/new/ hooks/scripts/workflow-guard.sh)" \ + "\`sed -i s/old/new/ hooks/scripts/workflow-guard.sh\`" \ + "\`touch hooks/scripts/workflow-guard.sh\`"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_substitution_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_substitution_miss" ]]; then + pass + else + fail "$builder_tester_substitution_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash dynamic mutating targets fail closed"; then + builder_tester_dynamic_miss="" + for command in \ + "sed -i 's/old/new/' \"\$TARGET_FILE\"" \ + "perl -pi -e 's/old/new/' \"\$TARGET_FILE\"" \ + "cp tests/fixture.sh \"\$TARGET_FILE\"" \ + "mv tests/generated-test.sh \"\$TARGET_FILE\"" \ + "rm \"\$TARGET_FILE\"" \ + "touch \"\$TARGET_FILE\"" \ + "install tests/fixture.sh \"\$TARGET_FILE\"" \ + "truncate -s 0 \"\$TARGET_FILE\"" \ + "dd if=/dev/zero of=\"\$TARGET_FILE\"" \ + "node -e 'const fs=require(\"fs\"); fs.writeFileSync(process.env.TARGET_FILE,\"x\")'"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_dynamic_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_dynamic_miss" ]]; then + pass + else + fail "$builder_tester_dynamic_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash shell functions fail closed"; then + builder_tester_function_miss="" + for command in \ + "f(){ sed -i s/old/new/ hooks/scripts/workflow-guard.sh; }; f" \ + "function f { touch hooks/scripts/workflow-guard.sh; }; f"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_function_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_function_miss" ]]; then + pass + else + fail "$builder_tester_function_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash loops and case fail closed"; then + builder_tester_compound_miss="" + for command in \ + "for f in hooks/scripts/workflow-guard.sh; do sed -i s/old/new/ \"\$f\"; done" \ + "while true; do touch hooks/scripts/workflow-guard.sh; break; done" \ + "until false; do touch hooks/scripts/workflow-guard.sh; break; done" \ + "case hooks/scripts/workflow-guard.sh in hooks/*) touch hooks/scripts/workflow-guard.sh ;; esac"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_compound_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_compound_miss" ]]; then + pass + else + fail "$builder_tester_compound_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash grouping and subshell fail closed"; then + builder_tester_grouping_miss="" + for command in \ + "{ sed -i s/old/new/ hooks/scripts/workflow-guard.sh; }" \ + "( sed -i s/old/new/ hooks/scripts/workflow-guard.sh )"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_grouping_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_grouping_miss" ]]; then + pass + else + fail "$builder_tester_grouping_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash alias eval and source fail closed"; then + builder_tester_indirection_miss="" + for command in \ + "alias mutate='touch hooks/scripts/workflow-guard.sh'; mutate" \ + "eval 'touch hooks/scripts/workflow-guard.sh'" \ + "source tests/generated-test.sh" \ + ". tests/generated-test.sh"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_indirection_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_indirection_miss" ]]; then + pass + else + fail "$builder_tester_indirection_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash find actions fail closed"; then + builder_tester_find_miss="" + for command in \ + "find hooks -name workflow-guard.sh -exec sed -i s/old/new/ {} \\;" \ + "find hooks -name workflow-guard.sh -execdir sed -i s/old/new/ {} \\;" \ + "find hooks -name workflow-guard.sh -delete" \ + "find hooks -name workflow-guard.sh -ok sed -i s/old/new/ {} \\;" \ + "find hooks -name workflow-guard.sh -okdir sed -i s/old/new/ {} \\;" \ + "find hooks -name workflow-guard.sh -fprint hooks/scripts/workflow-guard.sh" \ + "find hooks -name workflow-guard.sh -fprint0 hooks/scripts/workflow-guard.sh" \ + "find hooks -name workflow-guard.sh -fprintf hooks/scripts/workflow-guard.sh '%p\n'" \ + "find hooks -name workflow-guard.sh -fls hooks/scripts/workflow-guard.sh"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_find_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_find_miss" ]]; then + pass + else + fail "$builder_tester_find_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash inline interpreter evals fail closed"; then + builder_tester_inline_eval_miss="" + for command in \ + "ruby -e 'File.write(\"hooks/scripts/workflow-guard.sh\", \"x\")'" \ + "perl -e 'open(my \$fh, \">\", \"hooks/scripts/workflow-guard.sh\"); print \$fh \"x\"'" \ + "php -r 'file_put_contents(\"hooks/scripts/workflow-guard.sh\", \"x\");'" \ + "awk 'BEGIN { print \"x\" > \"hooks/scripts/workflow-guard.sh\" }'"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_inline_eval_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_inline_eval_miss" ]]; then + pass + else + fail "$builder_tester_inline_eval_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash combined inline eval flags fail closed"; then + builder_tester_combined_inline_eval_miss="" + for command in \ + "ruby -we 'File.write(\"hooks/scripts/workflow-guard.sh\", \"x\")'" \ + "perl -we 'open(my \$fh, \">\", \"hooks/scripts/workflow-guard.sh\"); print \$fh \"x\"'"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_combined_inline_eval_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_combined_inline_eval_miss" ]]; then + pass + else + fail "$builder_tester_combined_inline_eval_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash opaque Python filesystem mutations fail closed"; then + builder_tester_python_opaque_miss="" + for command in \ + "python3 -c 'import shutil; shutil.copyfile(\"tests/test-hooks.sh\",\"hooks/scripts/workflow-guard.sh\")'" \ + "python3 -c 'import shutil; shutil.copytree(\"tests\", \"hooks/scripts/workflow-guard.d/copied-tests\")'" \ + "python3 -c 'from shutil import copytree; copytree(\"tests\", \"hooks/scripts/workflow-guard.d/copied-tests\")'" \ + "python3 -c 'import os; os.remove(\"hooks/scripts/workflow-guard.sh\")'" \ + "python3 -c 'from os import replace; replace(\"tests/generated-test.sh\",\"hooks/scripts/workflow-guard.sh\")'" \ + "python3 -c 'from pathlib import Path; Path(\"hooks/scripts/generated.sh\").touch()'" \ + "python3 -c 'from pathlib import Path; Path(\"hooks/scripts/generated-dir\").mkdir()'" \ + "python3 -c 'from pathlib import Path; Path(\"hooks/scripts/generated-dir\").rmdir()'" \ + "python3 -c 'from pathlib import Path; Path(\"tests/generated-test.sh\").rename(\"hooks/scripts/workflow-guard.sh\")'" \ + "python3 -c 'from pathlib import Path; Path(\"tests/generated-test.sh\").replace(\"hooks/scripts/workflow-guard.sh\")'" \ + "python3 -c 'import os as o; o.replace(\"tests/generated-test.sh\", \"hooks/scripts/workflow-guard.sh\")'" \ + "python3 -c 'import shutil as s; s.copytree(\"tests\", \"hooks/scripts/workflow-guard.d/copied-tests\")'" \ + "python3 -c 'from os import remove as r; r(\"hooks/scripts/workflow-guard.sh\")'" \ + "python3 -c 'from shutil import copyfile as cf; cf(\"tests/test-hooks.sh\", \"hooks/scripts/workflow-guard.sh\")'" \ + "python3 -c 'import os, shutil as s; s.copyfile(\"tests/test-hooks.sh\", \"hooks/scripts/workflow-guard.sh\")'" \ + "python3 -c 'from pathlib import Path; Path(\"hooks/scripts/workflow-guard.sh\").unlink()'"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_python_opaque_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_python_opaque_miss" ]]; then + pass + else + fail "$builder_tester_python_opaque_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash xargs command execution fails closed"; then + builder_tester_xargs_miss="" + for command in \ + "printf '%s\n' hooks/scripts/workflow-guard.sh | xargs sed -i s/old/new/" \ + "printf '%s\n' hooks/scripts/workflow-guard.sh | xargs -I{} sh -c 'touch \"{}\"'"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_xargs_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_xargs_miss" ]]; then + pass + else + fail "$builder_tester_xargs_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash non-apply-patch heredoc and herestring fail closed"; then + builder_tester_here_miss="" + heredoc_command=$(printf "cat <<'EOF' > hooks/scripts/workflow-guard.sh\nx\nEOF") + herestring_command="cat > hooks/scripts/workflow-guard.sh <<< x" + for command in "$heredoc_command" "$herestring_command"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_here_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_here_miss" ]]; then + pass + else + fail "$builder_tester_here_miss" + fi +fi + +if test_start "workflow-guard: builder-tester Bash process substitution fails closed"; then + builder_tester_process_substitution_miss="" + for command in \ + "cat <(sed -i s/old/new/ hooks/scripts/workflow-guard.sh)" \ + "cat tests/test-hooks.sh >(sed -i s/old/new/ hooks/scripts/workflow-guard.sh)"; do + run_workflow_guard_builder_tester_bash "$command" + if ! assert_claude_pretooluse_deny_contains "unproven shell write target"; then + builder_tester_process_substitution_miss="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_process_substitution_miss" ]]; then + pass + else + fail "$builder_tester_process_substitution_miss" + fi +fi + +if test_start "workflow-guard: builder-tester test and build-config edits are allowed"; then + builder_tester_block="" + for target_path in "tests/test-hooks.sh" "tools/memory-graph/tests/MemoryGraph.Tests/MemoryGraph.Tests.csproj" "Directory.Build.props"; do + printf '{"tool_name":"Edit","agent_type":"builder-tester","tool_input":{"file_path":"%s"}}\n' "$target_path" | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if [[ $HOOK_EXIT -ne 0 ]] || echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1; then + builder_tester_block="$target_path exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_block" ]]; then + pass + else + fail "$builder_tester_block" + fi +fi + +if test_start "workflow-guard: builder-tester Bash test and build-config writes are allowed"; then + builder_tester_block="" + for command in \ + "printf x | tee tests/test-hooks.sh" \ + "printf x > tests/test-hooks.sh" \ + "python3 -c 'open(\"tests/test-hooks.sh\",\"w\").write(\"x\")'" \ + "/usr/bin/python3 -c 'open(\"tests/test-hooks.sh\",\"w\").write(\"x\")'" \ + "python3 -c 'open(\"tests/test-hooks.sh\", mode=\"w\").write(\"x\")'" \ + "python3 -c 'from pathlib import Path; Path(\"tests/generated-test.sh\").write_text(\"x\")'" \ + "python3 -c 'from pathlib import Path; Path(\"tests/generated-output.txt\").write_text(\"ok\")'" \ + "python3 -c 'from pathlib import Path; Path(\"tests/generated-output.txt\").open(\"w\").write(\"ok\")'" \ + "python3 -c 'from pathlib import Path; Path(\"tests/generated-output.txt\").open(mode=\"w\").write(\"ok\")'" \ + "printf x | tee tools/memory-graph/tests/MemoryGraph.Tests/MemoryGraph.Tests.csproj" \ + "printf x > Directory.Build.props" \ + "python3 -c 'open(\"tools/memory-graph/tests/MemoryGraph.Tests/MemoryGraph.Tests.csproj\",\"w\").write(\"x\")'" \ + "touch tests/generated-test.sh" \ + "cp tests/fixture.sh tests/generated-test.sh" \ + "sed -i 's/old/new/' tests/test-hooks.sh" \ + "truncate -s 0 Directory.Build.props"; do + run_workflow_guard_builder_tester_bash "$command" + if [[ $HOOK_EXIT -ne 0 ]] \ + || echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.permissionDecision == "deny" or .decision == "block"' >/dev/null 2>&1; then + builder_tester_block="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_block" ]]; then + pass + else + fail "$builder_tester_block" + fi +fi + +if test_start "workflow-guard: builder-tester Bash wrapper test redirections are allowed"; then + builder_tester_block="" + for command in \ + "bash -lc 'echo hi > tests/generated.txt'" \ + "sh -c 'printf hi > tests/generated.txt'" \ + "zsh -c 'echo hi > tests/generated.txt'" \ + "env bash -lc 'echo hi > tests/generated.txt'" \ + "bash -lc 'echo hi' > tests/generated.txt"; do + run_workflow_guard_builder_tester_bash "$command" + if [[ $HOOK_EXIT -ne 0 ]] \ + || echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.permissionDecision == "deny" or .decision == "block"' >/dev/null 2>&1; then + builder_tester_block="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_block" ]]; then + pass + else + fail "$builder_tester_block" + fi +fi + +if test_start "workflow-guard: builder-tester Bash nested test writes are allowed"; then + builder_tester_block="" + for command in \ + "bash -c \"touch tests/generated-test.sh\"" \ + "sh -c \"sed -i s/old/new/ tests/test-hooks.sh\"" \ + "zsh -c \"touch tests/generated-test.sh\"" \ + "env bash -c \"touch tests/generated-test.sh\"" \ + "env node -e 'const fs=require(\"fs\"); fs.writeFileSync(\"tests/generated-test.sh\",\"x\")'"; do + run_workflow_guard_builder_tester_bash "$command" + if [[ $HOOK_EXIT -ne 0 ]] \ + || echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.permissionDecision == "deny" or .decision == "block"' >/dev/null 2>&1; then + builder_tester_block="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_block" ]]; then + pass + else + fail "$builder_tester_block" + fi +fi + +if test_start "workflow-guard: builder-tester dotnet build and test commands are allowed"; then + builder_tester_block="" + for command in \ + "dotnet build tools/memory-graph/src/MemoryGraph/MemoryGraph.csproj --tl:on -v:minimal" \ + "dotnet test tools/memory-graph/tests/MemoryGraph.Tests/MemoryGraph.Tests.csproj --tl:on -v:minimal"; do + run_workflow_guard_builder_tester_bash "$command" + if [[ $HOOK_EXIT -ne 0 ]] \ + || echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.permissionDecision == "deny" or .decision == "block"' >/dev/null 2>&1; then + builder_tester_block="command=$command exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_block" ]]; then + pass + else + fail "$builder_tester_block" + fi +fi + +if test_start "workflow-guard: builder-tester Bash filtered hook tests are allowed"; then + run_workflow_guard_builder_tester_bash "bash tests/test-hooks.sh --filter workflow-guard" + if [[ $HOOK_EXIT -eq 0 ]] \ + && ! echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.permissionDecision == "deny" or .decision == "block"' >/dev/null 2>&1; then + pass + else + fail "exit=$HOOK_EXIT, expected filtered hook-test Bash command to be allowed; stdout='$HOOK_STDOUT'" + fi +fi + +if test_start "workflow-guard: builder-tester Bash apply_patch test and build-config edits are allowed"; then + builder_tester_block="" + for target_path in "tests/test-hooks.sh" "tools/memory-graph/tests/MemoryGraph.Tests/MemoryGraph.Tests.csproj"; do + jq -n --arg command "$(printf "apply_patch <<'PATCH'\n*** Begin Patch\n*** Update File: %s\n@@\n-old\n+new\n*** End Patch\nPATCH" "$target_path")" \ + '{tool_name: "Bash", agent_type: "builder-tester", tool_input: {command: $command}}' | \ + HOME="$TEST_AGENT_HOME" CLAUDE_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if [[ $HOOK_EXIT -ne 0 ]] \ + || echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.permissionDecision == "deny" or .decision == "block"' >/dev/null 2>&1; then + builder_tester_block="$target_path exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$builder_tester_block" ]]; then + pass + else + fail "$builder_tester_block" + fi +fi + if test_start "workflow-guard: Edit but no task journal → no output"; then rm -f "$TEST_PROJECT/.claude/task.md" echo '{"tool_name": "Edit", "tool_input": {}}' | \ @@ -6276,6 +7583,64 @@ if test_start "workflow-guard: Codex apply_patch only state artifacts → no out rm -f "$TEST_PROJECT/.codex/task.md" fi +if test_start "workflow-guard: Codex outside state artifact paths do not bypass unresolved policy"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + mkdir -p "$TEST_PROJECT/.codex" + echo -e "Status: BUILDING [step 2/3]" > "$TEST_PROJECT/.codex/task.md" + outside_state_path="$(dirname "$TEST_PROJECT")/outside-project/.codex/task.md" + state_bypass_miss="" + + for target_path in "../outside-project/.codex/task.md" "$outside_state_path"; do + jq -n --arg path "$target_path" \ + '{tool_name: "Edit", tool_input: {file_path: $path}}' | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if [[ $HOOK_EXIT -ne 0 ]] || ! echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ + || ! echo "$HOOK_STDOUT" | grep -q "subagent policy is unresolved"; then + state_bypass_miss="Edit path=$target_path exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + + if [[ -z "$state_bypass_miss" ]]; then + jq -n --arg patch $'*** Begin Patch\n*** Update File: ../outside-project/.codex/task.md\n@@\n-old\n+new\n*** End Patch' \ + '{tool_name: "apply_patch", tool_input: {patch: $patch}}' | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if [[ $HOOK_EXIT -ne 0 ]] || ! echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ + || ! echo "$HOOK_STDOUT" | grep -q "subagent policy is unresolved"; then + state_bypass_miss="apply_patch traversal exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + fi + fi + + if [[ -z "$state_bypass_miss" ]]; then + jq -n --arg patch "$(printf "*** Begin Patch\n*** Update File: %s\n@@\n-old\n+new\n*** End Patch" "$outside_state_path")" \ + '{tool_name: "apply_patch", tool_input: {patch: $patch}}' | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ + > /tmp/_wg_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wg_out) + rm -f /tmp/_wg_out + if [[ $HOOK_EXIT -ne 0 ]] || ! echo "$HOOK_STDOUT" | jq -e '.decision == "block"' >/dev/null 2>&1 \ + || ! echo "$HOOK_STDOUT" | grep -q "subagent policy is unresolved"; then + state_bypass_miss="apply_patch absolute exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + fi + fi + + if [[ -z "$state_bypass_miss" ]]; then + pass + else + fail "$state_bypass_miss" + fi + rm -rf "$TEST_PROJECT/.codex" +fi + if test_start "workflow-guard: Codex apply_patch mixed source and state with unresolved policy → blocks"; then mkdir -p "$TEST_PROJECT/.codex" echo -e "Status: BUILDING [step 2/3]" > "$TEST_PROJECT/.codex/task.md" @@ -6297,8 +7662,10 @@ fi if test_start "workflow-guard: Codex delegated Build without Code Writer event → blocks"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +Created: workflow-guard-missing-code-writer-task Status: BUILDING Triaged as: small Subagent policy state: delegation_authorized @@ -6325,8 +7692,10 @@ fi if test_start "workflow-guard: Codex delegated Build with Code Writer start event → warning only"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + clear_workflow_cache mkdir -p "$TEST_PROJECT/.codex" cat > "$TEST_PROJECT/.codex/task.md" <<'TASK' +Created: workflow-guard-code-writer-start-task Status: BUILDING Triaged as: small Subagent policy state: delegation_authorized @@ -6336,7 +7705,7 @@ Required agents: ## Agent Dispatch Log - Code Writer dispatch: event cw-1 TASK - echo '{"event":"SubagentStart","agent_type":"code-writer","agent_name":"code-writer","agent_id":"cw-1"}' > "$TEST_PROJECT/.codex/subagent-events.jsonl" + record_codex_subagent_event "SubagentStart" "code-writer" "cw-1" echo '{"tool_name": "apply_patch", "tool_input": {"command": "*** Begin Patch\n*** Update File: src/App.cs\n*** End Patch"}}' | \ HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-guard.sh" \ > /tmp/_wg_out 2>/dev/null @@ -6784,6 +8153,10 @@ if test_start "codex strict install: workflow-guard installs and legacy post-too fail "missing task-journal-resolver.sh after install" elif [[ ! -f "$INSTALL_TEST_HOME/.codex/hooks/assistant/workflow-phase-gates.sh" ]]; then fail "missing workflow-phase-gates.sh after install" + elif [[ ! -f "$INSTALL_TEST_HOME/.codex/hooks/assistant/workflow-guard.d/path-policy.sh" \ + || ! -f "$INSTALL_TEST_HOME/.codex/hooks/assistant/workflow-guard.d/shell-write-parser.sh" \ + || ! -f "$INSTALL_TEST_HOME/.codex/hooks/assistant/workflow-guard.d/workflow-state-artifacts.sh" ]]; then + fail "missing workflow-guard.d module after install" elif [[ ! -x "$INSTALL_TEST_HOME/.codex/hooks/assistant/post-tool-context.sh" \ || ! -x "$INSTALL_TEST_HOME/.codex/hooks/assistant/tool-failure-advisor.sh" ]]; then fail "legacy post-tool shim scripts should be installed as executable no-ops" @@ -6860,17 +8233,30 @@ if test_start "codex default install: workflow hooks install and compaction hook pre-compress.sh \ post-compact.sh \ task-journal-resolver.sh \ - workflow-phase-gates.sh; do + workflow-phase-gates.sh \ + hook-runtime.sh; do if [[ ! -x "$INSTALL_TEST_HOME/.codex/hooks/assistant/$required_hook" ]]; then missing_default_hook="$required_hook" break fi done + missing_workflow_guard_module="" + for required_guard_module in path-policy.sh shell-write-parser.sh workflow-state-artifacts.sh; do + if [[ ! -f "$INSTALL_TEST_HOME/.codex/hooks/assistant/workflow-guard.d/$required_guard_module" ]]; then + missing_workflow_guard_module="$required_guard_module" + break + fi + done + if [[ $HOOK_EXIT -ne 0 ]]; then fail "install exit=$HOOK_EXIT, stderr='$INSTALL_STDERR'" elif [[ -n "$missing_default_hook" ]]; then fail "Codex default install did not create executable $missing_default_hook" + elif [[ -n "$missing_workflow_guard_module" ]]; then + fail "Codex default install did not copy workflow-guard.d/$missing_workflow_guard_module" + elif [[ ! -f "$INSTALL_TEST_HOME/.codex/hooks/assistant/workflow-phase-gates.d/subagent-evidence.sh" ]]; then + fail "Codex default install did not copy workflow-phase-gates.d/subagent-evidence.sh" elif ! jq -e --arg command_dir "$INSTALL_TEST_HOME/.codex/hooks/assistant" ' { sessionStart: ([.hooks.SessionStart[]?.hooks[]?.command?] | any(. == ($command_dir + "/session-start.sh"))), @@ -7044,6 +8430,66 @@ if test_start "workflow-enforcer: Codex dev prompt without task asks once for de fi fi +if test_start "workflow-enforcer: Codex subagent approval parser rejects question quoted and meta prompts"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + parser_failure="" + for prompt in \ + "should I use agents to fix the hook?" \ + "The phrase \"use agents\" is only an example; fix the hook" \ + "do not assume use agents is approval; fix the hook"; do + jq -cn --arg prompt "$prompt" '{prompt: $prompt, hook_event_name: "UserPromptSubmit"}' | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-enforcer.sh" \ + > /tmp/_wf_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wf_out) + rm -f /tmp/_wf_out + if [[ $HOOK_EXIT -ne 0 ]] \ + || ! echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.additionalContext' >/dev/null 2>&1 \ + || ! echo "$HOOK_STDOUT" | grep -q "CODEX SUBAGENT AUTHORIZATION (ask-once)" \ + || echo "$HOOK_STDOUT" | grep -q "CODEX SUBAGENT AUTHORIZATION (denied)"; then + parser_failure="prompt='$prompt' exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$parser_failure" ]]; then + pass + else + fail "$parser_failure" + fi +fi + +if test_start "workflow-enforcer: Codex subagent approval parser accepts bounded approval prompts"; then + rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" + parser_failure="" + for prompt in \ + "Use delegation." \ + "yes, use subagents" \ + "approve subagents for this task" \ + "approve use subagents" \ + "delegate this work in parallel" \ + "spawn two agents" \ + "use one agent per point"; do + jq -cn --arg prompt "$prompt" '{prompt: $prompt, hook_event_name: "UserPromptSubmit"}' | \ + HOME="$TEST_AGENT_HOME" CODEX_PROJECT_DIR="$TEST_PROJECT" bash "$HOOKS_DIR/workflow-enforcer.sh" \ + > /tmp/_wf_out 2>/dev/null + HOOK_EXIT=$? + HOOK_STDOUT=$(cat /tmp/_wf_out) + rm -f /tmp/_wf_out + if [[ $HOOK_EXIT -ne 0 ]] \ + || ! echo "$HOOK_STDOUT" | jq -e '.hookSpecificOutput.additionalContext' >/dev/null 2>&1 \ + || echo "$HOOK_STDOUT" | grep -q "CODEX SUBAGENT AUTHORIZATION (ask-once)" \ + || echo "$HOOK_STDOUT" | grep -q "CODEX SUBAGENT AUTHORIZATION (denied)"; then + parser_failure="prompt='$prompt' exit=$HOOK_EXIT stdout='$HOOK_STDOUT'" + break + fi + done + if [[ -z "$parser_failure" ]]; then + pass + else + fail "$parser_failure" + fi +fi + if test_start "workflow-enforcer: Codex completed task dev prompt asks once for delegation authorization"; then rm -rf "$TEST_PROJECT/.claude" "$TEST_PROJECT/.gemini" "$TEST_PROJECT/.codex" mkdir -p "$TEST_PROJECT/.codex" diff --git a/tools/hooks/benchmark-hook-output.sh b/tools/hooks/benchmark-hook-output.sh index 50be1ce..044686a 100755 --- a/tools/hooks/benchmark-hook-output.sh +++ b/tools/hooks/benchmark-hook-output.sh @@ -4,6 +4,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" FRAMEWORK_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" HOOKS_DIR="$FRAMEWORK_DIR/hooks/scripts" +. "$HOOKS_DIR/hook-runtime.sh" METRIC_COLUMNS=(hook_name scenario stdout_bytes stdout_words stderr_bytes exit_code first_blocker_or_action) @@ -85,6 +86,10 @@ new_fixture() { mkdir -p "$PROJECT_DIR/.codex" "$AGENT_HOME/.codex" } +codex_lifecycle_events_file() { + HOME="$AGENT_HOME" assistant_hook_codex_subagent_events_file_for_project "$PROJECT_DIR" +} + run_hook_scenario() { local hook_name="$1" local scenario="$2" @@ -260,7 +265,7 @@ run_subagent_monitor_start() { --arg cwd "$PROJECT_DIR" \ '{hook_event_name:"SubagentStart",agent_type:"code-writer",agent_id:"cw-bench-1",turn_id:"turn-bench",session_id:"session-bench",agent_transcript_path:"/tmp/benchmark-transcript",cwd:$cwd}')" run_hook_scenario "subagent-monitor" "codex code-writer start" "$stdin_json" \ - "recorded SubagentStart lifecycle evidence" "$PROJECT_DIR/.codex/subagent-events.jsonl" \ + "recorded SubagentStart lifecycle evidence" "$(codex_lifecycle_events_file)" \ "SUBAGENT CONSTRAINT" "SubagentStart" "code-writer" } @@ -277,7 +282,7 @@ run_subagent_monitor_stop() { env HOME="$AGENT_HOME" CODEX_PROJECT_DIR="$PROJECT_DIR" bash "$HOOKS_DIR/subagent-monitor.sh" \ >/dev/null 2>/dev/null <<< "$stdin_start" run_hook_scenario "subagent-monitor" "codex code-writer stop" "$stdin_stop" \ - "recorded SubagentStop lifecycle evidence" "$PROJECT_DIR/.codex/subagent-events.jsonl" \ + "recorded SubagentStop lifecycle evidence" "$(codex_lifecycle_events_file)" \ "SubagentStart" "SubagentStop" "cw-bench-1" } @@ -327,7 +332,7 @@ EOF - `subagent-monitor` / `codex code-writer start`: `SUBAGENT CONSTRAINT`, `SubagentStart`, `code-writer` - `subagent-monitor` / `codex code-writer stop`: `SubagentStart`, `SubagentStop`, `cw-bench-1` -The subagent stop scenario emits no user-facing stdout in the current hook behavior. The benchmark records the locally feasible lifecycle evidence by checking the project-local `.codex/subagent-events.jsonl` file written by `subagent-monitor.sh`. +The subagent stop scenario emits no user-facing stdout in the current hook behavior. The benchmark records the locally feasible lifecycle evidence by checking the protected agent-owned workflow-state event file written by `subagent-monitor.sh`; project-local `.codex/subagent-events.jsonl` files are diagnostic only. EOF } From d766e7f7e12d771767108a715953a5d0767638f7 Mon Sep 17 00:00:00 2001 From: Laimis Date: Mon, 6 Jul 2026 11:43:42 +0300 Subject: [PATCH 7/9] Add loop readiness research guardrails --- .../assistant-workflow/contracts/output.yaml | 138 +++++++++++++++++- .../contracts/phase-gates.yaml | 10 ++ .../assistant-workflow/evals/cases.json | 60 ++++++++ .../references/plan-template.md | 77 ++++------ .../references/task-journal-template.md | 49 +++---- .../skills/assistant-research/SKILL.md | 6 +- .../assistant-research/contracts/output.yaml | 60 ++++++++ .../contracts/phase-gates.yaml | 15 ++ .../assistant-research/evals/cases.json | 59 ++++++++ .../skills/assistant-research/research.md | 15 ++ skills/assistant-research/SKILL.md | 6 +- .../assistant-research/contracts/output.yaml | 60 ++++++++ .../contracts/phase-gates.yaml | 15 ++ skills/assistant-research/evals/cases.json | 59 ++++++++ skills/assistant-research/research.md | 15 ++ .../assistant-workflow/contracts/output.yaml | 138 +++++++++++++++++- .../contracts/phase-gates.yaml | 10 ++ skills/assistant-workflow/evals/cases.json | 60 ++++++++ .../references/plan-template.md | 77 ++++------ .../references/task-journal-template.md | 49 +++---- tests/p0-p4/assistant-research-contracts.sh | 55 +++++++ tests/p0-p4/workflow-basics-contracts.sh | 58 ++++++++ tests/test-p0-p4-contracts.sh | 1 + 23 files changed, 926 insertions(+), 166 deletions(-) create mode 100644 tests/p0-p4/assistant-research-contracts.sh diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml index 31e1243..e033d33 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml @@ -13,17 +13,17 @@ skill: assistant-workflow completion_tiers: small: required_artifacts: [changed_files, validation_results, manual_test_steps, user_approval] - conditional_artifacts: [triage_result, subagent_evidence, test_results, debugging_result, artifact_contract] + conditional_artifacts: [triage_result, subagent_evidence, test_results, debugging_result, artifact_contract, workflow_experiment_ledger, loop_readiness_assessment] review_gate: "self-review or explicit validation summary unless risk_tier in [high, critical]" checkpoint_policy: "inline plan/check summary is enough; phase_checkpoints are strict-profile only" medium: required_artifacts: [triage_result, changed_files, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, plan_document, user_approval] - conditional_artifacts: [test_results, task_journal, context_map, slice_manifest, single_slice_rationale, slice_verification_summary, metrics_entry, candidate_search_result, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result] + conditional_artifacts: [test_results, task_journal, context_map, slice_manifest, single_slice_rationale, slice_verification_summary, metrics_entry, candidate_search_result, workflow_experiment_ledger, loop_readiness_assessment, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result] review_gate: "spec review plus code quality review, with independent QA evaluation when required and evidence matched to the approved plan" checkpoint_policy: "task journal or carried-forward state records phase transitions" large_critical: required_artifacts: [triage_result, phase_checkpoints, changed_files, test_results, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, task_journal, context_map, context_budget_note, slice_manifest, slice_verification_summary, plan_document, user_approval] - conditional_artifacts: [debugging_result, artifact_contract, pattern_retrieval_note, decomposition_plan_review, single_slice_rationale, metrics_entry, candidate_search_result, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result, verified_skill_distillation] + conditional_artifacts: [debugging_result, artifact_contract, pattern_retrieval_note, decomposition_plan_review, single_slice_rationale, metrics_entry, candidate_search_result, workflow_experiment_ledger, loop_readiness_assessment, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result, verified_skill_distillation] review_gate: "full strict evidence, security/operability review when applicable, QA evaluation when required, and all slice criteria verified" checkpoint_policy: "explicit phase checkpoints are required" @@ -1290,6 +1290,132 @@ artifacts: condition: "candidate search occurred after plan approval or selected candidate changes approved scope/files/behavior/risk" validation: "Records deviation, impact, rollback/re-approval status, and user approval when required" + - name: workflow_experiment_ledger + type: object[] + required: conditional + condition: "explicit workflow experiment, loop-optimization experiment, or approved plan labels the work as an experiment" + description: "Lightweight ledger for explicit workflow experiments only; not required for ordinary medium+ work." + validation: "Each entry includes experiment_id, hypothesis, intervention, expected_signal, measurement_method, baseline_or_control, result_status, evidence, decision, and next_check. The ledger does not by itself require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation." + on_fail: "When the approved plan names an explicit workflow experiment, add the missing ledger entry before running the experiment; otherwise record N/A and do not add harness artifacts unless harness_capable == true." + min_items: 1 + object_fields: + - name: experiment_id + type: string + required: true + validation: "Stable local id unique within the workflow task" + - name: hypothesis + type: string + required: true + validation: "Testable claim about workflow behavior, cost, speed, quality, or reliability" + - name: intervention + type: string + required: true + validation: "Specific workflow, contract, prompt, hook, or process change being tried" + - name: expected_signal + type: string + required: true + validation: "Observable signal expected to move if the hypothesis is useful" + - name: measurement_method + type: string + required: true + validation: "Command, metric, review criterion, or inspection that measures the signal" + - name: baseline_or_control + type: string + required: true + validation: "Previous behavior, control condition, or explicit no-baseline reason" + - name: result_status + type: enum + required: true + enum_values: [planned, running, measured, inconclusive, stopped] + - name: evidence + type: string[] + required: true + min_items: 1 + validation: "Plan ref, command result, review note, metric, or explicit pending-evidence marker for planned experiments" + - name: decision + type: enum + required: true + enum_values: [pending, continue, adjust, stop, inconclusive] + - name: next_check + type: string + required: true + validation: "Next measurement, review, or stop point" + + - name: loop_readiness_assessment + type: object + required: conditional + condition: "before starting an explicit repeat or optimization loop outside the standard required workflow phase gates" + description: "Pre-loop gate for repeat/optimization loops; not required for one-pass work or the normal required review loop." + validation: "Records loop_type, loop_trigger, readiness_state, verifier, stop_condition, max_iterations, budget_limit, tool_access, state_tracking, retry_or_empty_result_handling, tool_error_handling, low_confidence_escalation, rollback_or_exit, harness_routing, and evidence before the loop starts. loop artifacts alone do not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation." + on_fail: "Before running the repeat/optimization loop, add the readiness assessment or remove the loop from the plan. Set harness_routing=required only when harness_capable == true or independent QA criteria already apply." + object_fields: + - name: loop_type + type: enum + required: true + enum_values: [repeat, optimization, experiment] + - name: loop_trigger + type: string + required: true + validation: "Why a repeated loop is needed instead of a single planned pass" + - name: readiness_state + type: enum + required: true + enum_values: [ready, blocked] + validation: "Must be ready before the loop starts" + - name: verifier + type: string + required: true + validation: "Check, role, command, metric, or reviewer that judges each iteration" + - name: stop_condition + type: string + required: true + validation: "Concrete pass, budget, stagnation, or failure condition that ends the loop" + - name: max_iterations + type: int + required: true + validation: "Positive finite cap" + - name: budget_limit + type: string + required: true + validation: "Token, time, cost, run-count, or explicit local-effort cap" + - name: tool_access + type: string[] + required: true + validation: "Tools/commands/agents needed for the loop, or explicit none entry" + - name: state_tracking + type: string + required: true + validation: "Where iteration state, results, and decisions are recorded" + - name: retry_or_empty_result_handling + type: string + required: true + validation: "How the loop handles retries, empty verifier results, empty search/tool results, exhausted no-result paths, or no-result exit evidence before another iteration is attempted" + on_missing: fail + - name: tool_error_handling + type: string + required: true + validation: "How tool errors route before another iteration, including retry limits, fallback role/action, blocker classification, or stop condition" + on_missing: fail + - name: low_confidence_escalation + type: string + required: true + validation: "Escalation path when verifier confidence is low, evidence is inconclusive, or repeated iterations do not improve confidence" + on_missing: fail + - name: rollback_or_exit + type: string + required: true + validation: "Action when stop_condition fails or the loop exhausts its cap" + - name: harness_routing + type: enum + required: true + enum_values: [not_required, required] + validation: "not_required unless harness_capable == true or QA criteria already apply" + - name: evidence + type: string[] + required: true + min_items: 1 + validation: "Plan ref, task packet ref, or readiness check evidence" + - name: plan_document type: string required: conditional @@ -1317,8 +1443,10 @@ artifacts: # 5. If qa_evaluation_mode=required: qa_evaluation_result is populated after build/test and code-review evidence; QA evaluation does not replace review_result. Scoped domain rubrics require selected_domain_rubrics and domain_quality_scores; unscoped rubric scoring is invalid. # 6. For medium+ tasks, slice_verification_summary has one VERIFIED entry per slice # 7. candidate_search_result exists when search_mode == candidate_search, includes search_exit_summary, and any plan_deviation is approved before Build -# 8. For medium+ harness-capable work, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision when triggered, and artifact_reference_ledger exist and are current before completion or handoff -# 9. user_approval is confirmed, confirmed_with_issues, or not_required_small for eligible small tasks +# 8. workflow_experiment_ledger exists only for explicit workflow experiments; loop_readiness_assessment exists only before explicit repeat/optimization loops and includes retry_or_empty_result_handling, tool_error_handling, and low_confidence_escalation +# 9. Loop artifacts alone do not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless harness_capable or QA criteria independently apply +# 10. For medium+ harness-capable work, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision when triggered, and artifact_reference_ledger exist and are current before completion or handoff +# 11. user_approval is confirmed, confirmed_with_issues, or not_required_small for eligible small tasks # If any check fails, do NOT print --- WORKFLOW COMPLETE ---. State what is missing. - name: verified_skill_distillation type: object diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml index 5e657a3..4bdcb62 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml @@ -282,6 +282,16 @@ gates: check: "Each task packet includes a deviation/rollback rule, or small inline plan explicitly states how deviations are handled" on_fail: "Add deviation/rollback handling before seeking plan approval" + - id: P_WORKFLOW_EXPERIMENT_LEDGER + check: "When the approved work is an explicit workflow experiment: workflow_experiment_ledger records hypothesis, intervention, expected signal, measurement method, baseline/control, result status, evidence, decision, and next check before the experiment runs" + condition: "explicit workflow experiment, loop-optimization experiment, or approved plan labels the work as an experiment" + on_fail: "Add the workflow_experiment_ledger entry to the plan/task journal before running the experiment, or remove the experiment label. Do not add Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless harness_capable or QA criteria independently apply." + + - id: P_LOOP_READINESS + check: "Before any explicit repeat or optimization loop outside standard required workflow phase gates starts: loop_readiness_assessment records loop type, trigger, readiness state, verifier, stop condition, finite max iterations, budget limit, tool access, state tracking, retry_or_empty_result_handling, tool_error_handling, low_confidence_escalation, rollback/exit action, harness routing, and evidence" + condition: "task will start an explicit repeat or optimization loop" + on_fail: "Add loop_readiness_assessment before the loop starts, including retry/empty-result handling, tool-error routing, and low-confidence escalation, or remove the loop from the plan. Keep harness_routing=not_required unless harness_capable == true or QA criteria independently apply." + - id: CS1 check: "When search_mode == candidate_search: goal tree decomposes the existing acceptance criteria and slice acceptance/verification criteria" condition: "search_mode == candidate_search" diff --git a/plugins/assistant-dev/skills/assistant-workflow/evals/cases.json b/plugins/assistant-dev/skills/assistant-workflow/evals/cases.json index fce3c06..bddef94 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/evals/cases.json +++ b/plugins/assistant-dev/skills/assistant-workflow/evals/cases.json @@ -626,6 +626,66 @@ ] ] } + }, + { + "id": "loop-experiment-artifacts-stay-conditional", + "title": "Loop and experiment artifacts stay conditional", + "category": "loop_readiness_and_experiments", + "purpose": "Checks that workflow experiment and repeat/optimization loop artifacts are required only when explicitly scoped and do not turn ordinary medium work into harness work.", + "prompt": "Plan a medium Assistant Framework workflow experiment to compare two review-loop prompts. Keep ordinary harness routing off unless the plan explicitly needs trace/replay or QA.", + "setup_context": [ + "The assistant-workflow skill instructions are active.", + "The request explicitly labels the work as a workflow experiment.", + "The task may use a bounded repeat/optimization loop, but no harness, trace/replay, Done Contract, or QA acceptance criteria were requested." + ], + "expected_behavior": [ + "Keeps ordinary medium task routing at harness_capable=false unless independent harness criteria apply.", + "Adds workflow_experiment_ledger only because the prompt explicitly asks for a workflow experiment.", + "Adds loop_readiness_assessment only before starting an explicit repeat or optimization loop.", + "Records hypothesis, intervention, expected signal, measurement method, baseline/control, evidence, decision, and next check.", + "Records verifier, stop condition, max iterations, budget limit, tool access, state tracking, retry or empty-result handling, tool-error handling, low-confidence escalation, and rollback/exit before any loop starts.", + "Does not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation from loop artifacts alone." + ], + "pass_criteria": [ + "The response marks workflow_experiment_ledger as conditional on explicit workflow experiments.", + "The response marks loop_readiness_assessment as conditional before repeat/optimization loops.", + "The response keeps harness_capable=false for ordinary medium work.", + "The response says loop artifacts alone do not require harness or QA artifacts." + ], + "fail_signals": [ + "Requires workflow_experiment_ledger for every medium task.", + "Requires loop_readiness_assessment without an explicit repeat or optimization loop.", + "Treats loop artifacts as proof that harness_capable must be true.", + "Requires Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation from loop artifacts alone." + ], + "machine_expectations": { + "required_substrings": [ + "workflow_experiment_ledger", + "explicit workflow experiment", + "loop_readiness_assessment", + "repeat/optimization loop", + "harness_capable=false", + "hypothesis", + "measurement method", + "baseline/control", + "verifier", + "stop condition", + "max iterations", + "budget limit", + "state tracking", + "retry_or_empty_result_handling", + "tool_error_handling", + "low_confidence_escalation", + "loop artifacts alone do not require" + ], + "forbidden_substrings": [ + "workflow_experiment_ledger is required for every medium task", + "loop_readiness_assessment is always required", + "harness_capable=true by default", + "Done Contract is required by loop artifacts", + "QA evaluation is required by loop artifacts" + ] + } } ] } diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md index 321c7ca..f73feea 100755 --- a/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md @@ -1,11 +1,10 @@ # Plan Templates -Three tiers — match ceremony to task size (don't get fancy when N is small). +Three tiers — match ceremony to task size. -Harness details are optional appendices. For medium+ harness-capable work, keep -compact refs in the base plan and load `references/plan-harness-appendix.md` -for the full schema. For non-harness work, record `N/A: [reason]` instead of -copying harness sections into the base plan. +Harness details live in optional appendices. Base plans keep compact refs only: +load `references/plan-harness-appendix.md` for harness-capable work, otherwise +record `N/A: [reason]`. ## Small Tasks — Inline Plan @@ -60,15 +59,11 @@ For Medium and Large/Mega plans, write implementation work as executable task pa - Expected success signal: [exit code 0, passing test name, output marker, etc.] - Evidence to record: - [test result, eval fixture, changed file, review note, or artifact proof] -- Harness routing: - - applies: [required for medium+ harness-capable work; otherwise N/A: reason] - - appendix: [references/plan-harness-appendix.md, or N/A: reason] - - done_contract_ref: [section/ref, or N/A: reason] - - harness_recipe_ref: [section/ref, or N/A: reason] - - harness_run_state_ref: [section/ref, or N/A: reason] - - trace_ledger_ref: [section/ref, or N/A: reason] - - replay_packet_ref: [section/ref, or N/A: reason] - - artifact_reference_ledger_ref: [section/ref, or N/A: reason] +- Loop / Experiment Routing: + - workflow_experiment_ledger: [N/A unless explicit workflow experiment; otherwise ref] + - loop_readiness_assessment: [N/A unless explicit repeat/optimization loop; otherwise ref with retry_or_empty_result_handling, tool_error_handling, low_confidence_escalation] + - loop_harness_routing: [ordinary medium+ keeps harness_capable=false; loop artifacts alone do not require harness/QA artifacts] +- Harness routing: [N/A unless harness_capable=true or QA criteria independently apply; otherwise refs to appendix, Done Contract, Harness Recipe, run state, trace/replay, artifact ledger] - Deviation / rollback rule: - [what to do if required files/behavior differ from plan; include rollback/revert boundary] - Worker status / evidence: @@ -136,16 +131,10 @@ Covers the essentials without Security/Operability overhead. Fill this in during ## Architecture - Current architecture: [identified or "new project"] - Architecture for this change: [Clean/MVVM/Hexagonal/etc.] -- Layer rules: - - [e.g., Domain has no external dependencies] - - [e.g., ViewModels don't reference Views] +- Layer rules: [key dependency boundaries] - Dependency direction: [A → B → C] -- New files placement: - - [file → layer/folder rationale] -- SOLID design notes: - - SRP: [which classes own which responsibility — flag any class with >1 reason to change] - - OCP: [will new variants require modifying existing classes? If yes, plan extension points] - - DIP: [which high-level modules depend on abstractions vs concrete implementations?] +- New files placement: [file → layer/folder rationale] +- SOLID notes: [SRP/OCP/DIP risks or N/A] ## Analysis ### Candidate search summary @@ -185,12 +174,20 @@ Covers the essentials without Security/Operability overhead. Fill this in during - Owner/consumer: [user, reviewer, downstream tool, runtime] - Non-goals/exclusions: [what must not be produced] +## Loop / Experiment Routing + +Only for explicit workflow experiments or explicit repeat/optimization loops. +Ordinary medium+ tasks keep `harness_capable=false`; loop artifacts alone do not +require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact +Reference Ledger, or QA evaluation. + +- workflow_experiment_ledger: [N/A unless explicit workflow experiment; otherwise compact ref with hypothesis/intervention/signal/measurement/baseline/status/evidence/decision/next check] +- loop_readiness_assessment: [N/A unless explicit repeat/optimization loop; otherwise compact ref with loop type/trigger/verifier/stop/max iterations/budget/tool access/state tracking/retry_or_empty_result_handling/tool_error_handling/low_confidence_escalation/rollback/harness routing] + ## Harness Appendix Routing -Required only for medium+ harness-capable work; otherwise keep every field as -`N/A: [reason]`. Load `references/plan-harness-appendix.md` for the full Done -Contract, Harness Recipe, runtime artifact, typed artifact ref, and QA routing -schemas. +Required only for medium+ harness-capable work; otherwise use `N/A: [reason]`. +Load `references/plan-harness-appendix.md` for full harness and QA schemas. - appendix_status: [required | N/A: reason] - done_contract_ref: [section/ref, or N/A: reason] @@ -243,28 +240,6 @@ Start with the Medium template, then add the sections below. - Decomposition Plan Review: reuse Medium fields and keep `- Broad-split rejection:` proof for large/mega slice plans. - Harness Appendix Routing: reuse the Medium compact refs and load `references/plan-harness-appendix.md` only when harness-capable. -## Harness Appendix Routing - -Required only for medium+ harness-capable work; otherwise record -`N/A: [reason]`. Load `references/plan-harness-appendix.md` for full harness, -typed artifact reference, pivot/restart, and QA routing schemas. - -- appendix_status: [required | N/A: reason] -- done_contract_ref: [section/ref, or N/A: reason] -- harness_recipe_ref: [section/ref, or N/A: reason] -- harness_run_state_ref: [section/ref, or N/A: reason] -- trace_ledger_ref: [section/ref, or N/A: reason] -- replay_packet_ref: [section/ref, or N/A: reason] -- artifact_reference_ledger_ref: [section/ref, or N/A: reason] -- qa_evaluation_mode: [required | optional | not_required + reason] - -## Slice manifest from Decompose - -Use the shared Slice Manifest structure above. Paste the approved Decompose manifest verbatim and keep `single_slice_rationale` when exactly one slice exists. - -## Task packets -Use the Executable Task Packet structure above for each approved slice. Order packets by dependency, consume the Decompose slice manifest directly, and keep each slice independently verifiable before the next slice starts. - ## Tests to run - [command]: [what it validates] ``` @@ -281,8 +256,8 @@ Use the Executable Task Packet structure above for each approved slice. Order pa ## Context Budget -- Exact/pinned: [goal, acceptance criteria, safety constraints, exact errors, files in scope, validation requirements] -- Summarized: [logs, tool output, conversation history, repetitive evidence] +- Exact/pinned: [goal, criteria, constraints, errors, scoped files, validation] +- Summarized: [logs, tool output, history, repetitive evidence] - Omitted/deferred: [out-of-scope files/results and why] - Split/delegation plan: [slice/task split when material exceeds one faithful context] diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md index 94b6bae..016d8b7 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md @@ -9,15 +9,10 @@ Write to `{agent_state_dir}/task.md` in the project root when a local state dire ## When to update - When clarification questions are asked, answered, or resolved via explicit `defaults` -- After each Build step completes (update Progress, Artifact Registry, and check off Milestones) -- After each medium+ harness-capable event listed in `references/task-journal-harness-appendix.md` -- After any Pivot/Restart Decision, typed artifact reference update, or QA evaluation result when applicable -- When key decisions are made -- When constraints are added by the user -- After each review cycle pass (append to Review Log — never overwrite) -- At verification summary (all steps done) -- During user review feedback -- During Document, after review/build/user-correction evidence has been checked for durable lessons +- After each Build step: Progress, Artifact Registry, Milestones +- After key decisions, new user constraints, review passes, verification summary, or user review feedback +- After harness-capable events, Pivot/Restart Decisions, typed artifact refs, or QA results when applicable +- During Document, after review/build/user-correction evidence is checked for durable lessons ## Template @@ -50,15 +45,19 @@ Candidate scope scan: - Adjacent surfaces: [tests/docs/contracts/config/mirrors/hooks/runtime surfaces to inspect] - Confidence: [low | medium | high] - Unknowns: [none, or one short scope/risk unknown per line] +Loop / Experiment Routing: +- workflow_experiment_ledger: [N/A unless explicit workflow experiment; otherwise compact ref with id/hypothesis/intervention/signal/measurement/baseline/status/evidence/decision/next_check] +- loop_readiness_assessment: [N/A unless explicit repeat or optimization loop; otherwise compact ref with loop_type/trigger/verifier/stop/max_iterations/budget/tool_access/state_tracking/retry_or_empty_result_handling/tool_error_handling/low_confidence_escalation/rollback/harness_routing/evidence] +- loop_harness_routing: [ordinary medium+ keeps harness_capable=false; loop artifacts alone do not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation; appendix only when harness_capable=true or QA criteria independently apply] Plan approval: [yes/no + date] ## Agent Dispatch Log -[strict subagent evidence inspected by stop-review/phase gates] -- Required roles: Code Writer, Builder/Tester, Code Reviewer; QA Evaluator when `qa_evaluation_mode=required`; Code Mapper/Explorer/Architect by size/risk; Reviewer for legacy compatibility. +[subagent evidence inspected by stop-review/phase gates] +- Required roles: Code Writer, Builder/Tester, Code Reviewer; QA Evaluator when required; Code Mapper/Explorer/Architect by size/risk; Reviewer for legacy compatibility. - Execution mode: delegated | direct_fallback | not_applicable -- Codex lifecycle evidence: delegated Codex roles need hook-written protected workflow-state `SubagentStart`/`SubagentStop` records task-bound to this journal's `Created:` identity, plus dispatch/result refs with the same `agent_id`; project-local `.codex/subagent-events.jsonl` is diagnostic only and journal text alone is insufficient. +- Codex lifecycle evidence: delegated Codex roles need hook-written protected workflow-state `SubagentStart`/`SubagentStop` records, task-bound to this journal's `Created:` identity; project-local `.codex/subagent-events.jsonl` is diagnostic only. - Direct fallback reason: [authorization_denied | subagents_unavailable | policy_disallowed | N/A] -- Evidence shorthand: delegated = dispatch/result refs; direct_fallback = role-equivalent direct evidence; N/A only when role not required. +- Evidence shorthand: delegated refs | role-equivalent direct evidence | N/A only when role not required. - Code Mapper dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] - Explorer dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] - Architect dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] @@ -87,7 +86,7 @@ Plan approval: [yes/no + date] | [path] | [what and why] | Step N | ## Harness Appendix Routing -[required only for medium+ harness-capable work; otherwise record `N/A: [reason]`] +[required only for medium+ harness-capable work; otherwise `N/A: [reason]`; full schema in `references/task-journal-harness-appendix.md`] - Appendix: `references/task-journal-harness-appendix.md` - Done Contract: [section/ref, or N/A: reason] - Harness Recipe: [section/ref, or N/A: reason] @@ -110,6 +109,7 @@ Plan approval: [yes/no + date] ## Slice Verification Ledger [required for medium+ tasks; update after each slice before starting the next] +do not start the next slice until the current one is `VERIFIED` | Slice | Task Packet | RED Status | Implementation Status | Verification Command/Result | Criteria Checked | Self-Check Result | Final Status | |-----------|-------------|------------|-----------------------|-----------------------------|------------------|-------------------|--------------| | S1: [slice_id] [name] | [packet id] | [pass/fail/N/A] | [done/blocked] | `[command]` → [pass/fail + signal] | [X/Y passed] | [pass/fail + note] | [VERIFIED/BLOCKED] | @@ -144,7 +144,7 @@ Plan approval: [yes/no + date] - [anything not covered or deferred] ## Review Log -[append an entry each time a review stage runs — never overwrite previous entries] +[append entries; Spec Review first (structured PASS/FAIL from `references/prompts/spec-review.md`), then Quality Review (assistant-review quality loop); never overwrite previous entries] ### Spec Review #1 - Result: PASS | FAIL @@ -237,17 +237,14 @@ Plan approval: [yes/no + date] ## Lifecycle -1. **Created** during Discover when clarification state must be tracked. Any task that enters clarification wait creates it before the wait; medium+ tasks also create it before leaving Discover even when no clarification wait is needed. -2. **Triage metadata** — record `Task type`, `Risk tier`, `Required gates`, `Required agents`, `Subagent policy state`, `Subagent execution mode`, and `Subagent authorization scope` before leaving Triage. Discovery may re-triage these fields when code/context evidence changes the risk or required gates. -3. **Clarification** updates — question caps are maximums, not quotas. Clear medium+ tasks may record `Clarification questions asked: 0` with `Clarification confidence: high`. While waiting, keep `Status: DISCOVERING`, set `Clarification status: needs_clarification`, set `Clarification defaults applied: false`, set confidence/cap/admissibility fields, and list every unresolved implementation-shaping topic. On explicit answers, clear unresolved topics, keep `Clarification defaults applied: false`, and set `Clarification status: ready`. On explicit `defaults`, print the applied defaults, clear unresolved topics, set `Clarification defaults applied: true`, and set `Clarification status: ready`. -4. **Decompose** — medium+ tasks set `Status: DECOMPOSING` after Discover is ready, then persist the slice manifest before moving on to planning. Small tasks skip this state. -5. **Plan approval** — once ready to plan, set `Status: PLANNING`, include the slice manifest in the plan for medium+ tasks, capture the approved plan, and update `Plan approval`. -6. **Build** each step — update Progress, Artifact Registry, Key Decisions, Status, and the harness appendix refs when triggered. For medium+ tasks, update the Slice Verification Ledger after each slice and do not start the next slice until the current one is `VERIFIED`. Check off Milestones when reached. -7. **Review cycle** when all steps done — Spec Review first (structured PASS/FAIL from `references/prompts/spec-review.md`), then Quality Review (assistant-review quality loop), fix must-fix → re-test → re-review until clean or a Pivot/Restart Decision routes recovery, fill Final Result -8. **Document** after review cycle passes — fill Verification Summary, add the Learning Controller block, Status: DOCUMENTING -9. **Handoff** to user — they test manually and add Review Notes -10. **Review fixes** — fix issues, re-test, re-review, update Progress -11. **Done** — Status: DONE, promote only evidence-backed durable lessons to approved local memory if available, record backend_unavailable/policy_disallowed/no-save rationale when not saved, and leave the ignored state file in place unless the user asks for cleanup +1. **Create** during Discover for clarification waits and all medium+ tasks. +2. **Triage** records task/risk/gates/agents/subagent fields before leaving Triage; re-triage if evidence changes them. +3. **Clarification** caps are maximums, not quotas. Waiting state stays `DISCOVERING`; explicit answers clear unresolved topics, while explicit `defaults` also sets `Clarification defaults applied: true`. +4. **Decompose/Plan** persists the slice manifest for medium+ work, captures approval, then updates `Plan approval`. +5. **Build** updates Progress, Artifact Registry, Key Decisions, Status, triggered harness refs, Milestones, and Slice Verification Ledger before the next slice. +6. **Review** runs Spec Review, then Quality Review, fixing/re-testing/re-reviewing until clean or routed by Pivot/Restart; fill Final Result. +7. **Document/Handoff** fills Verification Summary, Learning Controller, Review Notes, and user manual-test notes. +8. **Done** sets `Status: DONE`, records only evidence-backed durable lessons when allowed, and leaves the ignored state file unless cleanup is requested. ## Rules diff --git a/plugins/assistant-research/skills/assistant-research/SKILL.md b/plugins/assistant-research/skills/assistant-research/SKILL.md index 7396df9..0531202 100644 --- a/plugins/assistant-research/skills/assistant-research/SKILL.md +++ b/plugins/assistant-research/skills/assistant-research/SKILL.md @@ -15,13 +15,14 @@ triggers: | Contract | File | Purpose | |---|---|---| | **Input** | `contracts/input.yaml` | Question, tier, tool selection | -| **Output** | `contracts/output.yaml` | Findings with confidence scores, conflicts, gaps | +| **Output** | `contracts/output.yaml` | Findings, candidate mechanisms, confidence scores, conflicts, gaps | | **Phase Gates** | `contracts/phase-gates.yaml` | Search → Synthesize → Verify pipeline gates | **Rules:** - Every finding must have a confidence level (HIGH/MEDIUM/LOW) based on source count - Every URL must be verified before presenting to user - Conflicts and gaps must be explicitly checked and reported (even if empty) +- Candidate mechanisms are hypotheses, not proven causes; include evidence, confidence, counterevidence/conflicts, gaps, and validation method On-demand investigation capabilities with tiered depth and URL verification. @@ -35,6 +36,7 @@ Answer research questions with evidence-weighted findings, verified URLs, explic - Findings include confidence levels based on source quality and agreement. - URLs are verified before presentation or explicitly omitted/flagged. - Conflicts and gaps are reported even when the answer is otherwise clear. +- Candidate mechanisms are clearly labeled as candidates and include validation methods before being used as explanations. ## Constraints @@ -80,6 +82,7 @@ Return: - **Status** - completion state and confidence for the research result. - **Answer** - concise synthesis of the research result. - **Findings** - confidence-scored findings with source attribution. +- **Candidate mechanisms** - when researching causes, why/how, or improvement mechanisms, evidence-backed hypotheses with confidence, counterevidence/conflicts, gaps, and validation method. - **Sources** - verified URLs only, with enough context to understand relevance. - **Conflicts** - conflicting evidence or interpretations, or "none found". - **Gaps** - unanswered questions, weak evidence, stale sources, or recommended next checks. @@ -89,4 +92,5 @@ Return: - Stop and ask one focused question only when scope or decision criteria would materially change source selection or interpretation. - Stop and report a gap when required sources are inaccessible, stale, conflicting, or too weak for the requested confidence. +- Do not present candidate mechanisms as proven causes without executed validation evidence. - Do not finalize with unverified URLs. diff --git a/plugins/assistant-research/skills/assistant-research/contracts/output.yaml b/plugins/assistant-research/skills/assistant-research/contracts/output.yaml index 324c9ec..8d17ac6 100644 --- a/plugins/assistant-research/skills/assistant-research/contracts/output.yaml +++ b/plugins/assistant-research/skills/assistant-research/contracts/output.yaml @@ -37,6 +37,58 @@ artifacts: description: "URLs that were verified as accessible" validation: "Every URL listed here was checked and returned a valid response" + - name: candidate_mechanisms + type: object[] + required: conditional + condition: "question asks why/how something happens, asks for causes, asks for improvement mechanisms, or requests candidate mechanisms" + description: "Evidence-backed hypotheses for mechanisms that could explain a pattern or guide validation; not proven causes." + min_items: 1 + validation: "Each candidate mechanism includes mechanism, claim_status, evidence, confidence, counterevidence_or_conflicts, gaps, and validation_method. Do not present candidate mechanisms as proven causes." + on_fail: "Add missing mechanism fields or downgrade the claim to a gap. If validation was not executed, label the item as candidate or needs_validation instead of proven." + object_fields: + - name: mechanism + type: string + required: true + description: "The hypothesized mechanism, causal path, or improvement lever" + - name: claim_status + type: enum + required: true + enum_values: [candidate, needs_validation, unsupported, rejected] + validation: "Must not use proven as a candidate mechanism status" + - name: evidence + type: object[] + required: true + min_items: 1 + validation: "Source-backed support, inference-only support, or explicit unresolved evidence marker" + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: evidence_status + type: enum + required: true + enum_values: [source_backed, inference_only, unresolved] + - name: confidence + type: enum + required: true + enum_values: [high, medium, low] + validation: "Assigned from evidence strength and conflict level; low when evidence is single-source, inference-only, unresolved, or materially conflicting" + - name: counterevidence_or_conflicts + type: string[] + required: true + description: "Counterevidence, conflicts, or explicit none-found entry" + - name: gaps + type: string[] + required: true + description: "Missing data, untested assumptions, stale sources, or unresolved checks" + - name: validation_method + type: string + required: true + validation: "Concrete test, metric, comparison, experiment, or next source check that could confirm or reject the mechanism" + - name: perspective_scan type: object[] required: conditional @@ -263,6 +315,13 @@ artifacts: # Verified URLs: {verified URL 1, verified URL 2 if any} # 2. ... # +# CANDIDATE MECHANISMS (required when the question asks why/how, causes, or improvement mechanisms) +# 1. {candidate mechanism} — confidence: {HIGH/MEDIUM/LOW} — status: {candidate/needs_validation/unsupported/rejected} +# Evidence: {source-backed support, inference-only support, or unresolved marker} +# Counterevidence/conflicts: {conflicts or none found} +# Gaps: {missing evidence or uncertainty} +# Validation method: {test, metric, comparison, experiment, or next source check} +# # CONFLICTS # - {source A} says X, {source B} says Y — {assessment} # @@ -282,3 +341,4 @@ artifacts: # 7. For five_lens_briefing, perspective_scan, question_trace, contradiction_map, synthesis_briefing, and peer_review are present # 8. For five_lens_briefing, each required lens has a question, answer or explicit gap, source notes/verified URLs, follow-up question, and evidence status before synthesis # 9. For high-stakes topics, recommendation is framed as educational due diligence with caveats and defaults to investigate_further unless stronger action is justified by verified evidence and user context +# 10. candidate_mechanisms exists when why/how/cause/improvement mechanisms were requested, and every candidate has evidence, confidence, counterevidence_or_conflicts, gaps, validation_method, and non-proven claim_status diff --git a/plugins/assistant-research/skills/assistant-research/contracts/phase-gates.yaml b/plugins/assistant-research/skills/assistant-research/contracts/phase-gates.yaml index ef87e67..aa04cc3 100644 --- a/plugins/assistant-research/skills/assistant-research/contracts/phase-gates.yaml +++ b/plugins/assistant-research/skills/assistant-research/contracts/phase-gates.yaml @@ -73,6 +73,11 @@ gates: check: "Each finding has a confidence level assigned based on source count and quality" on_fail: "Assign confidence levels per the HIGH/MEDIUM/LOW criteria in output contract" + - id: SY_CANDIDATE_MECHANISMS + check: "When candidate_mechanisms are produced: each mechanism is labeled candidate, needs_validation, unsupported, or rejected; includes evidence, confidence, counterevidence_or_conflicts, gaps, and validation_method; and is not presented as a proven cause" + condition: "question asks why/how something happens, asks for causes, asks for improvement mechanisms, or candidate_mechanisms is non-empty" + on_fail: "Add missing candidate_mechanism fields, downgrade unsupported items to gaps, or relabel causal language as a candidate mechanism before Verify." + - id: SY5 check: "For five_lens_briefing: a question trace/evidence ledger exists before the contradiction map and synthesis, and unresolved or inference-only answers are not presented as source-backed findings" condition: "research_method == five_lens_briefing" @@ -108,6 +113,11 @@ gates: check: "Low-confidence findings are explicitly flagged in the output" on_fail: "Add explicit LOW confidence labels to single-source or conflicting findings" + - id: VR_CANDIDATE_MECHANISMS + check: "Candidate mechanisms without executed validation evidence are presented as hypotheses to test, not conclusions or proven causes" + condition: "candidate_mechanisms is non-empty" + on_fail: "Rewrite candidate mechanisms as hypotheses, add validation_method and gaps, or move validated source-backed claims to findings with evidence." + # ───────────────────────────────────────────── # Research invariants # ───────────────────────────────────────────── @@ -141,3 +151,8 @@ invariants: check: "For finance/trading, legal, medical, safety-critical, or similarly high-impact topics, the output is framed as educational due diligence with risk/context caveats and defaults to investigate_further unless verified evidence, explicit user context, and peer review support a stronger recommendation" scope: all_phases on_fail: "Add high-stakes caveats and downgrade the recommendation to investigate_further unless the stronger recommendation is explicitly justified" + + - id: INV7 + check: "Candidate mechanisms are never presented as proven causes; executed validation evidence is recorded separately in findings" + scope: all_phases + on_fail: "Relabel the mechanism as candidate or needs_validation, record counterevidence/conflicts and gaps, and add a validation_method before presenting it" diff --git a/plugins/assistant-research/skills/assistant-research/evals/cases.json b/plugins/assistant-research/skills/assistant-research/evals/cases.json index c4683ce..6d6fb95 100644 --- a/plugins/assistant-research/skills/assistant-research/evals/cases.json +++ b/plugins/assistant-research/skills/assistant-research/evals/cases.json @@ -200,6 +200,65 @@ "source-grounded answers optional" ] } + }, + { + "id": "candidate-mechanisms-are-evidence-backed-hypotheses", + "title": "Candidate mechanisms are evidence-backed hypotheses", + "category": "candidate_mechanisms", + "purpose": "Checks that why/how or improvement-mechanism research presents candidate mechanisms with evidence, confidence, conflicts, gaps, and validation methods instead of proven-cause claims.", + "prompt": "Research candidate mechanisms behind why workflow review loops keep drifting, and suggest how we could validate each mechanism before changing the framework.", + "setup_context": [ + "The assistant-research skill instructions are active.", + "The user asks for candidate mechanisms and validation methods.", + "No mechanism has been experimentally validated in the current project yet." + ], + "expected_behavior": [ + "Includes a CANDIDATE MECHANISMS section separate from FINDINGS.", + "Labels mechanisms as candidate hypotheses or needs_validation, not proven causes.", + "Includes evidence and confidence for each candidate mechanism.", + "Includes counterevidence or conflicts for each candidate mechanism, or an explicit none-found entry.", + "Includes gaps for each candidate mechanism.", + "Includes a validation method for each candidate mechanism.", + "Keeps validated factual claims in FINDINGS with source attribution." + ], + "pass_criteria": [ + "The response includes candidate_mechanisms or a CANDIDATE MECHANISMS section.", + "Each mechanism includes evidence, confidence, counterevidence/conflicts, gaps, and validation method.", + "The response says mechanisms are hypotheses to test or candidates.", + "The response does not present candidate mechanisms as proven causes." + ], + "fail_signals": [ + "Presents the mechanisms as proven causes without executed validation evidence.", + "Omits counterevidence or conflicts.", + "Omits gaps.", + "Omits validation methods.", + "Uses confidence without evidence." + ], + "machine_expectations": { + "required_substrings": [ + "CANDIDATE MECHANISMS", + "candidate", + "hypotheses", + "Evidence", + "confidence", + "Counterevidence", + "conflicts", + "Gaps", + "Validation method", + "FINDINGS" + ], + "forbidden_substrings": [ + "claim_status: proven", + "\"claim_status\": \"proven\"", + "status: proven", + "\"status\": \"proven\"", + "definitely causes", + "No validation needed", + "counterevidence omitted", + "gaps omitted", + "confidence without evidence" + ] + } } ] } diff --git a/plugins/assistant-research/skills/assistant-research/research.md b/plugins/assistant-research/skills/assistant-research/research.md index 92a060e..8a4bac4 100644 --- a/plugins/assistant-research/skills/assistant-research/research.md +++ b/plugins/assistant-research/skills/assistant-research/research.md @@ -47,6 +47,14 @@ Use `memory_add_insight` to record notable findings in the knowledge graph for f Research angles are required; subagent dispatch is optional. If the active adapter and user/tool policy permit parallel agents, each angle can be delegated independently. If not, run the angles sequentially in the main session and record that delegation was unavailable. Never reduce source diversity just because delegation is unavailable. +## Candidate mechanisms + +When the question asks why/how something happens, what caused a pattern, or +which improvement mechanisms to try, produce candidate mechanisms. Treat them +as hypotheses to validate, not proven causes. Each candidate mechanism needs +supporting evidence, confidence, counterevidence/conflicts, gaps, and a +validation method. + ## Mandatory: URL Verification **Every URL presented to the user MUST be verified.** See `url-verify.md`. AI agents hallucinate plausible-looking URLs routinely. Never present an unverified URL. @@ -77,6 +85,13 @@ FINDINGS 3. [finding] — confidence: LOW (single source, unverified) Source: [source] +CANDIDATE MECHANISMS (when requested by why/how/causal/improvement questions) +1. [mechanism] — confidence: MEDIUM — status: candidate + Evidence: [source-backed evidence or inference notes] + Counterevidence/conflicts: [conflicting evidence or none found] + Gaps: [missing evidence or uncertainty] + Validation method: [test, metric, comparison, experiment, or next check] + CONFLICTS - [source A] says X, [source B] says Y — [assessment of which is more likely correct] diff --git a/skills/assistant-research/SKILL.md b/skills/assistant-research/SKILL.md index 7396df9..0531202 100644 --- a/skills/assistant-research/SKILL.md +++ b/skills/assistant-research/SKILL.md @@ -15,13 +15,14 @@ triggers: | Contract | File | Purpose | |---|---|---| | **Input** | `contracts/input.yaml` | Question, tier, tool selection | -| **Output** | `contracts/output.yaml` | Findings with confidence scores, conflicts, gaps | +| **Output** | `contracts/output.yaml` | Findings, candidate mechanisms, confidence scores, conflicts, gaps | | **Phase Gates** | `contracts/phase-gates.yaml` | Search → Synthesize → Verify pipeline gates | **Rules:** - Every finding must have a confidence level (HIGH/MEDIUM/LOW) based on source count - Every URL must be verified before presenting to user - Conflicts and gaps must be explicitly checked and reported (even if empty) +- Candidate mechanisms are hypotheses, not proven causes; include evidence, confidence, counterevidence/conflicts, gaps, and validation method On-demand investigation capabilities with tiered depth and URL verification. @@ -35,6 +36,7 @@ Answer research questions with evidence-weighted findings, verified URLs, explic - Findings include confidence levels based on source quality and agreement. - URLs are verified before presentation or explicitly omitted/flagged. - Conflicts and gaps are reported even when the answer is otherwise clear. +- Candidate mechanisms are clearly labeled as candidates and include validation methods before being used as explanations. ## Constraints @@ -80,6 +82,7 @@ Return: - **Status** - completion state and confidence for the research result. - **Answer** - concise synthesis of the research result. - **Findings** - confidence-scored findings with source attribution. +- **Candidate mechanisms** - when researching causes, why/how, or improvement mechanisms, evidence-backed hypotheses with confidence, counterevidence/conflicts, gaps, and validation method. - **Sources** - verified URLs only, with enough context to understand relevance. - **Conflicts** - conflicting evidence or interpretations, or "none found". - **Gaps** - unanswered questions, weak evidence, stale sources, or recommended next checks. @@ -89,4 +92,5 @@ Return: - Stop and ask one focused question only when scope or decision criteria would materially change source selection or interpretation. - Stop and report a gap when required sources are inaccessible, stale, conflicting, or too weak for the requested confidence. +- Do not present candidate mechanisms as proven causes without executed validation evidence. - Do not finalize with unverified URLs. diff --git a/skills/assistant-research/contracts/output.yaml b/skills/assistant-research/contracts/output.yaml index 324c9ec..8d17ac6 100644 --- a/skills/assistant-research/contracts/output.yaml +++ b/skills/assistant-research/contracts/output.yaml @@ -37,6 +37,58 @@ artifacts: description: "URLs that were verified as accessible" validation: "Every URL listed here was checked and returned a valid response" + - name: candidate_mechanisms + type: object[] + required: conditional + condition: "question asks why/how something happens, asks for causes, asks for improvement mechanisms, or requests candidate mechanisms" + description: "Evidence-backed hypotheses for mechanisms that could explain a pattern or guide validation; not proven causes." + min_items: 1 + validation: "Each candidate mechanism includes mechanism, claim_status, evidence, confidence, counterevidence_or_conflicts, gaps, and validation_method. Do not present candidate mechanisms as proven causes." + on_fail: "Add missing mechanism fields or downgrade the claim to a gap. If validation was not executed, label the item as candidate or needs_validation instead of proven." + object_fields: + - name: mechanism + type: string + required: true + description: "The hypothesized mechanism, causal path, or improvement lever" + - name: claim_status + type: enum + required: true + enum_values: [candidate, needs_validation, unsupported, rejected] + validation: "Must not use proven as a candidate mechanism status" + - name: evidence + type: object[] + required: true + min_items: 1 + validation: "Source-backed support, inference-only support, or explicit unresolved evidence marker" + object_fields: + - name: source + type: string + required: true + - name: detail + type: string + required: true + - name: evidence_status + type: enum + required: true + enum_values: [source_backed, inference_only, unresolved] + - name: confidence + type: enum + required: true + enum_values: [high, medium, low] + validation: "Assigned from evidence strength and conflict level; low when evidence is single-source, inference-only, unresolved, or materially conflicting" + - name: counterevidence_or_conflicts + type: string[] + required: true + description: "Counterevidence, conflicts, or explicit none-found entry" + - name: gaps + type: string[] + required: true + description: "Missing data, untested assumptions, stale sources, or unresolved checks" + - name: validation_method + type: string + required: true + validation: "Concrete test, metric, comparison, experiment, or next source check that could confirm or reject the mechanism" + - name: perspective_scan type: object[] required: conditional @@ -263,6 +315,13 @@ artifacts: # Verified URLs: {verified URL 1, verified URL 2 if any} # 2. ... # +# CANDIDATE MECHANISMS (required when the question asks why/how, causes, or improvement mechanisms) +# 1. {candidate mechanism} — confidence: {HIGH/MEDIUM/LOW} — status: {candidate/needs_validation/unsupported/rejected} +# Evidence: {source-backed support, inference-only support, or unresolved marker} +# Counterevidence/conflicts: {conflicts or none found} +# Gaps: {missing evidence or uncertainty} +# Validation method: {test, metric, comparison, experiment, or next source check} +# # CONFLICTS # - {source A} says X, {source B} says Y — {assessment} # @@ -282,3 +341,4 @@ artifacts: # 7. For five_lens_briefing, perspective_scan, question_trace, contradiction_map, synthesis_briefing, and peer_review are present # 8. For five_lens_briefing, each required lens has a question, answer or explicit gap, source notes/verified URLs, follow-up question, and evidence status before synthesis # 9. For high-stakes topics, recommendation is framed as educational due diligence with caveats and defaults to investigate_further unless stronger action is justified by verified evidence and user context +# 10. candidate_mechanisms exists when why/how/cause/improvement mechanisms were requested, and every candidate has evidence, confidence, counterevidence_or_conflicts, gaps, validation_method, and non-proven claim_status diff --git a/skills/assistant-research/contracts/phase-gates.yaml b/skills/assistant-research/contracts/phase-gates.yaml index ef87e67..aa04cc3 100644 --- a/skills/assistant-research/contracts/phase-gates.yaml +++ b/skills/assistant-research/contracts/phase-gates.yaml @@ -73,6 +73,11 @@ gates: check: "Each finding has a confidence level assigned based on source count and quality" on_fail: "Assign confidence levels per the HIGH/MEDIUM/LOW criteria in output contract" + - id: SY_CANDIDATE_MECHANISMS + check: "When candidate_mechanisms are produced: each mechanism is labeled candidate, needs_validation, unsupported, or rejected; includes evidence, confidence, counterevidence_or_conflicts, gaps, and validation_method; and is not presented as a proven cause" + condition: "question asks why/how something happens, asks for causes, asks for improvement mechanisms, or candidate_mechanisms is non-empty" + on_fail: "Add missing candidate_mechanism fields, downgrade unsupported items to gaps, or relabel causal language as a candidate mechanism before Verify." + - id: SY5 check: "For five_lens_briefing: a question trace/evidence ledger exists before the contradiction map and synthesis, and unresolved or inference-only answers are not presented as source-backed findings" condition: "research_method == five_lens_briefing" @@ -108,6 +113,11 @@ gates: check: "Low-confidence findings are explicitly flagged in the output" on_fail: "Add explicit LOW confidence labels to single-source or conflicting findings" + - id: VR_CANDIDATE_MECHANISMS + check: "Candidate mechanisms without executed validation evidence are presented as hypotheses to test, not conclusions or proven causes" + condition: "candidate_mechanisms is non-empty" + on_fail: "Rewrite candidate mechanisms as hypotheses, add validation_method and gaps, or move validated source-backed claims to findings with evidence." + # ───────────────────────────────────────────── # Research invariants # ───────────────────────────────────────────── @@ -141,3 +151,8 @@ invariants: check: "For finance/trading, legal, medical, safety-critical, or similarly high-impact topics, the output is framed as educational due diligence with risk/context caveats and defaults to investigate_further unless verified evidence, explicit user context, and peer review support a stronger recommendation" scope: all_phases on_fail: "Add high-stakes caveats and downgrade the recommendation to investigate_further unless the stronger recommendation is explicitly justified" + + - id: INV7 + check: "Candidate mechanisms are never presented as proven causes; executed validation evidence is recorded separately in findings" + scope: all_phases + on_fail: "Relabel the mechanism as candidate or needs_validation, record counterevidence/conflicts and gaps, and add a validation_method before presenting it" diff --git a/skills/assistant-research/evals/cases.json b/skills/assistant-research/evals/cases.json index c4683ce..6d6fb95 100644 --- a/skills/assistant-research/evals/cases.json +++ b/skills/assistant-research/evals/cases.json @@ -200,6 +200,65 @@ "source-grounded answers optional" ] } + }, + { + "id": "candidate-mechanisms-are-evidence-backed-hypotheses", + "title": "Candidate mechanisms are evidence-backed hypotheses", + "category": "candidate_mechanisms", + "purpose": "Checks that why/how or improvement-mechanism research presents candidate mechanisms with evidence, confidence, conflicts, gaps, and validation methods instead of proven-cause claims.", + "prompt": "Research candidate mechanisms behind why workflow review loops keep drifting, and suggest how we could validate each mechanism before changing the framework.", + "setup_context": [ + "The assistant-research skill instructions are active.", + "The user asks for candidate mechanisms and validation methods.", + "No mechanism has been experimentally validated in the current project yet." + ], + "expected_behavior": [ + "Includes a CANDIDATE MECHANISMS section separate from FINDINGS.", + "Labels mechanisms as candidate hypotheses or needs_validation, not proven causes.", + "Includes evidence and confidence for each candidate mechanism.", + "Includes counterevidence or conflicts for each candidate mechanism, or an explicit none-found entry.", + "Includes gaps for each candidate mechanism.", + "Includes a validation method for each candidate mechanism.", + "Keeps validated factual claims in FINDINGS with source attribution." + ], + "pass_criteria": [ + "The response includes candidate_mechanisms or a CANDIDATE MECHANISMS section.", + "Each mechanism includes evidence, confidence, counterevidence/conflicts, gaps, and validation method.", + "The response says mechanisms are hypotheses to test or candidates.", + "The response does not present candidate mechanisms as proven causes." + ], + "fail_signals": [ + "Presents the mechanisms as proven causes without executed validation evidence.", + "Omits counterevidence or conflicts.", + "Omits gaps.", + "Omits validation methods.", + "Uses confidence without evidence." + ], + "machine_expectations": { + "required_substrings": [ + "CANDIDATE MECHANISMS", + "candidate", + "hypotheses", + "Evidence", + "confidence", + "Counterevidence", + "conflicts", + "Gaps", + "Validation method", + "FINDINGS" + ], + "forbidden_substrings": [ + "claim_status: proven", + "\"claim_status\": \"proven\"", + "status: proven", + "\"status\": \"proven\"", + "definitely causes", + "No validation needed", + "counterevidence omitted", + "gaps omitted", + "confidence without evidence" + ] + } } ] } diff --git a/skills/assistant-research/research.md b/skills/assistant-research/research.md index 92a060e..8a4bac4 100644 --- a/skills/assistant-research/research.md +++ b/skills/assistant-research/research.md @@ -47,6 +47,14 @@ Use `memory_add_insight` to record notable findings in the knowledge graph for f Research angles are required; subagent dispatch is optional. If the active adapter and user/tool policy permit parallel agents, each angle can be delegated independently. If not, run the angles sequentially in the main session and record that delegation was unavailable. Never reduce source diversity just because delegation is unavailable. +## Candidate mechanisms + +When the question asks why/how something happens, what caused a pattern, or +which improvement mechanisms to try, produce candidate mechanisms. Treat them +as hypotheses to validate, not proven causes. Each candidate mechanism needs +supporting evidence, confidence, counterevidence/conflicts, gaps, and a +validation method. + ## Mandatory: URL Verification **Every URL presented to the user MUST be verified.** See `url-verify.md`. AI agents hallucinate plausible-looking URLs routinely. Never present an unverified URL. @@ -77,6 +85,13 @@ FINDINGS 3. [finding] — confidence: LOW (single source, unverified) Source: [source] +CANDIDATE MECHANISMS (when requested by why/how/causal/improvement questions) +1. [mechanism] — confidence: MEDIUM — status: candidate + Evidence: [source-backed evidence or inference notes] + Counterevidence/conflicts: [conflicting evidence or none found] + Gaps: [missing evidence or uncertainty] + Validation method: [test, metric, comparison, experiment, or next check] + CONFLICTS - [source A] says X, [source B] says Y — [assessment of which is more likely correct] diff --git a/skills/assistant-workflow/contracts/output.yaml b/skills/assistant-workflow/contracts/output.yaml index 31e1243..e033d33 100644 --- a/skills/assistant-workflow/contracts/output.yaml +++ b/skills/assistant-workflow/contracts/output.yaml @@ -13,17 +13,17 @@ skill: assistant-workflow completion_tiers: small: required_artifacts: [changed_files, validation_results, manual_test_steps, user_approval] - conditional_artifacts: [triage_result, subagent_evidence, test_results, debugging_result, artifact_contract] + conditional_artifacts: [triage_result, subagent_evidence, test_results, debugging_result, artifact_contract, workflow_experiment_ledger, loop_readiness_assessment] review_gate: "self-review or explicit validation summary unless risk_tier in [high, critical]" checkpoint_policy: "inline plan/check summary is enough; phase_checkpoints are strict-profile only" medium: required_artifacts: [triage_result, changed_files, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, plan_document, user_approval] - conditional_artifacts: [test_results, task_journal, context_map, slice_manifest, single_slice_rationale, slice_verification_summary, metrics_entry, candidate_search_result, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result] + conditional_artifacts: [test_results, task_journal, context_map, slice_manifest, single_slice_rationale, slice_verification_summary, metrics_entry, candidate_search_result, workflow_experiment_ledger, loop_readiness_assessment, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result] review_gate: "spec review plus code quality review, with independent QA evaluation when required and evidence matched to the approved plan" checkpoint_policy: "task journal or carried-forward state records phase transitions" large_critical: required_artifacts: [triage_result, phase_checkpoints, changed_files, test_results, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, task_journal, context_map, context_budget_note, slice_manifest, slice_verification_summary, plan_document, user_approval] - conditional_artifacts: [debugging_result, artifact_contract, pattern_retrieval_note, decomposition_plan_review, single_slice_rationale, metrics_entry, candidate_search_result, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result, verified_skill_distillation] + conditional_artifacts: [debugging_result, artifact_contract, pattern_retrieval_note, decomposition_plan_review, single_slice_rationale, metrics_entry, candidate_search_result, workflow_experiment_ledger, loop_readiness_assessment, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result, verified_skill_distillation] review_gate: "full strict evidence, security/operability review when applicable, QA evaluation when required, and all slice criteria verified" checkpoint_policy: "explicit phase checkpoints are required" @@ -1290,6 +1290,132 @@ artifacts: condition: "candidate search occurred after plan approval or selected candidate changes approved scope/files/behavior/risk" validation: "Records deviation, impact, rollback/re-approval status, and user approval when required" + - name: workflow_experiment_ledger + type: object[] + required: conditional + condition: "explicit workflow experiment, loop-optimization experiment, or approved plan labels the work as an experiment" + description: "Lightweight ledger for explicit workflow experiments only; not required for ordinary medium+ work." + validation: "Each entry includes experiment_id, hypothesis, intervention, expected_signal, measurement_method, baseline_or_control, result_status, evidence, decision, and next_check. The ledger does not by itself require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation." + on_fail: "When the approved plan names an explicit workflow experiment, add the missing ledger entry before running the experiment; otherwise record N/A and do not add harness artifacts unless harness_capable == true." + min_items: 1 + object_fields: + - name: experiment_id + type: string + required: true + validation: "Stable local id unique within the workflow task" + - name: hypothesis + type: string + required: true + validation: "Testable claim about workflow behavior, cost, speed, quality, or reliability" + - name: intervention + type: string + required: true + validation: "Specific workflow, contract, prompt, hook, or process change being tried" + - name: expected_signal + type: string + required: true + validation: "Observable signal expected to move if the hypothesis is useful" + - name: measurement_method + type: string + required: true + validation: "Command, metric, review criterion, or inspection that measures the signal" + - name: baseline_or_control + type: string + required: true + validation: "Previous behavior, control condition, or explicit no-baseline reason" + - name: result_status + type: enum + required: true + enum_values: [planned, running, measured, inconclusive, stopped] + - name: evidence + type: string[] + required: true + min_items: 1 + validation: "Plan ref, command result, review note, metric, or explicit pending-evidence marker for planned experiments" + - name: decision + type: enum + required: true + enum_values: [pending, continue, adjust, stop, inconclusive] + - name: next_check + type: string + required: true + validation: "Next measurement, review, or stop point" + + - name: loop_readiness_assessment + type: object + required: conditional + condition: "before starting an explicit repeat or optimization loop outside the standard required workflow phase gates" + description: "Pre-loop gate for repeat/optimization loops; not required for one-pass work or the normal required review loop." + validation: "Records loop_type, loop_trigger, readiness_state, verifier, stop_condition, max_iterations, budget_limit, tool_access, state_tracking, retry_or_empty_result_handling, tool_error_handling, low_confidence_escalation, rollback_or_exit, harness_routing, and evidence before the loop starts. loop artifacts alone do not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation." + on_fail: "Before running the repeat/optimization loop, add the readiness assessment or remove the loop from the plan. Set harness_routing=required only when harness_capable == true or independent QA criteria already apply." + object_fields: + - name: loop_type + type: enum + required: true + enum_values: [repeat, optimization, experiment] + - name: loop_trigger + type: string + required: true + validation: "Why a repeated loop is needed instead of a single planned pass" + - name: readiness_state + type: enum + required: true + enum_values: [ready, blocked] + validation: "Must be ready before the loop starts" + - name: verifier + type: string + required: true + validation: "Check, role, command, metric, or reviewer that judges each iteration" + - name: stop_condition + type: string + required: true + validation: "Concrete pass, budget, stagnation, or failure condition that ends the loop" + - name: max_iterations + type: int + required: true + validation: "Positive finite cap" + - name: budget_limit + type: string + required: true + validation: "Token, time, cost, run-count, or explicit local-effort cap" + - name: tool_access + type: string[] + required: true + validation: "Tools/commands/agents needed for the loop, or explicit none entry" + - name: state_tracking + type: string + required: true + validation: "Where iteration state, results, and decisions are recorded" + - name: retry_or_empty_result_handling + type: string + required: true + validation: "How the loop handles retries, empty verifier results, empty search/tool results, exhausted no-result paths, or no-result exit evidence before another iteration is attempted" + on_missing: fail + - name: tool_error_handling + type: string + required: true + validation: "How tool errors route before another iteration, including retry limits, fallback role/action, blocker classification, or stop condition" + on_missing: fail + - name: low_confidence_escalation + type: string + required: true + validation: "Escalation path when verifier confidence is low, evidence is inconclusive, or repeated iterations do not improve confidence" + on_missing: fail + - name: rollback_or_exit + type: string + required: true + validation: "Action when stop_condition fails or the loop exhausts its cap" + - name: harness_routing + type: enum + required: true + enum_values: [not_required, required] + validation: "not_required unless harness_capable == true or QA criteria already apply" + - name: evidence + type: string[] + required: true + min_items: 1 + validation: "Plan ref, task packet ref, or readiness check evidence" + - name: plan_document type: string required: conditional @@ -1317,8 +1443,10 @@ artifacts: # 5. If qa_evaluation_mode=required: qa_evaluation_result is populated after build/test and code-review evidence; QA evaluation does not replace review_result. Scoped domain rubrics require selected_domain_rubrics and domain_quality_scores; unscoped rubric scoring is invalid. # 6. For medium+ tasks, slice_verification_summary has one VERIFIED entry per slice # 7. candidate_search_result exists when search_mode == candidate_search, includes search_exit_summary, and any plan_deviation is approved before Build -# 8. For medium+ harness-capable work, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision when triggered, and artifact_reference_ledger exist and are current before completion or handoff -# 9. user_approval is confirmed, confirmed_with_issues, or not_required_small for eligible small tasks +# 8. workflow_experiment_ledger exists only for explicit workflow experiments; loop_readiness_assessment exists only before explicit repeat/optimization loops and includes retry_or_empty_result_handling, tool_error_handling, and low_confidence_escalation +# 9. Loop artifacts alone do not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless harness_capable or QA criteria independently apply +# 10. For medium+ harness-capable work, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision when triggered, and artifact_reference_ledger exist and are current before completion or handoff +# 11. user_approval is confirmed, confirmed_with_issues, or not_required_small for eligible small tasks # If any check fails, do NOT print --- WORKFLOW COMPLETE ---. State what is missing. - name: verified_skill_distillation type: object diff --git a/skills/assistant-workflow/contracts/phase-gates.yaml b/skills/assistant-workflow/contracts/phase-gates.yaml index 5e657a3..4bdcb62 100644 --- a/skills/assistant-workflow/contracts/phase-gates.yaml +++ b/skills/assistant-workflow/contracts/phase-gates.yaml @@ -282,6 +282,16 @@ gates: check: "Each task packet includes a deviation/rollback rule, or small inline plan explicitly states how deviations are handled" on_fail: "Add deviation/rollback handling before seeking plan approval" + - id: P_WORKFLOW_EXPERIMENT_LEDGER + check: "When the approved work is an explicit workflow experiment: workflow_experiment_ledger records hypothesis, intervention, expected signal, measurement method, baseline/control, result status, evidence, decision, and next check before the experiment runs" + condition: "explicit workflow experiment, loop-optimization experiment, or approved plan labels the work as an experiment" + on_fail: "Add the workflow_experiment_ledger entry to the plan/task journal before running the experiment, or remove the experiment label. Do not add Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless harness_capable or QA criteria independently apply." + + - id: P_LOOP_READINESS + check: "Before any explicit repeat or optimization loop outside standard required workflow phase gates starts: loop_readiness_assessment records loop type, trigger, readiness state, verifier, stop condition, finite max iterations, budget limit, tool access, state tracking, retry_or_empty_result_handling, tool_error_handling, low_confidence_escalation, rollback/exit action, harness routing, and evidence" + condition: "task will start an explicit repeat or optimization loop" + on_fail: "Add loop_readiness_assessment before the loop starts, including retry/empty-result handling, tool-error routing, and low-confidence escalation, or remove the loop from the plan. Keep harness_routing=not_required unless harness_capable == true or QA criteria independently apply." + - id: CS1 check: "When search_mode == candidate_search: goal tree decomposes the existing acceptance criteria and slice acceptance/verification criteria" condition: "search_mode == candidate_search" diff --git a/skills/assistant-workflow/evals/cases.json b/skills/assistant-workflow/evals/cases.json index fce3c06..bddef94 100644 --- a/skills/assistant-workflow/evals/cases.json +++ b/skills/assistant-workflow/evals/cases.json @@ -626,6 +626,66 @@ ] ] } + }, + { + "id": "loop-experiment-artifacts-stay-conditional", + "title": "Loop and experiment artifacts stay conditional", + "category": "loop_readiness_and_experiments", + "purpose": "Checks that workflow experiment and repeat/optimization loop artifacts are required only when explicitly scoped and do not turn ordinary medium work into harness work.", + "prompt": "Plan a medium Assistant Framework workflow experiment to compare two review-loop prompts. Keep ordinary harness routing off unless the plan explicitly needs trace/replay or QA.", + "setup_context": [ + "The assistant-workflow skill instructions are active.", + "The request explicitly labels the work as a workflow experiment.", + "The task may use a bounded repeat/optimization loop, but no harness, trace/replay, Done Contract, or QA acceptance criteria were requested." + ], + "expected_behavior": [ + "Keeps ordinary medium task routing at harness_capable=false unless independent harness criteria apply.", + "Adds workflow_experiment_ledger only because the prompt explicitly asks for a workflow experiment.", + "Adds loop_readiness_assessment only before starting an explicit repeat or optimization loop.", + "Records hypothesis, intervention, expected signal, measurement method, baseline/control, evidence, decision, and next check.", + "Records verifier, stop condition, max iterations, budget limit, tool access, state tracking, retry or empty-result handling, tool-error handling, low-confidence escalation, and rollback/exit before any loop starts.", + "Does not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation from loop artifacts alone." + ], + "pass_criteria": [ + "The response marks workflow_experiment_ledger as conditional on explicit workflow experiments.", + "The response marks loop_readiness_assessment as conditional before repeat/optimization loops.", + "The response keeps harness_capable=false for ordinary medium work.", + "The response says loop artifacts alone do not require harness or QA artifacts." + ], + "fail_signals": [ + "Requires workflow_experiment_ledger for every medium task.", + "Requires loop_readiness_assessment without an explicit repeat or optimization loop.", + "Treats loop artifacts as proof that harness_capable must be true.", + "Requires Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation from loop artifacts alone." + ], + "machine_expectations": { + "required_substrings": [ + "workflow_experiment_ledger", + "explicit workflow experiment", + "loop_readiness_assessment", + "repeat/optimization loop", + "harness_capable=false", + "hypothesis", + "measurement method", + "baseline/control", + "verifier", + "stop condition", + "max iterations", + "budget limit", + "state tracking", + "retry_or_empty_result_handling", + "tool_error_handling", + "low_confidence_escalation", + "loop artifacts alone do not require" + ], + "forbidden_substrings": [ + "workflow_experiment_ledger is required for every medium task", + "loop_readiness_assessment is always required", + "harness_capable=true by default", + "Done Contract is required by loop artifacts", + "QA evaluation is required by loop artifacts" + ] + } } ] } diff --git a/skills/assistant-workflow/references/plan-template.md b/skills/assistant-workflow/references/plan-template.md index 321c7ca..f73feea 100755 --- a/skills/assistant-workflow/references/plan-template.md +++ b/skills/assistant-workflow/references/plan-template.md @@ -1,11 +1,10 @@ # Plan Templates -Three tiers — match ceremony to task size (don't get fancy when N is small). +Three tiers — match ceremony to task size. -Harness details are optional appendices. For medium+ harness-capable work, keep -compact refs in the base plan and load `references/plan-harness-appendix.md` -for the full schema. For non-harness work, record `N/A: [reason]` instead of -copying harness sections into the base plan. +Harness details live in optional appendices. Base plans keep compact refs only: +load `references/plan-harness-appendix.md` for harness-capable work, otherwise +record `N/A: [reason]`. ## Small Tasks — Inline Plan @@ -60,15 +59,11 @@ For Medium and Large/Mega plans, write implementation work as executable task pa - Expected success signal: [exit code 0, passing test name, output marker, etc.] - Evidence to record: - [test result, eval fixture, changed file, review note, or artifact proof] -- Harness routing: - - applies: [required for medium+ harness-capable work; otherwise N/A: reason] - - appendix: [references/plan-harness-appendix.md, or N/A: reason] - - done_contract_ref: [section/ref, or N/A: reason] - - harness_recipe_ref: [section/ref, or N/A: reason] - - harness_run_state_ref: [section/ref, or N/A: reason] - - trace_ledger_ref: [section/ref, or N/A: reason] - - replay_packet_ref: [section/ref, or N/A: reason] - - artifact_reference_ledger_ref: [section/ref, or N/A: reason] +- Loop / Experiment Routing: + - workflow_experiment_ledger: [N/A unless explicit workflow experiment; otherwise ref] + - loop_readiness_assessment: [N/A unless explicit repeat/optimization loop; otherwise ref with retry_or_empty_result_handling, tool_error_handling, low_confidence_escalation] + - loop_harness_routing: [ordinary medium+ keeps harness_capable=false; loop artifacts alone do not require harness/QA artifacts] +- Harness routing: [N/A unless harness_capable=true or QA criteria independently apply; otherwise refs to appendix, Done Contract, Harness Recipe, run state, trace/replay, artifact ledger] - Deviation / rollback rule: - [what to do if required files/behavior differ from plan; include rollback/revert boundary] - Worker status / evidence: @@ -136,16 +131,10 @@ Covers the essentials without Security/Operability overhead. Fill this in during ## Architecture - Current architecture: [identified or "new project"] - Architecture for this change: [Clean/MVVM/Hexagonal/etc.] -- Layer rules: - - [e.g., Domain has no external dependencies] - - [e.g., ViewModels don't reference Views] +- Layer rules: [key dependency boundaries] - Dependency direction: [A → B → C] -- New files placement: - - [file → layer/folder rationale] -- SOLID design notes: - - SRP: [which classes own which responsibility — flag any class with >1 reason to change] - - OCP: [will new variants require modifying existing classes? If yes, plan extension points] - - DIP: [which high-level modules depend on abstractions vs concrete implementations?] +- New files placement: [file → layer/folder rationale] +- SOLID notes: [SRP/OCP/DIP risks or N/A] ## Analysis ### Candidate search summary @@ -185,12 +174,20 @@ Covers the essentials without Security/Operability overhead. Fill this in during - Owner/consumer: [user, reviewer, downstream tool, runtime] - Non-goals/exclusions: [what must not be produced] +## Loop / Experiment Routing + +Only for explicit workflow experiments or explicit repeat/optimization loops. +Ordinary medium+ tasks keep `harness_capable=false`; loop artifacts alone do not +require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact +Reference Ledger, or QA evaluation. + +- workflow_experiment_ledger: [N/A unless explicit workflow experiment; otherwise compact ref with hypothesis/intervention/signal/measurement/baseline/status/evidence/decision/next check] +- loop_readiness_assessment: [N/A unless explicit repeat/optimization loop; otherwise compact ref with loop type/trigger/verifier/stop/max iterations/budget/tool access/state tracking/retry_or_empty_result_handling/tool_error_handling/low_confidence_escalation/rollback/harness routing] + ## Harness Appendix Routing -Required only for medium+ harness-capable work; otherwise keep every field as -`N/A: [reason]`. Load `references/plan-harness-appendix.md` for the full Done -Contract, Harness Recipe, runtime artifact, typed artifact ref, and QA routing -schemas. +Required only for medium+ harness-capable work; otherwise use `N/A: [reason]`. +Load `references/plan-harness-appendix.md` for full harness and QA schemas. - appendix_status: [required | N/A: reason] - done_contract_ref: [section/ref, or N/A: reason] @@ -243,28 +240,6 @@ Start with the Medium template, then add the sections below. - Decomposition Plan Review: reuse Medium fields and keep `- Broad-split rejection:` proof for large/mega slice plans. - Harness Appendix Routing: reuse the Medium compact refs and load `references/plan-harness-appendix.md` only when harness-capable. -## Harness Appendix Routing - -Required only for medium+ harness-capable work; otherwise record -`N/A: [reason]`. Load `references/plan-harness-appendix.md` for full harness, -typed artifact reference, pivot/restart, and QA routing schemas. - -- appendix_status: [required | N/A: reason] -- done_contract_ref: [section/ref, or N/A: reason] -- harness_recipe_ref: [section/ref, or N/A: reason] -- harness_run_state_ref: [section/ref, or N/A: reason] -- trace_ledger_ref: [section/ref, or N/A: reason] -- replay_packet_ref: [section/ref, or N/A: reason] -- artifact_reference_ledger_ref: [section/ref, or N/A: reason] -- qa_evaluation_mode: [required | optional | not_required + reason] - -## Slice manifest from Decompose - -Use the shared Slice Manifest structure above. Paste the approved Decompose manifest verbatim and keep `single_slice_rationale` when exactly one slice exists. - -## Task packets -Use the Executable Task Packet structure above for each approved slice. Order packets by dependency, consume the Decompose slice manifest directly, and keep each slice independently verifiable before the next slice starts. - ## Tests to run - [command]: [what it validates] ``` @@ -281,8 +256,8 @@ Use the Executable Task Packet structure above for each approved slice. Order pa ## Context Budget -- Exact/pinned: [goal, acceptance criteria, safety constraints, exact errors, files in scope, validation requirements] -- Summarized: [logs, tool output, conversation history, repetitive evidence] +- Exact/pinned: [goal, criteria, constraints, errors, scoped files, validation] +- Summarized: [logs, tool output, history, repetitive evidence] - Omitted/deferred: [out-of-scope files/results and why] - Split/delegation plan: [slice/task split when material exceeds one faithful context] diff --git a/skills/assistant-workflow/references/task-journal-template.md b/skills/assistant-workflow/references/task-journal-template.md index 94b6bae..016d8b7 100644 --- a/skills/assistant-workflow/references/task-journal-template.md +++ b/skills/assistant-workflow/references/task-journal-template.md @@ -9,15 +9,10 @@ Write to `{agent_state_dir}/task.md` in the project root when a local state dire ## When to update - When clarification questions are asked, answered, or resolved via explicit `defaults` -- After each Build step completes (update Progress, Artifact Registry, and check off Milestones) -- After each medium+ harness-capable event listed in `references/task-journal-harness-appendix.md` -- After any Pivot/Restart Decision, typed artifact reference update, or QA evaluation result when applicable -- When key decisions are made -- When constraints are added by the user -- After each review cycle pass (append to Review Log — never overwrite) -- At verification summary (all steps done) -- During user review feedback -- During Document, after review/build/user-correction evidence has been checked for durable lessons +- After each Build step: Progress, Artifact Registry, Milestones +- After key decisions, new user constraints, review passes, verification summary, or user review feedback +- After harness-capable events, Pivot/Restart Decisions, typed artifact refs, or QA results when applicable +- During Document, after review/build/user-correction evidence is checked for durable lessons ## Template @@ -50,15 +45,19 @@ Candidate scope scan: - Adjacent surfaces: [tests/docs/contracts/config/mirrors/hooks/runtime surfaces to inspect] - Confidence: [low | medium | high] - Unknowns: [none, or one short scope/risk unknown per line] +Loop / Experiment Routing: +- workflow_experiment_ledger: [N/A unless explicit workflow experiment; otherwise compact ref with id/hypothesis/intervention/signal/measurement/baseline/status/evidence/decision/next_check] +- loop_readiness_assessment: [N/A unless explicit repeat or optimization loop; otherwise compact ref with loop_type/trigger/verifier/stop/max_iterations/budget/tool_access/state_tracking/retry_or_empty_result_handling/tool_error_handling/low_confidence_escalation/rollback/harness_routing/evidence] +- loop_harness_routing: [ordinary medium+ keeps harness_capable=false; loop artifacts alone do not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation; appendix only when harness_capable=true or QA criteria independently apply] Plan approval: [yes/no + date] ## Agent Dispatch Log -[strict subagent evidence inspected by stop-review/phase gates] -- Required roles: Code Writer, Builder/Tester, Code Reviewer; QA Evaluator when `qa_evaluation_mode=required`; Code Mapper/Explorer/Architect by size/risk; Reviewer for legacy compatibility. +[subagent evidence inspected by stop-review/phase gates] +- Required roles: Code Writer, Builder/Tester, Code Reviewer; QA Evaluator when required; Code Mapper/Explorer/Architect by size/risk; Reviewer for legacy compatibility. - Execution mode: delegated | direct_fallback | not_applicable -- Codex lifecycle evidence: delegated Codex roles need hook-written protected workflow-state `SubagentStart`/`SubagentStop` records task-bound to this journal's `Created:` identity, plus dispatch/result refs with the same `agent_id`; project-local `.codex/subagent-events.jsonl` is diagnostic only and journal text alone is insufficient. +- Codex lifecycle evidence: delegated Codex roles need hook-written protected workflow-state `SubagentStart`/`SubagentStop` records, task-bound to this journal's `Created:` identity; project-local `.codex/subagent-events.jsonl` is diagnostic only. - Direct fallback reason: [authorization_denied | subagents_unavailable | policy_disallowed | N/A] -- Evidence shorthand: delegated = dispatch/result refs; direct_fallback = role-equivalent direct evidence; N/A only when role not required. +- Evidence shorthand: delegated refs | role-equivalent direct evidence | N/A only when role not required. - Code Mapper dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] - Explorer dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] - Architect dispatch/result/direct evidence: [delegated refs | direct evidence | N/A] @@ -87,7 +86,7 @@ Plan approval: [yes/no + date] | [path] | [what and why] | Step N | ## Harness Appendix Routing -[required only for medium+ harness-capable work; otherwise record `N/A: [reason]`] +[required only for medium+ harness-capable work; otherwise `N/A: [reason]`; full schema in `references/task-journal-harness-appendix.md`] - Appendix: `references/task-journal-harness-appendix.md` - Done Contract: [section/ref, or N/A: reason] - Harness Recipe: [section/ref, or N/A: reason] @@ -110,6 +109,7 @@ Plan approval: [yes/no + date] ## Slice Verification Ledger [required for medium+ tasks; update after each slice before starting the next] +do not start the next slice until the current one is `VERIFIED` | Slice | Task Packet | RED Status | Implementation Status | Verification Command/Result | Criteria Checked | Self-Check Result | Final Status | |-----------|-------------|------------|-----------------------|-----------------------------|------------------|-------------------|--------------| | S1: [slice_id] [name] | [packet id] | [pass/fail/N/A] | [done/blocked] | `[command]` → [pass/fail + signal] | [X/Y passed] | [pass/fail + note] | [VERIFIED/BLOCKED] | @@ -144,7 +144,7 @@ Plan approval: [yes/no + date] - [anything not covered or deferred] ## Review Log -[append an entry each time a review stage runs — never overwrite previous entries] +[append entries; Spec Review first (structured PASS/FAIL from `references/prompts/spec-review.md`), then Quality Review (assistant-review quality loop); never overwrite previous entries] ### Spec Review #1 - Result: PASS | FAIL @@ -237,17 +237,14 @@ Plan approval: [yes/no + date] ## Lifecycle -1. **Created** during Discover when clarification state must be tracked. Any task that enters clarification wait creates it before the wait; medium+ tasks also create it before leaving Discover even when no clarification wait is needed. -2. **Triage metadata** — record `Task type`, `Risk tier`, `Required gates`, `Required agents`, `Subagent policy state`, `Subagent execution mode`, and `Subagent authorization scope` before leaving Triage. Discovery may re-triage these fields when code/context evidence changes the risk or required gates. -3. **Clarification** updates — question caps are maximums, not quotas. Clear medium+ tasks may record `Clarification questions asked: 0` with `Clarification confidence: high`. While waiting, keep `Status: DISCOVERING`, set `Clarification status: needs_clarification`, set `Clarification defaults applied: false`, set confidence/cap/admissibility fields, and list every unresolved implementation-shaping topic. On explicit answers, clear unresolved topics, keep `Clarification defaults applied: false`, and set `Clarification status: ready`. On explicit `defaults`, print the applied defaults, clear unresolved topics, set `Clarification defaults applied: true`, and set `Clarification status: ready`. -4. **Decompose** — medium+ tasks set `Status: DECOMPOSING` after Discover is ready, then persist the slice manifest before moving on to planning. Small tasks skip this state. -5. **Plan approval** — once ready to plan, set `Status: PLANNING`, include the slice manifest in the plan for medium+ tasks, capture the approved plan, and update `Plan approval`. -6. **Build** each step — update Progress, Artifact Registry, Key Decisions, Status, and the harness appendix refs when triggered. For medium+ tasks, update the Slice Verification Ledger after each slice and do not start the next slice until the current one is `VERIFIED`. Check off Milestones when reached. -7. **Review cycle** when all steps done — Spec Review first (structured PASS/FAIL from `references/prompts/spec-review.md`), then Quality Review (assistant-review quality loop), fix must-fix → re-test → re-review until clean or a Pivot/Restart Decision routes recovery, fill Final Result -8. **Document** after review cycle passes — fill Verification Summary, add the Learning Controller block, Status: DOCUMENTING -9. **Handoff** to user — they test manually and add Review Notes -10. **Review fixes** — fix issues, re-test, re-review, update Progress -11. **Done** — Status: DONE, promote only evidence-backed durable lessons to approved local memory if available, record backend_unavailable/policy_disallowed/no-save rationale when not saved, and leave the ignored state file in place unless the user asks for cleanup +1. **Create** during Discover for clarification waits and all medium+ tasks. +2. **Triage** records task/risk/gates/agents/subagent fields before leaving Triage; re-triage if evidence changes them. +3. **Clarification** caps are maximums, not quotas. Waiting state stays `DISCOVERING`; explicit answers clear unresolved topics, while explicit `defaults` also sets `Clarification defaults applied: true`. +4. **Decompose/Plan** persists the slice manifest for medium+ work, captures approval, then updates `Plan approval`. +5. **Build** updates Progress, Artifact Registry, Key Decisions, Status, triggered harness refs, Milestones, and Slice Verification Ledger before the next slice. +6. **Review** runs Spec Review, then Quality Review, fixing/re-testing/re-reviewing until clean or routed by Pivot/Restart; fill Final Result. +7. **Document/Handoff** fills Verification Summary, Learning Controller, Review Notes, and user manual-test notes. +8. **Done** sets `Status: DONE`, records only evidence-backed durable lessons when allowed, and leaves the ignored state file unless cleanup is requested. ## Rules diff --git a/tests/p0-p4/assistant-research-contracts.sh b/tests/p0-p4/assistant-research-contracts.sh new file mode 100644 index 0000000..778653c --- /dev/null +++ b/tests/p0-p4/assistant-research-contracts.sh @@ -0,0 +1,55 @@ +if [[ -z "${P0P4_HARNESS_LOADED:-}" ]]; then + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/p0p4-harness.sh" +fi +p0p4_bootstrap_suite "${BASH_SOURCE[0]}" + +research_skill="$FRAMEWORK_DIR/skills/assistant-research/SKILL.md" +research_reference="$FRAMEWORK_DIR/skills/assistant-research/research.md" +research_output="$FRAMEWORK_DIR/skills/assistant-research/contracts/output.yaml" +research_phase_gates="$FRAMEWORK_DIR/skills/assistant-research/contracts/phase-gates.yaml" +research_evals="$FRAMEWORK_DIR/skills/assistant-research/evals/cases.json" + +test_start "assistant-research candidate mechanisms stay evidence-backed and unproven" +missing_candidate_mechanism_terms=() +for term in \ + "candidate_mechanisms" \ + "counterevidence_or_conflicts" \ + "validation_method" \ + "claim_status"; do + if ! grep -Fq "$term" "$research_output"; then + missing_candidate_mechanism_terms+=("contracts/output.yaml: $term") + fi +done +for term in \ + "SY_CANDIDATE_MECHANISMS" \ + "VR_CANDIDATE_MECHANISMS" \ + "not presented as a proven cause"; do + if ! grep -Fq "$term" "$research_phase_gates"; then + missing_candidate_mechanism_terms+=("contracts/phase-gates.yaml: $term") + fi +done +for term in \ + "Candidate mechanisms" \ + "validation method"; do + if ! grep -Fq "$term" "$research_skill"; then + missing_candidate_mechanism_terms+=("SKILL.md: $term") + fi + if ! grep -Fq "$term" "$research_reference"; then + missing_candidate_mechanism_terms+=("research.md: $term") + fi +done +for term in \ + "candidate-mechanisms-are-evidence-backed-hypotheses" \ + "CANDIDATE MECHANISMS" \ + "Validation method"; do + if ! grep -Fq "$term" "$research_evals"; then + missing_candidate_mechanism_terms+=("evals/cases.json: $term") + fi +done +if [[ "${#missing_candidate_mechanism_terms[@]}" -eq 0 ]]; then + pass +else + fail "assistant-research candidate-mechanism contract missing terms: ${missing_candidate_mechanism_terms[*]}" +fi + +p0p4_finish_suite "${BASH_SOURCE[0]}" diff --git a/tests/p0-p4/workflow-basics-contracts.sh b/tests/p0-p4/workflow-basics-contracts.sh index dac316c..e8ecbaf 100644 --- a/tests/p0-p4/workflow-basics-contracts.sh +++ b/tests/p0-p4/workflow-basics-contracts.sh @@ -238,6 +238,64 @@ else fail "workflow candidate-search phase 1 contract missing terms: ${missing_candidate_terms[*]}" fi +test_start "workflow loop experiment artifacts stay conditional and non-harness" +missing_loop_artifact_terms=() +for term in \ + "workflow_experiment_ledger" \ + "explicit workflow experiment" \ + "loop_readiness_assessment" \ + "explicit repeat or optimization loop" \ + "retry_or_empty_result_handling" \ + "tool_error_handling" \ + "low_confidence_escalation" \ + "loop artifacts alone do not require"; do + if ! grep -Fq "$term" "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/output.yaml"; then + missing_loop_artifact_terms+=("output.yaml: $term") + fi +done +for term in \ + "P_WORKFLOW_EXPERIMENT_LEDGER" \ + "P_LOOP_READINESS" \ + "retry_or_empty_result_handling" \ + "tool_error_handling" \ + "low_confidence_escalation" \ + "Keep harness_routing=not_required unless harness_capable == true"; do + if ! grep -Fq "$term" "$FRAMEWORK_DIR/skills/assistant-workflow/contracts/phase-gates.yaml"; then + missing_loop_artifact_terms+=("phase-gates.yaml: $term") + fi +done +for term in \ + "Loop / Experiment Routing" \ + "harness_capable=false" \ + "retry_or_empty_result_handling" \ + "tool_error_handling" \ + "low_confidence_escalation" \ + "loop artifacts alone do not"; do + if ! grep -Fq "$term" "$FRAMEWORK_DIR/skills/assistant-workflow/references/plan-template.md"; then + missing_loop_artifact_terms+=("plan-template.md: $term") + fi + if ! grep -Fq "$term" "$FRAMEWORK_DIR/skills/assistant-workflow/references/task-journal-template.md"; then + missing_loop_artifact_terms+=("task-journal-template.md: $term") + fi +done +for term in \ + "loop-experiment-artifacts-stay-conditional" \ + "workflow_experiment_ledger" \ + "loop_readiness_assessment" \ + "retry_or_empty_result_handling" \ + "tool_error_handling" \ + "low_confidence_escalation" \ + "harness_capable=false"; do + if ! grep -Fq "$term" "$FRAMEWORK_DIR/skills/assistant-workflow/evals/cases.json"; then + missing_loop_artifact_terms+=("workflow eval: $term") + fi +done +if [[ "${#missing_loop_artifact_terms[@]}" -eq 0 ]]; then + pass +else + fail "workflow loop experiment contract missing terms: ${missing_loop_artifact_terms[*]}" +fi + test_start "workflow candidate-search root and assistant-dev plugin copies stay in sync" if [[ -d "$FRAMEWORK_DIR/plugins/assistant-dev/skills/assistant-workflow" ]] \ && diff -qr "$FRAMEWORK_DIR/skills/assistant-workflow" "$FRAMEWORK_DIR/plugins/assistant-dev/skills/assistant-workflow" >/tmp/p0p4-candidate-plugin-parity.out; then diff --git a/tests/test-p0-p4-contracts.sh b/tests/test-p0-p4-contracts.sh index 189ccde..02b0267 100755 --- a/tests/test-p0-p4-contracts.sh +++ b/tests/test-p0-p4-contracts.sh @@ -15,6 +15,7 @@ source "$P0P4_SUITE_DIR/installed-hook-smoke.sh" source "$P0P4_SUITE_DIR/skill-instruction-quality-contracts.sh" source "$P0P4_SUITE_DIR/assistant-ideate-reference-contracts.sh" source "$P0P4_SUITE_DIR/assistant-review-reference-contracts.sh" +source "$P0P4_SUITE_DIR/assistant-research-contracts.sh" source "$P0P4_SUITE_DIR/domain-rubrics-contracts.sh" source "$P0P4_SUITE_DIR/skill-validator-contracts.sh" source "$P0P4_SUITE_DIR/skill-eval-contracts.sh" From 96be5a602283b0d6baddb5a582de9883c403539f Mon Sep 17 00:00:00 2001 From: Laimis Date: Mon, 6 Jul 2026 12:00:11 +0300 Subject: [PATCH 8/9] Document loop readiness trigger --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6258aa7..d2f76ea 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,8 @@ Only tracked `assistant-*` directories are first-class release skills. ### assistant-workflow Core development pipeline: idea-to-action decomposition, triage, discover, plan, build & test, verify, document. +Loop readiness is conditional. Ordinary day-to-day development, such as taking a work item, prompting once, waiting for implementation, and manually testing the result, stays in the normal workflow. Add `loop_readiness_assessment` only before an explicit repeat, optimization, or experiment loop outside the standard phase gates. Examples include "keep fixing build/test failures until green", "iterate on this interface until the manual checklist passes", or "optimize until a measured target is reached". A loop plan must name the verifier, stop condition, finite max iterations, budget limit, retry/empty-result handling, tool-error handling, low-confidence escalation, rollback/exit action, and harness routing. Loop readiness alone does not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless harness or QA criteria independently apply. + Triggers on: build, implement, fix, refactor, plan, create, idea ### assistant-clarify From 24765375d2b8f385543a32017ccf65aeee36c6cc Mon Sep 17 00:00:00 2001 From: Laimis Date: Thu, 9 Jul 2026 11:58:27 +0300 Subject: [PATCH 9/9] Reduce workflow controller overhead --- docs/hook-output-benchmarks.md | 8 +- hooks/scripts/workflow-enforcer.sh | 64 ++++---- .../workflow-phase-gates.d/qa-controller.sh | 93 ++++++++++- .../subagent-evidence.sh | 2 +- .../skills/assistant-review/SKILL.md | 6 +- .../assistant-review/contracts/input.yaml | 14 +- .../references/qa-evaluation-loop.md | 9 +- .../skills/assistant-workflow/SKILL.md | 11 +- .../assistant-workflow/contracts/input.yaml | 30 +++- .../assistant-workflow/contracts/output.yaml | 20 ++- .../contracts/phase-gates.yaml | 18 ++- .../assistant-workflow/references/phases.md | 11 +- .../references/plan-harness-appendix.md | 11 +- .../references/plan-template.md | 4 + .../references/subagent-dispatch.md | 4 +- .../references/task-journal-template.md | 3 + .../references/triage-rubric.md | 11 ++ skills/assistant-review/SKILL.md | 6 +- skills/assistant-review/contracts/input.yaml | 14 +- .../references/qa-evaluation-loop.md | 9 +- skills/assistant-workflow/SKILL.md | 11 +- .../assistant-workflow/contracts/input.yaml | 30 +++- .../assistant-workflow/contracts/output.yaml | 20 ++- .../contracts/phase-gates.yaml | 18 ++- .../assistant-workflow/references/phases.md | 11 +- .../references/plan-harness-appendix.md | 11 +- .../references/plan-template.md | 4 + .../references/subagent-dispatch.md | 4 +- .../references/task-journal-template.md | 3 + .../references/triage-rubric.md | 11 ++ tests/p0-p4/harness-controller-contracts.sh | 23 +++ tests/p0-p4/instruction-overload-contracts.sh | 24 ++- tests/p0-p4/qa-evaluator-contracts.sh | 37 +++++ tests/p0-p4/runtime-phase-gate-contracts.sh | 144 +++++++++++++++++- tests/p0-p4/workflow-basics-contracts.sh | 8 +- 35 files changed, 588 insertions(+), 119 deletions(-) diff --git a/docs/hook-output-benchmarks.md b/docs/hook-output-benchmarks.md index fbd7c05..8eba54e 100644 --- a/docs/hook-output-benchmarks.md +++ b/docs/hook-output-benchmarks.md @@ -13,10 +13,10 @@ Current rows: 7 | hook_name | scenario | stdout_bytes | stdout_words | stderr_bytes | exit_code | first_blocker_or_action | |---|---|---:|---:|---:|---:|---| | session-start | codex active task journal | 980 | 102 | 0 | 0 | ACTIVE TASK JOURNAL AVAILABLE | -| workflow-enforcer | codex building phase gates | 1511 | 164 | 0 | 0 | WORKFLOW STATE (auto-injected every prompt): | +| workflow-enforcer | codex building phase gates | 1364 | 145 | 0 | 0 | WORKFLOW STATE (auto-injected every prompt): | | post-compact | codex restored active task | 749 | 75 | 0 | 0 | RESTORED AFTER COMPACTION — Active task journal available | | skill-router | codex assistant-workflow route | 498 | 47 | 0 | 0 | SKILL MATCH (1/1): This request matches assistant-workflow. You MUST load and follow this SKILL.md and its contracts before acting. At minimum: triage the task size, then build with tests included in the Build phase. Skipping workflow for speed is explicitly prohibited. Required inputs for this skill: [task_description,task_type,clarification_status,unresolved_clarification_topics,clarification_defaults_applied] | -| stop-review | codex missing spec review blocker | 207 | 26 | 0 | 0 | review_gate:no_spec_review missing=no Spec Review; ### Spec Review #N action=run Spec Review first and record PASS/FAIL plus scope, mismatch, and required-fix fields | +| stop-review | codex missing spec review blocker | 198 | 21 | 0 | 0 | review_gate:no_spec_review missing=no Spec Review; ### Spec Review #N action=run Spec Review first and record PASS/FAIL plus scope, mismatch, and required-fix fields | | subagent-monitor | codex code-writer start | 200 | 20 | 0 | 0 | SUBAGENT CONSTRAINT: You are a code writer. Write code only. Do NOT run builds or tests — builder-tester handles that. | | subagent-monitor | codex code-writer stop | 0 | 0 | 0 | 0 | recorded SubagentStop lifecycle evidence | @@ -29,8 +29,8 @@ The C slice trimmed repeated explanatory prose from `session-start`, `workflow-e | c-hook-output-trim | session-start | 2122 | 225 | 1766 | 183 | | c-hook-output-trim | workflow-enforcer | 1892 | 232 | 1625 | 184 | | c-hook-output-trim | post-compact | 2131 | 218 | 1526 | 154 | -| runtime-gate-output-trim | workflow-enforcer | 1626 | 184 | 1511 | 164 | -| runtime-gate-output-trim | stop-review | 371 | 54 | 207 | 26 | +| runtime-gate-output-trim | workflow-enforcer | 1626 | 184 | 1364 | 145 | +| runtime-gate-output-trim | stop-review | 371 | 54 | 198 | 21 | ## Signal Anchors Checked diff --git a/hooks/scripts/workflow-enforcer.sh b/hooks/scripts/workflow-enforcer.sh index ef07873..3b034e1 100755 --- a/hooks/scripts/workflow-enforcer.sh +++ b/hooks/scripts/workflow-enforcer.sh @@ -62,7 +62,7 @@ emit_workflow_rules_context() { - Phases: TRIAGE -> DISCOVER -> DECOMPOSE when needed -> PLAN -> DESIGN when needed -> BUILD -> REVIEW -> DOCUMENT - Do not skip phases; small tasks still use lightweight phases. - BUILD includes same-step tests for features. -- REVIEW loops until clean. +- REVIEW loops until clean or max 10 rounds. STATE BOOTSTRAP (when no active task journal is present): - For development/code-work, create or refresh $STATE_DIR/task.md before planning or implementation. @@ -260,6 +260,20 @@ subagent_policy_state=${subagent_policy_state:-unknown} subagent_execution_mode=${subagent_execution_mode:-unknown} subagent_authorization_scope=${subagent_authorization_scope:-} +subagent_scope_items=() +while IFS= read -r scope_item; do + subagent_scope_items+=("$scope_item") +done < <(printf '%s\n' "$subagent_authorization_scope" | sed '/^[[:space:]]*$/d') + +if (( ${#subagent_scope_items[@]} == 0 )) || [[ ${#subagent_scope_items[@]} -eq 1 && "${subagent_scope_items[0]}" == "none" ]]; then + subagent_authorization_scope_summary="none" +else + subagent_authorization_scope_summary="${subagent_scope_items[0]}" + for ((i = 1; i < ${#subagent_scope_items[@]}; i++)); do + subagent_authorization_scope_summary+=", ${subagent_scope_items[i]}" + done +fi + has_clarification_status_field="no" has_clarification_defaults_field="no" has_clarification_topics_field="no" @@ -415,40 +429,20 @@ if [[ "$STATE_DIR" == ".codex" && "$subagent_policy_state" == "authorization_req exit 0 fi -# Build phase-aware enforcement context +# Build compact phase-aware enforcement context. Detailed recovery text is +# appended below only for gates that are incomplete and actionable now. context="WORKFLOW STATE (auto-injected every prompt): -- Task: $task_name -- Size: $size -- Phase: $status -- Clarification status: $clarification_status -- Clarification defaults applied: $clarification_defaults -- Clarification confidence: $clarification_confidence -- Clarification questions: $clarification_questions_asked/$clarification_question_cap (cap is maximum, not quota) -- Clarification admissibility: $clarification_admissibility -- Unresolved clarification topics: $clarification_topics_summary -- Plan approved: $has_plan_approval -- Reviews completed: $review_count -- Final result: $has_final_result -- Review gate complete: $has_review_completion -- Subagent policy state: $subagent_policy_state -- Subagent execution mode: $subagent_execution_mode -- Subagent authorization scope: ${subagent_authorization_scope:-none} -- Subagent evidence gate: $subagent_gate_status -- Metrics today: $has_metrics_today - -PHASE RULES: -- Current phase is $status — stay until exit criteria pass. -- PLAN approval before BUILD; BUILD includes same-step tests for new components. -- REVIEW loops review -> fix -> re-review until clean or max 10 rounds. -- State the current phase before the next action." +state: Task: $task_name | Size: $size | Phase: $status +clarification: Clarification status: $clarification_status | Clarification defaults applied: $clarification_defaults | Clarification confidence: $clarification_confidence | Clarification questions: $clarification_questions_asked/$clarification_question_cap (cap is maximum, not quota) | Clarification admissibility: $clarification_admissibility | Unresolved clarification topics: $clarification_topics_summary +plan: Plan approved: $has_plan_approval +review: Reviews completed: $review_count | Final result: $has_final_result | Review gate complete: $has_review_completion +subagent: Subagent policy state: $subagent_policy_state | Subagent execution mode: $subagent_execution_mode | Subagent authorization scope: $subagent_authorization_scope_summary | Subagent evidence gate: $subagent_gate_status +metrics: Metrics today: $has_metrics_today +rule: Current phase is $status; state phase before action." context+=" -RUNTIME PHASE GATES: -- Plan approved: $has_plan_approval -- Review gate complete: $has_review_completion -- Subagent evidence gate: $subagent_gate_status -- Metrics today: $has_metrics_today" +RUNTIME PHASE GATES: Plan approved: $has_plan_approval | Review gate complete: $has_review_completion | Subagent evidence gate: $subagent_gate_status | Metrics today: $has_metrics_today" if [[ "$clarification_gate_active" == "yes" ]]; then context+=" @@ -500,28 +494,28 @@ if [[ "$is_building" == "yes" && "$has_plan_approval" == "no" && "$size" != "sma plan_missing_field="$(assistant_phase_reason_missing_field "plan_gate" "$plan_missing_key")" plan_action="$(assistant_phase_reason_action "plan_gate" "$plan_missing_key")" context+=" -WARNING: You are BUILDING without an approved plan ($plan_missing_key). missing=$plan_missing_field action=$plan_action" +WARNING: You are BUILDING without an approved plan ($plan_missing_key). plan_gate:$plan_missing_key missing=$plan_missing_field action=$plan_action" fi subagent_warning_key="$(assistant_phase_subagent_warning_reason_key "$TASK_FILE" "$subagent_gate_status" "$status" || true)" subagent_warning_action="$(assistant_phase_subagent_warning_action "$subagent_warning_key" "$status" || true)" if [[ -n "$subagent_warning_action" ]]; then context+=" -WARNING: Subagent evidence gate incomplete ($subagent_warning_key). $subagent_warning_action" +WARNING: Subagent evidence gate incomplete ($subagent_warning_key). subagent_evidence_gate:$subagent_warning_key $subagent_warning_action" fi if [[ ( "$is_reviewing" == "yes" || "$is_documenting" == "yes" ) && "$has_review_completion" == "no" ]]; then review_missing_field="$(assistant_phase_reason_missing_field "review_gate" "$review_gate_status")" review_action="$(assistant_phase_reason_action "review_gate" "$review_gate_status")" context+=" -WARNING: Review gate incomplete ($review_gate_status). missing=$review_missing_field action=$review_action" +WARNING: Review gate incomplete ($review_gate_status). review_gate:$review_gate_status missing=$review_missing_field action=$review_action" fi if [[ "$is_documenting" == "yes" && "$has_metrics_today" == "no" ]]; then metrics_missing_field="$(assistant_phase_reason_missing_field "metrics_gate" "missing_metrics_today")" metrics_action="$(assistant_phase_reason_action "metrics_gate" "missing_metrics_today")" context+=" -WARNING: Metrics gate incomplete. missing=$metrics_missing_field action=$metrics_action" +WARNING: Metrics gate incomplete. metrics_gate:missing_metrics_today missing=$metrics_missing_field action=$metrics_action" fi if [[ "$clarification_gate_active" == "yes" && "$requires_saved_clarification_state" == "yes" ]]; then diff --git a/hooks/scripts/workflow-phase-gates.d/qa-controller.sh b/hooks/scripts/workflow-phase-gates.d/qa-controller.sh index 345a129..cd0162c 100644 --- a/hooks/scripts/workflow-phase-gates.d/qa-controller.sh +++ b/hooks/scripts/workflow-phase-gates.d/qa-controller.sh @@ -13,10 +13,45 @@ assistant_phase_requires_qa_evaluator() { low = tolower(trim(value)) return low ~ /^(n\/a|na|not_required|not required|not-required|not-applicable|not_applicable|not applicable)([[:space:][:punct:]]|$)/ } + function explicit_optional(value, low) { + low = tolower(trim(value)) + return low ~ /^optional([[:space:][:punct:]]|$)/ + } function bracket_placeholder(value) { value = trim(value) return value ~ /^\[[^]]*\]$/ } + function non_trigger_text(value, low) { + low = tolower(trim(value)) + gsub(/`/, "", low) + return low == "" || + low ~ /\|/ || + low ~ /^\[[^]]*\]$/ || + explicit_not_required(low) || + explicit_optional(low) || + low ~ /(^|[[:space:][:punct:]])(n\/a|not_required|not required|not-required|not-applicable|not_applicable|not applicable|optional)([[:space:][:punct:]]|$)/ || + low ~ /(^|[[:space:][:punct:]])(no|without)[[:space:]]+(explicit qa|qa request|accepted done contract|done contract|harness-capable acceptance|harness capable acceptance|domain-scored|domain scored|ui|visual|product|ux|docs|dx)/ || + low ~ /(^|[[:space:][:punct:]])(pending|waiting|todo|tbd|not yet)([[:space:][:punct:]]|$)/ || + low ~ /(when|if|unless|only when)[^.\n]*(required|qa_evaluation_mode|qa evaluation mode)/ || + low ~ /(template|placeholder|generic acceptance criteria)/ + } + function positive_required_trigger(value, low) { + low = tolower(trim(value)) + gsub(/`/, "", low) + if (non_trigger_text(low)) { + return 0 + } + return low ~ /^required([[:space:][:punct:]]|$)/ || + low ~ /qa[ _-]?evaluation[ _-]?mode[[:space:]]*[:=][[:space:]]*required([[:space:][:punct:]]|$)/ || + low ~ /(^|[[:space:][:punct:]])qa[ _-]?evaluator([[:space:][:punct:]]|$)/ || + low ~ /(explicit|requested|user[^[:alnum:]]+asks)[^.\n]*(qa|acceptance[ _-]?evaluation)/ || + low ~ /(qa|acceptance[ _-]?evaluation)[^.\n]*(explicit|requested|required)/ || + low ~ /(accepted[ _-]?done[ _-]?contract|done[ _-]?contract[^.\n]*(accepted|accepted_by))/ || + low ~ /(harness[ _-]?capable|harness_capable|harness)[^.\n]*(acceptance[ _-]?(scope|evaluation)|acceptance)/ || + low ~ /(domain[ _-]?scored|domain[ _-]?quality|domain[ _-]?context|domain[ _-]?rubric|rubric_refs)/ || + (low ~ /(^|[[:space:][:punct:]])(ui|visual|product|ux|docs|dx)([[:space:][:punct:]]|$)/ && low ~ /(acceptance|quality|rubric|domain)/) || + low ~ /(multi_agent|agent_id|final[ _-]?verdict|qa[ _-]?result)/ + } function qa_mode_required_line(line, low) { low = tolower(trim(line)) sub(/^[[:space:]]*[-*][[:space:]]*/, "", low) @@ -29,6 +64,12 @@ assistant_phase_requires_qa_evaluator() { return low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]*[:=][[:space:]]*(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ || low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]+(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ } + function qa_mode_optional_line(line, low) { + low = tolower(trim(line)) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", low) + return low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]*[:=][[:space:]]*optional([[:space:][:punct:]]|$)/ || + low ~ /^(qa_evaluation_mode|qa[ _-]?evaluation[ _-]?mode)[[:space:]]+optional([[:space:][:punct:]]|$)/ + } function qa_section_mode_required_line(line, value, low) { line = trim(line) sub(/^[[:space:]]*[-*][[:space:]]*/, "", line) @@ -61,16 +102,33 @@ assistant_phase_requires_qa_evaluator() { low = tolower(value) return low ~ /^(n\/a|na|not_required|not required|not-required|not_applicable|not applicable|not-applicable)([[:space:][:punct:]]|$)/ } + function qa_section_mode_optional_line(line, value, low) { + line = trim(line) + sub(/^[[:space:]]*[-*][[:space:]]*/, "", line) + low = tolower(line) + if (low !~ /^mode[[:space:]]*:/) { + return 0 + } + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + value = trim(value) + if (value ~ /\|/ || bracket_placeholder(value)) { + return 0 + } + low = tolower(value) + return low ~ /^optional([[:space:][:punct:]]|$)/ + } function line_marks_not_required(line, low) { low = tolower(line) - return low ~ /(^|[[:space:][:punct:]])(n\/a|not_required|not required|not-required|not-applicable|not_applicable|not applicable)([[:space:][:punct:]]|$)/ + return low ~ /(^|[[:space:][:punct:]])(n\/a|not_required|not required|not-required|not-applicable|not_applicable|not applicable|optional)([[:space:][:punct:]]|$)/ || + low ~ /(when|if|unless|only when)[^.\n]*(required|qa_evaluation_mode|qa evaluation mode)/ } function qa_marker(line, low) { low = tolower(line) return low ~ /(qa[ _-]?evaluator|qaevaluator|qa[ _-]?evaluation|qa_evaluation_mode|qa[ _-]?loop)/ } function scan_required_line(line) { - if (qa_marker(line) && !line_marks_not_required(line)) { + if (qa_marker(line) && !line_marks_not_required(line) && positive_required_trigger(line)) { found = 1 } } @@ -80,13 +138,24 @@ assistant_phase_requires_qa_evaluator() { if (low ~ /^(qa evaluator dispatch|qa evaluator result|qa evaluator direct evidence):/) { value = line sub(/^[^:]*:[[:space:]]*/, "", value) - if (!bracket_placeholder(value) && !explicit_not_required(value)) { + if (!non_trigger_text(value) && (positive_required_trigger(value) || positive_required_trigger(line))) { found = 1 } } else if (low ~ /^qa evaluator dispatch\/result\/direct evidence:/) { value = line sub(/^[^:]*:[[:space:]]*/, "", value) - if (!bracket_placeholder(value) && !explicit_not_required(value)) { + if (!non_trigger_text(value) && (positive_required_trigger(value) || positive_required_trigger(line))) { + found = 1 + } + } + } + function scan_trigger_label(line, value, low) { + sub(/^[[:space:]]*[-*]?[[:space:]]*/, "", line) + low = tolower(line) + if (low ~ /^(qa trigger reason|qa_trigger_reason|done contract|done_contract|done_contract_ref|harness routing|harness_routing|harness_capable|acceptance scope|qa evaluation scope|qa scope|domain_context|domain context|rubric_refs|rubric refs)[[:space:]]*:/) { + value = line + sub(/^[^:]*:[[:space:]]*/, "", value) + if (!non_trigger_text(value) && (positive_required_trigger(value) || positive_required_trigger(line))) { found = 1 } } @@ -113,6 +182,10 @@ assistant_phase_requires_qa_evaluator() { not_required_mode = 1 next } + if (qa_section_mode_optional_line(line)) { + optional_mode = 1 + next + } } if (qa_mode_required_line(line)) { required_mode = 1 @@ -123,8 +196,13 @@ assistant_phase_requires_qa_evaluator() { not_required_mode = 1 next } + if (qa_mode_optional_line(line)) { + optional_mode = 1 + next + } scan_qa_label(line) + scan_trigger_label(line) if (low ~ /^required agents:[[:space:]]*$/) { in_required_agents = 1 @@ -151,10 +229,13 @@ assistant_phase_requires_qa_evaluator() { } } END { - if (not_required_mode && !required_mode) { + if (found) { + exit 0 + } + if ((not_required_mode || optional_mode) && !required_mode) { exit 1 } - exit found ? 0 : 1 + exit 1 } ' "$file" 2>/dev/null } diff --git a/hooks/scripts/workflow-phase-gates.d/subagent-evidence.sh b/hooks/scripts/workflow-phase-gates.d/subagent-evidence.sh index 07e75d6..4ae010a 100644 --- a/hooks/scripts/workflow-phase-gates.d/subagent-evidence.sh +++ b/hooks/scripts/workflow-phase-gates.d/subagent-evidence.sh @@ -352,7 +352,7 @@ assistant_phase_required_subagent_roles() { if (low ~ /architect/) emit("Architect") if (low ~ /code writer|code-writer/) emit("Code Writer") if (low ~ /builder\/tester|builder-tester/) emit("Builder/Tester") - if (low ~ /qa[ _-]?evaluator|qaevaluator/) emit("QA Evaluator") + if (qa_required == "yes" && low ~ /qa[ _-]?evaluator|qaevaluator/) emit("QA Evaluator") if (low ~ /code[ _-]?reviewer|codereviewer/) { emit("Code Reviewer") } else if (low ~ /(^|[^[:alnum:]_-])reviewer([^[:alnum:]_-]|$)/) { diff --git a/plugins/assistant-dev/skills/assistant-review/SKILL.md b/plugins/assistant-dev/skills/assistant-review/SKILL.md index 7765577..1fcc857 100644 --- a/plugins/assistant-dev/skills/assistant-review/SKILL.md +++ b/plugins/assistant-dev/skills/assistant-review/SKILL.md @@ -45,6 +45,8 @@ Find concrete defects, risks, regressions, and test gaps; fix them when in revie - In review-fix mode, must-fix and should-fix findings are addressed or explicitly deferred. - Validation runs after fixes, and a fresh review confirms the final state. - QA evaluation runs after code-review/build evidence when `qa_evaluation_mode=required`, returns score progression and a final acceptance verdict, and does not replace code-reviewer. +- QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. +- QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. - QA evaluation loads `references/domain-rubrics.md` only when `domain_context`, explicit `rubric_refs`, or subjective/UI/visual/product/UX/docs/DX/domain acceptance criteria require scoped domain-quality scoring. ## Constraints @@ -78,7 +80,7 @@ Use the smallest mode that answers the request; combine modes when reviewing imp - **Semantic contract review**: for skill/workflow/framework changes, check contract inheritance, method-template alignment, eval coverage, method-signature fidelity, and high-stakes recommendation guards before judging the change clean. - **Maintainability review**: apply SOLID/KISS/DRY/YAGNI/readability only when a concrete risk exists. - **Security handoff**: invoke `assistant-security` when the reviewed surface touches auth, permissions, secrets, input handling, persistence, shell commands, dependency/config changes, network calls, or external integrations. -- **QA evaluation**: after code-review/build evidence exists, dispatch QAEvaluator when `qa_evaluation_mode=required` for explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. Load `references/qa-evaluation-loop.md`. +- **QA evaluation**: after code-review/build evidence exists, dispatch QAEvaluator only when `qa_evaluation_mode=required`. Load `references/qa-evaluation-loop.md`. - **Domain rubric QA**: within QA evaluation, load `references/domain-rubrics.md` only when scoped by acceptance criteria, Done Contract, `domain_context`, or explicit `rubric_refs`. QAEvaluator selects rubric families and returns domain-quality scores; Code Reviewer still owns code defects, security, architecture, and test coverage. Finding format: @@ -223,7 +225,7 @@ while round <= 10: ## The QA Evaluation Loop -Run QA only when `qa_evaluation_mode=required` or the workflow Review phase requests QA evidence. Load `references/qa-evaluation-loop.md` before dispatching QAEvaluator; that reference owns the detailed algorithm, score progression, domain-rubric routing, pivot/restart behavior, and terminal round-10 cap. +Run QA only when `qa_evaluation_mode=required`; the workflow Review phase should set that mode only from the QA required positive triggers above. Load `references/qa-evaluation-loop.md` before dispatching QAEvaluator; that reference owns the detailed algorithm, score progression, domain-rubric routing, pivot/restart behavior, and terminal round-10 cap. At this level, keep only the routing boundary: QA runs after build/test evidence and Code Reviewer or Reviewer compatibility evidence exist, and it evaluates acceptance criteria, Done Contract evidence, verification evidence, scoped domain quality, score progression, and final readiness. QA does not replace code review or report general code defects unless they directly block acceptance. diff --git a/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml b/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml index 55137e2..69cdee8 100644 --- a/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml +++ b/plugins/assistant-dev/skills/assistant-review/contracts/input.yaml @@ -112,13 +112,19 @@ fields: required: false enum_values: [not_required, optional, required] description: "Whether the independent QA evaluation loop should run after code-review/build evidence" - validation: "required only for explicit QA/acceptance evaluation, harness_capable == true with acceptance evaluation in scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing scope; not_required must have a reason when scoped acceptance QA is skipped" + validation: "QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work." default: not_required on_missing: infer infer_from: > - Use required when the user explicitly requests QA/acceptance evaluation, an accepted Done Contract - exists, harness_capable == true and acceptance evaluation is in scope, the task is domain-scored, - or it affects UI/visual/product/UX/docs/DX quality. + Use required only for QA required positive triggers: explicit QA/acceptance evaluation request, + accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped + UI/visual/product/UX/docs/DX acceptance. + Use required when the user explicitly requests QA/acceptance evaluation, an accepted Done + Contract exists, harness_capable == true and acceptance evaluation is in scope, the task is + domain-scored, or scoped UI/visual/product/UX/docs/DX acceptance applies. + QA non-triggers: template labels/placeholders, generic acceptance criteria labels, + optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ + code-review-only/source-changing work. Use optional when acceptance material exists but the orchestrator has not marked QA as required. Use not_required for ordinary medium+ code-review-only/source-changing work with no separate acceptance evidence need. diff --git a/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md b/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md index 5219919..d228b3c 100644 --- a/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md +++ b/plugins/assistant-dev/skills/assistant-review/references/qa-evaluation-loop.md @@ -4,14 +4,17 @@ Use this reference after build/test evidence and the Code Reviewer loop exist. Q ## When QA Runs +QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. +QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. + Run QA evaluation when any of these are true: - The user explicitly asks for QA or acceptance evaluation. - The task has an accepted Done Contract. - The task is harness-capable and acceptance evaluation is in scope. -- The task is subjective, domain-scored, or UI/visual/product/UX/docs/DX-facing. -- Workflow Review needs a final acceptance verdict beyond code/security/architecture/test-coverage review. +- The task has domain-scored scope or scoped UI/visual/product/UX/docs/DX acceptance. +- Workflow Review needs a final acceptance verdict because one of the positive triggers above is present. -Skip QA evaluation when the task has no explicit QA request, Done Contract, harness-capable acceptance scope, domain-scored criteria, or UI/visual/product/UX/docs/DX scope, and the workflow output records `qa_evaluation_mode=not_required` with a reason. +Skip QA evaluation when the task has no explicit QA request, Done Contract, harness-capable acceptance scope, domain-scored criteria, or UI/visual/product/UX/docs/DX scope, and the workflow output records `qa_evaluation_mode=not_required` with a reason. Template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work do not force QA. ## Inputs diff --git a/plugins/assistant-dev/skills/assistant-workflow/SKILL.md b/plugins/assistant-dev/skills/assistant-workflow/SKILL.md index 64e16c9..3aadfd0 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/SKILL.md +++ b/plugins/assistant-dev/skills/assistant-workflow/SKILL.md @@ -26,7 +26,10 @@ This skill is intentionally agent-agnostic: it must work in restricted company e - Medium+ harness-capable work has an accepted Done Contract and Harness Recipe before Build. - Trace/replay-ready harness work maintains Harness Run State, Trace Ledger, and Replay Packet artifacts. - Ordinary medium+ workflow tasks default to `harness_capable=false` and do not inherit Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless explicit harness/QA criteria apply. -- Harness-capable acceptance, Done Contract, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work records independent QA Evaluator evidence after build/test and code-review evidence. +- `controller_intensity=standard` is the ordinary medium+ non-harness default; `strict` needs independent strict/harness/QA criteria. +- QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. +- QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. +- Required QA records independent QA Evaluator evidence after build/test and code-review evidence. - Behavior changes have tests or explicit validation attached to the implementation step they protect. - Final output reports changed files, verification evidence, residual risks, and next steps. - Candidate Search is used for explicit alternatives, open-ended architecture/design, optimization, high uncertainty, repeated failed attempts, unclear/flaky bugs, or reviewer-requested pivots — not as default ceremony. @@ -177,7 +180,7 @@ Load `references/triage-rubric.md`. Perform a quick read-only candidate scope sc [Design] = include if task has UI work, skip for backend-only. Print: `>> Triaged as: [SIZE] — phases: [list]` -Print: `>> Triage metadata: type=[TASK_TYPE] | risk=[RISK_TIER] | gates=[count] | agents=[count] | search=[search_mode] | scope_confidence=[low|medium|high]` +Print: `>> Triage metadata: type=[TASK_TYPE] | risk=[RISK_TIER] | intensity=[controller_intensity] | gates=[count] | agents=[count] | search=[search_mode] | scope_confidence=[low|medium|high]` If scope exceeds initial triage during any phase, stop and re-triage. @@ -212,7 +215,7 @@ Load `references/phases.md` and execute the phase matching your current stage. U For subagent roles and dispatch rules, load `references/subagent-dispatch.md` and resolve `subagent_policy_state`, `subagent_execution_mode`, and `subagent_authorization_scope` before any subagent spawn. Assistant Framework policy requires explicit user authorization before spawning subagents for development/code-work roles unless the current user prompt already explicitly authorizes them for this task. Ask once for the needed delegation scope and wait before continuing phases that require subagents. A sufficient prompt is: `This workflow expects Code Mapper, Architect, Code Writer, Builder/Tester, Code Reviewer, and QA Evaluator when QA is required for [scope]. May I use subagents for this task?` After authorization, use `delegated` mode and spawn the configured role agents. Use `direct_fallback` only when authorization is denied, policy disallows spawning, or a real spawn attempt fails because subagents/custom agents are unavailable; do not infer unavailability merely because no visible tool is named `Task`, `delegate`, or `subagent`. For BES-style option exploration, load `references/candidate-search.md` only when `search_mode: candidate_search` is selected. For mega tasks and anti-patterns, load `references/mega-and-patterns.md`. -Load `references/harness-controller.md` only when medium+ work is harness-capable: long-running, trace/replay-ready multi-slice, high-risk harness, subjective/domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested as harness work. +Load `references/harness-controller.md` only when medium+ work is harness-capable: long-running, trace/replay-ready multi-slice, high-risk harness, subjective/domain-scored, scoped UI/visual/product/UX/docs/DX acceptance, or explicitly requested as harness work. ## Planning Checklist @@ -245,7 +248,7 @@ Before final output for medium+ or high-risk work: - Review the diff against the acceptance criteria. - For bugfixes, verify the review material includes reproduction/root-cause evidence from `assistant-debugging` or a clear reason it was not applicable. - Use `assistant-review` for code quality/spec review when the change is non-trivial. -- Use QA Evaluator through `assistant-review` when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work after build/test and code-review evidence. +- Use QA Evaluator through `assistant-review` when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance after build/test and code-review evidence. - Use `assistant-security` when touching auth, user input, secrets, persistence, network calls, shell commands, dependency/config changes, or external integrations. Review findings must cite evidence and concrete risk. Avoid generic style feedback unless it affects correctness, security, maintainability, or test reliability. diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml index 9568c6d..0762017 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/input.yaml @@ -149,18 +149,40 @@ fields: or Harness Recipe. Do not infer true from size=medium+, delegation alone, or ordinary source-changing workflow tasks. + - name: controller_intensity + type: enum + required: false + enum_values: [light, standard, strict] + description: "Completion/gate intensity selected during Triage; maps onto existing completion_tiers and gate_tiers." + validation: "light uses small/low-risk guidance, standard uses ordinary medium+ completion/review without harness/QA defaults, and strict uses strict gates or harness/QA artifacts only when their existing conditions apply." + default: standard + on_missing: infer + infer_from: > + Use light for small low-risk localized work. Use standard for ordinary medium+ + source-changing work, including delegated work, when harness_capable == false + and qa_evaluation_mode == not_required. Use strict only for high/critical risk, + hook_profile == strict, harness_capable == true, qa_evaluation_mode == required, + explicit harness/QA acceptance work, or trace/replay-ready work. Do not infer + strict from size=medium+ or delegation alone. + - name: qa_evaluation_mode type: enum required: false enum_values: [not_required, optional, required] description: "Whether Review phase should run the independent QA Evaluator acceptance loop after build/test and code-review evidence" - validation: "required only for explicit QA/acceptance evaluation, harness_capable == true with acceptance evaluation in scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing scope; not_required is the default for ordinary medium+ source-changing work." + validation: "QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work." default: not_required on_missing: infer infer_from: > - Use required when the user explicitly requests QA/acceptance evaluation, the task has an accepted - Done Contract, harness_capable == true and acceptance evaluation is in scope, the task is - domain-scored, or it affects UI/visual/product/UX/docs/DX quality. + Use required only for QA required positive triggers: explicit QA/acceptance evaluation request, + accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped + UI/visual/product/UX/docs/DX acceptance. + Use required when the user explicitly requests QA/acceptance evaluation, the task has an + accepted Done Contract, harness_capable == true and acceptance evaluation is in scope, the task + is domain-scored, or scoped UI/visual/product/UX/docs/DX acceptance applies. + QA non-triggers: template labels/placeholders, generic acceptance criteria labels, + optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ + code-review-only/source-changing work. Use optional when acceptance material exists but QA is not required. Use not_required for ordinary medium+ code-review-only/source-changing work with no separate acceptance evidence need. diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml index e033d33..dcce215 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/output.yaml @@ -10,18 +10,25 @@ skill: assistant-workflow # Completion evidence is tiered so small/low-risk work does not inherit the # full medium+ harness. Artifact definitions below remain the source of truth; # these tiers define which artifacts are required by default for each class. +# controller_intensity maps onto these tiers and gate_tiers: light -> small, +# standard -> ordinary medium+ non-harness, strict -> large/critical or +# explicit strict/harness/QA criteria. completion_tiers: small: + controller_intensity: light required_artifacts: [changed_files, validation_results, manual_test_steps, user_approval] conditional_artifacts: [triage_result, subagent_evidence, test_results, debugging_result, artifact_contract, workflow_experiment_ledger, loop_readiness_assessment] review_gate: "self-review or explicit validation summary unless risk_tier in [high, critical]" checkpoint_policy: "inline plan/check summary is enough; phase_checkpoints are strict-profile only" medium: + controller_intensity: standard + intensity_policy: "ordinary medium+ non-harness work keeps harness_capable=false and qa_evaluation_mode=not_required by default; standard does not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation." required_artifacts: [triage_result, changed_files, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, plan_document, user_approval] conditional_artifacts: [test_results, task_journal, context_map, slice_manifest, single_slice_rationale, slice_verification_summary, metrics_entry, candidate_search_result, workflow_experiment_ledger, loop_readiness_assessment, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result] review_gate: "spec review plus code quality review, with independent QA evaluation when required and evidence matched to the approved plan" checkpoint_policy: "task journal or carried-forward state records phase transitions" large_critical: + controller_intensity: strict required_artifacts: [triage_result, phase_checkpoints, changed_files, test_results, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, task_journal, context_map, context_budget_note, slice_manifest, slice_verification_summary, plan_document, user_approval] conditional_artifacts: [debugging_result, artifact_contract, pattern_retrieval_note, decomposition_plan_review, single_slice_rationale, metrics_entry, candidate_search_result, workflow_experiment_ledger, loop_readiness_assessment, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result, verified_skill_distillation] review_gate: "full strict evidence, security/operability review when applicable, QA evaluation when required, and all slice criteria verified" @@ -32,8 +39,8 @@ artifacts: type: object required: conditional condition: "size in [medium, large, mega] or hook_profile == strict or risk_tier in [high, critical]" - validation: "Contains task_type, risk_tier, size, required_gates, required_agents, subagent policy state, execution mode, authorization scope, search_mode, and candidate_scope_scan selected during Triage" - on_fail: "Return to TRIAGE and populate task_type, risk_tier, size, required_gates, required_agents or fallback roles, subagent policy state, execution mode, authorization scope, search_mode, and candidate_scope_scan" + validation: "Contains task_type, risk_tier, size, controller_intensity, required_gates, required_agents, subagent policy state, execution mode, authorization scope, search_mode, and candidate_scope_scan selected during Triage" + on_fail: "Return to TRIAGE and populate task_type, risk_tier, size, controller_intensity, required_gates, required_agents or fallback roles, subagent policy state, execution mode, authorization scope, search_mode, and candidate_scope_scan" object_fields: - name: task_type type: enum @@ -47,6 +54,11 @@ artifacts: type: enum required: true enum_values: [small, medium, large, mega] + - name: controller_intensity + type: enum + required: true + enum_values: [light, standard, strict] + validation: "Maps to completion_tiers and gate_tiers. standard is used for ordinary medium+ non-harness work and does not require Done Contract, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation by default; strict requires independent strict, harness, or QA criteria. Do not infer strict from size=medium+ or delegation alone." - name: required_gates type: string[] required: true @@ -1436,7 +1448,7 @@ artifacts: # Completeness check: # Before printing --- WORKFLOW COMPLETE ---, verify: -# 1. All required artifacts for the task's completion_tier are present +# 1. All required artifacts for the task's completion_tier are present and controller_intensity matches the selected tier # 2. test_results.failed == 0 when tests ran; validation_results contains at least one result=passed entry for completed implementation workflows. Skipped/not_applicable entries do not satisfy completion evidence; docs-only/no-op/discovery exceptions must record the reason. # 3. When size is medium/large/mega, risk_tier is high/critical, or hook_profile == strict: spec_review_result.status == PASS and required_fixes are empty or resolved # 4. When size is medium/large/mega, risk_tier is high/critical, or hook_profile == strict: review_result.quality_review_status is not missing @@ -1445,7 +1457,7 @@ artifacts: # 7. candidate_search_result exists when search_mode == candidate_search, includes search_exit_summary, and any plan_deviation is approved before Build # 8. workflow_experiment_ledger exists only for explicit workflow experiments; loop_readiness_assessment exists only before explicit repeat/optimization loops and includes retry_or_empty_result_handling, tool_error_handling, and low_confidence_escalation # 9. Loop artifacts alone do not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless harness_capable or QA criteria independently apply -# 10. For medium+ harness-capable work, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision when triggered, and artifact_reference_ledger exist and are current before completion or handoff +# 10. For medium+ harness-capable work, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision when triggered, and artifact_reference_ledger exist and are current before completion or handoff. controller_intensity=standard with harness_capable=false does not require these artifacts. # 11. user_approval is confirmed, confirmed_with_issues, or not_required_small for eligible small tasks # If any check fails, do NOT print --- WORKFLOW COMPLETE ---. State what is missing. - name: verified_skill_distillation diff --git a/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml b/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml index 4bdcb62..5b4ceaf 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml +++ b/plugins/assistant-dev/skills/assistant-workflow/contracts/phase-gates.yaml @@ -23,6 +23,9 @@ gate_tiers: blocker: "Prevents phase transition; missing evidence would make the workflow unsafe or unverifiable." guidance: "Preferred discipline; record or follow when useful, but do not ask ritual questions or block low-risk progress solely for this item." strict_only: "Enforced only when hook_profile == strict or project policy explicitly requests full harness enforcement." +# controller_intensity maps onto these gate_tiers: light keeps low-risk guidance +# compact, standard uses normal medium blockers/review without harness/QA +# defaults, and strict may enforce strict_only gates when independent criteria apply. gates: @@ -50,6 +53,10 @@ gates: check: "risk_tier is set to one of: low, moderate, high, critical" on_fail: "Assess risk tier using impact, uncertainty, reversibility, security/data exposure, and behavior parity criteria" + - id: T_CONTROLLER_INTENSITY + check: "controller_intensity is set to one of: light, standard, strict; ordinary medium+ non-harness work uses standard; strict is selected only for high/critical risk, hook_profile == strict, harness_capable == true, qa_evaluation_mode == required, or explicit trace/replay/harness/QA criteria" + on_fail: "Set controller_intensity from existing completion_tiers/gate_tiers. Do not infer strict from size=medium+ or delegation alone; use standard when harness_capable=false and qa_evaluation_mode=not_required." + - id: T6 check: "required_gates includes common gates plus every applicable task-category gate pack" on_fail: "Load references/triage-rubric.md and add the required gate packs for the task type before continuing" @@ -59,9 +66,9 @@ gates: on_fail: "Select required workflow agents when subagent_execution_mode=delegated, or record direct fallback roles and responsibilities from references/subagent-dispatch.md and risk-specific gate packs. not_applicable is invalid for source-changing Build tasks." - id: T_QA_EVALUATOR - check: "When qa_evaluation_mode == required: required_agents includes QA Evaluator in addition to Code Writer, Builder/Tester, and Code Reviewer or Reviewer compatibility" + check: "When qa_evaluation_mode == required: required_agents includes QA Evaluator in addition to Code Writer, Builder/Tester, and Code Reviewer or Reviewer compatibility. QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance." condition: "qa_evaluation_mode == required" - on_fail: "Add QA Evaluator as a Review-phase role, or record qa_evaluation_mode=not_required with a concrete reason when QA is not applicable." + on_fail: "Add QA Evaluator as a Review-phase role, or record qa_evaluation_mode=not_required with a concrete reason when QA is not applicable. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work." - id: T9 check: "subagent_policy_state, subagent_execution_mode, and subagent_authorization_scope are initialized before any phase that may spawn subagents; development/code-work that needs workflow roles starts as authorization_required unless the current user prompt explicitly authorized subagents" @@ -80,7 +87,7 @@ gates: - id: T8 severity: guidance - check: "'>> Triage metadata: type={TASK_TYPE} | risk={RISK_TIER} | gates={count} | agents={count}' message was printed to user" + check: "'>> Triage metadata: type={TASK_TYPE} | risk={RISK_TIER} | intensity={controller_intensity} | gates={count} | agents={count}' message was printed to user" on_fail: "Print structured triage metadata when medium+ complexity or user visibility needs it" # ───────────────────────────────────────────── @@ -577,3 +584,8 @@ invariants: check: "Any pivot_restart_decision that changes approved scope, files, behavior, risk, verification, or acceptance criteria has reapproval_required=true and records approval before Build or Review continues" scope: all_phases on_fail: "Pause the workflow, mark reapproval_required=true, update the plan/task journal, and wait for user approval before continuing the changed path" + + - id: INV_CONTROLLER_INTENSITY_STANDARD + check: "controller_intensity == standard with harness_capable=false and qa_evaluation_mode=not_required does not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation" + scope: all_phases + on_fail: "Remove accidental strict/harness/QA requirements, or re-triage with explicit independent criteria before promoting controller_intensity to strict." diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/phases.md b/plugins/assistant-dev/skills/assistant-workflow/references/phases.md index 2521c94..89571a4 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/phases.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/phases.md @@ -18,6 +18,7 @@ For any task that needs clarification, create or update `{agent_state_dir}/task. - `Clarification question cap: N` where the cap is a maximum, never a quota - `Clarification admissibility: satisfied | needs_clarification | not_applicable` - `Unresolved clarification topics:` as a markdown list +- `Controller intensity: light | standard | strict` - `Search mode: none | lightweight | candidate_search` - `Candidate archive: {agent_state_dir}/candidate-search.md | inline | N/A` @@ -46,7 +47,7 @@ Print: `>> Direct fallback Explorer responsibility` (when `subagent_execution_mo If score ≤ 2: recommend fixing environment gaps before feature work. The agent isn't broken — the environment is. 5. Ask structured clarification Q&A with recommendations for any unresolved implementation-shaping field only when the question is admissible. Admissible means the answer affects correctness, scope, behavior, data, public contract, security, migration safety, or verification; cannot be discovered from code/context; has no safe default; and includes the risk if guessed. 6. Restate requirements in 1-3 sentences after clarification is resolved -7. Confirm or revise `Task type`, `Risk tier`, `Required gates`, `Required agents`, `subagent_policy_state`, and `subagent_execution_mode` from the saved Triage metadata after reading code/context. If discovery changes any of them, print `>> Re-triage required` and update the task journal before continuing. +7. Confirm or revise `Task type`, `Risk tier`, `Controller intensity`, `Required gates`, `Required agents`, `subagent_policy_state`, and `subagent_execution_mode` from the saved Triage metadata after reading code/context. If discovery changes any of them, print `>> Re-triage required` and update the task journal before continuing. 8. For `task_type: bugfix`, classify `debugging_mode`: if root cause is unknown or the reproduction path is unclear, load and follow `assistant-debugging` before planning a fix. Carry forward its reproduction status, hypotheses, root cause/confidence, and residual risks. If `assistant-debugging` is unavailable or policy-disallowed, do direct hypothesis-driven debugging with the same evidence requirements and record the fallback path. **Clarification format:** @@ -173,7 +174,7 @@ Print: `>> Direct fallback Architect responsibility` (when `subagent_execution_m **Entry rule:** Do not enter Plan while the saved clarification state is pending. Resume Plan only after Discover records `Clarification status: ready` and all implementation-shaping fields are explicit or explicitly defaulted. -Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. Treat `harness_capable` as false unless the task is long-running, trace/replay-ready multi-slice, high-risk harness, domain-scored, UI/visual/product/UX/docs/DX-facing, explicitly requested as harness/QA work, or already has an accepted Done Contract/Harness Recipe. For medium+ harness-capable work, load `references/harness-controller.md` plus `references/plan-harness-appendix.md`, then add compact Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger refs before task packets. Then read `references/plan-template.md` and use the correct tier: +Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. Select `controller_intensity`: light for small low-risk work, standard for ordinary medium+ non-harness work, and strict only for high/critical, strict hooks, harness/QA, or trace/replay criteria. Standard keeps `harness_capable=false` and `qa_evaluation_mode=not_required` defaults; it does not load Done Contract, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA artifacts. Treat `harness_capable` as false unless the task is long-running, trace/replay-ready multi-slice, high-risk harness, domain-scored, UI/visual/product/UX/docs/DX-facing, explicitly requested as harness/QA work, or already has an accepted Done Contract/Harness Recipe. For medium+ harness-capable work, load `references/harness-controller.md` plus `references/plan-harness-appendix.md`, then add compact Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger refs before task packets. Then read `references/plan-template.md` and use the correct tier: - Small: inline plan (goal, files, risks, tests). Do not wait for approval unless risk, ambiguity, user instruction, or a scope-changing decision makes approval necessary. - Medium: standard plan (drop Security/Operability unless the task touches auth, PII, payments, or infra) - Large/Mega: full plan (all sections including Security and Operability) @@ -187,7 +188,7 @@ Before writing the plan, load `references/artifact-first-output-contract.md` and 7. For medium+ tasks: consume the Decompose slice manifest directly in the plan and align each task packet to exactly one slice_id without rediscovering boundaries 8. Write ordered implementation steps with file paths 9. For large/mega: fill in Security and Operability sections. For medium: only if the task touches auth, PII, payments, or infra (promote to Full tier per plan-template.md) -10. Carry `Task type`, `Risk tier`, `Required gates`, `Required agents`, `subagent_policy_state`, `subagent_execution_mode`, `subagent_authorization_scope`, and `Search mode` into the plan. Each required gate must map to task packet criteria or explicit N/A rationale. +10. Carry `Task type`, `Risk tier`, `Controller intensity`, `Required gates`, `Required agents`, `subagent_policy_state`, `subagent_execution_mode`, `subagent_authorization_scope`, and `Search mode` into the plan. Each required gate must map to task packet criteria or explicit N/A rationale. 11. If `search_mode: candidate_search`, load `references/candidate-search.md`, create the goal tree from acceptance/slice criteria, score candidates, record the archive location, and treat post-approval pivots as plan deviations requiring re-approval when scope/files/behavior/risk change. 12. Load prompt packs only when applicable: - Refactors: `references/prompts/refactor-safety.md` @@ -255,7 +256,7 @@ Keep the runtime artifacts current as execution progresses: - **Code Reviewer** (`code-reviewer`, or `reviewer` compatibility through assistant-review delegated review): independent code/security/architecture/test coverage review evidence for Review - **QA Evaluator** (`qa-evaluator`): independent acceptance, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result evaluation when `qa_evaluation_mode=required` -Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Code Reviewer; `reviewer` remains compatibility routing for existing/legacy handoffs. Add QA Evaluator only when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. +Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Code Reviewer; `reviewer` remains compatibility routing for existing/legacy handoffs. Add QA Evaluator only when `qa_evaluation_mode=required`. QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. **Direct fallback mode (`subagent_execution_mode=direct_fallback`):** when authorization is denied, subagents are unavailable, or policy disallows spawning, the active agent may implement, test, and review directly, but must preserve the same phases, contracts, evidence requirements, and review/security gates. Do not pretend delegation happened; record `subagent_policy_state`, `subagent_execution_mode`, the explicit `Direct fallback reason: authorization_denied | subagents_unavailable | policy_disallowed`, and the direct-execution evidence. @@ -424,7 +425,7 @@ For medium+ tasks: full spec review plus autonomous code quality loop via `assis Print: `>> Stage 3: QA Evaluation — loading assistant-review references/qa-evaluation-loop.md` when QA is required. -Run QA Evaluation only when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. +Run QA Evaluation only when `qa_evaluation_mode=required`. QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. The QA Evaluator receives Done Contract when present, acceptance criteria, verification evidence, code review result, domain_context/rubric_refs when applicable, round 1-10, previously_failed_acceptance_items, and qa_filter_policy. It loads assistant-review `references/domain-rubrics.md` only when acceptance criteria, Done Contract, domain_context, or explicit rubric_refs require subjective/product/UX/docs/DX/UI/domain scoring. It returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression or score_entry, evidence, and open_questions when blocked. diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/plan-harness-appendix.md b/plugins/assistant-dev/skills/assistant-workflow/references/plan-harness-appendix.md index b280454..18bb72e 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/plan-harness-appendix.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/plan-harness-appendix.md @@ -3,8 +3,8 @@ Load this appendix only for medium+ harness-capable work: explicit harness work, long-running or trace/replay-ready multi-slice work, high-risk harness work, accepted Done Contract/Harness Recipe work, domain-scored work, or -UI/visual/product/UX/docs/DX-facing work. For ordinary medium+ source changes, -keep the base plan's harness fields as `N/A: [reason]`. +scoped UI/visual/product/UX/docs/DX acceptance. For ordinary medium+ source +changes, keep the base plan's harness fields as `N/A: [reason]`. ## Task Packet Harness Fields @@ -90,10 +90,13 @@ Each row is a typed producer/consumer record, not an ad hoc string reference. QA Evaluator remains separate from Code Reviewer. Use this routing only when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, accepted Done Contract, domain-scored work, or -UI/visual/product/UX/docs/DX-facing work. +scoped UI/visual/product/UX/docs/DX acceptance. + +QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. +QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. - qa_evaluation_mode: [required | optional | not_required] -- qa_trigger_reason: [explicit QA request, Done Contract, harness acceptance scope, domain-scored, UI/visual/product/UX/docs/DX, or N/A: reason] +- qa_trigger_reason: [explicit QA request, accepted Done Contract, harness acceptance scope, domain-scored, scoped UI/visual/product/UX/docs/DX acceptance, or N/A: reason] - code_review_result_ref: [Code Reviewer/Reviewer compatibility result ref] - qa_evaluation_result_ref: [QA Evaluator result ref, or N/A: reason] - domain_context_ref: [domain/rubric context ref, or N/A: reason] diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md index f73feea..2f6e821 100755 --- a/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/plan-template.md @@ -60,6 +60,7 @@ For Medium and Large/Mega plans, write implementation work as executable task pa - Evidence to record: - [test result, eval fixture, changed file, review note, or artifact proof] - Loop / Experiment Routing: + - controller_intensity: [light | standard | strict; standard keeps ordinary medium+ non-harness work out of harness/QA defaults] - workflow_experiment_ledger: [N/A unless explicit workflow experiment; otherwise ref] - loop_readiness_assessment: [N/A unless explicit repeat/optimization loop; otherwise ref with retry_or_empty_result_handling, tool_error_handling, low_confidence_escalation] - loop_harness_routing: [ordinary medium+ keeps harness_capable=false; loop artifacts alone do not require harness/QA artifacts] @@ -108,6 +109,7 @@ Covers the essentials without Security/Operability overhead. Fill this in during ## Triage result - Task type: [feature | bugfix | refactor | migration | rewrite | config | infra | security | docs | spike] - Risk tier: [low | moderate | high | critical] +- Controller intensity: [light | standard | strict] - Required gates: [common gates + task-category gate packs from references/triage-rubric.md] - Required agents: [roles/skills selected from size, task type, and risk] - Subagent policy state: [not_required | authorization_required | delegation_authorized | authorization_denied | subagents_unavailable | policy_disallowed] @@ -190,6 +192,7 @@ Required only for medium+ harness-capable work; otherwise use `N/A: [reason]`. Load `references/plan-harness-appendix.md` for full harness and QA schemas. - appendix_status: [required | N/A: reason] +- controller_intensity: [light | standard | strict + reason] - done_contract_ref: [section/ref, or N/A: reason] - harness_recipe_ref: [section/ref, or N/A: reason] - harness_run_state_ref: [section/ref, or N/A: reason] @@ -197,6 +200,7 @@ Load `references/plan-harness-appendix.md` for full harness and QA schemas. - replay_packet_ref: [section/ref, or N/A: reason] - artifact_reference_ledger_ref: [section/ref, or N/A: reason] - qa_evaluation_mode: [required | optional | not_required + reason] +- qa_trigger_reason: [QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work.] ## Slice manifest from Decompose diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md b/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md index 27cdd92..a18d9dc 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/subagent-dispatch.md @@ -55,7 +55,7 @@ Every phase has a declared role responsibility. Phases without a subagent role a | **BUILD** | Code Writer | All sizes | Implements code following the plan | | **BUILD** | Builder/Tester | All sizes | Builds, runs tests, returns concise results | | **REVIEW** | Code Reviewer, or Reviewer compatibility | All sizes | Independent code review via `assistant-review` skill | -| **REVIEW** | QA Evaluator | Medium+ harness/domain-scored/UI/visual/product/UX/docs/DX-facing or explicitly requested QA | Independent acceptance QA via `assistant-review` QA loop | +| **REVIEW** | QA Evaluator | `qa_evaluation_mode=required` only | Independent acceptance QA via `assistant-review` QA loop | | **DOCUMENT** | — (Orchestrator direct) | All sizes | Documentation generation is orchestrator's synthesis work | **Rule:** If `subagent_execution_mode=delegated` and a phase's subagent column shows a dispatch, you MUST dispatch that role. If `subagent_execution_mode=direct_fallback`, you MUST NOT spawn subagents; instead record which role responsibility was handled directly and what equivalent evidence proves it. @@ -73,7 +73,7 @@ Every phase has a declared role responsibility. Phases without a subagent role a - **Every task gets at minimum**: Code Writer → Builder/Tester → Code Reviewer responsibilities. `reviewer` remains valid compatibility routing for existing handoffs, but new dispatches should use `code-reviewer` for code defects, security, architecture, test coverage, and structural code issues. In delegated mode these are subagents; in direct fallback they are explicitly recorded role-equivalent steps. No code ships without build + test + review evidence. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts MUST infer required_agents with at least Code Writer, Builder/Tester, and Code Reviewer (or Reviewer compatibility); `not_applicable` is invalid for source-changing Build tasks. - **Strict evidence gate**: delegated mode is not complete until the task journal Agent Dispatch Log records Code Writer dispatch/result, Builder/Tester dispatch/result, and Code Reviewer dispatch/result evidence, or Reviewer dispatch/result evidence when compatibility routing is used. Medium+ delegated slice work also records per-slice dispatch evidence before each slice is marked verified. **Codex adds a stronger check:** delegated Codex evidence must correspond to real hook-written `SubagentStart`/`SubagentStop` lifecycle records in protected agent-owned workflow state; each dispatch/result entry must reference the matching lifecycle `agent_id`, and task-journal text or project-local `.codex/subagent-events.jsonl` diagnostics are insufficient because Codex can otherwise run phases inline and write claimed dispatch entries. Direct fallback is allowed only for explicit `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` reasons, and must record equivalent role, phase, verification, and review evidence; silent fallback fails the stop-review/phase gates. -- **QA evidence gate**: when QA is required, delegated mode is not complete until the task journal Agent Dispatch Log records QA Evaluator dispatch/result evidence after Builder/Tester and Code Reviewer evidence. Direct fallback must record fresh QA Evaluator direct evidence separately from Code Reviewer direct evidence. +- **QA evidence gate**: when QA is required, delegated mode is not complete until the task journal Agent Dispatch Log records QA Evaluator dispatch/result evidence after Builder/Tester and Code Reviewer evidence. Direct fallback must record fresh QA Evaluator direct evidence separately from Code Reviewer direct evidence. QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. - **Every medium+ task gets Architect decomposition responsibility**: In delegated mode the Architect proposes smallest iterable slice boundaries; in direct fallback the same criteria and evidence are recorded directly. - **Launch in parallel** when agents are independent (e.g., Code Mapper + Explorer on different modules) - **Code Mapper runs first** on medium+ tasks — its output feeds into Architect and Code Writer diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md index 016d8b7..8eeeb0f 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/task-journal-template.md @@ -23,6 +23,7 @@ Status: DISCOVERING | DECOMPOSING | PLANNING | BUILDING [step N/M] | REVIEWING | Triaged as: [small | medium | large | mega] Task type: [feature | bugfix | refactor | migration | rewrite | config | infra | security | docs | spike] Risk tier: [low | moderate | high | critical] +Controller intensity: [light | standard | strict] Clarification status: [ready | needs_clarification] Clarification defaults applied: [true | false] Clarification confidence: [low | medium | high] @@ -88,6 +89,7 @@ Plan approval: [yes/no + date] ## Harness Appendix Routing [required only for medium+ harness-capable work; otherwise `N/A: [reason]`; full schema in `references/task-journal-harness-appendix.md`] - Appendix: `references/task-journal-harness-appendix.md` +- Controller intensity: [light | standard | strict + reason] - Done Contract: [section/ref, or N/A: reason] - Harness Recipe: [section/ref, or N/A: reason] - Harness Run State: [section/ref, or N/A: reason] @@ -192,6 +194,7 @@ do not start the next slice until the current one is `VERIFIED` ### QA Evaluation - Mode: required | optional | not_required +- QA trigger reason: [QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work.] - QA Evaluator result: [final_verdict/result ref, or N/A: reason] - Selected domain rubrics: [families used, or N/A] - Domain quality scores: [compact scores, or N/A] diff --git a/plugins/assistant-dev/skills/assistant-workflow/references/triage-rubric.md b/plugins/assistant-dev/skills/assistant-workflow/references/triage-rubric.md index 9d1e179..dd0ad5c 100644 --- a/plugins/assistant-dev/skills/assistant-workflow/references/triage-rubric.md +++ b/plugins/assistant-dev/skills/assistant-workflow/references/triage-rubric.md @@ -9,6 +9,7 @@ Record these fields in the task journal for medium+ tasks and in the inline plan - `Task type`: `feature`, `bugfix`, `refactor`, `migration`, `rewrite`, `config`, `infra`, `security`, `docs`, or `spike` - `Risk tier`: `low`, `moderate`, `high`, or `critical` - `Triaged as`: `small`, `medium`, `large`, or `mega` +- `Controller intensity`: `light`, `standard`, or `strict` - `Required agents`: the roles required by size and risk - `Subagent policy state`: `not_required`, `authorization_required`, `delegation_authorized`, `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` - `Subagent execution mode`: `delegated`, `direct_fallback`, or `not_applicable` @@ -28,6 +29,16 @@ Record these fields in the task journal for medium+ tasks and in the inline plan Escalate size when risk exceeds file count. Auth, PII, payments, destructive data changes, public API changes, or behavior-preserving legacy migration are at least `large` unless discovery proves they are isolated and fully covered. +## Controller Intensity Rules + +| Intensity | Use when | +|---|---| +| `light` | Small low-risk localized work can use compact guidance and validation. | +| `standard` | Ordinary medium+ source-changing work, including delegated work, when `harness_capable=false` and `qa_evaluation_mode=not_required`. | +| `strict` | High/critical risk, `hook_profile == strict`, `harness_capable=true`, `qa_evaluation_mode=required`, or explicit trace/replay/harness/QA criteria. | + +Do not infer `strict` from `size=medium+` or delegation alone. `standard` keeps ordinary medium non-harness work out of Done Contract, Trace Ledger, Replay Packet, Artifact Reference Ledger, and QA defaults. + ## Candidate Scope Scan Before finalizing Triage, run a quick read-only scan to avoid classifying from the prompt alone. Keep it bounded: named files from the prompt, likely modules/directories, obvious symbols/search terms, nearby tests, docs, contracts, config, generated mirrors, and runtime surfaces that can change the risk tier. diff --git a/skills/assistant-review/SKILL.md b/skills/assistant-review/SKILL.md index 7765577..1fcc857 100644 --- a/skills/assistant-review/SKILL.md +++ b/skills/assistant-review/SKILL.md @@ -45,6 +45,8 @@ Find concrete defects, risks, regressions, and test gaps; fix them when in revie - In review-fix mode, must-fix and should-fix findings are addressed or explicitly deferred. - Validation runs after fixes, and a fresh review confirms the final state. - QA evaluation runs after code-review/build evidence when `qa_evaluation_mode=required`, returns score progression and a final acceptance verdict, and does not replace code-reviewer. +- QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. +- QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. - QA evaluation loads `references/domain-rubrics.md` only when `domain_context`, explicit `rubric_refs`, or subjective/UI/visual/product/UX/docs/DX/domain acceptance criteria require scoped domain-quality scoring. ## Constraints @@ -78,7 +80,7 @@ Use the smallest mode that answers the request; combine modes when reviewing imp - **Semantic contract review**: for skill/workflow/framework changes, check contract inheritance, method-template alignment, eval coverage, method-signature fidelity, and high-stakes recommendation guards before judging the change clean. - **Maintainability review**: apply SOLID/KISS/DRY/YAGNI/readability only when a concrete risk exists. - **Security handoff**: invoke `assistant-security` when the reviewed surface touches auth, permissions, secrets, input handling, persistence, shell commands, dependency/config changes, network calls, or external integrations. -- **QA evaluation**: after code-review/build evidence exists, dispatch QAEvaluator when `qa_evaluation_mode=required` for explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. Load `references/qa-evaluation-loop.md`. +- **QA evaluation**: after code-review/build evidence exists, dispatch QAEvaluator only when `qa_evaluation_mode=required`. Load `references/qa-evaluation-loop.md`. - **Domain rubric QA**: within QA evaluation, load `references/domain-rubrics.md` only when scoped by acceptance criteria, Done Contract, `domain_context`, or explicit `rubric_refs`. QAEvaluator selects rubric families and returns domain-quality scores; Code Reviewer still owns code defects, security, architecture, and test coverage. Finding format: @@ -223,7 +225,7 @@ while round <= 10: ## The QA Evaluation Loop -Run QA only when `qa_evaluation_mode=required` or the workflow Review phase requests QA evidence. Load `references/qa-evaluation-loop.md` before dispatching QAEvaluator; that reference owns the detailed algorithm, score progression, domain-rubric routing, pivot/restart behavior, and terminal round-10 cap. +Run QA only when `qa_evaluation_mode=required`; the workflow Review phase should set that mode only from the QA required positive triggers above. Load `references/qa-evaluation-loop.md` before dispatching QAEvaluator; that reference owns the detailed algorithm, score progression, domain-rubric routing, pivot/restart behavior, and terminal round-10 cap. At this level, keep only the routing boundary: QA runs after build/test evidence and Code Reviewer or Reviewer compatibility evidence exist, and it evaluates acceptance criteria, Done Contract evidence, verification evidence, scoped domain quality, score progression, and final readiness. QA does not replace code review or report general code defects unless they directly block acceptance. diff --git a/skills/assistant-review/contracts/input.yaml b/skills/assistant-review/contracts/input.yaml index 55137e2..69cdee8 100644 --- a/skills/assistant-review/contracts/input.yaml +++ b/skills/assistant-review/contracts/input.yaml @@ -112,13 +112,19 @@ fields: required: false enum_values: [not_required, optional, required] description: "Whether the independent QA evaluation loop should run after code-review/build evidence" - validation: "required only for explicit QA/acceptance evaluation, harness_capable == true with acceptance evaluation in scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing scope; not_required must have a reason when scoped acceptance QA is skipped" + validation: "QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work." default: not_required on_missing: infer infer_from: > - Use required when the user explicitly requests QA/acceptance evaluation, an accepted Done Contract - exists, harness_capable == true and acceptance evaluation is in scope, the task is domain-scored, - or it affects UI/visual/product/UX/docs/DX quality. + Use required only for QA required positive triggers: explicit QA/acceptance evaluation request, + accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped + UI/visual/product/UX/docs/DX acceptance. + Use required when the user explicitly requests QA/acceptance evaluation, an accepted Done + Contract exists, harness_capable == true and acceptance evaluation is in scope, the task is + domain-scored, or scoped UI/visual/product/UX/docs/DX acceptance applies. + QA non-triggers: template labels/placeholders, generic acceptance criteria labels, + optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ + code-review-only/source-changing work. Use optional when acceptance material exists but the orchestrator has not marked QA as required. Use not_required for ordinary medium+ code-review-only/source-changing work with no separate acceptance evidence need. diff --git a/skills/assistant-review/references/qa-evaluation-loop.md b/skills/assistant-review/references/qa-evaluation-loop.md index 5219919..d228b3c 100644 --- a/skills/assistant-review/references/qa-evaluation-loop.md +++ b/skills/assistant-review/references/qa-evaluation-loop.md @@ -4,14 +4,17 @@ Use this reference after build/test evidence and the Code Reviewer loop exist. Q ## When QA Runs +QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. +QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. + Run QA evaluation when any of these are true: - The user explicitly asks for QA or acceptance evaluation. - The task has an accepted Done Contract. - The task is harness-capable and acceptance evaluation is in scope. -- The task is subjective, domain-scored, or UI/visual/product/UX/docs/DX-facing. -- Workflow Review needs a final acceptance verdict beyond code/security/architecture/test-coverage review. +- The task has domain-scored scope or scoped UI/visual/product/UX/docs/DX acceptance. +- Workflow Review needs a final acceptance verdict because one of the positive triggers above is present. -Skip QA evaluation when the task has no explicit QA request, Done Contract, harness-capable acceptance scope, domain-scored criteria, or UI/visual/product/UX/docs/DX scope, and the workflow output records `qa_evaluation_mode=not_required` with a reason. +Skip QA evaluation when the task has no explicit QA request, Done Contract, harness-capable acceptance scope, domain-scored criteria, or UI/visual/product/UX/docs/DX scope, and the workflow output records `qa_evaluation_mode=not_required` with a reason. Template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work do not force QA. ## Inputs diff --git a/skills/assistant-workflow/SKILL.md b/skills/assistant-workflow/SKILL.md index 64e16c9..3aadfd0 100644 --- a/skills/assistant-workflow/SKILL.md +++ b/skills/assistant-workflow/SKILL.md @@ -26,7 +26,10 @@ This skill is intentionally agent-agnostic: it must work in restricted company e - Medium+ harness-capable work has an accepted Done Contract and Harness Recipe before Build. - Trace/replay-ready harness work maintains Harness Run State, Trace Ledger, and Replay Packet artifacts. - Ordinary medium+ workflow tasks default to `harness_capable=false` and do not inherit Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless explicit harness/QA criteria apply. -- Harness-capable acceptance, Done Contract, domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested QA work records independent QA Evaluator evidence after build/test and code-review evidence. +- `controller_intensity=standard` is the ordinary medium+ non-harness default; `strict` needs independent strict/harness/QA criteria. +- QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. +- QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. +- Required QA records independent QA Evaluator evidence after build/test and code-review evidence. - Behavior changes have tests or explicit validation attached to the implementation step they protect. - Final output reports changed files, verification evidence, residual risks, and next steps. - Candidate Search is used for explicit alternatives, open-ended architecture/design, optimization, high uncertainty, repeated failed attempts, unclear/flaky bugs, or reviewer-requested pivots — not as default ceremony. @@ -177,7 +180,7 @@ Load `references/triage-rubric.md`. Perform a quick read-only candidate scope sc [Design] = include if task has UI work, skip for backend-only. Print: `>> Triaged as: [SIZE] — phases: [list]` -Print: `>> Triage metadata: type=[TASK_TYPE] | risk=[RISK_TIER] | gates=[count] | agents=[count] | search=[search_mode] | scope_confidence=[low|medium|high]` +Print: `>> Triage metadata: type=[TASK_TYPE] | risk=[RISK_TIER] | intensity=[controller_intensity] | gates=[count] | agents=[count] | search=[search_mode] | scope_confidence=[low|medium|high]` If scope exceeds initial triage during any phase, stop and re-triage. @@ -212,7 +215,7 @@ Load `references/phases.md` and execute the phase matching your current stage. U For subagent roles and dispatch rules, load `references/subagent-dispatch.md` and resolve `subagent_policy_state`, `subagent_execution_mode`, and `subagent_authorization_scope` before any subagent spawn. Assistant Framework policy requires explicit user authorization before spawning subagents for development/code-work roles unless the current user prompt already explicitly authorizes them for this task. Ask once for the needed delegation scope and wait before continuing phases that require subagents. A sufficient prompt is: `This workflow expects Code Mapper, Architect, Code Writer, Builder/Tester, Code Reviewer, and QA Evaluator when QA is required for [scope]. May I use subagents for this task?` After authorization, use `delegated` mode and spawn the configured role agents. Use `direct_fallback` only when authorization is denied, policy disallows spawning, or a real spawn attempt fails because subagents/custom agents are unavailable; do not infer unavailability merely because no visible tool is named `Task`, `delegate`, or `subagent`. For BES-style option exploration, load `references/candidate-search.md` only when `search_mode: candidate_search` is selected. For mega tasks and anti-patterns, load `references/mega-and-patterns.md`. -Load `references/harness-controller.md` only when medium+ work is harness-capable: long-running, trace/replay-ready multi-slice, high-risk harness, subjective/domain-scored, UI/visual/product/UX/docs/DX-facing, or explicitly requested as harness work. +Load `references/harness-controller.md` only when medium+ work is harness-capable: long-running, trace/replay-ready multi-slice, high-risk harness, subjective/domain-scored, scoped UI/visual/product/UX/docs/DX acceptance, or explicitly requested as harness work. ## Planning Checklist @@ -245,7 +248,7 @@ Before final output for medium+ or high-risk work: - Review the diff against the acceptance criteria. - For bugfixes, verify the review material includes reproduction/root-cause evidence from `assistant-debugging` or a clear reason it was not applicable. - Use `assistant-review` for code quality/spec review when the change is non-trivial. -- Use QA Evaluator through `assistant-review` when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work after build/test and code-review evidence. +- Use QA Evaluator through `assistant-review` when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance after build/test and code-review evidence. - Use `assistant-security` when touching auth, user input, secrets, persistence, network calls, shell commands, dependency/config changes, or external integrations. Review findings must cite evidence and concrete risk. Avoid generic style feedback unless it affects correctness, security, maintainability, or test reliability. diff --git a/skills/assistant-workflow/contracts/input.yaml b/skills/assistant-workflow/contracts/input.yaml index 9568c6d..0762017 100644 --- a/skills/assistant-workflow/contracts/input.yaml +++ b/skills/assistant-workflow/contracts/input.yaml @@ -149,18 +149,40 @@ fields: or Harness Recipe. Do not infer true from size=medium+, delegation alone, or ordinary source-changing workflow tasks. + - name: controller_intensity + type: enum + required: false + enum_values: [light, standard, strict] + description: "Completion/gate intensity selected during Triage; maps onto existing completion_tiers and gate_tiers." + validation: "light uses small/low-risk guidance, standard uses ordinary medium+ completion/review without harness/QA defaults, and strict uses strict gates or harness/QA artifacts only when their existing conditions apply." + default: standard + on_missing: infer + infer_from: > + Use light for small low-risk localized work. Use standard for ordinary medium+ + source-changing work, including delegated work, when harness_capable == false + and qa_evaluation_mode == not_required. Use strict only for high/critical risk, + hook_profile == strict, harness_capable == true, qa_evaluation_mode == required, + explicit harness/QA acceptance work, or trace/replay-ready work. Do not infer + strict from size=medium+ or delegation alone. + - name: qa_evaluation_mode type: enum required: false enum_values: [not_required, optional, required] description: "Whether Review phase should run the independent QA Evaluator acceptance loop after build/test and code-review evidence" - validation: "required only for explicit QA/acceptance evaluation, harness_capable == true with acceptance evaluation in scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing scope; not_required is the default for ordinary medium+ source-changing work." + validation: "QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work." default: not_required on_missing: infer infer_from: > - Use required when the user explicitly requests QA/acceptance evaluation, the task has an accepted - Done Contract, harness_capable == true and acceptance evaluation is in scope, the task is - domain-scored, or it affects UI/visual/product/UX/docs/DX quality. + Use required only for QA required positive triggers: explicit QA/acceptance evaluation request, + accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped + UI/visual/product/UX/docs/DX acceptance. + Use required when the user explicitly requests QA/acceptance evaluation, the task has an + accepted Done Contract, harness_capable == true and acceptance evaluation is in scope, the task + is domain-scored, or scoped UI/visual/product/UX/docs/DX acceptance applies. + QA non-triggers: template labels/placeholders, generic acceptance criteria labels, + optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ + code-review-only/source-changing work. Use optional when acceptance material exists but QA is not required. Use not_required for ordinary medium+ code-review-only/source-changing work with no separate acceptance evidence need. diff --git a/skills/assistant-workflow/contracts/output.yaml b/skills/assistant-workflow/contracts/output.yaml index e033d33..dcce215 100644 --- a/skills/assistant-workflow/contracts/output.yaml +++ b/skills/assistant-workflow/contracts/output.yaml @@ -10,18 +10,25 @@ skill: assistant-workflow # Completion evidence is tiered so small/low-risk work does not inherit the # full medium+ harness. Artifact definitions below remain the source of truth; # these tiers define which artifacts are required by default for each class. +# controller_intensity maps onto these tiers and gate_tiers: light -> small, +# standard -> ordinary medium+ non-harness, strict -> large/critical or +# explicit strict/harness/QA criteria. completion_tiers: small: + controller_intensity: light required_artifacts: [changed_files, validation_results, manual_test_steps, user_approval] conditional_artifacts: [triage_result, subagent_evidence, test_results, debugging_result, artifact_contract, workflow_experiment_ledger, loop_readiness_assessment] review_gate: "self-review or explicit validation summary unless risk_tier in [high, critical]" checkpoint_policy: "inline plan/check summary is enough; phase_checkpoints are strict-profile only" medium: + controller_intensity: standard + intensity_policy: "ordinary medium+ non-harness work keeps harness_capable=false and qa_evaluation_mode=not_required by default; standard does not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation." required_artifacts: [triage_result, changed_files, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, plan_document, user_approval] conditional_artifacts: [test_results, task_journal, context_map, slice_manifest, single_slice_rationale, slice_verification_summary, metrics_entry, candidate_search_result, workflow_experiment_ledger, loop_readiness_assessment, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result] review_gate: "spec review plus code quality review, with independent QA evaluation when required and evidence matched to the approved plan" checkpoint_policy: "task journal or carried-forward state records phase transitions" large_critical: + controller_intensity: strict required_artifacts: [triage_result, phase_checkpoints, changed_files, test_results, validation_results, subagent_evidence, spec_review_result, review_result, learning_controller, manual_test_steps, task_journal, context_map, context_budget_note, slice_manifest, slice_verification_summary, plan_document, user_approval] conditional_artifacts: [debugging_result, artifact_contract, pattern_retrieval_note, decomposition_plan_review, single_slice_rationale, metrics_entry, candidate_search_result, workflow_experiment_ledger, loop_readiness_assessment, done_contract, harness_recipe, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision, artifact_reference_ledger, qa_evaluation_result, verified_skill_distillation] review_gate: "full strict evidence, security/operability review when applicable, QA evaluation when required, and all slice criteria verified" @@ -32,8 +39,8 @@ artifacts: type: object required: conditional condition: "size in [medium, large, mega] or hook_profile == strict or risk_tier in [high, critical]" - validation: "Contains task_type, risk_tier, size, required_gates, required_agents, subagent policy state, execution mode, authorization scope, search_mode, and candidate_scope_scan selected during Triage" - on_fail: "Return to TRIAGE and populate task_type, risk_tier, size, required_gates, required_agents or fallback roles, subagent policy state, execution mode, authorization scope, search_mode, and candidate_scope_scan" + validation: "Contains task_type, risk_tier, size, controller_intensity, required_gates, required_agents, subagent policy state, execution mode, authorization scope, search_mode, and candidate_scope_scan selected during Triage" + on_fail: "Return to TRIAGE and populate task_type, risk_tier, size, controller_intensity, required_gates, required_agents or fallback roles, subagent policy state, execution mode, authorization scope, search_mode, and candidate_scope_scan" object_fields: - name: task_type type: enum @@ -47,6 +54,11 @@ artifacts: type: enum required: true enum_values: [small, medium, large, mega] + - name: controller_intensity + type: enum + required: true + enum_values: [light, standard, strict] + validation: "Maps to completion_tiers and gate_tiers. standard is used for ordinary medium+ non-harness work and does not require Done Contract, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation by default; strict requires independent strict, harness, or QA criteria. Do not infer strict from size=medium+ or delegation alone." - name: required_gates type: string[] required: true @@ -1436,7 +1448,7 @@ artifacts: # Completeness check: # Before printing --- WORKFLOW COMPLETE ---, verify: -# 1. All required artifacts for the task's completion_tier are present +# 1. All required artifacts for the task's completion_tier are present and controller_intensity matches the selected tier # 2. test_results.failed == 0 when tests ran; validation_results contains at least one result=passed entry for completed implementation workflows. Skipped/not_applicable entries do not satisfy completion evidence; docs-only/no-op/discovery exceptions must record the reason. # 3. When size is medium/large/mega, risk_tier is high/critical, or hook_profile == strict: spec_review_result.status == PASS and required_fixes are empty or resolved # 4. When size is medium/large/mega, risk_tier is high/critical, or hook_profile == strict: review_result.quality_review_status is not missing @@ -1445,7 +1457,7 @@ artifacts: # 7. candidate_search_result exists when search_mode == candidate_search, includes search_exit_summary, and any plan_deviation is approved before Build # 8. workflow_experiment_ledger exists only for explicit workflow experiments; loop_readiness_assessment exists only before explicit repeat/optimization loops and includes retry_or_empty_result_handling, tool_error_handling, and low_confidence_escalation # 9. Loop artifacts alone do not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation unless harness_capable or QA criteria independently apply -# 10. For medium+ harness-capable work, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision when triggered, and artifact_reference_ledger exist and are current before completion or handoff +# 10. For medium+ harness-capable work, harness_run_state, trace_ledger, replay_packet, pivot_restart_decision when triggered, and artifact_reference_ledger exist and are current before completion or handoff. controller_intensity=standard with harness_capable=false does not require these artifacts. # 11. user_approval is confirmed, confirmed_with_issues, or not_required_small for eligible small tasks # If any check fails, do NOT print --- WORKFLOW COMPLETE ---. State what is missing. - name: verified_skill_distillation diff --git a/skills/assistant-workflow/contracts/phase-gates.yaml b/skills/assistant-workflow/contracts/phase-gates.yaml index 4bdcb62..5b4ceaf 100644 --- a/skills/assistant-workflow/contracts/phase-gates.yaml +++ b/skills/assistant-workflow/contracts/phase-gates.yaml @@ -23,6 +23,9 @@ gate_tiers: blocker: "Prevents phase transition; missing evidence would make the workflow unsafe or unverifiable." guidance: "Preferred discipline; record or follow when useful, but do not ask ritual questions or block low-risk progress solely for this item." strict_only: "Enforced only when hook_profile == strict or project policy explicitly requests full harness enforcement." +# controller_intensity maps onto these gate_tiers: light keeps low-risk guidance +# compact, standard uses normal medium blockers/review without harness/QA +# defaults, and strict may enforce strict_only gates when independent criteria apply. gates: @@ -50,6 +53,10 @@ gates: check: "risk_tier is set to one of: low, moderate, high, critical" on_fail: "Assess risk tier using impact, uncertainty, reversibility, security/data exposure, and behavior parity criteria" + - id: T_CONTROLLER_INTENSITY + check: "controller_intensity is set to one of: light, standard, strict; ordinary medium+ non-harness work uses standard; strict is selected only for high/critical risk, hook_profile == strict, harness_capable == true, qa_evaluation_mode == required, or explicit trace/replay/harness/QA criteria" + on_fail: "Set controller_intensity from existing completion_tiers/gate_tiers. Do not infer strict from size=medium+ or delegation alone; use standard when harness_capable=false and qa_evaluation_mode=not_required." + - id: T6 check: "required_gates includes common gates plus every applicable task-category gate pack" on_fail: "Load references/triage-rubric.md and add the required gate packs for the task type before continuing" @@ -59,9 +66,9 @@ gates: on_fail: "Select required workflow agents when subagent_execution_mode=delegated, or record direct fallback roles and responsibilities from references/subagent-dispatch.md and risk-specific gate packs. not_applicable is invalid for source-changing Build tasks." - id: T_QA_EVALUATOR - check: "When qa_evaluation_mode == required: required_agents includes QA Evaluator in addition to Code Writer, Builder/Tester, and Code Reviewer or Reviewer compatibility" + check: "When qa_evaluation_mode == required: required_agents includes QA Evaluator in addition to Code Writer, Builder/Tester, and Code Reviewer or Reviewer compatibility. QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance." condition: "qa_evaluation_mode == required" - on_fail: "Add QA Evaluator as a Review-phase role, or record qa_evaluation_mode=not_required with a concrete reason when QA is not applicable." + on_fail: "Add QA Evaluator as a Review-phase role, or record qa_evaluation_mode=not_required with a concrete reason when QA is not applicable. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work." - id: T9 check: "subagent_policy_state, subagent_execution_mode, and subagent_authorization_scope are initialized before any phase that may spawn subagents; development/code-work that needs workflow roles starts as authorization_required unless the current user prompt explicitly authorized subagents" @@ -80,7 +87,7 @@ gates: - id: T8 severity: guidance - check: "'>> Triage metadata: type={TASK_TYPE} | risk={RISK_TIER} | gates={count} | agents={count}' message was printed to user" + check: "'>> Triage metadata: type={TASK_TYPE} | risk={RISK_TIER} | intensity={controller_intensity} | gates={count} | agents={count}' message was printed to user" on_fail: "Print structured triage metadata when medium+ complexity or user visibility needs it" # ───────────────────────────────────────────── @@ -577,3 +584,8 @@ invariants: check: "Any pivot_restart_decision that changes approved scope, files, behavior, risk, verification, or acceptance criteria has reapproval_required=true and records approval before Build or Review continues" scope: all_phases on_fail: "Pause the workflow, mark reapproval_required=true, update the plan/task journal, and wait for user approval before continuing the changed path" + + - id: INV_CONTROLLER_INTENSITY_STANDARD + check: "controller_intensity == standard with harness_capable=false and qa_evaluation_mode=not_required does not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation" + scope: all_phases + on_fail: "Remove accidental strict/harness/QA requirements, or re-triage with explicit independent criteria before promoting controller_intensity to strict." diff --git a/skills/assistant-workflow/references/phases.md b/skills/assistant-workflow/references/phases.md index 2521c94..89571a4 100644 --- a/skills/assistant-workflow/references/phases.md +++ b/skills/assistant-workflow/references/phases.md @@ -18,6 +18,7 @@ For any task that needs clarification, create or update `{agent_state_dir}/task. - `Clarification question cap: N` where the cap is a maximum, never a quota - `Clarification admissibility: satisfied | needs_clarification | not_applicable` - `Unresolved clarification topics:` as a markdown list +- `Controller intensity: light | standard | strict` - `Search mode: none | lightweight | candidate_search` - `Candidate archive: {agent_state_dir}/candidate-search.md | inline | N/A` @@ -46,7 +47,7 @@ Print: `>> Direct fallback Explorer responsibility` (when `subagent_execution_mo If score ≤ 2: recommend fixing environment gaps before feature work. The agent isn't broken — the environment is. 5. Ask structured clarification Q&A with recommendations for any unresolved implementation-shaping field only when the question is admissible. Admissible means the answer affects correctness, scope, behavior, data, public contract, security, migration safety, or verification; cannot be discovered from code/context; has no safe default; and includes the risk if guessed. 6. Restate requirements in 1-3 sentences after clarification is resolved -7. Confirm or revise `Task type`, `Risk tier`, `Required gates`, `Required agents`, `subagent_policy_state`, and `subagent_execution_mode` from the saved Triage metadata after reading code/context. If discovery changes any of them, print `>> Re-triage required` and update the task journal before continuing. +7. Confirm or revise `Task type`, `Risk tier`, `Controller intensity`, `Required gates`, `Required agents`, `subagent_policy_state`, and `subagent_execution_mode` from the saved Triage metadata after reading code/context. If discovery changes any of them, print `>> Re-triage required` and update the task journal before continuing. 8. For `task_type: bugfix`, classify `debugging_mode`: if root cause is unknown or the reproduction path is unclear, load and follow `assistant-debugging` before planning a fix. Carry forward its reproduction status, hypotheses, root cause/confidence, and residual risks. If `assistant-debugging` is unavailable or policy-disallowed, do direct hypothesis-driven debugging with the same evidence requirements and record the fallback path. **Clarification format:** @@ -173,7 +174,7 @@ Print: `>> Direct fallback Architect responsibility` (when `subagent_execution_m **Entry rule:** Do not enter Plan while the saved clarification state is pending. Resume Plan only after Discover records `Clarification status: ready` and all implementation-shaping fields are explicit or explicitly defaulted. -Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. Treat `harness_capable` as false unless the task is long-running, trace/replay-ready multi-slice, high-risk harness, domain-scored, UI/visual/product/UX/docs/DX-facing, explicitly requested as harness/QA work, or already has an accepted Done Contract/Harness Recipe. For medium+ harness-capable work, load `references/harness-controller.md` plus `references/plan-harness-appendix.md`, then add compact Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger refs before task packets. Then read `references/plan-template.md` and use the correct tier: +Before writing the plan, load `references/artifact-first-output-contract.md` and define the Artifact Contract: artifact type, required files/deliverables, output format/schema, acceptance criteria, verification command or method, expected success signal, owner/consumer, and non-goals. Select `controller_intensity`: light for small low-risk work, standard for ordinary medium+ non-harness work, and strict only for high/critical, strict hooks, harness/QA, or trace/replay criteria. Standard keeps `harness_capable=false` and `qa_evaluation_mode=not_required` defaults; it does not load Done Contract, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA artifacts. Treat `harness_capable` as false unless the task is long-running, trace/replay-ready multi-slice, high-risk harness, domain-scored, UI/visual/product/UX/docs/DX-facing, explicitly requested as harness/QA work, or already has an accepted Done Contract/Harness Recipe. For medium+ harness-capable work, load `references/harness-controller.md` plus `references/plan-harness-appendix.md`, then add compact Done Contract, Harness Recipe, Harness Run State, Trace Ledger, Replay Packet, and Artifact Reference Ledger refs before task packets. Then read `references/plan-template.md` and use the correct tier: - Small: inline plan (goal, files, risks, tests). Do not wait for approval unless risk, ambiguity, user instruction, or a scope-changing decision makes approval necessary. - Medium: standard plan (drop Security/Operability unless the task touches auth, PII, payments, or infra) - Large/Mega: full plan (all sections including Security and Operability) @@ -187,7 +188,7 @@ Before writing the plan, load `references/artifact-first-output-contract.md` and 7. For medium+ tasks: consume the Decompose slice manifest directly in the plan and align each task packet to exactly one slice_id without rediscovering boundaries 8. Write ordered implementation steps with file paths 9. For large/mega: fill in Security and Operability sections. For medium: only if the task touches auth, PII, payments, or infra (promote to Full tier per plan-template.md) -10. Carry `Task type`, `Risk tier`, `Required gates`, `Required agents`, `subagent_policy_state`, `subagent_execution_mode`, `subagent_authorization_scope`, and `Search mode` into the plan. Each required gate must map to task packet criteria or explicit N/A rationale. +10. Carry `Task type`, `Risk tier`, `Controller intensity`, `Required gates`, `Required agents`, `subagent_policy_state`, `subagent_execution_mode`, `subagent_authorization_scope`, and `Search mode` into the plan. Each required gate must map to task packet criteria or explicit N/A rationale. 11. If `search_mode: candidate_search`, load `references/candidate-search.md`, create the goal tree from acceptance/slice criteria, score candidates, record the archive location, and treat post-approval pivots as plan deviations requiring re-approval when scope/files/behavior/risk change. 12. Load prompt packs only when applicable: - Refactors: `references/prompts/refactor-safety.md` @@ -255,7 +256,7 @@ Keep the runtime artifacts current as execution progresses: - **Code Reviewer** (`code-reviewer`, or `reviewer` compatibility through assistant-review delegated review): independent code/security/architecture/test coverage review evidence for Review - **QA Evaluator** (`qa-evaluator`): independent acceptance, Done Contract, verification evidence, UI/visual/product/UX/docs/DX/domain quality, score progression, and final result evaluation when `qa_evaluation_mode=required` -Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Code Reviewer; `reviewer` remains compatibility routing for existing/legacy handoffs. Add QA Evaluator only when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. +Any source-changing Build task must infer `required_agents` with at least Code Writer, Builder/Tester, and Code Reviewer; `reviewer` remains compatibility routing for existing/legacy handoffs. Add QA Evaluator only when `qa_evaluation_mode=required`. QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. `not_applicable` is invalid once project source, tests, docs, config, hooks, contracts, or generated project artifacts will change. For medium+ delegated work, also record per-slice dispatch evidence before a slice is marked `VERIFIED`. **Direct fallback mode (`subagent_execution_mode=direct_fallback`):** when authorization is denied, subagents are unavailable, or policy disallows spawning, the active agent may implement, test, and review directly, but must preserve the same phases, contracts, evidence requirements, and review/security gates. Do not pretend delegation happened; record `subagent_policy_state`, `subagent_execution_mode`, the explicit `Direct fallback reason: authorization_denied | subagents_unavailable | policy_disallowed`, and the direct-execution evidence. @@ -424,7 +425,7 @@ For medium+ tasks: full spec review plus autonomous code quality loop via `assis Print: `>> Stage 3: QA Evaluation — loading assistant-review references/qa-evaluation-loop.md` when QA is required. -Run QA Evaluation only when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, Done Contract presence, domain-scored work, or UI/visual/product/UX/docs/DX-facing work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. +Run QA Evaluation only when `qa_evaluation_mode=required`. QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. QA runs after build/test evidence and Code Reviewer or Reviewer compatibility result are available. Dispatch `qa-evaluator` in delegated mode, or record direct-fallback QA evidence when delegation is denied, unavailable after a real spawn failure, or policy-disallowed. The QA Evaluator receives Done Contract when present, acceptance criteria, verification evidence, code review result, domain_context/rubric_refs when applicable, round 1-10, previously_failed_acceptance_items, and qa_filter_policy. It loads assistant-review `references/domain-rubrics.md` only when acceptance criteria, Done Contract, domain_context, or explicit rubric_refs require subjective/product/UX/docs/DX/UI/domain scoring. It returns final_verdict/result, acceptance_findings, qa_scorecard, selected_domain_rubrics/domain_quality_scores when scoped, score_progression or score_entry, evidence, and open_questions when blocked. diff --git a/skills/assistant-workflow/references/plan-harness-appendix.md b/skills/assistant-workflow/references/plan-harness-appendix.md index b280454..18bb72e 100644 --- a/skills/assistant-workflow/references/plan-harness-appendix.md +++ b/skills/assistant-workflow/references/plan-harness-appendix.md @@ -3,8 +3,8 @@ Load this appendix only for medium+ harness-capable work: explicit harness work, long-running or trace/replay-ready multi-slice work, high-risk harness work, accepted Done Contract/Harness Recipe work, domain-scored work, or -UI/visual/product/UX/docs/DX-facing work. For ordinary medium+ source changes, -keep the base plan's harness fields as `N/A: [reason]`. +scoped UI/visual/product/UX/docs/DX acceptance. For ordinary medium+ source +changes, keep the base plan's harness fields as `N/A: [reason]`. ## Task Packet Harness Fields @@ -90,10 +90,13 @@ Each row is a typed producer/consumer record, not an ad hoc string reference. QA Evaluator remains separate from Code Reviewer. Use this routing only when `qa_evaluation_mode=required`: explicit QA/acceptance evaluation, harness-capable acceptance scope, accepted Done Contract, domain-scored work, or -UI/visual/product/UX/docs/DX-facing work. +scoped UI/visual/product/UX/docs/DX acceptance. + +QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. +QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. - qa_evaluation_mode: [required | optional | not_required] -- qa_trigger_reason: [explicit QA request, Done Contract, harness acceptance scope, domain-scored, UI/visual/product/UX/docs/DX, or N/A: reason] +- qa_trigger_reason: [explicit QA request, accepted Done Contract, harness acceptance scope, domain-scored, scoped UI/visual/product/UX/docs/DX acceptance, or N/A: reason] - code_review_result_ref: [Code Reviewer/Reviewer compatibility result ref] - qa_evaluation_result_ref: [QA Evaluator result ref, or N/A: reason] - domain_context_ref: [domain/rubric context ref, or N/A: reason] diff --git a/skills/assistant-workflow/references/plan-template.md b/skills/assistant-workflow/references/plan-template.md index f73feea..2f6e821 100755 --- a/skills/assistant-workflow/references/plan-template.md +++ b/skills/assistant-workflow/references/plan-template.md @@ -60,6 +60,7 @@ For Medium and Large/Mega plans, write implementation work as executable task pa - Evidence to record: - [test result, eval fixture, changed file, review note, or artifact proof] - Loop / Experiment Routing: + - controller_intensity: [light | standard | strict; standard keeps ordinary medium+ non-harness work out of harness/QA defaults] - workflow_experiment_ledger: [N/A unless explicit workflow experiment; otherwise ref] - loop_readiness_assessment: [N/A unless explicit repeat/optimization loop; otherwise ref with retry_or_empty_result_handling, tool_error_handling, low_confidence_escalation] - loop_harness_routing: [ordinary medium+ keeps harness_capable=false; loop artifacts alone do not require harness/QA artifacts] @@ -108,6 +109,7 @@ Covers the essentials without Security/Operability overhead. Fill this in during ## Triage result - Task type: [feature | bugfix | refactor | migration | rewrite | config | infra | security | docs | spike] - Risk tier: [low | moderate | high | critical] +- Controller intensity: [light | standard | strict] - Required gates: [common gates + task-category gate packs from references/triage-rubric.md] - Required agents: [roles/skills selected from size, task type, and risk] - Subagent policy state: [not_required | authorization_required | delegation_authorized | authorization_denied | subagents_unavailable | policy_disallowed] @@ -190,6 +192,7 @@ Required only for medium+ harness-capable work; otherwise use `N/A: [reason]`. Load `references/plan-harness-appendix.md` for full harness and QA schemas. - appendix_status: [required | N/A: reason] +- controller_intensity: [light | standard | strict + reason] - done_contract_ref: [section/ref, or N/A: reason] - harness_recipe_ref: [section/ref, or N/A: reason] - harness_run_state_ref: [section/ref, or N/A: reason] @@ -197,6 +200,7 @@ Load `references/plan-harness-appendix.md` for full harness and QA schemas. - replay_packet_ref: [section/ref, or N/A: reason] - artifact_reference_ledger_ref: [section/ref, or N/A: reason] - qa_evaluation_mode: [required | optional | not_required + reason] +- qa_trigger_reason: [QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work.] ## Slice manifest from Decompose diff --git a/skills/assistant-workflow/references/subagent-dispatch.md b/skills/assistant-workflow/references/subagent-dispatch.md index 27cdd92..a18d9dc 100644 --- a/skills/assistant-workflow/references/subagent-dispatch.md +++ b/skills/assistant-workflow/references/subagent-dispatch.md @@ -55,7 +55,7 @@ Every phase has a declared role responsibility. Phases without a subagent role a | **BUILD** | Code Writer | All sizes | Implements code following the plan | | **BUILD** | Builder/Tester | All sizes | Builds, runs tests, returns concise results | | **REVIEW** | Code Reviewer, or Reviewer compatibility | All sizes | Independent code review via `assistant-review` skill | -| **REVIEW** | QA Evaluator | Medium+ harness/domain-scored/UI/visual/product/UX/docs/DX-facing or explicitly requested QA | Independent acceptance QA via `assistant-review` QA loop | +| **REVIEW** | QA Evaluator | `qa_evaluation_mode=required` only | Independent acceptance QA via `assistant-review` QA loop | | **DOCUMENT** | — (Orchestrator direct) | All sizes | Documentation generation is orchestrator's synthesis work | **Rule:** If `subagent_execution_mode=delegated` and a phase's subagent column shows a dispatch, you MUST dispatch that role. If `subagent_execution_mode=direct_fallback`, you MUST NOT spawn subagents; instead record which role responsibility was handled directly and what equivalent evidence proves it. @@ -73,7 +73,7 @@ Every phase has a declared role responsibility. Phases without a subagent role a - **Every task gets at minimum**: Code Writer → Builder/Tester → Code Reviewer responsibilities. `reviewer` remains valid compatibility routing for existing handoffs, but new dispatches should use `code-reviewer` for code defects, security, architecture, test coverage, and structural code issues. In delegated mode these are subagents; in direct fallback they are explicitly recorded role-equivalent steps. No code ships without build + test + review evidence. Any development/code-work task that will change project source, tests, docs, config, hooks, contracts, or generated project artifacts MUST infer required_agents with at least Code Writer, Builder/Tester, and Code Reviewer (or Reviewer compatibility); `not_applicable` is invalid for source-changing Build tasks. - **Strict evidence gate**: delegated mode is not complete until the task journal Agent Dispatch Log records Code Writer dispatch/result, Builder/Tester dispatch/result, and Code Reviewer dispatch/result evidence, or Reviewer dispatch/result evidence when compatibility routing is used. Medium+ delegated slice work also records per-slice dispatch evidence before each slice is marked verified. **Codex adds a stronger check:** delegated Codex evidence must correspond to real hook-written `SubagentStart`/`SubagentStop` lifecycle records in protected agent-owned workflow state; each dispatch/result entry must reference the matching lifecycle `agent_id`, and task-journal text or project-local `.codex/subagent-events.jsonl` diagnostics are insufficient because Codex can otherwise run phases inline and write claimed dispatch entries. Direct fallback is allowed only for explicit `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` reasons, and must record equivalent role, phase, verification, and review evidence; silent fallback fails the stop-review/phase gates. -- **QA evidence gate**: when QA is required, delegated mode is not complete until the task journal Agent Dispatch Log records QA Evaluator dispatch/result evidence after Builder/Tester and Code Reviewer evidence. Direct fallback must record fresh QA Evaluator direct evidence separately from Code Reviewer direct evidence. +- **QA evidence gate**: when QA is required, delegated mode is not complete until the task journal Agent Dispatch Log records QA Evaluator dispatch/result evidence after Builder/Tester and Code Reviewer evidence. Direct fallback must record fresh QA Evaluator direct evidence separately from Code Reviewer direct evidence. QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work. - **Every medium+ task gets Architect decomposition responsibility**: In delegated mode the Architect proposes smallest iterable slice boundaries; in direct fallback the same criteria and evidence are recorded directly. - **Launch in parallel** when agents are independent (e.g., Code Mapper + Explorer on different modules) - **Code Mapper runs first** on medium+ tasks — its output feeds into Architect and Code Writer diff --git a/skills/assistant-workflow/references/task-journal-template.md b/skills/assistant-workflow/references/task-journal-template.md index 016d8b7..8eeeb0f 100644 --- a/skills/assistant-workflow/references/task-journal-template.md +++ b/skills/assistant-workflow/references/task-journal-template.md @@ -23,6 +23,7 @@ Status: DISCOVERING | DECOMPOSING | PLANNING | BUILDING [step N/M] | REVIEWING | Triaged as: [small | medium | large | mega] Task type: [feature | bugfix | refactor | migration | rewrite | config | infra | security | docs | spike] Risk tier: [low | moderate | high | critical] +Controller intensity: [light | standard | strict] Clarification status: [ready | needs_clarification] Clarification defaults applied: [true | false] Clarification confidence: [low | medium | high] @@ -88,6 +89,7 @@ Plan approval: [yes/no + date] ## Harness Appendix Routing [required only for medium+ harness-capable work; otherwise `N/A: [reason]`; full schema in `references/task-journal-harness-appendix.md`] - Appendix: `references/task-journal-harness-appendix.md` +- Controller intensity: [light | standard | strict + reason] - Done Contract: [section/ref, or N/A: reason] - Harness Recipe: [section/ref, or N/A: reason] - Harness Run State: [section/ref, or N/A: reason] @@ -192,6 +194,7 @@ do not start the next slice until the current one is `VERIFIED` ### QA Evaluation - Mode: required | optional | not_required +- QA trigger reason: [QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance. QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work.] - QA Evaluator result: [final_verdict/result ref, or N/A: reason] - Selected domain rubrics: [families used, or N/A] - Domain quality scores: [compact scores, or N/A] diff --git a/skills/assistant-workflow/references/triage-rubric.md b/skills/assistant-workflow/references/triage-rubric.md index 9d1e179..dd0ad5c 100644 --- a/skills/assistant-workflow/references/triage-rubric.md +++ b/skills/assistant-workflow/references/triage-rubric.md @@ -9,6 +9,7 @@ Record these fields in the task journal for medium+ tasks and in the inline plan - `Task type`: `feature`, `bugfix`, `refactor`, `migration`, `rewrite`, `config`, `infra`, `security`, `docs`, or `spike` - `Risk tier`: `low`, `moderate`, `high`, or `critical` - `Triaged as`: `small`, `medium`, `large`, or `mega` +- `Controller intensity`: `light`, `standard`, or `strict` - `Required agents`: the roles required by size and risk - `Subagent policy state`: `not_required`, `authorization_required`, `delegation_authorized`, `authorization_denied`, `subagents_unavailable`, or `policy_disallowed` - `Subagent execution mode`: `delegated`, `direct_fallback`, or `not_applicable` @@ -28,6 +29,16 @@ Record these fields in the task journal for medium+ tasks and in the inline plan Escalate size when risk exceeds file count. Auth, PII, payments, destructive data changes, public API changes, or behavior-preserving legacy migration are at least `large` unless discovery proves they are isolated and fully covered. +## Controller Intensity Rules + +| Intensity | Use when | +|---|---| +| `light` | Small low-risk localized work can use compact guidance and validation. | +| `standard` | Ordinary medium+ source-changing work, including delegated work, when `harness_capable=false` and `qa_evaluation_mode=not_required`. | +| `strict` | High/critical risk, `hook_profile == strict`, `harness_capable=true`, `qa_evaluation_mode=required`, or explicit trace/replay/harness/QA criteria. | + +Do not infer `strict` from `size=medium+` or delegation alone. `standard` keeps ordinary medium non-harness work out of Done Contract, Trace Ledger, Replay Packet, Artifact Reference Ledger, and QA defaults. + ## Candidate Scope Scan Before finalizing Triage, run a quick read-only scan to avoid classifying from the prompt alone. Keep it bounded: named files from the prompt, likely modules/directories, obvious symbols/search terms, nearby tests, docs, contracts, config, generated mirrors, and runtime surfaces that can change the risk tier. diff --git a/tests/p0-p4/harness-controller-contracts.sh b/tests/p0-p4/harness-controller-contracts.sh index a7e59f3..814c990 100644 --- a/tests/p0-p4/harness-controller-contracts.sh +++ b/tests/p0-p4/harness-controller-contracts.sh @@ -95,6 +95,29 @@ require_terms "workflow input contract" "$workflow_dir/contracts/input.yaml" \ "ordinary medium+ workflow tasks default to false" \ "source-changing workflow tasks" +test_start "controller intensity keeps ordinary medium work at standard" +require_terms "controller intensity input contract" "$workflow_dir/contracts/input.yaml" \ + "- name: controller_intensity" \ + "enum_values: [light, standard, strict]" \ + "ordinary medium+" \ + "harness_capable == false" \ + "qa_evaluation_mode == not_required" \ + "Do not infer" \ + "strict from size=medium+ or delegation alone" +require_terms "controller intensity phase gates" "$workflow_dir/contracts/phase-gates.yaml" \ + "T_CONTROLLER_INTENSITY" \ + "ordinary medium+ non-harness work uses standard" \ + "Do not infer strict from size=medium+ or delegation alone" \ + "controller_intensity == standard with harness_capable=false and qa_evaluation_mode=not_required does not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation" +if rg -n 'size=medium\+?[[:space:]]*->[[:space:]]*strict|strict (for|when|because of) size=medium\+?|size=medium\+?[^.\n]*(promote|requires|selects|uses)[^.\n]*strict|delegation alone[^.\n]*(promote|requires|selects|uses|means)[^.\n]*strict|strict (for|when|because of) delegation alone' \ + "$workflow_dir/contracts/input.yaml" \ + "$workflow_dir/contracts/phase-gates.yaml" \ + "$workflow_dir/references/triage-rubric.md" >/tmp/p0p4-controller-intensity-bad-promotion.out; then + fail "controller intensity must not promote medium size or delegation alone to strict; see /tmp/p0p4-controller-intensity-bad-promotion.out" +else + pass +fi + test_start "harness artifacts are not unconditional medium+ requirements" unconditional_harness_terms_output="$(mktemp)" p0p4_register_cleanup "$unconditional_harness_terms_output" diff --git a/tests/p0-p4/instruction-overload-contracts.sh b/tests/p0-p4/instruction-overload-contracts.sh index 5e89d2f..b11461d 100755 --- a/tests/p0-p4/instruction-overload-contracts.sh +++ b/tests/p0-p4/instruction-overload-contracts.sh @@ -28,6 +28,10 @@ if grep -Fq "completion_tiers:" "$workflow_output" \ && grep -Fq "small:" "$workflow_output" \ && grep -Fq "medium:" "$workflow_output" \ && grep -Fq "large_critical:" "$workflow_output" \ + && grep -Fq "controller_intensity: light" "$workflow_output" \ + && grep -Fq "controller_intensity: standard" "$workflow_output" \ + && grep -Fq "controller_intensity: strict" "$workflow_output" \ + && grep -Fq "standard does not require Done Contract, Harness Recipe, Trace Ledger, Replay Packet, Artifact Reference Ledger, or QA evaluation" "$workflow_output" \ && grep -Fq "self-review or explicit validation summary" "$workflow_output" \ && grep -Fq "phase_checkpoints are strict-profile only" "$workflow_output" \ && grep -Fq "condition: \"size in [medium, large, mega] or risk_tier in [high, critical] or hook_profile == strict\"" "$workflow_output" \ @@ -35,7 +39,7 @@ if grep -Fq "completion_tiers:" "$workflow_output" \ && grep -Fq "When size is medium/large/mega, risk_tier is high/critical, or hook_profile == strict: review_result.quality_review_status is not missing" "$workflow_output"; then pass else - fail "assistant-workflow output contract must define small/medium/large-critical completion tiers and conditional heavy review artifacts" + fail "assistant-workflow output contract must define small/medium/large-critical completion tiers, controller intensity mapping, and conditional heavy review artifacts" fi test_start "phase gates separate blockers from guidance" @@ -95,6 +99,24 @@ if [[ -f "$benchmark_doc" ]]; then if ! grep -Fq "runtime-gate-output-trim" "$benchmark_doc"; then benchmark_failures+=("benchmark doc missing runtime-gate-output-trim history row") fi + workflow_enforcer_budget="$(awk -F'|' ' + $2 ~ /^[[:space:]]*workflow-enforcer[[:space:]]*$/ && $3 ~ /^[[:space:]]*codex building phase gates[[:space:]]*$/ { + bytes = $4 + words = $5 + gsub(/[[:space:]]/, "", bytes) + gsub(/[[:space:]]/, "", words) + print bytes " " words + exit + } + ' "$benchmark_doc")" + if [[ -z "$workflow_enforcer_budget" ]]; then + benchmark_failures+=("benchmark doc missing workflow-enforcer active journal metrics") + else + read -r workflow_enforcer_bytes workflow_enforcer_words <<< "$workflow_enforcer_budget" + if (( workflow_enforcer_bytes >= 1511 || workflow_enforcer_words >= 164 )); then + benchmark_failures+=("workflow-enforcer active metrics not reduced below 1511 bytes / 164 words") + fi + fi fi if [[ -f "$benchmark_script" ]] && ! grep -Fq "runtime-gate-output-trim" "$benchmark_script"; then benchmark_failures+=("benchmark script does not render runtime-gate-output-trim history") diff --git a/tests/p0-p4/qa-evaluator-contracts.sh b/tests/p0-p4/qa-evaluator-contracts.sh index 91db67d..0a91d20 100644 --- a/tests/p0-p4/qa-evaluator-contracts.sh +++ b/tests/p0-p4/qa-evaluator-contracts.sh @@ -213,6 +213,8 @@ fi test_start "QA evaluation required mode is explicitly scoped" review_dir="$FRAMEWORK_DIR/skills/assistant-review" +qa_positive_trigger_wording="QA required positive triggers: explicit QA/acceptance evaluation request, accepted Done Contract, harness-capable acceptance scope, domain-scored scope, or scoped UI/visual/product/UX/docs/DX acceptance." +qa_negative_trigger_wording="QA non-triggers: template labels/placeholders, generic acceptance criteria labels, optional/not_required reasons, delegation/source-changing work alone, and ordinary medium+ code-review-only/source-changing work." require_terms "workflow QA mode" "$workflow_dir/contracts/input.yaml" \ "- name: harness_capable" \ "default: false" \ @@ -240,6 +242,41 @@ require_terms "workflow QA output mode" "$workflow_dir/contracts/output.yaml" \ 'condition: "qa_evaluation_mode == required"' \ "QA evaluation does not replace review_result" +test_start "workflow and review QA trigger wording is mirrored" +qa_trigger_wording_missing=() +for file in \ + "$workflow_dir/SKILL.md" \ + "$workflow_dir/contracts/input.yaml" \ + "$workflow_dir/contracts/phase-gates.yaml" \ + "$workflow_dir/references/phases.md" \ + "$workflow_dir/references/plan-harness-appendix.md" \ + "$workflow_dir/references/subagent-dispatch.md" \ + "$workflow_dir/references/plan-template.md" \ + "$workflow_dir/references/task-journal-template.md" \ + "$review_dir/SKILL.md" \ + "$review_dir/contracts/input.yaml" \ + "$review_dir/references/qa-evaluation-loop.md"; do + for term in "$qa_positive_trigger_wording" "$qa_negative_trigger_wording"; do + if [[ ! -f "$file" ]] || ! grep -Fq -- "$term" "$file"; then + qa_trigger_wording_missing+=("${file#$FRAMEWORK_DIR/}: $term") + fi + done +done +if [[ "${#qa_trigger_wording_missing[@]}" -eq 0 ]]; then + pass +else + fail "QA positive/negative trigger wording is not mirrored: ${qa_trigger_wording_missing[*]}" +fi + +test_start "workflow QA trigger wording avoids broad DX-facing scope" +if rg -n "UI/visual/product/UX/docs/DX-facing" \ + "$workflow_dir/SKILL.md" \ + "$workflow_dir/references/plan-harness-appendix.md" >/tmp/p0p4-stale-qa-facing-wording.out; then + fail "workflow root QA surfaces contain broad DX-facing wording; see /tmp/p0p4-stale-qa-facing-wording.out" +else + pass +fi + test_start "workflow QA Done Contract debate_record contract is mirrored" workflow_debate_failures=() for workflow_qa_dir in \ diff --git a/tests/p0-p4/runtime-phase-gate-contracts.sh b/tests/p0-p4/runtime-phase-gate-contracts.sh index 9bdf227..f3983e3 100644 --- a/tests/p0-p4/runtime-phase-gate-contracts.sh +++ b/tests/p0-p4/runtime-phase-gate-contracts.sh @@ -19,6 +19,14 @@ runtime_helper_term_present() { done return 1 } +qa_parser_requires_file() { + local task_file="$1" + bash -c '. "$1"; assistant_phase_requires_qa_evaluator "$2"' _ "$helper_file" "$task_file" +} +required_roles_for_file() { + local task_file="$1" + bash -c '. "$1"; assistant_phase_required_subagent_roles "$2"' _ "$helper_file" "$task_file" +} if [[ ! -f "$helper_file" ]]; then missing_runtime_helper_terms+=("workflow-phase-gates.sh exists") else @@ -86,10 +94,140 @@ else fail "runtime helper wiring missing terms: ${missing_runtime_helper_terms[*]}" fi +test_start "runtime QA parser detects explicit required mode" +qa_required_mode_file="$(mktemp)" +p0p4_register_cleanup "$qa_required_mode_file" +cat > "$qa_required_mode_file" <<'TASK' +# Task +Status: REVIEWING +Triaged as: medium +qa_evaluation_mode: required +TASK +if qa_parser_requires_file "$qa_required_mode_file" \ + && required_roles_for_file "$qa_required_mode_file" | grep -Fxq "QA Evaluator"; then + pass +else + fail "qa_evaluation_mode: required must require QA Evaluator evidence and role inference" +fi + +test_start "runtime QA parser detects scoped positive QA triggers" +qa_positive_missing=() +for spec in \ + "accepted_done_contract|QA trigger reason: accepted Done Contract" \ + "bracketed_done_contract_ref|Done Contract: accepted_by user [Plan Done Contract]" \ + "harness_acceptance|QA trigger reason: harness-capable acceptance scope" \ + "domain_scored|QA trigger reason: domain-scored scope" \ + "scoped_ui_acceptance|QA trigger reason: scoped UI/visual/product/UX/docs/DX acceptance" \ + "required_agent|Required agents: QA Evaluator"; do + label="${spec%%|*}" + trigger_line="${spec#*|}" + qa_positive_file="$(mktemp)" + p0p4_register_cleanup "$qa_positive_file" + cat > "$qa_positive_file" < "$qa_override_file" < "$qa_placeholder_file" <<'TASK' +# Task +Status: REVIEWING +Triaged as: medium +Done Contract: [Done Contract section/ref, or N/A: reason] +Done Contract: [Done Contract section/ref] +QA trigger reason: [generic acceptance criteria label] +QA Evaluator result: [final_verdict/result ref, or N/A: reason] +QA Evaluator result: [final_verdict/result ref] +TASK +if qa_parser_requires_file "$qa_placeholder_file"; then + fail "placeholder and generic trigger labels must not require QA" +else + pass +fi + +test_start "runtime QA parser ignores template, optional, not_required, and generic acceptance labels" +qa_negative_file="$(mktemp)" +p0p4_register_cleanup "$qa_negative_file" +cat > "$qa_negative_file" <<'TASK' +# Task +Status: REVIEWING +Triaged as: medium +Required agents: +- Code Writer +- Builder/Tester +- Code Reviewer +- QA Evaluator when required +Required gates: +- QA Evaluation Log: [section/ref when qa_evaluation_mode=required, or N/A: reason] +Acceptance criteria: +- [generic acceptance criteria label] +qa_evaluation_mode: optional +## Agent Dispatch Log +- QA Evaluator dispatch/result/direct evidence: [delegated QA refs | direct evidence | N/A when not required] +- QA Evaluator direct evidence: not_required +### QA Evaluation +- Mode: optional +- QA trigger reason: N/A: no explicit QA request, Done Contract, harness-capable acceptance scope, domain-scored criteria, or UI/visual/product/UX/docs/DX scope +- QA Evaluator result: [final_verdict/result ref, or N/A: reason] +TASK +qa_negative_roles="$(required_roles_for_file "$qa_negative_file")" +if qa_parser_requires_file "$qa_negative_file" || printf '%s\n' "$qa_negative_roles" | grep -Fxq "QA Evaluator"; then + fail "template labels, placeholders, optional/not_required reasons, and generic acceptance labels must not require QA" +else + pass +fi + test_start "workflow enforcer declares runtime phase gate warnings" missing_workflow_gate_terms=() for term in \ "RUNTIME PHASE GATES" \ + "state: Task:" \ + "clarification: Clarification status:" \ + "review: Reviews completed:" \ + "subagent: Subagent policy state:" \ + "metrics: Metrics today:" \ "Plan approved" \ "Review gate complete" \ "Metrics today" \ @@ -97,11 +235,15 @@ for term in \ "cap is maximum, not quota" \ "Question admissibility" \ "WARNING: You are BUILDING without an approved plan" \ + 'plan_gate:$plan_missing_key missing=' \ "WARNING: Subagent evidence gate incomplete" \ + 'subagent_evidence_gate:$subagent_warning_key' \ "assistant_phase_subagent_warning_reason_key" \ "assistant_phase_subagent_warning_action" \ "WARNING: Review gate incomplete" \ - "WARNING: Metrics gate incomplete"; do + 'review_gate:$review_gate_status missing=' \ + "WARNING: Metrics gate incomplete" \ + "metrics_gate:missing_metrics_today missing="; do if ! grep -Fq "$term" "$FRAMEWORK_DIR/hooks/scripts/workflow-enforcer.sh"; then missing_workflow_gate_terms+=("$term") fi diff --git a/tests/p0-p4/workflow-basics-contracts.sh b/tests/p0-p4/workflow-basics-contracts.sh index e8ecbaf..78e4226 100644 --- a/tests/p0-p4/workflow-basics-contracts.sh +++ b/tests/p0-p4/workflow-basics-contracts.sh @@ -78,6 +78,7 @@ for term in \ "Required Triage Output" \ "Task type" \ "Risk tier" \ + "Controller intensity" \ "Required gates" \ "Required agents" \ "Subagent policy state" \ @@ -96,13 +97,15 @@ for term in \ done for term in \ "references/triage-rubric.md" \ - "Triage metadata"; do + "Triage metadata" \ + "intensity=[controller_intensity]"; do if ! grep -Fq "$term" "$FRAMEWORK_DIR/skills/assistant-workflow/SKILL.md"; then missing_triage_terms+=("SKILL.md: $term") fi done for term in \ "risk_tier" \ + "controller_intensity" \ "required_gates" \ "required_agents" \ "subagent_policy_state" \ @@ -114,9 +117,11 @@ for term in \ done for term in \ "T4" \ + "T_CONTROLLER_INTENSITY" \ "T9" \ "T10" \ "risk_tier is set" \ + "controller_intensity is set" \ "required_gates includes common gates" \ "required_agents or fallback execution roles are populated" \ "subagent_policy_state, subagent_execution_mode, and subagent_authorization_scope are initialized" \ @@ -128,6 +133,7 @@ done for term in \ "Task type:" \ "Risk tier:" \ + "Controller intensity:" \ "Required gates:" \ "Required agents:" \ "Subagent policy state:" \