From 69e8ecec1212ff03ca48e596e8c7d2b2130f59d0 Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Thu, 2 Jul 2026 16:55:21 -0400 Subject: [PATCH 1/6] fix: align H1 headings with frontmatter title convention The experiment numbering convention (PR #35) set frontmatter titles to the `NNNN. Title` format but left H1 headings with their old inconsistent formats (e.g. "Experiment 002:", "PoC FullSend", "ADR 0022"). This causes the fullsend docs site sidebar to display wrong titles since it falls back to H1 when rendering directory entries. Update all 21 experiment H1 headings to match their frontmatter title field exactly. Assisted-by: Claude (fix) Signed-off-by: Wayne Sun --- 0001-agent-outage-fire-drill.md | 2 +- 0002-claude-scanner/README.md | 2 +- 0003-scanner/README.md | 2 +- 0004-meta-loop-self-improving-engine/README.md | 2 +- 0005-agent-scoped-tools-triage/README.md | 2 +- 0007-github-actions-agent-runtime-mvp/README.md | 2 +- 0008-guardrails-eval/README.md | 2 +- 0009-hermes-security-patterns/README.md | 2 +- 0010-host-side-api-server/README.md | 2 +- 0011-integration-service-design-drift/README.md | 2 +- 0012-model-armor-vs-agent-triage/README.md | 2 +- 0013-openshell-policy-bypass/README.md | 2 +- 0014-openshell-sandbox-evaluation.md | 2 +- 0015-prompt-injection-defense/README.md | 2 +- 0016-promptfoo-eval/README.md | 2 +- 0017-reasoning-monitor/README.md | 2 +- 0018-runner-hello-world/README.md | 2 +- 0019-skills/README.md | 2 +- 0020-target-repo-skills/README.md | 2 +- 0021-tool-scoping/README.md | 2 +- 0022-claude-github-app-auth/README.md | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/0001-agent-outage-fire-drill.md b/0001-agent-outage-fire-drill.md index ede62a4..3b3e785 100644 --- a/0001-agent-outage-fire-drill.md +++ b/0001-agent-outage-fire-drill.md @@ -6,7 +6,7 @@ topics: - incident-response --- -# Experiment 003: Agent Outage Fire Drill +# 1. Agent outage fire drill **Date:** 2026-03-16 **Status:** Proposed diff --git a/0002-claude-scanner/README.md b/0002-claude-scanner/README.md index cd899d9..73f0269 100644 --- a/0002-claude-scanner/README.md +++ b/0002-claude-scanner/README.md @@ -6,7 +6,7 @@ topics: - tooling --- -# Experiment 002: Claude-based ADR Drift Scanner +# 2. Claude-based ADR drift scanner A shell script that uses `claude` CLI to analyze Tekton tasks for ADR compliance. No hardcoded rules, no config files — all intelligence comes from claude's comprehension of the ADR text. diff --git a/0003-scanner/README.md b/0003-scanner/README.md index 8add62b..0437617 100644 --- a/0003-scanner/README.md +++ b/0003-scanner/README.md @@ -6,7 +6,7 @@ topics: - tooling --- -# Experiment 001: ADR-0046 Drift Scanner +# 3. ADR-0046 drift scanner A Python CLI tool that detects Tekton task steps drifting from [ADR-0046](https://github.com/konflux-ci/architecture/blob/main/ADR/0046-common-task-runner-image.md) (common task runner image) by comparing step images against a config-driven allowlist. diff --git a/0004-meta-loop-self-improving-engine/README.md b/0004-meta-loop-self-improving-engine/README.md index c2f79d4..71f98f3 100644 --- a/0004-meta-loop-self-improving-engine/README.md +++ b/0004-meta-loop-self-improving-engine/README.md @@ -7,7 +7,7 @@ topics: - evaluation --- -# Experiment 006: Zero-Config Autonomous Bug Fix Engine with Self-Improving Meta-Loop +# 4. Zero-config autonomous bug fix engine with self-improving meta-loop **Date:** 2026-03-29 **Status:** Complete diff --git a/0005-agent-scoped-tools-triage/README.md b/0005-agent-scoped-tools-triage/README.md index ef8195a..ea6bdc1 100644 --- a/0005-agent-scoped-tools-triage/README.md +++ b/0005-agent-scoped-tools-triage/README.md @@ -7,7 +7,7 @@ topics: - skills --- -# PoC FullSend — Triage Agents with scoped tools and sandbox +# 5. Agent scoped tools triage This experiment addresses: https://github.com/fullsend-ai/fullsend/issues/101 diff --git a/0007-github-actions-agent-runtime-mvp/README.md b/0007-github-actions-agent-runtime-mvp/README.md index 2dbf67b..dc0d56f 100644 --- a/0007-github-actions-agent-runtime-mvp/README.md +++ b/0007-github-actions-agent-runtime-mvp/README.md @@ -6,7 +6,7 @@ topics: - tooling --- -# Experiment: GitHub Actions Agent Runtime MVP +# 7. GitHub Actions agent runtime MVP ## Hypothesis diff --git a/0008-guardrails-eval/README.md b/0008-guardrails-eval/README.md index 8a3af77..5643fa4 100644 --- a/0008-guardrails-eval/README.md +++ b/0008-guardrails-eval/README.md @@ -6,7 +6,7 @@ topics: - evaluation --- -# Experiment: Pluggable Guardrails Evaluation +# 8. Guardrails evaluation Evaluates prompt injection scanning tools as pre-LLM guardrails for fullsend's autonomous SDLC pipeline. Extends the [Model Armor experiment](../0012-model-armor-vs-agent-triage/) (PR #117) with additional attack payloads and alternative scanning backends. diff --git a/0009-hermes-security-patterns/README.md b/0009-hermes-security-patterns/README.md index ae151ed..658666f 100644 --- a/0009-hermes-security-patterns/README.md +++ b/0009-hermes-security-patterns/README.md @@ -6,7 +6,7 @@ topics: - tooling --- -# Experiment: Hermes-Inspired Security Patterns for Fullsend +# 9. Hermes-inspired security patterns Evaluates security patterns from [Hermes Agent](https://github.com/NousResearch/hermes-agent) for integration into fullsend's autonomous SDLC pipeline. Tests two integration strategies: static file scanning via [Tirith](https://github.com/sheeki03/tirith) CLI and runtime SSRF protection via a Claude Code PreToolUse hook. diff --git a/0010-host-side-api-server/README.md b/0010-host-side-api-server/README.md index 47019bf..6f9147a 100644 --- a/0010-host-side-api-server/README.md +++ b/0010-host-side-api-server/README.md @@ -6,7 +6,7 @@ topics: - tooling --- -# Experiment: Host-Side API Server for Sandboxed Agents +# 10. Host-side API server for sandboxed agents Tracking issue: [fullsend-ai/experiments#25](https://github.com/fullsend-ai/experiments/issues/25) diff --git a/0011-integration-service-design-drift/README.md b/0011-integration-service-design-drift/README.md index d5fd884..471a114 100644 --- a/0011-integration-service-design-drift/README.md +++ b/0011-integration-service-design-drift/README.md @@ -6,7 +6,7 @@ topics: - tooling --- -# Experiment: Integration Service Design Doc Drift Analysis +# 11. Integration Service design doc drift analysis **Date:** 2026-03-20 diff --git a/0012-model-armor-vs-agent-triage/README.md b/0012-model-armor-vs-agent-triage/README.md index 2abc612..3adb60a 100644 --- a/0012-model-armor-vs-agent-triage/README.md +++ b/0012-model-armor-vs-agent-triage/README.md @@ -6,7 +6,7 @@ topics: - evaluation --- -# Experiment: Model Armor vs AI Agent Triage — Prompt Injection +# 12. Model Armor vs AI agent triage ## Hypothesis diff --git a/0013-openshell-policy-bypass/README.md b/0013-openshell-policy-bypass/README.md index 0c2cc08..e9552a4 100644 --- a/0013-openshell-policy-bypass/README.md +++ b/0013-openshell-policy-bypass/README.md @@ -6,7 +6,7 @@ topics: - sandbox --- -# OpenShell Binary Policy Bypass Experiment +# 13. OpenShell policy bypass This experiment tests whether OpenShell's binary-level network policy restrictions can be bypassed by an agent. It answers two questions: diff --git a/0014-openshell-sandbox-evaluation.md b/0014-openshell-sandbox-evaluation.md index 5570f71..645dece 100644 --- a/0014-openshell-sandbox-evaluation.md +++ b/0014-openshell-sandbox-evaluation.md @@ -6,7 +6,7 @@ topics: - evaluation --- -# Experiment: OpenShell Sandbox Evaluation +# 14. OpenShell sandbox evaluation **Date:** 2026-03-27 **Status:** Complete diff --git a/0015-prompt-injection-defense/README.md b/0015-prompt-injection-defense/README.md index 32d6f69..5ecb4bf 100644 --- a/0015-prompt-injection-defense/README.md +++ b/0015-prompt-injection-defense/README.md @@ -6,7 +6,7 @@ topics: - evaluation --- -# Experiment 002: Prompt Injection Defense-in-Depth +# 15. Prompt injection defense-in-depth ## Hypothesis diff --git a/0016-promptfoo-eval/README.md b/0016-promptfoo-eval/README.md index e669b75..fd26e37 100644 --- a/0016-promptfoo-eval/README.md +++ b/0016-promptfoo-eval/README.md @@ -6,7 +6,7 @@ topics: - tooling --- -# Experiment 004: Promptfoo for Agent Evaluation in CI +# 16. Promptfoo for agent evaluation in CI **Date:** 2026-03-20 **Status:** Complete diff --git a/0017-reasoning-monitor/README.md b/0017-reasoning-monitor/README.md index ddcf644..5fc5cc0 100644 --- a/0017-reasoning-monitor/README.md +++ b/0017-reasoning-monitor/README.md @@ -6,7 +6,7 @@ topics: - evaluation --- -# Experiment 005: Reasoning Monitor Agent for Prompt Injection Detection +# 17. Reasoning monitor agent for prompt injection detection **Status:** Implementation complete, results pending **Issue:** [fullsend #174](https://github.com/konflux-ci/fullsend/issues/174) diff --git a/0018-runner-hello-world/README.md b/0018-runner-hello-world/README.md index faf8da1..317b972 100644 --- a/0018-runner-hello-world/README.md +++ b/0018-runner-hello-world/README.md @@ -6,7 +6,7 @@ topics: - sandbox --- -# Runner Hello World Experiment +# 18. Runner hello world A minimal end-to-end test of the `fullsend run` CLI. It provisions an OpenShell sandbox from a pre-built container image, runs a Claude Code agent inside it, extracts output, and validates the result with a retry loop. diff --git a/0019-skills/README.md b/0019-skills/README.md index f528c59..315c26c 100644 --- a/0019-skills/README.md +++ b/0019-skills/README.md @@ -5,6 +5,6 @@ topics: - skills --- -# Experiment: Skills +# 19. Skills Experiments with skill authoring and consumption patterns. diff --git a/0020-target-repo-skills/README.md b/0020-target-repo-skills/README.md index c75c145..4525137 100644 --- a/0020-target-repo-skills/README.md +++ b/0020-target-repo-skills/README.md @@ -6,7 +6,7 @@ topics: - tooling --- -# Experiment: Target Repository Skills in Triage +# 20. Target repository skills in triage ## Hypothesis diff --git a/0021-tool-scoping/README.md b/0021-tool-scoping/README.md index 2638af3..407d77b 100644 --- a/0021-tool-scoping/README.md +++ b/0021-tool-scoping/README.md @@ -6,7 +6,7 @@ topics: - tooling --- -# ADR 0022 Tool Scoping Experiment +# 21. Tool scoping Evidence for [ADR 0022](https://github.com/fullsend-ai/fullsend-adr-tools/blob/main/docs/ADRs/0022-allowed-and-disallowed-tools-for-agents.md). diff --git a/0022-claude-github-app-auth/README.md b/0022-claude-github-app-auth/README.md index 638db53..08edfc2 100644 --- a/0022-claude-github-app-auth/README.md +++ b/0022-claude-github-app-auth/README.md @@ -6,7 +6,7 @@ topics: - security --- -# PoC FullSend #67 +# 22. Claude GitHub App auth This experiment folder addresses: https://github.com/konflux-ci/fullsend/issues/67 From d9e7732c2e0497796434e34ec4c6db4031f84397 Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Thu, 2 Jul 2026 16:56:50 -0400 Subject: [PATCH 2/6] fix(0023): apply experiment conventions to review-cache-experiment Rename review-cache-experiment to 0023-review-cache, add required frontmatter (title, status, topics), update H1 heading, and add entry to README index table. Assisted-by: Claude (fix) Signed-off-by: Wayne Sun --- .../README.md | 12 ++++++++++-- .../scripts/extract_functions.sh | 0 .../scripts/filter.py | 0 .../scripts/publish.py | 0 .../scripts/save.py | 0 .../scripts/schema.sql | 0 .../scripts/store.py | 0 .../test-data/results/results.md | 0 .../test-data/review/review1.json | 0 .../test-data/review/review2.json | 0 .../test-data/review/review3.json | 0 .../test_e2e.py | 0 .../test_filter.sh | 0 .../test_publication_policy.sh | 0 .../test_results.md | 0 README.md | 1 + 16 files changed, 11 insertions(+), 2 deletions(-) rename {review-cache-experiment => 0023-review-cache}/README.md (98%) rename {review-cache-experiment => 0023-review-cache}/scripts/extract_functions.sh (100%) rename {review-cache-experiment => 0023-review-cache}/scripts/filter.py (100%) rename {review-cache-experiment => 0023-review-cache}/scripts/publish.py (100%) rename {review-cache-experiment => 0023-review-cache}/scripts/save.py (100%) rename {review-cache-experiment => 0023-review-cache}/scripts/schema.sql (100%) rename {review-cache-experiment => 0023-review-cache}/scripts/store.py (100%) rename {review-cache-experiment => 0023-review-cache}/test-data/results/results.md (100%) rename {review-cache-experiment => 0023-review-cache}/test-data/review/review1.json (100%) rename {review-cache-experiment => 0023-review-cache}/test-data/review/review2.json (100%) rename {review-cache-experiment => 0023-review-cache}/test-data/review/review3.json (100%) rename {review-cache-experiment => 0023-review-cache}/test_e2e.py (100%) rename {review-cache-experiment => 0023-review-cache}/test_filter.sh (100%) rename {review-cache-experiment => 0023-review-cache}/test_publication_policy.sh (100%) rename {review-cache-experiment => 0023-review-cache}/test_results.md (100%) diff --git a/review-cache-experiment/README.md b/0023-review-cache/README.md similarity index 98% rename from review-cache-experiment/README.md rename to 0023-review-cache/README.md index 5600d1d..1478559 100644 --- a/review-cache-experiment/README.md +++ b/0023-review-cache/README.md @@ -1,4 +1,12 @@ -# Review Cache - Publication Policy Architecture +--- +title: "23. Review cache publication policy" +status: Concluded +topics: + - review + - deduplication +--- + +# 23. Review cache publication policy ## Problem @@ -356,4 +364,4 @@ To validate readiness for production integration, the experiment needs real data **Deduplication accuracy:** Needs real-world validation before production use. The 100% success rate on synthetic data validates the mechanism works, but doesn't yet prove it handles the messy reality of production code reviews. -**Recommendation:** Run real data validation to measure actual dedup rates and identify edge cases before committing to SQLite cache as the production solution. \ No newline at end of file +**Recommendation:** Run real data validation to measure actual dedup rates and identify edge cases before committing to SQLite cache as the production solution. diff --git a/review-cache-experiment/scripts/extract_functions.sh b/0023-review-cache/scripts/extract_functions.sh similarity index 100% rename from review-cache-experiment/scripts/extract_functions.sh rename to 0023-review-cache/scripts/extract_functions.sh diff --git a/review-cache-experiment/scripts/filter.py b/0023-review-cache/scripts/filter.py similarity index 100% rename from review-cache-experiment/scripts/filter.py rename to 0023-review-cache/scripts/filter.py diff --git a/review-cache-experiment/scripts/publish.py b/0023-review-cache/scripts/publish.py similarity index 100% rename from review-cache-experiment/scripts/publish.py rename to 0023-review-cache/scripts/publish.py diff --git a/review-cache-experiment/scripts/save.py b/0023-review-cache/scripts/save.py similarity index 100% rename from review-cache-experiment/scripts/save.py rename to 0023-review-cache/scripts/save.py diff --git a/review-cache-experiment/scripts/schema.sql b/0023-review-cache/scripts/schema.sql similarity index 100% rename from review-cache-experiment/scripts/schema.sql rename to 0023-review-cache/scripts/schema.sql diff --git a/review-cache-experiment/scripts/store.py b/0023-review-cache/scripts/store.py similarity index 100% rename from review-cache-experiment/scripts/store.py rename to 0023-review-cache/scripts/store.py diff --git a/review-cache-experiment/test-data/results/results.md b/0023-review-cache/test-data/results/results.md similarity index 100% rename from review-cache-experiment/test-data/results/results.md rename to 0023-review-cache/test-data/results/results.md diff --git a/review-cache-experiment/test-data/review/review1.json b/0023-review-cache/test-data/review/review1.json similarity index 100% rename from review-cache-experiment/test-data/review/review1.json rename to 0023-review-cache/test-data/review/review1.json diff --git a/review-cache-experiment/test-data/review/review2.json b/0023-review-cache/test-data/review/review2.json similarity index 100% rename from review-cache-experiment/test-data/review/review2.json rename to 0023-review-cache/test-data/review/review2.json diff --git a/review-cache-experiment/test-data/review/review3.json b/0023-review-cache/test-data/review/review3.json similarity index 100% rename from review-cache-experiment/test-data/review/review3.json rename to 0023-review-cache/test-data/review/review3.json diff --git a/review-cache-experiment/test_e2e.py b/0023-review-cache/test_e2e.py similarity index 100% rename from review-cache-experiment/test_e2e.py rename to 0023-review-cache/test_e2e.py diff --git a/review-cache-experiment/test_filter.sh b/0023-review-cache/test_filter.sh similarity index 100% rename from review-cache-experiment/test_filter.sh rename to 0023-review-cache/test_filter.sh diff --git a/review-cache-experiment/test_publication_policy.sh b/0023-review-cache/test_publication_policy.sh similarity index 100% rename from review-cache-experiment/test_publication_policy.sh rename to 0023-review-cache/test_publication_policy.sh diff --git a/review-cache-experiment/test_results.md b/0023-review-cache/test_results.md similarity index 100% rename from review-cache-experiment/test_results.md rename to 0023-review-cache/test_results.md diff --git a/README.md b/README.md index 482b172..2323d9a 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Experiments for the fullsend platform — each tests a hypothesis about autonomo | 0020 | [Target repository skills in triage](0020-target-repo-skills/) | Concluded | | 0021 | [Tool scoping](0021-tool-scoping/) | Concluded | | 0022 | [Claude GitHub App auth](0022-claude-github-app-auth/) | Concluded | +| 0023 | [Review cache publication policy](0023-review-cache/) | Concluded | ## Conventions From 7cc600022bd625eec19c816985214eb588ff26a6 Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Thu, 2 Jul 2026 17:00:39 -0400 Subject: [PATCH 3/6] feat: add H1-matches-frontmatter-title lint check Add validation to lint-experiment-frontmatter that enforces H1 heading matches the frontmatter title field exactly. Prevents future drift between the two after the convention was established. Also fixes 0006-code-agent-evaluation which had a pre-existing mismatch caught by the new check. Assisted-by: Claude (fix) Signed-off-by: Wayne Sun --- 0006-code-agent-evaluation/EXPERIMENT.md | 2 +- AGENTS.md | 2 +- hack/lint-experiment-frontmatter | 22 ++++++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/0006-code-agent-evaluation/EXPERIMENT.md b/0006-code-agent-evaluation/EXPERIMENT.md index a0ce700..68a04c4 100644 --- a/0006-code-agent-evaluation/EXPERIMENT.md +++ b/0006-code-agent-evaluation/EXPERIMENT.md @@ -6,7 +6,7 @@ topics: - security --- -# Code Agent Evaluation Experiment +# 6. Code agent evaluation **Date:** April 11–14, 2026 **Author:** Adam Scerra diff --git a/AGENTS.md b/AGENTS.md index a93603e..2257dc8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,7 +40,7 @@ Pre-commit hooks enforce experiment conventions. Always stage your changes befor | Hook | What it validates | |------|-------------------| | `lint-experiment-numbers` | Filenames match `^[0-9]{4}-`, no duplicate numbers, no leading zeros in titles | -| `lint-experiment-frontmatter` | Required fields (`title`, `status`), valid status values, `topics` is a list | +| `lint-experiment-frontmatter` | Required fields (`title`, `status`), valid status values, `topics` is a list, H1 heading matches frontmatter `title` | | `lint-experiment-index` | README.md index table lists every experiment on disk and vice versa | Run all hooks manually: `pre-commit run --all-files` diff --git a/hack/lint-experiment-frontmatter b/hack/lint-experiment-frontmatter index 3d8f9bb..b87828b 100755 --- a/hack/lint-experiment-frontmatter +++ b/hack/lint-experiment-frontmatter @@ -9,6 +9,7 @@ Checks: - Experiment files have required frontmatter fields (title, status) - Status is a valid value (Active, Concluded, Abandoned, Merged) - topics is a list if present +- H1 heading matches frontmatter title exactly Adapted from fullsend-ai/fullsend hack/lint-adr-frontmatter """ @@ -61,6 +62,16 @@ def find_experiment_md(entry_path): return None +def extract_h1(filepath): + """Extract the first H1 heading from a markdown file.""" + with open(filepath, encoding="utf-8") as f: + for line in f: + m = re.match(r"^#\s+(.+)$", line.rstrip()) + if m: + return m.group(1) + return None + + def validate_experiment(filepath): """Validate an experiment file's frontmatter. Returns list of error strings.""" errors = [] @@ -89,6 +100,17 @@ def validate_experiment(filepath): ) ) + if "title" in fm: + h1 = extract_h1(filepath) + if h1 is None: + errors.append(f"{rel_path}: missing H1 heading") + elif h1 != fm["title"]: + errors.append( + "{}: H1 heading does not match frontmatter title\n" + " title: {}\n" + " H1: {}".format(rel_path, fm["title"], h1) + ) + return errors From 59cfec83abc13f23722345fa7fa5bcc5fe2c3336 Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Thu, 2 Jul 2026 17:03:29 -0400 Subject: [PATCH 4/6] docs(0006): rename EXPERIMENT.md to README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align with repo convention — all experiments use README.md as the primary document. Update cross-references in RECOMMENDATION.md (4) and invoke-variant.sh (1). Assisted-by: Claude Signed-off-by: Wayne Sun --- 0006-code-agent-evaluation/{EXPERIMENT.md => README.md} | 0 0006-code-agent-evaluation/RECOMMENDATION.md | 8 ++++---- 0006-code-agent-evaluation/scripts/invoke-variant.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename 0006-code-agent-evaluation/{EXPERIMENT.md => README.md} (100%) diff --git a/0006-code-agent-evaluation/EXPERIMENT.md b/0006-code-agent-evaluation/README.md similarity index 100% rename from 0006-code-agent-evaluation/EXPERIMENT.md rename to 0006-code-agent-evaluation/README.md diff --git a/0006-code-agent-evaluation/RECOMMENDATION.md b/0006-code-agent-evaluation/RECOMMENDATION.md index 313be60..ec4b3b5 100644 --- a/0006-code-agent-evaluation/RECOMMENDATION.md +++ b/0006-code-agent-evaluation/RECOMMENDATION.md @@ -1,7 +1,7 @@ # Code Agent Recommendation — Best Configuration for PR #189 **Date:** April 14, 2026 (updated after Round 4) -**Based on:** [EXPERIMENT.md](EXPERIMENT.md) — 490+ trials across 20 scenarios, 8 variants, 4 rounds +**Based on:** [README.md](README.md) — 490+ trials across 20 scenarios, 8 variants, 4 rounds **Target:** [PR #189: Add code agent definition and skill](https://github.com/fullsend-ai/fullsend/pull/189) **Review feedback:** [ralphbean's preliminary review](https://github.com/fullsend-ai/fullsend/pull/189#issuecomment-1) @@ -36,7 +36,7 @@ further redesign is needed. For detailed scores by round and scenario, see -[EXPERIMENT.md sections 5–8](EXPERIMENT.md#5-round-1-results-all-variants). +[README.md sections 5–8](README.md#5-round-1-results-all-variants). --- @@ -44,7 +44,7 @@ For detailed scores by round and scenario, see All priority-1 changes from the initial recommendation have been implemented in the V8 hybrid variant, validated with 66 trials -([Round 4](EXPERIMENT.md#8-round-4-results-v8-hybrid-validation)), and +([Round 4](README.md#8-round-4-results-v8-hybrid-validation)), and committed as [`d70dcff`](https://github.com/fullsend-ai/fullsend/commit/d70dcff). @@ -110,7 +110,7 @@ structured variants) ## Cross-references -- **Full experiment:** [EXPERIMENT.md](EXPERIMENT.md) +- **Full experiment:** [README.md](README.md) - **V8 variant (PR #189 latest):** [variants/V8-hybrid/](variants/V8-hybrid/) - **All variants (V1–V7):** [variants/VARIANTS.md](variants/VARIANTS.md) — links to browse any variant - **PR #189:** [Add code agent definition and skill](https://github.com/fullsend-ai/fullsend/pull/189) diff --git a/0006-code-agent-evaluation/scripts/invoke-variant.sh b/0006-code-agent-evaluation/scripts/invoke-variant.sh index 00b1742..cc7f95d 100644 --- a/0006-code-agent-evaluation/scripts/invoke-variant.sh +++ b/0006-code-agent-evaluation/scripts/invoke-variant.sh @@ -134,7 +134,7 @@ case "${VARIANT}" in V4) # V4: CLAUDE.md only — infrastructure exists but V4 was excluded from - # scored results (started then stopped; see EXPERIMENT.md section 2) + # scored results (started then stopped; see README.md section 2) VARIANT_DIR="${EXPERIMENT_ROOT}/variants/V4-claudemd-only" # Copy CLAUDE.md to repo root From de0b2c64e6e635a8a905000673fd155a3c0cbea8 Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Thu, 2 Jul 2026 17:16:51 -0400 Subject: [PATCH 5/6] style(0023): apply pre-commit auto-fixes Trailing whitespace, ruff lint fixes (13 errors), and ruff format on review-cache experiment files. Assisted-by: Claude Signed-off-by: Wayne Sun --- 0023-review-cache/scripts/filter.py | 68 ++++--- 0023-review-cache/scripts/publish.py | 151 ++++++++------- 0023-review-cache/scripts/save.py | 56 +++--- 0023-review-cache/scripts/store.py | 153 ++++++++------- .../test-data/results/results.md | 2 +- 0023-review-cache/test_e2e.py | 177 +++++++++++------- 0023-review-cache/test_results.md | 14 +- 7 files changed, 356 insertions(+), 265 deletions(-) diff --git a/0023-review-cache/scripts/filter.py b/0023-review-cache/scripts/filter.py index b6d6979..47e6041 100755 --- a/0023-review-cache/scripts/filter.py +++ b/0023-review-cache/scripts/filter.py @@ -20,7 +20,8 @@ import json import sys from pathlib import Path -from store import ReviewMemoryStore, Finding + +from store import ReviewMemoryStore def filter_findings(findings_file: str, pr_number: int, db_path: str, output_file: str): @@ -40,15 +41,15 @@ def filter_findings(findings_file: str, pr_number: int, db_path: str, output_fil output_file: Classified output path """ # Read agent findings - with open(findings_file, 'r') as f: + with open(findings_file, "r") as f: data = json.load(f) - agent_findings = data.get('findings', []) + agent_findings = data.get("findings", []) if not agent_findings: print("No findings to filter") # Write empty output - with open(output_file, 'w') as f: + with open(output_file, "w") as f: json.dump(data, f, indent=2) return @@ -65,10 +66,10 @@ def filter_findings(findings_file: str, pr_number: int, db_path: str, output_fil for f_data in agent_findings: # Build semantic key from finding - file_path = f_data.get('file', '') - line = f_data.get('line', 0) - category = f_data.get('category', 'unknown') - function_name = f_data.get('function', 'package') + file_path = f_data.get("file", "") + line = f_data.get("line", 0) + category = f_data.get("category", "unknown") + function_name = f_data.get("function", "package") # Calculate semantic key (same logic as store.py) line_bucket = (line // 10) * 10 if line else 0 @@ -79,9 +80,11 @@ def filter_findings(findings_file: str, pr_number: int, db_path: str, output_fil if dismissal_reason: # CLASSIFY as dismissed - f_data['status'] = 'dismissed' - f_data['dismissal_reason'] = dismissal_reason - print(f"ℹ️ CLASSIFIED (dismissed): {category} in {file_path}:{function_name}") + f_data["status"] = "dismissed" + f_data["dismissal_reason"] = dismissal_reason + print( + f"ℹ️ CLASSIFIED (dismissed): {category} in {file_path}:{function_name}" + ) print(f" Reason: {dismissal_reason}") dismissed_count += 1 classified.append(f_data) @@ -92,55 +95,62 @@ def filter_findings(findings_file: str, pr_number: int, db_path: str, output_fil prior = prior_map[semantic_key] # Was it previously reported? - if prior.status in ['new', 'still_present']: + if prior.status in ["new", "still_present"]: # CLASSIFY as still_present - f_data['status'] = 'still_present' - f_data['first_seen_sha'] = prior.first_seen_sha - print(f"📌 CLASSIFIED (still-present): {category} in {file_path}:{function_name}") + f_data["status"] = "still_present" + f_data["first_seen_sha"] = prior.first_seen_sha + print( + f"📌 CLASSIFIED (still-present): {category} in {file_path}:{function_name}" + ) print(f" First seen: {prior.first_seen_sha}") still_present_count += 1 classified.append(f_data) continue # Check 3: New finding - f_data['status'] = 'new' + f_data["status"] = "new" print(f"🆕 CLASSIFIED (new): {category} in {file_path}:{function_name}") new_count += 1 classified.append(f_data) # Update findings in result - data['findings'] = classified + data["findings"] = classified # Write classified output - with open(output_file, 'w') as f: + with open(output_file, "w") as f: json.dump(data, f, indent=2) - print(f"\n✓ Classification complete:") + print("\n✓ Classification complete:") print(f" Total input: {len(agent_findings)}") print(f" New: {new_count}") print(f" Still present: {still_present_count}") print(f" Dismissed: {dismissed_count}") - print(f"\n All findings passed to publish.py for presentation policy") + print("\n All findings passed to publish.py for presentation policy") def main(): parser = argparse.ArgumentParser(description="Classify findings by lifecycle state") - parser.add_argument('--findings', required=True, help="Path to agent-result.json") - parser.add_argument('--pr', type=int, required=True, help="PR number") - parser.add_argument('--db', default=".fullsend/review-memory.db", help="Database path") - parser.add_argument('--output', required=True, help="Classified output path") + parser.add_argument("--findings", required=True, help="Path to agent-result.json") + parser.add_argument("--pr", type=int, required=True, help="PR number") + parser.add_argument( + "--db", default=".fullsend/review-memory.db", help="Database path" + ) + parser.add_argument("--output", required=True, help="Classified output path") args = parser.parse_args() # Check database exists if not Path(args.db).exists(): - print(f"No cache database found at {args.db} - marking all as 'new'", file=sys.stderr) + print( + f"No cache database found at {args.db} - marking all as 'new'", + file=sys.stderr, + ) # Mark all findings as 'new' since we have no prior context - with open(args.findings, 'r') as f: + with open(args.findings, "r") as f: data = json.load(f) - for finding in data.get('findings', []): - finding['status'] = 'new' - with open(args.output, 'w') as f: + for finding in data.get("findings", []): + finding["status"] = "new" + with open(args.output, "w") as f: json.dump(data, f, indent=2) return diff --git a/0023-review-cache/scripts/publish.py b/0023-review-cache/scripts/publish.py index 45e4713..31cefa9 100644 --- a/0023-review-cache/scripts/publish.py +++ b/0023-review-cache/scripts/publish.py @@ -44,45 +44,39 @@ def apply_policy(self, findings: List[dict]) -> Dict[str, List[dict]]: } """ result = { - 'inline_comments': [], - 'summary_resolved': [], - 'summary_unresolved': [], - 'summary_dismissed': [] + "inline_comments": [], + "summary_resolved": [], + "summary_unresolved": [], + "summary_dismissed": [], } for finding in findings: - status = finding.get('status', 'new') + status = finding.get("status", "new") - if status == 'new': + if status == "new": # New findings: post full inline comment - result['inline_comments'].append(finding) + result["inline_comments"].append(finding) - elif status == 'still_present': + elif status == "still_present": # Still present: mention in summary, don't post inline # This is the key to reducing duplicate noise - result['summary_unresolved'].append(finding) + result["summary_unresolved"].append(finding) - elif status == 'resolved': + elif status == "resolved": # Resolved: show progress - result['summary_resolved'].append(finding) + result["summary_resolved"].append(finding) - elif status == 'dismissed': + elif status == "dismissed": # Dismissed: show transparency (but don't post inline) - result['summary_dismissed'].append(finding) + result["summary_dismissed"].append(finding) return result def severity_icon(severity: str) -> str: """Return emoji for severity level.""" - icons = { - 'critical': '🔴', - 'high': '🔴', - 'medium': '🟡', - 'low': '🟢', - 'info': 'ℹ️' - } - return icons.get(severity.lower(), '⚪') + icons = {"critical": "🔴", "high": "🔴", "medium": "🟡", "low": "🟢", "info": "ℹ️"} + return icons.get(severity.lower(), "⚪") def format_inline_comment(finding: dict) -> str: @@ -99,18 +93,18 @@ def format_inline_comment(finding: dict) -> str: **Remediation:** Add mutex protection or use atomic operations. """ - icon = severity_icon(finding.get('severity', 'medium')) - category = finding.get('category', 'unknown') - file_path = finding.get('file', 'unknown') - line = finding.get('line', 0) - function = finding.get('function', '') - severity = finding.get('severity', 'medium') - description = finding.get('description', '') - remediation = finding.get('remediation', '') + icon = severity_icon(finding.get("severity", "medium")) + category = finding.get("category", "unknown") + file_path = finding.get("file", "unknown") + line = finding.get("line", 0) + function = finding.get("function", "") + severity = finding.get("severity", "medium") + description = finding.get("description", "") + remediation = finding.get("remediation", "") # Header location = f"{file_path}:{line}" - if function and function != 'package': + if function and function != "package": location += f" ({function})" comment = f"### {icon} {category} in {location}\n\n" @@ -130,20 +124,24 @@ def format_summary_item(finding: dict) -> str: Example: - race-condition in collector.go:89 (coldStart) """ - category = finding.get('category', 'unknown') - file_path = finding.get('file', 'unknown') - line = finding.get('line', 0) - function = finding.get('function', '') + category = finding.get("category", "unknown") + file_path = finding.get("file", "unknown") + line = finding.get("line", 0) + function = finding.get("function", "") item = f"- {category} in {file_path}:{line}" - if function and function != 'package': + if function and function != "package": item += f" ({function})" # Add dismissal reason if present - dismissal_reason = finding.get('dismissal_reason') + dismissal_reason = finding.get("dismissal_reason") if dismissal_reason: # Truncate long reasons - reason = dismissal_reason[:60] + '...' if len(dismissal_reason) > 60 else dismissal_reason + reason = ( + dismissal_reason[:60] + "..." + if len(dismissal_reason) > 60 + else dismissal_reason + ) item += f"\n *Reason: {reason}*" return item @@ -160,16 +158,16 @@ def generate_github_comment(policy_result: Dict[str, List[dict]], verdict: str) sections = [] # Section 1: Inline comments for new findings - inline = policy_result['inline_comments'] + inline = policy_result["inline_comments"] if inline: sections.append("## Security Issues\n") for finding in inline: sections.append(format_inline_comment(finding)) # Section 2: Progress summary - resolved = policy_result['summary_resolved'] - unresolved = policy_result['summary_unresolved'] - dismissed = policy_result['summary_dismissed'] + resolved = policy_result["summary_resolved"] + unresolved = policy_result["summary_unresolved"] + dismissed = policy_result["summary_dismissed"] if resolved or unresolved or dismissed: sections.append("\n---\n\n## Progress Summary\n") @@ -193,14 +191,17 @@ def generate_github_comment(policy_result: Dict[str, List[dict]], verdict: str) if not inline and not resolved and not unresolved and not dismissed: sections.append("## ✅ No Issues Found\n\nAll checks passed!\n") - return '\n'.join(sections) + return "\n".join(sections) -def compute_resolved_findings(pr_number: int, db_path: str, current_findings: List[dict]) -> List[dict]: +def compute_resolved_findings( + pr_number: int, db_path: str, current_findings: List[dict] +) -> List[dict]: """ Compute which findings from cache are now resolved (not in current output). """ from pathlib import Path + from store import ReviewMemoryStore if not Path(db_path).exists(): @@ -209,10 +210,10 @@ def compute_resolved_findings(pr_number: int, db_path: str, current_findings: Li # Build semantic keys for current findings current_keys = set() for f in current_findings: - file_path = f.get('file', '') - line = f.get('line', 0) - category = f.get('category', 'unknown') - function = f.get('function', 'package') + file_path = f.get("file", "") + line = f.get("line", 0) + category = f.get("category", "unknown") + function = f.get("function", "package") line_bucket = (line // 10) * 10 if line else 0 semantic_key = f"{file_path}:{function}:{category}:{line_bucket}" current_keys.add(semantic_key) @@ -225,15 +226,17 @@ def compute_resolved_findings(pr_number: int, db_path: str, current_findings: Li for prior in prior_findings: # Was in prior, not in current → resolved if prior.semantic_key not in current_keys: - if prior.status in ['new', 'still_present']: - resolved.append({ - 'file': prior.file_path, - 'line': prior.line_number, - 'function': prior.function_name, - 'category': prior.category, - 'severity': prior.severity, - 'status': 'resolved' - }) + if prior.status in ["new", "still_present"]: + resolved.append( + { + "file": prior.file_path, + "line": prior.line_number, + "function": prior.function_name, + "category": prior.category, + "severity": prior.severity, + "status": "resolved", + } + ) return resolved @@ -242,23 +245,29 @@ def main(): parser = argparse.ArgumentParser( description="Apply publication policy to classified findings" ) - parser.add_argument('--findings', required=True, - help="Path to classified findings JSON") - parser.add_argument('--output', required=True, - help="Path to output GitHub comment markdown") - parser.add_argument('--pr', type=int, - help="PR number (optional, for computing resolved findings)") - parser.add_argument('--db', default=".fullsend/review-memory.db", - help="Database path (optional, for computing resolved findings)") + parser.add_argument( + "--findings", required=True, help="Path to classified findings JSON" + ) + parser.add_argument( + "--output", required=True, help="Path to output GitHub comment markdown" + ) + parser.add_argument( + "--pr", type=int, help="PR number (optional, for computing resolved findings)" + ) + parser.add_argument( + "--db", + default=".fullsend/review-memory.db", + help="Database path (optional, for computing resolved findings)", + ) args = parser.parse_args() # Load classified findings - with open(args.findings, 'r') as f: + with open(args.findings, "r") as f: data = json.load(f) - findings = data.get('findings', []) - verdict = data.get('action', 'comment') + findings = data.get("findings", []) + verdict = data.get("action", "comment") # Apply publication policy policy = PublicationPolicy() @@ -267,7 +276,7 @@ def main(): # Compute resolved findings if cache is available if args.pr: resolved_findings = compute_resolved_findings(args.pr, args.db, findings) - policy_result['summary_resolved'] = resolved_findings + policy_result["summary_resolved"] = resolved_findings # Print policy decisions print("Publication Policy Applied:") @@ -280,14 +289,14 @@ def main(): comment = generate_github_comment(policy_result, verdict) # Write output - with open(args.output, 'w') as f: + with open(args.output, "w") as f: f.write(comment) print(f"\n✓ GitHub comment written to {args.output}") # Also write policy result as JSON for testing - policy_json = args.output.replace('.md', '-policy.json') - with open(policy_json, 'w') as f: + policy_json = args.output.replace(".md", "-policy.json") + with open(policy_json, "w") as f: json.dump(policy_result, f, indent=2) print(f"✓ Policy result written to {policy_json}") diff --git a/0023-review-cache/scripts/save.py b/0023-review-cache/scripts/save.py index 52fcbeb..e78de17 100644 --- a/0023-review-cache/scripts/save.py +++ b/0023-review-cache/scripts/save.py @@ -10,10 +10,11 @@ import argparse import json -import sys import subprocess +import sys from pathlib import Path -from store import ReviewMemoryStore, Finding + +from store import Finding, ReviewMemoryStore def extract_enrichment(file_path: str, line: int) -> tuple[str, str]: @@ -32,10 +33,7 @@ def extract_enrichment(file_path: str, line: int) -> tuple[str, str]: extract_function_name "{file_path}" {line} """ result = subprocess.run( - ["bash", "-c", bash_cmd], - capture_output=True, - text=True, - check=True + ["bash", "-c", bash_cmd], capture_output=True, text=True, check=True ) function_name = result.stdout.strip() or "package" @@ -44,21 +42,25 @@ def extract_enrichment(file_path: str, line: int) -> tuple[str, str]: extract_code_snippet "{file_path}" {line} """ result = subprocess.run( - ["bash", "-c", bash_cmd], - capture_output=True, - text=True, - check=True + ["bash", "-c", bash_cmd], capture_output=True, text=True, check=True ) code_snippet = result.stdout.strip() return function_name, code_snippet except subprocess.CalledProcessError as e: - print(f"Warning: Failed to extract for {file_path}:{line}: {e}", file=sys.stderr) + print( + f"Warning: Failed to extract for {file_path}:{line}: {e}", file=sys.stderr + ) return "package", "" -def save_findings(findings_file: str, pr_number: int, head_sha: str, db_path: str = ".fullsend/review-memory.db"): +def save_findings( + findings_file: str, + pr_number: int, + head_sha: str, + db_path: str = ".fullsend/review-memory.db", +): """ Save findings to SQLite, enriching with function names and code snippets. @@ -69,10 +71,10 @@ def save_findings(findings_file: str, pr_number: int, head_sha: str, db_path: st db_path: Database path """ # Read findings - with open(findings_file, 'r') as f: + with open(findings_file, "r") as f: data = json.load(f) - findings_data = data.get('findings', []) + findings_data = data.get("findings", []) if not findings_data: print("No findings to save") @@ -83,8 +85,8 @@ def save_findings(findings_file: str, pr_number: int, head_sha: str, db_path: st for f_data in findings_data: # Extract enrichment - file_path = f_data.get('file', '') - line = f_data.get('line', 0) + file_path = f_data.get("file", "") + line = f_data.get("line", 0) function_name, code_snippet = extract_enrichment(file_path, line) @@ -92,16 +94,16 @@ def save_findings(findings_file: str, pr_number: int, head_sha: str, db_path: st finding = Finding( file_path=file_path, function_name=function_name, - category=f_data.get('category', 'unknown'), - severity=f_data.get('severity', 'low'), - description=f_data.get('description', ''), - remediation=f_data.get('remediation', ''), + category=f_data.get("category", "unknown"), + severity=f_data.get("severity", "low"), + description=f_data.get("description", ""), + remediation=f_data.get("remediation", ""), code_snippet=code_snippet, line_number=line, pr_number=pr_number, first_seen_sha=head_sha, last_seen_sha=head_sha, - status="new" + status="new", ) enriched_findings.append(finding) @@ -113,7 +115,7 @@ def save_findings(findings_file: str, pr_number: int, head_sha: str, db_path: st # Save all findings saved_count = 0 - for status_group in ['new', 'still_present', 'resolved']: + for status_group in ["new", "still_present", "resolved"]: for finding in result[status_group]: store.save_finding(finding) saved_count += 1 @@ -126,10 +128,12 @@ def save_findings(findings_file: str, pr_number: int, head_sha: str, db_path: st def main(): parser = argparse.ArgumentParser(description="Save review findings to cache") - parser.add_argument('--findings', required=True, help="Path to agent-result.json") - parser.add_argument('--pr', type=int, required=True, help="PR number") - parser.add_argument('--sha', required=True, help="Current HEAD SHA") - parser.add_argument('--db', default=".fullsend/review-memory.db", help="Database path") + parser.add_argument("--findings", required=True, help="Path to agent-result.json") + parser.add_argument("--pr", type=int, required=True, help="PR number") + parser.add_argument("--sha", required=True, help="Current HEAD SHA") + parser.add_argument( + "--db", default=".fullsend/review-memory.db", help="Database path" + ) args = parser.parse_args() diff --git a/0023-review-cache/scripts/store.py b/0023-review-cache/scripts/store.py index 4763b23..e7ed85c 100644 --- a/0023-review-cache/scripts/store.py +++ b/0023-review-cache/scripts/store.py @@ -5,18 +5,18 @@ Used by save.py, load.py, and other scripts. """ -import sqlite3 import hashlib import re -from dataclasses import dataclass, asdict -from typing import List, Optional +import sqlite3 +from dataclasses import dataclass from datetime import datetime -import json +from typing import List, Optional @dataclass class Finding: """A single review finding with lifecycle tracking.""" + # Identity file_path: str function_name: str @@ -75,7 +75,7 @@ def to_dict(self) -> dict: } @classmethod - def from_dict(cls, data: dict) -> 'Finding': + def from_dict(cls, data: dict) -> "Finding": """Create Finding from dictionary.""" return cls( file_path=data.get("file", ""), @@ -181,14 +181,15 @@ def save_finding(self, finding: Finding): # Check if exists existing = self.conn.execute( "SELECT first_seen_sha, created_at FROM findings WHERE semantic_key = ?", - (finding.semantic_key,) + (finding.semantic_key,), ).fetchone() now = datetime.now().isoformat() if existing: # Update: preserve first_seen and created_at - self.conn.execute(""" + self.conn.execute( + """ UPDATE findings SET last_seen_sha = ?, status = ?, @@ -199,69 +200,79 @@ def save_finding(self, finding: Finding): code_hash = ?, updated_at = ? WHERE semantic_key = ? - """, ( - finding.last_seen_sha, - finding.status, - finding.line_number, - sanitize_text(finding.description, max_length=1000), - sanitize_text(finding.remediation, max_length=1000), - finding.code_snippet, - finding.code_hash, - now, - finding.semantic_key - )) + """, + ( + finding.last_seen_sha, + finding.status, + finding.line_number, + sanitize_text(finding.description, max_length=1000), + sanitize_text(finding.remediation, max_length=1000), + finding.code_snippet, + finding.code_hash, + now, + finding.semantic_key, + ), + ) else: # Insert new finding - self.conn.execute(""" + self.conn.execute( + """ INSERT INTO findings VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - finding.semantic_key, - finding.pr_number, - finding.file_path, - finding.function_name, - finding.line_number, - finding.category, - finding.severity, - sanitize_text(finding.description, max_length=1000), - sanitize_text(finding.remediation, max_length=1000), - finding.code_snippet, - finding.code_hash, - finding.status, - finding.first_seen_sha, - finding.last_seen_sha, - now, # created_at - now # updated_at - )) + """, + ( + finding.semantic_key, + finding.pr_number, + finding.file_path, + finding.function_name, + finding.line_number, + finding.category, + finding.severity, + sanitize_text(finding.description, max_length=1000), + sanitize_text(finding.remediation, max_length=1000), + finding.code_snippet, + finding.code_hash, + finding.status, + finding.first_seen_sha, + finding.last_seen_sha, + now, # created_at + now, # updated_at + ), + ) self.conn.commit() def get_pr_findings(self, pr_number: int) -> List[Finding]: """Get all findings for a PR.""" - cursor = self.conn.execute(""" + cursor = self.conn.execute( + """ SELECT * FROM findings WHERE pr_number = ? ORDER BY severity DESC, file_path, line_number - """, (pr_number,)) + """, + (pr_number,), + ) findings = [] for row in cursor.fetchall(): finding = Finding( - file_path=row['file_path'], - function_name=row['function_name'], - category=row['category'], - severity=row['severity'], - description=row['description'], - remediation=row['remediation'] or "", - code_snippet=row['code_snippet'] or "", - line_number=row['line_number'] or 0, - pr_number=row['pr_number'], - first_seen_sha=row['first_seen_sha'], - last_seen_sha=row['last_seen_sha'], - status=row['status'] + file_path=row["file_path"], + function_name=row["function_name"], + category=row["category"], + severity=row["severity"], + description=row["description"], + remediation=row["remediation"] or "", + code_snippet=row["code_snippet"] or "", + line_number=row["line_number"] or 0, + pr_number=row["pr_number"], + first_seen_sha=row["first_seen_sha"], + last_seen_sha=row["last_seen_sha"], + status=row["status"], ) findings.append(finding) return findings - def dismiss_finding(self, semantic_key: str, pr_number: int, reason: str, approved_by: str = "human"): + def dismiss_finding( + self, semantic_key: str, pr_number: int, reason: str, approved_by: str = "human" + ): """ Dismiss a finding with sanitized reasoning. @@ -269,26 +280,32 @@ def dismiss_finding(self, semantic_key: str, pr_number: int, reason: str, approv """ sanitized_reason = sanitize_text(reason, max_length=500) - self.conn.execute(""" + self.conn.execute( + """ INSERT OR REPLACE INTO intentional_exceptions VALUES (?, ?, ?, ?, ?) - """, ( - semantic_key, - pr_number, - sanitized_reason, - approved_by, - datetime.now().isoformat() - )) + """, + ( + semantic_key, + pr_number, + sanitized_reason, + approved_by, + datetime.now().isoformat(), + ), + ) self.conn.commit() def get_dismissal_reason(self, semantic_key: str, pr_number: int) -> Optional[str]: """Get dismissal reason for a finding (PR-scoped).""" - cursor = self.conn.execute(""" + cursor = self.conn.execute( + """ SELECT dismissal_reason FROM intentional_exceptions WHERE semantic_key = ? AND pr_number = ? - """, (semantic_key, pr_number)) + """, + (semantic_key, pr_number), + ) row = cursor.fetchone() - return row['dismissal_reason'] if row else None + return row["dismissal_reason"] if row else None def deduplicate_findings(self, new_findings: List[Finding], pr_number: int) -> dict: """ @@ -301,7 +318,7 @@ def deduplicate_findings(self, new_findings: List[Finding], pr_number: int) -> d 'resolved': [...] } """ - result = {'new': [], 'still_present': [], 'resolved': []} + result = {"new": [], "still_present": [], "resolved": []} # Get prior findings prior_findings = self.get_pr_findings(pr_number) @@ -317,19 +334,19 @@ def deduplicate_findings(self, new_findings: List[Finding], pr_number: int) -> d # Same finding from before finding.status = "still_present" finding.first_seen_sha = prior_map[key].first_seen_sha - result['still_present'].append(finding) + result["still_present"].append(finding) seen_keys.add(key) else: # New finding finding.status = "new" - result['new'].append(finding) + result["new"].append(finding) seen_keys.add(key) # Check for resolved findings for key, prior_finding in prior_map.items(): if key not in seen_keys: prior_finding.status = "resolved" - result['resolved'].append(prior_finding) + result["resolved"].append(prior_finding) return result @@ -360,7 +377,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): line_number=42, pr_number=123, first_seen_sha="abc123", - last_seen_sha="abc123" + last_seen_sha="abc123", ) store.save_finding(f) diff --git a/0023-review-cache/test-data/results/results.md b/0023-review-cache/test-data/results/results.md index 3333d38..2da9d4b 100644 --- a/0023-review-cache/test-data/results/results.md +++ b/0023-review-cache/test-data/results/results.md @@ -1,6 +1,6 @@ # Test Execution Results -**Date:** July 2, 2026 +**Date:** July 2, 2026 **Tests Run:** test_e2e.py, test_filter.sh, test_publication_policy.sh --- diff --git a/0023-review-cache/test_e2e.py b/0023-review-cache/test_e2e.py index 0716c68..254471d 100755 --- a/0023-review-cache/test_e2e.py +++ b/0023-review-cache/test_e2e.py @@ -18,7 +18,8 @@ import sys import tempfile from pathlib import Path -from scripts.store import ReviewMemoryStore, Finding + +from scripts.store import Finding, ReviewMemoryStore PASS = 0 FAIL = 0 @@ -41,15 +42,17 @@ def load_review_json(path: str) -> list[Finding]: findings = [] for fd in data["findings"]: - findings.append(Finding( - file_path=fd["file"], - function_name="", - category=fd["category"], - severity=fd["severity"], - description=fd["description"], - remediation=fd.get("remediation", ""), - line_number=fd.get("line", 0), - )) + findings.append( + Finding( + file_path=fd["file"], + function_name="", + category=fd["category"], + severity=fd["severity"], + description=fd["description"], + remediation=fd.get("remediation", ""), + line_number=fd.get("line", 0), + ) + ) return findings @@ -107,7 +110,7 @@ def test_kaexporter_three_round_review(): # ── ROUND 2: After fixes, bot re-reports + rephrases ────────── - print(f"\n ROUND 2: After developer fixes (commit f6e5d4)") + print("\n ROUND 2: After developer fixes (commit f6e5d4)") print("-" * 70) print("Developer fixed: WaitSumSeconds denominator, retryCount30d Reset,") print(" context.Background() (with AfterFunc pattern)") @@ -145,24 +148,38 @@ def test_kaexporter_three_round_review(): # The rephrased context.Background (line 327, bucket 320) is NEW # because the bot changed the line number to a different bucket check("R2 new count", len(result["new"]), 2) - check("R2 new contains rephrased context.Background", - "race-condition" in new_cats, True) - check("R2 new contains scrapeDurationGauge FP", - "race-condition" in new_cats, True) + check( + "R2 new contains rephrased context.Background", + "race-condition" in new_cats, + True, + ) + check( + "R2 new contains scrapeDurationGauge FP", + "race-condition" in new_cats, + True, + ) # coldStart, off-by-one, metric-label-injection: still_present check("R2 still_present count", len(result["still_present"]), 3) - check("R2 still_present has metric-label-injection", - "metric-label-injection" in still_cats, True) - check("R2 still_present has off-by-one", - "off-by-one" in still_cats, True) + check( + "R2 still_present has metric-label-injection", + "metric-label-injection" in still_cats, + True, + ) + check("R2 still_present has off-by-one", "off-by-one" in still_cats, True) # Fixed: original context.Background (line 221), retryCount30d, WaitSumSeconds check("R2 resolved count", len(result["resolved"]), 3) - check("R2 resolved has logic-error (WaitSumSeconds)", - "logic-error" in resolved_cats, True) - check("R2 resolved has error-handling-gap (retryCount30d)", - "error-handling-gap" in resolved_cats, True) + check( + "R2 resolved has logic-error (WaitSumSeconds)", + "logic-error" in resolved_cats, + True, + ) + check( + "R2 resolved has error-handling-gap (retryCount30d)", + "error-handling-gap" in resolved_cats, + True, + ) for status_group in ["new", "still_present", "resolved"]: for f in result[status_group]: @@ -171,54 +188,68 @@ def test_kaexporter_three_round_review(): # Verify first_seen preserved for coldStart all_findings = store.get_pr_findings(pr_number) coldstart = next( - (f for f in all_findings - if f.category == "race-condition" and f.line_number == 89), - None + ( + f + for f in all_findings + if f.category == "race-condition" and f.line_number == 89 + ), + None, ) if coldstart: - check("R2 coldStart first_seen preserved", - coldstart.first_seen_sha, sha1) + check( + "R2 coldStart first_seen preserved", coldstart.first_seen_sha, sha1 + ) # Human dismisses coldStart and off-by-one print("\n Developer dismisses false positives:") coldstart_key = coldstart.semantic_key if coldstart else "" store.dismiss_finding( - coldstart_key, pr_number, + coldstart_key, + pr_number, "Single-goroutine invariant: only runCollection goroutine " "reads coldStart, no concurrent access possible", - "senior-engineer" + "senior-engineer", + ) + print( + f" Dismissed coldStart: '{store.get_dismissal_reason(coldstart_key, pr_number)[:60]}...'" ) - print(f" Dismissed coldStart: '{store.get_dismissal_reason(coldstart_key, pr_number)[:60]}...'") offbyone = next( (f for f in all_findings if f.category == "off-by-one"), None ) if offbyone: store.dismiss_finding( - offbyone.semantic_key, pr_number, + offbyone.semantic_key, + pr_number, "dayOffset 0-29 inclusive = 30 calendar days. " "Buckets[29-30] would be index -1 (panic). Correct as-is.", - "senior-engineer" + "senior-engineer", + ) + print( + f" Dismissed off-by-one: '{store.get_dismissal_reason(offbyone.semantic_key, pr_number)[:60]}...'" ) - print(f" Dismissed off-by-one: '{store.get_dismissal_reason(offbyone.semantic_key, pr_number)[:60]}...'") # Also dismiss metric-label-injection mli = next( - (f for f in all_findings if f.category == "metric-label-injection"), None + (f for f in all_findings if f.category == "metric-label-injection"), + None, ) if mli: store.dismiss_finding( - mli.semantic_key, pr_number, + mli.semantic_key, + pr_number, "Existing mitigations sufficient: capped fetch limits, " "30-day expiry, fixed label dimensions. Enforce at scrape config level.", - "senior-engineer" + "senior-engineer", + ) + print( + f" Dismissed metric-label-injection: '{store.get_dismissal_reason(mli.semantic_key, pr_number)[:60]}...'" ) - print(f" Dismissed metric-label-injection: '{store.get_dismissal_reason(mli.semantic_key, pr_number)[:60]}...'") # ── ROUND 3: Dismissed issues STILL return ──────────────────── - print(f"\n ROUND 3: Bot re-reviews AGAIN (commit 9a8b7c)") + print("\n ROUND 3: Bot re-reviews AGAIN (commit 9a8b7c)") print("-" * 70) print("Despite dismissals, bot re-reports coldStart and metric-label-injection") print("Bot adds: collectMetrics nil-error (genuinely new)\n") @@ -245,8 +276,11 @@ def test_kaexporter_three_round_review(): # collectMetrics nil-error (line 298, bucket 290): genuinely new check("R3 new count", len(result["new"]), 1) - check("R3 new is collectMetrics error-handling-gap", - result["new"][0].category if result["new"] else "", "error-handling-gap") + check( + "R3 new is collectMetrics error-handling-gap", + result["new"][0].category if result["new"] else "", + "error-handling-gap", + ) # coldStart + metric-label-injection: still_present (despite dismissal) check("R3 still_present count", len(result["still_present"]), 2) @@ -262,31 +296,42 @@ def test_kaexporter_three_round_review(): # Verify dismissal reasons survive across rounds coldstart_r3 = next( - (f for f in result["still_present"] - if f.category == "race-condition"), - None + (f for f in result["still_present"] if f.category == "race-condition"), + None, ) if coldstart_r3: - reason = store.get_dismissal_reason(coldstart_r3.semantic_key, pr_number) - check("R3 coldStart dismissal reason preserved", - reason is not None, True) - print(f"\n Dismissal reason still available for coldStart:") + reason = store.get_dismissal_reason( + coldstart_r3.semantic_key, pr_number + ) + check( + "R3 coldStart dismissal reason preserved", reason is not None, True + ) + print("\n Dismissal reason still available for coldStart:") print(f" '{reason[:70]}...'") mli_r3 = next( - (f for f in result["still_present"] - if f.category == "metric-label-injection"), - None + ( + f + for f in result["still_present"] + if f.category == "metric-label-injection" + ), + None, ) if mli_r3: reason = store.get_dismissal_reason(mli_r3.semantic_key, pr_number) - check("R3 metric-label-injection dismissal preserved", - reason is not None, True) + check( + "R3 metric-label-injection dismissal preserved", + reason is not None, + True, + ) # Verify first_seen_sha tracks all the way back to round 1 if coldstart_r3: - check("R3 coldStart first_seen still points to R1", - coldstart_r3.first_seen_sha, sha1) + check( + "R3 coldStart first_seen still points to R1", + coldstart_r3.first_seen_sha, + sha1, + ) for status_group in ["new", "still_present", "resolved"]: for f in result[status_group]: @@ -348,7 +393,7 @@ def test_semantic_key_stability(): line_number=89, pr_number=456, first_seen_sha="abc", - last_seen_sha="abc" + last_seen_sha="abc", ) f2 = Finding( @@ -360,14 +405,17 @@ def test_semantic_key_stability(): line_number=85, pr_number=456, first_seen_sha="abc", - last_seen_sha="def" + last_seen_sha="def", ) print(f" Finding 1: line {f1.line_number} -> key: {f1.semantic_key}") print(f" Finding 2: line {f2.line_number} -> key: {f2.semantic_key}") - check("Same key for lines 89 and 85 (both bucket 80)", - f1.semantic_key, f2.semantic_key) + check( + "Same key for lines 89 and 85 (both bucket 80)", + f1.semantic_key, + f2.semantic_key, + ) # Different bucket = different key (known limitation) f3 = Finding( @@ -379,7 +427,7 @@ def test_semantic_key_stability(): line_number=221, pr_number=456, first_seen_sha="abc", - last_seen_sha="abc" + last_seen_sha="abc", ) f4 = Finding( @@ -391,14 +439,17 @@ def test_semantic_key_stability(): line_number=327, pr_number=456, first_seen_sha="def", - last_seen_sha="def" + last_seen_sha="def", ) print(f"\n Finding 3: line {f3.line_number} -> key: {f3.semantic_key}") print(f" Finding 4: line {f4.line_number} -> key: {f4.semantic_key}") - check("Different key for lines 221 vs 327 (bucket 220 vs 320)", - f3.semantic_key != f4.semantic_key, True) + check( + "Different key for lines 221 vs 327 (bucket 220 vs 320)", + f3.semantic_key != f4.semantic_key, + True, + ) print("\n Note: The rephrased context.Background finding at line 327") print(" gets a different key. This is a known limitation of bucket-based") diff --git a/0023-review-cache/test_results.md b/0023-review-cache/test_results.md index ffd0df1..e645650 100644 --- a/0023-review-cache/test_results.md +++ b/0023-review-cache/test_results.md @@ -224,16 +224,16 @@ test-data/review/ ### What Was Proven -✅ **Architecture is sound** - 5-layer separation of state from presentation works -✅ **Implementation is correct** - Core logic passes all checks -✅ **Pipeline integration works** - Agent stays context-free, post-scripts handle memory -✅ **Publication policy works** - 100% duplicate elimination on synthetic data +✅ **Architecture is sound** - 5-layer separation of state from presentation works +✅ **Implementation is correct** - Core logic passes all checks +✅ **Pipeline integration works** - Agent stays context-free, post-scripts handle memory +✅ **Publication policy works** - 100% duplicate elimination on synthetic data ### What Needs Validation -⚠️ **Real data testing** - Semantic keys need validation on actual agent output -⚠️ **Edge case handling** - Rebases, force-pushes, refactors need real-world testing -⚠️ **Dedup accuracy** - Measure actual rates on production code reviews +⚠️ **Real data testing** - Semantic keys need validation on actual agent output +⚠️ **Edge case handling** - Rebases, force-pushes, refactors need real-world testing +⚠️ **Dedup accuracy** - Measure actual rates on production code reviews ### Recommendation From 05bd450eb9ec986a27df195d6c5f10a7bd3df488 Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Thu, 2 Jul 2026 17:18:57 -0400 Subject: [PATCH 6/6] fix: address review findings from PR #36 - extract_h1() now skips YAML frontmatter to avoid matching comments - Fix stale EXPERIMENT.md self-reference in 0006 file tree - Fix stale review-cache-experiment/ directory name in 0023 file tree - Remove dead EXPERIMENT.md fallback from both lint scripts Assisted-by: Claude Signed-off-by: Wayne Sun --- 0006-code-agent-evaluation/README.md | 2 +- 0023-review-cache/README.md | 2 +- hack/lint-experiment-frontmatter | 17 +++++++++++++---- hack/lint-experiment-numbers | 2 -- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/0006-code-agent-evaluation/README.md b/0006-code-agent-evaluation/README.md index 68a04c4..d8f6093 100644 --- a/0006-code-agent-evaluation/README.md +++ b/0006-code-agent-evaluation/README.md @@ -853,7 +853,7 @@ V8 (hybrid — proposed in PR #189): ``` experiments/code-agent-evaluation/ -├── EXPERIMENT.md # This document +├── README.md # This document ├── RECOMMENDATION.md # Action summary for PR #189 ├── variants/ │ ├── VARIANTS.md # Index of all variants with git refs diff --git a/0023-review-cache/README.md b/0023-review-cache/README.md index 1478559..8d167a8 100644 --- a/0023-review-cache/README.md +++ b/0023-review-cache/README.md @@ -212,7 +212,7 @@ Total integration: ~40 lines of bash ## File Structure ``` -review-cache-experiment/ +0023-review-cache/ ├── scripts/ # Production code (post-scripts only) │ ├── store.py # Layer 1: SQLite storage │ ├── save.py # Layer 2: Deduplication diff --git a/hack/lint-experiment-frontmatter b/hack/lint-experiment-frontmatter index b87828b..4a5b28e 100755 --- a/hack/lint-experiment-frontmatter +++ b/hack/lint-experiment-frontmatter @@ -55,7 +55,7 @@ def find_experiment_md(entry_path): """Find the markdown file for an experiment entry.""" if os.path.isfile(entry_path) and entry_path.endswith(".md"): return entry_path - for candidate in ["README.md", "EXPERIMENT.md"]: + for candidate in ["README.md"]: path = os.path.join(entry_path, candidate) if os.path.isfile(path): return path @@ -63,10 +63,19 @@ def find_experiment_md(entry_path): def extract_h1(filepath): - """Extract the first H1 heading from a markdown file.""" + """Extract the first H1 heading from a markdown file, skipping frontmatter.""" + in_frontmatter = False with open(filepath, encoding="utf-8") as f: - for line in f: - m = re.match(r"^#\s+(.+)$", line.rstrip()) + for i, line in enumerate(f): + stripped = line.rstrip() + if i == 0 and stripped == "---": + in_frontmatter = True + continue + if in_frontmatter: + if stripped == "---": + in_frontmatter = False + continue + m = re.match(r"^#\s+(.+)$", stripped) if m: return m.group(1) return None diff --git a/hack/lint-experiment-numbers b/hack/lint-experiment-numbers index e03d2f1..ccd0ca6 100755 --- a/hack/lint-experiment-numbers +++ b/hack/lint-experiment-numbers @@ -82,8 +82,6 @@ for entry in "${experiments[@]}"; do md_file="$entry" elif [[ -d "$entry" && -f "$entry/README.md" ]]; then md_file="$entry/README.md" - elif [[ -d "$entry" && -f "$entry/EXPERIMENT.md" ]]; then - md_file="$entry/EXPERIMENT.md" fi if [[ -z "$md_file" ]]; then