diff --git a/.github/prompts/adversarial-review.md b/.github/prompts/adversarial-review.md new file mode 100644 index 0000000..86cf8de --- /dev/null +++ b/.github/prompts/adversarial-review.md @@ -0,0 +1,78 @@ +# Bash AST adversarial review + +You are an adversarial reviewer for `bash-ast`, a Rust CLI/library that uses GNU Bash's real parser through FFI to parse shell scripts into JSON AST and convert JSON AST back to bash. + +Your job is to add value beyond ordinary CI. Do not simply rerun the full test suite as your main contribution; the workflow has already captured baseline build/test logs for you. Instead, inspect the repository and the supplied context, identify parser behaviors worth challenging, and run a small number of targeted probes. + +## What to inspect first + +- `README.md`, `Cargo.toml`, `src/`, and relevant tests under `tests/`. +- `review-artifacts/pr-context.json` if present. +- `review-artifacts/base-diff.stat` and `review-artifacts/base-diff.patch` if present. +- `review-artifacts/build.log`, `review-artifacts/baseline-tests.log`, and status files if present. + +If this run is associated with a PR, extract 2-4 concrete, testable claims from the PR title/body/diff before running probes. If there is no PR context, pick high-risk parser/round-trip behaviors from the current checkout. + +## Probe guidance + +Prefer edge cases involving one or more of: + +- nested quotes and escaped newlines; +- command substitution and arithmetic expansion; +- heredocs and here-strings; +- process substitution; +- pipelines, negated pipelines, and lists; +- arrays and parameter expansion; +- case/select/for/while/function syntax; +- malformed syntax and graceful error handling; +- parse-to-JSON then `--to-bash` round trips. + +For each probe: + +1. Create temporary scripts/data only under `/tmp` or `review-artifacts/agent-probes/`. +2. Use the repository's actual binary/library/test harness whenever practical. The built CLI is usually `target/debug/bash-ast` after `cargo build`. +3. Capture concise evidence. If output is long, write full logs to `review-artifacts/agent-probes/` and summarize the relevant lines. +4. Decide whether the observed behavior supports or refutes the hypothesis. + +## Constraints + +- Do not modify repository source, tests, manifests, lockfiles, generated snapshots, or submodules. +- Do not install arbitrary dependencies. +- Do not run broad/unbounded commands that dump huge files or recursive listings. +- Do not use network access except GitHub context already provided by the workflow. +- Keep shell commands and outputs in the final response compact. +- If setup/build failures prevent runtime probes, perform source-level inspection and report `NEEDS_MORE_INVESTIGATION` with the best concrete blocker evidence. + +## Required final response format + +Return a concise human-readable review followed by a machine-readable JSON block between exact markers: + +`JSON_RESULT_START` + +```json +{ + "recommendation": "PASS|FAIL|NEEDS_MORE_INVESTIGATION", + "why": "One or two sentences explaining the recommendation and highest risk.", + "tests": [ + { + "title": "Short name", + "hypothesis": "What behavior was being tested", + "impact": "Why this matters if wrong", + "command": "Short command summary, not a giant script", + "output": "Concise observed output or pointer to artifact path", + "result": "PASS|FAIL", + "unitTestRecommendation": "What automated coverage should be added or why existing coverage is enough" + } + ], + "finalMessage": "Brief operator-facing summary" +} +``` + +`JSON_RESULT_END` + +Rules for the JSON block: + +- `recommendation` must be exactly `PASS`, `FAIL`, or `NEEDS_MORE_INVESTIGATION`. +- `tests` must contain at least one substantive probe or one clearly labeled blocker probe. +- Every test object must have non-empty string fields: `title`, `hypothesis`, `impact`, `command`, `output`, `result`, and `unitTestRecommendation`. +- Per-test `result` must be exactly `PASS` or `FAIL`. diff --git a/.github/scripts/render-adversarial-review-summary.py b/.github/scripts/render-adversarial-review-summary.py new file mode 100644 index 0000000..be6e77f --- /dev/null +++ b/.github/scripts/render-adversarial-review-summary.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Render a GitHub Actions summary from agent adversarial review output.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any + + +def read_text(path: Path | None) -> str: + if path is None or not path.exists(): + return "" + return path.read_text(encoding="utf-8", errors="replace") + + +def extract_json_blob(response: str) -> tuple[dict[str, Any] | None, str | None]: + marker_match = re.search( + r"JSON_RESULT_START\s*(?:```(?:json)?\s*)?(.*?)(?:```\s*)?JSON_RESULT_END", + response, + flags=re.IGNORECASE | re.DOTALL, + ) + candidates: list[str] = [] + if marker_match: + candidates.append(marker_match.group(1).strip()) + + candidates.extend(match.group(1).strip() for match in re.finditer(r"```json\s*(.*?)```", response, re.DOTALL | re.IGNORECASE)) + + for candidate in candidates: + try: + parsed = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + return parsed, None + + return None, "Could not find a valid JSON review block between JSON_RESULT_START and JSON_RESULT_END." + + +def normalize_tests(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, dict)] + + +def append_log_tail(lines: list[str], title: str, text: str, max_lines: int = 60) -> None: + if not text.strip(): + return + tail = "\n".join(text.rstrip().splitlines()[-max_lines:]) + lines.extend([ + f"### {title}", + "", + "```text", + tail, + "```", + "", + ]) + + +def render_summary(args: argparse.Namespace) -> str: + response = read_text(args.response) + build_log = read_text(args.build_log) + baseline_log = read_text(args.baseline_log) + build_status = read_text(args.build_status).strip() if args.build_status and args.build_status.exists() else "unknown" + baseline_status = read_text(args.baseline_status).strip() if args.baseline_status and args.baseline_status.exists() else "unknown" + review, warning = extract_json_blob(response) + + lines: list[str] = [ + "## Adversarial review", + "", + f"- **Build exit code:** `{build_status or 'unknown'}`", + f"- **Baseline test exit code:** `{baseline_status or 'unknown'}`", + ] + + if review is None: + lines.extend([ + "- **Recommendation:** `UNKNOWN`", + "", + f"> ⚠️ {warning}", + "", + ]) + else: + recommendation = str(review.get("recommendation", "UNKNOWN")) + why = str(review.get("why", "No rationale supplied.")) + final_message = str(review.get("finalMessage", "")) + tests = normalize_tests(review.get("tests")) + + lines.extend([ + f"- **Recommendation:** `{recommendation}`", + f"- **Why:** {why}", + ]) + if final_message: + lines.append(f"- **Final message:** {final_message}") + lines.extend(["", "### Structured probes", ""]) + + if not tests: + lines.extend(["No structured probes were parsed from the agent response.", ""]) + else: + for index, test in enumerate(tests, start=1): + title = str(test.get("title", f"Probe {index}")) + result = str(test.get("result", "UNKNOWN")) + lines.extend([ + f"#### {index}. {title} — `{result}`", + "", + f"- **Hypothesis:** {test.get('hypothesis', '')}", + f"- **Impact:** {test.get('impact', '')}", + f"- **Command:** `{test.get('command', '')}`", + f"- **Output:** {test.get('output', '')}", + f"- **Coverage recommendation:** {test.get('unitTestRecommendation', '')}", + "", + ]) + + lines.extend([ + "### Full agent response", + "", + "
", + "Expand raw response", + "", + "````text", + response[-12000:] if response else "(no agent response captured)", + "````", + "", + "
", + "", + ]) + + append_log_tail(lines, "Build log tail", build_log) + append_log_tail(lines, "Baseline test log tail", baseline_log) + + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--response", type=Path, required=True) + parser.add_argument("--build-log", type=Path) + parser.add_argument("--baseline-log", type=Path) + parser.add_argument("--build-status", type=Path) + parser.add_argument("--baseline-status", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(render_summary(args), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/adversarial-review.yml b/.github/workflows/adversarial-review.yml new file mode 100644 index 0000000..c267f7f --- /dev/null +++ b/.github/workflows/adversarial-review.yml @@ -0,0 +1,262 @@ +name: Adversarial review + +on: + workflow_dispatch: + inputs: + pr_number: + description: Optional pull request number to review + required: false + default: '' + ref: + description: Optional ref, branch, or SHA to review when pr_number is empty + required: false + default: '' + post_comment: + description: Post or update the sticky PR review comment + required: false + type: boolean + default: false + pull_request: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + pull-requests: read + issues: write + actions: read + +concurrency: + group: adversarial-review-${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.pr_number || github.event.inputs.ref || github.sha }} + cancel-in-progress: false + +env: + CARGO_TERM_COLOR: always + PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number || '' }} + BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + +jobs: + adversarial-review: + name: Adversarial review + # pull_request runs only for same-repository branches so repository/model secrets are not exposed to forked PR code. + if: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 45 + + steps: + - name: Checkout target + uses: actions/checkout@v6.0.2 + with: + submodules: recursive + fetch-depth: 0 + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event.inputs.ref || github.sha }} + + - name: Checkout workflow_dispatch PR + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number != '' }} + env: + GH_TOKEN: ${{ github.token }} + REQUESTED_PR: ${{ github.event.inputs.pr_number }} + run: | + set -euxo pipefail + gh pr checkout "$REQUESTED_PR" + git submodule update --init --recursive + echo "HEAD_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + echo "BASE_REF=$(gh pr view "$REQUESTED_PR" --json baseRefName --jq .baseRefName)" >> "$GITHUB_ENV" + + - name: Install OS dependencies + run: | + set -euxo pipefail + sudo apt-get update + sudo apt-get install -y libncurses-dev + + - name: Install Rust + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: stable + + - name: Setup Node.js for agent action + uses: actions/setup-node@v6 + with: + node-version: '25' + + - name: Build baseline binary + id: build + continue-on-error: true + run: | + set +e + mkdir -p review-artifacts + cargo build --verbose 2>&1 | tee review-artifacts/build.log + status=${PIPESTATUS[0]} + echo "$status" > review-artifacts/build-status.txt + exit "$status" + + - name: Run baseline tests + id: baseline_tests + continue-on-error: true + run: | + set +e + mkdir -p review-artifacts + cargo test --verbose -- --test-threads=1 2>&1 | tee review-artifacts/baseline-tests.log + status=${PIPESTATUS[0]} + echo "$status" > review-artifacts/baseline-test-status.txt + exit "$status" + + - name: Collect review context + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euxo pipefail + mkdir -p review-artifacts/agent-probes + git status --short > review-artifacts/git-status.txt + git log --oneline -n 20 > review-artifacts/recent-commits.txt + cargo metadata --no-deps --format-version 1 > review-artifacts/cargo-metadata.json || true + + git fetch origin "$BASE_REF" --depth=1 || true + if git rev-parse --verify "origin/$BASE_REF" >/dev/null 2>&1; then + git diff --stat "origin/$BASE_REF...HEAD" > review-artifacts/base-diff.stat || true + git diff --find-renames "origin/$BASE_REF...HEAD" > review-artifacts/base-diff.patch || true + else + : > review-artifacts/base-diff.stat + : > review-artifacts/base-diff.patch + fi + + if [ -n "$PR_NUMBER" ]; then + gh pr view "$PR_NUMBER" \ + --json number,title,author,body,baseRefName,headRefName,headRefOid,url,files,comments \ + > review-artifacts/pr-context.json || echo '{}' > review-artifacts/pr-context.json + gh pr diff "$PR_NUMBER" > review-artifacts/pr.diff || true + else + echo '{}' > review-artifacts/pr-context.json + : > review-artifacts/pr.diff + fi + + - name: Compose review prompt + id: compose_prompt + run: | + set -euo pipefail + delimiter="REVIEW_PROMPT_$(date +%s)_$$" + { + echo "prompt<<$delimiter" + cat .github/prompts/adversarial-review.md + echo + echo "## Workflow-provided context" + echo + echo "- Repository: $GITHUB_REPOSITORY" + echo "- Event: $GITHUB_EVENT_NAME" + echo "- PR number: ${PR_NUMBER:-none}" + echo "- Base ref: ${BASE_REF:-unknown}" + echo "- Head SHA: ${HEAD_SHA:-unknown}" + echo "- Build exit code: $(cat review-artifacts/build-status.txt 2>/dev/null || echo unknown)" + echo "- Baseline test exit code: $(cat review-artifacts/baseline-test-status.txt 2>/dev/null || echo unknown)" + echo + echo "Artifacts are available under ./review-artifacts/. Keep any additional probe artifacts under ./review-artifacts/agent-probes/." + echo "$delimiter" + } >> "$GITHUB_OUTPUT" + + - name: Run adversarial review + id: review_agent + continue-on-error: true + uses: cv/pi-action@main + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + output_mode: output + allowed_associations: OWNER,MEMBER,COLLABORATOR + prompt: ${{ steps.compose_prompt.outputs.prompt }} + pr_number: ${{ github.event.pull_request.number || github.event.inputs.pr_number || '' }} + timeout: '1800' + share_session: true + provider: ${{ vars.ADVERSARIAL_REVIEW_PROVIDER }} + model: ${{ vars.ADVERSARIAL_REVIEW_MODEL }} + api_key: ${{ secrets.ADVERSARIAL_REVIEW_API_KEY }} + provider_base_url: ${{ vars.ADVERSARIAL_REVIEW_PROVIDER_BASE_URL }} + provider_api: ${{ vars.ADVERSARIAL_REVIEW_PROVIDER_API }} + model_name: ${{ vars.ADVERSARIAL_REVIEW_MODEL_NAME }} + model_reasoning: ${{ vars.ADVERSARIAL_REVIEW_MODEL_REASONING }} + model_input: ${{ vars.ADVERSARIAL_REVIEW_MODEL_INPUT }} + model_context_window: ${{ vars.ADVERSARIAL_REVIEW_MODEL_CONTEXT_WINDOW }} + model_max_tokens: ${{ vars.ADVERSARIAL_REVIEW_MODEL_MAX_TOKENS }} + env: + NPM_CONFIG_IGNORE_SCRIPTS: 'true' + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + + - name: Persist agent response + if: always() + env: + AGENT_RESPONSE: ${{ steps.review_agent.outputs.response }} + AGENT_SUCCESS: ${{ steps.review_agent.outputs.success }} + AGENT_SHARE_URL: ${{ steps.review_agent.outputs.share_url }} + run: | + set -euo pipefail + mkdir -p review-artifacts + python3 - <<'PY' + import json + import os + from pathlib import Path + Path('review-artifacts/agent-response.md').write_text(os.environ.get('AGENT_RESPONSE', ''), encoding='utf-8') + Path('review-artifacts/agent-action-metadata.json').write_text( + json.dumps({ + 'success': os.environ.get('AGENT_SUCCESS', ''), + 'share_url': os.environ.get('AGENT_SHARE_URL', ''), + }, indent=2) + '\n', + encoding='utf-8', + ) + PY + + - name: Render review summary + if: always() + run: | + set -euxo pipefail + python3 .github/scripts/render-adversarial-review-summary.py \ + --response review-artifacts/agent-response.md \ + --build-log review-artifacts/build.log \ + --baseline-log review-artifacts/baseline-tests.log \ + --build-status review-artifacts/build-status.txt \ + --baseline-status review-artifacts/baseline-test-status.txt \ + --output review-artifacts/adversarial-review-summary.md + cat review-artifacts/adversarial-review-summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload review artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: adversarial-review-${{ github.run_id }}-${{ github.run_attempt }} + path: review-artifacts/** + if-no-files-found: warn + retention-days: 14 + + - name: Post or update sticky PR comment + if: ${{ always() && env.PR_NUMBER != '' }} + env: + GH_TOKEN: ${{ github.token }} + ADVERSARIAL_REVIEW_POST_COMMENTS: ${{ vars.ADVERSARIAL_REVIEW_POST_COMMENTS }} + run: | + set -euo pipefail + post_comment_input=$(jq -r '.inputs.post_comment // "false"' "$GITHUB_EVENT_PATH") + if [ "${ADVERSARIAL_REVIEW_POST_COMMENTS:-}" != "true" ] && [ "$post_comment_input" != "true" ]; then + echo "Sticky PR comment disabled; set ADVERSARIAL_REVIEW_POST_COMMENTS=true or workflow_dispatch post_comment=true to enable." + exit 0 + fi + + marker='' + body_file=$(mktemp) + { + echo "$marker" + echo "" + echo + cat review-artifacts/adversarial-review-summary.md + } > "$body_file" + + comment_id=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" --paginate \ + --jq ".[] | select(.body | contains(\"$marker\")) | .id" | tail -n 1) + + if [ -n "$comment_id" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$comment_id" -F "body=@$body_file" >/dev/null + else + gh pr comment "$PR_NUMBER" --body-file "$body_file" + fi