From 4dafbfcf5fdb731a6a297bb5cb794f8654fda4b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 17:16:35 +0000 Subject: [PATCH] Reduce cross-branch friction for concurrent development Three changes aimed at many branches being developed and merged in parallel: - Split the single CLI-surface snapshot module (one 1455-line .ambr that conflicted whenever two branches touched any command output) into per-group help modules (test_snapshots_help_{build,run,tools,history,account}.py) plus test_snapshots_errors.py, so each group's goldens live in their own .ambr file. tests/_snapshot_surface.py holds the HELP_GROUPS partition and shared helpers; test_snapshots_help_groups.py guards that the partition stays complete and disjoint, preserving the old file's "every command must have a help snapshot" property. The 45 goldens are byte-identical, just redistributed. - Baseline check.sh's escape-hatch diff and Any/cast count gates on the merge-base with origin/main instead of the origin/main tip (matching what diff-cover and the mutation gate already do), so an unrelated merge to main that lowers a baseline count can no longer fail an in-flight branch that added nothing. - Add merge_group triggers to ci.yml and codeql.yml so the required checks run on merge-queue refs; enabling the queue in branch protection then ensures two individually-green PRs can't land a semantic conflict on main together. https://claude.ai/code/session_01PC6rFfEQ83Jo4F72ANv9rz --- .github/workflows/ci.yml | 3 + .github/workflows/codeql.yml | 2 + AGENTS.md | 2 +- pyproject.toml | 3 +- scripts/check.sh | 24 +- .../__snapshots__/test_snapshots_errors.ambr | 65 ++ .../test_snapshots_help_account.ambr | 251 ++++++ .../test_snapshots_help_build.ambr | 175 ++++ .../test_snapshots_help_history.ambr | 120 +++ ...hots.ambr => test_snapshots_help_run.ambr} | 821 ------------------ .../test_snapshots_help_tools.ambr | 215 +++++ tests/_snapshot_surface.py | 61 ++ tests/conftest.py | 10 + tests/test_cli_output_snapshots.py | 89 -- tests/test_snapshots_errors.py | 47 + tests/test_snapshots_help_account.py | 25 + tests/test_snapshots_help_build.py | 25 + tests/test_snapshots_help_groups.py | 25 + tests/test_snapshots_help_history.py | 25 + tests/test_snapshots_help_run.py | 25 + tests/test_snapshots_help_tools.py | 25 + 21 files changed, 1118 insertions(+), 920 deletions(-) create mode 100644 tests/__snapshots__/test_snapshots_errors.ambr create mode 100644 tests/__snapshots__/test_snapshots_help_account.ambr create mode 100644 tests/__snapshots__/test_snapshots_help_build.ambr create mode 100644 tests/__snapshots__/test_snapshots_help_history.ambr rename tests/__snapshots__/{test_cli_output_snapshots.ambr => test_snapshots_help_run.ambr} (56%) create mode 100644 tests/__snapshots__/test_snapshots_help_tools.ambr create mode 100644 tests/_snapshot_surface.py delete mode 100644 tests/test_cli_output_snapshots.py create mode 100644 tests/test_snapshots_errors.py create mode 100644 tests/test_snapshots_help_account.py create mode 100644 tests/test_snapshots_help_build.py create mode 100644 tests/test_snapshots_help_groups.py create mode 100644 tests/test_snapshots_help_history.py create mode 100644 tests/test_snapshots_help_run.py create mode 100644 tests/test_snapshots_help_tools.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f19ae03b..1f8541d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,9 @@ on: pull_request: branches: [main] types: [opened, reopened, ready_for_review, synchronize] + merge_group: # Merge-queue runs: validate the queued merge result so two + # PRs that are each green against an older main can't land a + # semantic conflict together. push: branches: [main] # PRs are covered by pull_request (incl. synchronize); # scoping push to main avoids double-running every PR commit. diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 553669f6..8e12dbf1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -3,6 +3,8 @@ name: CodeQL on: pull_request: branches: [main] + merge_group: # Merge-queue runs, so a required CodeQL check doesn't + # block queued merges (mirrors ci.yml). push: branches: [main] # PRs are covered by pull_request; scoping push to main # avoids double-running every PR commit (mirrors ci.yml). diff --git a/AGENTS.md b/AGENTS.md index d38fc0b0..38b127f9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,7 +65,7 @@ uv run pytest -m install # installs each init template's requirements fo `check.sh` runs the default suite with a **90% branch-coverage gate** (`--cov-fail-under=90`). New code generally needs tests to clear that gate. -CLI output is pinned by **syrupy snapshot tests** (`tests/__snapshots__/*.ambr`). Changing help text, tables, or rendered output will fail those tests until you regenerate them with `uv run pytest --snapshot-update` and commit the updated `.ambr` files. The auto-format hook only touches `*.py`, and pre-commit's whitespace fixers deliberately skip `tests/__snapshots__/` (syrupy's indentation must stay byte-for-byte), so never hand-edit a snapshot — always regenerate. +CLI output is pinned by **syrupy snapshot tests** (`tests/__snapshots__/*.ambr`). Changing help text, tables, or rendered output will fail those tests until you regenerate them with `uv run pytest --snapshot-update` and commit the updated `.ambr` files. The auto-format hook only touches `*.py`, and pre-commit's whitespace fixers deliberately skip `tests/__snapshots__/` (syrupy's indentation must stay byte-for-byte), so never hand-edit a snapshot — always regenerate. The `--help` goldens are split per command group (`tests/test_snapshots_help_.py`) so concurrent branches touching different commands regenerate *different* `.ambr` files; a new top-level command must be added to `HELP_GROUPS` in `tests/_snapshot_surface.py` (the partition guard in `tests/test_snapshots_help_groups.py` fails until it is). The post-edit hook (`.claude/settings.json`) runs `ruff check --fix --unfixable F401` + `ruff format` on every edited `*.py`. `--unfixable F401` means a just-added import is **not** auto-deleted while it's momentarily unused — so adding an import in one edit and its usage in the next is safe. The flip side: a genuinely unused import survives the hook and only fails at `ruff check` in the gate, so still prefer making the import and its first usage land in the same edit. diff --git a/pyproject.toml b/pyproject.toml index 8d034daa..9074834a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -276,4 +276,5 @@ DEP002 = ["fastapi", "python-dotenv", "python-multipart", "uvicorn"] # coverage is read by scripts/mutation_gate.py and click by # scripts/generated_code_compile_gate.py (dev-only gates run from check.sh, never # shipped in the wheel), so deptry sees dev deps imported from non-test code. -DEP004 = ["fastapi", "httpx", "hypothesis", "pytest", "coverage", "click"] +# syrupy is imported by tests/_snapshot_surface.py (typing the snapshot fixture). +DEP004 = ["fastapi", "httpx", "hypothesis", "pytest", "coverage", "click", "syrupy"] diff --git a/scripts/check.sh b/scripts/check.sh index 690860f8..a51be1cd 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -209,10 +209,18 @@ fi echo "==> no new static-analysis escape hatches" # Existing escape hatches are tolerated for now; new ones must be refactored away or # justified by changing this gate deliberately. Broad noqa/type-ignore/no-cover are -# checked by added diff lines. `Any` and `cast(` are count-gated against origin/main -# so mechanical edits to existing uses don't fail, but net-new uses do. +# checked by added diff lines. `Any` and `cast(` are count-gated against the +# merge-base with origin/main so mechanical edits to existing uses don't fail, but +# net-new uses do. if git rev-parse --verify --quiet origin/main >/dev/null; then - escape_hatches="$(git diff -U0 origin/main -- aai_cli tests \ + # Diff and count against the MERGE-BASE, not the origin/main tip (which the + # mutation gate and diff-cover already do). With many concurrent branches, + # main moves constantly: a tip-based baseline makes this gate fail on a branch + # the moment an unrelated merge lowers the count (e.g. removes an `Any`), even + # though the branch itself added nothing. The merge-base only moves when the + # branch itself rebases. + gate_base="$(git merge-base origin/main HEAD || echo origin/main)" + escape_hatches="$(git diff -U0 "$gate_base" -- aai_cli tests \ | rg '^\+.*(# type: ignore|# noqa|pragma: no cover)' || true)" if [[ -n "$escape_hatches" ]]; then printf '%s\n' "$escape_hatches" @@ -226,7 +234,7 @@ if git rev-parse --verify --quiet origin/main >/dev/null; then # env-gated marker suites (e2e/install) and live on origin/main, so they aren't # added diff lines and don't trip this; a genuinely-needed new one must update this # gate deliberately. Scoped to tests/ — production sleeps are fine. - test_shortcuts="$(git diff -U0 origin/main -- tests \ + test_shortcuts="$(git diff -U0 "$gate_base" -- tests \ | rg '^\+.*(pytest\.skip\(|pytest\.xfail\(|@pytest\.mark\.(skip|xfail)|\btime\.sleep\()' || true)" if [[ -n "$test_shortcuts" ]]; then printf '%s\n' "$test_shortcuts" @@ -234,17 +242,17 @@ if git rev-parse --verify --quiet origin/main >/dev/null; then exit 1 fi - base_any_count="$({ git grep -n "Any" origin/main -- aai_cli tests || true; } | wc -l | tr -d '[:space:]')" + base_any_count="$({ git grep -n "Any" "$gate_base" -- aai_cli tests || true; } | wc -l | tr -d '[:space:]')" work_any_count="$({ rg -n "Any" aai_cli tests || true; } | wc -l | tr -d '[:space:]')" if (( work_any_count > base_any_count )); then - echo "New Any usage found: ${work_any_count} current vs ${base_any_count} on origin/main." + echo "New Any usage found: ${work_any_count} current vs ${base_any_count} at the merge-base with origin/main." exit 1 fi - base_cast_count="$({ git grep -n "cast(" origin/main -- aai_cli tests || true; } | wc -l | tr -d '[:space:]')" + base_cast_count="$({ git grep -n "cast(" "$gate_base" -- aai_cli tests || true; } | wc -l | tr -d '[:space:]')" work_cast_count="$({ rg -n "cast\\(" aai_cli tests || true; } | wc -l | tr -d '[:space:]')" if (( work_cast_count > base_cast_count )); then - echo "New cast() usage found: ${work_cast_count} current vs ${base_cast_count} on origin/main." + echo "New cast() usage found: ${work_cast_count} current vs ${base_cast_count} at the merge-base with origin/main." exit 1 fi else diff --git a/tests/__snapshots__/test_snapshots_errors.ambr b/tests/__snapshots__/test_snapshots_errors.ambr new file mode 100644 index 00000000..e2fcb025 --- /dev/null +++ b/tests/__snapshots__/test_snapshots_errors.ambr @@ -0,0 +1,65 @@ +# serializer version: 1 +# name: test_error_human_render_matches_snapshot[api_no_suggestion] + ''' + Error: Transcription request failed: boom. + + ''' +# --- +# name: test_error_human_render_matches_snapshot[not_authenticated] + ''' + Error: You're not signed in. + Suggestion: Run 'assembly onboard' for guided setup, 'assembly login' if you + have an account, or set ASSEMBLYAI_API_KEY. + + ''' +# --- +# name: test_error_human_render_matches_snapshot[plain_no_suggestion] + ''' + Error: Something went wrong. + + ''' +# --- +# name: test_error_human_render_matches_snapshot[rejected_key] + ''' + Error: Your API key was rejected. + Suggestion: Run 'assembly login' with a valid key, or set ASSEMBLYAI_API_KEY. + + ''' +# --- +# name: test_error_human_render_matches_snapshot[usage_with_suggestion] + ''' + Error: Unknown voice 'nope'. + Suggestion: Run 'assembly agent --list-voices' to see the options. + + ''' +# --- +# name: test_error_json_render_matches_snapshot[api_no_suggestion] + ''' + {"error": {"type": "api_error", "message": "Transcription request failed: boom."}} + + ''' +# --- +# name: test_error_json_render_matches_snapshot[not_authenticated] + ''' + {"error": {"type": "not_authenticated", "message": "You're not signed in.", "suggestion": "Run 'assembly onboard' for guided setup, 'assembly login' if you have an account, or set ASSEMBLYAI_API_KEY."}} + + ''' +# --- +# name: test_error_json_render_matches_snapshot[plain_no_suggestion] + ''' + {"error": {"type": "error", "message": "Something went wrong."}} + + ''' +# --- +# name: test_error_json_render_matches_snapshot[rejected_key] + ''' + {"error": {"type": "not_authenticated", "message": "Your API key was rejected.", "suggestion": "Run 'assembly login' with a valid key, or set ASSEMBLYAI_API_KEY."}} + + ''' +# --- +# name: test_error_json_render_matches_snapshot[usage_with_suggestion] + ''' + {"error": {"type": "usage_error", "message": "Unknown voice 'nope'.", "suggestion": "Run 'assembly agent --list-voices' to see the options."}} + + ''' +# --- diff --git a/tests/__snapshots__/test_snapshots_help_account.ambr b/tests/__snapshots__/test_snapshots_help_account.ambr new file mode 100644 index 00000000..d5f8dbba --- /dev/null +++ b/tests/__snapshots__/test_snapshots_help_account.ambr @@ -0,0 +1,251 @@ +# serializer version: 1 +# name: test_command_help_matches_snapshot[audit] + ''' + + Usage: assembly audit [OPTIONS] + + List recent audit-log entries for your account. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --limit INTEGER RANGE [x>=1] How many entries to show. │ + │ [default: 20] │ + │ --action TEXT Filter by raw action name. │ + │ --resource TEXT Filter by raw resource type. │ + │ --include-logins Show successful login │ + │ events. │ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Recent audit-log entries + $ assembly audit + Show more entries + $ assembly audit --limit 100 + Include login events + $ assembly audit --include-logins + Filter by action + $ assembly audit --action token.create + Filter by resource, as JSON + $ assembly audit --resource token --json + + + + ''' +# --- +# name: test_command_help_matches_snapshot[balance] + ''' + + Usage: assembly balance [OPTIONS] + + Show your remaining account balance. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Show your remaining balance + $ assembly balance + Get the raw cents for scripting + $ assembly balance --json | jq '.balance_in_cents' + + + + ''' +# --- +# name: test_command_help_matches_snapshot[keys_create] + ''' + + Usage: assembly keys create [OPTIONS] + + Create a new API key. Prints the key value once — copy it now. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ * --name TEXT A label for the new key. [required] │ + │ --project INTEGER Project id to create the key in (defaults to │ + │ your first). │ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Create a key in your default project + $ assembly keys create --name ci-pipeline + Create a key in a specific project + $ assembly keys create --name prod --project 7 + Capture the new key into an env var + $ export ASSEMBLYAI_API_KEY=$(assembly keys create --name ci --json | jq -r + '.api_key') + + + + ''' +# --- +# name: test_command_help_matches_snapshot[keys_list] + ''' + + Usage: assembly keys list [OPTIONS] + + List API keys across your projects (keys shown masked). + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + List your API keys (masked) + $ assembly keys list + As JSON for scripting + $ assembly keys list --json + Get key ids to use with rename + $ assembly keys list --json | jq '.[].id' + + + + ''' +# --- +# name: test_command_help_matches_snapshot[keys_rename] + ''' + + Usage: assembly keys rename [OPTIONS] TOKEN_ID NEW_NAME + + Rename an existing API key. + + ╭─ Arguments ──────────────────────────────────────────────────────────────────╮ + │ * token_id INTEGER The key id (see `assembly keys list`). │ + │ [required] │ + │ * new_name TEXT The new label. [required] │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Relabel a key (id from `assembly keys list`) + $ assembly keys rename 123 "prod" + + + + ''' +# --- +# name: test_command_help_matches_snapshot[limits] + ''' + + Usage: assembly limits [OPTIONS] + + Show your account's rate limits per service. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Show rate limits per service + $ assembly limits + As JSON for scripting + $ assembly limits --json + + + + ''' +# --- +# name: test_command_help_matches_snapshot[login] + ''' + + Usage: assembly login [OPTIONS] + + Authenticate via your browser; stores a CLI API key. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --api-key TEXT Provide key non-interactively. │ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Log in with your browser + $ assembly login + Log in non-interactively (CI) + $ assembly login --api-key sk_... + + + + ''' +# --- +# name: test_command_help_matches_snapshot[logout] + ''' + + Usage: assembly logout [OPTIONS] + + Clear stored credentials for the active profile. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Clear stored credentials for the active profile + $ assembly logout + + + + ''' +# --- +# name: test_command_help_matches_snapshot[usage] + ''' + + Usage: assembly usage [OPTIONS] + + Show usage over a date range (defaults to the last 30 days). + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --start TEXT Start date (YYYY-MM-DD). Default: 30d │ + │ ago. │ + │ --end TEXT End date (YYYY-MM-DD). Default: today. │ + │ --window TEXT Window size: 'day', 'week', or 'month'. │ + │ --include-zero,--all Include zero-usage windows (matches │ + │ --include-logins on `assembly audit`). │ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Usage over the last 30 days + $ assembly usage + A specific date range + $ assembly usage --start 2026-05-01 --end 2026-06-01 + Break spend down by month + $ assembly usage --window month + Total spend in cents for scripting + $ assembly usage --json | jq '[.usage_items[].line_items[].price] | add' + + + + ''' +# --- +# name: test_command_help_matches_snapshot[whoami] + ''' + + Usage: assembly whoami [OPTIONS] + + Show the active profile and whether its key is usable. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Show the active profile and whether its key works + $ assembly whoami + + + + ''' +# --- diff --git a/tests/__snapshots__/test_snapshots_help_build.ambr b/tests/__snapshots__/test_snapshots_help_build.ambr new file mode 100644 index 00000000..086558a9 --- /dev/null +++ b/tests/__snapshots__/test_snapshots_help_build.ambr @@ -0,0 +1,175 @@ +# serializer version: 1 +# name: test_command_help_matches_snapshot[deploy] + ''' + + Usage: assembly deploy [OPTIONS] + + Deploy the current project to Vercel (default), Railway, or Fly.io. + + Asks for confirmation first, then runs the target's CLI (`vercel deploy`, + `railway up`, or `fly launch`). Requires that target's CLI to be installed. + (Render deploys from a connected Git repo — see the project README.) + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --prod Deploy to production (Vercel only). │ + │ --vercel Deploy to Vercel (the default). │ + │ --railway Deploy to Railway. │ + │ --fly Deploy to Fly.io. │ + │ --yes -y Skip the confirmation prompt. │ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Deploy a preview to Vercel (asks first) + $ assembly deploy + Deploy to production on Vercel + $ assembly deploy --prod --yes + Deploy to Railway + $ assembly deploy --railway + Deploy to Fly.io + $ assembly deploy --fly + + + + ''' +# --- +# name: test_command_help_matches_snapshot[dev] + ''' + + Usage: assembly dev [OPTIONS] + + Launch the dev server for the app in the current directory. + + Run this from inside a project created by `assembly init`. It installs + dependencies + if needed, then starts the FastAPI server with live reload and opens the + browser. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --port INTEGER Local server port. [default: 3000] │ + │ --host TEXT Interface to bind. Loopback by default; pass │ + │ 0.0.0.0 to expose on your network. │ + │ [default: 127.0.0.1] │ + │ --no-open Launch, but don't open the browser. │ + │ --no-install Skip dependency install; launch directly. │ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Launch the app in the current directory + $ assembly dev + Use a specific port + $ assembly dev --port 8000 + Launch without opening a browser + $ assembly dev --no-open + Skip the dependency install step + $ assembly dev --no-install + + + + ''' +# --- +# name: test_command_help_matches_snapshot[init] + ''' + + Usage: assembly init [OPTIONS] [TEMPLATE] [DIRECTORY] + + Scaffold a new project from a template, then launch it. + + This is the starting point for creating an app — including a voice agent app + ('assembly init voice-agent'). The 'assembly agent' command only runs a live + mic + conversation and writes no code. + + ╭─ Arguments ──────────────────────────────────────────────────────────────────╮ + │ template [TEMPLATE] Template to scaffold: audio-transcription, │ + │ live-captions, voice-agent (omit to pick │ + │ interactively). │ + │ directory [DIRECTORY] Target directory (default: