From 75df4723dd7417ddb009b41d20bdb4288ef4db0e Mon Sep 17 00:00:00 2001 From: Baruch Sadogursky Date: Thu, 16 Jul 2026 13:16:33 -0500 Subject: [PATCH 01/19] fix(ci): discover and run Python test suites run-tests.sh matched test_*.sh only, so test_compute_lift.py (22 tests) and test_stamp_changelog.py (10 tests) were never executed by CI. Both pass when run by hand, which is what made this dangerous: the modules looked covered while stamp-changelog.py sat untested in the publish path. Discover test_*.{sh,py} and dispatch by extension. Both suite kinds are self-driving via the entry-point guard file-hygiene.md requires, so the runner picks the interpreter and reads the exit code. Suite count 17 -> 19. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JgdWmv7KBj91nxbVjMsj7X --- CHANGELOG.md | 4 +++ scripts/run-tests.sh | 38 ++++++++++++++++++++----- scripts/tests/test_run_tests.sh | 49 +++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b3e3b8d..843e0fe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### Skills + +- **`run-tests.sh` discovered shell suites only — 32 Python tests never ran in CI** — the `test_*.sh` glob silently orphaned `test_compute_lift.py` (22 tests) and `test_stamp_changelog.py` (10 tests); both passed when run by hand, so the modules looked covered while `stamp-changelog.py` sat untested in the publish path. Discovery now matches `test_*.{sh,py}` and dispatches by extension (both suite kinds self-drive via the entry-point guard `rules/file-hygiene.md` requires). Suite count 17 → 19. Full motivation: PR #181. + ## 0.3.93 — 2026-07-17 ### Skills diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index ff0f31ad..2ea6a5a0 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Discover and run every module's bash unit-test suite, in one pass. +# Discover and run every module's unit-test suite, in one pass. # # rules/testing-standards.md mandates that every module ship tests, but # nothing executed them: the publish pipeline ran lint + skill-review + @@ -8,8 +8,14 @@ # on every PR (tests.yml) and again before publish (publish.yml), so a # failing suite blocks the merge and the release. # -# Discovers /**/tests/test_*.sh, runs each in its own subshell so -# one suite's `set`/`cd`/traps can't leak into the next. +# Discovers /**/tests/test_*.sh and /**/tests/test_*.py, and +# runs each in its own subshell so one suite's `set`/`cd`/traps can't leak +# into the next. Discovery covers BOTH languages deliberately: a `.sh`-only +# glob silently orphaned every Python suite in the repo — they looked +# covered while never executing in CI, including one wired into the publish +# path. Each suite self-drives (a shell harness prints its own summary and +# exits non-zero; a unittest module does the same via its entry-point +# guard), so dispatch is by extension and nothing else. # # Output contract (rules/script-delegation.md — structured stdout): # stdout: one JSON object, the run summary — @@ -26,6 +32,24 @@ set -euo pipefail +# Dispatch a suite to its interpreter by extension. Both suite kinds are +# self-driving — a shell harness counts its own failures and exits +# non-zero; a unittest module does the same through the entry-point guard +# `rules/file-hygiene.md` requires — so the runner only picks the +# interpreter and reads the exit code. An unrecognized extension is a +# discovery/dispatch mismatch, not a test failure: exit 2, not 1. +run_suite() { + local suite="$1" + case "$suite" in + *.sh) bash "$suite" ;; + *.py) python3 "$suite" ;; + *) + echo "run-tests: no interpreter for suite type: $suite" >&2 + return 2 + ;; + esac +} + # JSON string escaper. A Unix path may contain any byte except NUL and # '/', so escape the JSON-mandatory characters AND the C0 control set # (newline/tab/CR/etc.) rather than assuming paths are control-free. @@ -72,7 +96,7 @@ main() { # the redirection, and the `if !` catches it — a command substitution # can't be used because bash strips NUL bytes from its output. local tmplist; tmplist="$(mktemp)" - if ! find "$base" -type f -path '*/tests/test_*.sh' -print0 | sort -z > "$tmplist"; then + if ! find "$base" -type f -path '*/tests/*' \( -name 'test_*.sh' -o -name 'test_*.py' \) -print0 | sort -z > "$tmplist"; then rm -f "$tmplist" echo "run-tests: suite discovery (find) failed under $base" >&2 printf '{"suites":0,"passed":0,"failed":0,"failures":[],"error":%s}\n' \ @@ -88,9 +112,9 @@ main() { rm -f "$tmplist" if [[ ${#suites[@]} -eq 0 ]]; then - echo "run-tests: no test suites found under ${base}/**/tests/test_*.sh" >&2 + echo "run-tests: no test suites found under ${base}/**/tests/test_*.{sh,py}" >&2 printf '{"suites":0,"passed":0,"failed":0,"failures":[],"error":%s}\n' \ - "$(json_str "no test suites found under ${base}/**/tests/test_*.sh")" + "$(json_str "no test suites found under ${base}/**/tests/test_*.{sh,py}")" return 2 fi @@ -100,7 +124,7 @@ main() { local failed=() for s in "${suites[@]}"; do echo "▶ $s" >&2 - if bash "$s" >&2; then + if run_suite "$s" >&2; then echo " ✓ $s" >&2 else echo " ✗ $s" >&2 diff --git a/scripts/tests/test_run_tests.sh b/scripts/tests/test_run_tests.sh index 05f66742..dbc02902 100755 --- a/scripts/tests/test_run_tests.sh +++ b/scripts/tests/test_run_tests.sh @@ -30,6 +30,18 @@ add_suite() { printf '#!/usr/bin/env bash\nexit %s\n' "$exit_code" > "$dir/test_$name.sh" } +# A Python suite, shaped like the real ones: self-driving via the +# entry-point guard rules/file-hygiene.md requires. Discovery matched +# `test_*.sh` only, so every Python suite in the repo was orphaned — +# present, passing when run by hand, never executed by CI. +add_py_suite() { + local base="$1" name="$2" exit_code="$3" + local dir="$base/skills/$name/tests" + mkdir -p "$dir" + printf 'import sys\nif __name__ == "__main__":\n sys.exit(%s)\n' \ + "$exit_code" > "$dir/test_$name.py" +} + # Runs the runner; sets OUT (stdout), ERR (stderr), CODE (exit). invoke() { local base="$1" errf; errf="$(mktemp)" @@ -124,6 +136,43 @@ else fi rm -rf "$base" +# --- a Python suite is discovered and run at all --- +# Pre-fix this exits 2 ("no test suites found"): the .sh-only glob matched +# nothing, so a tree of passing Python tests read as an empty tree. +base="$(make_base)"; add_py_suite "$base" pyalpha 0 +invoke "$base" +if [[ "$CODE" == 0 ]] && [[ "$(jq -r .suites <<<"$OUT")" == 1 ]] \ + && [[ "$(jq -r .passed <<<"$OUT")" == 1 ]]; then + pass "python-only tree -> discovered, exit 0, suites=1" +else + fail "python discovery: code=$CODE out=$OUT" +fi +rm -rf "$base" + +# --- a failing Python suite reddens the run --- +# Discovery alone isn't enough: a suite that runs but whose exit code is +# dropped would count as passed and let a real regression ship green. +base="$(make_base)"; add_py_suite "$base" pyalpha 0; add_py_suite "$base" pydoomed 1 +invoke "$base" +if [[ "$CODE" == 1 ]] && [[ "$(jq -r .failed <<<"$OUT")" == 1 ]] \ + && [[ "$(jq -r '.failures[0]' <<<"$OUT")" == *"test_pydoomed.py" ]]; then + pass "failing python suite -> exit 1, named in failures" +else + fail "python failure: code=$CODE out=$OUT" +fi +rm -rf "$base" + +# --- shell and python suites are counted in one run --- +base="$(make_base)"; add_suite "$base" alpha 0; add_py_suite "$base" pybeta 0 +invoke "$base" +if [[ "$CODE" == 0 ]] && [[ "$(jq -r .suites <<<"$OUT")" == 2 ]] \ + && [[ "$(jq -r .passed <<<"$OUT")" == 2 ]]; then + pass "mixed sh+py tree -> both counted, suites=2" +else + fail "mixed discovery: code=$CODE out=$OUT" +fi +rm -rf "$base" + echo "" echo "run-tests.sh: ${PASS_COUNT} passed, ${FAIL_COUNT} failed" [[ $FAIL_COUNT -eq 0 ]] || exit 1 From 432b59b937c83f880283def8ae66c55ad2918493 Mon Sep 17 00:00:00 2001 From: Baruch Sadogursky Date: Thu, 16 Jul 2026 13:16:43 -0500 Subject: [PATCH 02/19] fix(release): extract the registry baseline capture into a script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SKILL.md had the operator paste PRE=$(tessl plugin info /

| grep "Latest Version" | awk '{print $NF}') which shares its parse with verify-publish-landed.sh but none of its hardening. Two failures it could not see: a tessl warning on stderr merged into the parsed text, and a parse miss silently yielding an empty PRE. The empty one is the dangerous case — it flows into the release contract's conjunct 2, where every version compares as "advanced past empty", so an unpublished release reports confirmed. capture-registry-baseline.sh emits {"version": ""} or exits 2. It never emits an empty baseline. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JgdWmv7KBj91nxbVjMsj7X --- CHANGELOG.md | 1 + skills/release/SKILL.md | 5 +- skills/release/capture-registry-baseline.sh | 93 +++++++++++++++ .../tests/test_capture_registry_baseline.sh | 109 ++++++++++++++++++ 4 files changed, 206 insertions(+), 2 deletions(-) create mode 100755 skills/release/capture-registry-baseline.sh create mode 100755 skills/release/tests/test_capture_registry_baseline.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 843e0fe6..0f5c58c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Skills - **`run-tests.sh` discovered shell suites only — 32 Python tests never ran in CI** — the `test_*.sh` glob silently orphaned `test_compute_lift.py` (22 tests) and `test_stamp_changelog.py` (10 tests); both passed when run by hand, so the modules looked covered while `stamp-changelog.py` sat untested in the publish path. Discovery now matches `test_*.{sh,py}` and dispatches by extension (both suite kinds self-drive via the entry-point guard `rules/file-hygiene.md` requires). Suite count 17 → 19. Full motivation: PR #181. +- **`release`: registry baseline was a pasted pipeline with no hardening** — `SKILL.md` had the operator paste `tessl plugin info | grep | awk`, sharing its parse with `verify-publish-landed.sh` but none of its guards: a tessl warning on stderr could poison the parse, and a parse miss silently yielded an empty `PRE` that satisfied the release contract's conjunct 2 vacuously (every version "advances past" empty), confirming an unpublished release. Extracted to `skills/release/capture-registry-baseline.sh`, which emits `{"version": ""}` or exits 2 — never an empty baseline. Full motivation: PR #181. ## 0.3.93 — 2026-07-17 diff --git a/skills/release/SKILL.md b/skills/release/SKILL.md index 48bb0c75..d0fd0057 100644 --- a/skills/release/SKILL.md +++ b/skills/release/SKILL.md @@ -124,10 +124,11 @@ skills/release/dismiss-stale-reviews.sh Run it once Step 5's poll shows every bot's latest verdict clean. It emits a JSON summary of what it dismissed and what it left active, exits non-zero on API failure, and is idempotent on re-run. Which reviews it dismisses and which it leaves is the script's decision contract — see `skills/release/dismiss-stale-reviews.sh` header, not restated here (`rules/script-as-black-box.md`). -Before merging, capture the registry baseline so the post-merge check has something to compare against: +Before merging, capture the registry baseline so the post-merge check has something to compare against. Exit 0 emits `{"version": ""}`; exit 2 is a tool-state error or an unparseable registry output — the script never emits an empty baseline, so a failure here stops the release rather than feeding conjunct 2 a value that passes vacuously: ```bash -PRE=$(tessl plugin info / | grep "Latest Version" | awk '{print $NF}') +PRE=$(skills/release/capture-registry-baseline.sh | jq -r '.version') \ + || { echo "Baseline capture failed — see stderr" >&2; exit 1; } ``` Pick the right cleanup path based on where you ran the skill from. diff --git a/skills/release/capture-registry-baseline.sh b/skills/release/capture-registry-baseline.sh new file mode 100755 index 00000000..74bcfdc6 --- /dev/null +++ b/skills/release/capture-registry-baseline.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Capture a tile's current registry version as the pre-merge baseline for +# the release contract's conjunct 2 (registry advanced past the baseline — +# see rules/ci-safety.md "Always Watch CI"). +# +# Why this is a script and not a pasted pipeline: skills/release/SKILL.md +# previously had the operator paste +# PRE=$(tessl plugin info /

| grep "Latest Version" | awk '{print $NF}') +# which shares its parse with verify-publish-landed.sh but none of its +# hardening. Two failure modes it could not see: +# 1. A tessl warning on stderr merged into the parsed text and misread +# the version. +# 2. A parse miss (the "Latest Version" line absent, e.g. the registry +# changed its output shape) silently yielded an empty PRE, which then +# went on to verify-publish-landed.sh AS the baseline — every version +# compares as "advanced past empty", so conjunct 2 passes vacuously +# and an unpublished release reports as confirmed. +# Same parse in two places, one unguarded, feeding the gate that is +# supposed to catch exactly this (rules/script-delegation.md — deterministic +# work belongs in a script, not a code block for the agent to reproduce). +# +# Usage: capture-registry-baseline.sh +# Out: one JSON object on stdout: {"version": ""} +# Exit: 0 on a successful parse; 2 on tool-state error (tessl unreachable +# or absent, unparseable output). There is no exit 1: an absent +# baseline is never a valid input to the conjunction, so this script +# fails loud rather than emitting a value the caller might use. + +set -euo pipefail + +json_str() { + local s="$1" + s="${s//\\/\\\\}"; s="${s//\"/\\\"}" + printf '"%s"' "$s" +} + +# Script-global, NOT a main() local: the EXIT trap fires after main() +# returns on the happy path, and a local would be out of scope by then — +# `set -u` turns the cleanup into an unbound-variable failure that reddens +# a successful run. (verify-publish-landed.sh survives the same pattern +# only because every one of its paths exits from inside main().) +ERR_FILE="" +# `return 0` is load-bearing, not decoration: the EXIT trap's final status +# leaks into the script's exit status, so a bare `[[ -n "$ERR_FILE" ]]` +# guard returning 1 (path never set, e.g. the usage-error path) silently +# rewrote a deliberate `exit 2` into `exit 1`. Cleanup reports on cleanup; +# it must never speak for the script's outcome. +cleanup() { + if [[ -n "$ERR_FILE" ]]; then + rm -f "$ERR_FILE" + fi + return 0 +} +trap cleanup EXIT + +main() { + if [[ $# -ne 2 ]]; then + echo "usage: capture-registry-baseline.sh " >&2 + exit 2 + fi + local workspace="$1" tile="$2" + + command -v tessl >/dev/null \ + || { echo "error: tessl CLI not found on PATH — install it ('npm i -g @tessl/cli') or add it to PATH, then re-run" >&2; exit 2; } + + ERR_FILE=$(mktemp) \ + || { echo "error: mktemp failed — cannot run capture-registry-baseline.sh without a writable TMPDIR" >&2; exit 2; } + + # Capture stdout and stderr separately, then parse stdout on its own. + # A mixed `2>&1` capture would let a tessl warning line reach the parse + # and be misread as the version. + local tessl_output + tessl_output=$(tessl plugin info "${workspace}/${tile}" 2>"$ERR_FILE") \ + || { local err; err=$(cat "$ERR_FILE"); echo "error: 'tessl plugin info ${workspace}/${tile}' failed: ${err} — verify (1) the workspace/plugin slug is correct, (2) you have network access to the registry, (3) 'tessl status' shows you are logged in, then re-run 'tessl plugin info ${workspace}/${tile}' directly to inspect the failure" >&2; exit 2; } + + # Parse in its own step so a miss is distinguishable from a tool failure. + # `grep` exits 1 on no-match, which `set -e` + `pipefail` would turn into + # a bare exit before the actionable diagnostic below fires; an explicit + # `if` around the pipeline keeps the failure observable without + # suppressing it. + local version="" + if ! version=$(printf '%s\n' "$tessl_output" | grep "Latest Version" | awk '{print $NF}'); then + version="" + fi + if [[ -z "$version" ]]; then + echo "error: could not parse 'Latest Version' from 'tessl plugin info ${workspace}/${tile}' output — the registry output shape may have changed; inspect it directly and update this parse (output was: ${tessl_output})" >&2 + exit 2 + fi + + printf '{"version":%s}\n' "$(json_str "$version")" +} + +[[ "${BASH_SOURCE[0]}" == "${0}" ]] && main "$@" diff --git a/skills/release/tests/test_capture_registry_baseline.sh b/skills/release/tests/test_capture_registry_baseline.sh new file mode 100755 index 00000000..3a3ac87a --- /dev/null +++ b/skills/release/tests/test_capture_registry_baseline.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Outcome-based tests for capture-registry-baseline.sh. +# +# The contract under test is the one the pasted pipeline in SKILL.md could +# not hold: a parse miss or a tool failure must exit 2 with a diagnostic, +# never emit an empty baseline. An empty baseline is the dangerous output — +# it flows into verify-publish-landed.sh's conjunct 2, where every version +# compares as "advanced past empty", so an unpublished release confirms. +# +# Approach: source the script (its main() guard prevents auto-run when +# sourced) and override `tessl` with a shell function returning fixture +# text. jq is not used by the script, so stdout is asserted literally. +# +# Run: bash skills/release/tests/test_capture_registry_baseline.sh +# Exit 0 on all-pass; non-zero with a per-test diagnostic on failure. + +set -uo pipefail + +SCRIPT="$(cd "$(dirname "$0")/.." && pwd)/capture-registry-baseline.sh" +[[ -f "$SCRIPT" && -r "$SCRIPT" ]] || { echo "fatal: capture-registry-baseline.sh not readable at $SCRIPT" >&2; exit 2; } + +FAIL_COUNT=0 +PASS_COUNT=0 +pass() { PASS_COUNT=$((PASS_COUNT + 1)); echo " pass: $1"; } +fail() { FAIL_COUNT=$((FAIL_COUNT + 1)); echo " FAIL: $1" >&2; } + +# Invoke the script as a subprocess with a stubbed `tessl` on PATH, so the +# real `set -euo pipefail` and exit codes are exercised end-to-end rather +# than the sourced-function approximation of them. +STUBDIR="$(mktemp -d)" +trap 'rm -rf "$STUBDIR"' EXIT +export PATH="$STUBDIR:$PATH" + +# MOCK_MODE selects the fixture the stub emits. +cat > "$STUBDIR/tessl" <<'STUB' +#!/usr/bin/env bash +case "${MOCK_MODE:-}" in + ok) + echo "Plugin jbaruch/coding-policy" + echo " Latest Version 0.3.91" + ;; + warning_on_stderr) + echo "warn: registry cache stale, refetching" >&2 + echo " Latest Version 0.3.91" + ;; + no_version_line) + echo "Plugin jbaruch/coding-policy" + echo " Description nothing useful here" + ;; + tessl_fails) + echo "error: not authenticated" >&2 + exit 1 + ;; + *) echo "stub tessl: unknown MOCK_MODE='${MOCK_MODE:-}'" >&2; exit 99 ;; +esac +STUB +chmod +x "$STUBDIR/tessl" + +invoke() { OUT="$(bash "$SCRIPT" jbaruch coding-policy 2>"$STUBDIR/err")"; CODE=$?; ERR="$(cat "$STUBDIR/err")"; } + +echo "capture-registry-baseline.sh tests" + +# --- happy path: version parsed, emitted as JSON --- +MOCK_MODE=ok invoke +if [[ "$CODE" == 0 ]] && [[ "$OUT" == '{"version":"0.3.91"}' ]]; then + pass "parses Latest Version -> {\"version\":\"0.3.91\"}" +else + fail "happy path: code=$CODE out=$OUT err=$ERR" +fi + +# --- a tessl warning on stderr must not poison the parse --- +# This is the failure the pasted `tessl plugin info | grep | awk` could not +# see: with stdout and stderr merged, the warning line reaches the parse. +MOCK_MODE=warning_on_stderr invoke +if [[ "$CODE" == 0 ]] && [[ "$OUT" == '{"version":"0.3.91"}' ]]; then + pass "stderr warning does not poison the parsed version" +else + fail "stderr warning: code=$CODE out=$OUT err=$ERR" +fi + +# --- parse miss: exit 2, diagnostic, and NO empty baseline on stdout --- +# The pasted pipeline yielded an empty PRE here, which then satisfied +# conjunct 2 vacuously. +MOCK_MODE=no_version_line invoke +if [[ "$CODE" == 2 ]] && [[ -z "$OUT" ]] && [[ "$ERR" == *"could not parse 'Latest Version'"* ]]; then + pass "parse miss -> exit 2, empty stdout, actionable diagnostic" +else + fail "parse miss: code=$CODE out=$OUT err=$ERR" +fi + +# --- tessl itself failing is exit 2 with the tool's stderr surfaced --- +MOCK_MODE=tessl_fails invoke +if [[ "$CODE" == 2 ]] && [[ -z "$OUT" ]] && [[ "$ERR" == *"not authenticated"* ]]; then + pass "tessl failure -> exit 2, tool stderr surfaced" +else + fail "tessl failure: code=$CODE out=$OUT err=$ERR" +fi + +# --- wrong arity is a usage error, not a silent default --- +OUT="$(bash "$SCRIPT" jbaruch 2>"$STUBDIR/err")"; CODE=$?; ERR="$(cat "$STUBDIR/err")" +if [[ "$CODE" == 2 ]] && [[ "$ERR" == *"usage:"* ]]; then + pass "wrong arity -> exit 2 with usage" +else + fail "arity: code=$CODE out=$OUT err=$ERR" +fi + +echo "" +echo "capture-registry-baseline.sh: ${PASS_COUNT} passed, ${FAIL_COUNT} failed" +[[ $FAIL_COUNT -eq 0 ]] || exit 1 From 7da59a424fb7acbef44d4e69cd23092c235a6ef4 Mon Sep 17 00:00:00 2001 From: Baruch Sadogursky Date: Thu, 16 Jul 2026 13:24:29 -0500 Subject: [PATCH 03/19] fix(release): capture before parse in the Step 7 wrappers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-aw caught that the baseline wrapper reintroduced the exact bug this PR removes. Without pipefail a pipeline's status is the last command's, and `jq -r` exits 0 on empty stdin — so piping capture-registry-baseline.sh into jq swallowed its exit 2 and produced the empty PRE the script exists to prevent. Verified: the `||` guard never fired. resolve-publish-run.sh had the same shape three lines away, with no guard at all — a failed resolve yielded an empty run_id and `gh run watch ""`, replacing the resolver's actionable diagnostic with a confusing one. Both now capture, then parse, then assert non-empty. Also correct the test header: it describes a sourced-function harness, but the test runs the script as a subprocess against a PATH stub. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JgdWmv7KBj91nxbVjMsj7X --- skills/release/SKILL.md | 14 ++++++++++++-- .../tests/test_capture_registry_baseline.sh | 9 ++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/skills/release/SKILL.md b/skills/release/SKILL.md index d0fd0057..4684eefb 100644 --- a/skills/release/SKILL.md +++ b/skills/release/SKILL.md @@ -126,9 +126,14 @@ Run it once Step 5's poll shows every bot's latest verdict clean. It emits a JSO Before merging, capture the registry baseline so the post-merge check has something to compare against. Exit 0 emits `{"version": ""}`; exit 2 is a tool-state error or an unparseable registry output — the script never emits an empty baseline, so a failure here stops the release rather than feeding conjunct 2 a value that passes vacuously: +Capture first, parse second — never in one pipeline. Without `set -o pipefail` a pipeline's status is the LAST command's, and `jq` exits 0 on empty stdin, so piping the script straight into `jq` would swallow its exit 2 and hand you the empty `PRE` this script exists to prevent: + ```bash -PRE=$(skills/release/capture-registry-baseline.sh | jq -r '.version') \ +baseline_json=$(skills/release/capture-registry-baseline.sh ) \ || { echo "Baseline capture failed — see stderr" >&2; exit 1; } +PRE=$(jq -r '.version' <<<"$baseline_json") \ + || { echo "Baseline JSON parse failed" >&2; exit 1; } +[[ -n "$PRE" ]] || { echo "Baseline capture returned an empty version" >&2; exit 1; } ``` Pick the right cleanup path based on where you ran the skill from. @@ -176,9 +181,14 @@ After merge — per `rules/ci-safety.md`'s Always Watch CI duty extended through - Verify the merge landed on main (`git pull --ff-only` succeeds; `git log -1 --oneline` shows the merge commit) - Watch the publish workflow to a terminal state. Bind to the merge commit SHA + the `push` event, never to "latest on main". The publish workflow may take several seconds to be enqueued after merge, so the run-id lookup polls until the run is listed (2s interval, 30s budget): + Same capture-then-parse discipline as the baseline above, for the same reason — piping the resolver into `jq` would swallow its exit code and hand `gh run watch` an empty run id, replacing the resolver's actionable diagnostic with a confusing one: + ```bash merge_sha=$(gh pr view --json mergeCommit --jq '.mergeCommit.oid') - run_id=$(skills/release/resolve-publish-run.sh "$merge_sha" "" | jq -r '.database_id') + run_json=$(skills/release/resolve-publish-run.sh "$merge_sha" "") \ + || { echo "Publish run not resolved — see stderr" >&2; exit 1; } + run_id=$(jq -r '.database_id' <<<"$run_json") + [[ -n "$run_id" && "$run_id" != "null" ]] || { echo "Resolver returned no run id" >&2; exit 1; } gh run watch "$run_id" ``` diff --git a/skills/release/tests/test_capture_registry_baseline.sh b/skills/release/tests/test_capture_registry_baseline.sh index 3a3ac87a..d041efbf 100755 --- a/skills/release/tests/test_capture_registry_baseline.sh +++ b/skills/release/tests/test_capture_registry_baseline.sh @@ -7,9 +7,12 @@ # it flows into verify-publish-landed.sh's conjunct 2, where every version # compares as "advanced past empty", so an unpublished release confirms. # -# Approach: source the script (its main() guard prevents auto-run when -# sourced) and override `tessl` with a shell function returning fixture -# text. jq is not used by the script, so stdout is asserted literally. +# Approach: run the script as a subprocess with a stub `tessl` executable +# placed first on PATH, so the real `set -euo pipefail`, the EXIT trap, and +# the exit codes are all exercised end-to-end — a sourced-function harness +# would approximate them and miss exactly the trap/exit-status defects this +# script hit in development. The stub selects its fixture via MOCK_MODE. +# jq is not used by the script, so stdout is asserted literally. # # Run: bash skills/release/tests/test_capture_registry_baseline.sh # Exit 0 on all-pass; non-zero with a per-test diagnostic on failure. From 5044d4b45316cc4cb29dc2815ad7bfa0d06b7c4f Mon Sep 17 00:00:00 2001 From: Baruch Sadogursky Date: Thu, 16 Jul 2026 15:04:54 -0500 Subject: [PATCH 04/19] fix(release): enforce the semver contract the baseline script promises MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-aw caught that capture-registry-baseline.sh documents stdout as {"version": ""} but accepted any non-empty last field on the "Latest Version" line. A registry outage printing "Latest Version unavailable" would emit {"version":"unavailable"} and hand verify-publish-landed.sh a baseline its version comparison cannot order — a registry-side outage silently becoming a bogus conjunct-2 verdict. A script that promises a contract and does not enforce it is the same defect this PR removes, one layer up. Validate the parsed token against the semver shape before printing. Fixtures cover an outage token, prose on the version line (awk's $NF makes prose look like a plausible single token), and a prerelease+build version that must still be accepted. Also guard jq in the resolver snippet (a jq failure surfaced as "Resolver returned no run id", hiding the root cause), and correct the CHANGELOG archive pointers from #181 to #183. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JgdWmv7KBj91nxbVjMsj7X --- CHANGELOG.md | 4 +- skills/release/SKILL.md | 3 +- skills/release/capture-registry-baseline.sh | 13 ++++ .../tests/test_capture_registry_baseline.sh | 63 +++++++++++++++++++ 4 files changed, 80 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f5c58c0..99dba4eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,8 @@ ### Skills -- **`run-tests.sh` discovered shell suites only — 32 Python tests never ran in CI** — the `test_*.sh` glob silently orphaned `test_compute_lift.py` (22 tests) and `test_stamp_changelog.py` (10 tests); both passed when run by hand, so the modules looked covered while `stamp-changelog.py` sat untested in the publish path. Discovery now matches `test_*.{sh,py}` and dispatches by extension (both suite kinds self-drive via the entry-point guard `rules/file-hygiene.md` requires). Suite count 17 → 19. Full motivation: PR #181. -- **`release`: registry baseline was a pasted pipeline with no hardening** — `SKILL.md` had the operator paste `tessl plugin info | grep | awk`, sharing its parse with `verify-publish-landed.sh` but none of its guards: a tessl warning on stderr could poison the parse, and a parse miss silently yielded an empty `PRE` that satisfied the release contract's conjunct 2 vacuously (every version "advances past" empty), confirming an unpublished release. Extracted to `skills/release/capture-registry-baseline.sh`, which emits `{"version": ""}` or exits 2 — never an empty baseline. Full motivation: PR #181. +- **`run-tests.sh` discovered shell suites only — 32 Python tests never ran in CI** — the `test_*.sh` glob silently orphaned `test_compute_lift.py` (22 tests) and `test_stamp_changelog.py` (10 tests); both passed when run by hand, so the modules looked covered while `stamp-changelog.py` sat untested in the publish path. Discovery now matches `test_*.{sh,py}` and dispatches by extension (both suite kinds self-drive via the entry-point guard `rules/file-hygiene.md` requires). Suite count 17 → 19. Full motivation: PR #183. +- **`release`: registry baseline was a pasted pipeline with no hardening** — `SKILL.md` had the operator paste `tessl plugin info | grep | awk`, sharing its parse with `verify-publish-landed.sh` but none of its guards: a tessl warning on stderr could poison the parse, and a parse miss silently yielded an empty `PRE` that satisfied the release contract's conjunct 2 vacuously (every version "advances past" empty), confirming an unpublished release. Extracted to `skills/release/capture-registry-baseline.sh`, which emits `{"version": ""}` or exits 2 — never an empty baseline. Full motivation: PR #183. ## 0.3.93 — 2026-07-17 diff --git a/skills/release/SKILL.md b/skills/release/SKILL.md index 4684eefb..60a8344d 100644 --- a/skills/release/SKILL.md +++ b/skills/release/SKILL.md @@ -187,7 +187,8 @@ After merge — per `rules/ci-safety.md`'s Always Watch CI duty extended through merge_sha=$(gh pr view --json mergeCommit --jq '.mergeCommit.oid') run_json=$(skills/release/resolve-publish-run.sh "$merge_sha" "") \ || { echo "Publish run not resolved — see stderr" >&2; exit 1; } - run_id=$(jq -r '.database_id' <<<"$run_json") + run_id=$(jq -r '.database_id' <<<"$run_json") \ + || { echo "Resolver output is not valid JSON — see stderr" >&2; exit 1; } [[ -n "$run_id" && "$run_id" != "null" ]] || { echo "Resolver returned no run id" >&2; exit 1; } gh run watch "$run_id" ``` diff --git a/skills/release/capture-registry-baseline.sh b/skills/release/capture-registry-baseline.sh index 74bcfdc6..c0cad118 100755 --- a/skills/release/capture-registry-baseline.sh +++ b/skills/release/capture-registry-baseline.sh @@ -87,6 +87,19 @@ main() { exit 2 fi + # Validate the shape this script's contract promises. Taking the last + # field of the "Latest Version" line on faith accepts whatever the + # registry prints there — `Latest Version unavailable` would emit + # {"version":"unavailable"} and hand verify-publish-landed.sh a baseline + # its version comparison cannot order, converting a registry-side outage + # into a bogus conjunct-2 verdict. A contract that promises semver and + # ships an arbitrary token is the failure this whole script exists to + # remove, one layer up. + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then + echo "error: parsed 'Latest Version' token '${version}' from 'tessl plugin info ${workspace}/${tile}' is not a semver version — the registry may be reporting an outage or has changed its output shape; inspect it directly before retrying the release (output was: ${tessl_output})" >&2 + exit 2 + fi + printf '{"version":%s}\n' "$(json_str "$version")" } diff --git a/skills/release/tests/test_capture_registry_baseline.sh b/skills/release/tests/test_capture_registry_baseline.sh index d041efbf..0ffbec6f 100755 --- a/skills/release/tests/test_capture_registry_baseline.sh +++ b/skills/release/tests/test_capture_registry_baseline.sh @@ -50,6 +50,16 @@ case "${MOCK_MODE:-}" in echo "Plugin jbaruch/coding-policy" echo " Description nothing useful here" ;; + version_not_semver) + # The registry reporting an outage on the line we parse. Taking the + # last field on faith yielded {"version":"unavailable"} — a baseline + # verify-publish-landed.sh's version comparison cannot order. + echo "Plugin jbaruch/coding-policy" + echo " Latest Version unavailable" + ;; + version_is_prose) + echo " Latest Version see registry status page" + ;; tessl_fails) echo "error: not authenticated" >&2 exit 1 @@ -91,6 +101,59 @@ else fail "parse miss: code=$CODE out=$OUT err=$ERR" fi +# --- a non-semver token on the Latest Version line is exit 2, not a value --- +# The contract promises {"version": ""}. Emitting whatever the +# registry printed would hand verify-publish-landed.sh a baseline its +# version comparison cannot order — a registry outage silently becomes a +# bogus conjunct-2 verdict. +MOCK_MODE=version_not_semver invoke +if [[ "$CODE" == 2 ]] && [[ -z "$OUT" ]] && [[ "$ERR" == *"is not a semver version"* ]]; then + pass "non-semver token ('unavailable') -> exit 2, empty stdout" +else + fail "non-semver: code=$CODE out=$OUT err=$ERR" +fi + +# --- multi-word prose on the version line is also rejected --- +# awk '{print $NF}' takes the LAST field, so prose yields a plausible-looking +# single token ("page") that only a shape check catches. +MOCK_MODE=version_is_prose invoke +if [[ "$CODE" == 2 ]] && [[ -z "$OUT" ]] && [[ "$ERR" == *"is not a semver version"* ]]; then + pass "prose version line -> exit 2, empty stdout" +else + fail "prose version: code=$CODE out=$OUT err=$ERR" +fi + +# --- a prerelease/build semver is still a valid baseline --- +# The shape check must not be so strict it rejects versions the registry +# legitimately serves. +cat > "$STUBDIR/tessl" <<'STUB2' +#!/usr/bin/env bash +echo " Latest Version 1.2.3-rc.1+build.5" +STUB2 +chmod +x "$STUBDIR/tessl" +invoke +if [[ "$CODE" == 0 ]] && [[ "$OUT" == '{"version":"1.2.3-rc.1+build.5"}' ]]; then + pass "prerelease+build semver accepted" +else + fail "prerelease semver: code=$CODE out=$OUT err=$ERR" +fi +# restore the MOCK_MODE stub for the remaining cases +cat > "$STUBDIR/tessl" <<'STUB' +#!/usr/bin/env bash +case "${MOCK_MODE:-}" in + ok) + echo "Plugin jbaruch/coding-policy" + echo " Latest Version 0.3.91" + ;; + tessl_fails) + echo "error: not authenticated" >&2 + exit 1 + ;; + *) echo "stub tessl: unknown MOCK_MODE='${MOCK_MODE:-}'" >&2; exit 99 ;; +esac +STUB +chmod +x "$STUBDIR/tessl" + # --- tessl itself failing is exit 2 with the tool's stderr surfaced --- MOCK_MODE=tessl_fails invoke if [[ "$CODE" == 2 ]] && [[ -z "$OUT" ]] && [[ "$ERR" == *"not authenticated"* ]]; then From 2942902582d9997553570bfee047a3146b95bcaf Mon Sep 17 00:00:00 2001 From: Baruch Sadogursky Date: Thu, 16 Jul 2026 16:03:26 -0500 Subject: [PATCH 05/19] fix(release): bound the baseline shape by what the comparator can order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-aw caught that the semver regex accepted prerelease/build versions while verify-publish-landed.sh's version_gt() splits on '.' and feeds the fields to bash arithmetic. Verified: 1.2.3-rc.1+build.5 reaches (( )) as 3-rc.1+build.5, and the arithmetic error makes the comparison return "not greater" rather than raising — so a landed publish reports as "Latest Version is not greater than baseline, investigate the registry state". The test I added blessing that input is what created the bug. Fixing "accepts any token" by accepting more semver than the system consumes just moved the false verdict downstream. Restrict to numeric major.minor.patch and say why in the script: the accepted shape and version_gt() move together or not at all. Copilot findings, both correct: - run-tests.sh dispatched to python3 without checking it exists, so a runner lacking it gave the suite 127, which the caller counted as a failing test instead of the setup error (rc 2) the contract promises. Interpreters are now overridable (SH_BIN/PY_BIN) and the caller honours rc 2 rather than folding it into `failed`. - The baseline snippet's `[[ -n "$PRE" ]]` accepted jq's literal "null"; the run-id snippet already guarded against it. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JgdWmv7KBj91nxbVjMsj7X --- scripts/run-tests.sh | 47 +++++++++++++++++-- scripts/tests/test_run_tests.sh | 20 ++++++++ skills/release/SKILL.md | 3 +- skills/release/capture-registry-baseline.sh | 23 ++++++--- .../tests/test_capture_registry_baseline.sh | 34 ++++++++------ 5 files changed, 101 insertions(+), 26 deletions(-) diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 2ea6a5a0..892ada10 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -38,16 +38,33 @@ set -euo pipefail # `rules/file-hygiene.md` requires — so the runner only picks the # interpreter and reads the exit code. An unrecognized extension is a # discovery/dispatch mismatch, not a test failure: exit 2, not 1. +# Interpreters are overridable so a runner with a differently-named binary +# (python3.11, a venv shim) can point at it without editing this file, and +# so the missing-interpreter path is testable without breaking PATH for the +# harness itself. +SH_BIN="${SH_BIN:-bash}" +PY_BIN="${PY_BIN:-python3}" + run_suite() { - local suite="$1" + local suite="$1" interp case "$suite" in - *.sh) bash "$suite" ;; - *.py) python3 "$suite" ;; + *.sh) interp="$SH_BIN" ;; + *.py) interp="$PY_BIN" ;; *) echo "run-tests: no interpreter for suite type: $suite" >&2 return 2 ;; esac + # Check the interpreter exists before dispatching. Without this, a runner + # lacking python3 gives the suite exit 127, which the caller counts as a + # FAILING TEST (rc 1) rather than the setup error (rc 2) this contract + # promises — reporting a red suite where the truth is an unprovisioned + # runner, and sending whoever reads CI to debug a test that never ran. + if ! command -v "$interp" >/dev/null; then + echo "run-tests: '${interp}' not found on PATH — cannot run ${suite}; install it on this runner (this is a setup error, not a test failure)" >&2 + return 2 + fi + "$interp" "$suite" } # JSON string escaper. A Unix path may contain any byte except NUL and @@ -121,10 +138,22 @@ main() { echo "Running ${#suites[@]} test suite(s):" >&2 echo "" >&2 - local failed=() + # rc 2 from run_suite is a setup/dispatch error (interpreter absent, + # unknown suite type), NOT a test verdict. Folding it into `failed` would + # report "1 suite failed" for a runner that never executed the suite at + # all — a red CI pointing at innocent tests. Surface it as the contract's + # setup error (rc 2, suites=0) and stop: subsequent suites would hit the + # same missing interpreter and bury the cause under noise. + local failed=() setup_error="" for s in "${suites[@]}"; do echo "▶ $s" >&2 - if run_suite "$s" >&2; then + local suite_rc=0 + run_suite "$s" >&2 || suite_rc=$? + if [[ $suite_rc -eq 2 ]]; then + setup_error="$s" + break + fi + if [[ $suite_rc -eq 0 ]]; then echo " ✓ $s" >&2 else echo " ✗ $s" >&2 @@ -133,6 +162,14 @@ main() { echo "" >&2 done + if [[ -n "$setup_error" ]]; then + echo "─────────────────────────────────────────────" >&2 + echo "SETUP ERROR: cannot run ${setup_error} — see stderr above" >&2 + printf '{"suites":0,"passed":0,"failed":0,"failures":[],"error":%s}\n' \ + "$(json_str "cannot run ${setup_error}: interpreter missing or unknown suite type")" + return 2 + fi + echo "─────────────────────────────────────────────" >&2 if [[ ${#failed[@]} -gt 0 ]]; then echo "FAILED: ${#failed[@]}/${#suites[@]} suite(s):" >&2 diff --git a/scripts/tests/test_run_tests.sh b/scripts/tests/test_run_tests.sh index dbc02902..da87c277 100755 --- a/scripts/tests/test_run_tests.sh +++ b/scripts/tests/test_run_tests.sh @@ -173,6 +173,26 @@ else fi rm -rf "$base" +# --- a missing interpreter is a SETUP error (2), not a test failure (1) --- +# Dispatching to an absent python3 gives the suite exit 127. Counting that +# as a failing suite reports red tests for a runner that never ran them, +# sending whoever reads CI to debug a test that never executed. +base="$(make_base)"; add_py_suite "$base" pyalpha 0 +errf="$(mktemp)" +# Override PY_BIN rather than emptying PATH — the harness needs bash, find, +# and mktemp on PATH to run at all, so nuking it tests the shebang, not the +# dispatcher. +OUT="$(PY_BIN=definitely-not-a-real-python3 "$RUNNER" "$base" 2>"$errf")"; CODE=$? +ERR="$(cat "$errf")"; rm -f "$errf" +if [[ "$CODE" == 2 ]] \ + && [[ "$(jq -r .suites <<<"$OUT")" == 0 ]] \ + && [[ "$(jq -r '.error' <<<"$OUT")" == *"interpreter missing"* ]]; then + pass "missing interpreter -> exit 2 (setup error), not exit 1" +else + fail "missing interpreter: code=$CODE out=$OUT err=$ERR" +fi +rm -rf "$base" + echo "" echo "run-tests.sh: ${PASS_COUNT} passed, ${FAIL_COUNT} failed" [[ $FAIL_COUNT -eq 0 ]] || exit 1 diff --git a/skills/release/SKILL.md b/skills/release/SKILL.md index 60a8344d..736c05fe 100644 --- a/skills/release/SKILL.md +++ b/skills/release/SKILL.md @@ -133,7 +133,8 @@ baseline_json=$(skills/release/capture-registry-baseline.sh || { echo "Baseline capture failed — see stderr" >&2; exit 1; } PRE=$(jq -r '.version' <<<"$baseline_json") \ || { echo "Baseline JSON parse failed" >&2; exit 1; } -[[ -n "$PRE" ]] || { echo "Baseline capture returned an empty version" >&2; exit 1; } +[[ -n "$PRE" && "$PRE" != "null" ]] \ + || { echo "Baseline capture returned no version" >&2; exit 1; } ``` Pick the right cleanup path based on where you ran the skill from. diff --git a/skills/release/capture-registry-baseline.sh b/skills/release/capture-registry-baseline.sh index c0cad118..ebf75f95 100755 --- a/skills/release/capture-registry-baseline.sh +++ b/skills/release/capture-registry-baseline.sh @@ -91,12 +91,23 @@ main() { # field of the "Latest Version" line on faith accepts whatever the # registry prints there — `Latest Version unavailable` would emit # {"version":"unavailable"} and hand verify-publish-landed.sh a baseline - # its version comparison cannot order, converting a registry-side outage - # into a bogus conjunct-2 verdict. A contract that promises semver and - # ships an arbitrary token is the failure this whole script exists to - # remove, one layer up. - if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then - echo "error: parsed 'Latest Version' token '${version}' from 'tessl plugin info ${workspace}/${tile}' is not a semver version — the registry may be reporting an outage or has changed its output shape; inspect it directly before retrying the release (output was: ${tessl_output})" >&2 + # its comparator cannot order, converting a registry-side outage into a + # bogus conjunct-2 verdict. + # + # Numeric major.minor.patch ONLY — deliberately narrower than semver. + # The sole consumer of this baseline is verify-publish-landed.sh's + # version_gt(), which splits on '.' and feeds the fields to bash + # arithmetic. A prerelease/build value reaches `(( ))` as `3-rc.1+build.5`, + # and the resulting arithmetic error makes the comparison return "not + # greater" rather than raising: a landed publish is then reported as + # "Latest Version is not greater than baseline — investigate the registry + # state", sending the operator after a registry that did nothing wrong. + # This script must not accept a baseline shape its consumer cannot order. + # Widening here requires teaching version_gt() full semver precedence + # first (see verify-publish-landed.sh) — the accepted shape and the + # comparator move together or not at all. + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "error: parsed 'Latest Version' token '${version}' from 'tessl plugin info ${workspace}/${tile}' is not a numeric major.minor.patch version — the registry may be reporting an outage, or is publishing a prerelease/build version that verify-publish-landed.sh's comparator cannot order; inspect it directly before retrying the release (output was: ${tessl_output})" >&2 exit 2 fi diff --git a/skills/release/tests/test_capture_registry_baseline.sh b/skills/release/tests/test_capture_registry_baseline.sh index 0ffbec6f..0695e666 100755 --- a/skills/release/tests/test_capture_registry_baseline.sh +++ b/skills/release/tests/test_capture_registry_baseline.sh @@ -101,14 +101,14 @@ else fail "parse miss: code=$CODE out=$OUT err=$ERR" fi -# --- a non-semver token on the Latest Version line is exit 2, not a value --- -# The contract promises {"version": ""}. Emitting whatever the -# registry printed would hand verify-publish-landed.sh a baseline its -# version comparison cannot order — a registry outage silently becomes a -# bogus conjunct-2 verdict. +# --- a non-version token on the Latest Version line is exit 2, not a value --- +# The contract promises numeric major.minor.patch. Emitting whatever the +# registry printed hands verify-publish-landed.sh a baseline its comparator +# cannot order — a registry outage silently becomes a bogus conjunct-2 +# verdict. MOCK_MODE=version_not_semver invoke -if [[ "$CODE" == 2 ]] && [[ -z "$OUT" ]] && [[ "$ERR" == *"is not a semver version"* ]]; then - pass "non-semver token ('unavailable') -> exit 2, empty stdout" +if [[ "$CODE" == 2 ]] && [[ -z "$OUT" ]] && [[ "$ERR" == *"not a numeric major.minor.patch"* ]]; then + pass "non-version token ('unavailable') -> exit 2, empty stdout" else fail "non-semver: code=$CODE out=$OUT err=$ERR" fi @@ -117,25 +117,31 @@ fi # awk '{print $NF}' takes the LAST field, so prose yields a plausible-looking # single token ("page") that only a shape check catches. MOCK_MODE=version_is_prose invoke -if [[ "$CODE" == 2 ]] && [[ -z "$OUT" ]] && [[ "$ERR" == *"is not a semver version"* ]]; then +if [[ "$CODE" == 2 ]] && [[ -z "$OUT" ]] && [[ "$ERR" == *"not a numeric major.minor.patch"* ]]; then pass "prose version line -> exit 2, empty stdout" else fail "prose version: code=$CODE out=$OUT err=$ERR" fi -# --- a prerelease/build semver is still a valid baseline --- -# The shape check must not be so strict it rejects versions the registry -# legitimately serves. +# --- a prerelease/build version is REJECTED, not accepted --- +# The accepted shape is bounded by the consumer, not by semver. This +# baseline's only reader is verify-publish-landed.sh's version_gt(), which +# splits on '.' and feeds the fields to bash arithmetic: `1.2.3-rc.1+build.5` +# arrives at `(( ))` as `3-rc.1+build.5`, and the arithmetic error makes the +# comparison return "not greater" instead of raising — so a landed publish +# reports as "Latest Version is not greater than baseline — investigate the +# registry state". Accepting a shape the consumer cannot order buys a false +# release verdict, so this script rejects it at the boundary. cat > "$STUBDIR/tessl" <<'STUB2' #!/usr/bin/env bash echo " Latest Version 1.2.3-rc.1+build.5" STUB2 chmod +x "$STUBDIR/tessl" invoke -if [[ "$CODE" == 0 ]] && [[ "$OUT" == '{"version":"1.2.3-rc.1+build.5"}' ]]; then - pass "prerelease+build semver accepted" +if [[ "$CODE" == 2 ]] && [[ -z "$OUT" ]] && [[ "$ERR" == *"not a numeric major.minor.patch"* ]]; then + pass "prerelease+build version -> exit 2 (comparator cannot order it)" else - fail "prerelease semver: code=$CODE out=$OUT err=$ERR" + fail "prerelease rejection: code=$CODE out=$OUT err=$ERR" fi # restore the MOCK_MODE stub for the remaining cases cat > "$STUBDIR/tessl" <<'STUB' From cae2183e62382330326a66a8fd8312f78b3ca68a Mon Sep 17 00:00:00 2001 From: Baruch Sadogursky Date: Thu, 16 Jul 2026 21:04:00 -0500 Subject: [PATCH 06/19] fix(ci): use a dispatcher sentinel so a suite exiting 2 stays a failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-aw caught a collision I introduced. run_suite returned 2 for its own setup errors (missing interpreter, unknown extension), but the harnesses in this repo exit 2 on a fatal precondition — so a genuinely failing suite read as a broken runner, reported suites=0, and stopped the run. A red suite hidden behind a setup error nobody could act on. Use 125 as the dispatcher's private sentinel (outside the range any suite here produces; git-bisect uses it for the same "launcher failed, not the thing it launched" distinction) and map it to the documented runner exit 2. Every other non-zero, 2 included, is the suite's own verdict. Regression tests for both a shell and a python suite exiting 2. Also from review: - SKILL.md documented the baseline contract as while the script now accepts only numeric major.minor.patch (script-as-black-box: skill prose names the script's contract). - CHANGELOG claimed 19 suites; it is 20 (18 sh + 2 py) — this PR adds one. - The baseline snippet's jq failure path didn't point at stderr. - test_capture_registry_baseline.sh ran on with an empty STUBDIR if mktemp -d failed, since the harness deliberately has no set -e. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JgdWmv7KBj91nxbVjMsj7X --- CHANGELOG.md | 2 +- scripts/run-tests.sh | 33 ++++++++++++++----- scripts/tests/test_run_tests.sh | 27 +++++++++++++++ skills/release/SKILL.md | 4 +-- .../tests/test_capture_registry_baseline.sh | 7 +++- 5 files changed, 60 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99dba4eb..e2632a42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Skills -- **`run-tests.sh` discovered shell suites only — 32 Python tests never ran in CI** — the `test_*.sh` glob silently orphaned `test_compute_lift.py` (22 tests) and `test_stamp_changelog.py` (10 tests); both passed when run by hand, so the modules looked covered while `stamp-changelog.py` sat untested in the publish path. Discovery now matches `test_*.{sh,py}` and dispatches by extension (both suite kinds self-drive via the entry-point guard `rules/file-hygiene.md` requires). Suite count 17 → 19. Full motivation: PR #183. +- **`run-tests.sh` discovered shell suites only — 32 Python tests never ran in CI** — the `test_*.sh` glob silently orphaned `test_compute_lift.py` (22 tests) and `test_stamp_changelog.py` (10 tests); both passed when run by hand, so the modules looked covered while `stamp-changelog.py` sat untested in the publish path. Discovery now matches `test_*.{sh,py}` and dispatches by extension (both suite kinds self-drive via the entry-point guard `rules/file-hygiene.md` requires). Suite count 17 → 20 (18 shell + 2 Python, including this PR's new baseline suite). Full motivation: PR #183. - **`release`: registry baseline was a pasted pipeline with no hardening** — `SKILL.md` had the operator paste `tessl plugin info | grep | awk`, sharing its parse with `verify-publish-landed.sh` but none of its guards: a tessl warning on stderr could poison the parse, and a parse miss silently yielded an empty `PRE` that satisfied the release contract's conjunct 2 vacuously (every version "advances past" empty), confirming an unpublished release. Extracted to `skills/release/capture-registry-baseline.sh`, which emits `{"version": ""}` or exits 2 — never an empty baseline. Full motivation: PR #183. ## 0.3.93 — 2026-07-17 diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 892ada10..97f75c43 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -36,8 +36,8 @@ set -euo pipefail # self-driving — a shell harness counts its own failures and exits # non-zero; a unittest module does the same through the entry-point guard # `rules/file-hygiene.md` requires — so the runner only picks the -# interpreter and reads the exit code. An unrecognized extension is a -# discovery/dispatch mismatch, not a test failure: exit 2, not 1. +# interpreter and reads the exit code. +# # Interpreters are overridable so a runner with a differently-named binary # (python3.11, a venv shim) can point at it without editing this file, and # so the missing-interpreter path is testable without breaking PATH for the @@ -45,6 +45,17 @@ set -euo pipefail SH_BIN="${SH_BIN:-bash}" PY_BIN="${PY_BIN:-python3}" +# A dispatcher fault must be distinguishable from a suite's own exit code, +# and 2 cannot do that job: the repo's own harnesses exit 2 on a fatal +# precondition ("fatal: