diff --git a/.github/actions/skill-review/action.yml b/.github/actions/skill-review/action.yml index 41a4c4bc..5227bc3e 100644 --- a/.github/actions/skill-review/action.yml +++ b/.github/actions/skill-review/action.yml @@ -33,11 +33,37 @@ inputs: (workflow_dispatch / initial push). required: false default: '' + credit-outage: + description: | + How a tessl "out of credits" (403) billing outage during review is + handled. `fail` (default) preserves the Mandatory Review gate — any + review failure, credit outage included, blocks the publish. `skip` + is fail-safe and narrow: ONLY a review call failing with the + out-of-credits signature is tolerated (that skill publishes + unreviewed, named in the run summary and the `unreviewed-skills` + output); every other non-zero exit — a below-threshold score, an + auth error, a tooling bug — still hard-fails. Opting in requires the + `context-artifacts` Mandatory Review Credit-Outage carve-out. + required: false + default: 'fail' + +outputs: + unreviewed-skills: + description: | + Comma-separated skills that were NOT reviewed because the tessl org + was out of credits (empty when every changed skill was reviewed). + Only ever non-empty under `credit-outage: skip`. Gate follow-up + steps on it to flag or track an unreviewed publish. + value: ${{ steps.review.outputs.unreviewed-skills }} runs: using: 'composite' steps: + # Body lives in review-skills.sh (a real, unit-tested file per + # rules/script-delegation.md), invoked via bash so no exec-bit is + # required — tessl packaging normalizes plugin file modes. - name: Identify changed skills and review them + id: review shell: bash env: EVENT_BEFORE: ${{ github.event.before }} @@ -45,59 +71,5 @@ runs: BASE_OVERRIDE: ${{ inputs.base-ref }} THRESHOLD: ${{ inputs.threshold }} SKILLS_DIR: ${{ inputs.skills-dir }} - run: | - # Pipefail so a failing `git diff` propagates instead of silently - # producing an empty changed-skills set (which would skip the - # mandatory review per `rules/context-artifacts.md`). - set -eo pipefail - - if [ -n "$BASE_OVERRIDE" ]; then - base="$BASE_OVERRIDE" - elif [ "$EVENT_NAME" = "workflow_dispatch" ] \ - || [ -z "$EVENT_BEFORE" ] \ - || [ "$EVENT_BEFORE" = "0000000000000000000000000000000000000000" ]; then - base="" - else - base="$EVENT_BEFORE" - fi - if [ -z "$base" ]; then - mapfile -t skills < <(find "$SKILLS_DIR" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort) - echo "Reviewing all ${#skills[@]} skill(s) (manual trigger or initial push)." - else - # Validate the base is a real commit before relying on it for - # the diff. If not reachable in the local clone, try a shallow - # fetch; if that fails too, hard-fail rather than silently - # converting an unreachable base into "no changes". - if ! git rev-parse --verify "${base}^{commit}" >/dev/null 2>&1; then - echo "Base $base not reachable in clone; fetching..." - git fetch --no-tags --depth=50 origin "$base" 2>/dev/null \ - || git fetch --unshallow origin 2>/dev/null \ - || true - fi - if ! git rev-parse --verify "${base}^{commit}" >/dev/null 2>&1; then - echo "::error::Base $base is not a reachable commit; refusing to silently skip skill review." - echo "::error::Set actions/checkout fetch-depth: 0 in the calling workflow, or pass an explicit base-ref." - exit 1 - fi - # Strip the SKILLS_DIR prefix from each diff path and pull - # out the first path component beneath it — works for any - # depth (single-component `skills` or nested `pkg/skills`). - mapfile -t skills < <(git diff --name-only "$base..HEAD" -- "$SKILLS_DIR/" \ - | awk -v prefix="$SKILLS_DIR/" ' - index($0, prefix) == 1 { - rest = substr($0, length(prefix) + 1) - slash = index(rest, "/") - if (slash > 0) print substr(rest, 1, slash - 1) - }' | sort -u) - echo "Detected ${#skills[@]} changed skill(s) since ${base:0:7}." - fi - if [ "${#skills[@]}" -eq 0 ]; then - echo "No skill changes — review skipped." - exit 0 - fi - for skill in "${skills[@]}"; do - [ -f "$SKILLS_DIR/$skill/SKILL.md" ] || { echo " - $skill: deleted, skipping"; continue; } - echo "::group::Reviewing $skill" - tessl skill review --threshold "$THRESHOLD" "$SKILLS_DIR/$skill/SKILL.md" - echo "::endgroup::" - done + CREDIT_OUTAGE: ${{ inputs.credit-outage }} + run: bash "$GITHUB_ACTION_PATH/review-skills.sh" diff --git a/.github/actions/skill-review/review-skills.sh b/.github/actions/skill-review/review-skills.sh new file mode 100644 index 00000000..5d2d8bc6 --- /dev/null +++ b/.github/actions/skill-review/review-skills.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Identify the skills whose files changed since the previous push and run +# `tessl skill review --threshold N` on each, per jbaruch/coding-policy +# `context-artifacts` "Mandatory Review". Unchanged skills are not +# re-reviewed — that just burns runner time and Tessl credits. +# +# This is the executable body of the `skill-review` composite action +# (`action.yml` calls it via `bash "$GITHUB_ACTION_PATH/review-skills.sh"`). +# It lives as a real file, not an inline `run:` block, so its +# credit-outage classification is unit-testable (rules/script-delegation.md); +# tests source it and drive `run_reviews` with a mocked `tessl`. +# +# Environment contract (action.yml sets these from its inputs): +# THRESHOLD minimum passing score (default 85) +# SKILLS_DIR dir holding skill subdirectories (default skills) +# BASE_OVERRIDE explicit diff base, or empty +# EVENT_NAME github.event_name +# EVENT_BEFORE github.event.before +# CREDIT_OUTAGE fail | skip (default fail) +# GITHUB_OUTPUT / GITHUB_STEP_SUMMARY runner-provided sinks (optional +# off-runner; default to /dev/null) +# +# Out: writes `unreviewed-skills=` to $GITHUB_OUTPUT (skills skipped +# under a credit outage; empty when all were reviewed). +# Exit: 0 when every changed skill passed review (or was credit-skipped +# under CREDIT_OUTAGE=skip); the failing skill's exit code otherwise; +# 2 on setup error (unreachable base, invalid credit-outage value). + +set -euo pipefail + +THRESHOLD="${THRESHOLD:-85}" +SKILLS_DIR="${SKILLS_DIR:-skills}" +BASE_OVERRIDE="${BASE_OVERRIDE:-}" +EVENT_NAME="${EVENT_NAME:-}" +EVENT_BEFORE="${EVENT_BEFORE:-}" +CREDIT_OUTAGE="${CREDIT_OUTAGE:-fail}" + +# Skills skipped because the tessl org was out of credits. Populated by +# run_reviews; read by main for the action output. +UNREVIEWED=() + +# True only when review output carries the out-of-credits billing signature. +# Precision matters: this is the sole failure the skip mode tolerates, so a +# too-loose match would publish a genuinely-broken skill unreviewed. Require +# BOTH the credit phrase AND a 403, each a fixed string (grep -F, no regex +# metacharacters) — a real failure that merely quotes "run out of credits" +# in prose, or any future tessl wording change, falls through to hard-fail +# (the safe direction). tessl exposes no distinct exit code for the billing +# 403; if it ever does, prefer that over string matching. +is_credit_outage() { + printf '%s' "$1" | grep -qiF 'run out of credits' \ + && printf '%s' "$1" | grep -qF '403' +} + +# Emit the operator-facing warning and run-summary note for one skill that +# went unreviewed because the tessl org is out of credits. Credits reset +# monthly on the 1st, so name the resume date to make the skip window +# legible; a non-GNU `date` falls back to a generic phrase (explicit +# fallback, not error-swallowing per rules/error-handling.md). +credit_skip() { + local skill="$1" resume + resume="$(date -u -d "$(date -u +%Y-%m-01) +1 month" +%Y-%m-01 2>/dev/null)" \ + || resume="the 1st of next month" + echo "::warning::tessl org out of credits (403) — skipping review for '${skill}'. Publishing UNREVIEWED; review resumes by ${resume} (monthly credit reset)." + { + echo "### ⚠️ Skill review skipped — tessl credits exhausted" + echo "" + echo "\`${skill}\` was **not** reviewed: the tessl org is out of credits (403)." + echo "It is publishing **without** review. The gate self-heals — top up credits to restore it immediately, otherwise it resumes by **${resume}** (monthly reset)." + } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +} + +# Review each named skill. Reads THRESHOLD, SKILLS_DIR, CREDIT_OUTAGE. +# Under CREDIT_OUTAGE=skip a review call that fails with the out-of-credits +# signature is tolerated (recorded in UNREVIEWED, publish continues); every +# other non-zero exit — a below-threshold score, an auth error, a tooling +# bug — still hard-fails, whatever the mode. Returns the failing skill's +# exit code on a hard failure, 0 when all skills were handled. +run_reviews() { + case "$CREDIT_OUTAGE" in + fail|skip) ;; + *) + echo "::error::invalid credit-outage '${CREDIT_OUTAGE}' — expected 'fail' or 'skip'." >&2 + return 2 + ;; + esac + + UNREVIEWED=() + local skill review_out rc + for skill in "$@"; do + [ -f "$SKILLS_DIR/$skill/SKILL.md" ] || { echo " - ${skill}: deleted, skipping"; continue; } + echo "::group::Reviewing ${skill}" + # Capture output so an out-of-credits outage (a billing state, not a + # skill defect) can be told apart from a real failure. `|| rc=$?` + # captures the exit without aborting under `set -e` and without + # toggling `-e` (which would leak to callers); never blanket-swallow — + # only the credit signature under skip-mode is tolerated below. + rc=0 + review_out="$(tessl skill review --threshold "$THRESHOLD" "$SKILLS_DIR/$skill/SKILL.md" 2>&1)" || rc=$? + printf '%s\n' "$review_out" + echo "::endgroup::" + [ "$rc" -eq 0 ] && continue + if [ "$CREDIT_OUTAGE" = "skip" ] && is_credit_outage "$review_out"; then + credit_skip "$skill" + UNREVIEWED+=("$skill") + continue + fi + local why="" + [ "$CREDIT_OUTAGE" = "skip" ] && why=" — not a credits outage" + echo "::error::Skill review failed for '${skill}' (exit ${rc})${why} — blocking publish." + return "$rc" + done + return 0 +} + +# Resolve the diff base and echo the changed skill names (one per line). +# BASE_OVERRIDE wins; else github.event.before for push; else "" meaning +# review every skill (workflow_dispatch / initial push expect a full +# re-review). An unreachable non-empty base hard-fails rather than +# silently collapsing to "no changes". +identify_skills() { + local base + if [ -n "$BASE_OVERRIDE" ]; then + base="$BASE_OVERRIDE" + elif [ "$EVENT_NAME" = "workflow_dispatch" ] \ + || [ -z "$EVENT_BEFORE" ] \ + || [ "$EVENT_BEFORE" = "0000000000000000000000000000000000000000" ]; then + base="" + else + base="$EVENT_BEFORE" + fi + + if [ -z "$base" ]; then + # Immediate subdir basenames, sorted. `sed 's|.*/||'` instead of GNU + # `find -printf '%f'` keeps discovery portable (BSD find on macOS has no + # -printf) so this path is testable off the CI runner. Skill dir names + # are kebab-case with no newlines (rules/plugin-evals.md Naming). + find "$SKILLS_DIR" -mindepth 1 -maxdepth 1 -type d | sed 's|.*/||' | sort + return 0 + fi + + # Validate the base is a real commit before relying on it. If not + # reachable in the local clone, try a shallow fetch; if that fails too, + # hard-fail rather than silently converting an unreachable base into + # "no changes". The fetches are best-effort — the rev-parse check below + # is the real gate — so a failed fetch warns rather than aborting. + if ! git rev-parse --verify "${base}^{commit}" >/dev/null 2>&1; then + echo "Base ${base} not reachable in clone; fetching..." >&2 + git fetch --no-tags --depth=50 origin "$base" 2>/dev/null \ + || git fetch --unshallow origin 2>/dev/null \ + || echo "::warning::could not fetch base ${base}; re-checking reachability" >&2 + fi + if ! git rev-parse --verify "${base}^{commit}" >/dev/null 2>&1; then + echo "::error::Base ${base} is not a reachable commit; refusing to silently skip skill review." >&2 + echo "::error::Set actions/checkout fetch-depth: 0 in the calling workflow, or pass an explicit base-ref." >&2 + return 2 + fi + # Strip the SKILLS_DIR prefix from each diff path and pull out the first + # path component beneath it — works for any depth (single-component + # `skills` or nested `pkg/skills`). + git diff --name-only "$base..HEAD" -- "$SKILLS_DIR/" \ + | awk -v prefix="$SKILLS_DIR/" ' + index($0, prefix) == 1 { + rest = substr($0, length(prefix) + 1) + slash = index(rest, "/") + if (slash > 0) print substr(rest, 1, slash - 1) + }' | sort -u +} + +main() { + # Capture-and-check rather than `mapfile < <(identify_skills)`: a process + # substitution hides the function's non-zero return, which would drop the + # unreachable-base hard-fail and silently review nothing. + local skills_raw skills=() rc=0 + skills_raw="$(identify_skills)" || rc=$? + if [ "$rc" -ne 0 ]; then + return "$rc" + fi + if [ -n "$skills_raw" ]; then + mapfile -t skills <<< "$skills_raw" + fi + echo "Reviewing ${#skills[@]} skill(s) (credit-outage=${CREDIT_OUTAGE})." + + if [ "${#skills[@]}" -gt 0 ]; then + run_reviews "${skills[@]}" + else + echo "No skill changes — review skipped." + fi + + local csv="" + if [ "${#UNREVIEWED[@]}" -gt 0 ]; then + csv=$(IFS=,; printf '%s' "${UNREVIEWED[*]}") + fi + echo "unreviewed-skills=${csv}" >> "${GITHUB_OUTPUT:-/dev/null}" +} + +[[ "${BASH_SOURCE[0]}" == "${0}" ]] && main "$@" diff --git a/.github/actions/skill-review/tests/test_review_skills.sh b/.github/actions/skill-review/tests/test_review_skills.sh new file mode 100644 index 00000000..fa20ed42 --- /dev/null +++ b/.github/actions/skill-review/tests/test_review_skills.sh @@ -0,0 +1,316 @@ +#!/usr/bin/env bash +# Outcome-based tests for review-skills.sh's credit-outage classification — +# the fail-safe logic issue #188 asks the skill-review action to own: under +# CREDIT_OUTAGE=skip, ONLY a tessl "out of credits" (403) failure is +# tolerated (the skill publishes unreviewed, recorded in UNREVIEWED); every +# other non-zero exit still hard-fails, and CREDIT_OUTAGE=fail never +# tolerates anything. +# +# Approach: source review-skills.sh (its main() guard prevents auto-run when +# sourced) and drive its functions directly with a mocked `tessl` shell +# function — a command substitution inherits the caller's functions, so the +# mock intercepts the real call without touching PATH. Coverage: +# - run_reviews: the credit-outage classification (fail vs skip, the +# credit-phrase-plus-403 signature, non-credit hard-fail, mixed runs, +# deleted skills, input validation); +# - identify_skills: review-all fallback, git-diff changed-skill detection +# against a throwaway git tree, and unreachable-base hard-fail; +# - main: the unreviewed-skills output emission. +# +# Determinism (rules/testing-standards.md): assertions never check the +# credit-reset date credit_skip() computes from the wall clock — only that a +# skill was skipped/recorded/failed. Fixtures are built in setup, no binary +# or time-relative inputs. +# +# Run: bash .github/actions/skill-review/tests/test_review_skills.sh +# Exit 0 on all-pass; non-zero with a per-test diagnostic on failure. + +set -uo pipefail + +SCRIPT="$(cd "$(dirname "$0")/.." && pwd)/review-skills.sh" +[[ -f "$SCRIPT" && -r "$SCRIPT" ]] || { echo "fatal: review-skills.sh not found at $SCRIPT" >&2; exit 2; } + +# Sourcing runs the script's own `set -euo pipefail`; `|| true` then `set +e` +# restores the harness's discipline (rules/error-handling.md source carve-out). +# The config vars below are `export`ed, mirroring how the action passes them to +# review-skills.sh (via `env:`) — which also tells shellcheck they're used +# externally by the sourced functions, not dead (SC2034). +# shellcheck disable=SC1090 +source "$SCRIPT" || true +set +e + +FAIL_COUNT=0 +PASS_COUNT=0 + +# Explicit setup checks under `set +e`: a failed mktemp/mkdir would otherwise +# leave a bad fixture path and the assertions below would no longer mean what +# they report (rules/error-handling.md setup-failure clause). +FIXTURE=$(mktemp -d -t skill-review-test.XXXXXX) \ + || { echo "fatal: could not create fixture dir" >&2; exit 2; } +cleanup_tmp() { + if [[ -n "${FIXTURE:-}" ]]; then + if ! rm -rf "$FIXTURE"; then + echo "warning: could not remove temp dir $FIXTURE — remove it by hand" >&2 + fi + fi + return 0 +} +trap cleanup_tmp EXIT + +# Fixture skills. `needs-credits` is named so the by-path mock can single it +# out for a mixed run; the rest are plain. +for s in alpha beta needs-credits; do + mkdir -p "$FIXTURE/$s" || { echo "fatal: could not create fixture $s" >&2; exit 2; } + : > "$FIXTURE/$s/SKILL.md" || { echo "fatal: could not write fixture $s/SKILL.md" >&2; exit 2; } +done +# A skill dir whose SKILL.md is absent (a deleted skill) — run_reviews must +# skip it, never call tessl on it. +mkdir -p "$FIXTURE/deleted" || { echo "fatal: could not create fixture deleted" >&2; exit 2; } + +# Global mock. MOCK_MODE selects behaviour; the last arg is the SKILL.md +# path. tessl runs inside run_reviews' command substitution (a subshell), so +# the call count is logged to a file rather than a variable — a subshell's +# variable writes never reach the parent. +MOCK_MODE="success" +MOCK_CALLS_FILE="$FIXTURE/tessl-calls.log" +# Invoked only from run_reviews inside the sourced review-skills.sh (via a +# command substitution shellcheck cannot trace), never directly here. +# shellcheck disable=SC2329 +tessl() { + echo x >> "$MOCK_CALLS_FILE" + local path="${!#}" + case "$MOCK_MODE" in + success) return 0 ;; + credits) + echo "✘ 403 Forbidden" + echo "Your organization has run out of credits. Upgrade your plan or buy more credits to continue." + return 1 + ;; + threshold) + echo "Skill scored 70 — below threshold 85." + return 1 + ;; + phrase-only) + # The EXACT production-matched phrase ("run out of credits") but no + # 403 — must NOT be classed as an outage. Using the exact phrase is + # the point: if is_credit_outage ever stopped requiring 403 and matched + # the phrase alone, this case would flip to a wrongful skip and fail. + echo "Your organization has run out of credits." + return 1 + ;; + code-only) + # A 403 without the credit phrase — a different auth/forbidden error. + echo "✘ 403 Forbidden — token lacks scope for this workspace." + return 1 + ;; + by-path) + if [[ "$path" == *needs-credits* ]]; then + echo "✘ 403 Forbidden" + echo "Your organization has run out of credits." + return 1 + fi + return 0 + ;; + *) echo "mock: unknown MOCK_MODE '$MOCK_MODE'" >&2; return 99 ;; + esac +} + +# Drive run_reviews with the harness globals set, capturing rc, the emitted +# output (stdout+stderr, where ::warning:: lands), and the resulting +# UNREVIEWED array. run_reviews runs in THIS shell (output redirected to a +# file, not command-substituted) so its UNREVIEWED writes survive. Sinks are +# fixture files so credit_skip's GITHUB_STEP_SUMMARY note is inspectable. +drive() { + local mode="$1"; shift # CREDIT_OUTAGE value + : > "$MOCK_CALLS_FILE" + export SKILLS_DIR="$FIXTURE" THRESHOLD="85" CREDIT_OUTAGE="$mode" + export GITHUB_STEP_SUMMARY="$FIXTURE/summary.txt"; : > "$GITHUB_STEP_SUMMARY" + UNREVIEWED=() + run_reviews "$@" > "$FIXTURE/run.out" 2>&1 + DRIVE_RC=$? + DRIVE_OUT="$(cat "$FIXTURE/run.out")" + MOCK_CALLS=$(wc -l < "$MOCK_CALLS_FILE" | tr -d ' ') +} + +assert_rc() { + local want="$1" name="$2" + if [[ "$DRIVE_RC" -eq "$want" ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "FAIL: $name — expected rc $want, got $DRIVE_RC" >&2 + echo " output: $DRIVE_OUT" >&2 + fi +} + +assert_unreviewed() { + local want="$1" name="$2" + local got; got=$(IFS=,; printf '%s' "${UNREVIEWED[*]:-}") + if [[ "$got" == "$want" ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "FAIL: $name — expected UNREVIEWED='$want', got '$got'" >&2 + fi +} + +assert_contains() { + local hay="$1" needle="$2" name="$3" + if [[ "$hay" == *"$needle"* ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "FAIL: $name — expected to contain '$needle'" >&2 + echo " in: $hay" >&2 + fi +} + +assert_eq() { + local got="$1" want="$2" name="$3" + if [[ "$got" == "$want" ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "FAIL: $name — expected '$want', got '$got'" >&2 + fi +} + +# --- fail mode: nothing is tolerated --- + +MOCK_MODE="success"; drive fail alpha +assert_rc 0 "fail+success: clean review passes" +assert_unreviewed "" "fail+success: nothing unreviewed" + +MOCK_MODE="credits"; drive fail alpha +assert_rc 1 "fail+credits: credit outage still hard-fails under fail mode" +assert_unreviewed "" "fail+credits: nothing recorded as unreviewed" + +# --- skip mode: only the credit signature is tolerated --- + +MOCK_MODE="credits"; drive skip alpha +assert_rc 0 "skip+credits: credit outage tolerated" +assert_unreviewed "alpha" "skip+credits: skill recorded unreviewed" +assert_contains "$DRIVE_OUT" "::warning::" "skip+credits: emits a warning" +assert_contains "$DRIVE_OUT" "out of credits" "skip+credits: warning names the cause" +assert_contains "$(cat "$FIXTURE/summary.txt")" "not** reviewed" "skip+credits: writes a run-summary note" + +MOCK_MODE="threshold"; drive skip alpha +assert_rc 1 "skip+threshold: a real below-threshold failure still hard-fails" +assert_unreviewed "" "skip+threshold: not recorded as a credit skip" +assert_contains "$DRIVE_OUT" "not a credits outage" "skip+threshold: diagnostic distinguishes the cause" + +# Detection precision (issue #188 follow-up): skip requires the credit phrase +# AND a 403. Either alone is a real failure and must hard-fail. +MOCK_MODE="phrase-only"; drive skip alpha +assert_rc 1 "skip+phrase-without-403: credit phrase alone is not an outage — hard-fails" +assert_unreviewed "" "skip+phrase-without-403: not recorded as a credit skip" + +MOCK_MODE="code-only"; drive skip alpha +assert_rc 1 "skip+403-without-phrase: a different 403 is not an outage — hard-fails" +assert_unreviewed "" "skip+403-without-phrase: not recorded as a credit skip" + +MOCK_MODE="success"; drive skip alpha beta +assert_rc 0 "skip+success: all clean passes" +assert_unreviewed "" "skip+success: nothing unreviewed" + +# --- mixed: one credit outage among clean reviews (skip mode) --- + +MOCK_MODE="by-path"; drive skip alpha needs-credits beta +assert_rc 0 "skip+mixed: run continues past the credit skip" +assert_unreviewed "needs-credits" "skip+mixed: only the outage skill is recorded" + +# --- deleted skill (no SKILL.md) is skipped, never reviewed --- + +MOCK_MODE="success"; drive skip deleted +assert_rc 0 "deleted skill: skipped, no failure" +assert_eq "$MOCK_CALLS" "0" "deleted skill: tessl never invoked" +assert_unreviewed "" "deleted skill: nothing unreviewed" + +# --- input validation: an unknown mode is a setup error, not a silent fail --- + +MOCK_MODE="success"; drive bogus alpha +assert_rc 2 "invalid credit-outage: rejected with exit 2" +assert_contains "$DRIVE_OUT" "invalid credit-outage" "invalid credit-outage: names the bad value" + +# --- identify_skills: changed-skill detection, review-all fallback, base failure --- + +# Run identify_skills with the given event context, echoing its output. The +# config are `export`ed — the env contract review-skills.sh reads, and read +# as used-externally so the linter does not flag them dead. Callers invoke +# this inside a `$()` capture, so both the exports and the `cd` stay confined +# to that subshell. +capture_identify() { + local dir="$1" + export SKILLS_DIR="$2" BASE_OVERRIDE="$3" EVENT_NAME="$4" EVENT_BEFORE="$5" + cd "$dir" || return 9 + identify_skills +} + +# Review-all fallback (workflow_dispatch / no base) returns every immediate +# skill-dir basename, sorted. FIXTURE holds alpha, beta, deleted, needs-credits. +got=$(capture_identify "$FIXTURE" "$FIXTURE" "" "workflow_dispatch" "") +assert_eq "$got" "$(printf 'alpha\nbeta\ndeleted\nneeds-credits')" "identify_skills: review-all lists every skill dir, sorted" + +# A throwaway git tree with a base commit and a HEAD that modifies alpha and +# adds beta — the diff base is HEAD~1, so both count as changed. +GITREPO="$FIXTURE/gitrepo" +make_git_fixture() { + rm -rf "$GITREPO" + mkdir -p "$GITREPO" || return 1 + ( + set -e + cd "$GITREPO" + git init -q + git config user.email t@e.test + git config user.name tester + mkdir -p skills/alpha; echo a > skills/alpha/SKILL.md + git add -A; git commit -q -m base + mkdir -p skills/beta; echo b > skills/beta/SKILL.md + echo more >> skills/alpha/SKILL.md + git add -A; git commit -q -m head + ) +} +make_git_fixture || { echo "fatal: could not build git fixture" >&2; exit 2; } +base_sha=$( cd "$GITREPO" && git rev-parse HEAD~1 ) + +got=$(capture_identify "$GITREPO" "skills" "" "push" "$base_sha") +assert_eq "$got" "$(printf 'alpha\nbeta')" "identify_skills: git-diff detects the modified and added skills" + +# An unreachable (non-sentinel) base with no remote to fetch from must +# hard-fail, never collapse to "no changes". +rc=0 +capture_identify "$GITREPO" "skills" "" "push" "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" >/dev/null 2>&1 || rc=$? +assert_eq "$rc" "2" "identify_skills: unreachable base hard-fails with exit 2" + +# --- main: emits the unreviewed-skills output --- + +# Run main with the given context; its GITHUB_OUTPUT write is the artifact +# under test. Config are `export`ed like capture_identify's, but this runs in +# the current shell (not a `$()`), so the exports persist afterward — harmless +# here: these are the final cases and each overwrites the values it needs. +capture_main() { + export SKILLS_DIR="$1" EVENT_NAME="$2" EVENT_BEFORE="$3" CREDIT_OUTAGE="$4" \ + GITHUB_OUTPUT="$5" GITHUB_STEP_SUMMARY="$6" BASE_OVERRIDE="" THRESHOLD="85" + main +} + +# Review-all over the fixture with a per-path mock: needs-credits hits the +# credit outage (skip), alpha/beta pass, deleted has no SKILL.md. main must +# write the outage skill to $GITHUB_OUTPUT. +GH_OUT="$FIXTURE/gh_output"; : > "$GH_OUT" +MOCK_MODE="by-path" +capture_main "$FIXTURE" "workflow_dispatch" "" "skip" "$GH_OUT" "$FIXTURE/sum2" >/dev/null 2>&1 +assert_eq "$(cat "$GH_OUT")" "unreviewed-skills=needs-credits" "main: writes the credit-skipped skill to GITHUB_OUTPUT" + +# A fully-clean review-all writes an empty output value. +GH_OUT2="$FIXTURE/gh_output2"; : > "$GH_OUT2" +MOCK_MODE="success" +capture_main "$FIXTURE" "workflow_dispatch" "" "skip" "$GH_OUT2" "$FIXTURE/sum3" >/dev/null 2>&1 +assert_eq "$(cat "$GH_OUT2")" "unreviewed-skills=" "main: clean review-all writes an empty unreviewed-skills value" + +echo "─────────────────────────────────────────────" +if [[ "$FAIL_COUNT" -gt 0 ]]; then + echo "FAILED: $FAIL_COUNT failed, $PASS_COUNT passed." >&2 + exit 1 +fi +echo "PASSED: all $PASS_COUNT assertion(s)." diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0a75c1e2..42c87741 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -48,6 +48,13 @@ jobs: uses: ./.github/actions/skill-review with: threshold: '85' + # Opt into the fail-safe out-of-credits tolerance (issue #188): + # a tessl billing 403 skips the affected skill (flagged in the run + # summary and unreviewed-skills output) instead of blocking every + # skill-changing merge until credits reset. Every other review + # failure still hard-fails. Sanctioned by the context-artifacts + # Credit-Outage Review Carve-Out. + credit-outage: 'skip' # Dogfoods the reusable action consumers call as # `uses: jbaruch/coding-policy/.github/actions/stamp-changelog@`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c389b76..b84413a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Rules +- **`context-artifacts`: Credit-Outage Review Carve-Out** — a tessl out-of-credits (403) billing outage can now satisfy the Mandatory Review gate under an opt-in, fail-safe skip: only the credit-phrase-plus-403 signature skips the affected skill (flagged in the run summary and the `unreviewed-skills` output), every other review failure still hard-fails, and the gate self-heals when credits return. Mirrors the `ci-safety` Publish-Pipeline carve-out shape. Distinct from the forbidden review-step bypass — that dodges a verdict, this tolerates an outage that produced none. Motivation: issue #188. - **`error-handling`: new Shell Error Handling section** — the rule was entirely silent on shell (no `set -euo pipefail`, no suppression ban), so consumers installing the plugin inherited none of the discipline the repo itself practises in all 18 of its non-test scripts. Encodes: mandatory `set -euo pipefail`; no `|| true` / `|| :` / bare `2>/dev/null`; explicit `if`/`case` on exit codes for commands that can legitimately fail; the distinction between an expected non-result and a tool failure (`grep` exits 1 on no-match, 2 on error — `|| true` collapses both); that silencing a *diagnostic* while explicitly handling the *failure* is not suppression; and that "fail visibly" permits a stderr warning rather than requiring exit non-zero. Full motivation: PR #184. ### Skills @@ -9,6 +10,10 @@ - **All 16 production `|| true` sites converted to explicit exit-code handling** — landed with the rule above so it ships describing a repo that already obeys it (`rules/context-artifacts.md` Post-Edit Rule Audit). Three patterns: `grep`-no-match (`verify-publish-landed`, `scaffold`, `preflight`, `migrate`, `adopt`) now branch on rc so exit 2 is no longer read as "nothing found"; `jq` extraction (`verify-moderation-cleared`) validates the body once so a non-JSON response stops blaming the registry's shape; best-effort cleanup (`scaffold`, `adopt`) keeps its lenient behaviour but warns to stderr instead of skipping in silence. Full motivation: PR #184. - **Post-audit follow-through: remaining bare cleanup `rm` swept, and an EXIT-trap tempfile leak fixed** — the bare cleanup `rm` sites the `|| true` sweep left untouched (`scaffold` rollback + success paths, `run-diagnostics`, `run-tests`) now route through a `discard` helper that warns on a failed remove instead of aborting the operation (`set -e`) or silently continuing (`set +e`), so the committed repo satisfies the new "best-effort warns, never nothing" clause everywhere the reviewer flagged. Separately, `verify-publish-landed` and `verify-moderation-cleared` declared their EXIT-trap `err_file` as a `main`-local, but the trap fires after `main` returns — on the normal success path the handler saw an empty variable and leaked the tempfile every run; only the `exit`-from-inside-`main` error paths cleaned up. `err_file` is now script-global. Full motivation: PR #184. +### Build + +- **`skill-review` action: native out-of-credits tolerance (#188)** — new `credit-outage: fail | skip` input (default `fail` preserves today's Mandatory Review gate) so consumers no longer vendor the action to survive a tessl billing 403. `jbaruch/nanoclaw-travel` had copied the whole composite verbatim just to wrap the review call; that drift is what this removes. The composite body moved from an inline `run:` block into a real, unit-tested `review-skills.sh` (per `rules/script-delegation.md`). Detection requires the credit phrase **and** a `403`, both fixed strings (issue #188 follow-up: a looser substring match over review output would publish a genuinely-broken skill unreviewed) — anything else, including a future tessl wording change, falls through to hard-fail. Skipped skills surface on a new `unreviewed-skills` output and a `$GITHUB_STEP_SUMMARY` note. `run-diagnostics.sh` now shellchecks `.github/actions/` too, and this repo's own `publish.yml` opts into `credit-outage: skip`. Covered by `.github/actions/skill-review/tests/test_review_skills.sh`. + ## 0.3.93 — 2026-07-17 ### Skills diff --git a/rules/context-artifacts.md b/rules/context-artifacts.md index 14812bbc..f0a79ad9 100644 --- a/rules/context-artifacts.md +++ b/rules/context-artifacts.md @@ -37,6 +37,20 @@ description: Plugin structure, rule/skill format, review pipeline, surface sync, - Fallback: review every skill when the diff base is absent (manual `workflow_dispatch`, initial push, all-zeros sentinel SHA); hard-fail when the base is set but unreachable - Rubric verifies: frontmatter validity, execution-mode preamble matching the skill's shape (sequential workflow or action router per `rules/skill-authoring.md`), flat step numbering, typed `Skill()` calls, silence-rule compliance, channel-appropriate formatting - Act on concrete feedback (tighter triggers, extracted reference material, tightened descriptions); re-review until the gate passes +- Credit-outage tolerance is opt-in and narrow — see Credit-Outage Review Carve-Out + +## Credit-Outage Review Carve-Out + +- Narrow exception for skipping review during a tessl out-of-credits (403) billing outage +- Applies when `tessl skill review` fails with the out-of-credits signature; never a below-threshold score +- The `skill-review` action's `credit-outage: skip` input publishes the affected skill unreviewed rather than blocking every skill-changing merge until credits return +- Preconditions (all required): + 1. Opt-in explicit — the consumer sets `credit-outage: skip`; default `fail` preserves the gate. Classification is the action's decision contract — see `.github/actions/skill-review/review-skills.sh` `run_reviews` + 2. Fail-safe — only the out-of-credits signature skips; every other non-zero exit still hard-fails the publish (below-threshold score, auth error, tooling bug) + 3. Each unreviewed publish is flagged — a `::warning::` plus a `$GITHUB_STEP_SUMMARY` note names every skipped skill, surfaced on the `unreviewed-skills` action output + 4. The gate self-heals — every publish re-attempts review; a credit top-up or the monthly reset restores it +- The forbidden review-step bypass in Disagreeing With the Reviewer remains forbidden +- Every skill reviewed clean, and every non-credit failure, still blocks or passes exactly as Mandatory Review prescribes ## Disagreeing With the Reviewer diff --git a/scripts/run-diagnostics.sh b/scripts/run-diagnostics.sh index 8eabd431..427ec4e7 100755 --- a/scripts/run-diagnostics.sh +++ b/scripts/run-diagnostics.sh @@ -22,9 +22,10 @@ # Output: # stderr: each engine's native findings plus per-engine progress. # Usage: scripts/run-diagnostics.sh [base-dir] -# base-dir Tree whose skills/ and scripts/ are scanned for shell -# scripts. Defaults to the repo root. The optional arg exists -# so the runner's own test can point it at a fixture tree. +# base-dir Tree whose skills/, scripts/, and .github/actions/ are +# scanned for shell scripts. Defaults to the repo root. The +# optional arg exists so the runner's own test can point it at +# a fixture tree. # pyright always runs against pyrightconfig.json at the cwd, # independent of base-dir. # Exit: 0 if both engines report zero findings, 1 on any finding, 2 on @@ -63,8 +64,12 @@ main() { local roots=() [[ -d "$base/skills" ]] && roots+=("$base/skills") [[ -d "$base/scripts" ]] && roots+=("$base/scripts") + # Composite-action bodies extracted to real scripts per + # rules/script-delegation.md (e.g. skill-review/review-skills.sh) — gate + # them too so an action script is held to the same zero-findings bar. + [[ -d "$base/.github/actions" ]] && roots+=("$base/.github/actions") if [[ ${#roots[@]} -eq 0 ]]; then - echo "run-diagnostics: neither skills/ nor scripts/ found under $base" >&2 + echo "run-diagnostics: none of skills/, scripts/, .github/actions/ found under $base" >&2 return 2 fi @@ -84,7 +89,7 @@ main() { discard "$tmplist" if [[ ${#scripts[@]} -eq 0 ]]; then - echo "run-diagnostics: no shell scripts found under ${base}/{skills,scripts}" >&2 + echo "run-diagnostics: no shell scripts found under ${base}/{skills,scripts,.github/actions}" >&2 return 2 fi