diff --git a/.claude/scripts/agent-telemetry.sh b/.claude/scripts/agent-telemetry.sh index 8130edd5..eba932cf 100755 --- a/.claude/scripts/agent-telemetry.sh +++ b/.claude/scripts/agent-telemetry.sh @@ -235,13 +235,90 @@ fi # schemas. Behavioural metrics must never be grepped from raw transcript text: # untrusted prose that merely mentions `sleep 60` would otherwise manufacture a # busy-wait pattern, and busy-wait counts are evidence the improver acts on. +# +# Every command, unclassified. Launch-mode classification lives ONLY in +# tagged_commands_in — one classifier, so there is no second copy to drift. +# The ONE definition of "this tool call never executed", shared by the safety +# section's denial detector and the sleep classifier. Kept as a single constant +# because two hand-maintained copies of a shape list is exactly how detector +# parity broke six times on the redactor: the sleep classifier originally +# matched a narrower subset, so a permission-DENIED sleep still counted as a +# foreground launch. +# +# Deliberately NOT `is_error`, which is far broader than "never ran" — a +# TIMED-OUT sleep carries is_error:true and is the single most expensive real +# block in the corpus, so suppressing it would hide the worst case while +# claiming to measure it. +# +# ANCHORED to the harness's denial envelope, not searched anywhere in the text. +# An executed command that sleeps and later fails while printing application or +# test output containing "approval denied for tool" would otherwise be treated +# as never-run and removed from every class — silently deleting a real block, +# which is the same failure mode as the is_error over-match this replaced. +NEVER_RAN_SHAPES='Blocked:|Permission to use [A-Za-z_]+ with command|Claude requested permissions to use|approval (denied|required) for tool' +NEVER_RAN_RE='^[[:space:]]*()?[[:space:]]*('"$NEVER_RAN_SHAPES"')' + +# tool_use ids whose result shows the call never ran, resolved once per file. +# +# The content is NORMALISED to its text before matching, exactly as the safety +# detector does. A tool_result may carry `content` as a plain string OR as the +# array-of-text-blocks shape; `tostring` on the array yields `[{"type":"text"…`, +# so an anchored pattern never reaches the denial text and the call is counted +# as an executed foreground launch even though it never ran. Anchoring without +# normalising is precisely that bug. +denied_ids() { + jq -Rr --arg re "$NEVER_RAN_RE" 'select(length>0)|(try fromjson catch empty) + | select(.type=="user") | .message.content[]? + | select(.type=="tool_result" and .is_error==true) + | select((.content + | if type=="array" then (map(select(.type=="text").text // empty)|join(" ")) + elif type=="string" then . else tostring end) + | test($re)) + | .tool_use_id // empty' "$1" 2>/dev/null \ + | jq -Rs 'split("\n")|map(select(length>0))' 2>/dev/null +} + +# ONE traversal per transcript emitting EVERY command tagged with its launch +# class, so the three class counts come from a single consistent read instead of +# one re-parse per class. Each LINE of a command is prefixed \001\002 so a +# multi-line command survives intact; control characters are used because no +# real shell command line starts with one, which keeps untrusted command text +# from forging a tag and moving itself between classes. +tagged_commands_in() { + local f="$1" errs + errs=$(denied_ids "$f"); [ -n "$errs" ] || errs='[]' + jq -r --argjson errs "$errs" ' + .. | objects + | ( + (select(.type=="tool_use") + | ((if .input?.run_in_background == true then "BG" else "FG" end)) as $c + | (.id? // "") as $i + | select($i == "" or (($errs | index($i)) | not)) + | (.input?.command? // empty) | select(type=="string") | select(length>0) + | split("\n") | map("\u0001" + $c + "\u0002" + .) | .[]), + (select(.type=="function_call") + | (.arguments? // empty) + | (try (fromjson | (.command? // .cmd? // empty)) catch empty) + | select(type=="string") | select(length>0) + | split("\n") | map("\u0001CX\u0002" + .) | .[]), + (select(.type=="custom_tool_call") + | .input? // empty | select(type=="string") + | [scan("cmd:\\s*\"((?:[^\"\\\\]|\\\\.)*)\"")] | .[]? | .[0]? + | select(type=="string") | select(length>0) + | gsub("\\\\n"; "\n") | gsub("\\\\t"; " ") | gsub("\\\\\""; "\"") + | split("\n") | map("\u0001CX\u0002" + .) | .[]) + ) + ' "$f" 2>/dev/null +} + commands_in() { local f="$1" jq -r ' .. | objects | ( # Claude: tool_use carrying a Bash command. - (select(.type=="tool_use") | .input?.command? // empty), + (select(.type=="tool_use") + | .input?.command? // empty), # Codex, JSON-argument shape (function_call). (select(.type=="function_call") | .arguments? // empty @@ -527,14 +604,90 @@ if want efficiency; then # STRUCTURAL, not a text grep: only commands the agent actually ran count. # A grep would let untrusted prose that merely mentions `sleep 60` fabricate # a busy-wait pattern, and this metric is evidence for definition changes. - SLEEPS=$(printf '%s\n%s\n' "$SF_CACHE" "$CX_CACHE" | grep -v '^$' \ - | while IFS= read -r f; do commands_in "$f"; done \ - | strip_heredocs \ - | grep -cE '(^|[;&|(]|&&|\|\||[[:space:]](do|then|else)[[:space:]])[[:space:]]*sleep[[:space:]]+["'"'"']?[$0-9{]' || true) + # ONE definition of "this command sleeps", shared by the total and every + # class count below. Duplicating the regex is how the two would drift apart + # and stop summing. + count_sleeps() { + strip_heredocs \ + | grep -cE '(^|[;&|(]|&&|\|\||[[:space:]](do|then|else)[[:space:]])[[:space:]]*sleep[[:space:]]+["'"'"']?[$0-9{]' || true + } + # The corpus is LIVE: the sibling instance writes transcripts while we read. + # Scanning once per class could observe a different corpus each time, so a + # sleep appended mid-run would make the classes disagree with a separately + # scanned total. Snapshot each file ONCE and derive every class from that + # copy, so the counts are mutually consistent by construction rather than + # merely checked afterwards. + # ONE tagged pass per transcript, held in memory — no temp copy. + # + # The earlier design copied each transcript to a temp dir so the classes + # could not disagree. That copy was an UNREDACTED transcript holding + # potential credentials, and protecting it turned out to be unfixable in + # place: `main` is the left side of `main | redact`, so it runs in a + # pipeline SUBSHELL and a trap set here never fires when the scheduler + # signals the top-level PID. Rather than harden the copy, the copy is gone — + # a single pass is inherently self-consistent, needs no snapshot, and cannot + # leave anything on disk to clean up. + TAGGED=$(printf '%s\n%s\n' "$SF_CACHE" "$CX_CACHE" | grep -v '^$' \ + | while IFS= read -r f; do tagged_commands_in "$f"; done) + # Each LINE of a command carries its class tag, so multi-line commands keep + # their structure and their order within a class — which the separator- + # anchored sleep regex and the heredoc stripper both depend on. + class_lines() { printf '%s\n' "$TAGGED" | awk -v c="$1" ' + index($0, "\001" c "\002")==1 { print substr($0, length(c)+3) }'; } + SLEEP_FG=$(class_lines FG | count_sleeps) + SLEEP_BG=$(class_lines BG | count_sleeps) + SLEEP_CX=$(class_lines CX | count_sleeps) + # The total is the SUM of the classes, not a separate scan. That makes + # class-vs-total drift impossible instead of detectable — and a drift + # warning over a sum would be a vacuous guard, which is worse than none. + # Classification is exhaustive by construction (a Claude tool_use is FG or + # BG; a Codex call is CX; there is no fourth command source). + SLEEPS=$((SLEEP_FG + SLEEP_BG + SLEEP_CX)) echo " [BOTH instances: ${SF_COUNT} Claude + ${CX_COUNT} Codex sessions]" echo " bash timeouts .............. ${TIMEOUTS} (each = a foreground block that produced nothing)" echo " interrupted tool calls ..... ${INTERRUPT}" echo " explicit sleep/poll calls .. ${SLEEPS} (contract: arm a watcher, never busy-wait)" + # The raw total above cannot answer the question the contract actually asks, + # because it scores a CONTRACT-COMPLIANT backgrounded watcher (`sleep N && + # check`, run_in_background) identically to a foreground busy-wait — and the + # improver's own runs emit several compliant watchers each, landing as + # self-noise in the very bucket used to judge the agents. These lines split + # the two. This SHARPENS the measurement; it removes nothing. + echo " ├ foreground launch ...... ${SLEEP_FG} [Claude, synchronous]" + echo " ├ background launch ...... ${SLEEP_BG} [Claude, run_in_background]" + echo " └ launch mode unknown .... ${SLEEP_CX} [Codex — see gap note below]" + # Rates, because a raw total is not a rate. The window selects FILES by + # mtime, so session counts swing hard day to day and a raw count fell while + # the per-session rate ROSE (442→328 total, but 2.02→3.73/session). Every + # trend claim must name its denominator. + if [ "$SF_COUNT" -gt 0 ]; then + echo " per-session (Claude, n=${SF_COUNT}): foreground $(awk -v a="$SLEEP_FG" -v b="$SF_COUNT" 'BEGIN{printf "%.2f", a/b}')/session, deferred $(awk -v a="$SLEEP_BG" -v b="$SF_COUNT" 'BEGIN{printf "%.2f", a/b}')/session" + fi + if [ "$CX_COUNT" -gt 0 ]; then + echo " per-session (Codex, n=${CX_COUNT}): unclassified $(awk -v a="$SLEEP_CX" -v b="$CX_COUNT" 'BEGIN{printf "%.2f", a/b}')/session" + fi + echo " NOTE: this splits LAUNCH MODE, which is NOT a compliance verdict." + echo " run_in_background says how Bash started the command, never" + echo " why the sleep exists. The contract permits a FOREGROUND bare" + echo " sleep as a local timer for a process the agent itself" + echo " started, and a BACKGROUND sleep can still be a redundant" + echo " poll alongside foreground polling. So a foreground count is" + echo " a busy-wait CANDIDATE, not a violation, and a background" + echo " count is not an exoneration — correlate with what was being" + echo " waited on before drawing a conclusion." + echo " The split is STRUCTURAL (read off the tool call), so prose" + echo " cannot fake it. The key is OMITTED when false, so absence is" + echo " correctly read as foreground. HOOK-REJECTED and" + echo " PERMISSION-DENIED calls are excluded from these classes" + echo " because they never ran; a call that ran and FAILED still" + echo " counts, including a TIMED-OUT sleep — that one blocked" + echo " longest and is the last thing to hide." + echo " CODEX IS A STATED GAP, NOT A ZERO: 767 live exec_command" + echo " calls carried yield_time_ms (an output-read timeout) and" + echo " ZERO carried any background flag, so that runtime exposes no" + echo " backgrounding surface to classify. Codex sleeps are counted" + echo " but NOT attributed — never read 'unclassified' as compliant" + echo " or as a violation." echo echo " descriptions of commands that ACTUALLY timed out:" # Correlated by tool_use_id, not a bare grep over every description. The @@ -591,7 +744,7 @@ if want safety; then # counts decide guard-vs-agent, so inflating them argues for loosening a # guard that never actually fired. printf '%s\n%s\n' "$SF_CACHE" "$CX_CACHE" | grep -v '^$' | while IFS= read -r f; do - jq -r ' + jq -r --arg never_ran "$NEVER_RAN_RE" ' .. | objects | ( (select(.type=="tool_result" and .is_error==true) @@ -610,7 +763,7 @@ if want safety; then elif type=="string" then . else empty end) ) | select(type=="string") - | select(test("[[:space:]]*Blocked:|^[[:space:]]*Permission to use [A-Za-z_]+ with command|Claude requested permissions to use|approval (denied|required) for tool")) + | select(test($never_ran)) | .[0:80] ' "$f" 2>/dev/null done | redact | sed -E 's/[0-9]+//g' | sort | uniq -c | sort -rn | head -10 | sed 's/^/ /' diff --git a/.claude/scripts/agent-telemetry.test.sh b/.claude/scripts/agent-telemetry.test.sh index 6a28125e..75e31d4a 100755 --- a/.claude/scripts/agent-telemetry.test.sh +++ b/.claude/scripts/agent-telemetry.test.sh @@ -1023,6 +1023,163 @@ check "the Codex denial gap stays disclosed" "$OUT" "CLAUDE-SCHEMA ONLY" OUT=$(run --section reliability) check "the mtime window bound is disclosed" "$OUT" "selects FILES by mtime" +# ── 6f. sleep background classification ─────────────────────────────────────── +echo +echo "sleep classification (launch mode: foreground vs background)" + +# A BACKGROUNDED sleep is the contract's CHEAP WATCHER, not a busy-wait. Scoring +# it as waste is what made the raw total unusable as evidence — the improver's +# own runs arm several per tick, polluting the bucket used to judge the agents. +mkdir -p "$FIX/bgsleep" +cat > "$FIX/bgsleep/s.jsonl" <<'EOF' +{"type":"assistant","message":{"content":[{"type":"tool_use","id":"b1","name":"Bash","input":{"command":"sleep 300 && gh pr checks 1","run_in_background":true}}]}} +EOF +OUT=$(CLAUDE_PROJECTS_DIR="$FIX/bgsleep" CODEX_HOME="$FIX/nocodex" MONOREPO_DIR="$FIX/monorepo" HOME="$FIX" \ + bash "$TARGET" --since-days 3650 --section efficiency 2>&1) +if printf '%s' "$OUT" | grep -qE 'background launch \.+ 1'; then + ok "a backgrounded sleep is classified background-launch" +else bad "a backgrounded sleep is classified background-launch" "$(printf '%s' "$OUT" | grep -E 'foreground|deferred|unclass')"; fi +if printf '%s' "$OUT" | grep -qE 'foreground launch \.+ 0'; then + ok "...and is NOT classified foreground-launch" +else bad "...and is NOT classified foreground-launch" "$(printf '%s' "$OUT" | grep -E 'foreground')"; fi + +# The key is OMITTED when false, so ABSENCE must read as foreground. If this +# regressed to "absent => unknown", every ordinary busy-wait would vanish. +mkdir -p "$FIX/fgsleep" +cat > "$FIX/fgsleep/s.jsonl" <<'EOF' +{"type":"assistant","message":{"content":[{"type":"tool_use","id":"f1","name":"Bash","input":{"command":"sleep 60"}}]}} +{"type":"assistant","message":{"content":[{"type":"tool_use","id":"f2","name":"Bash","input":{"command":"sleep 30","run_in_background":false}}]}} +EOF +OUT=$(CLAUDE_PROJECTS_DIR="$FIX/fgsleep" CODEX_HOME="$FIX/nocodex" MONOREPO_DIR="$FIX/monorepo" HOME="$FIX" \ + bash "$TARGET" --since-days 3650 --section efficiency 2>&1) +if printf '%s' "$OUT" | grep -qE 'foreground launch \.+ 2'; then + ok "an omitted AND an explicit-false flag both read as foreground" +else bad "an omitted AND an explicit-false flag both read as foreground" "$(printf '%s' "$OUT" | grep -E 'foreground|deferred')"; fi + +# Codex exposes NO backgrounding surface (767 live exec calls: yield_time_ms +# only, zero background flags). Its sleeps must land in `unclassified` — folding +# them into foreground would invent an attribution the data cannot support. +OUT=$(CLAUDE_PROJECTS_DIR="$FIX/empty" CODEX_HOME="$FIX/cxreal" MONOREPO_DIR="$FIX/monorepo" HOME="$FIX" \ + bash "$TARGET" --since-days 3650 --section efficiency 2>&1) +if printf '%s' "$OUT" | grep -qE 'launch mode unknown \.+ 1'; then + ok "a Codex sleep has unknown launch mode" +else bad "a Codex sleep has unknown launch mode" "$(printf '%s' "$OUT" | grep -E 'unclass|foreground')"; fi +if printf '%s' "$OUT" | grep -qE 'foreground launch \.+ 0'; then + ok "...and is NOT attributed to foreground" +else bad "...and is NOT attributed to foreground" "$(printf '%s' "$OUT" | grep -E 'foreground')"; fi +check "the Codex classification gap is stated" "$OUT" "STATED GAP, NOT A ZERO" +check "launch mode is not presented as a compliance verdict" "$OUT" "NOT a compliance verdict" + +# The classes must SUM to the total, or a trend is read off a broken split. +# Mixed corpus: 2 Claude (1 fg + 1 bg) + 1 Codex = 3. +OUT=$(CLAUDE_PROJECTS_DIR="$FIX/bgsleep" CODEX_HOME="$FIX/cxreal" MONOREPO_DIR="$FIX/monorepo" HOME="$FIX" \ + bash "$TARGET" --since-days 3650 --section efficiency 2>&1) +if printf '%s' "$OUT" | grep -qE 'sleep/poll calls \.\. 2' \ + && printf '%s' "$OUT" | grep -qE 'background launch \.+ 1' \ + && printf '%s' "$OUT" | grep -qE 'launch mode unknown \.+ 1'; then + ok "classes sum to the total across both instances" +else bad "classes sum to the total across both instances" "$(printf '%s' "$OUT" | grep -E 'sleep/poll|foreground|deferred|unclass')"; fi +# The total is DERIVED from the classes now, so a drift warning would be a +# vacuous guard. Exhaustiveness is what must hold instead: every counted sleep +# lands in exactly one class, which the sum check above asserts directly. + +# Rates carry their denominator — a raw total is not a rate, and comparing one +# against the other is exactly how the 07-19 sleep reading was misread as a win. +check "per-session rate states its denominator" "$OUT" "per-session (Claude, n=" + +# A sleep the enforcement hook REJECTED never ran. Counting it as a foreground +# block reports time the agent never spent blocking, and inflates precisely the +# metric the hook exists to drive down. +mkdir -p "$FIX/blockedsleep" +cat > "$FIX/blockedsleep/s.jsonl" <<'EOF' +{"type":"assistant","message":{"content":[{"type":"tool_use","id":"blk1","name":"Bash","input":{"command":"sleep 60 && gh pr checks 1"}}]}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"blk1","is_error":true,"content":"Blocked: sleep 60 followed by: gh pr checks 1"}]}} +{"type":"assistant","message":{"content":[{"type":"tool_use","id":"ok1","name":"Bash","input":{"command":"sleep 5"}}]}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"ok1","is_error":false,"content":"done"}]}} +EOF +OUT=$(CLAUDE_PROJECTS_DIR="$FIX/blockedsleep" CODEX_HOME="$FIX/nocodex" MONOREPO_DIR="$FIX/monorepo" HOME="$FIX" \ + bash "$TARGET" --since-days 3650 --section efficiency 2>&1) +if printf '%s' "$OUT" | grep -qE 'foreground launch \.+ 1'; then + ok "a hook-BLOCKED sleep is excluded; the executed one still counts" +else bad "a hook-BLOCKED sleep is excluded; the executed one still counts" "$(printf '%s' "$OUT" | grep -E 'foreground|sleep/poll')"; fi + +# A TIMED-OUT sleep also carries is_error:true, but it RAN — and it is the most +# expensive block in the corpus. Excluding it (by keying on is_error rather than +# on the never-ran shapes) would hide the worst case while claiming to measure +# busy-waiting. This guard exists because that regression was written and caught. +mkdir -p "$FIX/timedoutsleep" +cat > "$FIX/timedoutsleep/s.jsonl" <<'EOF' +{"type":"assistant","message":{"content":[{"type":"tool_use","id":"to1","name":"Bash","input":{"command":"sleep 600"}}]}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"to1","is_error":true,"content":[{"type":"text","text":"Command timed out after 2m 0s"}]}]}} +EOF +OUT=$(CLAUDE_PROJECTS_DIR="$FIX/timedoutsleep" CODEX_HOME="$FIX/nocodex" MONOREPO_DIR="$FIX/monorepo" HOME="$FIX" \ + bash "$TARGET" --since-days 3650 --section efficiency 2>&1) +if printf '%s' "$OUT" | grep -qE 'foreground launch \.+ 1'; then + ok "a TIMED-OUT sleep still counts (it ran; is_error is not 'never ran')" +else bad "a TIMED-OUT sleep still counts (it ran; is_error is not 'never ran')" "$(printf '%s' "$OUT" | grep -E 'foreground|sleep/poll')"; fi + +# EVERY recognised never-ran shape must be excluded, not just the hook's. The +# sleep classifier and the safety detector share ONE regex constant precisely so +# a shape recognised by one cannot be missed by the other. +# NOTE the two content SHAPES here: a plain string and the array-of-text-blocks +# form. Both are supported by the harness, and an anchored pattern applied to +# `tostring` of the array sees `[{"type":"text"…` and never reaches the denial +# text — so the array case must be covered explicitly or the anchoring silently +# counts a never-run sleep as an executed foreground launch. +mkdir -p "$FIX/deniedsleep" +cat > "$FIX/deniedsleep/s.jsonl" <<'EOF' +{"type":"assistant","message":{"content":[{"type":"tool_use","id":"d1","name":"Bash","input":{"command":"sleep 60"}}]}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"d1","is_error":true,"content":"Claude requested permissions to use Bash, but you have not granted it yet"}]}} +{"type":"assistant","message":{"content":[{"type":"tool_use","id":"d2","name":"Bash","input":{"command":"sleep 45"}}]}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"d2","is_error":true,"content":[{"type":"text","text":"approval denied for tool Bash"}]}]}} +{"type":"assistant","message":{"content":[{"type":"tool_use","id":"d3","name":"Bash","input":{"command":"sleep 30"}}]}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"d3","is_error":true,"content":[{"type":"text","text":"Blocked: sleep 30 followed by: gh pr checks"}]}]}} +EOF +OUT=$(CLAUDE_PROJECTS_DIR="$FIX/deniedsleep" CODEX_HOME="$FIX/nocodex" MONOREPO_DIR="$FIX/monorepo" HOME="$FIX" \ + bash "$TARGET" --since-days 3650 --section efficiency 2>&1) +if printf '%s' "$OUT" | grep -qE 'foreground launch \.+ 0'; then + ok "permission-denied sleeps are excluded too, not just hook-blocked ones" +else bad "permission-denied sleeps are excluded too, not just hook-blocked ones" "$(printf '%s' "$OUT" | grep -E 'foreground|sleep/poll')"; fi + +# The snapshot holds an UNREDACTED transcript copy plus extracted command text, +# so a signal must not leave it in /tmp. +# +# STRUCTURAL check, and the limitation is stated rather than papered over: an +# end-to-end "kill the miner mid-snapshot" test was attempted TWICE and was +# VACUOUS both times — first the run finished before the signal landed, then the +# signal landed before the snapshot was created, and on a corpus large enough to +# widen the window the miner still self-cleaned before the TERM arrived. Both +# versions passed with SNAP deliberately removed from the traps, which is the +# definition of a guard that cannot fail. A deterministic assertion that the +# signal traps actually cover SNAP is worth more than a green test that proves +# nothing; the residual gap is that this reads the registration, not the effect. +# The classifier must never write an unredacted transcript copy to disk. An +# earlier revision snapshotted each transcript to a temp dir to keep the class +# counts consistent, which put credential-bearing content in /tmp — and it could +# not be protected in place, because `main` runs in a pipeline subshell so a trap +# set there never fires when the scheduler signals the top-level PID. The copy +# was removed rather than hardened; this guard stops it coming back. +SNAPLEFT=$(ls -d "${TMPDIR:-/tmp}"/.agtel_snap.* 2>/dev/null | wc -l | tr -d ' ') +OUT=$(CLAUDE_PROJECTS_DIR="$FIX/projects" CODEX_HOME="$FIX/codex" MONOREPO_DIR="$FIX/monorepo" HOME="$FIX" \ + bash "$TARGET" --since-days 3650 --section efficiency 2>&1) +SNAPNOW=$(ls -d "${TMPDIR:-/tmp}"/.agtel_snap.* 2>/dev/null | wc -l | tr -d ' ') +if [ "$SNAPNOW" -le "$SNAPLEFT" ]; then + ok "no transcript snapshot directory is left on disk" +else bad "no transcript snapshot directory is left on disk" "before=$SNAPLEFT after=$SNAPNOW"; fi +if grep -q 'cp "$f" "$SNAP/cur"' "$TARGET"; then + bad "the classifier does not copy raw transcripts to a temp dir" "found a raw transcript cp" +else ok "the classifier does not copy raw transcripts to a temp dir"; fi + +# A command's own text must not be able to forge a class tag and move itself +# between classes — the tag is control-character delimited for exactly this. +mkdir -p "$FIX/forgetag" +printf '%s\n' '{"type":"assistant","message":{"content":[{"type":"tool_use","id":"x1","name":"Bash","input":{"command":"echo BGnot-a-real-tag\nsleep 60"}}]}}' > "$FIX/forgetag/s.jsonl" +OUT=$(CLAUDE_PROJECTS_DIR="$FIX/forgetag" CODEX_HOME="$FIX/nocodex" MONOREPO_DIR="$FIX/monorepo" HOME="$FIX" \ + bash "$TARGET" --since-days 3650 --section efficiency 2>&1) +if printf '%s' "$OUT" | grep -qE 'foreground launch \.+ 1' && printf '%s' "$OUT" | grep -qE 'background launch \.+ 0'; then + ok "command text cannot forge a class tag" +else bad "command text cannot forge a class tag" "$(printf '%s' "$OUT" | grep -E 'foreground|background')"; fi + # ── 7. robustness ───────────────────────────────────────────────────────────── echo echo "robustness" diff --git a/.claude/skills/agent-improvement/SKILL.md b/.claude/skills/agent-improvement/SKILL.md index b4a8fed8..b2db6e34 100644 --- a/.claude/skills/agent-improvement/SKILL.md +++ b/.claude/skills/agent-improvement/SKILL.md @@ -65,7 +65,8 @@ than absolute values, and only a recorded number can trend. date_utc, window_days reliability: tool_error_count, top_signatures[], timeout_count safety: blocked_actions[], near_misses[], credential_hits, injection_attempts_in_corpus -efficiency: sleep_poll_calls, timeouts, redundant_call_patterns[] +efficiency: sleep_poll_calls{total, fg_per_session, bg_per_session, codex_unclassified}, + timeouts, redundant_call_patterns[] quality: merged_prs, reverts, post_merge_red, review_findings_per_pr coordination: two_writer_races, push_collisions, duplicate_artifacts[] currency: loader_drift[], stale_memory[], unused_capabilities[] @@ -78,6 +79,20 @@ The `flow` row is the Kanban Kata's measurement surface ([monorepo#2271](https:/ it comes from `flow-scorecard.sh`, and its stated gaps (UI-only column limits, lifetime as a cycle-time proxy) are part of the record — never silently paper over them. +**Record sleeps as RATES, split by class — a raw total is not a rate.** The window selects files by +mtime, so session counts swing hard day to day: a raw sleep total once fell 442→328 while the +per-session rate *rose* 2.02→3.73, and the fall was read as a win. Always state the denominator, and +never compare a raw count against a per-session one. + +**The class is a LAUNCH MODE, never a compliance verdict.** `foreground launch` and `background +launch` say only how the command was started. The contract permits a foreground bare sleep as a +local timer for a process the agent itself started, and a backgrounded sleep can still be a +redundant poll running alongside foreground polling — so a foreground count is a busy-wait +*candidate* to investigate, not a violation to report, and a background count is not an +exoneration. Correlate with what was actually being waited on before drawing any conclusion. Codex's +sleeps carry no launch mode at all (that runtime exposes no backgrounding flag) and are +**unattributed** — never fold them into either class. + **A metric that moved the wrong way since yesterday outranks a new finding** — regression first. Verify any change from a previous run that is awaiting confirmation (step 5) before starting new work.