From d0b984d318eb422d9653f9d7ccbb5a7cc97bd409 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 10:34:51 +1000 Subject: [PATCH 01/17] feat: balance dispatch by quota run rate --- bin/fm-bootstrap.sh | 71 ++++++ bin/fm-dispatch-select.sh | 362 ++++++++++++++++++++++++++----- docs/configuration.md | 49 ++++- tests/fm-bootstrap.test.sh | 6 + tests/fm-dispatch-select.test.sh | 347 ++++++++++++++++++++--------- 5 files changed, 673 insertions(+), 162 deletions(-) diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 753c5cec3..723ecdacf 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -679,6 +679,76 @@ crew_dispatch_validate() { | map(select(. as $p | effort_ok($p.h; $p.e) | not)) | map("\(.h):\(.e)") | unique; + def nonnegative_number($value): ($value | type) == "number" and $value >= 0; + def positive_integer($value): + ($value | type) == "number" and $value > 0 and ($value | floor) == $value; + def nonnegative_integer($value): + ($value | type) == "number" and $value >= 0 and ($value | floor) == $value; + def quota_balanced_error: + (.quotaBalanced? // {}) as $quota + | if has("quotaBalanced") and ((.quotaBalanced | type) != "object") then + "quotaBalanced must be an object" + elif (($quota | keys_unsorted) - ["providers","reservePercent","runRate","staleClearMargin"] | length) > 0 then + "quotaBalanced has unknown fields" + elif $quota.reservePercent? != null + and ((nonnegative_number($quota.reservePercent) | not) or $quota.reservePercent > 100) then + "quotaBalanced.reservePercent must be between 0 and 100" + elif $quota.staleClearMargin? != null and (nonnegative_number($quota.staleClearMargin) | not) then + "quotaBalanced.staleClearMargin must be non-negative" + elif $quota.runRate? != null and (($quota.runRate | type) != "object") then + "quotaBalanced.runRate must be an object" + elif ((($quota.runRate? // {}) | keys_unsorted) + - ["historyMaxAgeSeconds","minimumObservationSeconds","recentWeight"] | length) > 0 then + "quotaBalanced.runRate has unknown fields" + elif $quota.runRate.minimumObservationSeconds? != null + and (positive_integer($quota.runRate.minimumObservationSeconds) | not) then + "quotaBalanced.runRate.minimumObservationSeconds must be a positive integer" + elif $quota.runRate.historyMaxAgeSeconds? != null + and (positive_integer($quota.runRate.historyMaxAgeSeconds) | not) then + "quotaBalanced.runRate.historyMaxAgeSeconds must be a positive integer" + elif $quota.runRate.recentWeight? != null + and ((nonnegative_number($quota.runRate.recentWeight) | not) or $quota.runRate.recentWeight > 1) then + "quotaBalanced.runRate.recentWeight must be between 0 and 1" + elif $quota.providers? != null and (($quota.providers | type) != "object") then + "quotaBalanced.providers must be an object" + elif ((($quota.providers? // {}) | keys_unsorted) + - ["claude","codex","copilot","cursor","grok"] | length) > 0 then + "quotaBalanced.providers has an unsupported provider" + elif [($quota.providers? // {})[] | select(type != "object")] | length > 0 then + "each quotaBalanced provider must be an object" + elif [($quota.providers? // {})[] + | select(((keys_unsorted - ["reservePercent","windows"]) | length) > 0)] | length > 0 then + "quotaBalanced provider has unknown fields" + elif [($quota.providers? // {})[] | .reservePercent? // empty + | select((nonnegative_number(.) | not) or . > 100)] | length > 0 then + "quotaBalanced provider reservePercent must be between 0 and 100" + elif [($quota.providers? // {})[] + | select(.windows? != null and ((.windows | type) != "array"))] | length > 0 then + "quotaBalanced provider windows must be an array" + elif [($quota.providers? // {})[] | .windows[]? | select(type != "object")] | length > 0 then + "each quotaBalanced window must be an object" + elif [($quota.providers? // {})[] | .windows[]? + | select(((keys_unsorted - ["durationSeconds","extraResets","scope"]) | length) > 0)] | length > 0 then + "quotaBalanced window has unknown fields" + elif [($quota.providers? // {})[] | .windows[]? + | select(positive_integer(.durationSeconds?) | not)] | length > 0 then + "quotaBalanced window durationSeconds must be a positive integer" + elif [($quota.providers? // {})[] | .windows[]? + | select(nonnegative_integer(.extraResets?) | not)] | length > 0 then + "quotaBalanced window extraResets must be a non-negative integer" + elif [($quota.providers? // {})[] | .windows[]? + | select(.scope != "session" and .scope != "weekly")] | length > 0 then + "quotaBalanced window scope must be session or weekly" + elif [($quota.providers? // {})[] | .windows[]? + | select((.durationSeconds == 18000 and .scope != "session") + or (.durationSeconds == 604800 and .scope != "weekly"))] | length > 0 then + "quotaBalanced window scope must match canonical 5-hour or 7-day duration" + elif [($quota.providers? // {})[] + | [(.windows // [])[] | [.scope, .durationSeconds]] + | group_by(.)[] | select(length > 1)] | length > 0 then + "quotaBalanced provider windows must not duplicate scope and durationSeconds" + else "" + end; if type != "object" then "top-level value must be an object" elif has("rules") and (.rules | type) != "array" then "rules must be an array" elif [(.rules // [])[]? | select(type != "object")] | length > 0 then "each rule must be an object" @@ -692,6 +762,7 @@ crew_dispatch_validate() { "unknown select: " + ([ (.rules // [])[]? | .select? // empty | select(. != "quota-balanced") ] | unique | join(", ")) elif has("default") and (.default | type) != "object" then "default must be an object" elif has("default") and ((.default.harness? | type) != "string" or (.default.harness | length) == 0) then "default needs harness when present" + elif (quota_balanced_error | length) > 0 then quota_balanced_error else ([(.rules // [])[]? | use_profiles(.use?)[]?.harness] + [.default?.harness?] | map(select(. != null)) diff --git a/bin/fm-dispatch-select.sh b/bin/fm-dispatch-select.sh index c626e9a95..08980d54c 100755 --- a/bin/fm-dispatch-select.sh +++ b/bin/fm-dispatch-select.sh @@ -1,39 +1,68 @@ #!/usr/bin/env bash # Resolve one already-matched crew-dispatch rule to a concrete profile. # Usage: -# fm-dispatch-select.sh [--select ] [--quota-json ] [] +# fm-dispatch-select.sh [--select ] [--quota-json ] +# [--config ] [--state-file ] [--now-epoch ] +# [] # # Input may be a full rule object with `use` and optional `select`, a single # profile object, or an ordered array of profile objects. # Output is one compact JSON profile object on stdout. # # quota-balanced is deterministic, and this header is the single owner of its -# contract: -# - It runs quota-axi --json (or the --quota-json fixture). -# - Per candidate vendor it takes the minimum percentRemaining across that -# vendor's GENERAL windows only - Claude five_hour and seven_day, Codex -# five_hour and weekly - ignoring model-scoped windows such as model:fable -# and model:codex_bengalfox:*. -# - The vendor with the higher minimum remaining quota wins; an exact tie -# between equally trusted candidates uses the first array element. +# scoring contract: +# - It consumes quota-axi schema version 2 through `quota-axi --full --json` +# (or the --quota-json fixture). +# - General windows have provider kind `session` or `weekly`; model-scoped +# windows are excluded regardless of label or id. An explicit positive +# windowSeconds is authoritative. Actual 5-hour and 7-day durations define +# session and weekly capacity scope even when provider kind is misleading. +# Claude's known five_hour and seven_day windows fall back to those canonical +# durations because Claude omits the field. No Codex duration is inferred +# from its misleading five_hour id. +# - A window score is percentage-capacity headroom through its reset: +# remaining + (100 * configured extra resets) - reserve +# - (estimated burn per second * seconds until reset) +# This is normalized provider-relative capacity, not a token or per-task +# prediction. A provider's score is its lowest general-window score. +# - On a first sample, burn is percent used divided by elapsed window time. +# A compatible recent sample can raise that estimate from actual usage +# delta; recentWeight controls burst sensitivity. Rollover, decreasing +# usage, old history, or clock reversal discards the recent delta. Reset +# time is clamped to the actual duration to contain source clock skew. +# - The vendor with the higher score wins; an exact tie between equally +# trusted candidates uses the first array element. # - Stale-but-cached general-window numbers are usable, but a fresh candidate -# wins unless the stale candidate's minimum is at least the stale-clear -# margin higher (default 20 points - the definition of "clearly less -# constrained"). +# wins unless the stale candidate's score is at least the stale-clear margin +# higher (default 20 percentage-capacity points). # - A vendor absent from quota output, or with no usable general windows, is # unavailable; selection happens among available candidates. -# - If quota-axi is missing, exits non-zero, returns unparseable JSON, or no -# candidate is usable, the reason is logged to stderr and the first array -# element is printed - quota trouble never blocks dispatch. +# - Candidate/window scores and their inputs are logged without account data +# so every decision is auditable. +# - If quota-axi is missing, exits non-zero, returns unparseable JSON, has an +# unsupported schema, or no candidate is usable, the reason is logged and +# the first array element is printed. Quota trouble never blocks dispatch. # -# quota-balanced uses quota-axi --json unless --quota-json supplies a fixture. +# quota-balanced uses quota-axi --full --json unless --quota-json supplies a +# fixture. Live calls keep private samples in state/.dispatch-quota-samples.json; +# fixture calls are stateless unless --state-file is explicit. +# The quotaBalanced object in config/crew-dispatch.json owns local tuning. # FM_DISPATCH_QUOTA_AXI overrides the quota command. -# FM_DISPATCH_STALE_CLEAR_MARGIN overrides the default 20 point stale margin. +# FM_DISPATCH_STALE_CLEAR_MARGIN overrides configured/default stale margin. set -u -STALE_CLEAR_MARGIN=${FM_DISPATCH_STALE_CLEAR_MARGIN:-20} +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +FM_DISPATCH_HOME=${FM_HOME:-$ROOT} +CONFIG_ROOT=${FM_CONFIG_OVERRIDE:-$FM_DISPATCH_HOME/config} +STATE_ROOT=${FM_STATE_OVERRIDE:-$FM_DISPATCH_HOME/state} +STALE_CLEAR_OVERRIDE=${FM_DISPATCH_STALE_CLEAR_MARGIN:-} SELECT_OVERRIDE= QUOTA_JSON_FILE= +CONFIG_FILE= +CONFIG_FILE_EXPLICIT=0 +STATE_FILE= +STATE_FILE_EXPLICIT=0 +NOW_EPOCH= ARGS=() usage() { @@ -68,6 +97,37 @@ while [ "$#" -gt 0 ]; do QUOTA_JSON_FILE=${1#--quota-json=} shift ;; + --config) + [ "$#" -gt 1 ] || { echo "error: --config requires a file" >&2; exit 2; } + CONFIG_FILE=$2 + CONFIG_FILE_EXPLICIT=1 + shift 2 + ;; + --config=*) + CONFIG_FILE=${1#--config=} + CONFIG_FILE_EXPLICIT=1 + shift + ;; + --state-file) + [ "$#" -gt 1 ] || { echo "error: --state-file requires a file" >&2; exit 2; } + STATE_FILE=$2 + STATE_FILE_EXPLICIT=1 + shift 2 + ;; + --state-file=*) + STATE_FILE=${1#--state-file=} + STATE_FILE_EXPLICIT=1 + shift + ;; + --now-epoch) + [ "$#" -gt 1 ] || { echo "error: --now-epoch requires seconds" >&2; exit 2; } + NOW_EPOCH=$2 + shift 2 + ;; + --now-epoch=*) + NOW_EPOCH=${1#--now-epoch=} + shift + ;; -h|--help) usage exit 0 @@ -93,6 +153,13 @@ done [ "${#ARGS[@]}" -le 1 ] || { echo "error: expected at most one JSON argument" >&2; exit 2; } command -v jq >/dev/null 2>&1 || { echo "error: jq is required" >&2; exit 2; } +if [ -z "$NOW_EPOCH" ]; then + NOW_EPOCH=$(date -u +%s) +fi +case "$NOW_EPOCH" in + ''|*[!0-9]*) echo "error: --now-epoch must be a non-negative integer" >&2; exit 2 ;; +esac + if [ "${#ARGS[@]}" -eq 1 ]; then SPEC_JSON=${ARGS[0]} else @@ -135,6 +202,28 @@ if [ "$select_strategy" != quota-balanced ]; then exit 0 fi +if [ "$CONFIG_FILE_EXPLICIT" -eq 0 ]; then + CONFIG_FILE="$CONFIG_ROOT/crew-dispatch.json" +fi +if [ -f "$CONFIG_FILE" ]; then + if ! dispatch_config=$(cat "$CONFIG_FILE" 2>/dev/null); then + log "cannot read dispatch config; using first profile" + first_profile + exit 0 + fi + if ! printf '%s\n' "$dispatch_config" | jq -e 'type == "object"' >/dev/null 2>&1; then + log "dispatch config is invalid; using first profile" + first_profile + exit 0 + fi +elif [ "$CONFIG_FILE_EXPLICIT" -eq 1 ]; then + log "cannot read dispatch config; using first profile" + first_profile + exit 0 +else + dispatch_config='{}' +fi + if [ -n "$QUOTA_JSON_FILE" ]; then if ! quota_json=$(cat "$QUOTA_JSON_FILE" 2>/dev/null); then log "cannot read quota JSON; using first profile" @@ -148,7 +237,7 @@ else first_profile exit 0 fi - quota_json=$("$quota_cmd" --json 2>/dev/null) + quota_json=$("$quota_cmd" --full --json 2>/dev/null) quota_status=$? if [ "$quota_status" -ne 0 ]; then log "quota-axi exited $quota_status; using first profile" @@ -157,78 +246,239 @@ else fi fi -if ! printf '%s\n' "$quota_json" | jq -e 'type == "object" and (.providers | type) == "array"' >/dev/null 2>&1; then - log "quota-axi returned unparseable JSON; using first profile" +if ! printf '%s\n' "$quota_json" | jq -e ' + type == "object" and .schemaVersion == 2 and (.providers | type) == "array" +' >/dev/null 2>&1; then + log "quota-axi returned invalid or unsupported schema JSON; using first profile" first_profile exit 0 fi -selection=$(printf '%s\n' "$quota_json" | jq -ec \ +if [ "$STATE_FILE_EXPLICIT" -eq 0 ] && [ -z "$QUOTA_JSON_FILE" ]; then + STATE_FILE="$STATE_ROOT/.dispatch-quota-samples.json" +fi +history_json='{"schemaVersion":1,"samples":[]}' +if [ -n "$STATE_FILE" ] && [ -f "$STATE_FILE" ]; then + if candidate_history=$(cat "$STATE_FILE" 2>/dev/null) \ + && printf '%s\n' "$candidate_history" | jq -e ' + type == "object" and .schemaVersion == 1 and (.samples | type) == "array" + ' >/dev/null 2>&1; then + history_json=$candidate_history + else + log "quota sample history is invalid; ignoring it" + fi +fi + +selection=$(jq -nec \ + --argjson quota "$quota_json" \ --argjson profiles "$profiles_json" \ - --argjson margin "$STALE_CLEAR_MARGIN" ' + --argjson config "$dispatch_config" \ + --argjson history "$history_json" \ + --argjson now "$NOW_EPOCH" \ + --arg staleOverride "$STALE_CLEAR_OVERRIDE" ' def clean($p): {harness: $p.harness} + (if ($p.model? | type) == "string" then {model: $p.model} else {} end) + (if ($p.effort? | type) == "string" then {effort: $p.effort} else {} end); - def provider_for($h): [.providers[]? | select(.provider == $h)][0]; - def general_ids($h): - if $h == "claude" then ["five_hour", "seven_day"] - elif $h == "codex" then ["five_hour", "weekly"] - else [] + def clamp($n; $low; $high): if $n < $low then $low elif $n > $high then $high else $n end; + def positive_number($v): ($v | type) == "number" and $v > 0; + def iso_epoch($value): + if ($value | type) != "string" then null + else ($value + | sub("\\+00:00$"; "Z") + | sub("\\.[0-9]+Z$"; "Z") + | try fromdateiso8601 catch null) + end; + def duration_for($provider; $window): + if positive_number($window.windowSeconds?) then ($window.windowSeconds | floor) + elif $provider == "claude" and $window.id == "five_hour" and $window.kind == "session" then 18000 + elif $provider == "claude" and $window.id == "seven_day" and $window.kind == "weekly" then 604800 + else null end; - def candidate_metric($p; $i): - . as $root - | ($p.harness // "") as $h - | ($root | provider_for($h)) as $provider - | if ($provider == null) or ((general_ids($h) | length) == 0) then empty + def scope_for($window; $duration): + if $duration == 18000 then "session" + elif $duration == 604800 then "weekly" + else $window.kind + end; + def settings: ($config.quotaBalanced? // {}); + def run_settings: (settings.runRate? // {}); + def reserve_for($provider): + (settings.providers?[$provider].reservePercent? // settings.reservePercent? // 0); + def extra_resets($provider; $scope; $duration): + ([settings.providers?[$provider].windows[]? + | select(.durationSeconds == $duration and .scope == $scope) + | (.extraResets // 0)] | add // 0); + def history_for($provider; $id; $kind; $duration): + [$history.samples[]? + | select(.provider == $provider and .id == $id and .kind == $kind + and .durationSeconds == $duration)][0]; + def window_metric($provider; $window): + (duration_for($provider; $window)) as $duration + | (iso_epoch($window.resetsAt?)) as $reset + | (if ($window.percentRemaining? | type) == "number" then $window.percentRemaining + elif ($window.percentUsed? | type) == "number" then (100 - $window.percentUsed) + else null end) as $raw_remaining + | if $duration == null or $reset == null or $raw_remaining == null then empty else - (($provider.windows // []) - | map(. as $window - | select(((general_ids($h) | index($window.id)) != null) - and (($window.kind? // "") != "model") - and (($window.percentRemaining? | type) == "number")))) as $windows - | if ($windows | length) == 0 then empty - else { - index: $i, - profile: clean($p), - harness: $h, - min: ($windows | map(.percentRemaining) | min), - fresh: (($provider.state.status? // "") == "fresh") + (clamp($raw_remaining; 0; 100)) as $remaining + | (scope_for($window; $duration)) as $scope + | (if ($window.percentUsed? | type) == "number" then clamp($window.percentUsed; 0; 100) + else (100 - $remaining) end) as $used + | (clamp(($reset - $now); 0; $duration)) as $until_reset + | ($duration - $until_reset) as $elapsed + | (run_settings.minimumObservationSeconds? // 300) as $minimum_observation + | (run_settings.historyMaxAgeSeconds? // 86400) as $history_max_age + | (run_settings.recentWeight? // 1) as $recent_weight + | (history_for($provider; ($window.id // ""); $window.kind; $duration)) as $prior + | (($prior != null) + and (($now - ($prior.sampledAtEpoch // ($now + 1))) > 0) + and (($now - $prior.sampledAtEpoch) <= $history_max_age) + and (((($prior.resetsAtEpoch // 0) - $reset) | fabs) <= 60) + and ($used >= ($prior.percentUsed // 101))) as $recent_valid + | ($used / ([$elapsed, $minimum_observation] | max)) as $cycle_rate + | (if $recent_valid then + (($used - $prior.percentUsed) / ($now - $prior.sampledAtEpoch)) + else 0 end) as $recent_rate + | ([$cycle_rate, ($recent_rate * $recent_weight)] | max) as $burn_rate + | (extra_resets($provider; $scope; $duration)) as $extra + | (reserve_for($provider)) as $reserve + | ($burn_rate * $until_reset) as $projected + | ($remaining + (100 * $extra)) as $capacity + | { + id: ($window.id // ""), + kind: $window.kind, + scope: $scope, + durationSeconds: $duration, + remaining: $remaining, + extraResets: $extra, + reserve: $reserve, + secondsUntilReset: $until_reset, + burnPerHour: ($burn_rate * 3600), + projectedBurn: $projected, + score: ($capacity - $reserve - $projected), + rateSource: (if $recent_valid and (($recent_rate * $recent_weight) > $cycle_rate) + then "recent" else "cycle" end), + clockClamped: (($reset - $now) < 0 or ($reset - $now) > $duration), + sample: { + provider: $provider, + id: ($window.id // ""), + kind: $window.kind, + durationSeconds: $duration, + resetsAtEpoch: $reset, + sampledAtEpoch: $now, + percentUsed: $used + } } + end; + def provider_for($name): [$quota.providers[]? | select(.provider == $name)][0]; + def candidate_metric($profile; $index): + ($profile.harness // "") as $provider_name + | (provider_for($provider_name)) as $provider + | if $provider == null then empty + else + ([$provider.windows[]? + | select((.kind == "session" or .kind == "weekly") + and (((.id? // "") | startswith("model:")) | not)) + | window_metric($provider_name; .)]) as $windows + | if ($windows | length) == 0 then empty + else ($windows | min_by(.score)) as $bottleneck + | { + index: $index, + profile: clean($profile), + harness: $provider_name, + fresh: (($provider.state.status? // "") == "fresh"), + score: $bottleneck.score, + bottleneck: $bottleneck, + windows: $windows + } end end; def better($a; $b): if $a == null then $b elif $b == null then $a - elif ($b.min > $a.min) then $b - elif ($b.min == $a.min and $b.index < $a.index) then $b + elif $b.score > $a.score then $b + elif $b.score == $a.score and $b.index < $a.index then $b else $a end; - def best_by_min($xs): reduce $xs[] as $x (null; better(.; $x)); - . as $quota_root - | ([$profiles | to_entries[] | . as $entry | ($quota_root | candidate_metric($entry.value; $entry.key))]) as $candidates + def best($items): reduce $items[] as $item (null; better(.; $item)); + ([range(0; $profiles | length) as $index + | candidate_metric($profiles[$index]; $index)]) as $candidates + | (if $staleOverride == "" then (settings.staleClearMargin? // 20) + else ($staleOverride | tonumber) end) as $stale_margin | if ($candidates | length) == 0 then { fallback: true, reason: "no usable quota windows for candidate vendors", - profile: clean($profiles[0]) + profile: clean($profiles[0]), + candidates: [], + samples: [] } else - (best_by_min($candidates | map(select(.fresh)))) as $fresh_best - | (best_by_min($candidates | map(select(.fresh | not)))) as $stale_best + (best($candidates | map(select(.fresh)))) as $fresh_best + | (best($candidates | map(select(.fresh | not)))) as $stale_best | (if $fresh_best != null and $stale_best != null then - if $stale_best.min >= ($fresh_best.min + $margin) then $stale_best else $fresh_best end + if $stale_best.score >= ($fresh_best.score + $stale_margin) + then $stale_best else $fresh_best end elif $fresh_best != null then $fresh_best else $stale_best end) as $chosen - | {fallback: false, profile: $chosen.profile} + | { + fallback: false, + profile: $chosen.profile, + chosen: $chosen.harness, + candidates: $candidates, + samples: ([$candidates[] | select(.fresh) | .windows[].sample] + | unique_by([.provider, .id, .kind, .durationSeconds])) + } end ' 2>/dev/null) || { - log "quota-axi data could not be evaluated; using first profile" + log "quota-axi data or quota-balanced configuration could not be evaluated; using first profile" first_profile exit 0 } +while IFS= read -r diagnostic; do + [ -n "$diagnostic" ] && log "$diagnostic" +done < <(printf '%s\n' "$selection" | jq -r ' + .candidates[]? + | . as $candidate + | $candidate.windows[] + | "quota-balanced candidate \($candidate.harness) trust=\(if $candidate.fresh then "fresh" else "stale" end)" + + " window=\(.scope)/\(.durationSeconds)s source-kind=\(.kind) score=\(.score * 100 | round / 100)" + + " remaining=\(.remaining) extra-resets=\(.extraResets) reserve=\(.reserve)" + + " burn-per-hour=\(.burnPerHour * 100 | round / 100) projected=\(.projectedBurn * 100 | round / 100)" + + " rate-source=\(.rateSource) clock-clamped=\(.clockClamped)" +') + if [ "$(printf '%s\n' "$selection" | jq -r '.fallback')" = true ]; then log "$(printf '%s\n' "$selection" | jq -r '.reason'); using first profile" +else + log "quota-balanced selected $(printf '%s\n' "$selection" | jq -r '.chosen') by bottleneck sustainable-headroom score" fi + +if [ -n "$STATE_FILE" ] && [ "$(printf '%s\n' "$selection" | jq '.samples | length')" -gt 0 ]; then + state_parent=$(dirname "$STATE_FILE") + if mkdir -p "$state_parent" 2>/dev/null; then + state_tmp=$(umask 077; mktemp "$state_parent/.dispatch-quota-samples.XXXXXX" 2>/dev/null) || state_tmp= + if [ -n "$state_tmp" ]; then + if jq -nec --argjson old "$history_json" --argjson current "$selection" ' + def key: [.provider, .id, .kind, .durationSeconds]; + {schemaVersion: 1, + samples: ([($old.samples[]? // empty), $current.samples[]] + | sort_by(key) + | group_by(key) + | map(last))} + ' > "$state_tmp" && chmod 600 "$state_tmp" && mv "$state_tmp" "$STATE_FILE"; then + : + else + rm -f "$state_tmp" + log "could not update quota sample history; selection remains valid" + fi + else + log "could not prepare quota sample history; selection remains valid" + fi + else + log "could not create quota sample state directory; selection remains valid" + fi +fi + printf '%s\n' "$selection" | jq -c '.profile' diff --git a/docs/configuration.md b/docs/configuration.md index 9e193cb97..4ff817142 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -194,6 +194,23 @@ This section is the single owner of the canonical schema and its per-field seman ```json { + "quotaBalanced": { + "reservePercent": 0, + "staleClearMargin": 20, + "runRate": { + "minimumObservationSeconds": 300, + "historyMaxAgeSeconds": 86400, + "recentWeight": 1 + }, + "providers": { + "": { + "reservePercent": 0, + "windows": [ + { "scope": "", "durationSeconds": 604800, "extraResets": 0 } + ] + } + } + }, "rules": [ { "when": "", @@ -216,7 +233,37 @@ Absent `select` means use the first array element, or the only object in the sin `default` is optional. An omitted model or effort means the selected harness uses its own default for that axis. If a selected profile carries an effort value the chosen harness does not accept, `fm-spawn.sh` records the requested `effort=` in task meta for traceability but omits the launch flag, and bootstrap reports the invalid harness/effort pair as a `CREW_DISPATCH` diagnostic when it is visible in the file. -`quota-balanced` selection is deterministic and implemented by `bin/fm-dispatch-select.sh`, whose header owns the general-window rules, the 20 point stale-clear freshness margin, vendor-availability handling, and the degrade-to-first-element fallbacks; quota trouble never blocks dispatch. +`quotaBalanced` is optional local tuning used only by rules with `select: "quota-balanced"`. +`reservePercent` keeps percentage-capacity headroom out of the usable score and defaults to 0, while a provider-level value overrides the global value. +`staleClearMargin` defaults to 20 percentage-capacity points and preserves the policy that cached stale quota must be clearly better than fresh quota to win. +`runRate.minimumObservationSeconds` defaults to 300 and bounds first-sample burst amplification near a reset. +`runRate.historyMaxAgeSeconds` defaults to 86400 and prevents old local samples from influencing recent burn. +`runRate.recentWeight` ranges from 0 to 1, defaults to 1, and controls how strongly an observed recent usage delta can raise the cycle-average burn estimate. +Each provider `windows` entry grants explicit manual reset capacity only to the matching canonical `scope` and `durationSeconds`. +The selector derives session versus weekly scope from actual 18000-second or 604800-second duration before considering provider labels, ids, or misleading kinds. +Weekly reset credits therefore never apply to an independent five-hour session window. +`extraResets` is a non-negative integer and adds one full normalized window of capacity per configured reset. +It represents currently unused manual resets and must be reduced locally when a reset is consumed. +The selector keeps account-free burn samples in `state/.dispatch-quota-samples.json`; the file is volatile private state and is not a source of configured reset credits. +`quota-balanced` selection is deterministic and implemented by `bin/fm-dispatch-select.sh`, whose header owns the exact score, rollover, clock-skew, diagnostics, vendor-availability, and degrade-to-first-element mechanics; quota trouble never blocks dispatch. + +To declare three Codex manual weekly resets after this version lands, merge this local object into `config/crew-dispatch.json` without adding a session entry: + +```json +{ + "quotaBalanced": { + "providers": { + "codex": { + "windows": [ + { "scope": "weekly", "durationSeconds": 604800, "extraResets": 3 } + ] + } + } + } +} +``` + +This declaration is local and gitignored, so the three-reset reserve is not a shared default and no account identity appears in tracked files or selector diagnostics. See [`docs/examples/crew-dispatch.json`](examples/crew-dispatch.json) for a starting point to copy into local `config/crew-dispatch.json`. When the file exists, bootstrap validates it with `jq`. Valid files stay silent by default; with `FM_BOOTSTRAP_VERBOSE_FACTS=1`, bootstrap emits `BOOTSTRAP_INFO: crew dispatch active config/crew-dispatch.json` plus one `BOOTSTRAP_INFO:` fact per rule and default profile. diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index 851ba66ec..130fb5929 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -782,6 +782,12 @@ empty array use is flagged^{"rules":[{"when":"big feature","use":[]}]}^exact^CRE array profile without harness is flagged^{"rules":[{"when":"big feature","use":[{"model":"gpt-5.5"}]}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - each use profile needs harness unknown select is flagged^{"rules":[{"when":"big feature","use":[{"harness":"claude"},{"harness":"codex"}],"select":"mystery"}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - unknown select: mystery array profile unsupported effort is flagged^{"rules":[{"when":"big feature","use":[{"harness":"codex","effort":"max"}]}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - invalid effort: codex:max +quota-balanced tuning is accepted^{"rules":[{"when":"big feature","use":[{"harness":"claude"},{"harness":"codex"}],"select":"quota-balanced"}],"quotaBalanced":{"reservePercent":5,"staleClearMargin":20,"runRate":{"minimumObservationSeconds":300,"historyMaxAgeSeconds":86400,"recentWeight":0.5},"providers":{"codex":{"reservePercent":10,"windows":[{"scope":"weekly","durationSeconds":604800,"extraResets":3}]}}}}^empty^ +quota-balanced reset count must be an integer^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"weekly","durationSeconds":604800,"extraResets":1.5}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced window extraResets must be a non-negative integer +quota-balanced duration must be positive^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"weekly","durationSeconds":0,"extraResets":3}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced window durationSeconds must be a positive integer +quota-balanced recent weight is bounded^{"quotaBalanced":{"runRate":{"recentWeight":1.5}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced.runRate.recentWeight must be between 0 and 1 +quota-balanced weekly resets cannot target session scope^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"session","durationSeconds":604800,"extraResets":3}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced window scope must match canonical 5-hour or 7-day duration +quota-balanced duplicate windows are flagged^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"weekly","durationSeconds":604800,"extraResets":1},{"scope":"weekly","durationSeconds":604800,"extraResets":2}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced provider windows must not duplicate scope and durationSeconds ROWS pass "bootstrap validates crew-dispatch.json and reports malformed or unverified configs" } diff --git a/tests/fm-dispatch-select.test.sh b/tests/fm-dispatch-select.test.sh index a9ae5b7bf..10f713358 100755 --- a/tests/fm-dispatch-select.test.sh +++ b/tests/fm-dispatch-select.test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Behavior tests for deterministic crew-dispatch profile selection. +# Behavior tests for deterministic, run-rate-aware crew dispatch selection. set -u # shellcheck source=tests/lib.sh @@ -9,148 +9,284 @@ BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin} TMP_ROOT=$(fm_test_tmproot fm-dispatch-select-tests) mkdir -p "$TMP_ROOT" -write_quota() { - local file=$1 claude_status=$2 claude_five=$3 claude_week=$4 codex_status=$5 codex_five=$6 codex_week=$7 - mkdir -p "$(dirname "$file")" - cat > "$file" < "$err" +} + +test_low_remaining_slow_burn_beats_high_remaining_burst() { + local quota state out err + quota="$TMP_ROOT/run-rate.json" + state="$TMP_ROOT/run-rate-state.json" + err="$TMP_ROOT/run-rate.err" + cat > "$quota" <<'JSON' { + "schemaVersion": 2, "providers": [ { "provider": "claude", - "state": { "status": "$claude_status" }, + "state": {"status": "fresh"}, "windows": [ - { "id": "five_hour", "kind": "session", "percentRemaining": $claude_five }, - { "id": "seven_day", "kind": "weekly", "percentRemaining": $claude_week }, - { "id": "model:fable", "kind": "model", "percentRemaining": 100 } + {"id": "five_hour", "kind": "session", "percentUsed": 80, "percentRemaining": 20, "resetsAt": "2026-07-19T00:10:00Z"}, + {"id": "seven_day", "kind": "weekly", "percentUsed": 80, "percentRemaining": 20, "resetsAt": "2026-07-20T00:00:00Z"}, + {"id": "model:fable", "kind": "model", "percentUsed": 99, "percentRemaining": 1, "resetsAt": "2026-07-20T00:00:00Z"} ] }, { "provider": "codex", - "state": { "status": "$codex_status" }, + "state": {"status": "fresh"}, "windows": [ - { "id": "five_hour", "kind": "session", "percentRemaining": $codex_five }, - { "id": "weekly", "kind": "weekly", "percentRemaining": $codex_week }, - { "id": "model:codex_bengalfox:5h", "kind": "model", "percentRemaining": 100 } + {"id": "five_hour", "label": "session", "kind": "session", "windowSeconds": 604800, "percentUsed": 20, "percentRemaining": 80, "resetsAt": "2026-07-24T00:00:00Z"} ] } ] } JSON + cat > "$state" < "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + { + "provider": "claude", + "state": {"status": "fresh"}, + "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 10, "percentRemaining": 90, "resetsAt": "2026-07-19T01:00:00Z"}, + {"id": "seven_day", "kind": "weekly", "percentUsed": 70, "percentRemaining": 30, "resetsAt": "2026-07-24T00:00:00Z"} + ] + }, + { + "provider": "codex", + "state": {"status": "fresh"}, + "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 50, "percentRemaining": 50, "resetsAt": "2026-07-19T00:00:00Z"} + ] + } + ] +} +JSON -test_higher_min_vendor_wins() { - local quota out - quota="$TMP_ROOT/higher.json" - write_quota "$quota" fresh 80 30 fresh 70 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") + out=$(run_fixture "$quota" "$err") + diagnostics=$(cat "$err") [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "higher-min vendor should win, got: $out" - pass "quota-balanced picks the candidate with the higher general-window minimum" + || fail "Claude weekly bottleneck should constrain an ample session window, got: $out" + assert_contains "$diagnostics" "candidate claude trust=fresh window=session/18000s" \ + "Claude session should use its canonical five-hour duration" + assert_contains "$diagnostics" "candidate claude trust=fresh window=weekly/604800s" \ + "Claude weekly window should be scored independently" + pass "Claude selection uses the lower sustainable headroom across session and weekly windows" } -test_exact_tie_uses_first_profile() { - local quota out - quota="$TMP_ROOT/tie.json" - write_quota "$quota" fresh 90 50 fresh 60 50 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "exact tie should pick first profile, got: $out" - pass "quota-balanced exact tie uses the first ordered profile" +test_codex_mislabeled_window_uses_actual_duration() { + local quota out err diagnostics + quota="$TMP_ROOT/codex-duration.json" + err="$TMP_ROOT/codex-duration.err" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 60, "percentRemaining": 40, "resetsAt": "2026-07-19T00:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "label": "session", "kind": "session", "windowSeconds": 604800, "percentUsed": 20, "percentRemaining": 80, "resetsAt": "2026-07-24T00:00:00Z"}, + {"id": "model:codex_bengalfox:5h", "kind": "model", "windowSeconds": 604800, "percentUsed": 99, "percentRemaining": 1, "resetsAt": "2026-07-24T00:00:00Z"} + ]} + ] } +JSON -test_quota_missing_falls_back_to_first() { - local fakebin out err status - fakebin=$(fm_fakebin "$TMP_ROOT/missing") - out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced "$profiles" 2>"$TMP_ROOT/missing.err") - status=$? - err=$(cat "$TMP_ROOT/missing.err") - expect_code 0 "$status" "missing quota-axi should not fail dispatch" + out=$(run_fixture "$quota" "$err") + diagnostics=$(cat "$err") [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "missing quota-axi should fall back to first, got: $out" - assert_contains "$err" "quota-axi missing" "missing quota-axi fallback should be logged" - pass "quota-axi missing falls back to the first profile and logs" + || fail "Codex seven-day burn should lose despite its five_hour id, got: $out" + assert_contains "$diagnostics" "candidate codex trust=fresh window=weekly/604800s source-kind=session" \ + "Codex diagnostics should expose the actual seven-day weekly scope" + assert_not_contains "$diagnostics" "codex_bengalfox" "model windows must remain excluded" + pass "Codex's misleading five_hour id cannot override its actual seven-day duration" } -test_quota_error_falls_back_to_first() { - local fakebin out err status - fakebin=$(fm_fakebin "$TMP_ROOT/error") - cat > "$fakebin/quota-axi" <<'SH' -#!/usr/bin/env bash -exit 42 -SH - chmod +x "$fakebin/quota-axi" - out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced "$profiles" 2>"$TMP_ROOT/error.err") - status=$? - err=$(cat "$TMP_ROOT/error.err") - expect_code 0 "$status" "quota-axi error should not fail dispatch" - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "quota-axi error should fall back to first, got: $out" - assert_contains "$err" "quota-axi exited 42" "quota-axi error fallback should be logged" - pass "quota-axi non-zero exit falls back to the first profile and logs" +test_configured_codex_extra_resets_add_capacity() { + local quota config out err diagnostics + quota="$TMP_ROOT/extra-resets.json" + config="$TMP_ROOT/extra-resets-config.json" + err="$TMP_ROOT/extra-resets.err" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 60, "percentRemaining": 40, "resetsAt": "2026-07-19T00:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 100, "percentRemaining": 0, "resetsAt": "2026-07-24T00:00:00Z"}, + {"id": "independent_five_hour", "kind": "session", "windowSeconds": 18000, "percentUsed": 0, "percentRemaining": 100, "resetsAt": "2026-07-19T00:00:00Z"} + ]} + ] +} +JSON + cat > "$config" <<'JSON' +{ + "quotaBalanced": { + "providers": { + "codex": { + "windows": [ + {"scope": "weekly", "durationSeconds": 604800, "extraResets": 3} + ] + } + } + } } +JSON -test_bad_quota_json_falls_back_to_first() { - local quota out err - quota="$TMP_ROOT/bad.json" - printf '%s\n' 'not-json' > "$quota" - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles" 2>"$TMP_ROOT/bad.err") - err=$(cat "$TMP_ROOT/bad.err") - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "bad quota JSON should fall back to first, got: $out" - assert_contains "$err" "unparseable JSON" "bad quota JSON fallback should be logged" - pass "unparseable quota JSON falls back to the first profile and logs" + out=$(run_fixture "$quota" "$err" --config "$config") + diagnostics=$(cat "$err") + [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ + || fail "three manual Codex resets should add three windows of capacity, got: $out" + assert_contains "$(printf '%s\n' "$diagnostics" | grep 'candidate codex.*window=weekly')" \ + "extra-resets=3" "weekly diagnostics should disclose configured reset capacity" + assert_contains "$(printf '%s\n' "$diagnostics" | grep 'candidate codex.*window=session')" \ + "extra-resets=0" "independent five-hour capacity must not receive weekly reset credits" + pass "three Codex weekly resets add weekly capacity while the independent five-hour window gets zero" } -test_stale_with_cache_needs_clear_margin_to_beat_fresh() { - local quota out - quota="$TMP_ROOT/stale-margin.json" - write_quota "$quota" stale 85 70 fresh 65 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") +test_stale_candidate_keeps_existing_clear_margin_policy() { + local quota out err + quota="$TMP_ROOT/stale.json" + err="$TMP_ROOT/stale.err" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "stale"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 70, "percentRemaining": 30, "resetsAt": "2026-07-19T00:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 80, "percentRemaining": 20, "resetsAt": "2026-07-19T00:00:00Z"} + ]} + ] +} +JSON + out=$(run_fixture "$quota" "$err") [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "fresh vendor should win when stale lead is below margin, got: $out" + || fail "fresh candidate should win when stale lead is under 20, got: $out" - write_quota "$quota" stale 90 85 fresh 65 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") + jq '(.providers[] | select(.provider == "claude").windows[0]) |= (.percentUsed = 50 | .percentRemaining = 50)' \ + "$quota" > "$TMP_ROOT/stale-clear.json" + out=$(run_fixture "$TMP_ROOT/stale-clear.json" "$err") [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "stale vendor should win when lead clears margin, got: $out" - pass "stale cached quota is usable only when it clears the documented margin over fresh" + || fail "stale candidate should win when sustainable score clears 20, got: $out" + pass "stale cached data remains usable only when its score clears the existing margin" } -test_vendor_absent_or_unusable_falls_back_conservatively() { - local quota out err - quota="$TMP_ROOT/absent.json" +test_rollover_discards_incompatible_recent_sample() { + local quota state out err new_reset + quota="$TMP_ROOT/rollover.json" + state="$TMP_ROOT/rollover-state.json" + err="$TMP_ROOT/rollover.err" + new_reset=$(jq -nr '"2026-07-26T00:00:00Z" | fromdateiso8601') cat > "$quota" <<'JSON' { + "schemaVersion": 2, "providers": [ - { - "provider": "codex", - "state": { "status": "fresh" }, - "windows": [ - { "id": "five_hour", "kind": "session", "percentRemaining": 40 }, - { "id": "weekly", "kind": "weekly", "percentRemaining": 50 } - ] - } + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 5, "percentRemaining": 95, "resetsAt": "2026-07-19T01:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 5, "percentRemaining": 95, "resetsAt": "2026-07-26T00:00:00Z"} + ]} ] } JSON - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") - [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "available candidate should win over absent vendor, got: $out" + cat > "$state" < "$quota" <<'JSON' -{ "providers": [] } +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 0, "percentRemaining": 100, "resetsAt": "2026-07-19T06:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 0, "percentRemaining": 100, "resetsAt": "2026-07-26T00:00:00Z"} + ]} + ] +} JSON - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles" 2>"$TMP_ROOT/none.err") - err=$(cat "$TMP_ROOT/none.err") + + out=$(run_fixture "$quota" "$err") + diagnostics=$(cat "$err") + [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ + || fail "Codex without authoritative duration should be unavailable, got: $out" + assert_contains "$diagnostics" "clock-clamped=true" "future reset beyond duration should be clock-clamped" + assert_not_contains "$diagnostics" "candidate codex" "missing Codex duration should not trust the five_hour id" + pass "missing durations are skipped and reset clock skew is bounded by actual duration" +} + +test_quota_failures_and_old_schema_fall_back_to_first() { + local fakebin out err status quota + fakebin=$(fm_fakebin "$TMP_ROOT/failure") + cat > "$fakebin/quota-axi" <<'SH' +#!/usr/bin/env bash +exit 42 +SH + chmod +x "$fakebin/quota-axi" + out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced \ + "$profiles" 2> "$TMP_ROOT/failure.err") + status=$? + err=$(cat "$TMP_ROOT/failure.err") + expect_code 0 "$status" "quota-axi error should not fail dispatch" + [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ + || fail "quota-axi failure should fall back to first, got: $out" + assert_contains "$err" "quota-axi exited 42" "quota-axi failure should be diagnosed" + + quota="$TMP_ROOT/schema-one.json" + printf '%s\n' '{"schemaVersion":1,"providers":[]}' > "$quota" + out=$(run_fixture "$quota" "$TMP_ROOT/schema-one.err") [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "no usable vendors should fall back to first, got: $out" - assert_contains "$err" "no usable quota windows" "no usable vendor fallback should be logged" - pass "absent or unusable vendors resolve to an available candidate or the first fallback" + || fail "unsupported schema should fall back to first, got: $out" + assert_contains "$(cat "$TMP_ROOT/schema-one.err")" "unsupported schema" \ + "unsupported schema should be diagnosed" + pass "quota-axi failure and unsupported schemas retain the first-profile fail-safe" } -test_backward_compatible_first_selection() { +test_backward_compatible_non_selector_paths() { local fakebin marker out single array_rule fakebin=$(fm_fakebin "$TMP_ROOT/no-call") marker="$TMP_ROOT/quota-called" @@ -171,16 +307,17 @@ SH [ "$out" = '{"harness":"claude","effort":"high"}' ] \ || fail "array without select should resolve to first, got: $out" [ ! -e "$marker" ] || fail "quota-axi should not be called without quota-balanced select" - pass "single-object use and no-select arrays preserve first-profile selection" + pass "single-object and no-select arrays preserve first-profile selection" } -test_higher_min_vendor_wins -test_exact_tie_uses_first_profile -test_quota_missing_falls_back_to_first -test_quota_error_falls_back_to_first -test_bad_quota_json_falls_back_to_first -test_stale_with_cache_needs_clear_margin_to_beat_fresh -test_vendor_absent_or_unusable_falls_back_conservatively -test_backward_compatible_first_selection +test_low_remaining_slow_burn_beats_high_remaining_burst +test_claude_session_and_weekly_bottlenecks +test_codex_mislabeled_window_uses_actual_duration +test_configured_codex_extra_resets_add_capacity +test_stale_candidate_keeps_existing_clear_margin_policy +test_rollover_discards_incompatible_recent_sample +test_missing_duration_and_clock_skew_are_contained +test_quota_failures_and_old_schema_fall_back_to_first +test_backward_compatible_non_selector_paths echo "# all fm-dispatch-select tests passed" From f3a212c889007d004458f523fdad36cc99d8b5a9 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 10:34:51 +1000 Subject: [PATCH 02/17] feat: balance dispatch by quota run rate --- bin/fm-bootstrap.sh | 71 ++++++ bin/fm-dispatch-select.sh | 362 ++++++++++++++++++++++++++----- docs/configuration.md | 49 ++++- tests/fm-bootstrap.test.sh | 6 + tests/fm-dispatch-select.test.sh | 347 ++++++++++++++++++++--------- 5 files changed, 673 insertions(+), 162 deletions(-) diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 753c5cec3..723ecdacf 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -679,6 +679,76 @@ crew_dispatch_validate() { | map(select(. as $p | effort_ok($p.h; $p.e) | not)) | map("\(.h):\(.e)") | unique; + def nonnegative_number($value): ($value | type) == "number" and $value >= 0; + def positive_integer($value): + ($value | type) == "number" and $value > 0 and ($value | floor) == $value; + def nonnegative_integer($value): + ($value | type) == "number" and $value >= 0 and ($value | floor) == $value; + def quota_balanced_error: + (.quotaBalanced? // {}) as $quota + | if has("quotaBalanced") and ((.quotaBalanced | type) != "object") then + "quotaBalanced must be an object" + elif (($quota | keys_unsorted) - ["providers","reservePercent","runRate","staleClearMargin"] | length) > 0 then + "quotaBalanced has unknown fields" + elif $quota.reservePercent? != null + and ((nonnegative_number($quota.reservePercent) | not) or $quota.reservePercent > 100) then + "quotaBalanced.reservePercent must be between 0 and 100" + elif $quota.staleClearMargin? != null and (nonnegative_number($quota.staleClearMargin) | not) then + "quotaBalanced.staleClearMargin must be non-negative" + elif $quota.runRate? != null and (($quota.runRate | type) != "object") then + "quotaBalanced.runRate must be an object" + elif ((($quota.runRate? // {}) | keys_unsorted) + - ["historyMaxAgeSeconds","minimumObservationSeconds","recentWeight"] | length) > 0 then + "quotaBalanced.runRate has unknown fields" + elif $quota.runRate.minimumObservationSeconds? != null + and (positive_integer($quota.runRate.minimumObservationSeconds) | not) then + "quotaBalanced.runRate.minimumObservationSeconds must be a positive integer" + elif $quota.runRate.historyMaxAgeSeconds? != null + and (positive_integer($quota.runRate.historyMaxAgeSeconds) | not) then + "quotaBalanced.runRate.historyMaxAgeSeconds must be a positive integer" + elif $quota.runRate.recentWeight? != null + and ((nonnegative_number($quota.runRate.recentWeight) | not) or $quota.runRate.recentWeight > 1) then + "quotaBalanced.runRate.recentWeight must be between 0 and 1" + elif $quota.providers? != null and (($quota.providers | type) != "object") then + "quotaBalanced.providers must be an object" + elif ((($quota.providers? // {}) | keys_unsorted) + - ["claude","codex","copilot","cursor","grok"] | length) > 0 then + "quotaBalanced.providers has an unsupported provider" + elif [($quota.providers? // {})[] | select(type != "object")] | length > 0 then + "each quotaBalanced provider must be an object" + elif [($quota.providers? // {})[] + | select(((keys_unsorted - ["reservePercent","windows"]) | length) > 0)] | length > 0 then + "quotaBalanced provider has unknown fields" + elif [($quota.providers? // {})[] | .reservePercent? // empty + | select((nonnegative_number(.) | not) or . > 100)] | length > 0 then + "quotaBalanced provider reservePercent must be between 0 and 100" + elif [($quota.providers? // {})[] + | select(.windows? != null and ((.windows | type) != "array"))] | length > 0 then + "quotaBalanced provider windows must be an array" + elif [($quota.providers? // {})[] | .windows[]? | select(type != "object")] | length > 0 then + "each quotaBalanced window must be an object" + elif [($quota.providers? // {})[] | .windows[]? + | select(((keys_unsorted - ["durationSeconds","extraResets","scope"]) | length) > 0)] | length > 0 then + "quotaBalanced window has unknown fields" + elif [($quota.providers? // {})[] | .windows[]? + | select(positive_integer(.durationSeconds?) | not)] | length > 0 then + "quotaBalanced window durationSeconds must be a positive integer" + elif [($quota.providers? // {})[] | .windows[]? + | select(nonnegative_integer(.extraResets?) | not)] | length > 0 then + "quotaBalanced window extraResets must be a non-negative integer" + elif [($quota.providers? // {})[] | .windows[]? + | select(.scope != "session" and .scope != "weekly")] | length > 0 then + "quotaBalanced window scope must be session or weekly" + elif [($quota.providers? // {})[] | .windows[]? + | select((.durationSeconds == 18000 and .scope != "session") + or (.durationSeconds == 604800 and .scope != "weekly"))] | length > 0 then + "quotaBalanced window scope must match canonical 5-hour or 7-day duration" + elif [($quota.providers? // {})[] + | [(.windows // [])[] | [.scope, .durationSeconds]] + | group_by(.)[] | select(length > 1)] | length > 0 then + "quotaBalanced provider windows must not duplicate scope and durationSeconds" + else "" + end; if type != "object" then "top-level value must be an object" elif has("rules") and (.rules | type) != "array" then "rules must be an array" elif [(.rules // [])[]? | select(type != "object")] | length > 0 then "each rule must be an object" @@ -692,6 +762,7 @@ crew_dispatch_validate() { "unknown select: " + ([ (.rules // [])[]? | .select? // empty | select(. != "quota-balanced") ] | unique | join(", ")) elif has("default") and (.default | type) != "object" then "default must be an object" elif has("default") and ((.default.harness? | type) != "string" or (.default.harness | length) == 0) then "default needs harness when present" + elif (quota_balanced_error | length) > 0 then quota_balanced_error else ([(.rules // [])[]? | use_profiles(.use?)[]?.harness] + [.default?.harness?] | map(select(. != null)) diff --git a/bin/fm-dispatch-select.sh b/bin/fm-dispatch-select.sh index c626e9a95..08980d54c 100755 --- a/bin/fm-dispatch-select.sh +++ b/bin/fm-dispatch-select.sh @@ -1,39 +1,68 @@ #!/usr/bin/env bash # Resolve one already-matched crew-dispatch rule to a concrete profile. # Usage: -# fm-dispatch-select.sh [--select ] [--quota-json ] [] +# fm-dispatch-select.sh [--select ] [--quota-json ] +# [--config ] [--state-file ] [--now-epoch ] +# [] # # Input may be a full rule object with `use` and optional `select`, a single # profile object, or an ordered array of profile objects. # Output is one compact JSON profile object on stdout. # # quota-balanced is deterministic, and this header is the single owner of its -# contract: -# - It runs quota-axi --json (or the --quota-json fixture). -# - Per candidate vendor it takes the minimum percentRemaining across that -# vendor's GENERAL windows only - Claude five_hour and seven_day, Codex -# five_hour and weekly - ignoring model-scoped windows such as model:fable -# and model:codex_bengalfox:*. -# - The vendor with the higher minimum remaining quota wins; an exact tie -# between equally trusted candidates uses the first array element. +# scoring contract: +# - It consumes quota-axi schema version 2 through `quota-axi --full --json` +# (or the --quota-json fixture). +# - General windows have provider kind `session` or `weekly`; model-scoped +# windows are excluded regardless of label or id. An explicit positive +# windowSeconds is authoritative. Actual 5-hour and 7-day durations define +# session and weekly capacity scope even when provider kind is misleading. +# Claude's known five_hour and seven_day windows fall back to those canonical +# durations because Claude omits the field. No Codex duration is inferred +# from its misleading five_hour id. +# - A window score is percentage-capacity headroom through its reset: +# remaining + (100 * configured extra resets) - reserve +# - (estimated burn per second * seconds until reset) +# This is normalized provider-relative capacity, not a token or per-task +# prediction. A provider's score is its lowest general-window score. +# - On a first sample, burn is percent used divided by elapsed window time. +# A compatible recent sample can raise that estimate from actual usage +# delta; recentWeight controls burst sensitivity. Rollover, decreasing +# usage, old history, or clock reversal discards the recent delta. Reset +# time is clamped to the actual duration to contain source clock skew. +# - The vendor with the higher score wins; an exact tie between equally +# trusted candidates uses the first array element. # - Stale-but-cached general-window numbers are usable, but a fresh candidate -# wins unless the stale candidate's minimum is at least the stale-clear -# margin higher (default 20 points - the definition of "clearly less -# constrained"). +# wins unless the stale candidate's score is at least the stale-clear margin +# higher (default 20 percentage-capacity points). # - A vendor absent from quota output, or with no usable general windows, is # unavailable; selection happens among available candidates. -# - If quota-axi is missing, exits non-zero, returns unparseable JSON, or no -# candidate is usable, the reason is logged to stderr and the first array -# element is printed - quota trouble never blocks dispatch. +# - Candidate/window scores and their inputs are logged without account data +# so every decision is auditable. +# - If quota-axi is missing, exits non-zero, returns unparseable JSON, has an +# unsupported schema, or no candidate is usable, the reason is logged and +# the first array element is printed. Quota trouble never blocks dispatch. # -# quota-balanced uses quota-axi --json unless --quota-json supplies a fixture. +# quota-balanced uses quota-axi --full --json unless --quota-json supplies a +# fixture. Live calls keep private samples in state/.dispatch-quota-samples.json; +# fixture calls are stateless unless --state-file is explicit. +# The quotaBalanced object in config/crew-dispatch.json owns local tuning. # FM_DISPATCH_QUOTA_AXI overrides the quota command. -# FM_DISPATCH_STALE_CLEAR_MARGIN overrides the default 20 point stale margin. +# FM_DISPATCH_STALE_CLEAR_MARGIN overrides configured/default stale margin. set -u -STALE_CLEAR_MARGIN=${FM_DISPATCH_STALE_CLEAR_MARGIN:-20} +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +FM_DISPATCH_HOME=${FM_HOME:-$ROOT} +CONFIG_ROOT=${FM_CONFIG_OVERRIDE:-$FM_DISPATCH_HOME/config} +STATE_ROOT=${FM_STATE_OVERRIDE:-$FM_DISPATCH_HOME/state} +STALE_CLEAR_OVERRIDE=${FM_DISPATCH_STALE_CLEAR_MARGIN:-} SELECT_OVERRIDE= QUOTA_JSON_FILE= +CONFIG_FILE= +CONFIG_FILE_EXPLICIT=0 +STATE_FILE= +STATE_FILE_EXPLICIT=0 +NOW_EPOCH= ARGS=() usage() { @@ -68,6 +97,37 @@ while [ "$#" -gt 0 ]; do QUOTA_JSON_FILE=${1#--quota-json=} shift ;; + --config) + [ "$#" -gt 1 ] || { echo "error: --config requires a file" >&2; exit 2; } + CONFIG_FILE=$2 + CONFIG_FILE_EXPLICIT=1 + shift 2 + ;; + --config=*) + CONFIG_FILE=${1#--config=} + CONFIG_FILE_EXPLICIT=1 + shift + ;; + --state-file) + [ "$#" -gt 1 ] || { echo "error: --state-file requires a file" >&2; exit 2; } + STATE_FILE=$2 + STATE_FILE_EXPLICIT=1 + shift 2 + ;; + --state-file=*) + STATE_FILE=${1#--state-file=} + STATE_FILE_EXPLICIT=1 + shift + ;; + --now-epoch) + [ "$#" -gt 1 ] || { echo "error: --now-epoch requires seconds" >&2; exit 2; } + NOW_EPOCH=$2 + shift 2 + ;; + --now-epoch=*) + NOW_EPOCH=${1#--now-epoch=} + shift + ;; -h|--help) usage exit 0 @@ -93,6 +153,13 @@ done [ "${#ARGS[@]}" -le 1 ] || { echo "error: expected at most one JSON argument" >&2; exit 2; } command -v jq >/dev/null 2>&1 || { echo "error: jq is required" >&2; exit 2; } +if [ -z "$NOW_EPOCH" ]; then + NOW_EPOCH=$(date -u +%s) +fi +case "$NOW_EPOCH" in + ''|*[!0-9]*) echo "error: --now-epoch must be a non-negative integer" >&2; exit 2 ;; +esac + if [ "${#ARGS[@]}" -eq 1 ]; then SPEC_JSON=${ARGS[0]} else @@ -135,6 +202,28 @@ if [ "$select_strategy" != quota-balanced ]; then exit 0 fi +if [ "$CONFIG_FILE_EXPLICIT" -eq 0 ]; then + CONFIG_FILE="$CONFIG_ROOT/crew-dispatch.json" +fi +if [ -f "$CONFIG_FILE" ]; then + if ! dispatch_config=$(cat "$CONFIG_FILE" 2>/dev/null); then + log "cannot read dispatch config; using first profile" + first_profile + exit 0 + fi + if ! printf '%s\n' "$dispatch_config" | jq -e 'type == "object"' >/dev/null 2>&1; then + log "dispatch config is invalid; using first profile" + first_profile + exit 0 + fi +elif [ "$CONFIG_FILE_EXPLICIT" -eq 1 ]; then + log "cannot read dispatch config; using first profile" + first_profile + exit 0 +else + dispatch_config='{}' +fi + if [ -n "$QUOTA_JSON_FILE" ]; then if ! quota_json=$(cat "$QUOTA_JSON_FILE" 2>/dev/null); then log "cannot read quota JSON; using first profile" @@ -148,7 +237,7 @@ else first_profile exit 0 fi - quota_json=$("$quota_cmd" --json 2>/dev/null) + quota_json=$("$quota_cmd" --full --json 2>/dev/null) quota_status=$? if [ "$quota_status" -ne 0 ]; then log "quota-axi exited $quota_status; using first profile" @@ -157,78 +246,239 @@ else fi fi -if ! printf '%s\n' "$quota_json" | jq -e 'type == "object" and (.providers | type) == "array"' >/dev/null 2>&1; then - log "quota-axi returned unparseable JSON; using first profile" +if ! printf '%s\n' "$quota_json" | jq -e ' + type == "object" and .schemaVersion == 2 and (.providers | type) == "array" +' >/dev/null 2>&1; then + log "quota-axi returned invalid or unsupported schema JSON; using first profile" first_profile exit 0 fi -selection=$(printf '%s\n' "$quota_json" | jq -ec \ +if [ "$STATE_FILE_EXPLICIT" -eq 0 ] && [ -z "$QUOTA_JSON_FILE" ]; then + STATE_FILE="$STATE_ROOT/.dispatch-quota-samples.json" +fi +history_json='{"schemaVersion":1,"samples":[]}' +if [ -n "$STATE_FILE" ] && [ -f "$STATE_FILE" ]; then + if candidate_history=$(cat "$STATE_FILE" 2>/dev/null) \ + && printf '%s\n' "$candidate_history" | jq -e ' + type == "object" and .schemaVersion == 1 and (.samples | type) == "array" + ' >/dev/null 2>&1; then + history_json=$candidate_history + else + log "quota sample history is invalid; ignoring it" + fi +fi + +selection=$(jq -nec \ + --argjson quota "$quota_json" \ --argjson profiles "$profiles_json" \ - --argjson margin "$STALE_CLEAR_MARGIN" ' + --argjson config "$dispatch_config" \ + --argjson history "$history_json" \ + --argjson now "$NOW_EPOCH" \ + --arg staleOverride "$STALE_CLEAR_OVERRIDE" ' def clean($p): {harness: $p.harness} + (if ($p.model? | type) == "string" then {model: $p.model} else {} end) + (if ($p.effort? | type) == "string" then {effort: $p.effort} else {} end); - def provider_for($h): [.providers[]? | select(.provider == $h)][0]; - def general_ids($h): - if $h == "claude" then ["five_hour", "seven_day"] - elif $h == "codex" then ["five_hour", "weekly"] - else [] + def clamp($n; $low; $high): if $n < $low then $low elif $n > $high then $high else $n end; + def positive_number($v): ($v | type) == "number" and $v > 0; + def iso_epoch($value): + if ($value | type) != "string" then null + else ($value + | sub("\\+00:00$"; "Z") + | sub("\\.[0-9]+Z$"; "Z") + | try fromdateiso8601 catch null) + end; + def duration_for($provider; $window): + if positive_number($window.windowSeconds?) then ($window.windowSeconds | floor) + elif $provider == "claude" and $window.id == "five_hour" and $window.kind == "session" then 18000 + elif $provider == "claude" and $window.id == "seven_day" and $window.kind == "weekly" then 604800 + else null end; - def candidate_metric($p; $i): - . as $root - | ($p.harness // "") as $h - | ($root | provider_for($h)) as $provider - | if ($provider == null) or ((general_ids($h) | length) == 0) then empty + def scope_for($window; $duration): + if $duration == 18000 then "session" + elif $duration == 604800 then "weekly" + else $window.kind + end; + def settings: ($config.quotaBalanced? // {}); + def run_settings: (settings.runRate? // {}); + def reserve_for($provider): + (settings.providers?[$provider].reservePercent? // settings.reservePercent? // 0); + def extra_resets($provider; $scope; $duration): + ([settings.providers?[$provider].windows[]? + | select(.durationSeconds == $duration and .scope == $scope) + | (.extraResets // 0)] | add // 0); + def history_for($provider; $id; $kind; $duration): + [$history.samples[]? + | select(.provider == $provider and .id == $id and .kind == $kind + and .durationSeconds == $duration)][0]; + def window_metric($provider; $window): + (duration_for($provider; $window)) as $duration + | (iso_epoch($window.resetsAt?)) as $reset + | (if ($window.percentRemaining? | type) == "number" then $window.percentRemaining + elif ($window.percentUsed? | type) == "number" then (100 - $window.percentUsed) + else null end) as $raw_remaining + | if $duration == null or $reset == null or $raw_remaining == null then empty else - (($provider.windows // []) - | map(. as $window - | select(((general_ids($h) | index($window.id)) != null) - and (($window.kind? // "") != "model") - and (($window.percentRemaining? | type) == "number")))) as $windows - | if ($windows | length) == 0 then empty - else { - index: $i, - profile: clean($p), - harness: $h, - min: ($windows | map(.percentRemaining) | min), - fresh: (($provider.state.status? // "") == "fresh") + (clamp($raw_remaining; 0; 100)) as $remaining + | (scope_for($window; $duration)) as $scope + | (if ($window.percentUsed? | type) == "number" then clamp($window.percentUsed; 0; 100) + else (100 - $remaining) end) as $used + | (clamp(($reset - $now); 0; $duration)) as $until_reset + | ($duration - $until_reset) as $elapsed + | (run_settings.minimumObservationSeconds? // 300) as $minimum_observation + | (run_settings.historyMaxAgeSeconds? // 86400) as $history_max_age + | (run_settings.recentWeight? // 1) as $recent_weight + | (history_for($provider; ($window.id // ""); $window.kind; $duration)) as $prior + | (($prior != null) + and (($now - ($prior.sampledAtEpoch // ($now + 1))) > 0) + and (($now - $prior.sampledAtEpoch) <= $history_max_age) + and (((($prior.resetsAtEpoch // 0) - $reset) | fabs) <= 60) + and ($used >= ($prior.percentUsed // 101))) as $recent_valid + | ($used / ([$elapsed, $minimum_observation] | max)) as $cycle_rate + | (if $recent_valid then + (($used - $prior.percentUsed) / ($now - $prior.sampledAtEpoch)) + else 0 end) as $recent_rate + | ([$cycle_rate, ($recent_rate * $recent_weight)] | max) as $burn_rate + | (extra_resets($provider; $scope; $duration)) as $extra + | (reserve_for($provider)) as $reserve + | ($burn_rate * $until_reset) as $projected + | ($remaining + (100 * $extra)) as $capacity + | { + id: ($window.id // ""), + kind: $window.kind, + scope: $scope, + durationSeconds: $duration, + remaining: $remaining, + extraResets: $extra, + reserve: $reserve, + secondsUntilReset: $until_reset, + burnPerHour: ($burn_rate * 3600), + projectedBurn: $projected, + score: ($capacity - $reserve - $projected), + rateSource: (if $recent_valid and (($recent_rate * $recent_weight) > $cycle_rate) + then "recent" else "cycle" end), + clockClamped: (($reset - $now) < 0 or ($reset - $now) > $duration), + sample: { + provider: $provider, + id: ($window.id // ""), + kind: $window.kind, + durationSeconds: $duration, + resetsAtEpoch: $reset, + sampledAtEpoch: $now, + percentUsed: $used + } } + end; + def provider_for($name): [$quota.providers[]? | select(.provider == $name)][0]; + def candidate_metric($profile; $index): + ($profile.harness // "") as $provider_name + | (provider_for($provider_name)) as $provider + | if $provider == null then empty + else + ([$provider.windows[]? + | select((.kind == "session" or .kind == "weekly") + and (((.id? // "") | startswith("model:")) | not)) + | window_metric($provider_name; .)]) as $windows + | if ($windows | length) == 0 then empty + else ($windows | min_by(.score)) as $bottleneck + | { + index: $index, + profile: clean($profile), + harness: $provider_name, + fresh: (($provider.state.status? // "") == "fresh"), + score: $bottleneck.score, + bottleneck: $bottleneck, + windows: $windows + } end end; def better($a; $b): if $a == null then $b elif $b == null then $a - elif ($b.min > $a.min) then $b - elif ($b.min == $a.min and $b.index < $a.index) then $b + elif $b.score > $a.score then $b + elif $b.score == $a.score and $b.index < $a.index then $b else $a end; - def best_by_min($xs): reduce $xs[] as $x (null; better(.; $x)); - . as $quota_root - | ([$profiles | to_entries[] | . as $entry | ($quota_root | candidate_metric($entry.value; $entry.key))]) as $candidates + def best($items): reduce $items[] as $item (null; better(.; $item)); + ([range(0; $profiles | length) as $index + | candidate_metric($profiles[$index]; $index)]) as $candidates + | (if $staleOverride == "" then (settings.staleClearMargin? // 20) + else ($staleOverride | tonumber) end) as $stale_margin | if ($candidates | length) == 0 then { fallback: true, reason: "no usable quota windows for candidate vendors", - profile: clean($profiles[0]) + profile: clean($profiles[0]), + candidates: [], + samples: [] } else - (best_by_min($candidates | map(select(.fresh)))) as $fresh_best - | (best_by_min($candidates | map(select(.fresh | not)))) as $stale_best + (best($candidates | map(select(.fresh)))) as $fresh_best + | (best($candidates | map(select(.fresh | not)))) as $stale_best | (if $fresh_best != null and $stale_best != null then - if $stale_best.min >= ($fresh_best.min + $margin) then $stale_best else $fresh_best end + if $stale_best.score >= ($fresh_best.score + $stale_margin) + then $stale_best else $fresh_best end elif $fresh_best != null then $fresh_best else $stale_best end) as $chosen - | {fallback: false, profile: $chosen.profile} + | { + fallback: false, + profile: $chosen.profile, + chosen: $chosen.harness, + candidates: $candidates, + samples: ([$candidates[] | select(.fresh) | .windows[].sample] + | unique_by([.provider, .id, .kind, .durationSeconds])) + } end ' 2>/dev/null) || { - log "quota-axi data could not be evaluated; using first profile" + log "quota-axi data or quota-balanced configuration could not be evaluated; using first profile" first_profile exit 0 } +while IFS= read -r diagnostic; do + [ -n "$diagnostic" ] && log "$diagnostic" +done < <(printf '%s\n' "$selection" | jq -r ' + .candidates[]? + | . as $candidate + | $candidate.windows[] + | "quota-balanced candidate \($candidate.harness) trust=\(if $candidate.fresh then "fresh" else "stale" end)" + + " window=\(.scope)/\(.durationSeconds)s source-kind=\(.kind) score=\(.score * 100 | round / 100)" + + " remaining=\(.remaining) extra-resets=\(.extraResets) reserve=\(.reserve)" + + " burn-per-hour=\(.burnPerHour * 100 | round / 100) projected=\(.projectedBurn * 100 | round / 100)" + + " rate-source=\(.rateSource) clock-clamped=\(.clockClamped)" +') + if [ "$(printf '%s\n' "$selection" | jq -r '.fallback')" = true ]; then log "$(printf '%s\n' "$selection" | jq -r '.reason'); using first profile" +else + log "quota-balanced selected $(printf '%s\n' "$selection" | jq -r '.chosen') by bottleneck sustainable-headroom score" fi + +if [ -n "$STATE_FILE" ] && [ "$(printf '%s\n' "$selection" | jq '.samples | length')" -gt 0 ]; then + state_parent=$(dirname "$STATE_FILE") + if mkdir -p "$state_parent" 2>/dev/null; then + state_tmp=$(umask 077; mktemp "$state_parent/.dispatch-quota-samples.XXXXXX" 2>/dev/null) || state_tmp= + if [ -n "$state_tmp" ]; then + if jq -nec --argjson old "$history_json" --argjson current "$selection" ' + def key: [.provider, .id, .kind, .durationSeconds]; + {schemaVersion: 1, + samples: ([($old.samples[]? // empty), $current.samples[]] + | sort_by(key) + | group_by(key) + | map(last))} + ' > "$state_tmp" && chmod 600 "$state_tmp" && mv "$state_tmp" "$STATE_FILE"; then + : + else + rm -f "$state_tmp" + log "could not update quota sample history; selection remains valid" + fi + else + log "could not prepare quota sample history; selection remains valid" + fi + else + log "could not create quota sample state directory; selection remains valid" + fi +fi + printf '%s\n' "$selection" | jq -c '.profile' diff --git a/docs/configuration.md b/docs/configuration.md index 25560837d..e82d144c1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -194,6 +194,23 @@ This section is the single owner of the canonical schema and its per-field seman ```json { + "quotaBalanced": { + "reservePercent": 0, + "staleClearMargin": 20, + "runRate": { + "minimumObservationSeconds": 300, + "historyMaxAgeSeconds": 86400, + "recentWeight": 1 + }, + "providers": { + "": { + "reservePercent": 0, + "windows": [ + { "scope": "", "durationSeconds": 604800, "extraResets": 0 } + ] + } + } + }, "rules": [ { "when": "", @@ -216,7 +233,37 @@ Absent `select` means use the first array element, or the only object in the sin `default` is optional. An omitted model or effort means the selected harness uses its own default for that axis. If a selected profile carries an effort value the chosen harness does not accept, `fm-spawn.sh` records the requested `effort=` in task meta for traceability but omits the launch flag, and bootstrap reports the invalid harness/effort pair as a `CREW_DISPATCH` diagnostic when it is visible in the file. -`quota-balanced` selection is deterministic and implemented by `bin/fm-dispatch-select.sh`, whose header owns the general-window rules, the 20 point stale-clear freshness margin, vendor-availability handling, and the degrade-to-first-element fallbacks; quota trouble never blocks dispatch. +`quotaBalanced` is optional local tuning used only by rules with `select: "quota-balanced"`. +`reservePercent` keeps percentage-capacity headroom out of the usable score and defaults to 0, while a provider-level value overrides the global value. +`staleClearMargin` defaults to 20 percentage-capacity points and preserves the policy that cached stale quota must be clearly better than fresh quota to win. +`runRate.minimumObservationSeconds` defaults to 300 and bounds first-sample burst amplification near a reset. +`runRate.historyMaxAgeSeconds` defaults to 86400 and prevents old local samples from influencing recent burn. +`runRate.recentWeight` ranges from 0 to 1, defaults to 1, and controls how strongly an observed recent usage delta can raise the cycle-average burn estimate. +Each provider `windows` entry grants explicit manual reset capacity only to the matching canonical `scope` and `durationSeconds`. +The selector derives session versus weekly scope from actual 18000-second or 604800-second duration before considering provider labels, ids, or misleading kinds. +Weekly reset credits therefore never apply to an independent five-hour session window. +`extraResets` is a non-negative integer and adds one full normalized window of capacity per configured reset. +It represents currently unused manual resets and must be reduced locally when a reset is consumed. +The selector keeps account-free burn samples in `state/.dispatch-quota-samples.json`; the file is volatile private state and is not a source of configured reset credits. +`quota-balanced` selection is deterministic and implemented by `bin/fm-dispatch-select.sh`, whose header owns the exact score, rollover, clock-skew, diagnostics, vendor-availability, and degrade-to-first-element mechanics; quota trouble never blocks dispatch. + +To declare three Codex manual weekly resets after this version lands, merge this local object into `config/crew-dispatch.json` without adding a session entry: + +```json +{ + "quotaBalanced": { + "providers": { + "codex": { + "windows": [ + { "scope": "weekly", "durationSeconds": 604800, "extraResets": 3 } + ] + } + } + } +} +``` + +This declaration is local and gitignored, so the three-reset reserve is not a shared default and no account identity appears in tracked files or selector diagnostics. See [`docs/examples/crew-dispatch.json`](examples/crew-dispatch.json) for a starting point to copy into local `config/crew-dispatch.json`. When the file exists, bootstrap validates it with `jq`. Valid files stay silent by default; with `FM_BOOTSTRAP_VERBOSE_FACTS=1`, bootstrap emits `BOOTSTRAP_INFO: crew dispatch active config/crew-dispatch.json` plus one `BOOTSTRAP_INFO:` fact per rule and default profile. diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index 851ba66ec..130fb5929 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -782,6 +782,12 @@ empty array use is flagged^{"rules":[{"when":"big feature","use":[]}]}^exact^CRE array profile without harness is flagged^{"rules":[{"when":"big feature","use":[{"model":"gpt-5.5"}]}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - each use profile needs harness unknown select is flagged^{"rules":[{"when":"big feature","use":[{"harness":"claude"},{"harness":"codex"}],"select":"mystery"}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - unknown select: mystery array profile unsupported effort is flagged^{"rules":[{"when":"big feature","use":[{"harness":"codex","effort":"max"}]}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - invalid effort: codex:max +quota-balanced tuning is accepted^{"rules":[{"when":"big feature","use":[{"harness":"claude"},{"harness":"codex"}],"select":"quota-balanced"}],"quotaBalanced":{"reservePercent":5,"staleClearMargin":20,"runRate":{"minimumObservationSeconds":300,"historyMaxAgeSeconds":86400,"recentWeight":0.5},"providers":{"codex":{"reservePercent":10,"windows":[{"scope":"weekly","durationSeconds":604800,"extraResets":3}]}}}}^empty^ +quota-balanced reset count must be an integer^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"weekly","durationSeconds":604800,"extraResets":1.5}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced window extraResets must be a non-negative integer +quota-balanced duration must be positive^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"weekly","durationSeconds":0,"extraResets":3}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced window durationSeconds must be a positive integer +quota-balanced recent weight is bounded^{"quotaBalanced":{"runRate":{"recentWeight":1.5}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced.runRate.recentWeight must be between 0 and 1 +quota-balanced weekly resets cannot target session scope^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"session","durationSeconds":604800,"extraResets":3}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced window scope must match canonical 5-hour or 7-day duration +quota-balanced duplicate windows are flagged^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"weekly","durationSeconds":604800,"extraResets":1},{"scope":"weekly","durationSeconds":604800,"extraResets":2}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced provider windows must not duplicate scope and durationSeconds ROWS pass "bootstrap validates crew-dispatch.json and reports malformed or unverified configs" } diff --git a/tests/fm-dispatch-select.test.sh b/tests/fm-dispatch-select.test.sh index a9ae5b7bf..10f713358 100755 --- a/tests/fm-dispatch-select.test.sh +++ b/tests/fm-dispatch-select.test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Behavior tests for deterministic crew-dispatch profile selection. +# Behavior tests for deterministic, run-rate-aware crew dispatch selection. set -u # shellcheck source=tests/lib.sh @@ -9,148 +9,284 @@ BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin} TMP_ROOT=$(fm_test_tmproot fm-dispatch-select-tests) mkdir -p "$TMP_ROOT" -write_quota() { - local file=$1 claude_status=$2 claude_five=$3 claude_week=$4 codex_status=$5 codex_five=$6 codex_week=$7 - mkdir -p "$(dirname "$file")" - cat > "$file" < "$err" +} + +test_low_remaining_slow_burn_beats_high_remaining_burst() { + local quota state out err + quota="$TMP_ROOT/run-rate.json" + state="$TMP_ROOT/run-rate-state.json" + err="$TMP_ROOT/run-rate.err" + cat > "$quota" <<'JSON' { + "schemaVersion": 2, "providers": [ { "provider": "claude", - "state": { "status": "$claude_status" }, + "state": {"status": "fresh"}, "windows": [ - { "id": "five_hour", "kind": "session", "percentRemaining": $claude_five }, - { "id": "seven_day", "kind": "weekly", "percentRemaining": $claude_week }, - { "id": "model:fable", "kind": "model", "percentRemaining": 100 } + {"id": "five_hour", "kind": "session", "percentUsed": 80, "percentRemaining": 20, "resetsAt": "2026-07-19T00:10:00Z"}, + {"id": "seven_day", "kind": "weekly", "percentUsed": 80, "percentRemaining": 20, "resetsAt": "2026-07-20T00:00:00Z"}, + {"id": "model:fable", "kind": "model", "percentUsed": 99, "percentRemaining": 1, "resetsAt": "2026-07-20T00:00:00Z"} ] }, { "provider": "codex", - "state": { "status": "$codex_status" }, + "state": {"status": "fresh"}, "windows": [ - { "id": "five_hour", "kind": "session", "percentRemaining": $codex_five }, - { "id": "weekly", "kind": "weekly", "percentRemaining": $codex_week }, - { "id": "model:codex_bengalfox:5h", "kind": "model", "percentRemaining": 100 } + {"id": "five_hour", "label": "session", "kind": "session", "windowSeconds": 604800, "percentUsed": 20, "percentRemaining": 80, "resetsAt": "2026-07-24T00:00:00Z"} ] } ] } JSON + cat > "$state" < "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + { + "provider": "claude", + "state": {"status": "fresh"}, + "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 10, "percentRemaining": 90, "resetsAt": "2026-07-19T01:00:00Z"}, + {"id": "seven_day", "kind": "weekly", "percentUsed": 70, "percentRemaining": 30, "resetsAt": "2026-07-24T00:00:00Z"} + ] + }, + { + "provider": "codex", + "state": {"status": "fresh"}, + "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 50, "percentRemaining": 50, "resetsAt": "2026-07-19T00:00:00Z"} + ] + } + ] +} +JSON -test_higher_min_vendor_wins() { - local quota out - quota="$TMP_ROOT/higher.json" - write_quota "$quota" fresh 80 30 fresh 70 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") + out=$(run_fixture "$quota" "$err") + diagnostics=$(cat "$err") [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "higher-min vendor should win, got: $out" - pass "quota-balanced picks the candidate with the higher general-window minimum" + || fail "Claude weekly bottleneck should constrain an ample session window, got: $out" + assert_contains "$diagnostics" "candidate claude trust=fresh window=session/18000s" \ + "Claude session should use its canonical five-hour duration" + assert_contains "$diagnostics" "candidate claude trust=fresh window=weekly/604800s" \ + "Claude weekly window should be scored independently" + pass "Claude selection uses the lower sustainable headroom across session and weekly windows" } -test_exact_tie_uses_first_profile() { - local quota out - quota="$TMP_ROOT/tie.json" - write_quota "$quota" fresh 90 50 fresh 60 50 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "exact tie should pick first profile, got: $out" - pass "quota-balanced exact tie uses the first ordered profile" +test_codex_mislabeled_window_uses_actual_duration() { + local quota out err diagnostics + quota="$TMP_ROOT/codex-duration.json" + err="$TMP_ROOT/codex-duration.err" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 60, "percentRemaining": 40, "resetsAt": "2026-07-19T00:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "label": "session", "kind": "session", "windowSeconds": 604800, "percentUsed": 20, "percentRemaining": 80, "resetsAt": "2026-07-24T00:00:00Z"}, + {"id": "model:codex_bengalfox:5h", "kind": "model", "windowSeconds": 604800, "percentUsed": 99, "percentRemaining": 1, "resetsAt": "2026-07-24T00:00:00Z"} + ]} + ] } +JSON -test_quota_missing_falls_back_to_first() { - local fakebin out err status - fakebin=$(fm_fakebin "$TMP_ROOT/missing") - out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced "$profiles" 2>"$TMP_ROOT/missing.err") - status=$? - err=$(cat "$TMP_ROOT/missing.err") - expect_code 0 "$status" "missing quota-axi should not fail dispatch" + out=$(run_fixture "$quota" "$err") + diagnostics=$(cat "$err") [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "missing quota-axi should fall back to first, got: $out" - assert_contains "$err" "quota-axi missing" "missing quota-axi fallback should be logged" - pass "quota-axi missing falls back to the first profile and logs" + || fail "Codex seven-day burn should lose despite its five_hour id, got: $out" + assert_contains "$diagnostics" "candidate codex trust=fresh window=weekly/604800s source-kind=session" \ + "Codex diagnostics should expose the actual seven-day weekly scope" + assert_not_contains "$diagnostics" "codex_bengalfox" "model windows must remain excluded" + pass "Codex's misleading five_hour id cannot override its actual seven-day duration" } -test_quota_error_falls_back_to_first() { - local fakebin out err status - fakebin=$(fm_fakebin "$TMP_ROOT/error") - cat > "$fakebin/quota-axi" <<'SH' -#!/usr/bin/env bash -exit 42 -SH - chmod +x "$fakebin/quota-axi" - out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced "$profiles" 2>"$TMP_ROOT/error.err") - status=$? - err=$(cat "$TMP_ROOT/error.err") - expect_code 0 "$status" "quota-axi error should not fail dispatch" - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "quota-axi error should fall back to first, got: $out" - assert_contains "$err" "quota-axi exited 42" "quota-axi error fallback should be logged" - pass "quota-axi non-zero exit falls back to the first profile and logs" +test_configured_codex_extra_resets_add_capacity() { + local quota config out err diagnostics + quota="$TMP_ROOT/extra-resets.json" + config="$TMP_ROOT/extra-resets-config.json" + err="$TMP_ROOT/extra-resets.err" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 60, "percentRemaining": 40, "resetsAt": "2026-07-19T00:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 100, "percentRemaining": 0, "resetsAt": "2026-07-24T00:00:00Z"}, + {"id": "independent_five_hour", "kind": "session", "windowSeconds": 18000, "percentUsed": 0, "percentRemaining": 100, "resetsAt": "2026-07-19T00:00:00Z"} + ]} + ] +} +JSON + cat > "$config" <<'JSON' +{ + "quotaBalanced": { + "providers": { + "codex": { + "windows": [ + {"scope": "weekly", "durationSeconds": 604800, "extraResets": 3} + ] + } + } + } } +JSON -test_bad_quota_json_falls_back_to_first() { - local quota out err - quota="$TMP_ROOT/bad.json" - printf '%s\n' 'not-json' > "$quota" - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles" 2>"$TMP_ROOT/bad.err") - err=$(cat "$TMP_ROOT/bad.err") - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "bad quota JSON should fall back to first, got: $out" - assert_contains "$err" "unparseable JSON" "bad quota JSON fallback should be logged" - pass "unparseable quota JSON falls back to the first profile and logs" + out=$(run_fixture "$quota" "$err" --config "$config") + diagnostics=$(cat "$err") + [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ + || fail "three manual Codex resets should add three windows of capacity, got: $out" + assert_contains "$(printf '%s\n' "$diagnostics" | grep 'candidate codex.*window=weekly')" \ + "extra-resets=3" "weekly diagnostics should disclose configured reset capacity" + assert_contains "$(printf '%s\n' "$diagnostics" | grep 'candidate codex.*window=session')" \ + "extra-resets=0" "independent five-hour capacity must not receive weekly reset credits" + pass "three Codex weekly resets add weekly capacity while the independent five-hour window gets zero" } -test_stale_with_cache_needs_clear_margin_to_beat_fresh() { - local quota out - quota="$TMP_ROOT/stale-margin.json" - write_quota "$quota" stale 85 70 fresh 65 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") +test_stale_candidate_keeps_existing_clear_margin_policy() { + local quota out err + quota="$TMP_ROOT/stale.json" + err="$TMP_ROOT/stale.err" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "stale"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 70, "percentRemaining": 30, "resetsAt": "2026-07-19T00:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 80, "percentRemaining": 20, "resetsAt": "2026-07-19T00:00:00Z"} + ]} + ] +} +JSON + out=$(run_fixture "$quota" "$err") [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "fresh vendor should win when stale lead is below margin, got: $out" + || fail "fresh candidate should win when stale lead is under 20, got: $out" - write_quota "$quota" stale 90 85 fresh 65 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") + jq '(.providers[] | select(.provider == "claude").windows[0]) |= (.percentUsed = 50 | .percentRemaining = 50)' \ + "$quota" > "$TMP_ROOT/stale-clear.json" + out=$(run_fixture "$TMP_ROOT/stale-clear.json" "$err") [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "stale vendor should win when lead clears margin, got: $out" - pass "stale cached quota is usable only when it clears the documented margin over fresh" + || fail "stale candidate should win when sustainable score clears 20, got: $out" + pass "stale cached data remains usable only when its score clears the existing margin" } -test_vendor_absent_or_unusable_falls_back_conservatively() { - local quota out err - quota="$TMP_ROOT/absent.json" +test_rollover_discards_incompatible_recent_sample() { + local quota state out err new_reset + quota="$TMP_ROOT/rollover.json" + state="$TMP_ROOT/rollover-state.json" + err="$TMP_ROOT/rollover.err" + new_reset=$(jq -nr '"2026-07-26T00:00:00Z" | fromdateiso8601') cat > "$quota" <<'JSON' { + "schemaVersion": 2, "providers": [ - { - "provider": "codex", - "state": { "status": "fresh" }, - "windows": [ - { "id": "five_hour", "kind": "session", "percentRemaining": 40 }, - { "id": "weekly", "kind": "weekly", "percentRemaining": 50 } - ] - } + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 5, "percentRemaining": 95, "resetsAt": "2026-07-19T01:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 5, "percentRemaining": 95, "resetsAt": "2026-07-26T00:00:00Z"} + ]} ] } JSON - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") - [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "available candidate should win over absent vendor, got: $out" + cat > "$state" < "$quota" <<'JSON' -{ "providers": [] } +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 0, "percentRemaining": 100, "resetsAt": "2026-07-19T06:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 0, "percentRemaining": 100, "resetsAt": "2026-07-26T00:00:00Z"} + ]} + ] +} JSON - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles" 2>"$TMP_ROOT/none.err") - err=$(cat "$TMP_ROOT/none.err") + + out=$(run_fixture "$quota" "$err") + diagnostics=$(cat "$err") + [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ + || fail "Codex without authoritative duration should be unavailable, got: $out" + assert_contains "$diagnostics" "clock-clamped=true" "future reset beyond duration should be clock-clamped" + assert_not_contains "$diagnostics" "candidate codex" "missing Codex duration should not trust the five_hour id" + pass "missing durations are skipped and reset clock skew is bounded by actual duration" +} + +test_quota_failures_and_old_schema_fall_back_to_first() { + local fakebin out err status quota + fakebin=$(fm_fakebin "$TMP_ROOT/failure") + cat > "$fakebin/quota-axi" <<'SH' +#!/usr/bin/env bash +exit 42 +SH + chmod +x "$fakebin/quota-axi" + out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced \ + "$profiles" 2> "$TMP_ROOT/failure.err") + status=$? + err=$(cat "$TMP_ROOT/failure.err") + expect_code 0 "$status" "quota-axi error should not fail dispatch" + [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ + || fail "quota-axi failure should fall back to first, got: $out" + assert_contains "$err" "quota-axi exited 42" "quota-axi failure should be diagnosed" + + quota="$TMP_ROOT/schema-one.json" + printf '%s\n' '{"schemaVersion":1,"providers":[]}' > "$quota" + out=$(run_fixture "$quota" "$TMP_ROOT/schema-one.err") [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "no usable vendors should fall back to first, got: $out" - assert_contains "$err" "no usable quota windows" "no usable vendor fallback should be logged" - pass "absent or unusable vendors resolve to an available candidate or the first fallback" + || fail "unsupported schema should fall back to first, got: $out" + assert_contains "$(cat "$TMP_ROOT/schema-one.err")" "unsupported schema" \ + "unsupported schema should be diagnosed" + pass "quota-axi failure and unsupported schemas retain the first-profile fail-safe" } -test_backward_compatible_first_selection() { +test_backward_compatible_non_selector_paths() { local fakebin marker out single array_rule fakebin=$(fm_fakebin "$TMP_ROOT/no-call") marker="$TMP_ROOT/quota-called" @@ -171,16 +307,17 @@ SH [ "$out" = '{"harness":"claude","effort":"high"}' ] \ || fail "array without select should resolve to first, got: $out" [ ! -e "$marker" ] || fail "quota-axi should not be called without quota-balanced select" - pass "single-object use and no-select arrays preserve first-profile selection" + pass "single-object and no-select arrays preserve first-profile selection" } -test_higher_min_vendor_wins -test_exact_tie_uses_first_profile -test_quota_missing_falls_back_to_first -test_quota_error_falls_back_to_first -test_bad_quota_json_falls_back_to_first -test_stale_with_cache_needs_clear_margin_to_beat_fresh -test_vendor_absent_or_unusable_falls_back_conservatively -test_backward_compatible_first_selection +test_low_remaining_slow_burn_beats_high_remaining_burst +test_claude_session_and_weekly_bottlenecks +test_codex_mislabeled_window_uses_actual_duration +test_configured_codex_extra_resets_add_capacity +test_stale_candidate_keeps_existing_clear_margin_policy +test_rollover_discards_incompatible_recent_sample +test_missing_duration_and_clock_skew_are_contained +test_quota_failures_and_old_schema_fall_back_to_first +test_backward_compatible_non_selector_paths echo "# all fm-dispatch-select tests passed" From 4dc0e9a5a61049f2392189184c77188e03b53691 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 10:47:34 +1000 Subject: [PATCH 03/17] no-mistakes(review): Captain, prevent quota account data from entering process arguments --- bin/fm-dispatch-select.sh | 12 ++++++------ tests/fm-dispatch-select.test.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/bin/fm-dispatch-select.sh b/bin/fm-dispatch-select.sh index 08980d54c..866793909 100755 --- a/bin/fm-dispatch-select.sh +++ b/bin/fm-dispatch-select.sh @@ -11,7 +11,7 @@ # # quota-balanced is deterministic, and this header is the single owner of its # scoring contract: -# - It consumes quota-axi schema version 2 through `quota-axi --full --json` +# - It consumes quota-axi schema version 2 through `quota-axi --json` # (or the --quota-json fixture). # - General windows have provider kind `session` or `weekly`; model-scoped # windows are excluded regardless of label or id. An explicit positive @@ -43,7 +43,7 @@ # unsupported schema, or no candidate is usable, the reason is logged and # the first array element is printed. Quota trouble never blocks dispatch. # -# quota-balanced uses quota-axi --full --json unless --quota-json supplies a +# quota-balanced uses quota-axi --json unless --quota-json supplies a # fixture. Live calls keep private samples in state/.dispatch-quota-samples.json; # fixture calls are stateless unless --state-file is explicit. # The quotaBalanced object in config/crew-dispatch.json owns local tuning. @@ -237,7 +237,7 @@ else first_profile exit 0 fi - quota_json=$("$quota_cmd" --full --json 2>/dev/null) + quota_json=$("$quota_cmd" --json 2>/dev/null) quota_status=$? if [ "$quota_status" -ne 0 ]; then log "quota-axi exited $quota_status; using first profile" @@ -269,14 +269,14 @@ if [ -n "$STATE_FILE" ] && [ -f "$STATE_FILE" ]; then fi fi -selection=$(jq -nec \ - --argjson quota "$quota_json" \ +selection=$(printf '%s\n' "$quota_json" | jq -ec \ --argjson profiles "$profiles_json" \ --argjson config "$dispatch_config" \ --argjson history "$history_json" \ --argjson now "$NOW_EPOCH" \ --arg staleOverride "$STALE_CLEAR_OVERRIDE" ' - def clean($p): + . as $quota + | def clean($p): {harness: $p.harness} + (if ($p.model? | type) == "string" then {model: $p.model} else {} end) + (if ($p.effort? | type) == "string" then {effort: $p.effort} else {} end); diff --git a/tests/fm-dispatch-select.test.sh b/tests/fm-dispatch-select.test.sh index 10f713358..b3b7fa173 100755 --- a/tests/fm-dispatch-select.test.sh +++ b/tests/fm-dispatch-select.test.sh @@ -286,6 +286,35 @@ SH pass "quota-axi failure and unsupported schemas retain the first-profile fail-safe" } +test_live_quota_uses_account_free_json_and_private_diagnostics() { + local fakebin args out diagnostics + fakebin=$(fm_fakebin "$TMP_ROOT/live-quota") + args="$TMP_ROOT/live-quota.args" + cat > "$fakebin/quota-axi" < '$args' +cat <<'JSON' +{"schemaVersion":2,"account":{"email":"private@example.com"},"providers":[ + {"provider":"claude","state":{"status":"fresh"},"windows":[ + {"id":"five_hour","kind":"session","percentUsed":10,"percentRemaining":90,"resetsAt":"2026-07-19T01:00:00Z"} + ]} +]} +JSON +SH + chmod +x "$fakebin/quota-axi" + + out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced \ + --now-epoch "$NOW" "$profiles" 2> "$TMP_ROOT/live-quota.err") + diagnostics=$(cat "$TMP_ROOT/live-quota.err") + [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ + || fail "live quota JSON should select from schema-v2 windows, got: $out" + [ "$(cat "$args")" = "--json" ] \ + || fail "live quota command should request only normal account-free JSON" + assert_not_contains "$diagnostics" "private@example.com" \ + "quota diagnostics must not expose account identity" + pass "live quota requests normal JSON and keeps account data out of diagnostics" +} + test_backward_compatible_non_selector_paths() { local fakebin marker out single array_rule fakebin=$(fm_fakebin "$TMP_ROOT/no-call") @@ -318,6 +347,7 @@ test_stale_candidate_keeps_existing_clear_margin_policy test_rollover_discards_incompatible_recent_sample test_missing_duration_and_clock_skew_are_contained test_quota_failures_and_old_schema_fall_back_to_first +test_live_quota_uses_account_free_json_and_private_diagnostics test_backward_compatible_non_selector_paths echo "# all fm-dispatch-select tests passed" From 9b65efd51fd63a2399d3d77a079a4dfb7d084840 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 11:22:15 +1000 Subject: [PATCH 04/17] no-mistakes(test): Captain, make composer glyph parsing locale-safe --- bin/backends/herdr.sh | 4 ++-- bin/fm-composer-lib.sh | 8 ++++++-- docs/configuration.md | 2 +- tests/fm-backend-herdr.test.sh | 2 +- tests/fm-composer-lib.test.sh | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 260d2b666..29de3ddb5 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -739,7 +739,7 @@ FM_BACKEND_HERDR_IDLE_RE=${FM_BACKEND_HERDR_IDLE_RE:-'^Type a message\.\.\.$'} # Known bare (unbordered) prompt glyphs a composer row may start with: ❯ # (claude) and › (codex) only. Generic shell-style glyphs > $ % # are still # recognized after a bordered composer row has already been structurally found. -FM_BACKEND_HERDR_BARE_PROMPT_RE=${FM_BACKEND_HERDR_BARE_PROMPT_RE:-'^[❯›]'} +FM_BACKEND_HERDR_BARE_PROMPT_RE=${FM_BACKEND_HERDR_BARE_PROMPT_RE:-'^(❯|›)'} # Pi allows a multi-line composer between its horizontal separators. Bound the # structural candidate so two unrelated transcript rules with an arbitrarily # large region between them can never be promoted into a composer. @@ -903,7 +903,7 @@ EOF fi # Delegate the empty/pending/unknown decision to the shared owner. The bare # shape only ever starts with an AGENT glyph (FM_BACKEND_HERDR_BARE_PROMPT_RE - # is '^[❯›]'), so a bare shell prompt never reaches here - it stays 'unknown' + # is '^(❯|›)'), so a bare shell prompt never reaches here - it stays 'unknown' # via the no-composer-row path above, exactly as before. fm_composer_classify_content "$bordered" "$stripped" "$FM_BACKEND_HERDR_IDLE_RE" } diff --git a/bin/fm-composer-lib.sh b/bin/fm-composer-lib.sh index 437b8c689..4f065d36f 100644 --- a/bin/fm-composer-lib.sh +++ b/bin/fm-composer-lib.sh @@ -207,8 +207,12 @@ fm_composer_classify_content() { # [idle_re] [idle_case] [ fi # Strip a leading prompt glyph, then re-judge the remainder. case "$content" in - '❯ '*|'› '*|'> '*|'$ '*|'% '*|'# '*) content=${content#??} ;; - '❯'*|'›'*|'>'*|'$'*|'%'*|'#'*) content=${content#?} ;; + '❯ '*) content=${content#'❯ '} ;; + '› '*) content=${content#'› '} ;; + '❯'*) content=${content#'❯'} ;; + '›'*) content=${content#'›'} ;; + '> '*|'$ '*|'% '*|'# '*) content=${content#??} ;; + '>'*|'$'*|'%'*|'#'*) content=${content#?} ;; esac content="${content#"${content%%[![:space:]]*}"}" content="${content%"${content##*[![:space:]]}"}" diff --git a/docs/configuration.md b/docs/configuration.md index e82d144c1..6e4b75704 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -396,7 +396,7 @@ FM_BACKEND= # optional runtime backend override for new spawns; tmux HERDR_SESSION=default # herdr-only: named session for normal backend ops; not enough for destructive cleanup (docs/herdr-backend.md) FM_BACKEND_HERDR_COMPOSER_LINES=20 # herdr-only: tail lines scanned by composer-state guard/fallback paths; idle-baseline submit confirmation uses agent-state FM_BACKEND_HERDR_IDLE_RE='^Type a message\.\.\.$' # herdr-only: empty-composer placeholder regex after shared ghost extraction plus border and prompt stripping -FM_BACKEND_HERDR_BARE_PROMPT_RE='^[❯›]' # herdr-only: verified agent glyphs recognized as an UNBORDERED (bare) composer row, e.g. claude's ❯ or codex's ›; shell glyphs remain unknown rather than empty, and de-emphasised ghost/placeholder text (dim or dark-truecolor) after an agent prompt reads empty via the shared fm_composer_strip_ghost (docs/herdr-backend.md "Incident (2026-07-08)", "Incident (2026-07-10)") +FM_BACKEND_HERDR_BARE_PROMPT_RE='^(❯|›)' # herdr-only: verified agent glyphs recognized as an UNBORDERED (bare) composer row, e.g. claude's ❯ or codex's ›; shell glyphs remain unknown rather than empty, and de-emphasised ghost/placeholder text (dim or dark-truecolor) after an agent prompt reads empty via the shared fm_composer_strip_ghost (docs/herdr-backend.md "Incident (2026-07-08)", "Incident (2026-07-10)") FM_BACKEND_HERDR_PI_COMPOSER_MAX_LINES=8 # herdr-only: maximum rows admitted between Pi's native-identity-corroborated separator pair; taller or ambiguous candidates stay unknown (docs/herdr-backend.md "Incident (2026-07-14)") FM_BACKEND_HERDR_SUBMIT_POLLS=6 # herdr-only: agent-state samples spread across each Enter attempt's budget when confirming a submit (docs/herdr-backend.md "Native agent-state submit confirmation") FM_BACKEND_HERDR_SUBMIT_MIN_SLEEP=0.6 # herdr-only: minimum per-Enter confirmation budget before polling agent-state after an idle baseline diff --git a/tests/fm-backend-herdr.test.sh b/tests/fm-backend-herdr.test.sh index 5186cca2c..b52b23feb 100755 --- a/tests/fm-backend-herdr.test.sh +++ b/tests/fm-backend-herdr.test.sh @@ -817,7 +817,7 @@ test_composer_state_bare_prompt_is_empty() { dir="$TMP_ROOT/composer-bare"; mkdir -p "$dir/responses"; log="$dir/log"; resp="$dir/responses"; : > "$log" printf ' ╭────────────────────────╮\n │ ❯ │\n ╰──────── Composer ─────╯\n\n Shift+Tab:mode\n' > "$resp/1.out" fb=$(make_herdr_fakebin "$dir") - out=$( PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ + out=$( LC_ALL=C PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ bash -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_composer_state default:w1:p2' "$ROOT" ) [ "$out" = empty ] || fail "a bare prompt glyph should read as empty, got '$out'" pass "fm_backend_herdr_composer_state: a bare '❯' composer row reads empty" diff --git a/tests/fm-composer-lib.test.sh b/tests/fm-composer-lib.test.sh index 53cb83114..a620b5677 100755 --- a/tests/fm-composer-lib.test.sh +++ b/tests/fm-composer-lib.test.sh @@ -97,7 +97,7 @@ test_idle_placeholder_is_empty() { out=$(classify 1 'Type a message...' "$idle") [ "$out" = empty ] || fail "the grok idle placeholder should read empty, got '$out'" # Placeholder after an agent glyph (post-strip match). - out=$(classify 0 '❯ Type a message...' "$idle") + out=$(LC_ALL=C classify 0 '❯ Type a message...' "$idle") [ "$out" = empty ] || fail "the idle placeholder after a glyph should read empty, got '$out'" # Without the idle regex it is just text -> pending. out=$(classify 1 'Type a message...') From 0c2664d01eb631f421b5982183673c4becb7a55c Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 11:53:50 +1000 Subject: [PATCH 05/17] no-mistakes(document): Document quota tuning schema constraints --- docs/configuration.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6e4b75704..1b78f147d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -234,12 +234,15 @@ Absent `select` means use the first array element, or the only object in the sin An omitted model or effort means the selected harness uses its own default for that axis. If a selected profile carries an effort value the chosen harness does not accept, `fm-spawn.sh` records the requested `effort=` in task meta for traceability but omits the launch flag, and bootstrap reports the invalid harness/effort pair as a `CREW_DISPATCH` diagnostic when it is visible in the file. `quotaBalanced` is optional local tuning used only by rules with `select: "quota-balanced"`. -`reservePercent` keeps percentage-capacity headroom out of the usable score and defaults to 0, while a provider-level value overrides the global value. -`staleClearMargin` defaults to 20 percentage-capacity points and preserves the policy that cached stale quota must be clearly better than fresh quota to win. -`runRate.minimumObservationSeconds` defaults to 300 and bounds first-sample burst amplification near a reset. -`runRate.historyMaxAgeSeconds` defaults to 86400 and prevents old local samples from influencing recent burn. +`reservePercent` is a number from 0 through 100 that keeps percentage-capacity headroom out of the usable score and defaults to 0, while a provider-level value overrides the global value. +`staleClearMargin` is a non-negative number that defaults to 20 percentage-capacity points and preserves the policy that cached stale quota must be clearly better than fresh quota to win. +`runRate.minimumObservationSeconds` is a positive integer that defaults to 300 and bounds first-sample burst amplification near a reset. +`runRate.historyMaxAgeSeconds` is a positive integer that defaults to 86400 and prevents old local samples from influencing recent burn. `runRate.recentWeight` ranges from 0 to 1, defaults to 1, and controls how strongly an observed recent usage delta can raise the cycle-average burn estimate. -Each provider `windows` entry grants explicit manual reset capacity only to the matching canonical `scope` and `durationSeconds`. +Provider tuning keys are limited to `claude`, `codex`, `copilot`, `cursor`, and `grok`. +Each provider `windows` entry requires a `session` or `weekly` scope, a positive-integer `durationSeconds`, and a non-negative-integer `extraResets`, and duplicate scope-duration pairs are invalid within one provider. +The canonical 18000-second duration requires `session`, and the canonical 604800-second duration requires `weekly`. +Each valid entry grants explicit manual reset capacity only to its matching scope and duration. The selector derives session versus weekly scope from actual 18000-second or 604800-second duration before considering provider labels, ids, or misleading kinds. Weekly reset credits therefore never apply to an independent five-hour session window. `extraResets` is a non-negative integer and adds one full normalized window of capacity per configured reset. @@ -267,7 +270,7 @@ This declaration is local and gitignored, so the three-reset reserve is not a sh See [`docs/examples/crew-dispatch.json`](examples/crew-dispatch.json) for a starting point to copy into local `config/crew-dispatch.json`. When the file exists, bootstrap validates it with `jq`. Valid files stay silent by default; with `FM_BOOTSTRAP_VERBOSE_FACTS=1`, bootstrap emits `BOOTSTRAP_INFO: crew dispatch active config/crew-dispatch.json` plus one `BOOTSTRAP_INFO:` fact per rule and default profile. -Malformed JSON, an unverified harness, a malformed array profile, an unknown `select`, or an effort value unsupported by that harness is reported as `CREW_DISPATCH: invalid config/crew-dispatch.json - ...`; missing `jq` is reported through the normal `MISSING: jq` install-consent flow. +Malformed JSON, an invalid `quotaBalanced` schema or value, an unverified harness, a malformed array profile, an unknown `select`, or an effort value unsupported by that harness is reported as `CREW_DISPATCH: invalid config/crew-dispatch.json - ...`; missing `jq` is reported through the normal `MISSING: jq` install-consent flow. If no dispatch rule fits, firstmate uses the dispatch profile `default` when present, then falls back to `config/crew-harness`. Because the spawn backstop is gated by file presence, any fallback path after a missing match, validation error, or missing `jq` still passes a resolved harness explicitly until the file is fixed or removed. Secondmate homes inherit this file from the primary, so a secondmate's own crewmates apply the same dispatch profile behavior. From af144a43ed81619fe6d3d94b32aba1796ca0322b Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 10:34:51 +1000 Subject: [PATCH 06/17] feat: balance dispatch by quota run rate --- bin/fm-bootstrap.sh | 71 ++++++ bin/fm-dispatch-select.sh | 362 ++++++++++++++++++++++++++----- docs/configuration.md | 49 ++++- tests/fm-bootstrap.test.sh | 6 + tests/fm-dispatch-select.test.sh | 347 ++++++++++++++++++++--------- 5 files changed, 673 insertions(+), 162 deletions(-) diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 753c5cec3..723ecdacf 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -679,6 +679,76 @@ crew_dispatch_validate() { | map(select(. as $p | effort_ok($p.h; $p.e) | not)) | map("\(.h):\(.e)") | unique; + def nonnegative_number($value): ($value | type) == "number" and $value >= 0; + def positive_integer($value): + ($value | type) == "number" and $value > 0 and ($value | floor) == $value; + def nonnegative_integer($value): + ($value | type) == "number" and $value >= 0 and ($value | floor) == $value; + def quota_balanced_error: + (.quotaBalanced? // {}) as $quota + | if has("quotaBalanced") and ((.quotaBalanced | type) != "object") then + "quotaBalanced must be an object" + elif (($quota | keys_unsorted) - ["providers","reservePercent","runRate","staleClearMargin"] | length) > 0 then + "quotaBalanced has unknown fields" + elif $quota.reservePercent? != null + and ((nonnegative_number($quota.reservePercent) | not) or $quota.reservePercent > 100) then + "quotaBalanced.reservePercent must be between 0 and 100" + elif $quota.staleClearMargin? != null and (nonnegative_number($quota.staleClearMargin) | not) then + "quotaBalanced.staleClearMargin must be non-negative" + elif $quota.runRate? != null and (($quota.runRate | type) != "object") then + "quotaBalanced.runRate must be an object" + elif ((($quota.runRate? // {}) | keys_unsorted) + - ["historyMaxAgeSeconds","minimumObservationSeconds","recentWeight"] | length) > 0 then + "quotaBalanced.runRate has unknown fields" + elif $quota.runRate.minimumObservationSeconds? != null + and (positive_integer($quota.runRate.minimumObservationSeconds) | not) then + "quotaBalanced.runRate.minimumObservationSeconds must be a positive integer" + elif $quota.runRate.historyMaxAgeSeconds? != null + and (positive_integer($quota.runRate.historyMaxAgeSeconds) | not) then + "quotaBalanced.runRate.historyMaxAgeSeconds must be a positive integer" + elif $quota.runRate.recentWeight? != null + and ((nonnegative_number($quota.runRate.recentWeight) | not) or $quota.runRate.recentWeight > 1) then + "quotaBalanced.runRate.recentWeight must be between 0 and 1" + elif $quota.providers? != null and (($quota.providers | type) != "object") then + "quotaBalanced.providers must be an object" + elif ((($quota.providers? // {}) | keys_unsorted) + - ["claude","codex","copilot","cursor","grok"] | length) > 0 then + "quotaBalanced.providers has an unsupported provider" + elif [($quota.providers? // {})[] | select(type != "object")] | length > 0 then + "each quotaBalanced provider must be an object" + elif [($quota.providers? // {})[] + | select(((keys_unsorted - ["reservePercent","windows"]) | length) > 0)] | length > 0 then + "quotaBalanced provider has unknown fields" + elif [($quota.providers? // {})[] | .reservePercent? // empty + | select((nonnegative_number(.) | not) or . > 100)] | length > 0 then + "quotaBalanced provider reservePercent must be between 0 and 100" + elif [($quota.providers? // {})[] + | select(.windows? != null and ((.windows | type) != "array"))] | length > 0 then + "quotaBalanced provider windows must be an array" + elif [($quota.providers? // {})[] | .windows[]? | select(type != "object")] | length > 0 then + "each quotaBalanced window must be an object" + elif [($quota.providers? // {})[] | .windows[]? + | select(((keys_unsorted - ["durationSeconds","extraResets","scope"]) | length) > 0)] | length > 0 then + "quotaBalanced window has unknown fields" + elif [($quota.providers? // {})[] | .windows[]? + | select(positive_integer(.durationSeconds?) | not)] | length > 0 then + "quotaBalanced window durationSeconds must be a positive integer" + elif [($quota.providers? // {})[] | .windows[]? + | select(nonnegative_integer(.extraResets?) | not)] | length > 0 then + "quotaBalanced window extraResets must be a non-negative integer" + elif [($quota.providers? // {})[] | .windows[]? + | select(.scope != "session" and .scope != "weekly")] | length > 0 then + "quotaBalanced window scope must be session or weekly" + elif [($quota.providers? // {})[] | .windows[]? + | select((.durationSeconds == 18000 and .scope != "session") + or (.durationSeconds == 604800 and .scope != "weekly"))] | length > 0 then + "quotaBalanced window scope must match canonical 5-hour or 7-day duration" + elif [($quota.providers? // {})[] + | [(.windows // [])[] | [.scope, .durationSeconds]] + | group_by(.)[] | select(length > 1)] | length > 0 then + "quotaBalanced provider windows must not duplicate scope and durationSeconds" + else "" + end; if type != "object" then "top-level value must be an object" elif has("rules") and (.rules | type) != "array" then "rules must be an array" elif [(.rules // [])[]? | select(type != "object")] | length > 0 then "each rule must be an object" @@ -692,6 +762,7 @@ crew_dispatch_validate() { "unknown select: " + ([ (.rules // [])[]? | .select? // empty | select(. != "quota-balanced") ] | unique | join(", ")) elif has("default") and (.default | type) != "object" then "default must be an object" elif has("default") and ((.default.harness? | type) != "string" or (.default.harness | length) == 0) then "default needs harness when present" + elif (quota_balanced_error | length) > 0 then quota_balanced_error else ([(.rules // [])[]? | use_profiles(.use?)[]?.harness] + [.default?.harness?] | map(select(. != null)) diff --git a/bin/fm-dispatch-select.sh b/bin/fm-dispatch-select.sh index c626e9a95..08980d54c 100755 --- a/bin/fm-dispatch-select.sh +++ b/bin/fm-dispatch-select.sh @@ -1,39 +1,68 @@ #!/usr/bin/env bash # Resolve one already-matched crew-dispatch rule to a concrete profile. # Usage: -# fm-dispatch-select.sh [--select ] [--quota-json ] [] +# fm-dispatch-select.sh [--select ] [--quota-json ] +# [--config ] [--state-file ] [--now-epoch ] +# [] # # Input may be a full rule object with `use` and optional `select`, a single # profile object, or an ordered array of profile objects. # Output is one compact JSON profile object on stdout. # # quota-balanced is deterministic, and this header is the single owner of its -# contract: -# - It runs quota-axi --json (or the --quota-json fixture). -# - Per candidate vendor it takes the minimum percentRemaining across that -# vendor's GENERAL windows only - Claude five_hour and seven_day, Codex -# five_hour and weekly - ignoring model-scoped windows such as model:fable -# and model:codex_bengalfox:*. -# - The vendor with the higher minimum remaining quota wins; an exact tie -# between equally trusted candidates uses the first array element. +# scoring contract: +# - It consumes quota-axi schema version 2 through `quota-axi --full --json` +# (or the --quota-json fixture). +# - General windows have provider kind `session` or `weekly`; model-scoped +# windows are excluded regardless of label or id. An explicit positive +# windowSeconds is authoritative. Actual 5-hour and 7-day durations define +# session and weekly capacity scope even when provider kind is misleading. +# Claude's known five_hour and seven_day windows fall back to those canonical +# durations because Claude omits the field. No Codex duration is inferred +# from its misleading five_hour id. +# - A window score is percentage-capacity headroom through its reset: +# remaining + (100 * configured extra resets) - reserve +# - (estimated burn per second * seconds until reset) +# This is normalized provider-relative capacity, not a token or per-task +# prediction. A provider's score is its lowest general-window score. +# - On a first sample, burn is percent used divided by elapsed window time. +# A compatible recent sample can raise that estimate from actual usage +# delta; recentWeight controls burst sensitivity. Rollover, decreasing +# usage, old history, or clock reversal discards the recent delta. Reset +# time is clamped to the actual duration to contain source clock skew. +# - The vendor with the higher score wins; an exact tie between equally +# trusted candidates uses the first array element. # - Stale-but-cached general-window numbers are usable, but a fresh candidate -# wins unless the stale candidate's minimum is at least the stale-clear -# margin higher (default 20 points - the definition of "clearly less -# constrained"). +# wins unless the stale candidate's score is at least the stale-clear margin +# higher (default 20 percentage-capacity points). # - A vendor absent from quota output, or with no usable general windows, is # unavailable; selection happens among available candidates. -# - If quota-axi is missing, exits non-zero, returns unparseable JSON, or no -# candidate is usable, the reason is logged to stderr and the first array -# element is printed - quota trouble never blocks dispatch. +# - Candidate/window scores and their inputs are logged without account data +# so every decision is auditable. +# - If quota-axi is missing, exits non-zero, returns unparseable JSON, has an +# unsupported schema, or no candidate is usable, the reason is logged and +# the first array element is printed. Quota trouble never blocks dispatch. # -# quota-balanced uses quota-axi --json unless --quota-json supplies a fixture. +# quota-balanced uses quota-axi --full --json unless --quota-json supplies a +# fixture. Live calls keep private samples in state/.dispatch-quota-samples.json; +# fixture calls are stateless unless --state-file is explicit. +# The quotaBalanced object in config/crew-dispatch.json owns local tuning. # FM_DISPATCH_QUOTA_AXI overrides the quota command. -# FM_DISPATCH_STALE_CLEAR_MARGIN overrides the default 20 point stale margin. +# FM_DISPATCH_STALE_CLEAR_MARGIN overrides configured/default stale margin. set -u -STALE_CLEAR_MARGIN=${FM_DISPATCH_STALE_CLEAR_MARGIN:-20} +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +FM_DISPATCH_HOME=${FM_HOME:-$ROOT} +CONFIG_ROOT=${FM_CONFIG_OVERRIDE:-$FM_DISPATCH_HOME/config} +STATE_ROOT=${FM_STATE_OVERRIDE:-$FM_DISPATCH_HOME/state} +STALE_CLEAR_OVERRIDE=${FM_DISPATCH_STALE_CLEAR_MARGIN:-} SELECT_OVERRIDE= QUOTA_JSON_FILE= +CONFIG_FILE= +CONFIG_FILE_EXPLICIT=0 +STATE_FILE= +STATE_FILE_EXPLICIT=0 +NOW_EPOCH= ARGS=() usage() { @@ -68,6 +97,37 @@ while [ "$#" -gt 0 ]; do QUOTA_JSON_FILE=${1#--quota-json=} shift ;; + --config) + [ "$#" -gt 1 ] || { echo "error: --config requires a file" >&2; exit 2; } + CONFIG_FILE=$2 + CONFIG_FILE_EXPLICIT=1 + shift 2 + ;; + --config=*) + CONFIG_FILE=${1#--config=} + CONFIG_FILE_EXPLICIT=1 + shift + ;; + --state-file) + [ "$#" -gt 1 ] || { echo "error: --state-file requires a file" >&2; exit 2; } + STATE_FILE=$2 + STATE_FILE_EXPLICIT=1 + shift 2 + ;; + --state-file=*) + STATE_FILE=${1#--state-file=} + STATE_FILE_EXPLICIT=1 + shift + ;; + --now-epoch) + [ "$#" -gt 1 ] || { echo "error: --now-epoch requires seconds" >&2; exit 2; } + NOW_EPOCH=$2 + shift 2 + ;; + --now-epoch=*) + NOW_EPOCH=${1#--now-epoch=} + shift + ;; -h|--help) usage exit 0 @@ -93,6 +153,13 @@ done [ "${#ARGS[@]}" -le 1 ] || { echo "error: expected at most one JSON argument" >&2; exit 2; } command -v jq >/dev/null 2>&1 || { echo "error: jq is required" >&2; exit 2; } +if [ -z "$NOW_EPOCH" ]; then + NOW_EPOCH=$(date -u +%s) +fi +case "$NOW_EPOCH" in + ''|*[!0-9]*) echo "error: --now-epoch must be a non-negative integer" >&2; exit 2 ;; +esac + if [ "${#ARGS[@]}" -eq 1 ]; then SPEC_JSON=${ARGS[0]} else @@ -135,6 +202,28 @@ if [ "$select_strategy" != quota-balanced ]; then exit 0 fi +if [ "$CONFIG_FILE_EXPLICIT" -eq 0 ]; then + CONFIG_FILE="$CONFIG_ROOT/crew-dispatch.json" +fi +if [ -f "$CONFIG_FILE" ]; then + if ! dispatch_config=$(cat "$CONFIG_FILE" 2>/dev/null); then + log "cannot read dispatch config; using first profile" + first_profile + exit 0 + fi + if ! printf '%s\n' "$dispatch_config" | jq -e 'type == "object"' >/dev/null 2>&1; then + log "dispatch config is invalid; using first profile" + first_profile + exit 0 + fi +elif [ "$CONFIG_FILE_EXPLICIT" -eq 1 ]; then + log "cannot read dispatch config; using first profile" + first_profile + exit 0 +else + dispatch_config='{}' +fi + if [ -n "$QUOTA_JSON_FILE" ]; then if ! quota_json=$(cat "$QUOTA_JSON_FILE" 2>/dev/null); then log "cannot read quota JSON; using first profile" @@ -148,7 +237,7 @@ else first_profile exit 0 fi - quota_json=$("$quota_cmd" --json 2>/dev/null) + quota_json=$("$quota_cmd" --full --json 2>/dev/null) quota_status=$? if [ "$quota_status" -ne 0 ]; then log "quota-axi exited $quota_status; using first profile" @@ -157,78 +246,239 @@ else fi fi -if ! printf '%s\n' "$quota_json" | jq -e 'type == "object" and (.providers | type) == "array"' >/dev/null 2>&1; then - log "quota-axi returned unparseable JSON; using first profile" +if ! printf '%s\n' "$quota_json" | jq -e ' + type == "object" and .schemaVersion == 2 and (.providers | type) == "array" +' >/dev/null 2>&1; then + log "quota-axi returned invalid or unsupported schema JSON; using first profile" first_profile exit 0 fi -selection=$(printf '%s\n' "$quota_json" | jq -ec \ +if [ "$STATE_FILE_EXPLICIT" -eq 0 ] && [ -z "$QUOTA_JSON_FILE" ]; then + STATE_FILE="$STATE_ROOT/.dispatch-quota-samples.json" +fi +history_json='{"schemaVersion":1,"samples":[]}' +if [ -n "$STATE_FILE" ] && [ -f "$STATE_FILE" ]; then + if candidate_history=$(cat "$STATE_FILE" 2>/dev/null) \ + && printf '%s\n' "$candidate_history" | jq -e ' + type == "object" and .schemaVersion == 1 and (.samples | type) == "array" + ' >/dev/null 2>&1; then + history_json=$candidate_history + else + log "quota sample history is invalid; ignoring it" + fi +fi + +selection=$(jq -nec \ + --argjson quota "$quota_json" \ --argjson profiles "$profiles_json" \ - --argjson margin "$STALE_CLEAR_MARGIN" ' + --argjson config "$dispatch_config" \ + --argjson history "$history_json" \ + --argjson now "$NOW_EPOCH" \ + --arg staleOverride "$STALE_CLEAR_OVERRIDE" ' def clean($p): {harness: $p.harness} + (if ($p.model? | type) == "string" then {model: $p.model} else {} end) + (if ($p.effort? | type) == "string" then {effort: $p.effort} else {} end); - def provider_for($h): [.providers[]? | select(.provider == $h)][0]; - def general_ids($h): - if $h == "claude" then ["five_hour", "seven_day"] - elif $h == "codex" then ["five_hour", "weekly"] - else [] + def clamp($n; $low; $high): if $n < $low then $low elif $n > $high then $high else $n end; + def positive_number($v): ($v | type) == "number" and $v > 0; + def iso_epoch($value): + if ($value | type) != "string" then null + else ($value + | sub("\\+00:00$"; "Z") + | sub("\\.[0-9]+Z$"; "Z") + | try fromdateiso8601 catch null) + end; + def duration_for($provider; $window): + if positive_number($window.windowSeconds?) then ($window.windowSeconds | floor) + elif $provider == "claude" and $window.id == "five_hour" and $window.kind == "session" then 18000 + elif $provider == "claude" and $window.id == "seven_day" and $window.kind == "weekly" then 604800 + else null end; - def candidate_metric($p; $i): - . as $root - | ($p.harness // "") as $h - | ($root | provider_for($h)) as $provider - | if ($provider == null) or ((general_ids($h) | length) == 0) then empty + def scope_for($window; $duration): + if $duration == 18000 then "session" + elif $duration == 604800 then "weekly" + else $window.kind + end; + def settings: ($config.quotaBalanced? // {}); + def run_settings: (settings.runRate? // {}); + def reserve_for($provider): + (settings.providers?[$provider].reservePercent? // settings.reservePercent? // 0); + def extra_resets($provider; $scope; $duration): + ([settings.providers?[$provider].windows[]? + | select(.durationSeconds == $duration and .scope == $scope) + | (.extraResets // 0)] | add // 0); + def history_for($provider; $id; $kind; $duration): + [$history.samples[]? + | select(.provider == $provider and .id == $id and .kind == $kind + and .durationSeconds == $duration)][0]; + def window_metric($provider; $window): + (duration_for($provider; $window)) as $duration + | (iso_epoch($window.resetsAt?)) as $reset + | (if ($window.percentRemaining? | type) == "number" then $window.percentRemaining + elif ($window.percentUsed? | type) == "number" then (100 - $window.percentUsed) + else null end) as $raw_remaining + | if $duration == null or $reset == null or $raw_remaining == null then empty else - (($provider.windows // []) - | map(. as $window - | select(((general_ids($h) | index($window.id)) != null) - and (($window.kind? // "") != "model") - and (($window.percentRemaining? | type) == "number")))) as $windows - | if ($windows | length) == 0 then empty - else { - index: $i, - profile: clean($p), - harness: $h, - min: ($windows | map(.percentRemaining) | min), - fresh: (($provider.state.status? // "") == "fresh") + (clamp($raw_remaining; 0; 100)) as $remaining + | (scope_for($window; $duration)) as $scope + | (if ($window.percentUsed? | type) == "number" then clamp($window.percentUsed; 0; 100) + else (100 - $remaining) end) as $used + | (clamp(($reset - $now); 0; $duration)) as $until_reset + | ($duration - $until_reset) as $elapsed + | (run_settings.minimumObservationSeconds? // 300) as $minimum_observation + | (run_settings.historyMaxAgeSeconds? // 86400) as $history_max_age + | (run_settings.recentWeight? // 1) as $recent_weight + | (history_for($provider; ($window.id // ""); $window.kind; $duration)) as $prior + | (($prior != null) + and (($now - ($prior.sampledAtEpoch // ($now + 1))) > 0) + and (($now - $prior.sampledAtEpoch) <= $history_max_age) + and (((($prior.resetsAtEpoch // 0) - $reset) | fabs) <= 60) + and ($used >= ($prior.percentUsed // 101))) as $recent_valid + | ($used / ([$elapsed, $minimum_observation] | max)) as $cycle_rate + | (if $recent_valid then + (($used - $prior.percentUsed) / ($now - $prior.sampledAtEpoch)) + else 0 end) as $recent_rate + | ([$cycle_rate, ($recent_rate * $recent_weight)] | max) as $burn_rate + | (extra_resets($provider; $scope; $duration)) as $extra + | (reserve_for($provider)) as $reserve + | ($burn_rate * $until_reset) as $projected + | ($remaining + (100 * $extra)) as $capacity + | { + id: ($window.id // ""), + kind: $window.kind, + scope: $scope, + durationSeconds: $duration, + remaining: $remaining, + extraResets: $extra, + reserve: $reserve, + secondsUntilReset: $until_reset, + burnPerHour: ($burn_rate * 3600), + projectedBurn: $projected, + score: ($capacity - $reserve - $projected), + rateSource: (if $recent_valid and (($recent_rate * $recent_weight) > $cycle_rate) + then "recent" else "cycle" end), + clockClamped: (($reset - $now) < 0 or ($reset - $now) > $duration), + sample: { + provider: $provider, + id: ($window.id // ""), + kind: $window.kind, + durationSeconds: $duration, + resetsAtEpoch: $reset, + sampledAtEpoch: $now, + percentUsed: $used + } } + end; + def provider_for($name): [$quota.providers[]? | select(.provider == $name)][0]; + def candidate_metric($profile; $index): + ($profile.harness // "") as $provider_name + | (provider_for($provider_name)) as $provider + | if $provider == null then empty + else + ([$provider.windows[]? + | select((.kind == "session" or .kind == "weekly") + and (((.id? // "") | startswith("model:")) | not)) + | window_metric($provider_name; .)]) as $windows + | if ($windows | length) == 0 then empty + else ($windows | min_by(.score)) as $bottleneck + | { + index: $index, + profile: clean($profile), + harness: $provider_name, + fresh: (($provider.state.status? // "") == "fresh"), + score: $bottleneck.score, + bottleneck: $bottleneck, + windows: $windows + } end end; def better($a; $b): if $a == null then $b elif $b == null then $a - elif ($b.min > $a.min) then $b - elif ($b.min == $a.min and $b.index < $a.index) then $b + elif $b.score > $a.score then $b + elif $b.score == $a.score and $b.index < $a.index then $b else $a end; - def best_by_min($xs): reduce $xs[] as $x (null; better(.; $x)); - . as $quota_root - | ([$profiles | to_entries[] | . as $entry | ($quota_root | candidate_metric($entry.value; $entry.key))]) as $candidates + def best($items): reduce $items[] as $item (null; better(.; $item)); + ([range(0; $profiles | length) as $index + | candidate_metric($profiles[$index]; $index)]) as $candidates + | (if $staleOverride == "" then (settings.staleClearMargin? // 20) + else ($staleOverride | tonumber) end) as $stale_margin | if ($candidates | length) == 0 then { fallback: true, reason: "no usable quota windows for candidate vendors", - profile: clean($profiles[0]) + profile: clean($profiles[0]), + candidates: [], + samples: [] } else - (best_by_min($candidates | map(select(.fresh)))) as $fresh_best - | (best_by_min($candidates | map(select(.fresh | not)))) as $stale_best + (best($candidates | map(select(.fresh)))) as $fresh_best + | (best($candidates | map(select(.fresh | not)))) as $stale_best | (if $fresh_best != null and $stale_best != null then - if $stale_best.min >= ($fresh_best.min + $margin) then $stale_best else $fresh_best end + if $stale_best.score >= ($fresh_best.score + $stale_margin) + then $stale_best else $fresh_best end elif $fresh_best != null then $fresh_best else $stale_best end) as $chosen - | {fallback: false, profile: $chosen.profile} + | { + fallback: false, + profile: $chosen.profile, + chosen: $chosen.harness, + candidates: $candidates, + samples: ([$candidates[] | select(.fresh) | .windows[].sample] + | unique_by([.provider, .id, .kind, .durationSeconds])) + } end ' 2>/dev/null) || { - log "quota-axi data could not be evaluated; using first profile" + log "quota-axi data or quota-balanced configuration could not be evaluated; using first profile" first_profile exit 0 } +while IFS= read -r diagnostic; do + [ -n "$diagnostic" ] && log "$diagnostic" +done < <(printf '%s\n' "$selection" | jq -r ' + .candidates[]? + | . as $candidate + | $candidate.windows[] + | "quota-balanced candidate \($candidate.harness) trust=\(if $candidate.fresh then "fresh" else "stale" end)" + + " window=\(.scope)/\(.durationSeconds)s source-kind=\(.kind) score=\(.score * 100 | round / 100)" + + " remaining=\(.remaining) extra-resets=\(.extraResets) reserve=\(.reserve)" + + " burn-per-hour=\(.burnPerHour * 100 | round / 100) projected=\(.projectedBurn * 100 | round / 100)" + + " rate-source=\(.rateSource) clock-clamped=\(.clockClamped)" +') + if [ "$(printf '%s\n' "$selection" | jq -r '.fallback')" = true ]; then log "$(printf '%s\n' "$selection" | jq -r '.reason'); using first profile" +else + log "quota-balanced selected $(printf '%s\n' "$selection" | jq -r '.chosen') by bottleneck sustainable-headroom score" fi + +if [ -n "$STATE_FILE" ] && [ "$(printf '%s\n' "$selection" | jq '.samples | length')" -gt 0 ]; then + state_parent=$(dirname "$STATE_FILE") + if mkdir -p "$state_parent" 2>/dev/null; then + state_tmp=$(umask 077; mktemp "$state_parent/.dispatch-quota-samples.XXXXXX" 2>/dev/null) || state_tmp= + if [ -n "$state_tmp" ]; then + if jq -nec --argjson old "$history_json" --argjson current "$selection" ' + def key: [.provider, .id, .kind, .durationSeconds]; + {schemaVersion: 1, + samples: ([($old.samples[]? // empty), $current.samples[]] + | sort_by(key) + | group_by(key) + | map(last))} + ' > "$state_tmp" && chmod 600 "$state_tmp" && mv "$state_tmp" "$STATE_FILE"; then + : + else + rm -f "$state_tmp" + log "could not update quota sample history; selection remains valid" + fi + else + log "could not prepare quota sample history; selection remains valid" + fi + else + log "could not create quota sample state directory; selection remains valid" + fi +fi + printf '%s\n' "$selection" | jq -c '.profile' diff --git a/docs/configuration.md b/docs/configuration.md index 25560837d..e82d144c1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -194,6 +194,23 @@ This section is the single owner of the canonical schema and its per-field seman ```json { + "quotaBalanced": { + "reservePercent": 0, + "staleClearMargin": 20, + "runRate": { + "minimumObservationSeconds": 300, + "historyMaxAgeSeconds": 86400, + "recentWeight": 1 + }, + "providers": { + "": { + "reservePercent": 0, + "windows": [ + { "scope": "", "durationSeconds": 604800, "extraResets": 0 } + ] + } + } + }, "rules": [ { "when": "", @@ -216,7 +233,37 @@ Absent `select` means use the first array element, or the only object in the sin `default` is optional. An omitted model or effort means the selected harness uses its own default for that axis. If a selected profile carries an effort value the chosen harness does not accept, `fm-spawn.sh` records the requested `effort=` in task meta for traceability but omits the launch flag, and bootstrap reports the invalid harness/effort pair as a `CREW_DISPATCH` diagnostic when it is visible in the file. -`quota-balanced` selection is deterministic and implemented by `bin/fm-dispatch-select.sh`, whose header owns the general-window rules, the 20 point stale-clear freshness margin, vendor-availability handling, and the degrade-to-first-element fallbacks; quota trouble never blocks dispatch. +`quotaBalanced` is optional local tuning used only by rules with `select: "quota-balanced"`. +`reservePercent` keeps percentage-capacity headroom out of the usable score and defaults to 0, while a provider-level value overrides the global value. +`staleClearMargin` defaults to 20 percentage-capacity points and preserves the policy that cached stale quota must be clearly better than fresh quota to win. +`runRate.minimumObservationSeconds` defaults to 300 and bounds first-sample burst amplification near a reset. +`runRate.historyMaxAgeSeconds` defaults to 86400 and prevents old local samples from influencing recent burn. +`runRate.recentWeight` ranges from 0 to 1, defaults to 1, and controls how strongly an observed recent usage delta can raise the cycle-average burn estimate. +Each provider `windows` entry grants explicit manual reset capacity only to the matching canonical `scope` and `durationSeconds`. +The selector derives session versus weekly scope from actual 18000-second or 604800-second duration before considering provider labels, ids, or misleading kinds. +Weekly reset credits therefore never apply to an independent five-hour session window. +`extraResets` is a non-negative integer and adds one full normalized window of capacity per configured reset. +It represents currently unused manual resets and must be reduced locally when a reset is consumed. +The selector keeps account-free burn samples in `state/.dispatch-quota-samples.json`; the file is volatile private state and is not a source of configured reset credits. +`quota-balanced` selection is deterministic and implemented by `bin/fm-dispatch-select.sh`, whose header owns the exact score, rollover, clock-skew, diagnostics, vendor-availability, and degrade-to-first-element mechanics; quota trouble never blocks dispatch. + +To declare three Codex manual weekly resets after this version lands, merge this local object into `config/crew-dispatch.json` without adding a session entry: + +```json +{ + "quotaBalanced": { + "providers": { + "codex": { + "windows": [ + { "scope": "weekly", "durationSeconds": 604800, "extraResets": 3 } + ] + } + } + } +} +``` + +This declaration is local and gitignored, so the three-reset reserve is not a shared default and no account identity appears in tracked files or selector diagnostics. See [`docs/examples/crew-dispatch.json`](examples/crew-dispatch.json) for a starting point to copy into local `config/crew-dispatch.json`. When the file exists, bootstrap validates it with `jq`. Valid files stay silent by default; with `FM_BOOTSTRAP_VERBOSE_FACTS=1`, bootstrap emits `BOOTSTRAP_INFO: crew dispatch active config/crew-dispatch.json` plus one `BOOTSTRAP_INFO:` fact per rule and default profile. diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index 851ba66ec..130fb5929 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -782,6 +782,12 @@ empty array use is flagged^{"rules":[{"when":"big feature","use":[]}]}^exact^CRE array profile without harness is flagged^{"rules":[{"when":"big feature","use":[{"model":"gpt-5.5"}]}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - each use profile needs harness unknown select is flagged^{"rules":[{"when":"big feature","use":[{"harness":"claude"},{"harness":"codex"}],"select":"mystery"}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - unknown select: mystery array profile unsupported effort is flagged^{"rules":[{"when":"big feature","use":[{"harness":"codex","effort":"max"}]}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - invalid effort: codex:max +quota-balanced tuning is accepted^{"rules":[{"when":"big feature","use":[{"harness":"claude"},{"harness":"codex"}],"select":"quota-balanced"}],"quotaBalanced":{"reservePercent":5,"staleClearMargin":20,"runRate":{"minimumObservationSeconds":300,"historyMaxAgeSeconds":86400,"recentWeight":0.5},"providers":{"codex":{"reservePercent":10,"windows":[{"scope":"weekly","durationSeconds":604800,"extraResets":3}]}}}}^empty^ +quota-balanced reset count must be an integer^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"weekly","durationSeconds":604800,"extraResets":1.5}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced window extraResets must be a non-negative integer +quota-balanced duration must be positive^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"weekly","durationSeconds":0,"extraResets":3}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced window durationSeconds must be a positive integer +quota-balanced recent weight is bounded^{"quotaBalanced":{"runRate":{"recentWeight":1.5}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced.runRate.recentWeight must be between 0 and 1 +quota-balanced weekly resets cannot target session scope^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"session","durationSeconds":604800,"extraResets":3}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced window scope must match canonical 5-hour or 7-day duration +quota-balanced duplicate windows are flagged^{"quotaBalanced":{"providers":{"codex":{"windows":[{"scope":"weekly","durationSeconds":604800,"extraResets":1},{"scope":"weekly","durationSeconds":604800,"extraResets":2}]}}}}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - quotaBalanced provider windows must not duplicate scope and durationSeconds ROWS pass "bootstrap validates crew-dispatch.json and reports malformed or unverified configs" } diff --git a/tests/fm-dispatch-select.test.sh b/tests/fm-dispatch-select.test.sh index a9ae5b7bf..10f713358 100755 --- a/tests/fm-dispatch-select.test.sh +++ b/tests/fm-dispatch-select.test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Behavior tests for deterministic crew-dispatch profile selection. +# Behavior tests for deterministic, run-rate-aware crew dispatch selection. set -u # shellcheck source=tests/lib.sh @@ -9,148 +9,284 @@ BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin} TMP_ROOT=$(fm_test_tmproot fm-dispatch-select-tests) mkdir -p "$TMP_ROOT" -write_quota() { - local file=$1 claude_status=$2 claude_five=$3 claude_week=$4 codex_status=$5 codex_five=$6 codex_week=$7 - mkdir -p "$(dirname "$file")" - cat > "$file" < "$err" +} + +test_low_remaining_slow_burn_beats_high_remaining_burst() { + local quota state out err + quota="$TMP_ROOT/run-rate.json" + state="$TMP_ROOT/run-rate-state.json" + err="$TMP_ROOT/run-rate.err" + cat > "$quota" <<'JSON' { + "schemaVersion": 2, "providers": [ { "provider": "claude", - "state": { "status": "$claude_status" }, + "state": {"status": "fresh"}, "windows": [ - { "id": "five_hour", "kind": "session", "percentRemaining": $claude_five }, - { "id": "seven_day", "kind": "weekly", "percentRemaining": $claude_week }, - { "id": "model:fable", "kind": "model", "percentRemaining": 100 } + {"id": "five_hour", "kind": "session", "percentUsed": 80, "percentRemaining": 20, "resetsAt": "2026-07-19T00:10:00Z"}, + {"id": "seven_day", "kind": "weekly", "percentUsed": 80, "percentRemaining": 20, "resetsAt": "2026-07-20T00:00:00Z"}, + {"id": "model:fable", "kind": "model", "percentUsed": 99, "percentRemaining": 1, "resetsAt": "2026-07-20T00:00:00Z"} ] }, { "provider": "codex", - "state": { "status": "$codex_status" }, + "state": {"status": "fresh"}, "windows": [ - { "id": "five_hour", "kind": "session", "percentRemaining": $codex_five }, - { "id": "weekly", "kind": "weekly", "percentRemaining": $codex_week }, - { "id": "model:codex_bengalfox:5h", "kind": "model", "percentRemaining": 100 } + {"id": "five_hour", "label": "session", "kind": "session", "windowSeconds": 604800, "percentUsed": 20, "percentRemaining": 80, "resetsAt": "2026-07-24T00:00:00Z"} ] } ] } JSON + cat > "$state" < "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + { + "provider": "claude", + "state": {"status": "fresh"}, + "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 10, "percentRemaining": 90, "resetsAt": "2026-07-19T01:00:00Z"}, + {"id": "seven_day", "kind": "weekly", "percentUsed": 70, "percentRemaining": 30, "resetsAt": "2026-07-24T00:00:00Z"} + ] + }, + { + "provider": "codex", + "state": {"status": "fresh"}, + "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 50, "percentRemaining": 50, "resetsAt": "2026-07-19T00:00:00Z"} + ] + } + ] +} +JSON -test_higher_min_vendor_wins() { - local quota out - quota="$TMP_ROOT/higher.json" - write_quota "$quota" fresh 80 30 fresh 70 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") + out=$(run_fixture "$quota" "$err") + diagnostics=$(cat "$err") [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "higher-min vendor should win, got: $out" - pass "quota-balanced picks the candidate with the higher general-window minimum" + || fail "Claude weekly bottleneck should constrain an ample session window, got: $out" + assert_contains "$diagnostics" "candidate claude trust=fresh window=session/18000s" \ + "Claude session should use its canonical five-hour duration" + assert_contains "$diagnostics" "candidate claude trust=fresh window=weekly/604800s" \ + "Claude weekly window should be scored independently" + pass "Claude selection uses the lower sustainable headroom across session and weekly windows" } -test_exact_tie_uses_first_profile() { - local quota out - quota="$TMP_ROOT/tie.json" - write_quota "$quota" fresh 90 50 fresh 60 50 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "exact tie should pick first profile, got: $out" - pass "quota-balanced exact tie uses the first ordered profile" +test_codex_mislabeled_window_uses_actual_duration() { + local quota out err diagnostics + quota="$TMP_ROOT/codex-duration.json" + err="$TMP_ROOT/codex-duration.err" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 60, "percentRemaining": 40, "resetsAt": "2026-07-19T00:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "label": "session", "kind": "session", "windowSeconds": 604800, "percentUsed": 20, "percentRemaining": 80, "resetsAt": "2026-07-24T00:00:00Z"}, + {"id": "model:codex_bengalfox:5h", "kind": "model", "windowSeconds": 604800, "percentUsed": 99, "percentRemaining": 1, "resetsAt": "2026-07-24T00:00:00Z"} + ]} + ] } +JSON -test_quota_missing_falls_back_to_first() { - local fakebin out err status - fakebin=$(fm_fakebin "$TMP_ROOT/missing") - out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced "$profiles" 2>"$TMP_ROOT/missing.err") - status=$? - err=$(cat "$TMP_ROOT/missing.err") - expect_code 0 "$status" "missing quota-axi should not fail dispatch" + out=$(run_fixture "$quota" "$err") + diagnostics=$(cat "$err") [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "missing quota-axi should fall back to first, got: $out" - assert_contains "$err" "quota-axi missing" "missing quota-axi fallback should be logged" - pass "quota-axi missing falls back to the first profile and logs" + || fail "Codex seven-day burn should lose despite its five_hour id, got: $out" + assert_contains "$diagnostics" "candidate codex trust=fresh window=weekly/604800s source-kind=session" \ + "Codex diagnostics should expose the actual seven-day weekly scope" + assert_not_contains "$diagnostics" "codex_bengalfox" "model windows must remain excluded" + pass "Codex's misleading five_hour id cannot override its actual seven-day duration" } -test_quota_error_falls_back_to_first() { - local fakebin out err status - fakebin=$(fm_fakebin "$TMP_ROOT/error") - cat > "$fakebin/quota-axi" <<'SH' -#!/usr/bin/env bash -exit 42 -SH - chmod +x "$fakebin/quota-axi" - out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced "$profiles" 2>"$TMP_ROOT/error.err") - status=$? - err=$(cat "$TMP_ROOT/error.err") - expect_code 0 "$status" "quota-axi error should not fail dispatch" - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "quota-axi error should fall back to first, got: $out" - assert_contains "$err" "quota-axi exited 42" "quota-axi error fallback should be logged" - pass "quota-axi non-zero exit falls back to the first profile and logs" +test_configured_codex_extra_resets_add_capacity() { + local quota config out err diagnostics + quota="$TMP_ROOT/extra-resets.json" + config="$TMP_ROOT/extra-resets-config.json" + err="$TMP_ROOT/extra-resets.err" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 60, "percentRemaining": 40, "resetsAt": "2026-07-19T00:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 100, "percentRemaining": 0, "resetsAt": "2026-07-24T00:00:00Z"}, + {"id": "independent_five_hour", "kind": "session", "windowSeconds": 18000, "percentUsed": 0, "percentRemaining": 100, "resetsAt": "2026-07-19T00:00:00Z"} + ]} + ] +} +JSON + cat > "$config" <<'JSON' +{ + "quotaBalanced": { + "providers": { + "codex": { + "windows": [ + {"scope": "weekly", "durationSeconds": 604800, "extraResets": 3} + ] + } + } + } } +JSON -test_bad_quota_json_falls_back_to_first() { - local quota out err - quota="$TMP_ROOT/bad.json" - printf '%s\n' 'not-json' > "$quota" - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles" 2>"$TMP_ROOT/bad.err") - err=$(cat "$TMP_ROOT/bad.err") - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "bad quota JSON should fall back to first, got: $out" - assert_contains "$err" "unparseable JSON" "bad quota JSON fallback should be logged" - pass "unparseable quota JSON falls back to the first profile and logs" + out=$(run_fixture "$quota" "$err" --config "$config") + diagnostics=$(cat "$err") + [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ + || fail "three manual Codex resets should add three windows of capacity, got: $out" + assert_contains "$(printf '%s\n' "$diagnostics" | grep 'candidate codex.*window=weekly')" \ + "extra-resets=3" "weekly diagnostics should disclose configured reset capacity" + assert_contains "$(printf '%s\n' "$diagnostics" | grep 'candidate codex.*window=session')" \ + "extra-resets=0" "independent five-hour capacity must not receive weekly reset credits" + pass "three Codex weekly resets add weekly capacity while the independent five-hour window gets zero" } -test_stale_with_cache_needs_clear_margin_to_beat_fresh() { - local quota out - quota="$TMP_ROOT/stale-margin.json" - write_quota "$quota" stale 85 70 fresh 65 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") +test_stale_candidate_keeps_existing_clear_margin_policy() { + local quota out err + quota="$TMP_ROOT/stale.json" + err="$TMP_ROOT/stale.err" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "stale"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 70, "percentRemaining": 30, "resetsAt": "2026-07-19T00:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 80, "percentRemaining": 20, "resetsAt": "2026-07-19T00:00:00Z"} + ]} + ] +} +JSON + out=$(run_fixture "$quota" "$err") [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "fresh vendor should win when stale lead is below margin, got: $out" + || fail "fresh candidate should win when stale lead is under 20, got: $out" - write_quota "$quota" stale 90 85 fresh 65 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") + jq '(.providers[] | select(.provider == "claude").windows[0]) |= (.percentUsed = 50 | .percentRemaining = 50)' \ + "$quota" > "$TMP_ROOT/stale-clear.json" + out=$(run_fixture "$TMP_ROOT/stale-clear.json" "$err") [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "stale vendor should win when lead clears margin, got: $out" - pass "stale cached quota is usable only when it clears the documented margin over fresh" + || fail "stale candidate should win when sustainable score clears 20, got: $out" + pass "stale cached data remains usable only when its score clears the existing margin" } -test_vendor_absent_or_unusable_falls_back_conservatively() { - local quota out err - quota="$TMP_ROOT/absent.json" +test_rollover_discards_incompatible_recent_sample() { + local quota state out err new_reset + quota="$TMP_ROOT/rollover.json" + state="$TMP_ROOT/rollover-state.json" + err="$TMP_ROOT/rollover.err" + new_reset=$(jq -nr '"2026-07-26T00:00:00Z" | fromdateiso8601') cat > "$quota" <<'JSON' { + "schemaVersion": 2, "providers": [ - { - "provider": "codex", - "state": { "status": "fresh" }, - "windows": [ - { "id": "five_hour", "kind": "session", "percentRemaining": 40 }, - { "id": "weekly", "kind": "weekly", "percentRemaining": 50 } - ] - } + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 5, "percentRemaining": 95, "resetsAt": "2026-07-19T01:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 5, "percentRemaining": 95, "resetsAt": "2026-07-26T00:00:00Z"} + ]} ] } JSON - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") - [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "available candidate should win over absent vendor, got: $out" + cat > "$state" < "$quota" <<'JSON' -{ "providers": [] } +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 0, "percentRemaining": 100, "resetsAt": "2026-07-19T06:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 0, "percentRemaining": 100, "resetsAt": "2026-07-26T00:00:00Z"} + ]} + ] +} JSON - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles" 2>"$TMP_ROOT/none.err") - err=$(cat "$TMP_ROOT/none.err") + + out=$(run_fixture "$quota" "$err") + diagnostics=$(cat "$err") + [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ + || fail "Codex without authoritative duration should be unavailable, got: $out" + assert_contains "$diagnostics" "clock-clamped=true" "future reset beyond duration should be clock-clamped" + assert_not_contains "$diagnostics" "candidate codex" "missing Codex duration should not trust the five_hour id" + pass "missing durations are skipped and reset clock skew is bounded by actual duration" +} + +test_quota_failures_and_old_schema_fall_back_to_first() { + local fakebin out err status quota + fakebin=$(fm_fakebin "$TMP_ROOT/failure") + cat > "$fakebin/quota-axi" <<'SH' +#!/usr/bin/env bash +exit 42 +SH + chmod +x "$fakebin/quota-axi" + out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced \ + "$profiles" 2> "$TMP_ROOT/failure.err") + status=$? + err=$(cat "$TMP_ROOT/failure.err") + expect_code 0 "$status" "quota-axi error should not fail dispatch" + [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ + || fail "quota-axi failure should fall back to first, got: $out" + assert_contains "$err" "quota-axi exited 42" "quota-axi failure should be diagnosed" + + quota="$TMP_ROOT/schema-one.json" + printf '%s\n' '{"schemaVersion":1,"providers":[]}' > "$quota" + out=$(run_fixture "$quota" "$TMP_ROOT/schema-one.err") [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "no usable vendors should fall back to first, got: $out" - assert_contains "$err" "no usable quota windows" "no usable vendor fallback should be logged" - pass "absent or unusable vendors resolve to an available candidate or the first fallback" + || fail "unsupported schema should fall back to first, got: $out" + assert_contains "$(cat "$TMP_ROOT/schema-one.err")" "unsupported schema" \ + "unsupported schema should be diagnosed" + pass "quota-axi failure and unsupported schemas retain the first-profile fail-safe" } -test_backward_compatible_first_selection() { +test_backward_compatible_non_selector_paths() { local fakebin marker out single array_rule fakebin=$(fm_fakebin "$TMP_ROOT/no-call") marker="$TMP_ROOT/quota-called" @@ -171,16 +307,17 @@ SH [ "$out" = '{"harness":"claude","effort":"high"}' ] \ || fail "array without select should resolve to first, got: $out" [ ! -e "$marker" ] || fail "quota-axi should not be called without quota-balanced select" - pass "single-object use and no-select arrays preserve first-profile selection" + pass "single-object and no-select arrays preserve first-profile selection" } -test_higher_min_vendor_wins -test_exact_tie_uses_first_profile -test_quota_missing_falls_back_to_first -test_quota_error_falls_back_to_first -test_bad_quota_json_falls_back_to_first -test_stale_with_cache_needs_clear_margin_to_beat_fresh -test_vendor_absent_or_unusable_falls_back_conservatively -test_backward_compatible_first_selection +test_low_remaining_slow_burn_beats_high_remaining_burst +test_claude_session_and_weekly_bottlenecks +test_codex_mislabeled_window_uses_actual_duration +test_configured_codex_extra_resets_add_capacity +test_stale_candidate_keeps_existing_clear_margin_policy +test_rollover_discards_incompatible_recent_sample +test_missing_duration_and_clock_skew_are_contained +test_quota_failures_and_old_schema_fall_back_to_first +test_backward_compatible_non_selector_paths echo "# all fm-dispatch-select tests passed" From 3e25d72b165ab2dae383596118875526110dd97a Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 10:47:34 +1000 Subject: [PATCH 07/17] no-mistakes(review): Captain, prevent quota account data from entering process arguments --- bin/fm-dispatch-select.sh | 12 ++++++------ tests/fm-dispatch-select.test.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/bin/fm-dispatch-select.sh b/bin/fm-dispatch-select.sh index 08980d54c..866793909 100755 --- a/bin/fm-dispatch-select.sh +++ b/bin/fm-dispatch-select.sh @@ -11,7 +11,7 @@ # # quota-balanced is deterministic, and this header is the single owner of its # scoring contract: -# - It consumes quota-axi schema version 2 through `quota-axi --full --json` +# - It consumes quota-axi schema version 2 through `quota-axi --json` # (or the --quota-json fixture). # - General windows have provider kind `session` or `weekly`; model-scoped # windows are excluded regardless of label or id. An explicit positive @@ -43,7 +43,7 @@ # unsupported schema, or no candidate is usable, the reason is logged and # the first array element is printed. Quota trouble never blocks dispatch. # -# quota-balanced uses quota-axi --full --json unless --quota-json supplies a +# quota-balanced uses quota-axi --json unless --quota-json supplies a # fixture. Live calls keep private samples in state/.dispatch-quota-samples.json; # fixture calls are stateless unless --state-file is explicit. # The quotaBalanced object in config/crew-dispatch.json owns local tuning. @@ -237,7 +237,7 @@ else first_profile exit 0 fi - quota_json=$("$quota_cmd" --full --json 2>/dev/null) + quota_json=$("$quota_cmd" --json 2>/dev/null) quota_status=$? if [ "$quota_status" -ne 0 ]; then log "quota-axi exited $quota_status; using first profile" @@ -269,14 +269,14 @@ if [ -n "$STATE_FILE" ] && [ -f "$STATE_FILE" ]; then fi fi -selection=$(jq -nec \ - --argjson quota "$quota_json" \ +selection=$(printf '%s\n' "$quota_json" | jq -ec \ --argjson profiles "$profiles_json" \ --argjson config "$dispatch_config" \ --argjson history "$history_json" \ --argjson now "$NOW_EPOCH" \ --arg staleOverride "$STALE_CLEAR_OVERRIDE" ' - def clean($p): + . as $quota + | def clean($p): {harness: $p.harness} + (if ($p.model? | type) == "string" then {model: $p.model} else {} end) + (if ($p.effort? | type) == "string" then {effort: $p.effort} else {} end); diff --git a/tests/fm-dispatch-select.test.sh b/tests/fm-dispatch-select.test.sh index 10f713358..b3b7fa173 100755 --- a/tests/fm-dispatch-select.test.sh +++ b/tests/fm-dispatch-select.test.sh @@ -286,6 +286,35 @@ SH pass "quota-axi failure and unsupported schemas retain the first-profile fail-safe" } +test_live_quota_uses_account_free_json_and_private_diagnostics() { + local fakebin args out diagnostics + fakebin=$(fm_fakebin "$TMP_ROOT/live-quota") + args="$TMP_ROOT/live-quota.args" + cat > "$fakebin/quota-axi" < '$args' +cat <<'JSON' +{"schemaVersion":2,"account":{"email":"private@example.com"},"providers":[ + {"provider":"claude","state":{"status":"fresh"},"windows":[ + {"id":"five_hour","kind":"session","percentUsed":10,"percentRemaining":90,"resetsAt":"2026-07-19T01:00:00Z"} + ]} +]} +JSON +SH + chmod +x "$fakebin/quota-axi" + + out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced \ + --now-epoch "$NOW" "$profiles" 2> "$TMP_ROOT/live-quota.err") + diagnostics=$(cat "$TMP_ROOT/live-quota.err") + [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ + || fail "live quota JSON should select from schema-v2 windows, got: $out" + [ "$(cat "$args")" = "--json" ] \ + || fail "live quota command should request only normal account-free JSON" + assert_not_contains "$diagnostics" "private@example.com" \ + "quota diagnostics must not expose account identity" + pass "live quota requests normal JSON and keeps account data out of diagnostics" +} + test_backward_compatible_non_selector_paths() { local fakebin marker out single array_rule fakebin=$(fm_fakebin "$TMP_ROOT/no-call") @@ -318,6 +347,7 @@ test_stale_candidate_keeps_existing_clear_margin_policy test_rollover_discards_incompatible_recent_sample test_missing_duration_and_clock_skew_are_contained test_quota_failures_and_old_schema_fall_back_to_first +test_live_quota_uses_account_free_json_and_private_diagnostics test_backward_compatible_non_selector_paths echo "# all fm-dispatch-select tests passed" From 276bfd627e2f841e318d177f5ae7973d0ad60dbf Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 11:22:15 +1000 Subject: [PATCH 08/17] no-mistakes(test): Captain, make composer glyph parsing locale-safe --- bin/backends/herdr.sh | 4 ++-- bin/fm-composer-lib.sh | 8 ++++++-- docs/configuration.md | 2 +- tests/fm-backend-herdr.test.sh | 2 +- tests/fm-composer-lib.test.sh | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 260d2b666..29de3ddb5 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -739,7 +739,7 @@ FM_BACKEND_HERDR_IDLE_RE=${FM_BACKEND_HERDR_IDLE_RE:-'^Type a message\.\.\.$'} # Known bare (unbordered) prompt glyphs a composer row may start with: ❯ # (claude) and › (codex) only. Generic shell-style glyphs > $ % # are still # recognized after a bordered composer row has already been structurally found. -FM_BACKEND_HERDR_BARE_PROMPT_RE=${FM_BACKEND_HERDR_BARE_PROMPT_RE:-'^[❯›]'} +FM_BACKEND_HERDR_BARE_PROMPT_RE=${FM_BACKEND_HERDR_BARE_PROMPT_RE:-'^(❯|›)'} # Pi allows a multi-line composer between its horizontal separators. Bound the # structural candidate so two unrelated transcript rules with an arbitrarily # large region between them can never be promoted into a composer. @@ -903,7 +903,7 @@ EOF fi # Delegate the empty/pending/unknown decision to the shared owner. The bare # shape only ever starts with an AGENT glyph (FM_BACKEND_HERDR_BARE_PROMPT_RE - # is '^[❯›]'), so a bare shell prompt never reaches here - it stays 'unknown' + # is '^(❯|›)'), so a bare shell prompt never reaches here - it stays 'unknown' # via the no-composer-row path above, exactly as before. fm_composer_classify_content "$bordered" "$stripped" "$FM_BACKEND_HERDR_IDLE_RE" } diff --git a/bin/fm-composer-lib.sh b/bin/fm-composer-lib.sh index 437b8c689..4f065d36f 100644 --- a/bin/fm-composer-lib.sh +++ b/bin/fm-composer-lib.sh @@ -207,8 +207,12 @@ fm_composer_classify_content() { # [idle_re] [idle_case] [ fi # Strip a leading prompt glyph, then re-judge the remainder. case "$content" in - '❯ '*|'› '*|'> '*|'$ '*|'% '*|'# '*) content=${content#??} ;; - '❯'*|'›'*|'>'*|'$'*|'%'*|'#'*) content=${content#?} ;; + '❯ '*) content=${content#'❯ '} ;; + '› '*) content=${content#'› '} ;; + '❯'*) content=${content#'❯'} ;; + '›'*) content=${content#'›'} ;; + '> '*|'$ '*|'% '*|'# '*) content=${content#??} ;; + '>'*|'$'*|'%'*|'#'*) content=${content#?} ;; esac content="${content#"${content%%[![:space:]]*}"}" content="${content%"${content##*[![:space:]]}"}" diff --git a/docs/configuration.md b/docs/configuration.md index e82d144c1..6e4b75704 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -396,7 +396,7 @@ FM_BACKEND= # optional runtime backend override for new spawns; tmux HERDR_SESSION=default # herdr-only: named session for normal backend ops; not enough for destructive cleanup (docs/herdr-backend.md) FM_BACKEND_HERDR_COMPOSER_LINES=20 # herdr-only: tail lines scanned by composer-state guard/fallback paths; idle-baseline submit confirmation uses agent-state FM_BACKEND_HERDR_IDLE_RE='^Type a message\.\.\.$' # herdr-only: empty-composer placeholder regex after shared ghost extraction plus border and prompt stripping -FM_BACKEND_HERDR_BARE_PROMPT_RE='^[❯›]' # herdr-only: verified agent glyphs recognized as an UNBORDERED (bare) composer row, e.g. claude's ❯ or codex's ›; shell glyphs remain unknown rather than empty, and de-emphasised ghost/placeholder text (dim or dark-truecolor) after an agent prompt reads empty via the shared fm_composer_strip_ghost (docs/herdr-backend.md "Incident (2026-07-08)", "Incident (2026-07-10)") +FM_BACKEND_HERDR_BARE_PROMPT_RE='^(❯|›)' # herdr-only: verified agent glyphs recognized as an UNBORDERED (bare) composer row, e.g. claude's ❯ or codex's ›; shell glyphs remain unknown rather than empty, and de-emphasised ghost/placeholder text (dim or dark-truecolor) after an agent prompt reads empty via the shared fm_composer_strip_ghost (docs/herdr-backend.md "Incident (2026-07-08)", "Incident (2026-07-10)") FM_BACKEND_HERDR_PI_COMPOSER_MAX_LINES=8 # herdr-only: maximum rows admitted between Pi's native-identity-corroborated separator pair; taller or ambiguous candidates stay unknown (docs/herdr-backend.md "Incident (2026-07-14)") FM_BACKEND_HERDR_SUBMIT_POLLS=6 # herdr-only: agent-state samples spread across each Enter attempt's budget when confirming a submit (docs/herdr-backend.md "Native agent-state submit confirmation") FM_BACKEND_HERDR_SUBMIT_MIN_SLEEP=0.6 # herdr-only: minimum per-Enter confirmation budget before polling agent-state after an idle baseline diff --git a/tests/fm-backend-herdr.test.sh b/tests/fm-backend-herdr.test.sh index 5186cca2c..b52b23feb 100755 --- a/tests/fm-backend-herdr.test.sh +++ b/tests/fm-backend-herdr.test.sh @@ -817,7 +817,7 @@ test_composer_state_bare_prompt_is_empty() { dir="$TMP_ROOT/composer-bare"; mkdir -p "$dir/responses"; log="$dir/log"; resp="$dir/responses"; : > "$log" printf ' ╭────────────────────────╮\n │ ❯ │\n ╰──────── Composer ─────╯\n\n Shift+Tab:mode\n' > "$resp/1.out" fb=$(make_herdr_fakebin "$dir") - out=$( PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ + out=$( LC_ALL=C PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ bash -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_composer_state default:w1:p2' "$ROOT" ) [ "$out" = empty ] || fail "a bare prompt glyph should read as empty, got '$out'" pass "fm_backend_herdr_composer_state: a bare '❯' composer row reads empty" diff --git a/tests/fm-composer-lib.test.sh b/tests/fm-composer-lib.test.sh index 53cb83114..a620b5677 100755 --- a/tests/fm-composer-lib.test.sh +++ b/tests/fm-composer-lib.test.sh @@ -97,7 +97,7 @@ test_idle_placeholder_is_empty() { out=$(classify 1 'Type a message...' "$idle") [ "$out" = empty ] || fail "the grok idle placeholder should read empty, got '$out'" # Placeholder after an agent glyph (post-strip match). - out=$(classify 0 '❯ Type a message...' "$idle") + out=$(LC_ALL=C classify 0 '❯ Type a message...' "$idle") [ "$out" = empty ] || fail "the idle placeholder after a glyph should read empty, got '$out'" # Without the idle regex it is just text -> pending. out=$(classify 1 'Type a message...') From 47883c5e16170cb0604bdc3dd8b161e8576b76cd Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 11:53:50 +1000 Subject: [PATCH 09/17] no-mistakes(document): Document quota tuning schema constraints --- docs/configuration.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6e4b75704..1b78f147d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -234,12 +234,15 @@ Absent `select` means use the first array element, or the only object in the sin An omitted model or effort means the selected harness uses its own default for that axis. If a selected profile carries an effort value the chosen harness does not accept, `fm-spawn.sh` records the requested `effort=` in task meta for traceability but omits the launch flag, and bootstrap reports the invalid harness/effort pair as a `CREW_DISPATCH` diagnostic when it is visible in the file. `quotaBalanced` is optional local tuning used only by rules with `select: "quota-balanced"`. -`reservePercent` keeps percentage-capacity headroom out of the usable score and defaults to 0, while a provider-level value overrides the global value. -`staleClearMargin` defaults to 20 percentage-capacity points and preserves the policy that cached stale quota must be clearly better than fresh quota to win. -`runRate.minimumObservationSeconds` defaults to 300 and bounds first-sample burst amplification near a reset. -`runRate.historyMaxAgeSeconds` defaults to 86400 and prevents old local samples from influencing recent burn. +`reservePercent` is a number from 0 through 100 that keeps percentage-capacity headroom out of the usable score and defaults to 0, while a provider-level value overrides the global value. +`staleClearMargin` is a non-negative number that defaults to 20 percentage-capacity points and preserves the policy that cached stale quota must be clearly better than fresh quota to win. +`runRate.minimumObservationSeconds` is a positive integer that defaults to 300 and bounds first-sample burst amplification near a reset. +`runRate.historyMaxAgeSeconds` is a positive integer that defaults to 86400 and prevents old local samples from influencing recent burn. `runRate.recentWeight` ranges from 0 to 1, defaults to 1, and controls how strongly an observed recent usage delta can raise the cycle-average burn estimate. -Each provider `windows` entry grants explicit manual reset capacity only to the matching canonical `scope` and `durationSeconds`. +Provider tuning keys are limited to `claude`, `codex`, `copilot`, `cursor`, and `grok`. +Each provider `windows` entry requires a `session` or `weekly` scope, a positive-integer `durationSeconds`, and a non-negative-integer `extraResets`, and duplicate scope-duration pairs are invalid within one provider. +The canonical 18000-second duration requires `session`, and the canonical 604800-second duration requires `weekly`. +Each valid entry grants explicit manual reset capacity only to its matching scope and duration. The selector derives session versus weekly scope from actual 18000-second or 604800-second duration before considering provider labels, ids, or misleading kinds. Weekly reset credits therefore never apply to an independent five-hour session window. `extraResets` is a non-negative integer and adds one full normalized window of capacity per configured reset. @@ -267,7 +270,7 @@ This declaration is local and gitignored, so the three-reset reserve is not a sh See [`docs/examples/crew-dispatch.json`](examples/crew-dispatch.json) for a starting point to copy into local `config/crew-dispatch.json`. When the file exists, bootstrap validates it with `jq`. Valid files stay silent by default; with `FM_BOOTSTRAP_VERBOSE_FACTS=1`, bootstrap emits `BOOTSTRAP_INFO: crew dispatch active config/crew-dispatch.json` plus one `BOOTSTRAP_INFO:` fact per rule and default profile. -Malformed JSON, an unverified harness, a malformed array profile, an unknown `select`, or an effort value unsupported by that harness is reported as `CREW_DISPATCH: invalid config/crew-dispatch.json - ...`; missing `jq` is reported through the normal `MISSING: jq` install-consent flow. +Malformed JSON, an invalid `quotaBalanced` schema or value, an unverified harness, a malformed array profile, an unknown `select`, or an effort value unsupported by that harness is reported as `CREW_DISPATCH: invalid config/crew-dispatch.json - ...`; missing `jq` is reported through the normal `MISSING: jq` install-consent flow. If no dispatch rule fits, firstmate uses the dispatch profile `default` when present, then falls back to `config/crew-harness`. Because the spawn backstop is gated by file presence, any fallback path after a missing match, validation error, or missing `jq` still passes a resolved harness explicitly until the file is fixed or removed. Secondmate homes inherit this file from the primary, so a secondmate's own crewmates apply the same dispatch profile behavior. From 0d6047091e19ba454911f57f0dddb376962fb38b Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 14:18:54 +1000 Subject: [PATCH 10/17] no-mistakes(review): Captain, preserve concurrent quota history updates --- bin/fm-dispatch-select.sh | 20 +++++++++-- tests/fm-dispatch-select.test.sh | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/bin/fm-dispatch-select.sh b/bin/fm-dispatch-select.sh index 866793909..0707fbe9c 100755 --- a/bin/fm-dispatch-select.sh +++ b/bin/fm-dispatch-select.sh @@ -458,15 +458,29 @@ fi if [ -n "$STATE_FILE" ] && [ "$(printf '%s\n' "$selection" | jq '.samples | length')" -gt 0 ]; then state_parent=$(dirname "$STATE_FILE") if mkdir -p "$state_parent" 2>/dev/null; then + history_lock="$STATE_FILE.lock" + # shellcheck source=bin/fm-wake-lib.sh + FM_STATE_OVERRIDE="$state_parent" . "$ROOT/bin/fm-wake-lib.sh" + fm_lock_acquire_wait "$history_lock" + trap 'fm_lock_release "$history_lock"' EXIT + current_history='{"schemaVersion":1,"samples":[]}' + if [ -f "$STATE_FILE" ]; then + if locked_history=$(cat "$STATE_FILE" 2>/dev/null) \ + && printf '%s\n' "$locked_history" | jq -e ' + type == "object" and .schemaVersion == 1 and (.samples | type) == "array" + ' >/dev/null 2>&1; then + current_history=$locked_history + fi + fi state_tmp=$(umask 077; mktemp "$state_parent/.dispatch-quota-samples.XXXXXX" 2>/dev/null) || state_tmp= if [ -n "$state_tmp" ]; then - if jq -nec --argjson old "$history_json" --argjson current "$selection" ' + if jq -nec --argjson old "$current_history" --argjson current "$selection" ' def key: [.provider, .id, .kind, .durationSeconds]; {schemaVersion: 1, samples: ([($old.samples[]? // empty), $current.samples[]] | sort_by(key) | group_by(key) - | map(last))} + | map(max_by([.sampledAtEpoch, .resetsAtEpoch, .percentUsed])))} ' > "$state_tmp" && chmod 600 "$state_tmp" && mv "$state_tmp" "$STATE_FILE"; then : else @@ -476,6 +490,8 @@ if [ -n "$STATE_FILE" ] && [ "$(printf '%s\n' "$selection" | jq '.samples | leng else log "could not prepare quota sample history; selection remains valid" fi + fm_lock_release "$history_lock" + trap - EXIT else log "could not create quota sample state directory; selection remains valid" fi diff --git a/tests/fm-dispatch-select.test.sh b/tests/fm-dispatch-select.test.sh index b3b7fa173..6d3daa497 100755 --- a/tests/fm-dispatch-select.test.sh +++ b/tests/fm-dispatch-select.test.sh @@ -232,6 +232,64 @@ JSON pass "window rollover discards incompatible history and records the new cycle" } +test_concurrent_history_updates_preserve_both_samples() { + local state sync fakebin real_cat quota_a quota_b out_a out_b err_a err_b pid_a pid_b + state="$TMP_ROOT/concurrent-state.json" + sync="$TMP_ROOT/concurrent-sync" + fakebin=$(fm_fakebin "$TMP_ROOT/concurrent") + real_cat=$(command -v cat) + quota_a="$TMP_ROOT/concurrent-a.json" + quota_b="$TMP_ROOT/concurrent-b.json" + out_a="$TMP_ROOT/concurrent-a.out" + out_b="$TMP_ROOT/concurrent-b.out" + err_a="$TMP_ROOT/concurrent-a.err" + err_b="$TMP_ROOT/concurrent-b.err" + mkdir -p "$sync" + printf '%s\n' '{"schemaVersion":1,"samples":[]}' > "$state" + cat > "$fakebin/cat" <<'SH' +#!/usr/bin/env bash +if [ "$#" -eq 1 ] && [ "$1" = "$SYNC_STATE" ] && [ ! -e "$SYNC_DIR/$PPID.seen" ]; then + attempts=0 + : > "$SYNC_DIR/$PPID.seen" + "$REAL_CAT" "$1" + : > "$SYNC_DIR/$PPID.ready" + while [ "$attempts" -lt 500 ]; do + ready=0 + for marker in "$SYNC_DIR"/*.ready; do + [ -e "$marker" ] && ready=$((ready + 1)) + done + [ "$ready" -ge 2 ] && exit 0 + attempts=$((attempts + 1)) + sleep 0.01 + done + exit 9 +fi +exec "$REAL_CAT" "$@" +SH + chmod +x "$fakebin/cat" + cat > "$quota_a" <<'JSON' +{"schemaVersion":2,"providers":[{"provider":"claude","state":{"status":"fresh"},"windows":[{"id":"concurrent-a","kind":"session","windowSeconds":18000,"percentUsed":10,"percentRemaining":90,"resetsAt":"2026-07-19T01:00:00Z"}]}]} +JSON + cat > "$quota_b" <<'JSON' +{"schemaVersion":2,"providers":[{"provider":"claude","state":{"status":"fresh"},"windows":[{"id":"concurrent-b","kind":"session","windowSeconds":18000,"percentUsed":20,"percentRemaining":80,"resetsAt":"2026-07-19T01:00:00Z"}]}]} +JSON + + PATH="$fakebin:$BASE_PATH" SYNC_STATE="$state" SYNC_DIR="$sync" REAL_CAT="$real_cat" \ + run_fixture "$quota_a" "$err_a" --state-file "$state" > "$out_a" & + pid_a=$! + PATH="$fakebin:$BASE_PATH" SYNC_STATE="$state" SYNC_DIR="$sync" REAL_CAT="$real_cat" \ + run_fixture "$quota_b" "$err_b" --state-file "$state" > "$out_b" & + pid_b=$! + wait "$pid_a" || fail "first concurrent selector failed: $(cat "$err_a")" + wait "$pid_b" || fail "second concurrent selector failed: $(cat "$err_b")" + + [ "$(jq '.samples | length' "$state")" -eq 2 ] \ + || fail "concurrent selectors should preserve both samples: $(cat "$state")" + [ "$(jq -r '[.samples[].id] | sort | join(",")' "$state")" = "concurrent-a,concurrent-b" ] \ + || fail "concurrent selectors wrote unexpected history: $(cat "$state")" + pass "concurrent history updates preserve every selector's sample" +} + test_missing_duration_and_clock_skew_are_contained() { local quota out err diagnostics quota="$TMP_ROOT/missing-duration.json" @@ -345,6 +403,7 @@ test_codex_mislabeled_window_uses_actual_duration test_configured_codex_extra_resets_add_capacity test_stale_candidate_keeps_existing_clear_margin_policy test_rollover_discards_incompatible_recent_sample +test_concurrent_history_updates_preserve_both_samples test_missing_duration_and_clock_skew_are_contained test_quota_failures_and_old_schema_fall_back_to_first test_live_quota_uses_account_free_json_and_private_diagnostics From 71d3eb99cf0c40fb57c95f125729c48c0dd091ac Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 14:23:16 +1000 Subject: [PATCH 11/17] no-mistakes(review): Captain, bound quota history lock acquisition --- bin/fm-dispatch-select.sh | 65 +++++++++++++++++++------------- tests/fm-dispatch-select.test.sh | 38 +++++++++++++++++++ 2 files changed, 77 insertions(+), 26 deletions(-) diff --git a/bin/fm-dispatch-select.sh b/bin/fm-dispatch-select.sh index 0707fbe9c..30f829422 100755 --- a/bin/fm-dispatch-select.sh +++ b/bin/fm-dispatch-select.sh @@ -461,37 +461,50 @@ if [ -n "$STATE_FILE" ] && [ "$(printf '%s\n' "$selection" | jq '.samples | leng history_lock="$STATE_FILE.lock" # shellcheck source=bin/fm-wake-lib.sh FM_STATE_OVERRIDE="$state_parent" . "$ROOT/bin/fm-wake-lib.sh" - fm_lock_acquire_wait "$history_lock" - trap 'fm_lock_release "$history_lock"' EXIT - current_history='{"schemaVersion":1,"samples":[]}' - if [ -f "$STATE_FILE" ]; then - if locked_history=$(cat "$STATE_FILE" 2>/dev/null) \ - && printf '%s\n' "$locked_history" | jq -e ' - type == "object" and .schemaVersion == 1 and (.samples | type) == "array" - ' >/dev/null 2>&1; then - current_history=$locked_history + history_lock_attempt=0 + history_lock_acquired=0 + while [ "$history_lock_attempt" -lt 10 ]; do + if fm_lock_try_acquire "$history_lock"; then + history_lock_acquired=1 + break fi - fi - state_tmp=$(umask 077; mktemp "$state_parent/.dispatch-quota-samples.XXXXXX" 2>/dev/null) || state_tmp= - if [ -n "$state_tmp" ]; then - if jq -nec --argjson old "$current_history" --argjson current "$selection" ' - def key: [.provider, .id, .kind, .durationSeconds]; - {schemaVersion: 1, - samples: ([($old.samples[]? // empty), $current.samples[]] - | sort_by(key) - | group_by(key) - | map(max_by([.sampledAtEpoch, .resetsAtEpoch, .percentUsed])))} - ' > "$state_tmp" && chmod 600 "$state_tmp" && mv "$state_tmp" "$STATE_FILE"; then - : + history_lock_attempt=$((history_lock_attempt + 1)) + [ "$history_lock_attempt" -ge 10 ] || sleep 0.1 + done + if [ "$history_lock_acquired" -eq 1 ]; then + trap 'fm_lock_release "$history_lock"' EXIT + current_history='{"schemaVersion":1,"samples":[]}' + if [ -f "$STATE_FILE" ]; then + if locked_history=$(cat "$STATE_FILE" 2>/dev/null) \ + && printf '%s\n' "$locked_history" | jq -e ' + type == "object" and .schemaVersion == 1 and (.samples | type) == "array" + ' >/dev/null 2>&1; then + current_history=$locked_history + fi + fi + state_tmp=$(umask 077; mktemp "$state_parent/.dispatch-quota-samples.XXXXXX" 2>/dev/null) || state_tmp= + if [ -n "$state_tmp" ]; then + if jq -nec --argjson old "$current_history" --argjson current "$selection" ' + def key: [.provider, .id, .kind, .durationSeconds]; + {schemaVersion: 1, + samples: ([($old.samples[]? // empty), $current.samples[]] + | sort_by(key) + | group_by(key) + | map(max_by([.sampledAtEpoch, .resetsAtEpoch, .percentUsed])))} + ' > "$state_tmp" && chmod 600 "$state_tmp" && mv "$state_tmp" "$STATE_FILE"; then + : + else + rm -f "$state_tmp" + log "could not update quota sample history; selection remains valid" + fi else - rm -f "$state_tmp" - log "could not update quota sample history; selection remains valid" + log "could not prepare quota sample history; selection remains valid" fi + fm_lock_release "$history_lock" + trap - EXIT else - log "could not prepare quota sample history; selection remains valid" + log "could not acquire quota sample history lock; selection remains valid" fi - fm_lock_release "$history_lock" - trap - EXIT else log "could not create quota sample state directory; selection remains valid" fi diff --git a/tests/fm-dispatch-select.test.sh b/tests/fm-dispatch-select.test.sh index 6d3daa497..c751619c7 100755 --- a/tests/fm-dispatch-select.test.sh +++ b/tests/fm-dispatch-select.test.sh @@ -290,6 +290,43 @@ JSON pass "concurrent history updates preserve every selector's sample" } +test_held_history_lock_does_not_block_dispatch() { + local quota state lock out err pid attempts + quota="$TMP_ROOT/held-lock.json" + state="$TMP_ROOT/held-lock-state.json" + lock="$state.lock" + out="$TMP_ROOT/held-lock.out" + err="$TMP_ROOT/held-lock.err" + cat > "$quota" <<'JSON' +{"schemaVersion":2,"providers":[{"provider":"claude","state":{"status":"fresh"},"windows":[{"id":"five_hour","kind":"session","percentUsed":10,"percentRemaining":90,"resetsAt":"2026-07-19T01:00:00Z"}]}]} +JSON + printf '%s\n' '{"schemaVersion":1,"samples":[]}' > "$state" + mkdir "$lock" + printf '%s\n' "$$" > "$lock/pid" + + run_fixture "$quota" "$err" --state-file "$state" > "$out" & + pid=$! + attempts=0 + while [ ! -s "$out" ] && [ "$attempts" -lt 30 ]; do + sleep 0.1 + attempts=$((attempts + 1)) + done + if [ ! -s "$out" ]; then + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fail "held history lock blocked dispatch output" + fi + wait "$pid" || fail "selector failed while history lock was held: $(cat "$err")" + + [ "$(cat "$out")" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ + || fail "held history lock should not change profile selection: $(cat "$out")" + assert_contains "$(cat "$err")" "could not acquire quota sample history lock" \ + "held history lock should explain the skipped history update" + [ "$(jq '.samples | length' "$state")" -eq 0 ] \ + || fail "held history lock should leave history unchanged: $(cat "$state")" + pass "held history lock skips persistence without blocking dispatch" +} + test_missing_duration_and_clock_skew_are_contained() { local quota out err diagnostics quota="$TMP_ROOT/missing-duration.json" @@ -404,6 +441,7 @@ test_configured_codex_extra_resets_add_capacity test_stale_candidate_keeps_existing_clear_margin_policy test_rollover_discards_incompatible_recent_sample test_concurrent_history_updates_preserve_both_samples +test_held_history_lock_does_not_block_dispatch test_missing_duration_and_clock_skew_are_contained test_quota_failures_and_old_schema_fall_back_to_first test_live_quota_uses_account_free_json_and_private_diagnostics From f5d6a2d62dd91325b6d1ffb1f65d16f15635453b Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 14:58:23 +1000 Subject: [PATCH 12/17] no-mistakes(document): Remove stale quota rollout wording --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 1b78f147d..99ccc2d02 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -250,7 +250,7 @@ It represents currently unused manual resets and must be reduced locally when a The selector keeps account-free burn samples in `state/.dispatch-quota-samples.json`; the file is volatile private state and is not a source of configured reset credits. `quota-balanced` selection is deterministic and implemented by `bin/fm-dispatch-select.sh`, whose header owns the exact score, rollover, clock-skew, diagnostics, vendor-availability, and degrade-to-first-element mechanics; quota trouble never blocks dispatch. -To declare three Codex manual weekly resets after this version lands, merge this local object into `config/crew-dispatch.json` without adding a session entry: +To declare three Codex manual weekly resets, merge this local object into `config/crew-dispatch.json` without adding a session entry: ```json { From a5a0cd9873b4ff59c8f461bb30002c13f50bdeaf Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 15:13:53 +1000 Subject: [PATCH 13/17] no-mistakes(review): Exclude code-review windows from general quota scoring --- bin/fm-dispatch-select.sh | 21 ++++++++++++--------- tests/fm-dispatch-select.test.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/bin/fm-dispatch-select.sh b/bin/fm-dispatch-select.sh index 30f829422..4a55d89d9 100755 --- a/bin/fm-dispatch-select.sh +++ b/bin/fm-dispatch-select.sh @@ -13,13 +13,13 @@ # scoring contract: # - It consumes quota-axi schema version 2 through `quota-axi --json` # (or the --quota-json fixture). -# - General windows have provider kind `session` or `weekly`; model-scoped -# windows are excluded regardless of label or id. An explicit positive -# windowSeconds is authoritative. Actual 5-hour and 7-day durations define -# session and weekly capacity scope even when provider kind is misleading. -# Claude's known five_hour and seven_day windows fall back to those canonical -# durations because Claude omits the field. No Codex duration is inferred -# from its misleading five_hour id. +# - General windows have provider kind `session` or `weekly`; model-scoped and +# code-review-scoped windows are excluded regardless of kind or duration. An +# explicit positive windowSeconds is authoritative. Actual 5-hour and 7-day +# durations define session and weekly capacity scope even when provider kind +# is misleading. Claude's known five_hour and seven_day windows fall back to +# those canonical durations because Claude omits the field. No Codex duration +# is inferred from its misleading five_hour id. # - A window score is percentage-capacity headroom through its reset: # remaining + (100 * configured extra resets) - reserve # - (estimated burn per second * seconds until reset) @@ -300,6 +300,10 @@ selection=$(printf '%s\n' "$quota_json" | jq -ec \ elif $duration == 604800 then "weekly" else $window.kind end; + def general_window($window): + (($window.kind == "session" or $window.kind == "weekly") + and (((($window.id? // "") | startswith("model:")) or + (($window.id? // "") | startswith("code_review_"))) | not)); def settings: ($config.quotaBalanced? // {}); def run_settings: (settings.runRate? // {}); def reserve_for($provider): @@ -377,8 +381,7 @@ selection=$(printf '%s\n' "$quota_json" | jq -ec \ | if $provider == null then empty else ([$provider.windows[]? - | select((.kind == "session" or .kind == "weekly") - and (((.id? // "") | startswith("model:")) | not)) + | select(general_window(.)) | window_metric($provider_name; .)]) as $windows | if ($windows | length) == 0 then empty else ($windows | min_by(.score)) as $bottleneck diff --git a/tests/fm-dispatch-select.test.sh b/tests/fm-dispatch-select.test.sh index c751619c7..981f92758 100755 --- a/tests/fm-dispatch-select.test.sh +++ b/tests/fm-dispatch-select.test.sh @@ -127,6 +127,35 @@ JSON pass "Codex's misleading five_hour id cannot override its actual seven-day duration" } +test_codex_code_review_windows_do_not_constrain_general_quota() { + local quota out err diagnostics + quota="$TMP_ROOT/codex-code-review.json" + err="$TMP_ROOT/codex-code-review.err" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider": "claude", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "percentUsed": 50, "percentRemaining": 50, "resetsAt": "2026-07-19T01:00:00Z"} + ]}, + {"provider": "codex", "state": {"status": "fresh"}, "windows": [ + {"id": "five_hour", "kind": "session", "windowSeconds": 604800, "percentUsed": 10, "percentRemaining": 90, "resetsAt": "2026-07-24T00:00:00Z"}, + {"id": "code_review_five_hour", "kind": "session", "windowSeconds": 18000, "percentUsed": 99, "percentRemaining": 1, "resetsAt": "2026-07-19T01:00:00Z"}, + {"id": "code_review_weekly", "kind": "weekly", "windowSeconds": 604800, "percentUsed": 99, "percentRemaining": 1, "resetsAt": "2026-07-24T00:00:00Z"} + ]} + ] +} +JSON + + out=$(run_fixture "$quota" "$err") + diagnostics=$(cat "$err") + [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ + || fail "code-review quota should not constrain Codex general quota, got: $out" + assert_not_contains "$diagnostics" "code_review" \ + "code-review windows must not enter general quota diagnostics" + pass "Codex code-review windows do not constrain general quota selection" +} + test_configured_codex_extra_resets_add_capacity() { local quota config out err diagnostics quota="$TMP_ROOT/extra-resets.json" @@ -437,6 +466,7 @@ SH test_low_remaining_slow_burn_beats_high_remaining_burst test_claude_session_and_weekly_bottlenecks test_codex_mislabeled_window_uses_actual_duration +test_codex_code_review_windows_do_not_constrain_general_quota test_configured_codex_extra_resets_add_capacity test_stale_candidate_keeps_existing_clear_margin_policy test_rollover_discards_incompatible_recent_sample From 4622c4b68a10a846e82d81d5744763e42a6cd775 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 16:15:53 +1000 Subject: [PATCH 14/17] no-mistakes(test): Captain: stabilize secondmate seed rollback cleanup --- bin/fm-home-seed.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/fm-home-seed.sh b/bin/fm-home-seed.sh index d506b95e4..21d38cd95 100755 --- a/bin/fm-home-seed.sh +++ b/bin/fm-home-seed.sh @@ -651,9 +651,16 @@ seed_return_treehouse_home() { } seed_remove_created_home() { - local home=$1 abs_home + local home=$1 abs_home delay abs_home=$(seed_rollback_target "$home" "created home") || return 0 + for delay in 0.05 0.1 0.2; do + rm -rf -- "$abs_home" 2>/dev/null || true + sleep "$delay" + done rm -rf -- "$abs_home" 2>/dev/null || true + if [ -e "$abs_home" ] || [ -L "$abs_home" ]; then + echo "warning: failed to remove created home $abs_home during seed rollback" >&2 + fi } seed_project_rollback_target() { From 8557d4cd532732f0f829eb70799d7b300bdec15a Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 16:55:46 +1000 Subject: [PATCH 15/17] no-mistakes(test): Stabilize watcher stale-absorb test synchronization --- tests/fm-watch-triage.test.sh | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 4cbc4de4f..07ae7b0a4 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -427,9 +427,10 @@ test_stale_terminal_status_overridden_by_active_run() { FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! - if ! wait_live "$pid" 30; then - reap "$pid"; fail "watcher exited for a stale terminal-looking status the run-step overrides (should absorb): $(cat "$out")" - fi + wait_numeric_file "$state/.stale-since-$key" 50 \ + || { reap "$pid"; fail "watcher did not absorb a stale terminal-looking status the run-step overrides: $(cat "$out")"; } + kill -0 "$pid" 2>/dev/null \ + || { wait "$pid" 2>/dev/null || true; fail "watcher exited after absorbing a stale terminal-looking status: $(cat "$out")"; } [ ! -s "$out" ] || fail "the overridden stale terminal status printed a wake reason during absorb" [ ! -s "$state/.wake-queue" ] || fail "the overridden stale terminal status enqueued a wake during absorb" [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on absorb" @@ -480,9 +481,10 @@ test_nonterminal_stale_provably_working_absorbed_then_escalated() { FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! - if ! wait_live "$pid" 30; then - reap "$pid"; fail "watcher exited for a fresh provably-working non-terminal stale (should absorb): $(cat "$out")" - fi + wait_numeric_file "$state/.stale-since-$key" 50 \ + || { reap "$pid"; fail "watcher did not absorb a fresh provably-working non-terminal stale: $(cat "$out")"; } + kill -0 "$pid" 2>/dev/null \ + || { wait "$pid" 2>/dev/null || true; fail "watcher exited after absorbing a fresh provably-working non-terminal stale: $(cat "$out")"; } [ ! -s "$out" ] || fail "fresh provably-working stale printed a wake reason during absorb" [ ! -s "$state/.wake-queue" ] || fail "fresh provably-working stale enqueued a wake during absorb" [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on absorb" @@ -554,7 +556,7 @@ test_nonterminal_stale_not_working_surfaced() { # the pause's own status-file age, so a churny idle pane cannot reset the cadence) # for a recheck, so a forgotten pause cannot rot invisibly. test_nonterminal_stale_paused_absorbed_then_resurfaced() { - local dir state fakebin out drain_out capture_file window key pane_hash sig pid back statusf + local dir state fakebin out drain_out capture_file window key pane_hash sig pid back statusf i dir=$(make_case nonterminal-stale-paused); state="$dir/state"; fakebin="$dir/fakebin" out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" window="test:fm-held" @@ -578,9 +580,15 @@ test_nonterminal_stale_paused_absorbed_then_resurfaced() { FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! - if ! wait_live "$pid" 30; then - reap "$pid"; fail "watcher exited for a fresh declared pause (should absorb): $(cat "$out")" - fi + i=0 + while [ ! -e "$state/.paused-$key" ] && [ "$i" -lt 50 ]; do + kill -0 "$pid" 2>/dev/null \ + || { wait "$pid" 2>/dev/null || true; fail "watcher exited before absorbing a fresh declared pause: $(cat "$out")"; } + sleep 0.1 + i=$((i + 1)) + done + [ -e "$state/.paused-$key" ] \ + || { reap "$pid"; fail "watcher did not absorb a fresh declared pause: $(cat "$out")"; } [ ! -s "$out" ] || fail "fresh paused stale printed a wake reason during absorb" [ ! -s "$state/.wake-queue" ] || fail "fresh paused stale enqueued a wake during absorb" [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on paused absorb" From ed0d00c4c0a117d3b161e1e4e6a26d5c4c9996c1 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 17:32:49 +1000 Subject: [PATCH 16/17] no-mistakes(test): Captain: stabilize bootstrap timeout fixture synchronization --- tests/fm-bootstrap.test.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index 130fb5929..c8dd317e5 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -126,9 +126,9 @@ make_fake_fleet_sync_root() { mkdir -p "$fake_root/bin" cat > "$fake_root/bin/fm-fleet-sync.sh" <<'SH' #!/usr/bin/env bash -[ -z "${FM_FAKE_FLEET_SYNC_STARTED_MARKER:-}" ] || : > "$FM_FAKE_FLEET_SYNC_STARTED_MARKER" printf '%s\n' 'alpha: synced' printf '%s\n' 'beta: skipped: no origin remote' +[ -z "${FM_FAKE_FLEET_SYNC_STARTED_MARKER:-}" ] || : > "$FM_FAKE_FLEET_SYNC_STARTED_MARKER" exec perl -e 'sleep 300' SH chmod +x "$fake_root/bin/fm-fleet-sync.sh" @@ -168,7 +168,17 @@ run_bootstrap_timeout_case() { ( # shellcheck disable=SC2317,SC2329 # Exported and invoked by the bootstrap subprocess. sleep() { - local inc=${1:-1} + local inc=${1:-1} tries + if [ "${FM_FAKE_GIT_WAIT_FOR_FLEET_START:-}" = 1 ] \ + && [ "${FM_FAKE_FLEET_START_WAITED:-0}" -eq 0 ] \ + && [ -n "${FM_FAKE_FLEET_SYNC_STARTED_MARKER:-}" ]; then + FM_FAKE_FLEET_START_WAITED=1 + tries=0 + while [ "$tries" -lt 500 ] && [ ! -e "$FM_FAKE_FLEET_SYNC_STARTED_MARKER" ]; do + command sleep 0.01 + tries=$((tries + 1)) + done + fi SECONDS=$((SECONDS + inc)) # Advance fake time quickly, but yield on every tick so the background # fleet-sync process can deterministically write its partial output before From 5aa87f206053b1a007dc93744bdc27afa6d4566d Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 19 Jul 2026 19:39:14 +1000 Subject: [PATCH 17/17] ci: preserve no-mistakes signature rechecks --- .github/workflows/no-mistakes-required.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/no-mistakes-required.yml b/.github/workflows/no-mistakes-required.yml index ab1224d39..86f3e8ace 100644 --- a/.github/workflows/no-mistakes-required.yml +++ b/.github/workflows/no-mistakes-required.yml @@ -11,7 +11,7 @@ permissions: concurrency: group: no-mistakes-required-${{ github.event.pull_request.number }} - cancel-in-progress: true + cancel-in-progress: false jobs: check: