Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 59 additions & 21 deletions setup/js/ai_credits_context.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ const AI_CREDITS_RATE_LIMIT_TEXT_FIELDS = new Set(["error", "message", "reason",
const AI_CREDITS_RATE_LIMIT_PATTERNS = [/ai[\s_-]*credits?.*(?:rate[\s-]*limit|limit exceeded|budget exceeded|exceeded)/i, /(?:rate[\s-]*limit|too many requests).*(?:ai[\s_-]*credits?)/i, /\bai_credits_limit_exceeded\b/i];
const MAX_AI_CREDITS_EXCEEDED_FIELDS = new Set(["max_ai_credits_exceeded", "maxAiCreditsExceeded"]);
const BUDGET_EXCEEDED_EVENT = "budget_exceeded";
// The literal error type emitted by the AWF API proxy (HTTP 400) when maxAiCredits is active
// and the requested model is not in the built-in pricing table.
const UNKNOWN_MODEL_AI_CREDITS_TYPE = "unknown_model_ai_credits";
const MAX_AI_CREDITS_EXCEEDED_STDIO_RE = /maximum ai credits exceeded(?:\s*\((\d+(?:\.\d+)?)\s*\/\s*(\d+(?:\.\d+)?)\))?/i;
const DEFAULT_AGENT_STDIO_LOG = "/tmp/gh-aw/agent-stdio.log";
const AGENT_STDIO_LOG_MAX_TAIL = 64 * 1024; // 64 KB — sufficient for any realistic error block
Expand Down Expand Up @@ -251,6 +254,37 @@ function parseMaxAICreditsExceededFromAuditLog(auditJsonlPathOverride) {
);
}

/**
* Detects an `unknown_model_ai_credits` error from the firewall audit log.
* This HTTP 400 error is emitted by the AWF API proxy when `maxAiCredits` is active and
* the requested model is not in the built-in pricing table and no `defaultAiCreditsPricing`
* fallback is configured.
*
* @param {string} [auditJsonlPathOverride]
* @returns {boolean}
*/
function parseUnknownModelAICreditsFromAuditLog(auditJsonlPathOverride) {
return iterateAuditEntries(
auditJsonlPathOverride,
false,
content => content.includes(UNKNOWN_MODEL_AI_CREDITS_TYPE),
(acc, entry) => {
if (acc) return true;
if (!entry || typeof entry !== "object") return false;
const stack = [entry];
while (stack.length > 0) {
const node = stack.pop();
if (!node || typeof node !== "object") continue;
for (const [, value] of Object.entries(node)) {
if (value === UNKNOWN_MODEL_AI_CREDITS_TYPE) return true;
if (value && typeof value === "object") stack.push(value);
}
}
return false;
}
);
}

/**
* Single-pass combined read of the audit log, returning all AI credits fields at once.
* Used by resolveAICreditsFailureState to avoid reading the same file twice.
Expand All @@ -277,37 +311,40 @@ function parseAuditLogCombined(auditJsonlPathOverride) {
}

/**
* @param {{ logProvenance?: boolean }} [options]
* @returns {{ aiCredits: string, maxAICredits: string, aiCreditsRateLimitError: boolean, maxAICreditsExceeded: boolean }}
*/
function resolveAICreditsFailureState() {
function resolveAICreditsFailureState({ logProvenance = true } = {}) {
const stdioSignals = parseAICreditsExceededFromAgentStdio();
const { aiCredits: auditAICredits, maxAICredits: auditMaxAICredits, rateLimitError: auditRateLimitError, maxAICreditsExceeded: auditMaxAICreditsExceeded } = parseAuditLogCombined();
const envAICredits = parsePositiveNumberString(process.env.GH_AW_AIC);
const envMaxAICredits = parsePositiveNumberString(process.env.GH_AW_MAX_AI_CREDITS);

// Log provenance so failing issues can be diagnosed when credit data is missing.
if (auditAICredits) {
console.log(`[ai-credits] aiCredits source=audit_log value=${auditAICredits}`);
} else if (stdioSignals.aiCredits) {
console.log(`[ai-credits] aiCredits source=agent_stdio value=${stdioSignals.aiCredits}`);
} else if (envAICredits) {
console.log(`[ai-credits] aiCredits source=env(GH_AW_AIC) value=${envAICredits}`);
} else {
console.log(`[ai-credits] aiCredits source=none GH_AW_AIC=${process.env.GH_AW_AIC || "(unset)"}`);
}
if (logProvenance) {
if (auditAICredits) {
console.log(`[ai-credits] aiCredits source=audit_log value=${auditAICredits}`);
} else if (stdioSignals.aiCredits) {
console.log(`[ai-credits] aiCredits source=agent_stdio value=${stdioSignals.aiCredits}`);
} else if (envAICredits) {
console.log(`[ai-credits] aiCredits source=env(GH_AW_AIC) value=${envAICredits}`);
} else {
console.log(`[ai-credits] aiCredits source=none GH_AW_AIC=${process.env.GH_AW_AIC || "(unset)"}`);
}

if (auditMaxAICredits) {
console.log(`[ai-credits] maxAICredits source=audit_log value=${auditMaxAICredits}`);
} else if (stdioSignals.maxAICredits) {
console.log(`[ai-credits] maxAICredits source=agent_stdio value=${stdioSignals.maxAICredits}`);
} else if (envMaxAICredits) {
console.log(`[ai-credits] maxAICredits source=env(GH_AW_MAX_AI_CREDITS) value=${envMaxAICredits}`);
} else {
console.log(`[ai-credits] maxAICredits source=none GH_AW_MAX_AI_CREDITS=${process.env.GH_AW_MAX_AI_CREDITS || "(unset)"}`);
}
if (auditMaxAICredits) {
console.log(`[ai-credits] maxAICredits source=audit_log value=${auditMaxAICredits}`);
} else if (stdioSignals.maxAICredits) {
console.log(`[ai-credits] maxAICredits source=agent_stdio value=${stdioSignals.maxAICredits}`);
} else if (envMaxAICredits) {
console.log(`[ai-credits] maxAICredits source=env(GH_AW_MAX_AI_CREDITS) value=${envMaxAICredits}`);
} else {
console.log(`[ai-credits] maxAICredits source=none GH_AW_MAX_AI_CREDITS=${process.env.GH_AW_MAX_AI_CREDITS || "(unset)"}`);
}

const rawRateLimitSignalSource = auditRateLimitError ? "audit_log" : stdioSignals.rateLimitError ? "agent_stdio" : process.env.GH_AW_AI_CREDITS_RATE_LIMIT_ERROR === "true" ? "env(GH_AW_AI_CREDITS_RATE_LIMIT_ERROR)" : "none";
console.log(`[ai-credits] rateLimitSignal source=${rawRateLimitSignalSource}`);
const rawRateLimitSignalSource = auditRateLimitError ? "audit_log" : stdioSignals.rateLimitError ? "agent_stdio" : process.env.GH_AW_AI_CREDITS_RATE_LIMIT_ERROR === "true" ? "env(GH_AW_AI_CREDITS_RATE_LIMIT_ERROR)" : "none";
console.log(`[ai-credits] rateLimitSignal source=${rawRateLimitSignalSource}`);
}

const aiCredits = auditAICredits || stdioSignals.aiCredits || envAICredits || "";
const maxAICredits = auditMaxAICredits || stdioSignals.maxAICredits || envMaxAICredits || "";
Expand Down Expand Up @@ -366,5 +403,6 @@ module.exports = {
parseMaxAICreditsFromAuditLog,
parseAICreditsErrorInfoFromAuditLog,
parseMaxAICreditsExceededFromAuditLog,
parseUnknownModelAICreditsFromAuditLog,
resolveAICreditsFailureState,
};
88 changes: 82 additions & 6 deletions setup/js/handle_agent_failure.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ const COPILOT_SESSION_STATE_DIR = path.join(os.tmpdir(), "gh-aw", "sandbox", "ag
// - Copilot/CAPI "CAPIError: 429" and utility-model quota text
// - retry wrapper text that includes the canonical "Failed to get response..." phrase
const ENGINE_RATE_LIMIT_429_RE =
/(?:\b429\b[\s\S]{0,120}(?:too many requests|rate[\s-]*limit)|rate_limit_(?:error|exceeded)|capierror:\s*429|failed to get response from the ai model[\s\S]{0,120}\b429\b|exceeded your rate limit for utility models)/i;
/(?:\b429\b[\s\S]{0,120}(?:too many requests|rate[\s-]*limit)|\brate_limit_(?:error|exceeded)\b|capierror:\s*429|failed to get response from the ai model[\s\S]{0,120}\b429\b|exceeded your rate limit for utility models)/i;

/**
* Parse action failure issue expiration from environment.
Expand Down Expand Up @@ -195,6 +195,7 @@ function buildFailureMatchCategories(options) {
if (options.mcpPolicyError) categories.push("mcp_policy_error");
if (options.modelNotSupportedError) categories.push("model_not_supported_error");
if (options.aiCreditsRateLimitError) categories.push("ai_credits_rate_limit_error");
if (options.unknownModelAICredits) categories.push("unknown_model_ai_credits");
if (options.maxAICreditsExceeded) categories.push("max_ai_credits_exceeded");
if (options.hasAppTokenMintingFailed) categories.push("app_token_minting_failed");
if (options.hasLockdownCheckFailed) categories.push("lockdown_check_failed");
Expand All @@ -208,6 +209,44 @@ function buildFailureMatchCategories(options) {
return categories.sort();
}

/**
* Build a precise failure issue title for known failure classes.
* Falls back to the generic failure title when no specific class matches.
* @param {Object} options
* @param {string} options.workflowName
* @param {boolean} options.isTimedOut
* @param {boolean} options.hasMissingSafeOutputs
* @param {boolean} options.hasReportIncomplete
* @param {boolean} options.hasMissingTool
* @param {boolean} options.hasMissingData
* @param {boolean} options.hasCacheMissMisconfiguration
* @param {boolean} options.hasToolDenialsExceeded
* @param {boolean} options.hasAppTokenMintingFailed
* @param {boolean} options.hasLockdownCheckFailed
* @param {boolean} options.hasStaleLockFileFailed
* @param {boolean} options.hasDailyAICExceeded
* @param {boolean} options.aiCreditsRateLimitError
* @param {boolean} options.maxAICreditsExceeded
* @returns {string}
*/
function buildFailureIssueTitle(options) {
const { workflowName } = options;
if (options.hasDailyAICExceeded) return `[aw] ${workflowName} exceeded daily effective workflow budget`;
if (options.maxAICreditsExceeded) return `[aw] ${workflowName} exceeded max AI credits`;
if (options.aiCreditsRateLimitError) return `[aw] ${workflowName} hit AI credits rate limit`;
if (options.hasAppTokenMintingFailed) return `[aw] ${workflowName} failed to mint GitHub App token`;
if (options.hasLockdownCheckFailed) return `[aw] ${workflowName} failed lockdown check`;
if (options.hasStaleLockFileFailed) return `[aw] ${workflowName} has stale lock file`;
if (options.isTimedOut) return `[aw] ${workflowName} timed out`;
if (options.hasToolDenialsExceeded) return `[aw] ${workflowName} exceeded tool denial limit`;
if (options.hasCacheMissMisconfiguration) return `[aw] ${workflowName} has cache-memory miss misconfiguration`;
if (options.hasReportIncomplete) return `[aw] ${workflowName} reported incomplete result`;
if (options.hasMissingSafeOutputs) return `[aw] ${workflowName} produced no safe outputs`;
if (options.hasMissingTool) return `[aw] ${workflowName} is missing required tool`;
if (options.hasMissingData) return `[aw] ${workflowName} is missing required data`;
return `[aw] ${workflowName} failed`;
}

/**
* Generate a precise failure-match marker for failure issue bodies.
* @param {Object} options - Marker options
Expand Down Expand Up @@ -1417,6 +1456,19 @@ function buildModelNotSupportedErrorContext(hasModelNotSupportedError) {
return "\n" + renderPromptTemplate("model_not_supported_error.md");
}

/**
* Builds the unknown_model_ai_credits failure context block for templates.
* @param {boolean} hasUnknownModelAICreditsError
* @returns {string}
*/
function buildUnknownModelAICreditsContext(hasUnknownModelAICreditsError) {
if (!hasUnknownModelAICreditsError) {
return "";
}

return "\n" + renderPromptTemplate("unknown_model_ai_credits.md");
}

/**
* Detect HTTP 429/rate-limit engine failures in text payloads.
* @param {string} content
Expand Down Expand Up @@ -2030,8 +2082,8 @@ const CASCADE_ROLLUP_LABEL = "cascade-rollup";
/** Daily-cap rollup constants */
const DAILY_CAP_ROLLUP_TITLE = "[aw] Daily failure issue cap exceeded";
const DAILY_CAP_ROLLUP_LABEL = "daily-cap-exceeded";
/** Matches the exact title pattern produced by handle_agent_failure for individual failure issues */
const FAILURE_TITLE_PATTERN = /^\[aw\] .+ failed$/;
/** Matches individual failure issue titles produced by handle_agent_failure */
const FAILURE_TITLE_PATTERN = /^\[aw\] \S.*$/;

/**
* Ensure a GitHub label exists in the repository, creating it with a deterministic
Expand Down Expand Up @@ -2070,7 +2122,7 @@ async function ensureLabelExists(owner, repo, labelName) {
}

/**
* Detect whether a failure cascade is active by counting `[aw] * failed` issues
* Detect whether a failure cascade is active by counting `[aw] *` failure issues
* created within the last CASCADE_WINDOW_MINUTES minutes.
*
* @param {string} owner
Expand All @@ -2083,7 +2135,7 @@ async function findRecentFailureIssues(owner, repo) {
const since = windowStart.toISOString().slice(0, 19) + "Z"; // e.g. "2026-05-22T02:00:00Z"

// GitHub search API supports `created:>=YYYY-MM-DDTHH:MM:SSZ`
const searchQuery = `repo:${owner}/${repo} is:issue is:open label:agentic-workflows "[aw]" "failed" in:title created:>=${since}`;
const searchQuery = `repo:${owner}/${repo} is:issue is:open label:agentic-workflows "[aw]" in:title created:>=${since}`;

try {
const result = await github.rest.search.issuesAndPullRequests({
Expand Down Expand Up @@ -2303,6 +2355,7 @@ async function main() {
const mcpPolicyError = process.env.GH_AW_MCP_POLICY_ERROR === "true";
const agenticEngineTimeout = process.env.GH_AW_AGENTIC_ENGINE_TIMEOUT === "true";
const modelNotSupportedError = process.env.GH_AW_MODEL_NOT_SUPPORTED_ERROR === "true";
const unknownModelAICredits = process.env.GH_AW_UNKNOWN_MODEL_AI_CREDITS === "true";
const pushRepoMemoryResult = process.env.GH_AW_PUSH_REPO_MEMORY_RESULT || "";
const reportFailureAsIssue = process.env.GH_AW_FAILURE_REPORT_AS_ISSUE !== "false"; // Default to true
// Feature flags: control whether missing_tool/missing_data signals trigger agent failure handling.
Expand Down Expand Up @@ -2369,6 +2422,7 @@ async function main() {
core.info(`MCP policy error: ${mcpPolicyError}`);
core.info(`Agentic engine timeout: ${agenticEngineTimeout}`);
core.info(`Model not supported error: ${modelNotSupportedError}`);
core.info(`Unknown model AI credits error: ${unknownModelAICredits}`);
core.info(`Push repo-memory result: ${pushRepoMemoryResult}`);
core.info(`App token minting failed (safe_outputs/conclusion/activation): ${safeOutputsAppTokenMintingFailed}/${conclusionAppTokenMintingFailed}/${activationAppTokenMintingFailed}`);
core.info(`Lockdown check failed: ${hasLockdownCheckFailed}`);
Expand Down Expand Up @@ -2622,7 +2676,22 @@ async function main() {

// Sanitize workflow name for title
const sanitizedWorkflowName = sanitizeContent(workflowName, { maxLength: 100 });
const issueTitle = `[aw] ${sanitizedWorkflowName} failed`;
const issueTitle = buildFailureIssueTitle({
workflowName: sanitizedWorkflowName,
isTimedOut,
hasMissingSafeOutputs,
hasReportIncomplete,
hasMissingTool,
hasMissingData,
hasCacheMissMisconfiguration,
hasToolDenialsExceeded,
hasAppTokenMintingFailed,
hasLockdownCheckFailed,
hasStaleLockFileFailed,
hasDailyAICExceeded,
aiCreditsRateLimitError,
maxAICreditsExceeded,
});
const failureCategories = buildFailureMatchCategories({
agentConclusion,
isTimedOut,
Expand All @@ -2643,6 +2712,7 @@ async function main() {
mcpPolicyError,
modelNotSupportedError,
aiCreditsRateLimitError,
unknownModelAICredits,
maxAICreditsExceeded,
hasAppTokenMintingFailed,
hasLockdownCheckFailed,
Expand Down Expand Up @@ -2774,6 +2844,7 @@ async function main() {
// Build model not supported error context
const modelNotSupportedErrorContext = buildModelNotSupportedErrorContext(modelNotSupportedError);
const aiCreditsRateLimitErrorContext = buildAICreditsRateLimitErrorContext(aiCreditsRateLimitError || maxAICreditsExceeded, aiCredits, maxAICredits, runUrl);
const unknownModelAICreditsContext = buildUnknownModelAICreditsContext(unknownModelAICredits);

// Build GitHub App token minting failure context
const appTokenMintingFailedContext = buildAppTokenMintingFailedContext(hasAppTokenMintingFailed);
Expand Down Expand Up @@ -2824,6 +2895,7 @@ async function main() {
mcp_policy_error_context: mcpPolicyErrorContext,
model_not_supported_error_context: modelNotSupportedErrorContext,
ai_credits_rate_limit_error_context: aiCreditsRateLimitErrorContext,
unknown_model_ai_credits_context: unknownModelAICreditsContext,
app_token_minting_failed_context: appTokenMintingFailedContext,
lockdown_check_failed_context: lockdownCheckFailedContext,
stale_lock_file_failed_context: staleLockFileFailedContext,
Expand Down Expand Up @@ -3000,6 +3072,7 @@ async function main() {
// Build model not supported error context
const modelNotSupportedErrorContext = buildModelNotSupportedErrorContext(modelNotSupportedError);
const aiCreditsRateLimitErrorContext = buildAICreditsRateLimitErrorContext(aiCreditsRateLimitError || maxAICreditsExceeded, aiCredits, maxAICredits, runUrl);
const unknownModelAICreditsContext = buildUnknownModelAICreditsContext(unknownModelAICredits);

// Build GitHub App token minting failure context
const appTokenMintingFailedContext = buildAppTokenMintingFailedContext(hasAppTokenMintingFailed);
Expand Down Expand Up @@ -3051,6 +3124,7 @@ async function main() {
mcp_policy_error_context: mcpPolicyErrorContext,
model_not_supported_error_context: modelNotSupportedErrorContext,
ai_credits_rate_limit_error_context: aiCreditsRateLimitErrorContext,
unknown_model_ai_credits_context: unknownModelAICreditsContext,
app_token_minting_failed_context: appTokenMintingFailedContext,
lockdown_check_failed_context: lockdownCheckFailedContext,
stale_lock_file_failed_context: staleLockFileFailedContext,
Expand Down Expand Up @@ -3154,6 +3228,7 @@ module.exports = {
buildToolDenialsExceededContext,
buildCredentialAuthErrorContext,
buildAICreditsRateLimitErrorContext,
buildUnknownModelAICreditsContext,
hasEngineRateLimit429Signal,
hasEngineRateLimit429InOTELMirror,
buildEngineRateLimit429Context,
Expand All @@ -3173,5 +3248,6 @@ module.exports = {
CASCADE_ROLLUP_TITLE,
FAILURE_TITLE_PATTERN,
buildFailureMatchCategories,
buildFailureIssueTitle,
FAILURE_CATEGORIES_PATH,
};
Loading
Loading