From 5beb7e53a28a35cf29749fae5cfe84c983fd7134 Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Tue, 21 Jul 2026 16:55:49 +0530 Subject: [PATCH 1/4] feat: graceful shutdown on ctrl+c during opfor run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thread an AbortSignal through RunAllOptions → EvaluatorLoopContext so the evaluator loop checks for cancellation between evaluators and between attacks. The in-flight attack finishes, completed results are kept, and a partial report is written with stopReason: "user-interrupted". CLI behavior: - First Ctrl+C: abort signal fires, engine finishes current work, writes a partial report, exits cleanly. - Second Ctrl+C: process.exit(130) — immediate force kill. - Handler is cleaned up in a finally block. Also fixes the customer-support test agent Dockerfile which referenced a workspace not registered in the root package.json. Closes #209 Co-authored-by: Cursor --- core/src/execute/evaluatorLoop.ts | 15 ++++++ core/src/execute/runAll.ts | 8 ++++ runners/cli/src/commands/run.ts | 48 ++++++++++++++++--- tests/e2e/agents/customer-support/Dockerfile | 14 ++---- .../customer-support/docker-compose.yml | 4 +- .../agents/customer-support/opfor.config.json | 29 +++++++++++ 6 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 tests/e2e/agents/customer-support/opfor.config.json diff --git a/core/src/execute/evaluatorLoop.ts b/core/src/execute/evaluatorLoop.ts index f71199fa..306da051 100644 --- a/core/src/execute/evaluatorLoop.ts +++ b/core/src/execute/evaluatorLoop.ts @@ -41,6 +41,8 @@ export interface EvaluatorLoopContext { /** Pre-built agent target; when omitted a fresh one is created per attack. */ agentTarget?: AgentTarget; notify: (event: ProgressEvent) => void; + /** Cancellation signal — when aborted, the loop finishes the in-flight attack then stops. */ + signal?: AbortSignal; } /** @@ -60,11 +62,19 @@ export async function runEvaluatorAttacks( tools, traceContext, notify, + signal, } = ctx; const sessionMap = new Map(); const evaluatorResults: EvaluatorResult[] = []; for (const evaluator of ordered) { + if (signal?.aborted) { + const stopReason = "user-interrupted"; + log.info(`\n⏹ Run interrupted — skipping remaining evaluators`); + notify({ type: "run_stopped", reason: stopReason }); + return { evaluatorResults, stopReason }; + } + notify({ type: "evaluator_start", evaluatorId: evaluator.id, evaluatorName: evaluator.name }); const deps = evaluator.dependsOn ?? []; @@ -144,6 +154,11 @@ export async function runEvaluatorAttacks( }; for (const attack of attacks) { + if (signal?.aborted) { + log.info(` ⏹ Interrupted — skipping remaining attacks for ${evaluator.name}`); + break; + } + if (upstreamSessions?.length) { attack.upstreamSessions = upstreamSessions; } diff --git a/core/src/execute/runAll.ts b/core/src/execute/runAll.ts index 987ef01e..c21a0474 100644 --- a/core/src/execute/runAll.ts +++ b/core/src/execute/runAll.ts @@ -28,6 +28,13 @@ export interface RunAllOptions { outputDir?: string; /** Pre-built agent target. When omitted, createAgentTarget is called using config.target. */ agentTarget?: AgentTarget; + /** + * Cancellation signal. When aborted the run finishes in-flight work, skips + * remaining evaluators/attacks, and returns a partial report with + * `stopReason: "user-interrupted"`. Callers (CLI, SDK) wire this to + * SIGINT / programmatic cancellation. + */ + signal?: AbortSignal; } // Re-exported for callers importing it from this module; defined in ./types.js @@ -122,6 +129,7 @@ export async function runAll( traceContext, agentTarget: options?.agentTarget, notify, + signal: options?.signal, }); evaluatorResults.push(...loop.evaluatorResults); const stopReason = loop.stopReason; diff --git a/runners/cli/src/commands/run.ts b/runners/cli/src/commands/run.ts index 7e9d7e6b..1b153721 100644 --- a/runners/cli/src/commands/run.ts +++ b/runners/cli/src/commands/run.ts @@ -188,12 +188,43 @@ export function registerRunCommand(program: Command): void { return; } } - const report = await runAll(runConfig, { - outputDir: path.resolve(OPFOR_DIR), - listeners, - }); - log.info("\n\nWriting report..."); + // Graceful shutdown: first Ctrl+C aborts the signal so the engine + // finishes its in-flight attack and writes a partial report; second + // Ctrl+C force-kills immediately. + const ac = new AbortController(); + let sigintCount = 0; + const onSigint = () => { + sigintCount++; + if (sigintCount === 1) { + log.warn( + "\n\nCaught interrupt — finishing in-flight attack, skipping remaining evaluators…" + ); + log.warn("Press Ctrl+C again to force quit.\n"); + ac.abort(); + } else { + process.exit(130); + } + }; + process.on("SIGINT", onSigint); + + let report; + try { + report = await runAll(runConfig, { + outputDir: path.resolve(OPFOR_DIR), + listeners, + signal: ac.signal, + }); + } finally { + process.off("SIGINT", onSigint); + } + + const isPartial = !!(report as { stopReason?: string }).stopReason; + if (isPartial) { + log.warn("\n\nWriting partial report…"); + } else { + log.info("\n\nWriting report..."); + } const outputDir = path.resolve(opts.output ?? OPFOR_REPORTS_DIR); const { html, json } = await writeReport(report, outputDir); @@ -202,8 +233,11 @@ export function registerRunCommand(program: Command): void { `\nResults: ${summary.passed} passed, ${summary.failed} failed, ${summary.errors} errors` ); - // Warn loudly if there were errors (infra/config issues) - if (summary.errors > 0 && summary.passed === 0 && summary.failed === 0) { + if (isPartial) { + log.warn( + `\n⚠️ Run interrupted — results are partial. Re-run for a complete assessment.` + ); + } else if (summary.errors > 0 && summary.passed === 0 && summary.failed === 0) { log.warn( `\n⚠️ Assessment incomplete: all ${summary.errors} attack(s) failed due to errors.` ); diff --git a/tests/e2e/agents/customer-support/Dockerfile b/tests/e2e/agents/customer-support/Dockerfile index dbb445d9..58a821d9 100644 --- a/tests/e2e/agents/customer-support/Dockerfile +++ b/tests/e2e/agents/customer-support/Dockerfile @@ -2,19 +2,13 @@ FROM node:22-alpine WORKDIR /app -# Copy workspace root manifests for npm workspaces install COPY package.json package-lock.json ./ -COPY tests/e2e/agents/customer-support/package.json ./tests/e2e/agents/customer-support/ -# Install only this workspace package's deps (devDependencies included — needed at runtime via tsx) -RUN npm install --workspace=@keyvaluesystems/agent-opfor-test-agent-customer-support --ignore-scripts +RUN npm install --ignore-scripts -# Copy agent source -COPY tests/e2e/agents/customer-support/src ./tests/e2e/agents/customer-support/src -COPY tests/e2e/agents/customer-support/public ./tests/e2e/agents/customer-support/public -COPY tests/e2e/agents/customer-support/tsconfig.json ./tests/e2e/agents/customer-support/ - -WORKDIR /app/tests/e2e/agents/customer-support +COPY src ./src +COPY public ./public +COPY tsconfig.json ./ EXPOSE 4001 diff --git a/tests/e2e/agents/customer-support/docker-compose.yml b/tests/e2e/agents/customer-support/docker-compose.yml index b697cfb6..278d1fb3 100644 --- a/tests/e2e/agents/customer-support/docker-compose.yml +++ b/tests/e2e/agents/customer-support/docker-compose.yml @@ -16,8 +16,8 @@ services: customer-support: build: - context: ../../../../ # repo root — Dockerfile copies from here - dockerfile: tests/e2e/agents/customer-support/Dockerfile + context: . + dockerfile: Dockerfile ports: - "4001:4001" environment: diff --git a/tests/e2e/agents/customer-support/opfor.config.json b/tests/e2e/agents/customer-support/opfor.config.json new file mode 100644 index 00000000..80b44006 --- /dev/null +++ b/tests/e2e/agents/customer-support/opfor.config.json @@ -0,0 +1,29 @@ +{ + "target": { + "kind": "agent", + "name": "customer-support", + "description": "AcmeCorp customer support agent with tools (lookup_order, lookup_user_profile, list_my_orders, create_ticket, process_refund) backed by PostgreSQL", + "type": "http-endpoint", + "endpoint": "http://localhost:4001/chat", + "requestFormat": "json", + "session": { + "send": { "in": "body", "name": "sessionId" } + }, + "promptPath": "prompt", + "responsePath": "response", + "stateful": true + }, + "selection": { + "mode": "evaluators", + "evaluators": ["bola", "system-prompt-leakage", "jailbreaking"] + }, + "attackerLlm": { + "provider": "openai-compatible", + "model": "deepseek/deepseek-v4-flash", + "apiKeyEnv": "OPFOR_API_KEY", + "baseURL": "https://llm.keyvalue.systems/v1" + }, + "effort": "adaptive", + "turnMode": "multi", + "turns": 3 +} From fe38a74322b0e0b5dd4fd77ec96aee4775ff1e0c Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Tue, 21 Jul 2026 17:04:26 +0530 Subject: [PATCH 2/4] docs: document graceful shutdown on ctrl+c Co-authored-by: Cursor --- AGENTS.md | 2 ++ docs/cli.md | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 07c4f349..01d318ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -273,6 +273,8 @@ There is no longer a separate `generate` step. `opfor run --config ` does **MCP targets** additionally run baseline pre-flight scans (`runBaselineScans`) before the evaluator loop — these enumerate `tools/list` + `resources/list` and judge them for poisoning / leakage independent of any evaluator. Throughout the run, `runAll` fans lifecycle events to registered `RunListener`s (progress reporting, NDJSON streaming) rather than only a callback. +**Cancellation.** `RunAllOptions` accepts an optional `signal?: AbortSignal`. When aborted, the evaluator loop finishes the in-flight attack, skips remaining evaluators/attacks, and returns a partial report with `stopReason: "user-interrupted"`. The CLI wires this to SIGINT (first Ctrl+C = graceful stop, second = force kill). The SDK can reuse the same mechanism for programmatic cancellation. + `runAllBrowser` is the same loop in browser-safe form: takes preloaded `EvaluatorSpec[]` + a pre-built `AgentTarget` (e.g. `DomTarget`), skips disk reads. --- diff --git a/docs/cli.md b/docs/cli.md index f618917e..47fc3754 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -196,6 +196,23 @@ Where `` is the target name slugified (e.g. `erkala-travel-support-agent`) --- +## Graceful shutdown (Ctrl+C) + +Pressing Ctrl+C during `opfor run` triggers a graceful shutdown instead of killing the process: + +| Press | Behavior | +| ------ | --------------------------------------------------------------------------------------------------------- | +| First | Finishes the in-flight attack, skips remaining evaluators/attacks, writes a partial report, exits cleanly | +| Second | Force-kills immediately (`exit 130`) | + +Partial reports include all completed evaluator results and are marked with `stopReason: "user-interrupted"` in the JSON output. The CLI prints a warning: + +``` +⚠️ Run interrupted — results are partial. Re-run for a complete assessment. +``` + +--- + ## Effort: adaptive vs comprehensive | Effort | What it does | From db11d39f273258c9dd9d97310ade5b8f942e275e Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Tue, 21 Jul 2026 17:05:30 +0530 Subject: [PATCH 3/4] chore: remove local test config from tracking Co-authored-by: Cursor --- .../agents/customer-support/opfor.config.json | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 tests/e2e/agents/customer-support/opfor.config.json diff --git a/tests/e2e/agents/customer-support/opfor.config.json b/tests/e2e/agents/customer-support/opfor.config.json deleted file mode 100644 index 80b44006..00000000 --- a/tests/e2e/agents/customer-support/opfor.config.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "target": { - "kind": "agent", - "name": "customer-support", - "description": "AcmeCorp customer support agent with tools (lookup_order, lookup_user_profile, list_my_orders, create_ticket, process_refund) backed by PostgreSQL", - "type": "http-endpoint", - "endpoint": "http://localhost:4001/chat", - "requestFormat": "json", - "session": { - "send": { "in": "body", "name": "sessionId" } - }, - "promptPath": "prompt", - "responsePath": "response", - "stateful": true - }, - "selection": { - "mode": "evaluators", - "evaluators": ["bola", "system-prompt-leakage", "jailbreaking"] - }, - "attackerLlm": { - "provider": "openai-compatible", - "model": "deepseek/deepseek-v4-flash", - "apiKeyEnv": "OPFOR_API_KEY", - "baseURL": "https://llm.keyvalue.systems/v1" - }, - "effort": "adaptive", - "turnMode": "multi", - "turns": 3 -} From 5314366d888177c215e9de584d8706e5bc8dd45a Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Wed, 22 Jul 2026 11:32:12 +0530 Subject: [PATCH 4/4] fix: narrow partial-report check to user-interrupted only Co-authored-by: Cursor --- runners/cli/src/commands/run.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/runners/cli/src/commands/run.ts b/runners/cli/src/commands/run.ts index 1b153721..c533dcdc 100644 --- a/runners/cli/src/commands/run.ts +++ b/runners/cli/src/commands/run.ts @@ -219,8 +219,9 @@ export function registerRunCommand(program: Command): void { process.off("SIGINT", onSigint); } - const isPartial = !!(report as { stopReason?: string }).stopReason; - if (isPartial) { + const isUserInterrupted = + (report as { stopReason?: string }).stopReason === "user-interrupted"; + if (isUserInterrupted) { log.warn("\n\nWriting partial report…"); } else { log.info("\n\nWriting report..."); @@ -233,7 +234,7 @@ export function registerRunCommand(program: Command): void { `\nResults: ${summary.passed} passed, ${summary.failed} failed, ${summary.errors} errors` ); - if (isPartial) { + if (isUserInterrupted) { log.warn( `\n⚠️ Run interrupted — results are partial. Re-run for a complete assessment.` );