diff --git a/AGENTS.md b/AGENTS.md index 07c4f34..01d318a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -273,6 +273,8 @@ There is no longer a separate `generate` step. `opfor run --config ` does **MCP targets** additionally run baseline pre-flight scans (`runBaselineScans`) before the evaluator loop — these enumerate `tools/list` + `resources/list` and judge them for poisoning / leakage independent of any evaluator. Throughout the run, `runAll` fans lifecycle events to registered `RunListener`s (progress reporting, NDJSON streaming) rather than only a callback. +**Cancellation.** `RunAllOptions` accepts an optional `signal?: AbortSignal`. When aborted, the evaluator loop finishes the in-flight attack, skips remaining evaluators/attacks, and returns a partial report with `stopReason: "user-interrupted"`. The CLI wires this to SIGINT (first Ctrl+C = graceful stop, second = force kill). The SDK can reuse the same mechanism for programmatic cancellation. + `runAllBrowser` is the same loop in browser-safe form: takes preloaded `EvaluatorSpec[]` + a pre-built `AgentTarget` (e.g. `DomTarget`), skips disk reads. --- diff --git a/core/src/execute/evaluatorLoop.ts b/core/src/execute/evaluatorLoop.ts index f71199f..306da05 100644 --- a/core/src/execute/evaluatorLoop.ts +++ b/core/src/execute/evaluatorLoop.ts @@ -41,6 +41,8 @@ export interface EvaluatorLoopContext { /** Pre-built agent target; when omitted a fresh one is created per attack. */ agentTarget?: AgentTarget; notify: (event: ProgressEvent) => void; + /** Cancellation signal — when aborted, the loop finishes the in-flight attack then stops. */ + signal?: AbortSignal; } /** @@ -60,11 +62,19 @@ export async function runEvaluatorAttacks( tools, traceContext, notify, + signal, } = ctx; const sessionMap = new Map(); const evaluatorResults: EvaluatorResult[] = []; for (const evaluator of ordered) { + if (signal?.aborted) { + const stopReason = "user-interrupted"; + log.info(`\n⏹ Run interrupted — skipping remaining evaluators`); + notify({ type: "run_stopped", reason: stopReason }); + return { evaluatorResults, stopReason }; + } + notify({ type: "evaluator_start", evaluatorId: evaluator.id, evaluatorName: evaluator.name }); const deps = evaluator.dependsOn ?? []; @@ -144,6 +154,11 @@ export async function runEvaluatorAttacks( }; for (const attack of attacks) { + if (signal?.aborted) { + log.info(` ⏹ Interrupted — skipping remaining attacks for ${evaluator.name}`); + break; + } + if (upstreamSessions?.length) { attack.upstreamSessions = upstreamSessions; } diff --git a/core/src/execute/runAll.ts b/core/src/execute/runAll.ts index 987ef01..c21a047 100644 --- a/core/src/execute/runAll.ts +++ b/core/src/execute/runAll.ts @@ -28,6 +28,13 @@ export interface RunAllOptions { outputDir?: string; /** Pre-built agent target. When omitted, createAgentTarget is called using config.target. */ agentTarget?: AgentTarget; + /** + * Cancellation signal. When aborted the run finishes in-flight work, skips + * remaining evaluators/attacks, and returns a partial report with + * `stopReason: "user-interrupted"`. Callers (CLI, SDK) wire this to + * SIGINT / programmatic cancellation. + */ + signal?: AbortSignal; } // Re-exported for callers importing it from this module; defined in ./types.js @@ -122,6 +129,7 @@ export async function runAll( traceContext, agentTarget: options?.agentTarget, notify, + signal: options?.signal, }); evaluatorResults.push(...loop.evaluatorResults); const stopReason = loop.stopReason; diff --git a/docs/cli.md b/docs/cli.md index f618917..47fc375 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -196,6 +196,23 @@ Where `` is the target name slugified (e.g. `erkala-travel-support-agent`) --- +## Graceful shutdown (Ctrl+C) + +Pressing Ctrl+C during `opfor run` triggers a graceful shutdown instead of killing the process: + +| Press | Behavior | +| ------ | --------------------------------------------------------------------------------------------------------- | +| First | Finishes the in-flight attack, skips remaining evaluators/attacks, writes a partial report, exits cleanly | +| Second | Force-kills immediately (`exit 130`) | + +Partial reports include all completed evaluator results and are marked with `stopReason: "user-interrupted"` in the JSON output. The CLI prints a warning: + +``` +⚠️ Run interrupted — results are partial. Re-run for a complete assessment. +``` + +--- + ## Effort: adaptive vs comprehensive | Effort | What it does | diff --git a/runners/cli/src/commands/run.ts b/runners/cli/src/commands/run.ts index 7e9d7e6..c533dcd 100644 --- a/runners/cli/src/commands/run.ts +++ b/runners/cli/src/commands/run.ts @@ -188,12 +188,44 @@ export function registerRunCommand(program: Command): void { return; } } - const report = await runAll(runConfig, { - outputDir: path.resolve(OPFOR_DIR), - listeners, - }); - log.info("\n\nWriting report..."); + // Graceful shutdown: first Ctrl+C aborts the signal so the engine + // finishes its in-flight attack and writes a partial report; second + // Ctrl+C force-kills immediately. + const ac = new AbortController(); + let sigintCount = 0; + const onSigint = () => { + sigintCount++; + if (sigintCount === 1) { + log.warn( + "\n\nCaught interrupt — finishing in-flight attack, skipping remaining evaluators…" + ); + log.warn("Press Ctrl+C again to force quit.\n"); + ac.abort(); + } else { + process.exit(130); + } + }; + process.on("SIGINT", onSigint); + + let report; + try { + report = await runAll(runConfig, { + outputDir: path.resolve(OPFOR_DIR), + listeners, + signal: ac.signal, + }); + } finally { + process.off("SIGINT", onSigint); + } + + const isUserInterrupted = + (report as { stopReason?: string }).stopReason === "user-interrupted"; + if (isUserInterrupted) { + log.warn("\n\nWriting partial report…"); + } else { + log.info("\n\nWriting report..."); + } const outputDir = path.resolve(opts.output ?? OPFOR_REPORTS_DIR); const { html, json } = await writeReport(report, outputDir); @@ -202,8 +234,11 @@ export function registerRunCommand(program: Command): void { `\nResults: ${summary.passed} passed, ${summary.failed} failed, ${summary.errors} errors` ); - // Warn loudly if there were errors (infra/config issues) - if (summary.errors > 0 && summary.passed === 0 && summary.failed === 0) { + if (isUserInterrupted) { + log.warn( + `\n⚠️ Run interrupted — results are partial. Re-run for a complete assessment.` + ); + } else if (summary.errors > 0 && summary.passed === 0 && summary.failed === 0) { log.warn( `\n⚠️ Assessment incomplete: all ${summary.errors} attack(s) failed due to errors.` ); diff --git a/tests/e2e/agents/customer-support/Dockerfile b/tests/e2e/agents/customer-support/Dockerfile index dbb445d..58a821d 100644 --- a/tests/e2e/agents/customer-support/Dockerfile +++ b/tests/e2e/agents/customer-support/Dockerfile @@ -2,19 +2,13 @@ FROM node:22-alpine WORKDIR /app -# Copy workspace root manifests for npm workspaces install COPY package.json package-lock.json ./ -COPY tests/e2e/agents/customer-support/package.json ./tests/e2e/agents/customer-support/ -# Install only this workspace package's deps (devDependencies included — needed at runtime via tsx) -RUN npm install --workspace=@keyvaluesystems/agent-opfor-test-agent-customer-support --ignore-scripts +RUN npm install --ignore-scripts -# Copy agent source -COPY tests/e2e/agents/customer-support/src ./tests/e2e/agents/customer-support/src -COPY tests/e2e/agents/customer-support/public ./tests/e2e/agents/customer-support/public -COPY tests/e2e/agents/customer-support/tsconfig.json ./tests/e2e/agents/customer-support/ - -WORKDIR /app/tests/e2e/agents/customer-support +COPY src ./src +COPY public ./public +COPY tsconfig.json ./ EXPOSE 4001 diff --git a/tests/e2e/agents/customer-support/docker-compose.yml b/tests/e2e/agents/customer-support/docker-compose.yml index b697cfb..278d1fb 100644 --- a/tests/e2e/agents/customer-support/docker-compose.yml +++ b/tests/e2e/agents/customer-support/docker-compose.yml @@ -16,8 +16,8 @@ services: customer-support: build: - context: ../../../../ # repo root — Dockerfile copies from here - dockerfile: tests/e2e/agents/customer-support/Dockerfile + context: . + dockerfile: Dockerfile ports: - "4001:4001" environment: