feat: deployment-aware agentic risk amplification scoring

README.md

@@ -111,6 +111,12 @@ When you run a scan, opfor:
 
 Each run lands in its own subfolder under `.opfor/reports/run-report-<compactTs>-<slug>-<shortId>/` containing `<slug>-report.html` and `<slug>-report.json`. Autonomous `opfor hunt` runs use the same layout under `hunt-report-<compactTs>-<slug>-<shortId>/`.
 
+## Risk scoring
+
+Every finding gets a **deployment-aware risk score (0–10)**, not just a static `low`/`high`/`critical` label. The same flaw is far more dangerous on an agent that can move money, act across tenants, or retain memory than on a read-only chatbot — so opfor takes each finding's base severity and amplifies it by the target's _agentic power_, inferred automatically from the business context and target metadata you already provide (no extra setup questions).
+
+In the report's results table, findings show a worst-case red risk number (a `high` BOLA on a money-moving, multi-tenant agent surfaces as `9.6`), while every defended evaluator shows a green `0.0`. A caption explains, in plain English, why the findings were amplified. This follows the OWASP [AIVSS](https://aivss.owasp.org/) amplification model, reduced to something fully automatic.
+
 ## Evaluator coverage
 
 Opfor ships with curated suites that map to industry standards. Pick a suite or run individual evaluators.

core/src/execute/agentProfile.ts

@@ -0,0 +1,85 @@
+// Derives a target's agentic "power profile" from context OPFOR already has —
+// no new setup questions. The `businessUseCase` free-text (already collected and
+// used to ground attack generation) plus structured target metadata carry the
+// signals that matter for risk amplification.
+//
+// This is a deterministic heuristic (no LLM call) so a headline-adjacent number
+// stays reproducible and free. An LLM-based classifier over the same inputs is a
+// planned enrichment — see the design doc / follow-up issue.
+
+import type { AgentProfile, UnifiedTargetConfig } from "./types.js";
+
+export interface ProfileInput {
+  /** Free-text domain/business context for the target (RunConfig.businessUseCase). */
+  businessUseCase?: string;
+  /** Structured target config, when available (absent on the browser path). */
+  target?: UnifiedTargetConfig;
+}
+
+// Patterns use a leading word boundary only (no trailing one) so common suffixes
+// match too — "refund" hits "refunds", "postgres" hits "postgresql".
+//
+// Side-effecting actions the agent can take → real blast radius, not read-only Q&A.
+const ACTION_WORDS =
+  /\b(refund|payment|charge|purchase|transfer|delete|remove|deploy|execute|send|email|provision|revoke|cancel|write|modify|update|create)/i;
+// Sensitive data the agent can reach.
+const DATA_WORDS =
+  /\b(database|sql|postgres|mysql|record|user data|customer data|personal data|pii|file|document|knowledge base|vector)/i;
+// Crossing identity / tenant / role boundaries.
+const IDENTITY_WORDS =
+  /\b(multi-?tenant|tenant|tier|role|admin|rbac|permission|account|impersonat)/i;
+// Long-lived memory / persistence.
+const MEMORY_WORDS = /\b(memory|remember|history|long-?term|persistent|persist|session)/i;
+
+/**
+ * Infer an {@link AgentProfile} from the run's business context + target metadata.
+ * Each factor is scored 0 / 0.5 / 1.0; `power` is their mean, normalized to [0,1].
+ * Always returns a profile (defaults to a low-power baseline when nothing fires).
+ */
+export function deriveAgentProfile(input: ProfileInput): AgentProfile {
+  const text = (input.businessUseCase ?? "").toLowerCase();
+  const target = input.target;
+  const reasons: string[] = [];
+  const factors: Record<string, number> = {};
+
+  // Autonomy — does it commit actions on its own? Tool-calling agents / MCP
+  // servers act without a human co-sign; action verbs confirm side effects.
+  let autonomy = 0.5;
+  if (ACTION_WORDS.test(text)) {
+    autonomy = 1.0;
+    reasons.push("acts on side-effecting tools without a human approval step");
+  }
+  factors.autonomy = autonomy;
+
+  // Tools — breadth/privilege of what it can touch.
+  let tools = 0.5;
+  if (ACTION_WORDS.test(text) || DATA_WORDS.test(text)) {
+    tools = 1.0;
+    reasons.push("has broad, high-authority tool/data access");
+  }
+  factors.tools = tools;
+
+  // Identity — can it act across users / tenants / roles?
+  let identity = 0;
+  if (IDENTITY_WORDS.test(text)) {
+    identity = 1.0;
+    reasons.push("operates across user / tenant / role boundaries");
+  }
+  factors.identity = identity;
+
+  // Persistence — session/long-term memory that survives across turns.
+  let persistence = 0;
+  if (target && "stateful" in target && target.stateful) persistence = 0.5;
+  if (MEMORY_WORDS.test(text)) persistence = Math.max(persistence, 0.5);
+  if (persistence > 0) reasons.push("retains state/memory across the conversation");
+  factors.persistence = persistence;
+
+  const values = Object.values(factors);
+  const power = values.reduce((sum, v) => sum + v, 0) / values.length;
+
+  const rationale = reasons.length
+    ? `Amplified because this agent ${reasons.join(", ")}.`
+    : "No strong agentic amplifiers detected; findings score near their base severity.";
+
+  return { power, factors, rationale };
+}

core/src/execute/aggregate.ts

@@ -7,7 +7,14 @@
 // every copy or the Node and browser paths would silently drift. Both paths now
 // funnel through these helpers, so that math lives in exactly one place.
 
-import type { AttackResult, EvaluatorResult, UnifiedRunReport, Effort } from "./types.js";
+import type {
+  AttackResult,
+  EvaluatorResult,
+  UnifiedRunReport,
+  Effort,
+  AgentProfile,
+} from "./types.js";
+import { amplifiedRisk } from "./amplify.js";
 
 /** Minimal shape needed to tally — any object carrying a judge verdict. */
 type Judged = { judge: { verdict: "PASS" | "FAIL" | "ERROR" } };
@@ -105,6 +112,13 @@ export interface ReportMeta {
   effort: Effort;
   attackModel: string;
   judgeModel: string;
+  /**
+   * Target's derived agentic power profile. When present, each evaluator gets a
+   * deployment-aware `risk` score amplified by `agentProfile.power`, and the
+   * profile is attached to the report. Omitted → evaluators carry no `risk`
+   * (e.g. direct helper calls in tests). Never affects the summary shape.
+   */
+  agentProfile?: AgentProfile;
 }
 
 /**
@@ -121,8 +135,20 @@ export function buildUnifiedReport(
   meta: ReportMeta,
   evaluators: EvaluatorResult[]
 ): UnifiedRunReport {
-  const { total, passed, failed, errors } = summarizeVerdicts(evaluators.flatMap((e) => e.attacks));
-  const { safetyScore, attackSuccessRate } = computeWeightedScores(evaluators);
+  // When an agent profile is available, amplify each evaluator's severity floor
+  // into a deployment-aware 0..10 risk score. Worst-case at the evaluator level:
+  // risk is >0 only for findings (failed > 0), 0.0 for evaluators that held.
+  // This is additive metadata — the summary shape and the weighted headline
+  // scores below are computed exactly as before.
+  const scored = meta.agentProfile
+    ? evaluators.map((ev) => ({
+        ...ev,
+        risk: amplifiedRisk(ev.severity, ev.failed > 0, meta.agentProfile!.power),
+      }))
+    : evaluators;
+
+  const { total, passed, failed, errors } = summarizeVerdicts(scored.flatMap((e) => e.attacks));
+  const { safetyScore, attackSuccessRate } = computeWeightedScores(scored);
 
   return {
     reportId: meta.reportId,
@@ -133,7 +159,8 @@ export function buildUnifiedReport(
     attackModel: meta.attackModel,
     judgeModel: meta.judgeModel,
     summary: { total, passed, failed, errors, safetyScore, attackSuccessRate },
-    evaluators,
+    evaluators: scored,
+    ...(meta.agentProfile ? { agentProfile: meta.agentProfile } : {}),
   };
 }
 

core/src/execute/amplify.ts

@@ -0,0 +1,51 @@
+// Agentic risk amplification — turns an evaluator finding's static severity into
+// a deployment-aware 0..10 risk score.
+//
+// Rationale: the same technical flaw (e.g. a broken object-level authorization
+// check) is far more dangerous on an autonomous agent that can move money and
+// read other users' data than on a read-only chatbot. Classic severity labels
+// (critical/high/medium/low) are context-blind — they never change per target.
+// This module keeps the label as a floor and lets the agent's *power* close the
+// gap toward 10, mirroring the OWASP AIVSS "amplification" model in a form that
+// is fully derivable from data OPFOR already has (see agentProfile.ts).
+//
+// The score is a RISK scale: higher = more dangerous. A non-finding (the agent
+// held) has no risk and scores 0.0.
+
+/**
+ * Technical-severity floors, one per severity label. Adopted from the
+ * CVSS/AIVSS severity bands (Critical ≥ 9, High ≥ 7, Medium ≥ 4, Low ≥ 0.1) so
+ * an un-amplified finding still lands in its expected band.
+ */
+export const BASE_RISK: Record<string, number> = {
+  critical: 9.0,
+  high: 7.0,
+  medium: 4.0,
+  low: 1.0,
+};
+
+/** Round to one decimal (nearest tenth), matching how the score is reported. */
+export function roundTo1(n: number): number {
+  return Math.round(n * 10) / 10;
+}
+
+/**
+ * Amplified risk for one evaluator, on a 0..10 scale (higher = more dangerous).
+ *
+ * - `isFinding` is worst-case at the evaluator level: if *any* attack broke
+ *   through, the evaluator is a finding. A finding scores from its severity
+ *   floor plus the agentic uplift; a clean evaluator scores 0.0. Averaging is
+ *   deliberately avoided — one successful breach is a breach regardless of how
+ *   many sibling attempts passed.
+ * - `power` (0..1) is the normalized agentic power of the target (see
+ *   `deriveAgentProfile`). It closes the "risk gap" `(10 - base)` toward 10.
+ *
+ *     risk = base + (10 - base) * power
+ */
+export function amplifiedRisk(severity: string, isFinding: boolean, power: number): number {
+  if (!isFinding) return 0;
+  const base = BASE_RISK[severity.toLowerCase()] ?? BASE_RISK.medium;
+  const p = Math.min(1, Math.max(0, power));
+  const uplift = (10 - base) * p;
+  return roundTo1(Math.min(10, base + uplift));
+}

core/src/execute/runAll.ts

@@ -12,6 +12,7 @@ import { runBaselineScans } from "./baselineScanner.js";
 import { dispatchProgress, notifyListeners, type RunListener } from "./runListener.js";
 import { runEvaluatorAttacks } from "./evaluatorLoop.js";
 import { buildUnifiedReport, modelLabel } from "./aggregate.js";
+import { deriveAgentProfile } from "./agentProfile.js";
 import { createModel } from "../providers/factory.js";
 import type { LlmConfig } from "../config/types.js";
 import { getAdapter } from "../telemetry/adapter.js";
@@ -292,6 +293,13 @@ async function curateTracesIfConfigured(
 
 function buildReport(config: RunConfig, evaluators: EvaluatorResult[]): UnifiedRunReport {
   const { attackModel, judgeModel } = modelLabel(config.attackerLlm, config.judgeLlm);
+  // Derive the target's agentic power profile once (deterministic, no LLM) from
+  // the business context + target metadata already in the config. Drives the
+  // per-evaluator risk amplification in buildUnifiedReport.
+  const agentProfile = deriveAgentProfile({
+    businessUseCase: config.businessUseCase,
+    target: config.target,
+  });
   return buildUnifiedReport(
     {
       reportId: randomUUID(),
@@ -301,6 +309,7 @@ function buildReport(config: RunConfig, evaluators: EvaluatorResult[]): UnifiedR
       effort: config.effort,
       attackModel,
       judgeModel,
+      agentProfile,
     },
     evaluators
   );

core/src/execute/runAllBrowser.ts

@@ -19,6 +19,7 @@ import {
   buildUnifiedReport,
   modelLabel,
 } from "./aggregate.js";
+import { deriveAgentProfile } from "./agentProfile.js";
 import type {
   AgentAttackSpec,
   AttackResult,
@@ -244,6 +245,9 @@ function buildBrowserReport(
   stopReason?: string
 ): UnifiedRunReport {
   const { attackModel, judgeModel } = modelLabel(config.attackerLlm, config.judgeLlm);
+  // The browser path has no structured target config, so the profile is derived
+  // from the operator's business-use-case text alone (see deriveAgentProfile).
+  const agentProfile = deriveAgentProfile({ businessUseCase: config.businessUseCase });
   return {
     ...buildUnifiedReport(
       {
@@ -254,6 +258,7 @@ function buildBrowserReport(
         effort: config.effort,
         attackModel,
         judgeModel,
+        agentProfile,
       },
       evaluators
     ),

core/src/execute/types.ts

@@ -237,6 +237,20 @@ export interface McpAttackResult extends BaseAttackResult {
 
 export type AttackResult = AgentAttackResult | McpAttackResult;
 
+/**
+ * Deployment-aware "power profile" of a target agent, derived once per run (see
+ * `deriveAgentProfile`). Drives risk amplification: the same finding scores
+ * higher on a more autonomous / tool-rich / multi-tenant agent.
+ */
+export interface AgentProfile {
+  /** Normalized agentic power in [0,1] — the uplift fed to `amplifiedRisk`. */
+  power: number;
+  /** Per-factor scores (0 / 0.5 / 1.0) that were inferred, keyed by factor name. */
+  factors: Record<string, number>;
+  /** Plain-English explanation of which signals fired — surfaced in the report. */
+  rationale: string;
+}
+
 export interface EvaluatorResult {
   evaluatorId: string;
   evaluatorName: string;
@@ -248,6 +262,14 @@ export interface EvaluatorResult {
   errors: number;
   passRate: number;
   attacks: AttackResult[];
+  /**
+   * Deployment-aware risk on a 0..10 scale (higher = more dangerous), computed
+   * from the evaluator's severity floor amplified by the target's agentic power.
+   * Worst-case: >0 only when the evaluator failed (a finding); 0.0 when it held.
+   * Undefined when no agent profile was available (e.g. direct buildUnifiedReport
+   * calls without a profile). See `amplifiedRisk`.
+   */
+  risk?: number;
 }
 
 export interface UnifiedRunReport {
@@ -267,6 +289,12 @@ export interface UnifiedRunReport {
     attackSuccessRate: number;
   };
   evaluators: EvaluatorResult[];
+  /**
+   * Target's derived agentic power profile. Present when a profile was derived
+   * for the run; drives the per-evaluator `risk` amplification. Its `rationale`
+   * is surfaced in the report to explain why findings were bumped.
+   */
+  agentProfile?: AgentProfile;
   /** Set when the run was stopped early due to a non-retryable LLM error. */
   stopReason?: string;
 }

core/src/report/buildReport.ts

@@ -71,6 +71,9 @@ function toReportViewModel(report: UnifiedRunReport): ReportViewModel {
     target: { name: report.targetName },
     summary: report.summary,
     evaluators: report.evaluators.map(toEvaluatorViewModel),
+    agentProfile: report.agentProfile
+      ? { power: report.agentProfile.power, rationale: report.agentProfile.rationale }
+      : undefined,
     stopReason: report.stopReason,
   };
 }
@@ -87,6 +90,7 @@ function toEvaluatorViewModel(ev: EvaluatorResult): EvaluatorViewModel {
     errors: ev.errors,
     passRate: ev.passRate,
     results: ev.attacks.map(toResultViewModel),
+    risk: ev.risk,
   };
 }
 

core/src/report/render.ts

@@ -176,11 +176,11 @@ export function renderReport(model: ReportViewModel): string {
       const sevColor = SEV_HEX[e.severity] || "#64748B";
       const passDenom = e.passed + e.failed;
       const passRate = passDenom > 0 ? Math.round((e.passed / passDenom) * 100) : 0;
-      const scoreable = e.results.filter((r) => r.judge.verdict !== "ERROR");
-      const avgScore =
-        scoreable.length > 0
-          ? (scoreable.reduce((s, r) => s + r.judge.score, 0) / scoreable.length).toFixed(1)
-          : "—";
+      // Deployment-aware risk (0..10, higher = more dangerous): red for a
+      // finding, green 0.0 when the evaluator held. "—" when no agent profile
+      // was available so no risk could be computed.
+      const risk = typeof e.risk === "number" ? e.risk : null;
+      const riskColor = risk === null ? "#94A3B8" : risk > 0 ? "#DC2626" : "#059669";
       const evalVerdict =
         e.errors > 0 && e.passed === 0 && e.failed === 0
           ? "ERROR"
@@ -204,7 +204,7 @@ export function renderReport(model: ReportViewModel): string {
           <td style="color:#DC2626;font-weight:600">${e.failed}</td>
           ${anyErrors ? `<td style="color:#D97706;font-weight:600">${e.errors > 0 ? e.errors : "—"}</td>` : ""}
           <td>${passRate}%</td>
-          <td class="td-score">${avgScore !== "—" ? `${avgScore}<span style="color:#94A3B8">/10</span>` : "—"}</td>
+          <td class="td-score">${risk === null ? "—" : `<span style="color:${riskColor};font-weight:700">${risk.toFixed(1)}</span><span style="color:#94A3B8">/10</span>`}</td>
         </tr>`;
     })
     .join("");
@@ -626,11 +626,12 @@ export function renderReport(model: ReportViewModel): string {
     <div class="results-table-wrap" style="margin-bottom:16px">
       <table class="results">
         <thead><tr>
-          <th>#</th><th>Evaluator</th><th>Severity</th><th>Verdict</th><th>Tests</th><th>Passed</th><th>Failed</th>${anyErrors ? "<th>Errors</th>" : ""}<th>Pass Rate</th><th>Avg Score</th>
+          <th>#</th><th>Evaluator</th><th>Base Sev</th><th>Verdict</th><th>Tests</th><th>Passed</th><th>Failed</th>${anyErrors ? "<th>Errors</th>" : ""}<th>Pass Rate</th><th>Risk (this agent)</th>
         </tr></thead>
         <tbody>${tableRows}</tbody>
       </table>
     </div>
+    ${model.agentProfile ? `<div style="font-size:12px;color:var(--muted);line-height:1.5"><strong>Risk (this agent)</strong> takes each finding's base severity and amplifies it by how much damage this specific agent can do (worst-case per evaluator; a defended test scores 0.0). ${esc(model.agentProfile.rationale)}</div>` : ""}
   </div>
 
   <!-- 5. Detailed Results -->