diff --git a/.changeset/add-candidate-benchmark-task.md b/.changeset/add-candidate-benchmark-task.md new file mode 100644 index 0000000..c20e140 --- /dev/null +++ b/.changeset/add-candidate-benchmark-task.md @@ -0,0 +1,5 @@ +--- +"@tangle-network/agent-interface": minor +--- + +Add one immutable candidate experiment shared by evaluation and execution, with exact baseline and candidate bundles, benchmark tasks, repetitions, seeds, and Runtime receipts. diff --git a/packages/agent-interface/package.json b/packages/agent-interface/package.json index 3f962f6..5e07f94 100644 --- a/packages/agent-interface/package.json +++ b/packages/agent-interface/package.json @@ -46,7 +46,7 @@ ], "scripts": { "build": "tsc -p tsconfig.json", - "check-types": "tsc --noEmit", + "check-types": "tsc --noEmit && tsc --noEmit -p tsconfig.test.json", "clean": "rm -rf dist tsconfig.tsbuildinfo", "test": "vitest run", "test:watch": "vitest", diff --git a/packages/agent-interface/src/agent-candidate-code-schema.test.ts b/packages/agent-interface/src/agent-candidate-code-schema.test.ts index b71bf0a..3475d96 100644 --- a/packages/agent-interface/src/agent-candidate-code-schema.test.ts +++ b/packages/agent-interface/src/agent-candidate-code-schema.test.ts @@ -21,9 +21,7 @@ const instructionDelivery = { kind: "argv-append" as const }; describe("candidate code and execution schemas", () => { it("distinguishes disabled, proposer no-op, and real changes", () => { - expect(() => - agentCandidateCodeSchema.parse({ kind: "disabled", reason: "control" }), - ).not.toThrow(); + expect(() => agentCandidateCodeSchema.parse({ kind: "disabled" })).not.toThrow(); expect(() => agentCandidateCodeSchema.parse({ kind: "no-op", diff --git a/packages/agent-interface/src/agent-candidate-code-schema.ts b/packages/agent-interface/src/agent-candidate-code-schema.ts index e92365f..be898b2 100644 --- a/packages/agent-interface/src/agent-candidate-code-schema.ts +++ b/packages/agent-interface/src/agent-candidate-code-schema.ts @@ -62,7 +62,6 @@ const executableSchema = z export const agentCandidateCodeDisabledSchema = z .object({ kind: z.literal("disabled"), - reason: z.enum(["control", "not-applicable"]), }) .strict() satisfies z.ZodType; diff --git a/packages/agent-interface/src/agent-candidate-execution-plan-schema.test.ts b/packages/agent-interface/src/agent-candidate-execution-plan-schema.test.ts index e0e9aea..20c6e83 100644 --- a/packages/agent-interface/src/agent-candidate-execution-plan-schema.test.ts +++ b/packages/agent-interface/src/agent-candidate-execution-plan-schema.test.ts @@ -47,37 +47,20 @@ function planFixture() { }; return { kind: "agent-candidate-execution-plan-material" as const, - bundleDigest: candidateSha("1"), - executionId: "execution-1", - attempt: { - number: 1, - maxAttempts: 1, - retryPolicy: "pre-model-infrastructure-only" as const, - }, - task: { - benchmark: "pier", - benchmarkVersion: "0.3", - taskId: "task-1", - splitDigest: candidateSha("2"), - instruction: { - encoding: "utf8" as const, - sha256: candidateSha("3"), - byteLength: 42, - delivery: { - kind: "utf8-file" as const, - env: "TANGLE_CANDIDATE_TASK_PATH" as const, - path: "/tangle/input/task.txt" as const, - }, - }, - repository: { - identity: "tangle-network/agent-runtime", - rootIdentity: "tangle-network/agent-runtime", - baseCommit: "1".repeat(40), - baseTree: "2".repeat(40), - }, - outcome: { kind: "workspace" as const }, - workspace: workspace("src/task.ts", "4"), + runCell: { + kind: "agent-candidate-run-cell" as const, + experimentDigest: candidateSha("0"), + arm: "candidate" as const, + bundleDigest: candidateSha("1"), + suiteDigest: candidateSha("9"), + taskDigest: candidateSha("2"), + taskIndex: 0, + repetition: 0, + seed: 42, + attempt: 1, + digest: candidateSha("3"), }, + executionId: "execution-1", workspaces: { taskRoot: "/work/task", candidateRoot: "/work/candidate", @@ -91,6 +74,19 @@ function planFixture() { }, harness: "codex" as const, harnessVersion: "0.1.0", + instructionDelivery: { + kind: "utf8-file" as const, + env: "TANGLE_CANDIDATE_TASK_PATH" as const, + path: "/tangle/input/task.txt" as const, + }, + limits: { + timeoutMs: 120_000, + maxSteps: 50, + maxModelCalls: 12, + maxInputTokens: 100_000, + maxOutputTokens: 20_000, + maxCostUsd: 5, + }, container: { source: "evaluator-task-container" as const, image: "pier-task:0.3", @@ -118,15 +114,6 @@ function planFixture() { }, ], }, - grader: { - name: "fixture-grader", - version: "1.0.0", - artifact: { - locator: { kind: "s3" as const, bucket: "test-artifacts", key: "grader" }, - sha256: candidateSha("a"), - byteLength: 1, - }, - }, launch: { executable: "node", args: [{ kind: "public" as const, value: "dist/agent.js" }], @@ -139,14 +126,6 @@ function planFixture() { cwd: { workspace: "task" as const, path: "." }, }, memory: { mode: "disabled" as const }, - limits: { - timeoutMs: 60_000, - maxSteps: 200, - maxModelCalls: 20, - maxInputTokens: 200_000, - maxOutputTokens: 20_000, - maxCostUsd: 20, - }, network: { mode: "disabled" as const }, }; } @@ -157,6 +136,7 @@ describe("agentCandidateExecutionPlanMaterialSchema", () => { kind: "agent-profile-workspace-plan" as const, digest: candidateSha("1"), material: { + sourceProfileDigest: candidateSha("0"), harness: "codex" as const, files: [ { @@ -223,36 +203,8 @@ describe("agentCandidateExecutionPlanMaterialSchema", () => { ).toThrow(/Unrecognized key/); }); - it("requires the split, fixed disjoint roots, and evaluator model access", () => { + it("requires fixed disjoint roots and evaluator model access", () => { const plan = planFixture(); - const { splitDigest: _splitDigest, ...taskWithoutSplit } = plan.task; - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...plan, - task: taskWithoutSplit, - }), - ).toThrow(); - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...plan, - task: { - ...plan.task, - repository: { - ...plan.task.repository, - baseTree: "2".repeat(64), - }, - }, - }), - ).toThrow(/one object format/); - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...plan, - task: { - ...plan.task, - instruction: { ...plan.task.instruction, byteLength: 0 }, - }, - }), - ).toThrow(); expect(() => agentCandidateExecutionPlanMaterialSchema.parse({ ...plan, @@ -262,71 +214,45 @@ describe("agentCandidateExecutionPlanMaterialSchema", () => { }, }), ).toThrow(/disjoint/); - const { access: _access, ...modelWithoutAccess } = plan.model; expect(() => agentCandidateExecutionPlanMaterialSchema.parse({ ...plan, - model: modelWithoutAccess, + workspaces: { ...plan.workspaces, taskRoot: "/tangle/input" }, }), - ).toThrow(); - }); - - it("accepts a bounded exact output instead of a Git result", () => { - const plan = planFixture(); - const outputPlan = { - ...plan, - task: { - ...plan.task, - workspace: { - ...plan.task.workspace, - material: { ...plan.task.workspace.material, files: [] }, - }, - outcome: { - kind: "output" as const, - mediaType: "application/json", - maxBytes: 1_048_576, - }, - }, - }; - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse(outputPlan), - ).not.toThrow(); - const { repository: _repository, ...taskWithoutRepository } = outputPlan.task; - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...outputPlan, - task: taskWithoutRepository, - }), - ).not.toThrow(); + ).toThrow(/task file delivery must be outside/); expect(() => agentCandidateExecutionPlanMaterialSchema.parse({ - ...outputPlan, - task: { - ...outputPlan.task, - outcome: { ...outputPlan.task.outcome, mediaType: "Application/JSON" }, + ...plan, + launch: { + ...plan.launch, + env: { + TANGLE_CANDIDATE_TASK_PATH: { + kind: "public", + value: "/tmp/task.txt", + }, + }, }, }), - ).toThrow(); + ).toThrow(/fixed evaluator-owned path/); + const { access: _access, ...modelWithoutAccess } = plan.model; expect(() => agentCandidateExecutionPlanMaterialSchema.parse({ - ...outputPlan, - task: { - ...outputPlan.task, - outcome: { ...outputPlan.task.outcome, maxBytes: 64 * 1024 * 1024 + 1 }, - }, + ...plan, + model: modelWithoutAccess, }), ).toThrow(); }); - it("requires repository provenance only for workspace outcomes", () => { + it("rejects task-owned fields so they cannot drift from the signed suite", () => { const plan = planFixture(); - const { repository: _repository, ...taskWithoutRepository } = plan.task; - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...plan, - task: taskWithoutRepository, - }), - ).toThrow(/require task repository provenance/); + for (const field of ["task", "grader", "limits"] as const) { + expect( + agentCandidateExecutionPlanMaterialSchema.safeParse({ + ...plan, + [field]: {}, + }).success, + ).toBe(false); + } }); it("rejects ambiguous roots and silently omitted profile behavior", () => { @@ -341,6 +267,7 @@ describe("agentCandidateExecutionPlanMaterialSchema", () => { } const profilePlan = { + sourceProfileDigest: candidateSha("0"), harness: "codex", files: [], env: {}, @@ -398,7 +325,7 @@ describe("agentCandidateExecutionPlanMaterialSchema", () => { expect(() => agentCandidateResolvedModelSchema.parse(withoutEffort)).toThrow(); }); - it("freezes only exact model-gateway domains and disables them for zero-call plans", () => { + it("freezes only exact model-gateway domains", () => { const plan = planFixture(); for (const domain of [ "*.tangle.tools", @@ -436,75 +363,22 @@ describe("agentCandidateExecutionPlanMaterialSchema", () => { }, }), ).toThrow(/lexicographically sorted/); - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...plan, - limits: { ...plan.limits, maxModelCalls: 0 }, - model: { - ...plan.model, - access: { ...plan.model.access, network: { mode: "disabled" } }, - }, - }), - ).not.toThrow(); - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...plan, - model: { - ...plan.model, - access: { ...plan.model.access, network: { mode: "disabled" } }, - }, - }), - ).toThrow(/require one frozen gateway allowlist/); }); - it("freezes counted attempts, retry policy, and tool-loop steps", () => { + it("keeps only the exact attempt number in the run cell", () => { const plan = planFixture(); expect(() => agentCandidateExecutionPlanMaterialSchema.parse({ ...plan, - attempt: { ...plan.attempt, number: 2 }, - }), - ).toThrow(/cannot exceed/); - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...plan, - attempt: { number: 1, maxAttempts: 2, retryPolicy: "none" }, - }), - ).toThrow(/exactly one attempt/); - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...plan, - limits: { ...plan.limits, maxSteps: 0 }, + runCell: { ...plan.runCell, attempt: 0 }, }), ).toThrow(); - }); - - it("binds exact task-instruction delivery outside both workspaces", () => { - const plan = planFixture(); - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...plan, - launch: { ...plan.launch, env: {} }, - }), - ).toThrow(/signed fixed task path/); - expect(() => - agentCandidateExecutionPlanMaterialSchema.parse({ - ...plan, - workspaces: { ...plan.workspaces, taskRoot: "/tangle" }, - }), - ).toThrow(/outside both workspaces/); expect(() => agentCandidateExecutionPlanMaterialSchema.parse({ ...plan, - task: { - ...plan.task, - instruction: { - ...plan.task.instruction, - delivery: { kind: "argv-append" }, - }, - }, + runCell: { ...plan.runCell, maxAttempts: 2 }, }), - ).toThrow(/cannot expose an instruction path/); + ).toThrow(/Unrecognized key/); }); it("requires active workspace bytes and fresh reset evidence", () => { diff --git a/packages/agent-interface/src/agent-candidate-execution-plan-schema.ts b/packages/agent-interface/src/agent-candidate-execution-plan-schema.ts index b429c05..31b5fd2 100644 --- a/packages/agent-interface/src/agent-candidate-execution-plan-schema.ts +++ b/packages/agent-interface/src/agent-candidate-execution-plan-schema.ts @@ -1,14 +1,21 @@ import { z } from "zod"; import type { + AgentCandidateBenchmarkCellRef, + AgentCandidateBenchmarkGraderIdentity, AgentCandidateEffectiveMemory, AgentCandidateExecutionLimits, AgentCandidateExecutionPlanEvidence, AgentCandidateExecutionPlanMaterial, + AgentCandidateInstructionDelivery, AgentCandidateModelAccessNetwork, AgentCandidateProfileActivation, AgentCandidateProfilePlanEvidence, AgentCandidateProfilePlanMaterial, AgentCandidateResolvedModel, + AgentCandidateResolvedTaskContainer, + AgentCandidateRunCell, + AgentCandidateRunCellMaterial, + AgentCandidateTaskRepository, AgentCandidateTaskOutcomeSpec, } from "./agent-candidate.js"; import { @@ -21,6 +28,7 @@ import { agentCandidateInstructionDeliverySchema, agentCandidateWorkingDirectorySchema, } from "./agent-candidate-code-schema.js"; +import { reasoningEffortSchema } from "./profile-schema.js"; import { addDuplicateIssues, agentCandidateConfigValueSchema, @@ -38,16 +46,6 @@ import { } from "./agent-candidate-schema-common.js"; import { harnessTypeSchema } from "./harness.js"; -const reasoningEffortSchema = z.enum([ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh", - "ultracode", -]); - function isCanonicalAbsolutePath(value: string): boolean { return ( value.startsWith("/") && @@ -113,8 +111,64 @@ export const agentCandidateResolvedModelSchema = z }) .strict() satisfies z.ZodType; +export const agentCandidateBenchmarkGraderIdentitySchema = z + .object({ + name: z.string().min(1), + version: z.string().min(1), + format: z.literal("tangle-grader"), + artifact: agentCandidateArtifactRefSchema, + }) + .strict() + .superRefine((grader, ctx) => { + if (grader.artifact.byteLength === 0) { + ctx.addIssue({ + code: "custom", + path: ["artifact", "byteLength"], + message: "pinned grader artifact must contain executable grader bytes", + }); + } + }) satisfies z.ZodType; + +export const agentCandidateTaskRepositorySchema = z + .object({ + identity: z.string().min(1).max(512), + rootIdentity: z.string().min(1).max(512), + baseCommit: gitObjectSchema, + baseTree: gitObjectSchema, + }) + .strict() + .superRefine((repository, ctx) => { + if (!sameGitObjectFormat(repository.baseCommit, repository.baseTree)) { + ctx.addIssue({ + code: "custom", + message: "task Git object ids must use one object format", + }); + } + }) satisfies z.ZodType; + +export const agentCandidateResolvedTaskContainerSchema = + agentCandidateContainerSchema + .extend({ + source: z.literal("evaluator-task-container"), + manifestDigest: sha256DigestSchema, + platform: z + .object({ + os: z.string().min(1).max(100), + architecture: z.string().min(1).max(100), + variant: z.string().min(1).max(100).optional(), + }) + .strict(), + }) + .strict() satisfies z.ZodType; + +export const agentCandidateRetryPolicySchema = z.enum([ + "pre-model-infrastructure-only", + "none", +]); + export const agentCandidateProfilePlanMaterialSchema = z .object({ + sourceProfileDigest: sha256DigestSchema, harness: harnessTypeSchema, files: z.array( z @@ -224,45 +278,38 @@ export const agentCandidateTaskOutcomeSpecSchema = z.discriminatedUnion("kind", .strict(), ]) satisfies z.ZodType; +export const agentCandidateBenchmarkCellRefSchema = z + .object({ + suiteDigest: sha256DigestSchema, + taskIndex: z.number().int().nonnegative(), + repetition: z.number().int().nonnegative(), + }) + .strict() satisfies z.ZodType; + +export const agentCandidateRunCellMaterialSchema = z + .object({ + kind: z.literal("agent-candidate-run-cell"), + experimentDigest: sha256DigestSchema, + arm: z.enum(["baseline", "candidate"]), + bundleDigest: sha256DigestSchema, + suiteDigest: sha256DigestSchema, + taskDigest: sha256DigestSchema, + taskIndex: z.number().int().nonnegative(), + repetition: z.number().int().nonnegative(), + seed: z.number().int().safe(), + attempt: z.number().int().positive(), + }) + .strict() satisfies z.ZodType; + +export const agentCandidateRunCellSchema = agentCandidateRunCellMaterialSchema + .extend({ digest: sha256DigestSchema }) + .strict() satisfies z.ZodType; + export const agentCandidateExecutionPlanMaterialSchema = z .object({ kind: z.literal("agent-candidate-execution-plan-material"), - bundleDigest: sha256DigestSchema, + runCell: agentCandidateRunCellSchema, executionId: z.string().min(1), - attempt: z - .object({ - number: z.number().int().min(1), - maxAttempts: z.number().int().min(1), - retryPolicy: z.enum(["pre-model-infrastructure-only", "none"]), - }) - .strict(), - task: z - .object({ - benchmark: z.string().min(1), - benchmarkVersion: z.string().min(1), - taskId: z.string().min(1), - splitDigest: sha256DigestSchema, - instruction: z - .object({ - encoding: z.literal("utf8"), - sha256: sha256DigestSchema, - byteLength: z.number().int().positive(), - delivery: agentCandidateInstructionDeliverySchema, - }) - .strict(), - repository: z - .object({ - identity: z.string().min(1), - rootIdentity: z.string().min(1), - baseCommit: gitObjectSchema, - baseTree: gitObjectSchema, - }) - .strict() - .optional(), - outcome: agentCandidateTaskOutcomeSpecSchema, - workspace: agentCandidateWorkspaceSnapshotEvidenceSchema, - }) - .strict(), workspaces: z .object({ taskRoot: z @@ -295,6 +342,9 @@ export const agentCandidateExecutionPlanMaterialSchema = z .strict(), harness: harnessTypeSchema, harnessVersion: z.string().min(1), + instructionDelivery: + agentCandidateInstructionDeliverySchema satisfies z.ZodType, + limits: agentCandidateExecutionLimitsSchema, container: agentCandidateContainerSchema .extend({ source: z.enum(["pinned-container", "evaluator-task-container"]), @@ -353,13 +403,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z .min(1), }) .strict(), - grader: z - .object({ - name: z.string().min(1), - version: z.string().min(1), - artifact: agentCandidateArtifactRefSchema, - }) - .strict(), launch: z .object({ executable: z @@ -372,51 +415,10 @@ export const agentCandidateExecutionPlanMaterialSchema = z .strict(), knowledgeManifestDigest: sha256DigestSchema.optional(), memory: agentCandidateEffectiveMemorySchema, - limits: agentCandidateExecutionLimitsSchema, network: z.object({ mode: z.literal("disabled") }).strict(), }) .strict() .superRefine((material, ctx) => { - if ( - material.task.repository !== undefined && - !sameGitObjectFormat( - material.task.repository.baseCommit, - material.task.repository.baseTree, - ) - ) { - ctx.addIssue({ - code: "custom", - path: ["task", "repository"], - message: "task Git object ids must use one object format", - }); - } - if ( - material.task.outcome.kind === "workspace" && - material.task.repository === undefined - ) { - ctx.addIssue({ - code: "custom", - path: ["task", "repository"], - message: "workspace outcomes require task repository provenance", - }); - } - if (material.attempt.number > material.attempt.maxAttempts) { - ctx.addIssue({ - code: "custom", - path: ["attempt", "number"], - message: "attempt number cannot exceed the frozen maximum", - }); - } - if ( - material.attempt.retryPolicy === "none" && - material.attempt.maxAttempts !== 1 - ) { - ctx.addIssue({ - code: "custom", - path: ["attempt", "maxAttempts"], - message: "a no-retry plan must allow exactly one attempt", - }); - } const routeIds = material.model.routes.map((route) => route.kind === "mode" || route.kind === "subagent" ? `${route.kind}:${route.name}` @@ -460,20 +462,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z } } const modelNetwork = material.model.access.network; - if (material.limits.maxModelCalls === 0 && modelNetwork.mode !== "disabled") { - ctx.addIssue({ - code: "custom", - path: ["model", "access", "network"], - message: "zero-call plans cannot expose a model gateway", - }); - } - if (material.limits.maxModelCalls > 0 && modelNetwork.mode !== "gateway-only") { - ctx.addIssue({ - code: "custom", - path: ["model", "access", "network"], - message: "model-calling plans require one frozen gateway allowlist", - }); - } if (modelNetwork.mode === "gateway-only") { addDuplicateIssues( modelNetwork.domains, @@ -515,6 +503,27 @@ export const agentCandidateExecutionPlanMaterialSchema = z message: "task and candidate workspace roots must be disjoint", }); } + const taskPath = material.launch.env.TANGLE_CANDIDATE_TASK_PATH; + if (taskPath !== undefined) { + if (taskPath.value !== "/tangle/input/task.txt") { + ctx.addIssue({ + code: "custom", + path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"], + message: "task file delivery must use the fixed evaluator-owned path", + }); + } + if ( + pathsOverlap(taskPath.value, material.workspaces.taskRoot) || + (material.workspaces.candidateRoot !== undefined && + pathsOverlap(taskPath.value, material.workspaces.candidateRoot)) + ) { + ctx.addIssue({ + code: "custom", + path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"], + message: "task file delivery must be outside both workspaces", + }); + } + } if ( material.launch.cwd.workspace === "candidate" && material.workspaces.candidateRoot === undefined @@ -536,34 +545,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z "candidate-targeted profile files require a candidate workspace root", }); } - const delivery = material.task.instruction.delivery; - const taskPath = material.launch.env.TANGLE_CANDIDATE_TASK_PATH; - if (delivery.kind === "utf8-file") { - if (taskPath?.value !== delivery.path) { - ctx.addIssue({ - code: "custom", - path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"], - message: "file delivery requires the signed fixed task path", - }); - } - if ( - pathsOverlap(material.workspaces.taskRoot, delivery.path) || - (material.workspaces.candidateRoot !== undefined && - pathsOverlap(material.workspaces.candidateRoot, delivery.path)) - ) { - ctx.addIssue({ - code: "custom", - path: ["workspaces"], - message: "the task instruction file must be outside both workspaces", - }); - } - } else if (taskPath !== undefined) { - ctx.addIssue({ - code: "custom", - path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"], - message: "non-file delivery cannot expose an instruction path", - }); - } if (activeCode && material.candidateWorkspace?.material.files.length === 0) { ctx.addIssue({ code: "custom", @@ -571,16 +552,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z message: "active candidate workspaces cannot be empty", }); } - if ( - material.task.outcome.kind === "workspace" && - material.task.workspace.material.files.length === 0 - ) { - ctx.addIssue({ - code: "custom", - path: ["task", "workspace", "material", "files"], - message: "task workspace snapshots cannot be empty", - }); - } if (!isCanonicalJsonValue(material)) { ctx.addIssue({ code: "custom", diff --git a/packages/agent-interface/src/agent-candidate-lineage-schema.ts b/packages/agent-interface/src/agent-candidate-lineage-schema.ts index ecd4026..eba7d0f 100644 --- a/packages/agent-interface/src/agent-candidate-lineage-schema.ts +++ b/packages/agent-interface/src/agent-candidate-lineage-schema.ts @@ -3,7 +3,6 @@ import type { AgentCandidateKnowledge, AgentCandidateLineage, AgentCandidateMemoryPolicy, - AgentCandidateSpend, } from "./agent-candidate.js"; import { agentCandidateArtifactRefSchema, @@ -46,16 +45,6 @@ export const agentCandidateMemoryPolicySchema = z.discriminatedUnion("mode", [ .strict(), ]) satisfies z.ZodType; -export const agentCandidateSpendSchema = z - .object({ - costUsd: z.number().finite().nonnegative(), - inputTokens: z.number().int().nonnegative(), - outputTokens: z.number().int().nonnegative(), - cachedInputTokens: z.number().int().nonnegative().optional(), - modelCalls: z.number().int().nonnegative(), - }) - .strict() satisfies z.ZodType; - export const agentCandidateLineageSchema = z .object({ source: z.enum(["optimizer", "human", "import", "compound"]), @@ -63,21 +52,7 @@ export const agentCandidateLineageSchema = z runIds: z.array(z.string().min(1)).optional(), profileDiffIds: z.array(z.string().min(1)).optional(), modelSnapshots: z.array(z.string().min(1)).optional(), - benchmark: z - .object({ - name: z.string().min(1), - version: z.string().min(1), - splitDigest: sha256DigestSchema, - }) - .strict() - .optional(), - spend: z - .object({ - proposal: agentCandidateSpendSchema, - evaluation: agentCandidateSpendSchema, - }) - .strict() - .optional(), + developmentSplitDigest: sha256DigestSchema.optional(), }) .strict() .superRefine((lineage, ctx) => { @@ -110,19 +85,12 @@ export const agentCandidateLineageSchema = z message: "generated candidates must name their producing run", }); } - if (lineage.benchmark === undefined) { + if (lineage.developmentSplitDigest === undefined) { ctx.addIssue({ code: "custom", - path: ["benchmark"], + path: ["developmentSplitDigest"], message: "generated candidates must pin their development split", }); } - if (lineage.spend === undefined) { - ctx.addIssue({ - code: "custom", - path: ["spend"], - message: "generated candidates must record proposal and evaluation spend", - }); - } } }) satisfies z.ZodType; diff --git a/packages/agent-interface/src/agent-candidate-outcome-schema.test.ts b/packages/agent-interface/src/agent-candidate-outcome-schema.test.ts index a8e081c..a4335c0 100644 --- a/packages/agent-interface/src/agent-candidate-outcome-schema.test.ts +++ b/packages/agent-interface/src/agent-candidate-outcome-schema.test.ts @@ -24,6 +24,7 @@ import { candidateGit, candidateSha, } from "./agent-candidate.test-fixture.js"; +import { canonicalCandidateDigest } from "./agent-candidate-schema-common.js"; function durableArtifact(key: string, digit: string, byteLength = 1) { return { @@ -66,11 +67,21 @@ const resolvedModel = { reasoningEffort: "high" as const, }; -function materializationReceipt(receipt: ReturnType) { +function materializationReceipt( + receipt: ReturnType, + options: { + executionId: string; + experimentDigest: ReturnType; + arm: "baseline" | "candidate"; + sourceProfileDigest: ReturnType; + repetition?: number; + }, +) { const profilePlan = { kind: "agent-profile-workspace-plan" as const, digest: candidateSha("1"), material: { + sourceProfileDigest: options.sourceProfileDigest, harness: "codex" as const, files: [], env: {}, @@ -81,39 +92,39 @@ function materializationReceipt(receipt: ReturnType) { }; const executionPlanMaterial = { kind: "agent-candidate-execution-plan-material" as const, - bundleDigest: receipt.bundleDigest, - executionId: "candidate-execution-1", - attempt: { number: 1, maxAttempts: 1, retryPolicy: "none" as const }, - task: { - benchmark: "output", - benchmarkVersion: "1.0.0", - taskId: "task-1", - splitDigest: candidateSha("2"), - instruction: { - encoding: "utf8" as const, - sha256: candidateSha("3"), - byteLength: 1, - delivery: { kind: "stdin-utf8" as const }, - }, - repository: { - identity: receipt.taskOutcome.material.outcome.baseRepository.identity, - rootIdentity: receipt.taskOutcome.material.outcome.baseRepository.rootIdentity, - baseCommit: receipt.taskOutcome.material.outcome.baseRepository.commit, - baseTree: receipt.taskOutcome.material.outcome.baseRepository.tree, - }, - outcome: { kind: "workspace" as const }, - workspace: workspaceSnapshot("input", "4"), + runCell: { + kind: "agent-candidate-run-cell" as const, + experimentDigest: options.experimentDigest, + arm: options.arm, + bundleDigest: receipt.bundleDigest, + suiteDigest: candidateSha("9"), + taskDigest: candidateSha("0"), + taskIndex: 0, + repetition: options.repetition ?? 0, + seed: 42 + (options.repetition ?? 0), + attempt: 1, + digest: receipt.runCellDigest, }, + executionId: options.executionId, workspaces: { taskRoot: "/work/task" }, codeKind: "disabled" as const, profile: { planDigest: profilePlan.digest, targetWorkspace: "task" as const, mountPaths: [] }, harness: "codex" as const, harnessVersion: "0.1.0", + instructionDelivery: { kind: "stdin-utf8" as const }, + limits: { + timeoutMs: 60_000, + maxSteps: 20, + maxModelCalls: 10, + maxInputTokens: 100_000, + maxOutputTokens: 20_000, + maxCostUsd: 5, + }, container: { source: "evaluator-task-container" as const, image: "candidate:1.0.0", - indexDigest: candidateSha("5"), - manifestDigest: candidateSha("6"), + indexDigest: candidateSha("6"), + manifestDigest: candidateSha("7"), platform: { os: "linux", architecture: "amd64" }, }, model: { @@ -121,16 +132,14 @@ function materializationReceipt(receipt: ReturnType) { resolved: resolvedModel, access: { kind: "evaluator-mediated" as const, - grantDigest: candidateSha("7"), - network: { mode: "disabled" as const }, + grantDigest: candidateSha("4"), + network: { + mode: "gateway-only" as const, + domains: ["router.tangle.tools"], + }, }, routes: [{ kind: "primary" as const, requested: "openai/gpt-5.4" }], }, - grader: { - name: "output-grader", - version: "1.0.0", - artifact: durableArtifact("graders/output.tar", "8", 100), - }, launch: { executable: "node", args: [], @@ -138,26 +147,36 @@ function materializationReceipt(receipt: ReturnType) { cwd: { workspace: "task" as const, path: "." }, }, memory: { mode: "disabled" as const }, - limits: { - timeoutMs: 1_000, - maxSteps: 1, - maxModelCalls: 0, - maxInputTokens: 0, - maxOutputTokens: 0, - maxCostUsd: 0, - }, network: { mode: "disabled" as const }, }; return { kind: "agent-candidate-materialization" as const, digestAlgorithm: "rfc8785-sha256" as const, bundleDigest: receipt.bundleDigest, - profilePlan, + benchmark: { + suite: { + digest: candidateSha("9"), + material: durableArtifact("benchmarks/suite.json", "9", 100), + }, + task: { + digest: candidateSha("0"), + material: durableArtifact("benchmarks/task-1.json", "0", 100), + }, + }, + profileActivation: { + kind: "agent-candidate-profile-activation" as const, + profilePlan, + files: [], + digest: candidateSha(options.arm === "baseline" ? "6" : "7"), + }, executionPlan: { kind: "agent-candidate-execution-plan" as const, digest: receipt.executionPlanDigest, material: executionPlanMaterial, - artifact: durableArtifact("plans/execution.json", "3", 800), + artifact: { + ...durableArtifact(`plans/${options.executionId}.json`, "3", 800), + sha256: receipt.executionPlanDigest, + }, }, codeKind: "disabled" as const, harness: "codex" as const, @@ -168,8 +187,14 @@ function materializationReceipt(receipt: ReturnType) { }; } -function runReceipt() { - const executionPlanDigest = candidateSha("3"); +function runReceipt(options: { + bundleDigest?: ReturnType; + score?: number; + startedAtMs?: number; + endedAtMs?: number; + identityDigit?: string; +} = {}) { + const executionPlanDigest = candidateSha(options.identityDigit ?? "3"); const fixedUsage = { inputTokens: 30, outputTokens: 12, @@ -192,8 +217,8 @@ function runReceipt() { traceSpanId: "span-1", status: "succeeded" as const, model: "gpt-5.4", - startedAtMs: 120, - endedAtMs: 180, + startedAtMs: 1_010, + endedAtMs: 1_030, inputTokens: 10, outputTokens: 5, cachedInputTokens: 3, @@ -206,8 +231,8 @@ function runReceipt() { traceSpanId: "span-2", status: "succeeded" as const, model: "gpt-5.4", - startedAtMs: 220, - endedAtMs: 280, + startedAtMs: 1_040, + endedAtMs: 1_060, inputTokens: 20, outputTokens: 7, cachedInputTokens: 4, @@ -257,19 +282,25 @@ function runReceipt() { kind: "agent-candidate-benchmark-result-material" as const, executionPlanDigest, taskOutcomeDigest: taskOutcome.digest, - benchmark: { - name: "pier", - version: "0.3.0", - taskId: "task-1", - splitDigest: candidateSha("9"), - }, grader: { name: "pier-executable-grader", version: "0.3.0", + format: "tangle-grader" as const, artifact: durableArtifact("graders/pier-0.3.0.tar", "a", 1_000), }, evidence: durableArtifact("results/run-1/grader-output.json", "c", 500), - score: 0.75, + grading: { + usage: { + inputTokens: 20, + outputTokens: 5, + cachedInputTokens: 0, + reasoningTokens: 0, + modelCalls: 1, + costUsdNanos: 25_000_000, + }, + timing: { startedAtMs: 1_101, endedAtMs: 1_121, durationMs: 20 }, + }, + score: options.score ?? 0.75, passed: true, dimensions: [ { name: "lint", score: 1 }, @@ -286,9 +317,15 @@ function runReceipt() { return { kind: "agent-candidate-run" as const, digestAlgorithm: "rfc8785-sha256" as const, - bundleDigest: candidateSha("c"), - materializationReceiptDigest: candidateSha("d"), + bundleDigest: options.bundleDigest ?? candidateSha("c"), + runCellDigest: candidateSha(options.identityDigit ?? "4"), + materializationReceiptDigest: candidateSha(options.identityDigit ?? "d"), executionPlanDigest, + timing: { + startedAtMs: options.startedAtMs ?? 1_000, + endedAtMs: options.endedAtMs ?? 1_100, + durationMs: (options.endedAtMs ?? 1_100) - (options.startedAtMs ?? 1_000), + }, memory: { mode: "disabled" as const }, trace: { artifact: durableArtifact("traces/run-1.json", "e", 500), @@ -300,18 +337,189 @@ function runReceipt() { modelSettlement, taskOutcome, benchmarkResult, - digest: candidateSha("f"), + digest: candidateSha(options.identityDigit ?? "f"), + }; +} + +function measuredBundle(digit: string, prompt: string) { + const fixture = candidateFixture(); + const { knowledge: _knowledge, ...withoutKnowledge } = fixture; + return { + ...withoutKnowledge, + profile: { + ...fixture.profile, + prompt: { systemPrompt: prompt }, + }, + code: { + kind: "disabled" as const, + }, + execution: { + harness: "codex" as const, + harnessVersion: "0.1.0", + launch: { + kind: "container-command" as const, + executable: "node", + args: [], + }, + instructionDelivery: { kind: "stdin-utf8" as const }, + cwd: { workspace: "task" as const, path: "." }, + environment: { kind: "evaluator-task-container" as const }, + isolation: { + network: "disabled" as const, + remoteIntegrations: "disabled" as const, + candidateSecrets: "disabled" as const, + }, + }, + memory: { mode: "disabled" as const }, + digest: candidateSha(digit), + }; +} + +function executionEvidence( + experimentDigest: ReturnType, + arm: "baseline" | "candidate", + bundle: ReturnType, + score: number, + repetition = 0, +) { + const identityDigit = arm === "baseline" ? String(repetition + 1) : String(repetition + 4); + const receipt = runReceipt({ + bundleDigest: bundle.digest, + score, + identityDigit, + }); + const executionId = `${arm}-execution-${repetition + 1}`; + const materialization = materializationReceipt(receipt, { + executionId, + experimentDigest, + arm, + sourceProfileDigest: canonicalCandidateDigest(bundle.profile), + repetition, + }); + return { + kind: "agent-candidate-execution-evidence" as const, + materializationReceipt: materialization, + receipt, + digest: candidateSha(identityDigit), }; } describe("candidate outcome contracts", () => { it("owns the current proposal, review, and receipt-bearing execution evidence", () => { - const bundle = candidateFixture(); + const baselineBundle = measuredBundle("b", "Solve the task."); + const candidateBundle = measuredBundle( + "a", + "Solve the task and verify the result.", + ); + const benchmarkTask = { + kind: "agent-candidate-benchmark-task" as const, + digestAlgorithm: "rfc8785-sha256" as const, + benchmark: { + name: "pier", + version: "0.3", + splitDigest: candidateSha("9"), + }, + scenario: { + id: "task-1", + kind: "repository-task", + scenarioDigest: candidateSha("1"), + }, + instruction: "Implement the requested repository change.", + repository: { + identity: "pier/task-1", + rootIdentity: "pier", + baseCommit: candidateGit("1"), + baseTree: candidateGit("2"), + }, + outcome: { kind: "workspace" as const }, + workspace: workspaceSnapshot("benchmark-task", "4"), + grader: { + name: "pier-executable-grader", + version: "0.3.0", + format: "tangle-grader" as const, + artifact: durableArtifact("graders/pier-0.3.0.tar", "a", 1_000), + }, + model: resolvedModel, + attempt: { maxAttempts: 1, retryPolicy: "none" as const }, + evaluatorTaskContainer: { + source: "evaluator-task-container" as const, + image: "candidate:1.0.0", + indexDigest: candidateSha("6"), + manifestDigest: candidateSha("7"), + platform: { os: "linux", architecture: "amd64" }, + }, + limits: { + timeoutMs: 60_000, + maxSteps: 20, + maxModelCalls: 10, + maxInputTokens: 100_000, + maxOutputTokens: 20_000, + maxCostUsd: 5, + }, + digest: candidateSha("0"), + }; + const suite = { + kind: "agent-candidate-benchmark-suite" as const, + digestAlgorithm: "rfc8785-sha256" as const, + taskDigests: [benchmarkTask.digest] as const, + reps: 3, + seeds: [42, 43, 44] as const, + digest: candidateSha("9"), + }; + const experimentDigest = candidateSha("e"); + const experiment = { + kind: "agent-candidate-experiment" as const, + digestAlgorithm: "rfc8785-sha256" as const, + baseline: baselineBundle, + candidate: candidateBundle, + candidateLineage: { + source: "optimizer" as const, + parentDigests: [baselineBundle.digest], + runIds: ["eval-1"], + developmentSplitDigest: candidateSha("8"), + }, + benchmark: { suite, tasks: [benchmarkTask] as const }, + policy: { + confidenceLevel: 0.95, + resamples: 2_000, + bootstrapSeed: 1_337, + deltaThreshold: 0, + minProductiveRuns: 3, + budgetUsd: 10, + criticalDimensions: ["tests"], + regressionTolerance: 0.05, + }, + digest: experimentDigest, + }; + const baselineEvidence = executionEvidence( + experimentDigest, + "baseline", + baselineBundle, + 0.5, + ); + const candidateEvidence = executionEvidence( + experimentDigest, + "candidate", + candidateBundle, + 0.75, + ); const evaluation = { kind: "agent-improvement-measured-comparison" as const, - benchmark: { name: "pier", version: "0.3", splitDigest: candidateSha("9") }, - baselineProfileDigest: candidateSha("8"), - candidateBundleDigest: bundle.digest, + experiment, + measurements: [ + { + baseline: baselineEvidence, + candidate: candidateEvidence, + }, + { + baseline: executionEvidence(experimentDigest, "baseline", baselineBundle, 0.5, 1), + candidate: executionEvidence(experimentDigest, "candidate", candidateBundle, 0.75, 1), + }, + { + baseline: executionEvidence(experimentDigest, "baseline", baselineBundle, 0.5, 2), + candidate: executionEvidence(experimentDigest, "candidate", candidateBundle, 0.75, 2), + }, + ], overall: { name: "composite" as const, baseline: 0.5, @@ -325,7 +533,7 @@ describe("candidate outcome contracts", () => { statistic: "mean" as const, resamples: 2_000, }, - n: 12, + n: 3, direction: "higher-is-better" as const, unit: "score" as const, }, @@ -345,7 +553,7 @@ describe("candidate outcome contracts", () => { statistic: "mean" as const, resamples: 2_000, }, - n: 12, + n: 3, direction: "higher-is-better" as const, unit: "score" as const, }, @@ -365,7 +573,7 @@ describe("candidate outcome contracts", () => { statistic: "mean" as const, resamples: 2_000, }, - n: 12, + n: 3, direction: "higher-is-better" as const, unit: "score" as const, }, @@ -373,18 +581,18 @@ describe("candidate outcome contracts", () => { kind: "cost" as const, name: "cost", availability: "measured" as const, - baseline: 0.01, - candidate: 0.02, - delta: 0.01, + baseline: 1.275, + candidate: 1.275, + delta: 0, confidenceInterval: { level: 0.95, lower: 0, - upper: 0.02, + upper: 0, method: "paired-bootstrap" as const, statistic: "mean" as const, resamples: 2_000, }, - n: 12, + n: 3, direction: "lower-is-better" as const, unit: "usd" as const, }, @@ -392,18 +600,18 @@ describe("candidate outcome contracts", () => { kind: "latency" as const, name: "latency", availability: "measured" as const, - baseline: 100, - candidate: 90, - delta: -10, + baseline: 120, + candidate: 120, + delta: 0, confidenceInterval: { level: 0.95, - lower: -20, + lower: 0, upper: 0, method: "paired-bootstrap" as const, statistic: "mean" as const, resamples: 2_000, }, - n: 12, + n: 3, direction: "lower-is-better" as const, unit: "milliseconds" as const, }, @@ -418,7 +626,7 @@ describe("candidate outcome contracts", () => { }, power: { sufficient: true, - n: 12, + n: 3, minimumDetectableDelta: 0.1, confidenceLevel: 0.95, scaleAssumed: true, @@ -434,24 +642,380 @@ describe("candidate outcome contracts", () => { candidateContentHash: candidateSha("3"), }, diff: "-old\n+new", - evaluation: { generationsExplored: 1, durationMs: 100, totalCostUsd: 0.5 }, + evaluation: { + generationsExplored: 1, + searchDurationMs: 40, + executionDurationMs: 60, + durationMs: 100, + searchCostUsd: 0.2, + executionCostUsd: 0.3, + totalCostUsd: 0.5, + }, }; const proposal = { kind: "agent-improvement-proposal" as const, runId: "eval-1", changedSurfaces: ["prompt"] as const, proposedAt: "2026-07-13T00:00:00.000Z", - baselineProfile: { name: "candidate" }, findings: [{ claim: "prompt omitted the requirement" }], evaluation, - candidateBundle: bundle, digest: candidateSha("2"), }; expect(agentImprovementProposalSchema.parse(proposal)).toEqual(proposal); + const firstCandidate = evaluation.measurements[0]!.candidate; + expect( + candidateExecutionEvidenceSchema.safeParse({ + ...firstCandidate, + receipt: { + ...firstCandidate.receipt, + modelSettlement: { + ...firstCandidate.receipt.modelSettlement, + material: { + ...firstCandidate.receipt.modelSettlement.material, + grantDigest: candidateSha("f"), + }, + }, + }, + }).success, + ).toBe(false); expect( agentImprovementProposalSchema.safeParse({ ...proposal, - evaluation: { ...evaluation, candidateBundleDigest: candidateSha("0") }, + evaluation: { + ...evaluation, + objectives: evaluation.objectives.map((objective) => + objective.kind === "cost" + ? { ...objective, candidate: 0.02, delta: 0.02 } + : objective, + ), + }, + }).success, + ).toBe(false); + for (const materialMutation of [ + { + limits: { + ...firstCandidate.materializationReceipt.executionPlan.material.limits, + maxCostUsd: 4, + }, + }, + { + container: { + ...firstCandidate.materializationReceipt.executionPlan.material.container, + manifestDigest: candidateSha("f"), + }, + }, + { + runCell: { + ...firstCandidate.materializationReceipt.executionPlan.material.runCell, + attempt: 2, + }, + }, + ]) { + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + measurements: evaluation.measurements.map((measurement, index) => + index === 0 + ? { + ...measurement, + candidate: { + ...measurement.candidate, + materializationReceipt: { + ...measurement.candidate.materializationReceipt, + executionPlan: { + ...measurement.candidate.materializationReceipt.executionPlan, + material: { + ...measurement.candidate.materializationReceipt.executionPlan.material, + ...materialMutation, + }, + }, + }, + }, + } + : measurement, + ), + }, + }).success, + ).toBe(false); + } + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + measurements: evaluation.measurements.map((measurement, index) => + index === 1 + ? { + ...measurement, + candidate: { + ...measurement.candidate, + receipt: { + ...measurement.candidate.receipt, + digest: evaluation.measurements[0]!.candidate.receipt.digest, + }, + }, + } + : measurement, + ), + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + experiment: { + ...experiment, + policy: { ...experiment.policy, confidenceLevel: 0.99 }, + }, + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + measurements: evaluation.measurements.map((measurement, index) => + index === 0 + ? { + ...measurement, + candidate: { + ...measurement.candidate, + receipt: { + ...measurement.candidate.receipt, + benchmarkResult: { + ...measurement.candidate.receipt.benchmarkResult, + material: { + ...measurement.candidate.receipt.benchmarkResult.material, + grader: { + ...measurement.candidate.receipt.benchmarkResult.material.grader, + version: "0.4.0", + }, + }, + }, + }, + }, + } + : measurement, + ), + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + experiment: { + ...experiment, + candidateLineage: { + source: "optimizer", + parentDigests: [baselineBundle.digest], + runIds: ["eval-1"], + }, + }, + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + experiment: { + ...experiment, + candidateLineage: { + ...experiment.candidateLineage, + developmentSplitDigest: benchmarkTask.benchmark.splitDigest, + }, + }, + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + measurements: evaluation.measurements.map((measurement) => ({ + ...measurement, + candidate: { + ...measurement.candidate, + materializationReceipt: { + ...measurement.candidate.materializationReceipt, + executionPlan: { + ...measurement.candidate.materializationReceipt.executionPlan, + material: { + ...measurement.candidate.materializationReceipt.executionPlan.material, + model: { + ...measurement.candidate.materializationReceipt.executionPlan.material.model, + access: { + ...measurement.candidate.materializationReceipt.executionPlan.material.model + .access, + network: { mode: "disabled" as const }, + }, + }, + }, + }, + }, + }, + })), + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + overall: { ...evaluation.overall, n: 12 }, + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + measurements: [ + { + ...evaluation.measurements[0], + candidate: { + ...candidateEvidence, + materializationReceipt: { + ...candidateEvidence.materializationReceipt, + codeKind: "patch" as const, + executionPlan: { + ...candidateEvidence.materializationReceipt.executionPlan, + material: { + ...candidateEvidence.materializationReceipt.executionPlan.material, + codeKind: "patch" as const, + }, + }, + }, + }, + }, + ], + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + experiment: { + ...experiment, + benchmark: { + ...experiment.benchmark, + tasks: [ + { + ...benchmarkTask, + repository: { + ...benchmarkTask.repository, + baseCommit: candidateGit("5"), + }, + }, + ] as const, + }, + }, + }, + }).success, + ).toBe(false); + const outputEvidence = ( + evidence: typeof baselineEvidence | typeof candidateEvidence, + mediaType: string, + ) => ({ + ...evidence, + receipt: { + ...evidence.receipt, + taskOutcome: { + ...evidence.receipt.taskOutcome, + material: { + ...evidence.receipt.taskOutcome.material, + outcome: { + kind: "output" as const, + spec: { mediaType, maxBytes: 100 }, + artifact: durableArtifact("outcomes/run-1/output.json", "7", 50), + }, + }, + }, + }, + }); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + experiment: { + ...experiment, + benchmark: { + ...experiment.benchmark, + tasks: [ + { + ...benchmarkTask, + repository: undefined, + outcome: { + kind: "output" as const, + mediaType: "application/json", + maxBytes: 100, + }, + }, + ] as const, + }, + }, + measurements: [ + { + ...evaluation.measurements[0], + baseline: outputEvidence(baselineEvidence, "application/json"), + candidate: outputEvidence(candidateEvidence, "text/plain"), + }, + ], + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + overall: { ...evaluation.overall, baseline: 0.9, delta: -0.15 }, + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + measurements: [ + { + ...evaluation.measurements[0], + candidate: { ...baselineEvidence, arm: "candidate" as const }, + }, + ], + }, + }).success, + ).toBe(false); + expect( + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + measurements: [ + { + ...evaluation.measurements[0], + candidate: { + ...candidateEvidence, + experimentDigest: candidateSha("0"), + }, + }, + ], + }, }).success, ).toBe(false); expect( @@ -473,7 +1037,7 @@ describe("candidate outcome contracts", () => { ), }, }).success, - ).toBe(true); + ).toBe(false); expect( agentImprovementProposalSchema.safeParse({ ...proposal, @@ -529,7 +1093,6 @@ describe("candidate outcome contracts", () => { const review = { kind: "agent-improvement-review" as const, proposalDigest: proposal.digest, - candidateBundleDigest: proposal.candidateBundle.digest, decision: "approve" as const, reviewedBy: "operator@example.com", reviewedAt: "2026-07-13T00:01:00.000Z", @@ -540,51 +1103,56 @@ describe("candidate outcome contracts", () => { expect( agentImprovementReviewSchema.safeParse({ ...review, - candidateBundleDigest: undefined, + proposalDigest: undefined, + }).success, + ).toBe(false); + expect( + agentImprovementReviewSchema.safeParse({ + ...review, + candidateBundleDigest: candidateBundle.digest, }).success, ).toBe(false); - const receipt = runReceipt(); - const materialization = materializationReceipt(receipt); - const evidence = { - kind: "agent-candidate-execution-evidence" as const, - proposalDigest: proposal.digest, - reviewDigest: review.digest, - executionId: "candidate-execution-1", - succeeded: true as const, - materializationReceipt: materialization, - profileActivation: { - kind: "agent-candidate-profile-activation" as const, - profilePlan: materialization.profilePlan, - files: [], - digest: candidateSha("6"), - }, - receipt, - digest: candidateSha("5"), - }; + const evidence = candidateEvidence; + const receipt = evidence.receipt; expect(candidateExecutionEvidenceSchema.parse(evidence)).toEqual(evidence); expect( candidateExecutionEvidenceSchema.safeParse({ ...evidence, receipt: { ...receipt, termination: { kind: "timeout", timeoutMs: 1_000 } }, }).success, - ).toBe(false); + ).toBe(true); expect( - candidateExecutionEvidenceSchema.safeParse({ - ...evidence, - receipt: { - ...receipt, - taskOutcome: { - ...receipt.taskOutcome, - material: { - ...receipt.taskOutcome.material, - outcome: { - kind: "output", - spec: { mediaType: "text/plain", maxBytes: 1024 }, - artifact: durableArtifact("outcomes/run-1/output.txt", "7", 10), + agentImprovementProposalSchema.safeParse({ + ...proposal, + evaluation: { + ...evaluation, + measurements: [ + { + ...evaluation.measurements[0], + candidate: { + ...evidence, + receipt: { + ...receipt, + taskOutcome: { + ...receipt.taskOutcome, + material: { + ...receipt.taskOutcome.material, + outcome: { + kind: "output", + spec: { mediaType: "text/plain", maxBytes: 1024 }, + artifact: durableArtifact( + "outcomes/run-1/output.txt", + "7", + 10, + ), + }, + }, + }, + }, }, }, - }, + ], }, }).success, ).toBe(false); @@ -604,7 +1172,12 @@ describe("candidate outcome contracts", () => { it("rejects obsolete schema version markers", () => { const receipt = runReceipt(); - const materialization = materializationReceipt(receipt); + const materialization = materializationReceipt(receipt, { + executionId: "candidate-execution-1", + experimentDigest: candidateSha("e"), + arm: "candidate", + sourceProfileDigest: candidateSha("f"), + }); expect( agentCandidateRunReceiptSchema.safeParse({ ...receipt, schemaVersion: 3 }).success, ).toBe(false); @@ -640,6 +1213,40 @@ describe("candidate outcome contracts", () => { trace: { ...receipt.trace, modelCallCount: 3 }, }), ).toThrow(/model-call count/); + expect(() => + agentCandidateRunReceiptSchema.parse({ + ...receipt, + modelSettlement: { + ...receipt.modelSettlement, + material: { + ...receipt.modelSettlement.material, + calls: [ + { ...receipt.modelSettlement.material.calls[0], startedAtMs: 999 }, + receipt.modelSettlement.material.calls[1], + ], + }, + }, + }), + ).toThrow(/within the recorded run/); + expect(() => + agentCandidateRunReceiptSchema.parse({ + ...receipt, + benchmarkResult: { + ...receipt.benchmarkResult, + material: { + ...receipt.benchmarkResult.material, + grading: { + ...receipt.benchmarkResult.material.grading, + timing: { + startedAtMs: 1_099, + endedAtMs: 1_119, + durationMs: 20, + }, + }, + }, + }, + }), + ).toThrow(/after candidate execution/); }); it("requires router provenance in every model settlement", () => { @@ -859,15 +1466,6 @@ describe("candidate outcome contracts", () => { passed: false, }), ).not.toThrow(); - expect(() => - agentCandidateBenchmarkResultMaterialSchema.parse({ - ...material, - grader: { - ...material.grader, - artifact: { ...material.grader.artifact, byteLength: 0 }, - }, - }), - ).toThrow(/grader bytes/); const { evidence: _evidence, ...withoutEvidence } = material; expect(() => agentCandidateBenchmarkResultMaterialSchema.parse(withoutEvidence), @@ -878,15 +1476,6 @@ describe("candidate outcome contracts", () => { evidence: { ...material.evidence, byteLength: 0 }, }), ).toThrow(/non-empty durable grading evidence/); - expect(() => - agentCandidateBenchmarkResultMaterialSchema.parse({ - ...material, - evidence: { - ...material.evidence, - sha256: material.grader.artifact.sha256, - }, - }), - ).toThrow(/distinct from the grader implementation/); expect(() => agentCandidateBenchmarkResultMaterialSchema.parse({ ...material, diff --git a/packages/agent-interface/src/agent-candidate-outcome-schema.ts b/packages/agent-interface/src/agent-candidate-outcome-schema.ts index 76f84d9..aa2d947 100644 --- a/packages/agent-interface/src/agent-candidate-outcome-schema.ts +++ b/packages/agent-interface/src/agent-candidate-outcome-schema.ts @@ -14,7 +14,10 @@ import { agentCandidateCapturedArtifactSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js"; -import { agentCandidateResolvedModelSchema } from "./agent-candidate-execution-plan-schema.js"; +import { + agentCandidateBenchmarkGraderIdentitySchema, + agentCandidateResolvedModelSchema, +} from "./agent-candidate-execution-plan-schema.js"; import { agentCandidateMediaTypeSchema, gitObjectSchema, @@ -332,35 +335,26 @@ export const agentCandidateBenchmarkResultMaterialSchema = z kind: z.literal("agent-candidate-benchmark-result-material"), executionPlanDigest: sha256DigestSchema, taskOutcomeDigest: sha256DigestSchema, - benchmark: z - .object({ - name: z.string().min(1), - version: z.string().min(1), - taskId: z.string().min(1), - splitDigest: sha256DigestSchema, - }) - .strict(), - grader: z + grader: agentCandidateBenchmarkGraderIdentitySchema, + evidence: agentCandidateArtifactRefSchema, + grading: z .object({ - name: z.string().min(1), - version: z.string().min(1), - artifact: agentCandidateArtifactRefSchema, + usage: agentCandidateFixedSpendSchema, + timing: z + .object({ + startedAtMs: safeCountSchema, + endedAtMs: safeCountSchema, + durationMs: safeCountSchema, + }) + .strict(), }) .strict(), - evidence: agentCandidateArtifactRefSchema, score: normalizedScoreSchema, passed: z.boolean(), dimensions: z.array(agentCandidateBenchmarkDimensionSchema), }) .strict() .superRefine((material, ctx) => { - if (material.grader.artifact.byteLength === 0) { - ctx.addIssue({ - code: "custom", - path: ["grader", "artifact", "byteLength"], - message: "pinned grader artifact must contain executable grader bytes", - }); - } if (material.evidence.byteLength === 0) { ctx.addIssue({ code: "custom", @@ -368,11 +362,17 @@ export const agentCandidateBenchmarkResultMaterialSchema = z message: "benchmark result must contain non-empty durable grading evidence", }); } - if (material.evidence.sha256 === material.grader.artifact.sha256) { + if ( + material.grading.timing.endedAtMs < + material.grading.timing.startedAtMs || + material.grading.timing.durationMs !== + material.grading.timing.endedAtMs - + material.grading.timing.startedAtMs + ) { ctx.addIssue({ code: "custom", - path: ["evidence", "sha256"], - message: "grading evidence must be distinct from the grader implementation", + path: ["grading", "timing"], + message: "grader timing must be ordered and internally consistent", }); } for (let index = 1; index < material.dimensions.length; index++) { diff --git a/packages/agent-interface/src/agent-candidate-promotion-schema.test.ts b/packages/agent-interface/src/agent-candidate-promotion-schema.test.ts new file mode 100644 index 0000000..e99e1db --- /dev/null +++ b/packages/agent-interface/src/agent-candidate-promotion-schema.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { canonicalCandidateDigest } from "./agent-candidate-schema-common.js"; +import { agentImprovementActivationSchema } from "./agent-candidate-promotion-schema.js"; + +const sha = (digit: string) => `sha256:${digit.repeat(64)}` as const; + +function activation() { + const material = { + kind: "agent-improvement-activation" as const, + proposalDigest: sha("1"), + reviewDigest: sha("2"), + experimentDigest: sha("3"), + candidateBundleDigest: sha("4"), + targets: [ + { + surface: "prompt" as const, + identity: "agent-profile:support", + expectedBaseDigest: sha("5"), + }, + ] as const, + fundingOwner: "tenant:test", + authorizedBy: "policy:tenant-admin", + authorizedAt: "2026-07-15T00:00:00.000Z", + }; + return { ...material, digest: canonicalCandidateDigest(material) }; +} + +describe("agentImprovementActivationSchema", () => { + it("accepts one exact activation authority receipt", () => { + expect(agentImprovementActivationSchema.parse(activation())).toEqual(activation()); + }); + + it("rejects duplicate surface identities", () => { + const input = activation(); + expect(() => + agentImprovementActivationSchema.parse({ + ...input, + targets: [...input.targets, input.targets[0]], + }), + ).toThrow(/unique by surface and identity/); + }); + + it("rejects values outside canonical JSON", () => { + expect(() => + agentImprovementActivationSchema.parse({ + ...activation(), + targets: [ + { + ...activation().targets[0], + identity: undefined, + }, + ], + }), + ).toThrow(); + }); +}); diff --git a/packages/agent-interface/src/agent-candidate-promotion-schema.ts b/packages/agent-interface/src/agent-candidate-promotion-schema.ts index dad5e0b..f733166 100644 --- a/packages/agent-interface/src/agent-candidate-promotion-schema.ts +++ b/packages/agent-interface/src/agent-candidate-promotion-schema.ts @@ -1,14 +1,18 @@ import { z } from "zod"; import type { + AgentCandidateExperiment, AgentCandidateJsonValue, AgentImprovementMeasuredComparison, + AgentImprovementActivation, AgentImprovementProposal, AgentImprovementReview, CandidateExecutionEvidence, } from "./agent-candidate.js"; import { agentCandidateBundleSchema } from "./agent-candidate-schema.js"; -import { agentCandidateProfileActivationSchema } from "./agent-candidate-execution-plan-schema.js"; +import { agentCandidateLineageSchema } from "./agent-candidate-lineage-schema.js"; +import { agentCandidateBenchmarkSuiteInputsSchema } from "./agent-candidate-task-schema.js"; import { + canonicalCandidateDigest, isCanonicalJsonValue, sha256DigestSchema, } from "./agent-candidate-schema-common.js"; @@ -16,7 +20,6 @@ import { agentCandidateMaterializationReceiptSchema, agentCandidateRunReceiptSchema, } from "./agent-candidate-receipt-schema.js"; -import { agentProfileSchema } from "./profile-schema.js"; const canonicalJsonSchema = z.custom( isCanonicalJsonValue, @@ -97,9 +100,7 @@ const measuredObjectiveSchema = z.union([ measuredObjectiveVariant(qualityDimensionFields), unavailableObjectiveVariant(qualityDimensionFields), measuredObjectiveVariant(costObjectiveFields), - unavailableObjectiveVariant(costObjectiveFields), measuredObjectiveVariant(latencyObjectiveFields), - unavailableObjectiveVariant(latencyObjectiveFields), ]); const improvementSurfaceSchema = z.enum([ @@ -115,18 +116,154 @@ const improvementSurfaceSchema = z.enum([ "knowledge", ]); +export const agentCandidateEvaluationPolicySchema = z + .object({ + confidenceLevel: z.number().finite().gt(0).lt(1), + resamples: z.number().int().min(100), + bootstrapSeed: z.number().int().safe(), + deltaThreshold: z.number().finite().nonnegative(), + minProductiveRuns: z.number().int().min(3), + budgetUsd: z.number().finite().nonnegative().optional(), + criticalDimensions: z.array(z.string().min(1)), + regressionTolerance: z.number().finite().nonnegative(), + }) + .strict() + .superRefine((policy, ctx) => { + if ( + new Set(policy.criticalDimensions).size !== policy.criticalDimensions.length || + policy.criticalDimensions.some( + (name, index) => index > 0 && policy.criticalDimensions[index - 1]! >= name, + ) + ) { + ctx.addIssue({ + code: "custom", + path: ["criticalDimensions"], + message: "critical dimensions must be sorted and unique", + }); + } + }); + +export const agentCandidateExperimentSchema = z + .object({ + kind: z.literal("agent-candidate-experiment"), + digestAlgorithm: z.literal("rfc8785-sha256"), + baseline: agentCandidateBundleSchema, + candidate: agentCandidateBundleSchema, + candidateLineage: agentCandidateLineageSchema, + benchmark: agentCandidateBenchmarkSuiteInputsSchema, + policy: agentCandidateEvaluationPolicySchema, + digest: sha256DigestSchema, + }) + .strict() + .superRefine((experiment, ctx) => { + const source = experiment.candidateLineage.source; + if ( + (source === "optimizer" || source === "compound") && + !experiment.candidateLineage.parentDigests?.includes(experiment.baseline.digest) + ) { + ctx.addIssue({ + code: "custom", + path: ["candidateLineage", "parentDigests"], + message: "generated candidate lineage must include the experiment baseline", + }); + } + if (experiment.candidateLineage.parentDigests?.includes(experiment.candidate.digest)) { + ctx.addIssue({ + code: "custom", + path: ["candidateLineage", "parentDigests"], + message: "candidate lineage cannot name the candidate itself as a parent", + }); + } + if ( + experiment.candidateLineage.developmentSplitDigest !== undefined && + experiment.benchmark.tasks.some( + (task) => + task.benchmark.splitDigest === + experiment.candidateLineage.developmentSplitDigest, + ) + ) { + ctx.addIssue({ + code: "custom", + path: ["candidateLineage", "developmentSplitDigest"], + message: "candidate development and held-out splits must be disjoint", + }); + } + if (!isCanonicalJsonValue(experiment)) { + ctx.addIssue({ + code: "custom", + message: "candidate experiment must contain only RFC 8785 JSON values", + }); + } + }) satisfies z.ZodType; + +export const candidateExecutionEvidenceSchema = z + .object({ + kind: z.literal("agent-candidate-execution-evidence"), + materializationReceipt: agentCandidateMaterializationReceiptSchema, + receipt: agentCandidateRunReceiptSchema, + digest: sha256DigestSchema, + }) + .strict() + .superRefine((evidence, ctx) => { + const materialization = evidence.materializationReceipt; + const plan = materialization.executionPlan; + const checks: Array<[boolean, (string | number)[], string]> = [ + [ + evidence.receipt.materializationReceiptDigest === materialization.digest, + ["receipt", "materializationReceiptDigest"], + "run receipt must bind the included materialization receipt", + ], + [ + evidence.receipt.executionPlanDigest === plan.digest, + ["receipt", "executionPlanDigest"], + "run receipt must bind the included execution plan", + ], + [ + evidence.receipt.bundleDigest === materialization.bundleDigest, + ["receipt", "bundleDigest"], + "run receipt and materialization must bind one bundle", + ], + [ + evidence.receipt.runCellDigest === plan.material.runCell.digest, + ["receipt", "runCellDigest"], + "run receipt must bind the materialized run cell", + ], + [ + materialization.profileActivation.profilePlan.digest === + plan.material.profile.planDigest, + ["materializationReceipt", "profileActivation", "profilePlan", "digest"], + "profile activation must bind the materialized execution plan", + ], + [ + evidence.receipt.modelSettlement.material.grantDigest === + plan.material.model.access.grantDigest, + ["receipt", "modelSettlement", "material", "grantDigest"], + "model settlement must bind the execution plan grant", + ], + ]; + for (const [valid, path, message] of checks) { + if (!valid) ctx.addIssue({ code: "custom", path, message }); + } + if (!isCanonicalJsonValue(evidence)) { + ctx.addIssue({ + code: "custom", + message: "candidate execution evidence must contain only RFC 8785 JSON values", + }); + } + }) satisfies z.ZodType; + export const agentImprovementMeasuredComparisonSchema = z .object({ kind: z.literal("agent-improvement-measured-comparison"), - benchmark: z - .object({ - name: z.string().min(1), - version: z.string().min(1), - splitDigest: sha256DigestSchema, - }) - .strict(), - baselineProfileDigest: sha256DigestSchema, - candidateBundleDigest: sha256DigestSchema, + experiment: agentCandidateExperimentSchema, + measurements: z.array( + z + .object({ + baseline: candidateExecutionEvidenceSchema, + candidate: candidateExecutionEvidenceSchema, + }) + .strict(), + ), overall: z .object({ name: z.literal("composite"), @@ -187,7 +324,11 @@ export const agentImprovementMeasuredComparisonSchema = z evaluation: z .object({ generationsExplored: z.number().int().nonnegative(), + searchDurationMs: z.number().finite().nonnegative(), + executionDurationMs: z.number().finite().nonnegative(), durationMs: z.number().finite().nonnegative(), + searchCostUsd: z.number().finite().nonnegative(), + executionCostUsd: z.number().finite().nonnegative(), totalCostUsd: z.number().finite().nonnegative(), }) .strict(), @@ -196,6 +337,276 @@ export const agentImprovementMeasuredComparisonSchema = z .strict() .superRefine((comparison, ctx) => { refineEstimate(comparison.overall, ["overall"], ctx); + if ( + !approximatelyEqual( + comparison.evaluation.durationMs, + comparison.evaluation.searchDurationMs + comparison.evaluation.executionDurationMs, + ) || + !approximatelyEqual( + comparison.evaluation.totalCostUsd, + comparison.evaluation.searchCostUsd + comparison.evaluation.executionCostUsd, + ) + ) { + ctx.addIssue({ + code: "custom", + path: ["evaluation"], + message: "evaluation totals must equal their search and execution components", + }); + } + const { suite, tasks } = comparison.experiment.benchmark; + const expectedN = suite.taskDigests.length * suite.reps; + if (comparison.measurements.length !== expectedN) { + ctx.addIssue({ + code: "custom", + path: ["measurements"], + message: "measured comparison must contain every signed benchmark cell", + }); + } + const executionIdentities = { + execution: new Set(), + runCell: new Set(), + materialization: new Set(), + receipt: new Set(), + evidence: new Set(), + }; + for (let taskIndex = 0; taskIndex < suite.taskDigests.length; taskIndex += 1) { + const task = tasks[taskIndex]; + if (!task) continue; + for (let repetition = 0; repetition < suite.reps; repetition += 1) { + const index = taskIndex * suite.reps + repetition; + const measurement = comparison.measurements[index]; + if (!measurement) continue; + const seed = suite.seeds[index]; + for (const arm of ["baseline", "candidate"] as const) { + const evidence = measurement[arm]; + const bundle = comparison.experiment[arm]; + const materialization = evidence.materializationReceipt; + const plan = materialization.executionPlan; + const runCell = plan.material.runCell; + const result = evidence.receipt.benchmarkResult.material; + const outcome = evidence.receipt.taskOutcome.material.outcome; + const armPath = ["measurements", index, arm] as (string | number)[]; + const expectedTree = + bundle.code.kind === "disabled" + ? undefined + : bundle.code.kind === "no-op" + ? bundle.code.baseTree + : bundle.code.candidateTree; + const containerMatches = + bundle.execution.environment.kind === "evaluator-task-container" + ? task.evaluatorTaskContainer !== undefined && + plan.material.container.source === "evaluator-task-container" && + JSON.stringify(plan.material.container) === + JSON.stringify(task.evaluatorTaskContainer) + : plan.material.container.source === "pinned-container" && + plan.material.container.image === + bundle.execution.environment.container.image && + plan.material.container.indexDigest === + bundle.execution.environment.container.indexDigest; + const checks: Array<[boolean, (string | number)[], string]> = [ + [ + runCell.experimentDigest === comparison.experiment.digest, + [...armPath, "materializationReceipt", "executionPlan", "material", "runCell", "experimentDigest"], + "execution evidence must bind the measured experiment", + ], + [ + runCell.arm === arm, + [...armPath, "materializationReceipt", "executionPlan", "material", "runCell", "arm"], + "execution evidence must bind its measured arm", + ], + [ + runCell.bundleDigest === bundle.digest && materialization.bundleDigest === bundle.digest, + [...armPath, "materializationReceipt", "bundleDigest"], + "execution evidence must bind the experiment arm bundle", + ], + [ + runCell.suiteDigest === suite.digest && + runCell.taskDigest === task.digest && + runCell.taskIndex === taskIndex && + runCell.repetition === repetition && + runCell.seed === seed && + runCell.attempt === 1, + [...armPath, "materializationReceipt", "executionPlan", "material", "runCell"], + "publishable execution evidence must use the first signed task attempt", + ], + [ + materialization.codeKind === bundle.code.kind, + [...armPath, "materializationReceipt", "codeKind"], + "materialized code must match the experiment arm bundle", + ], + [ + materialization.benchmark.suite.digest === suite.digest && + materialization.benchmark.task.digest === task.digest, + [...armPath, "materializationReceipt", "benchmark"], + "execution evidence must capture the signed suite and selected task", + ], + [ + materialization.harness === bundle.execution.harness && + materialization.harnessVersion === bundle.execution.harnessVersion && + plan.material.harness === bundle.execution.harness && + plan.material.harnessVersion === bundle.execution.harnessVersion, + [...armPath, "materializationReceipt", "harness"], + "execution evidence must bind the candidate harness and version", + ], + [ + JSON.stringify(plan.material.instructionDelivery) === + JSON.stringify(bundle.execution.instructionDelivery), + [...armPath, "materializationReceipt", "executionPlan", "material", "instructionDelivery"], + "execution plan must bind the candidate instruction delivery", + ], + [ + JSON.stringify(plan.material.limits) === JSON.stringify(task.limits), + [...armPath, "materializationReceipt", "executionPlan", "material", "limits"], + "execution plan must bind every signed task limit", + ], + [ + containerMatches, + [...armPath, "materializationReceipt", "executionPlan", "material", "container"], + "execution plan must bind the candidate or evaluator task container", + ], + [ + JSON.stringify(plan.material.candidateWorkspace) === + JSON.stringify(bundle.execution.workspace) && + JSON.stringify(materialization.candidateWorkspace) === + JSON.stringify(bundle.execution.workspace), + [...armPath, "materializationReceipt", "candidateWorkspace"], + "execution evidence must bind the candidate workspace", + ], + [ + materialization.materializedTree === expectedTree, + [...armPath, "materializationReceipt", "materializedTree"], + "materialized tree must match the candidate code", + ], + [ + plan.material.launch.cwd.workspace === bundle.execution.cwd.workspace && + plan.material.launch.cwd.path === bundle.execution.cwd.path, + [...armPath, "materializationReceipt", "executionPlan", "material", "launch", "cwd"], + "execution plan must bind the candidate working directory", + ], + [ + plan.material.knowledgeManifestDigest === bundle.knowledge?.snapshot.digest && + materialization.knowledgeManifestDigest === bundle.knowledge?.snapshot.digest, + [...armPath, "materializationReceipt", "knowledgeManifestDigest"], + "execution evidence must bind the candidate knowledge snapshot", + ], + [ + (bundle.memory.mode === "disabled" && plan.material.memory.mode === "disabled") || + (bundle.memory.mode === "isolated" && + plan.material.memory.mode === "isolated" && + plan.material.memory.seedDigest === bundle.memory.seed?.sha256), + [...armPath, "materializationReceipt", "executionPlan", "material", "memory"], + "execution plan must bind the candidate memory policy", + ], + [ + result.evidence.sha256 !== task.grader.artifact.sha256, + [...armPath, "receipt", "benchmarkResult", "material", "evidence", "sha256"], + "grading evidence must be distinct from the signed grader implementation", + ], + [ + JSON.stringify(result.grader) === JSON.stringify(task.grader), + [...armPath, "receipt", "benchmarkResult", "material", "grader"], + "benchmark result must bind the signed grader", + ], + [ + materialization.profileActivation.profilePlan.material.sourceProfileDigest === + canonicalCandidateDigest(bundle.profile as AgentCandidateJsonValue), + [...armPath, "materializationReceipt", "profileActivation", "profilePlan", "material", "sourceProfileDigest"], + "materialized profile files must bind the experiment arm profile", + ], + [ + JSON.stringify(materialization.resolvedModel) === JSON.stringify(task.model), + [...armPath, "materializationReceipt", "resolvedModel"], + "execution must use the selected task model", + ], + [ + (task.limits.maxModelCalls === 0 && + materialization.executionPlan.material.model.access.network.mode === + "disabled") || + (task.limits.maxModelCalls > 0 && + materialization.executionPlan.material.model.access.network.mode === + "gateway-only"), + [ + ...armPath, + "materializationReceipt", + "executionPlan", + "material", + "model", + "access", + "network", + ], + "model gateway access must match the signed model-call limit", + ], + [ + outcome.kind === task.outcome.kind, + [...armPath, "receipt", "taskOutcome", "material", "outcome", "kind"], + "captured outcome must match the selected task contract", + ], + [ + task.outcome.kind !== "output" || + (outcome.kind === "output" && + outcome.spec.mediaType === task.outcome.mediaType && + outcome.spec.maxBytes === task.outcome.maxBytes), + [...armPath, "receipt", "taskOutcome", "material", "outcome", "spec"], + "captured output must match the selected task specification", + ], + [ + task.outcome.kind !== "workspace" || + (outcome.kind === "workspace" && + task.repository !== undefined && + outcome.baseRepository.identity === task.repository.identity && + outcome.baseRepository.rootIdentity === task.repository.rootIdentity && + outcome.baseRepository.commit === task.repository.baseCommit && + outcome.baseRepository.tree === task.repository.baseTree), + [...armPath, "receipt", "taskOutcome", "material", "outcome", "baseRepository"], + "captured workspace must start from the selected task repository", + ], + ]; + for (const [valid, path, message] of checks) { + if (!valid) ctx.addIssue({ code: "custom", path, message }); + } + const identitiesForRun = { + execution: plan.material.executionId, + runCell: runCell.digest, + materialization: materialization.digest, + receipt: evidence.receipt.digest, + evidence: evidence.digest, + }; + for (const [kind, identity] of Object.entries(identitiesForRun) as Array< + [keyof typeof executionIdentities, string] + >) { + if (executionIdentities[kind].has(identity)) { + ctx.addIssue({ + code: "custom", + path: armPath, + message: `measured executions must not reuse ${kind} identity`, + }); + } + executionIdentities[kind].add(identity); + } + } + } + } + if (comparison.overall.n !== expectedN) { + ctx.addIssue({ + code: "custom", + path: ["overall", "n"], + message: "measured sample count must equal the complete benchmark suite", + }); + } + if (comparison.measurements.length > 0) { + refineMeasuredMean( + comparison.overall.baseline, + comparison.measurements.map((row) => row.baseline.receipt.benchmarkResult.material.score), + ["overall", "baseline"], + ctx, + ); + refineMeasuredMean( + comparison.overall.candidate, + comparison.measurements.map((row) => row.candidate.receipt.benchmarkResult.material.score), + ["overall", "candidate"], + ctx, + ); + } const identities = new Set(); const qualityObjectives = new Set(); const dimensionParents: Array<{ index: number; objective: string }> = []; @@ -204,6 +615,41 @@ export const agentImprovementMeasuredComparisonSchema = z for (const [index, objective] of comparison.objectives.entries()) { if (objective.availability === "measured") { refineEstimate(objective, ["objectives", index], ctx); + if (objective.n !== expectedN) { + ctx.addIssue({ + code: "custom", + path: ["objectives", index, "n"], + message: "measured objective count must equal the complete benchmark suite", + }); + } + if (comparison.measurements.length > 0 && objective.kind === "cost") { + refineMeasuredMean( + objective.baseline, + comparison.measurements.map((row) => executionCostUsd(row.baseline)), + ["objectives", index, "baseline"], + ctx, + ); + refineMeasuredMean( + objective.candidate, + comparison.measurements.map((row) => executionCostUsd(row.candidate)), + ["objectives", index, "candidate"], + ctx, + ); + } + if (comparison.measurements.length > 0 && objective.kind === "latency") { + refineMeasuredMean( + objective.baseline, + comparison.measurements.map((row) => executionLatencyMs(row.baseline)), + ["objectives", index, "baseline"], + ctx, + ); + refineMeasuredMean( + objective.candidate, + comparison.measurements.map((row) => executionLatencyMs(row.candidate)), + ["objectives", index, "candidate"], + ctx, + ); + } } const identity = objective.kind === "dimension" @@ -257,6 +703,33 @@ export const agentImprovementMeasuredComparisonSchema = z message: "power analysis must use the paired held-out sample", }); } + if ( + comparison.overall.confidenceInterval.level !== + comparison.experiment.policy.confidenceLevel || + comparison.overall.confidenceInterval.resamples !== + comparison.experiment.policy.resamples || + comparison.power.confidenceLevel !== comparison.experiment.policy.confidenceLevel + ) { + ctx.addIssue({ + code: "custom", + path: ["experiment", "policy"], + message: "reported uncertainty must use the frozen evaluation policy", + }); + } + for (const [index, objective] of comparison.objectives.entries()) { + if ( + objective.availability === "measured" && + (objective.confidenceInterval.level !== + comparison.experiment.policy.confidenceLevel || + objective.confidenceInterval.resamples !== comparison.experiment.policy.resamples) + ) { + ctx.addIssue({ + code: "custom", + path: ["objectives", index, "confidenceInterval"], + message: "objective uncertainty must use the frozen evaluation policy", + }); + } + } if (!isCanonicalJsonValue(comparison)) { ctx.addIssue({ code: "custom", @@ -277,33 +750,38 @@ export const agentImprovementProposalSchema = z "changed surfaces must be unique", ), proposedAt: z.iso.datetime(), - baselineProfile: agentProfileSchema, findings: z.array(canonicalJsonObjectSchema), evaluation: agentImprovementMeasuredComparisonSchema, - candidateBundle: agentCandidateBundleSchema, digest: sha256DigestSchema, }) .strict() .superRefine((proposal, ctx) => { - const measuredBenchmark = proposal.evaluation.benchmark; - const bundleBenchmark = proposal.candidateBundle.lineage.benchmark; + if (proposal.evaluation.decision.outcome !== "ship") { + ctx.addIssue({ + code: "custom", + path: ["evaluation", "decision", "outcome"], + message: "an improvement proposal requires a passing measured comparison", + }); + } if ( - !bundleBenchmark || - measuredBenchmark.name !== bundleBenchmark.name || - measuredBenchmark.version !== bundleBenchmark.version || - measuredBenchmark.splitDigest !== bundleBenchmark.splitDigest + !proposal.evaluation.power.sufficient || + proposal.evaluation.overall.n < + proposal.evaluation.experiment.policy.minProductiveRuns ) { ctx.addIssue({ code: "custom", - path: ["evaluation", "benchmark"], - message: "measured comparison must bind the candidate development split", + path: ["evaluation", "power"], + message: "an improvement proposal requires sufficient pre-registered power", }); } - if (proposal.evaluation.candidateBundleDigest !== proposal.candidateBundle.digest) { + if ( + proposal.evaluation.experiment.baseline.digest === + proposal.evaluation.experiment.candidate.digest + ) { ctx.addIssue({ code: "custom", - path: ["evaluation", "candidateBundleDigest"], - message: "measured comparison must bind the exact candidate bundle", + path: ["evaluation", "experiment", "candidate", "digest"], + message: "an improvement proposal requires a changed candidate bundle", }); } if (!isCanonicalJsonValue(proposal)) { @@ -346,11 +824,46 @@ function refineEstimate( } } +function refineMeasuredMean( + reported: number, + values: number[], + path: (string | number)[], + ctx: z.RefinementCtx, +): void { + const measured = values.reduce((sum, value) => sum + value, 0) / values.length; + const tolerance = Number.EPSILON * Math.max(1, Math.abs(measured)) * values.length * 8; + if (Math.abs(reported - measured) > tolerance) { + ctx.addIssue({ + code: "custom", + path, + message: "reported mean must equal the signed per-cell results", + }); + } +} + +function approximatelyEqual(left: number, right: number): boolean { + const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 16; + return Math.abs(left - right) <= tolerance; +} + +function executionCostUsd(evidence: CandidateExecutionEvidence): number { + return ( + evidence.receipt.modelSettlement.material.usage.costUsdNanos + + evidence.receipt.benchmarkResult.material.grading.usage.costUsdNanos + ) / 1_000_000_000; +} + +function executionLatencyMs(evidence: CandidateExecutionEvidence): number { + return ( + evidence.receipt.timing.durationMs + + evidence.receipt.benchmarkResult.material.grading.timing.durationMs + ); +} + export const agentImprovementReviewSchema = z .object({ kind: z.literal("agent-improvement-review"), proposalDigest: sha256DigestSchema, - candidateBundleDigest: sha256DigestSchema, decision: z.enum(["approve", "reject", "request-changes"]), reviewedBy: z.string().min(1), reviewedAt: z.iso.datetime(), @@ -361,94 +874,50 @@ export const agentImprovementReviewSchema = z .strict() .refine(isCanonicalJsonValue, "review must contain only RFC 8785 JSON values") satisfies z.ZodType; -export const candidateExecutionEvidenceSchema = z +export const agentImprovementActivationSchema = z .object({ - kind: z.literal("agent-candidate-execution-evidence"), + kind: z.literal("agent-improvement-activation"), proposalDigest: sha256DigestSchema, reviewDigest: sha256DigestSchema, - executionId: z.string().regex(/^[A-Za-z0-9._:-]{1,200}$/), - succeeded: z.literal(true), - materializationReceipt: agentCandidateMaterializationReceiptSchema, - profileActivation: agentCandidateProfileActivationSchema, - receipt: agentCandidateRunReceiptSchema, + experimentDigest: sha256DigestSchema, + candidateBundleDigest: sha256DigestSchema, + targets: z + .tuple([ + z + .object({ + surface: improvementSurfaceSchema, + identity: z.string().min(1).max(500), + expectedBaseDigest: sha256DigestSchema, + }) + .strict(), + ]) + .rest( + z + .object({ + surface: improvementSurfaceSchema, + identity: z.string().min(1).max(500), + expectedBaseDigest: sha256DigestSchema, + }) + .strict(), + ), + fundingOwner: z.string().min(1).max(500), + authorizedBy: z.string().min(1).max(500), + authorizedAt: z.iso.datetime(), digest: sha256DigestSchema, }) .strict() - .superRefine((evidence, ctx) => { - const materialization = evidence.materializationReceipt; - const plan = materialization.executionPlan; - const plannedOutcome = plan.material.task.outcome; - const capturedOutcome = evidence.receipt.taskOutcome.material.outcome; - const checks: Array<[boolean, (string | number)[], string]> = [ - [ - evidence.receipt.materializationReceiptDigest === materialization.digest, - ["receipt", "materializationReceiptDigest"], - "run receipt must bind the included materialization receipt", - ], - [ - evidence.receipt.executionPlanDigest === plan.digest, - ["receipt", "executionPlanDigest"], - "run receipt must bind the included execution plan", - ], - [ - evidence.receipt.bundleDigest === materialization.bundleDigest, - ["receipt", "bundleDigest"], - "run receipt and materialization must bind one bundle", - ], - [ - evidence.executionId === plan.material.executionId, - ["executionId"], - "execution evidence must bind the materialized execution id", - ], - [ - evidence.profileActivation.profilePlan.digest === - materialization.profilePlan.digest, - ["profileActivation", "profilePlan", "digest"], - "profile activation must bind the materialized profile plan", - ], - [ - capturedOutcome.kind === plannedOutcome.kind, - ["receipt", "taskOutcome", "material", "outcome", "kind"], - "task outcome kind must match the signed execution plan", - ], - [ - evidence.receipt.termination.kind === "exit" && - evidence.receipt.termination.exitCode === 0, - ["receipt", "termination"], - "successful execution evidence requires a zero exit status", - ], - ]; - if (plannedOutcome.kind === "output" && capturedOutcome.kind === "output") { - checks.push([ - capturedOutcome.spec.mediaType === plannedOutcome.mediaType && - capturedOutcome.spec.maxBytes === plannedOutcome.maxBytes, - ["receipt", "taskOutcome", "material", "outcome", "spec"], - "task output constraints must match the signed execution plan", - ]); - } - if ( - plannedOutcome.kind === "workspace" && - capturedOutcome.kind === "workspace" && - plan.material.task.repository - ) { - const repository = plan.material.task.repository; - const base = capturedOutcome.baseRepository; - checks.push([ - base.identity === repository.identity && - base.rootIdentity === repository.rootIdentity && - base.commit === repository.baseCommit && - base.tree === repository.baseTree, - ["receipt", "taskOutcome", "material", "outcome", "baseRepository"], - "workspace outcome must bind the signed repository base", - ]); - } - for (const [valid, path, message] of checks) { - if (!valid) ctx.addIssue({ code: "custom", path, message }); - } - if (!isCanonicalJsonValue(evidence)) { + .superRefine((activation, ctx) => { + const identities = activation.targets.map( + (target) => `${target.surface}\u0000${target.identity}`, + ); + if (new Set(identities).size !== identities.length) { ctx.addIssue({ code: "custom", - message: "candidate execution evidence must contain only RFC 8785 JSON values", + path: ["targets"], + message: "activation targets must be unique by surface and identity", }); } - }) satisfies z.ZodType; + if (!isCanonicalJsonValue(activation)) { + ctx.addIssue({ code: "custom", message: "activation must contain only RFC 8785 JSON values" }); + } + }) satisfies z.ZodType; diff --git a/packages/agent-interface/src/agent-candidate-receipt-schema.ts b/packages/agent-interface/src/agent-candidate-receipt-schema.ts index d76072d..121aecb 100644 --- a/packages/agent-interface/src/agent-candidate-receipt-schema.ts +++ b/packages/agent-interface/src/agent-candidate-receipt-schema.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import type { + AgentCandidateBenchmarkInputEvidence, AgentCandidateMaterializationReceipt, AgentCandidateMemoryReceipt, AgentCandidateRunReceipt, @@ -13,7 +14,7 @@ import { } from "./agent-candidate-artifact-schema.js"; import { agentCandidateExecutionPlanEvidenceSchema, - agentCandidateProfilePlanEvidenceSchema, + agentCandidateProfileActivationSchema, agentCandidateResolvedModelSchema, } from "./agent-candidate-execution-plan-schema.js"; import { @@ -50,6 +51,36 @@ const ociPlatformSchema = z }) .strict(); +const benchmarkMaterialEvidenceSchema = z + .object({ + digest: sha256DigestSchema, + material: agentCandidateCapturedArtifactSchema, + }) + .strict() + .superRefine((evidence, ctx) => { + if (evidence.material.sha256 !== evidence.digest) { + ctx.addIssue({ + code: "custom", + path: ["material", "sha256"], + message: "benchmark material artifact must match its canonical digest", + }); + } + if (evidence.material.byteLength === 0) { + ctx.addIssue({ + code: "custom", + path: ["material", "byteLength"], + message: "benchmark material artifact must contain canonical bytes", + }); + } + }); + +export const agentCandidateBenchmarkInputEvidenceSchema = z + .object({ + suite: benchmarkMaterialEvidenceSchema, + task: benchmarkMaterialEvidenceSchema, + }) + .strict() satisfies z.ZodType; + export const agentCandidateTraceEvidenceSchema = z .object({ artifact: agentCandidateCapturedArtifactSchema, @@ -110,7 +141,8 @@ export const agentCandidateMaterializationReceiptSchema = z kind: z.literal("agent-candidate-materialization"), digestAlgorithm: z.literal("rfc8785-sha256"), bundleDigest: sha256DigestSchema, - profilePlan: agentCandidateProfilePlanEvidenceSchema, + benchmark: agentCandidateBenchmarkInputEvidenceSchema, + profileActivation: agentCandidateProfileActivationSchema, executionPlan: agentCandidateExecutionPlanEvidenceSchema, candidateWorkspace: agentCandidateWorkspaceSnapshotEvidenceSchema.optional(), codeKind: z.enum(["disabled", "no-op", "git-patch"]), @@ -172,19 +204,29 @@ export const agentCandidateMaterializationReceiptSchema = z } const checks: Array<[boolean, (string | number)[], string]> = [ [ - receipt.bundleDigest === plan.bundleDigest, - ["executionPlan", "material", "bundleDigest"], + receipt.bundleDigest === plan.runCell.bundleDigest, + ["executionPlan", "material", "runCell", "bundleDigest"], "execution plan must bind the receipt bundle", ], [ - receipt.profilePlan.digest === plan.profile.planDigest, + receipt.benchmark.suite.digest === plan.runCell.suiteDigest, + ["benchmark", "suite", "digest"], + "execution plan must bind the captured benchmark suite", + ], + [ + receipt.benchmark.task.digest === plan.runCell.taskDigest, + ["benchmark", "task", "digest"], + "execution plan must bind the captured benchmark task", + ], + [ + receipt.profileActivation.profilePlan.digest === plan.profile.planDigest, ["executionPlan", "material", "profile", "planDigest"], "execution plan must bind the exact profile plan", ], [ JSON.stringify(plan.profile.mountPaths) === JSON.stringify( - receipt.profilePlan.material.files.map((file) => file.relPath), + receipt.profileActivation.profilePlan.material.files.map((file) => file.relPath), ), ["executionPlan", "material", "profile", "mountPaths"], "execution plan must bind every profile mount path", @@ -230,8 +272,8 @@ export const agentCandidateMaterializationReceiptSchema = z "execution plan must bind the exact uploaded candidate workspace", ], [ - receipt.profilePlan.material.harness === receipt.harness, - ["profilePlan", "material", "harness"], + receipt.profileActivation.profilePlan.material.harness === receipt.harness, + ["profileActivation", "profilePlan", "material", "harness"], "profile plan harness must match materialization", ], ]; @@ -251,8 +293,16 @@ export const agentCandidateRunReceiptSchema = z kind: z.literal("agent-candidate-run"), digestAlgorithm: z.literal("rfc8785-sha256"), bundleDigest: sha256DigestSchema, + runCellDigest: sha256DigestSchema, materializationReceiptDigest: sha256DigestSchema, executionPlanDigest: sha256DigestSchema, + timing: z + .object({ + startedAtMs: z.number().int().nonnegative().safe(), + endedAtMs: z.number().int().nonnegative().safe(), + durationMs: z.number().int().nonnegative().safe(), + }) + .strict(), memory: agentCandidateMemoryReceiptSchema, trace: agentCandidateTraceEvidenceSchema, termination: agentCandidateTerminationSchema, @@ -264,6 +314,17 @@ export const agentCandidateRunReceiptSchema = z }) .strict() .superRefine((receipt, ctx) => { + if ( + receipt.timing.endedAtMs < receipt.timing.startedAtMs || + receipt.timing.durationMs !== + receipt.timing.endedAtMs - receipt.timing.startedAtMs + ) { + ctx.addIssue({ + code: "custom", + path: ["timing"], + message: "run timing must be ordered and internally consistent", + }); + } if ( receipt.modelSettlement.material.executionPlanDigest !== receipt.executionPlanDigest @@ -274,6 +335,28 @@ export const agentCandidateRunReceiptSchema = z message: "model settlement must bind the executed plan", }); } + for (const [index, call] of receipt.modelSettlement.material.calls.entries()) { + if ( + call.startedAtMs < receipt.timing.startedAtMs || + call.endedAtMs > receipt.timing.endedAtMs + ) { + ctx.addIssue({ + code: "custom", + path: ["modelSettlement", "material", "calls", index], + message: "model calls must occur within the recorded run", + }); + } + } + if ( + receipt.benchmarkResult.material.grading.timing.startedAtMs < + receipt.timing.endedAtMs + ) { + ctx.addIssue({ + code: "custom", + path: ["benchmarkResult", "material", "grading", "timing", "startedAtMs"], + message: "grading must start after candidate execution ends", + }); + } if ( receipt.trace.modelCallCount !== receipt.modelSettlement.material.usage.modelCalls diff --git a/packages/agent-interface/src/agent-candidate-schema-common.ts b/packages/agent-interface/src/agent-candidate-schema-common.ts index cbecd75..dbf3b3b 100644 --- a/packages/agent-interface/src/agent-candidate-schema-common.ts +++ b/packages/agent-interface/src/agent-candidate-schema-common.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import type { AgentCandidateConfigValue, AgentCandidateGitHubRepository, + AgentCandidateJsonValue, Sha256Digest, } from "./agent-candidate.js"; @@ -44,11 +45,59 @@ export const headerNameSchema = z.string().regex(headerNamePattern); export const agentCandidateMediaTypeSchema = z.string().regex(mediaTypePattern); export function sha256Utf8(value: string): Sha256Digest { - const bytes = sha256(new TextEncoder().encode(value)); + return sha256Bytes(new TextEncoder().encode(value)); +} + +export function sha256Bytes(value: Uint8Array): Sha256Digest { + const bytes = sha256(value); const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); return `sha256:${hex}`; } +/** RFC 8785 serialization used by every candidate-producing package. */ +export function canonicalCandidateJson(value: unknown): string { + if (!isCanonicalJsonValue(value)) { + throw new Error("candidate document must be finite, acyclic RFC 8785 JSON"); + } + return serializeCanonicalCandidate(value as AgentCandidateJsonValue); +} + +export function canonicalCandidateBytes( + value: unknown, +): Uint8Array { + return new TextEncoder().encode(canonicalCandidateJson(value)); +} + +export function canonicalCandidateDigest( + value: unknown, +): Sha256Digest { + return sha256Bytes(canonicalCandidateBytes(value)); +} + +export function omitTopLevelDigest( + value: T, +): Omit { + const { digest: _digest, ...material } = value; + return material; +} + +function serializeCanonicalCandidate(value: AgentCandidateJsonValue): string { + if (value === null || typeof value === "boolean" || typeof value === "number") { + return JSON.stringify(value); + } + if (typeof value === "string") return JSON.stringify(value); + if (Array.isArray(value)) { + return `[${value.map(serializeCanonicalCandidate).join(",")}]`; + } + return `{${Object.keys(value) + .sort() + .map( + (key) => + `${JSON.stringify(key)}:${serializeCanonicalCandidate(value[key] as AgentCandidateJsonValue)}`, + ) + .join(",")}}`; +} + export function isWellFormedUnicode(value: string): boolean { for (let index = 0; index < value.length; index++) { const code = value.charCodeAt(index); diff --git a/packages/agent-interface/src/agent-candidate-schema.test.ts b/packages/agent-interface/src/agent-candidate-schema.test.ts index 581b83e..825c32f 100644 --- a/packages/agent-interface/src/agent-candidate-schema.test.ts +++ b/packages/agent-interface/src/agent-candidate-schema.test.ts @@ -63,19 +63,18 @@ describe("agentCandidateBundleSchema", () => { expect(() => agentCandidateBundleSchema.parse({ ...candidateFixture(), - code: { kind: "disabled", reason: "control" }, + code: { kind: "disabled" }, execution: { ...executionWithoutWorkspace, launch: { kind: "container-command", executable: "codex" }, cwd: { workspace: "task", path: "." }, }, - lineage: { source: "human" }, }), ).not.toThrow(); expect(() => agentCandidateBundleSchema.parse({ ...candidateFixture(), - code: { kind: "disabled", reason: "control" }, + code: { kind: "disabled" }, }), ).toThrow(/disabled code controls/); }); @@ -87,7 +86,7 @@ describe("agentCandidateBundleSchema", () => { expect(() => agentCandidateBundleSchema.parse({ ...candidate, - code: { kind: "disabled", reason: "not-applicable" }, + code: { kind: "disabled" }, execution: { ...executionWithoutWorkspace, launch: { kind: "container-command", executable: "codex" }, @@ -95,17 +94,6 @@ describe("agentCandidateBundleSchema", () => { }, }), ).not.toThrow(); - expect(() => - agentCandidateBundleSchema.parse({ - ...candidate, - code: { kind: "disabled", reason: "control" }, - execution: { - ...executionWithoutWorkspace, - launch: { kind: "container-command", executable: "codex" }, - cwd: { workspace: "task", path: "." }, - }, - }), - ).toThrow(/fixed controls cannot claim proposer lineage/); }); it("requires active code to use its structured candidate entrypoint", () => { @@ -195,80 +183,6 @@ describe("agentCandidateBundleSchema", () => { } }); - it("requires compound lineage to retain distinct parents", () => { - expect(() => - agentCandidateBundleSchema.parse({ - ...candidateFixture(), - lineage: { - ...candidateFixture().lineage, - source: "compound", - parentDigests: [candidateSha("1")], - }, - }), - ).toThrow(/at least two/); - expect(() => - agentCandidateBundleSchema.parse({ - ...candidateFixture(), - lineage: { - ...candidateFixture().lineage, - source: "compound", - parentDigests: [candidateSha("1"), candidateSha("1")], - }, - }), - ).toThrow(/duplicate/); - }); - - it("fails closed when a generated candidate omits run, split, or spend", () => { - for (const lineage of [ - { - ...candidateFixture().lineage, - parentDigests: [], - }, - { - ...candidateFixture().lineage, - runIds: [], - }, - { - ...candidateFixture().lineage, - benchmark: undefined, - }, - { - ...candidateFixture().lineage, - spend: undefined, - }, - ]) { - expect(() => - agentCandidateBundleSchema.parse({ ...candidateFixture(), lineage }), - ).toThrow(/generated candidates/); - } - }); - - it("keeps proposer no-ops and parent identities honest", () => { - const candidate = candidateFixture(); - expect(() => - agentCandidateBundleSchema.parse({ - ...candidate, - code: { - kind: "no-op", - reason: "proposer-no-change", - repository: candidate.code.repository, - baseCommit: candidate.code.baseCommit, - baseTree: candidate.code.baseTree, - }, - lineage: { source: "human" }, - }), - ).toThrow(/proposer no-op/); - expect(() => - agentCandidateBundleSchema.parse({ - ...candidate, - lineage: { - ...candidate.lineage, - parentDigests: [candidate.digest], - }, - }), - ).toThrow(/cannot name itself/); - }); - it("rejects malformed identities and unknown wire fields", () => { expect(() => agentCandidateBundleSchema.parse({ @@ -360,6 +274,7 @@ describe("candidate receipts", () => { reasoningEffort: "high" as const, }; const profilePlanMaterial = { + sourceProfileDigest: candidateSha("f"), harness: "codex" as const, files: [], env: {}, @@ -374,35 +289,33 @@ describe("candidate receipts", () => { artifact: capturedProfilePlan.artifact, }; const resetEvidence = embeddedBytes("fresh-memory-reset"); + const capturedSuite = capturedMaterial({ + kind: "agent-candidate-benchmark-suite", + digestAlgorithm: "rfc8785-sha256", + taskDigests: [candidateSha("0")], + reps: 1, + seeds: [42], + }); + const capturedTask = capturedMaterial({ + kind: "agent-candidate-benchmark-task", + taskId: "pier-task-1", + }); const executionPlanMaterial = { kind: "agent-candidate-execution-plan-material" as const, - bundleDigest: candidateSha("1"), - executionId: "run-1", - attempt: { - number: 1, - maxAttempts: 1, - retryPolicy: "pre-model-infrastructure-only" as const, - }, - task: { - benchmark: "pier", - benchmarkVersion: "0.3", - taskId: "pier-task-1", - splitDigest: candidateSha("f"), - instruction: { - encoding: "utf8" as const, - sha256: candidateSha("e"), - byteLength: 37, - delivery: { kind: "argv-append" as const }, - }, - repository: { - identity: "r360/pier-synthetic-task-1", - rootIdentity: "r360/pier-synthetic", - baseCommit: candidateGit("a"), - baseTree: candidateGit("b"), - }, - outcome: { kind: "workspace" as const }, - workspace: taskWorkspace, + runCell: { + kind: "agent-candidate-run-cell" as const, + experimentDigest: candidateSha("e"), + arm: "candidate" as const, + bundleDigest: candidateSha("1"), + suiteDigest: capturedSuite.digest, + taskDigest: capturedTask.digest, + taskIndex: 0, + repetition: 0, + seed: 42, + attempt: 1, + digest: candidateSha("d"), }, + executionId: "run-1", workspaces: { taskRoot: "/work/task", candidateRoot: "/work/candidate", @@ -416,6 +329,15 @@ describe("candidate receipts", () => { }, harness: "codex" as const, harnessVersion: "0.1.0", + instructionDelivery: { kind: "stdin-utf8" as const }, + limits: { + timeoutMs: 120_000, + maxSteps: 50, + maxModelCalls: 12, + maxInputTokens: 100_000, + maxOutputTokens: 20_000, + maxCostUsd: 5, + }, container: { source: "evaluator-task-container" as const, image: "pier-task:0.3", @@ -436,15 +358,6 @@ describe("candidate receipts", () => { }, routes: [{ kind: "primary" as const, requested: "openai/gpt-5.4" }], }, - grader: { - name: "fixture-grader", - version: "1.0.0", - artifact: { - locator: { kind: "s3" as const, bucket: "test-artifacts", key: "grader" }, - sha256: candidateSha("c"), - byteLength: 1, - }, - }, launch: { executable: "node", args: [{ kind: "public" as const, value: "dist/agent.js" }], @@ -463,14 +376,6 @@ describe("candidate receipts", () => { }, beforeState: memoryWorkspace, }, - limits: { - timeoutMs: 600_000, - maxSteps: 500, - maxModelCalls: 100, - maxInputTokens: 1_000_000, - maxOutputTokens: 100_000, - maxCostUsd: 100, - }, network: { mode: "disabled" as const }, }; const capturedExecutionPlan = capturedMaterial(executionPlanMaterial); @@ -478,7 +383,22 @@ describe("candidate receipts", () => { kind: "agent-candidate-materialization", digestAlgorithm: "rfc8785-sha256", bundleDigest: candidateSha("1"), - profilePlan, + benchmark: { + suite: { + digest: capturedSuite.digest, + material: capturedSuite.artifact, + }, + task: { + digest: capturedTask.digest, + material: capturedTask.artifact, + }, + }, + profileActivation: { + kind: "agent-candidate-profile-activation", + profilePlan, + files: [], + digest: candidateSha("c"), + }, executionPlan: { kind: "agent-candidate-execution-plan", digest: capturedExecutionPlan.digest, @@ -565,12 +485,15 @@ describe("candidate receipts", () => { expect(() => agentCandidateMaterializationReceiptSchema.parse({ ...materialization, - profilePlan: { - ...materialization.profilePlan, - artifact: { - ...materialization.profilePlan.artifact, - content: "", - byteLength: 0, + profileActivation: { + ...materialization.profileActivation, + profilePlan: { + ...materialization.profileActivation.profilePlan, + artifact: { + ...materialization.profileActivation.profilePlan.artifact, + content: "", + byteLength: 0, + }, }, }, }), diff --git a/packages/agent-interface/src/agent-candidate-schema.ts b/packages/agent-interface/src/agent-candidate-schema.ts index ca366ee..56f35d9 100644 --- a/packages/agent-interface/src/agent-candidate-schema.ts +++ b/packages/agent-interface/src/agent-candidate-schema.ts @@ -4,7 +4,6 @@ import { agentCandidateCodeSchema, agentCandidateExecutionSchema } from "./agent import { agentCandidateProfileSchema } from "./agent-candidate-profile-schema.js"; import { agentCandidateKnowledgeSchema, - agentCandidateLineageSchema, agentCandidateMemoryPolicySchema, } from "./agent-candidate-lineage-schema.js"; import { @@ -30,7 +29,6 @@ export const agentCandidateBundleSchema = z execution: agentCandidateExecutionSchema, knowledge: agentCandidateKnowledgeSchema.optional(), memory: agentCandidateMemoryPolicySchema, - lineage: agentCandidateLineageSchema, digest: sha256DigestSchema, }) .strict() @@ -52,14 +50,6 @@ export const agentCandidateBundleSchema = z message: "execution harness must match the candidate profile preference", }); } - if (bundle.lineage.parentDigests?.includes(bundle.digest)) { - ctx.addIssue({ - code: "custom", - path: ["lineage", "parentDigests"], - message: "a candidate cannot name itself as a parent", - }); - } - if (bundle.code.kind === "disabled") { if (bundle.execution.cwd.workspace !== "task") { ctx.addIssue({ @@ -82,17 +72,6 @@ export const agentCandidateBundleSchema = z message: "disabled code cannot carry a candidate workspace", }); } - if ( - bundle.code.reason === "control" && - (bundle.lineage.source === "optimizer" || - bundle.lineage.source === "compound") - ) { - ctx.addIssue({ - code: "custom", - path: ["lineage", "source"], - message: "fixed controls cannot claim proposer lineage", - }); - } } else { const launch = bundle.execution.launch; if (launch.kind !== "candidate-entrypoint") { @@ -137,17 +116,6 @@ export const agentCandidateBundleSchema = z }); } } - if ( - bundle.code.kind === "no-op" && - bundle.lineage.source !== "optimizer" && - bundle.lineage.source !== "compound" - ) { - ctx.addIssue({ - code: "custom", - path: ["lineage", "source"], - message: "proposer no-op candidates require optimizer or compound lineage", - }); - } } }) satisfies z.ZodType; @@ -170,4 +138,13 @@ export * from "./agent-candidate-lineage-schema.js"; export * from "./agent-candidate-outcome-schema.js"; export * from "./agent-candidate-profile-schema.js"; export * from "./agent-candidate-receipt-schema.js"; -export { sha256DigestSchema } from "./agent-candidate-schema-common.js"; +export * from "./agent-candidate-task-schema.js"; +export { + canonicalCandidateBytes, + canonicalCandidateDigest, + canonicalCandidateJson, + omitTopLevelDigest, + sha256Bytes, + sha256DigestSchema, + sha256Utf8, +} from "./agent-candidate-schema-common.js"; diff --git a/packages/agent-interface/src/agent-candidate-task-schema.test.ts b/packages/agent-interface/src/agent-candidate-task-schema.test.ts new file mode 100644 index 0000000..44eeeb6 --- /dev/null +++ b/packages/agent-interface/src/agent-candidate-task-schema.test.ts @@ -0,0 +1,216 @@ +import { describe, expect, it } from "vitest"; +import { candidateSha } from "./agent-candidate.test-fixture.js"; +import { + agentCandidateBenchmarkSuiteInputsSchema, + agentCandidateBenchmarkSuiteSchema, + agentCandidateBenchmarkTaskSchema, +} from "./agent-candidate-task-schema.js"; + +function artifact(key: string, digit: string) { + return { + locator: { kind: "s3" as const, bucket: "candidate-artifacts", key }, + sha256: candidateSha(digit), + byteLength: 2, + }; +} + +function task() { + return { + kind: "agent-candidate-benchmark-task" as const, + digestAlgorithm: "rfc8785-sha256" as const, + benchmark: { + name: "support-agent", + version: "2026-07-15", + splitDigest: candidateSha("3"), + }, + scenario: { + id: "case-17", + kind: "support-ticket", + scenarioDigest: candidateSha("2"), + }, + datasetSnapshot: artifact("datasets/support.json", "1"), + instruction: "Resolve the customer request using the available tools.", + outcome: { + kind: "output" as const, + mediaType: "text/plain", + maxBytes: 64_000, + }, + workspace: { + kind: "agent-candidate-workspace-snapshot" as const, + digest: candidateSha("4"), + material: { + kind: "agent-candidate-workspace-manifest" as const, + files: [], + }, + manifest: artifact("tasks/case-17.manifest", "4"), + archive: artifact("tasks/case-17.tar", "6"), + }, + grader: { + name: "exact-reference", + version: "1.0.0", + format: "tangle-grader" as const, + artifact: artifact("graders/case-17.json", "7"), + }, + model: { + requested: "openai/gpt-5.4", + provider: "openai", + model: "gpt-5.4", + snapshot: "gpt-5.4-2026-07-15", + reasoningEffort: "high" as const, + }, + attempt: { + maxAttempts: 2, + retryPolicy: "pre-model-infrastructure-only" as const, + }, + evaluatorTaskContainer: { + source: "evaluator-task-container" as const, + image: "ghcr.io/tangle-network/agent:sha-abc", + indexDigest: candidateSha("8"), + manifestDigest: candidateSha("9"), + platform: { os: "linux", architecture: "amd64" }, + }, + limits: { + timeoutMs: 120_000, + maxSteps: 50, + maxModelCalls: 12, + maxInputTokens: 100_000, + maxOutputTokens: 20_000, + maxCostUsd: 5, + }, + digest: candidateSha("a"), + }; +} + +function suite() { + const benchmarkTask = task(); + return { + kind: "agent-candidate-benchmark-suite" as const, + digestAlgorithm: "rfc8785-sha256" as const, + taskDigests: [benchmarkTask.digest] as const, + reps: 2, + seeds: [42, 43] as const, + digest: candidateSha("f"), + }; +} + +describe("agent candidate benchmark task", () => { + it("parses one exact output task shared by evaluation and execution", () => { + expect(agentCandidateBenchmarkTaskSchema.parse(task())).toEqual(task()); + }); + + it("requires workspace tasks to pin their source repository", () => { + const value = task(); + const invalid = { + ...value, + outcome: { kind: "workspace" as const }, + }; + expect(agentCandidateBenchmarkTaskSchema.safeParse(invalid).success).toBe(false); + }); + + it("rejects unbounded or unknown execution policy", () => { + const value = task(); + expect( + agentCandidateBenchmarkTaskSchema.safeParse({ + ...value, + limits: { ...value.limits, maxCostUsd: Number.POSITIVE_INFINITY }, + }).success, + ).toBe(false); + expect( + agentCandidateBenchmarkTaskSchema.safeParse({ + ...value, + funding: { owner: "platform" }, + }).success, + ).toBe(false); + expect( + agentCandidateBenchmarkTaskSchema.safeParse({ + ...value, + attempt: { maxAttempts: 2, retryPolicy: "none" }, + }).success, + ).toBe(false); + expect( + agentCandidateBenchmarkTaskSchema.safeParse({ + ...value, + grader: { + ...value.grader, + artifact: { ...value.grader.artifact, byteLength: 0 }, + }, + }).success, + ).toBe(false); + }); + + it("rejects mixed repository object formats", () => { + const value = task(); + expect( + agentCandidateBenchmarkTaskSchema.safeParse({ + ...value, + repository: { + identity: "owner/repository", + rootIdentity: "owner/repository", + baseCommit: "a".repeat(40), + baseTree: "b".repeat(64), + }, + outcome: { kind: "workspace" }, + }).success, + ).toBe(false); + }); + + it("reuses the candidate container safety policy", () => { + const value = task(); + expect( + agentCandidateBenchmarkTaskSchema.safeParse({ + ...value, + evaluatorTaskContainer: { + ...value.evaluatorTaskContainer, + image: "http://127.0.0.1/private", + }, + }).success, + ).toBe(false); + }); +}); + +describe("agent candidate benchmark suite", () => { + it("enumerates the complete ordered task and repetition denominator", () => { + expect(agentCandidateBenchmarkSuiteSchema.parse(suite())).toEqual(suite()); + expect( + agentCandidateBenchmarkSuiteInputsSchema.parse({ suite: suite(), tasks: [task()] }), + ).toEqual({ suite: suite(), tasks: [task()] }); + }); + + it("rejects omitted seeds, duplicate tasks, or substituted task documents", () => { + const value = suite(); + expect( + agentCandidateBenchmarkSuiteSchema.safeParse({ + ...value, + seeds: value.seeds.slice(0, 1), + }).success, + ).toBe(false); + expect( + agentCandidateBenchmarkSuiteSchema.safeParse({ + ...value, + taskDigests: [task().digest, task().digest], + seeds: [42, 43, 44, 45], + }).success, + ).toBe(false); + expect( + agentCandidateBenchmarkSuiteInputsSchema.safeParse({ + suite: value, + tasks: [{ ...task(), digest: candidateSha("e") }], + }).success, + ).toBe(false); + const alias = { + ...task(), + digest: candidateSha("e"), + scenario: { ...task().scenario, id: "case-18" }, + }; + expect( + agentCandidateBenchmarkSuiteInputsSchema.safeParse({ + suite: { + ...value, + taskDigests: [task().digest, alias.digest], + seeds: [42, 43, 44, 45], + }, + tasks: [task(), alias], + }).success, + ).toBe(false); + }); +}); diff --git a/packages/agent-interface/src/agent-candidate-task-schema.ts b/packages/agent-interface/src/agent-candidate-task-schema.ts new file mode 100644 index 0000000..bfd6323 --- /dev/null +++ b/packages/agent-interface/src/agent-candidate-task-schema.ts @@ -0,0 +1,226 @@ +import { z } from "zod"; +import type { + AgentCandidateBenchmarkSuite, + AgentCandidateBenchmarkSuiteInputs, + AgentCandidateBenchmarkSuiteMaterial, + AgentCandidateBenchmarkTask, + AgentCandidateBenchmarkTaskMaterial, +} from "./agent-candidate.js"; +import { + agentCandidateArtifactRefSchema, + agentCandidateWorkspaceSnapshotEvidenceSchema, +} from "./agent-candidate-artifact-schema.js"; +import { + agentCandidateBenchmarkGraderIdentitySchema, + agentCandidateBenchmarkCellRefSchema, + agentCandidateExecutionLimitsSchema, + agentCandidateResolvedModelSchema, + agentCandidateResolvedTaskContainerSchema, + agentCandidateRetryPolicySchema, + agentCandidateTaskRepositorySchema, + agentCandidateTaskOutcomeSpecSchema, +} from "./agent-candidate-execution-plan-schema.js"; +import { + isCanonicalJsonValue, + isWellFormedUnicode, + sha256DigestSchema, +} from "./agent-candidate-schema-common.js"; + +export const agentCandidateBenchmarkTaskMaterialSchema = z + .object({ + kind: z.literal("agent-candidate-benchmark-task"), + digestAlgorithm: z.literal("rfc8785-sha256"), + benchmark: z + .object({ + name: z.string().min(1).max(512), + version: z.string().min(1).max(256), + splitDigest: sha256DigestSchema, + }) + .strict(), + scenario: z + .object({ + id: z.string().min(1).max(512), + kind: z.string().min(1).max(512), + scenarioDigest: sha256DigestSchema, + }) + .strict(), + datasetSnapshot: agentCandidateArtifactRefSchema.optional(), + instruction: z + .string() + .min(1) + .max(4 * 1024 * 1024) + .refine(isWellFormedUnicode, "task instruction must be well-formed Unicode"), + repository: agentCandidateTaskRepositorySchema.optional(), + outcome: agentCandidateTaskOutcomeSpecSchema, + workspace: agentCandidateWorkspaceSnapshotEvidenceSchema, + grader: agentCandidateBenchmarkGraderIdentitySchema, + model: agentCandidateResolvedModelSchema, + attempt: z + .object({ + maxAttempts: z.number().int().min(1), + retryPolicy: agentCandidateRetryPolicySchema, + }) + .strict(), + evaluatorTaskContainer: agentCandidateResolvedTaskContainerSchema.optional(), + limits: agentCandidateExecutionLimitsSchema, + }) + .strict() + .superRefine((task, ctx) => { + if (!isCanonicalJsonValue(task)) { + ctx.addIssue({ + code: "custom", + message: "candidate benchmark task must contain finite, acyclic canonical JSON", + }); + } + if (task.outcome.kind === "workspace" && task.repository === undefined) { + ctx.addIssue({ + code: "custom", + path: ["repository"], + message: "workspace outcomes require an exact source repository", + }); + } + if (task.attempt.retryPolicy === "none" && task.attempt.maxAttempts !== 1) { + ctx.addIssue({ + code: "custom", + path: ["attempt", "maxAttempts"], + message: "a no-retry task must allow exactly one attempt", + }); + } + if (task.datasetSnapshot?.byteLength === 0) { + ctx.addIssue({ + code: "custom", + path: ["datasetSnapshot", "byteLength"], + message: "dataset snapshot provenance must contain bytes", + }); + } + }) satisfies z.ZodType; + +/** Structural parse only; Runtime recomputes the canonical digest before use. */ +export const agentCandidateBenchmarkTaskSchema = + agentCandidateBenchmarkTaskMaterialSchema + .extend({ digest: sha256DigestSchema }) + .strict() satisfies z.ZodType; + +export const agentCandidateBenchmarkSuiteMaterialSchema = z + .object({ + kind: z.literal("agent-candidate-benchmark-suite"), + digestAlgorithm: z.literal("rfc8785-sha256"), + taskDigests: z + .tuple([sha256DigestSchema]) + .rest(sha256DigestSchema), + reps: z.number().int().positive(), + seeds: z + .tuple([ + z + .number() + .int() + .min(Number.MIN_SAFE_INTEGER) + .max(Number.MAX_SAFE_INTEGER), + ]) + .rest( + z + .number() + .int() + .min(Number.MIN_SAFE_INTEGER) + .max(Number.MAX_SAFE_INTEGER), + ), + }) + .strict() + .superRefine((suite, ctx) => { + const taskDigests = new Set(); + for (const [index, digest] of suite.taskDigests.entries()) { + if (taskDigests.has(digest)) { + ctx.addIssue({ + code: "custom", + path: ["taskDigests", index], + message: "benchmark suite task digests must be unique", + }); + } + taskDigests.add(digest); + } + const expectedSeeds = suite.taskDigests.length * suite.reps; + if (suite.seeds.length !== expectedSeeds) { + ctx.addIssue({ + code: "custom", + path: ["seeds"], + message: "benchmark suite must provide one seed per task repetition", + }); + } + if (!isCanonicalJsonValue(suite)) { + ctx.addIssue({ + code: "custom", + message: "benchmark suite must contain finite, acyclic canonical JSON", + }); + } + }) satisfies z.ZodType; + +export const agentCandidateBenchmarkSuiteSchema = + agentCandidateBenchmarkSuiteMaterialSchema + .extend({ digest: sha256DigestSchema }) + .strict() satisfies z.ZodType; + +export const agentCandidateBenchmarkSuiteInputsSchema = z + .object({ + suite: agentCandidateBenchmarkSuiteSchema, + tasks: z + .tuple([agentCandidateBenchmarkTaskSchema]) + .rest(agentCandidateBenchmarkTaskSchema), + }) + .strict() + .superRefine((input, ctx) => { + if (input.tasks.length !== input.suite.taskDigests.length) { + ctx.addIssue({ + code: "custom", + path: ["tasks"], + message: "benchmark suite inputs must contain every signed task exactly once", + }); + } + const taskIds = new Set(); + const scenarioDigests = new Set(); + const benchmark = input.tasks[0]?.benchmark; + for (const [index, task] of input.tasks.entries()) { + if (task.digest !== input.suite.taskDigests[index]) { + ctx.addIssue({ + code: "custom", + path: ["tasks", index, "digest"], + message: "benchmark task order must match the signed suite", + }); + } + if (taskIds.has(task.scenario.id)) { + ctx.addIssue({ + code: "custom", + path: ["tasks", index, "scenario", "id"], + message: "benchmark suite task ids must be unique", + }); + } + taskIds.add(task.scenario.id); + if (scenarioDigests.has(task.scenario.scenarioDigest)) { + ctx.addIssue({ + code: "custom", + path: ["tasks", index, "scenario", "scenarioDigest"], + message: "benchmark suite scenario digests must be unique", + }); + } + scenarioDigests.add(task.scenario.scenarioDigest); + if ( + benchmark && + (task.benchmark.name !== benchmark.name || + task.benchmark.version !== benchmark.version || + task.benchmark.splitDigest !== benchmark.splitDigest) + ) { + ctx.addIssue({ + code: "custom", + path: ["tasks", index, "benchmark"], + message: "benchmark suite tasks must share one benchmark identity", + }); + } + } + if (!isCanonicalJsonValue(input)) { + ctx.addIssue({ + code: "custom", + message: "benchmark suite inputs must contain finite, acyclic canonical JSON", + }); + } + }) satisfies z.ZodType; + +export { agentCandidateBenchmarkCellRefSchema }; diff --git a/packages/agent-interface/src/agent-candidate.test-fixture.ts b/packages/agent-interface/src/agent-candidate.test-fixture.ts index 7e7adfb..37a1846 100644 --- a/packages/agent-interface/src/agent-candidate.test-fixture.ts +++ b/packages/agent-interface/src/agent-candidate.test-fixture.ts @@ -151,32 +151,6 @@ export function candidateFixture() { mode: "isolated", scope: "task", }, - lineage: { - source: "optimizer", - parentDigests: [candidateSha("8")], - runIds: ["r360-search"], - profileDiffIds: ["profile-diff-3"], - modelSnapshots: ["openai/gpt-5.4-2026-06-15"], - benchmark: { - name: "pier", - version: "0.3", - splitDigest: candidateSha("9"), - }, - spend: { - proposal: { - costUsd: 3.5, - inputTokens: 100, - outputTokens: 20, - modelCalls: 2, - }, - evaluation: { - costUsd: 9, - inputTokens: 900, - outputTokens: 200, - modelCalls: 8, - }, - }, - }, digest: candidateSha("a"), }); } diff --git a/packages/agent-interface/src/agent-candidate.ts b/packages/agent-interface/src/agent-candidate.ts index 133097e..b59a1ad 100644 --- a/packages/agent-interface/src/agent-candidate.ts +++ b/packages/agent-interface/src/agent-candidate.ts @@ -180,8 +180,6 @@ export interface AgentCandidateProfile export interface AgentCandidateCodeDisabled { kind: "disabled"; - /** `control` marks a comparison arm; `not-applicable` disables only the code surface. */ - reason: "control" | "not-applicable"; } /** A code proposer ran against this exact tree and returned no change. */ @@ -318,15 +316,6 @@ export type AgentCandidateMemoryPolicy = seed?: AgentCandidateArtifactRef; }; -/** Captured model spend for one phase of candidate production. */ -export interface AgentCandidateSpend { - costUsd: number; - inputTokens: number; - outputTokens: number; - cachedInputTokens?: number; - modelCalls: number; -} - /** Lossless evaluator-owned usage totals for one candidate execution. */ export interface AgentCandidateFixedSpend { inputTokens: number; @@ -345,15 +334,8 @@ export interface AgentCandidateLineage { runIds?: string[]; profileDiffIds?: string[]; modelSnapshots?: string[]; - benchmark?: { - name: string; - version: string; - splitDigest: Sha256Digest; - }; - spend?: { - proposal: AgentCandidateSpend; - evaluation: AgentCandidateSpend; - }; + /** Exact development split used to produce a generated candidate. */ + developmentSplitDigest?: Sha256Digest; } /** @@ -372,7 +354,6 @@ export interface AgentCandidateBundle { execution: AgentCandidateExecution; knowledge?: AgentCandidateKnowledge; memory: AgentCandidateMemoryPolicy; - lineage: AgentCandidateLineage; digest: Sha256Digest; } @@ -388,6 +369,15 @@ export interface AgentCandidateOciPlatform { variant?: string; } +/** Exact evaluator-selected task image used when the candidate does not pin one. */ +export interface AgentCandidateResolvedTaskContainer { + source: "evaluator-task-container"; + image: string; + indexDigest: Sha256Digest; + manifestDigest: Sha256Digest; + platform: AgentCandidateOciPlatform; +} + export interface AgentCandidateResolvedModel { requested: string; provider: string; @@ -398,6 +388,8 @@ export interface AgentCandidateResolvedModel { /** Canonical, digest-free profile-plan identity document. */ export interface AgentCandidateProfilePlanMaterial { + /** Canonical digest of the complete frozen profile that produced this plan. */ + sourceProfileDigest: Sha256Digest; harness: HarnessType; files: Array<{ relPath: string; @@ -495,6 +487,119 @@ export type AgentCandidateTaskOutcomeSpec = | { kind: "workspace" } | ({ kind: "output" } & AgentCandidateTaskOutputSpec); +/** Immutable grader identity admitted for one benchmark task. */ +export interface AgentCandidateBenchmarkGraderIdentity { + name: string; + version: string; + format: "tangle-grader"; + artifact: AgentCandidateArtifactRef; +} + +/** Portable task bytes shared by evaluation, approval, and execution. */ +export interface AgentCandidateBenchmarkTaskMaterial { + kind: "agent-candidate-benchmark-task"; + digestAlgorithm: AgentCandidateDigestAlgorithm; + benchmark: { + name: string; + version: string; + splitDigest: Sha256Digest; + }; + scenario: { + id: string; + kind: string; + scenarioDigest: Sha256Digest; + }; + datasetSnapshot?: AgentCandidateArtifactRef; + instruction: string; + repository?: AgentCandidateTaskRepository; + outcome: AgentCandidateTaskOutcomeSpec; + workspace: AgentCandidateWorkspaceSnapshotEvidence; + grader: AgentCandidateBenchmarkGraderIdentity; + model: AgentCandidateResolvedModel; + attempt: Omit; + evaluatorTaskContainer?: AgentCandidateResolvedTaskContainer; + limits: AgentCandidateExecutionLimits; +} + +/** Content-addressed benchmark task approved and executed without reinterpretation. */ +export interface AgentCandidateBenchmarkTask + extends AgentCandidateBenchmarkTaskMaterial { + digest: Sha256Digest; +} + +/** Complete measured denominator shared by evaluation and execution. */ +export interface AgentCandidateBenchmarkSuiteMaterial { + kind: "agent-candidate-benchmark-suite"; + digestAlgorithm: AgentCandidateDigestAlgorithm; + taskDigests: [Sha256Digest, ...Sha256Digest[]]; + reps: number; + /** Task-major, then repetition-major: seeds[taskIndex * reps + repetition]. */ + seeds: [number, ...number[]]; +} + +export interface AgentCandidateBenchmarkSuite + extends AgentCandidateBenchmarkSuiteMaterial { + digest: Sha256Digest; +} + +/** Canonical task documents transported alongside their signed suite. */ +export interface AgentCandidateBenchmarkSuiteInputs { + suite: AgentCandidateBenchmarkSuite; + tasks: [AgentCandidateBenchmarkTask, ...AgentCandidateBenchmarkTask[]]; +} + +/** One cell in a signed suite; task identity and seed are derived by position. */ +export interface AgentCandidateBenchmarkCellRef { + suiteDigest: Sha256Digest; + taskIndex: number; + repetition: number; +} + +/** Decision rules frozen before either experiment arm executes. */ +export interface AgentCandidateEvaluationPolicy { + confidenceLevel: number; + resamples: number; + bootstrapSeed: number; + deltaThreshold: number; + minProductiveRuns: number; + budgetUsd?: number; + criticalDimensions: string[]; + regressionTolerance: number; +} + +/** Both complete agent states and the exact held-out work used to compare them. */ +export interface AgentCandidateExperimentMaterial { + kind: "agent-candidate-experiment"; + digestAlgorithm: AgentCandidateDigestAlgorithm; + baseline: AgentCandidateBundle; + candidate: AgentCandidateBundle; + candidateLineage: AgentCandidateLineage; + benchmark: AgentCandidateBenchmarkSuiteInputs; + policy: AgentCandidateEvaluationPolicy; +} + +export interface AgentCandidateExperiment + extends AgentCandidateExperimentMaterial { + digest: Sha256Digest; +} + +/** Digest-free identity of one exact attempt in a frozen experiment. */ +export interface AgentCandidateRunCellMaterial + extends AgentCandidateBenchmarkCellRef { + kind: "agent-candidate-run-cell"; + experimentDigest: Sha256Digest; + arm: "baseline" | "candidate"; + bundleDigest: Sha256Digest; + taskDigest: Sha256Digest; + seed: number; + attempt: number; +} + +/** One immutable experiment attempt used by plans, receipts, traces, and results. */ +export interface AgentCandidateRunCell extends AgentCandidateRunCellMaterial { + digest: Sha256Digest; +} + /** * Canonical, digest-free per-task execution identity document. * @@ -503,24 +608,8 @@ export type AgentCandidateTaskOutcomeSpec = */ export interface AgentCandidateExecutionPlanMaterial { kind: "agent-candidate-execution-plan-material"; - bundleDigest: Sha256Digest; + runCell: AgentCandidateRunCell; executionId: string; - attempt: AgentCandidateAttemptPolicy; - task: { - benchmark: string; - benchmarkVersion: string; - taskId: string; - splitDigest: Sha256Digest; - instruction: { - encoding: "utf8"; - sha256: Sha256Digest; - byteLength: number; - delivery: AgentCandidateInstructionDelivery; - }; - repository?: AgentCandidateTaskRepository; - outcome: AgentCandidateTaskOutcomeSpec; - workspace: AgentCandidateWorkspaceSnapshotEvidence; - }; workspaces: { taskRoot: string; candidateRoot?: string; @@ -530,6 +619,8 @@ export interface AgentCandidateExecutionPlanMaterial { profile: AgentCandidateProfileApplication; harness: HarnessType; harnessVersion: string; + instructionDelivery: AgentCandidateInstructionDelivery; + limits: AgentCandidateExecutionLimits; container: { source: AgentCandidateExecutionEnvironment["kind"]; image: string; @@ -552,12 +643,6 @@ export interface AgentCandidateExecutionPlanMaterial { | { kind: "subagent"; name: string; requested: string } >; }; - /** Exact evaluator grader implementation admitted for this plan. */ - grader: { - name: string; - version: string; - artifact: AgentCandidateArtifactRef; - }; launch: { executable: string; args: AgentCandidateConfigValue[]; @@ -566,7 +651,6 @@ export interface AgentCandidateExecutionPlanMaterial { }; knowledgeManifestDigest?: Sha256Digest; memory: AgentCandidateEffectiveMemory; - limits: AgentCandidateExecutionLimits; network: { mode: "disabled" }; } @@ -596,6 +680,18 @@ export interface AgentCandidateExecutionPlanEvidence { artifact: AgentCandidateCapturedArtifact; } +/** Exact suite and task material selected for one execution plan. */ +export interface AgentCandidateBenchmarkInputEvidence { + suite: { + digest: Sha256Digest; + material: AgentCandidateCapturedArtifact; + }; + task: { + digest: Sha256Digest; + material: AgentCandidateCapturedArtifact; + }; +} + export interface AgentCandidateTraceEvidence { artifact: AgentCandidateCapturedArtifact; eventCount: number; @@ -618,7 +714,8 @@ export interface AgentCandidateMaterializationReceipt { kind: "agent-candidate-materialization"; digestAlgorithm: AgentCandidateDigestAlgorithm; bundleDigest: Sha256Digest; - profilePlan: AgentCandidateProfilePlanEvidence; + benchmark: AgentCandidateBenchmarkInputEvidence; + profileActivation: AgentCandidateProfileActivation; executionPlan: AgentCandidateExecutionPlanEvidence; candidateWorkspace?: AgentCandidateWorkspaceSnapshotEvidence; codeKind: AgentCandidateCode["kind"]; @@ -650,8 +747,14 @@ export interface AgentCandidateRunReceipt { kind: "agent-candidate-run"; digestAlgorithm: AgentCandidateDigestAlgorithm; bundleDigest: Sha256Digest; + runCellDigest: Sha256Digest; materializationReceiptDigest: Sha256Digest; executionPlanDigest: Sha256Digest; + timing: { + startedAtMs: number; + endedAtMs: number; + durationMs: number; + }; memory: AgentCandidateMemoryReceipt; trace: AgentCandidateTraceEvidence; termination: AgentCandidateTermination; @@ -674,16 +777,17 @@ export type AgentImprovementSurface = | "code" | "knowledge"; +/** One paired Runtime execution from the exact signed experiment. */ +export interface AgentCandidateExperimentMeasurement { + baseline: CandidateExecutionEvidence; + candidate: CandidateExecutionEvidence; +} + /** Portable paired held-out comparison produced by an evaluation package. */ export interface AgentImprovementMeasuredComparison { kind: "agent-improvement-measured-comparison"; - benchmark: { - name: string; - version: string; - splitDigest: Sha256Digest; - }; - baselineProfileDigest: Sha256Digest; - candidateBundleDigest: Sha256Digest; + experiment: AgentCandidateExperiment; + measurements: AgentCandidateExperimentMeasurement[]; overall: { name: "composite"; baseline: number; @@ -782,7 +886,11 @@ export interface AgentImprovementMeasuredComparison { diff: string; evaluation: { generationsExplored: number; + searchDurationMs: number; + executionDurationMs: number; durationMs: number; + searchCostUsd: number; + executionCostUsd: number; totalCostUsd: number; }; metadata?: { [key: string]: AgentCandidateJsonValue }; @@ -793,10 +901,8 @@ export interface AgentImprovementProposal { runId: string; changedSurfaces: [AgentImprovementSurface, ...AgentImprovementSurface[]]; proposedAt: string; - baselineProfile: AgentProfile; findings: { [key: string]: AgentCandidateJsonValue }[]; evaluation: AgentImprovementMeasuredComparison; - candidateBundle: AgentCandidateBundle; digest: Sha256Digest; } @@ -809,7 +915,6 @@ export type AgentImprovementReviewDecision = export interface AgentImprovementReview { kind: "agent-improvement-review"; proposalDigest: Sha256Digest; - candidateBundleDigest: Sha256Digest; decision: AgentImprovementReviewDecision; reviewedBy: string; reviewedAt: string; @@ -818,15 +923,32 @@ export interface AgentImprovementReview { digest: Sha256Digest; } -/** Successful post-approval execution, carrying the exact Runtime receipt. */ -export interface CandidateExecutionEvidence { - kind: "agent-candidate-execution-evidence"; +export interface AgentImprovementActivationTarget { + surface: AgentImprovementSurface; + /** Product-owned stable identity, such as an agent profile, repository, or knowledge base. */ + identity: string; + /** Current target state that activation is allowed to replace. */ + expectedBaseDigest: Sha256Digest; +} + +/** Authority receipt permitting activation of one already-measured candidate. */ +export interface AgentImprovementActivation { + kind: "agent-improvement-activation"; proposalDigest: Sha256Digest; reviewDigest: Sha256Digest; - executionId: string; - succeeded: true; + experimentDigest: Sha256Digest; + candidateBundleDigest: Sha256Digest; + targets: [AgentImprovementActivationTarget, ...AgentImprovementActivationTarget[]]; + fundingOwner: string; + authorizedBy: string; + authorizedAt: string; + digest: Sha256Digest; +} + +/** Complete execution of one exact experiment attempt. */ +export interface CandidateExecutionEvidence { + kind: "agent-candidate-execution-evidence"; materializationReceipt: AgentCandidateMaterializationReceipt; - profileActivation: AgentCandidateProfileActivation; receipt: AgentCandidateRunReceipt; digest: Sha256Digest; } @@ -913,19 +1035,18 @@ export interface AgentCandidateBenchmarkResultMaterial { kind: "agent-candidate-benchmark-result-material"; executionPlanDigest: Sha256Digest; taskOutcomeDigest: Sha256Digest; - benchmark: { - name: string; - version: string; - taskId: string; - splitDigest: Sha256Digest; - }; - grader: { - name: string; - version: string; - artifact: AgentCandidateArtifactRef; - }; + grader: AgentCandidateBenchmarkGraderIdentity; /** Raw grader output required to independently audit the reported verdict. */ evidence: AgentCandidateArtifactRef; + /** Evaluator-owned model usage and elapsed time, separate from candidate usage. */ + grading: { + usage: AgentCandidateFixedSpend; + timing: { + startedAtMs: number; + endedAtMs: number; + durationMs: number; + }; + }; score: number; passed: boolean; dimensions: AgentCandidateBenchmarkDimension[]; diff --git a/packages/agent-interface/src/profile-schema.ts b/packages/agent-interface/src/profile-schema.ts index 7eaf440..6288323 100644 --- a/packages/agent-interface/src/profile-schema.ts +++ b/packages/agent-interface/src/profile-schema.ts @@ -50,13 +50,21 @@ export const agentProfileResourcesSchema = z.object({ failOnError: z.boolean().optional(), }); +export const reasoningEffortSchema = z.enum([ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "ultracode", +]); + export const agentProfileModelHintsSchema = z.object({ default: z.string().optional(), small: z.string().optional(), provider: z.string().optional(), - reasoningEffort: z - .enum(["none", "minimal", "low", "medium", "high", "xhigh", "ultracode"]) - .optional(), + reasoningEffort: reasoningEffortSchema.optional(), metadata: z.record(z.string(), z.unknown()).optional(), }); diff --git a/packages/agent-interface/tsconfig.test.json b/packages/agent-interface/tsconfig.test.json new file mode 100644 index 0000000..9622e8a --- /dev/null +++ b/packages/agent-interface/tsconfig.test.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "composite": false, + "declaration": false, + "noEmit": true, + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*.ts"], + "exclude": [] +}