From 52951ccf494e62844db7a51fc1567726ae6fb4fa Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 13 Jul 2026 18:53:17 -0600 Subject: [PATCH 1/5] feat(improvement): widen failure-distiller notes to 1500 chars so tracebacks survive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default generationFailureDistiller clipped judge notes at 400 chars and errors at 200 — an executable judge's traceback/failing assertion arrived at the proposer as a stub. 1500 keeps a real traceback intact (error bound 500); the shape of the findings is unchanged. Doc references to the old ~400-char digest updated. --- src/improvement/improve.ts | 9 ++++++--- src/improvement/raw-trace-distiller.ts | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 312b08de..266c93f1 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -95,7 +95,7 @@ export interface ImproveOptions { * trace-analyst over the runDir's traces) to replace it; pass `null` to disable * and keep the static `findings` all the way through. */ analyzeGeneration?: SelfImproveOptions['analyzeGeneration'] | null - /** META-HARNESS mode: instead of the ~400-char distilled findings, feed the + /** META-HARNESS mode: instead of the ~1500-char distilled findings, feed the * proposer RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's * real run traces under `runDir` (per-cell `spans.jsonl` event logs + * `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose @@ -251,16 +251,19 @@ function generationFailureDistiller( ? 0 : judgeScores.reduce((sum, j) => sum + (j.composite ?? 0), 0) / judgeScores.length if (!error && composite >= 0.999) continue + // 1500 chars keeps a real traceback / failing assertion intact — the old + // 400 clipped executable-judge notes down to a stub, leaving the proposer + // trace-blind. Errors stay tighter (they are usually one-line). const notes = judgeScores .map((j) => j.notes) .filter((n): n is string => typeof n === 'string' && n.length > 0) .join('; ') - .slice(0, 400) + .slice(0, 1500) failures.push({ scenario, composite: Number(composite.toFixed(3)), notes, - ...(error ? { error: error.slice(0, 200) } : {}), + ...(error ? { error: error.slice(0, 500) } : {}), }) } } diff --git a/src/improvement/raw-trace-distiller.ts b/src/improvement/raw-trace-distiller.ts index aba84aa1..6e46f3d5 100644 --- a/src/improvement/raw-trace-distiller.ts +++ b/src/improvement/raw-trace-distiller.ts @@ -3,7 +3,7 @@ * `rawTraceDistiller` — the meta-harness `analyzeGeneration` producer. * * The default `generationFailureDistiller` (in `improve.ts`) COMPRESSES each - * generation's failing cells into ~400-char structured findings before the next + * generation's failing cells into ~1500-char structured findings before the next * proposal round. That is the ACE-style recipe: a small summary is the proposer's * whole view of what went wrong. This producer does the opposite — the * meta-harness recipe (yoonholee.com/meta-harness): it does NOT summarize. It @@ -11,7 +11,7 @@ * disk under `runDir` — the durable per-cell `spans.jsonl` event logs, * `cached-result.json` scores, and any artifacts the substrate persisted — and * instructs the agent to `grep`/`cat`/`ls` them to diagnose the failures itself - * (up to the harness's full context, ~millions of tokens, vs a ~400-char digest). + * (up to the harness's full context, ~millions of tokens, vs a ~1500-char digest). * * It emits `AnalystFinding[]` so it drops into the SAME `opts.analyzeGeneration` * slot the default distiller uses, and renders through the same From d4fcf3ddc424aa4ab91243e03668f6cfacf1ccd9 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 15 Jul 2026 09:43:41 -0600 Subject: [PATCH 2/5] merge origin/main into feat/gepa-trace-evidence (keep 1500/500 evidence widths + main's claim field) --- .github/workflows/ci.yml | 28 + .github/workflows/publish.yml | 83 +- README.md | 85 +- bench/CHANGELOG.md | 9 + bench/HARNESS.md | 72 +- bench/README.md | 27 +- .../no-model-task/environment/Dockerfile | 16 + .../environment/seed/src/status.txt | 1 + .../pier-agent/no-model-task/instruction.md | 6 + .../pier-agent/no-model-task/pre_artifacts.sh | 6 + .../pier-agent/no-model-task/task.toml | 35 + .../pier-agent/no-model-task/tests/Dockerfile | 17 + .../no-model-task/tests/seed/src/status.txt | 1 + .../pier-agent/no-model-task/tests/test.sh | 19 + bench/package.json | 35 +- bench/pier_agents/__init__.py | 18 + bench/pier_agents/candidate_contract.py | 747 + bench/pier_agents/process_boundary.py | 321 + bench/pier_agents/tangle_candidate.py | 907 ++ bench/pier_agents/tangle_candidate_test.py | 1022 ++ bench/pier_agents/workspace_boundary.py | 368 + bench/pnpm-lock.yaml | 315 +- bench/scripts/run-package-tests.mjs | 56 + bench/scripts/terminate-pier-trial.mts | 66 + bench/scripts/verify-packed-consumer.mjs | 175 + bench/scripts/verify-pier-agent.mts | 679 + bench/scripts/verify-pier-pair.mts | 74 + bench/scripts/verify-pier-recovery.mts | 139 + bench/src/benchmarks/_harness.test.mts | 178 + bench/src/benchmarks/_harness.ts | 255 +- bench/src/benchmarks/humaneval.ts | 16 +- bench/src/benchmarks/swe-bench.test.mts | 58 + bench/src/benchmarks/swe-bench.ts | 133 +- bench/src/benchmarks/types.ts | 28 + bench/src/hev-eval.mts | 69 + bench/src/hev-improve.mts | 168 + bench/src/hev-structural.mts | 688 + bench/src/index.ts | 37 + bench/src/mbpp-structural.mts | 662 + bench/src/pier-agent.test-fixtures.mts | 19 + bench/src/pier-agent.test.mts | 350 + bench/src/pier-agent.ts | 631 + bench/src/pier-result-grader.mjs | 30 + bench/src/pier-result-grader.test.mts | 62 + bench/src/pier-result-grader.ts | 108 + bench/src/pier-task-outcome.test.mts | 144 + bench/src/pier-task-outcome.ts | 251 + bench/src/pier-trial-controller.test.mts | 412 + bench/src/pier-trial-controller.ts | 825 ++ bench/src/pier-trial-supervisor.mjs | 352 + bench/src/smoke-structural-rollout.mts | 393 + bench/src/swe-bench-env.test.ts | 49 +- bench/src/tb-container-executor.test.mts | 2 +- bench/tsconfig.public.json | 16 + docs/agent-optimization-map.md | 91 - docs/api/README.md | 2 + docs/api/agent.md | 90 +- docs/api/candidate-execution.md | 577 + docs/api/index.md | 12079 +++++++++++----- docs/api/intelligence.md | 2852 +++- docs/api/mcp.md | 615 +- docs/api/primeintellect.md | 830 ++ docs/api/primitive-catalog.md | 299 +- docs/api/runtime.md | 1837 ++- docs/archive/go-live-plan.md | 6 +- docs/canonical-api.md | 13 +- docs/concepts.md | 4 +- docs/design/structural-rollout-integration.md | 45 + docs/intelligence-sdk.md | 58 +- examples/README.md | 2 +- examples/ablation-suite/aisdk-env.ts | 279 + examples/ablation-suite/extract-contracts.mjs | 50 + .../multi-contract/contracts.all.json | 135 + .../fixtures/multi-contract/contracts.json | 72 + examples/ablation-suite/gepa-driver-prompt.ts | 48 +- examples/ablation-suite/multi-contract-env.ts | 284 + examples/ablation-suite/run-aisdk.ts | 153 + examples/ablation-suite/run-multi-contract.ts | 189 + .../agentic-data-creation.ts | 79 +- .../python-agno/agno_to_intelligence.py | 5 +- examples/agents-of-all-shapes/run.ts | 5 +- examples/agents-of-all-shapes/shapes.ts | 4 +- examples/coding-benchmark/dispatch.ts | 28 +- examples/improve/improve.ts | 18 +- examples/intelligence-drop-in/README.md | 10 +- .../intelligence-drop-in.ts | 24 +- examples/intelligence-recommend/README.md | 4 +- .../intelligence-recommend.ts | 4 +- examples/intelligence-webcode/README.md | 2 +- .../intelligence-webcode.ts | 12 +- examples/webcode-matrix/webcode-matrix.ts | 24 +- package.json | 30 +- pnpm-lock.yaml | 235 +- scripts/gen-primitive-catalog.mjs | 2 + scripts/verify-package-exports.mjs | 125 +- scripts/verify-primeintellect-v1.mjs | 192 + skills/build-with-agent-runtime/SKILL.md | 40 +- src/agent/surfaces.ts | 136 +- src/candidate-execution/artifacts.ts | 271 + src/candidate-execution/benchmark-grader.ts | 142 + src/candidate-execution/builder.ts | 158 + src/candidate-execution/bundle.ts | 18 + src/candidate-execution/claim-file-formats.ts | 34 + src/candidate-execution/claim-file-store.ts | 417 + src/candidate-execution/claim-plan.ts | 107 + src/candidate-execution/claim-terminal.ts | 516 + src/candidate-execution/claim.ts | 946 ++ src/candidate-execution/cleanup.ts | 114 + src/candidate-execution/digest.ts | 76 + src/candidate-execution/dispose.ts | 88 + src/candidate-execution/exact-object.ts | 21 + src/candidate-execution/execute.ts | 918 ++ src/candidate-execution/execution-window.ts | 41 + src/candidate-execution/executor-capture.ts | 90 + src/candidate-execution/finalize.ts | 336 + src/candidate-execution/git-materialize.ts | 515 + src/candidate-execution/index.ts | 119 + src/candidate-execution/model-settlement.ts | 281 + src/candidate-execution/outcome-evidence.ts | 333 + src/candidate-execution/output-artifacts.ts | 39 + src/candidate-execution/prepare.ts | 983 ++ src/candidate-execution/prepared-state.ts | 608 + src/candidate-execution/profile.ts | 361 + .../protected-model-port.ts | 504 + .../protected-redaction.ts | 176 + .../protected-trace-store.ts | 160 + src/candidate-execution/recover.ts | 175 + src/candidate-execution/types.ts | 577 + src/candidate-execution/verify.ts | 167 + src/candidate-execution/workspace-archive.ts | 986 ++ src/candidate-execution/workspace-snapshot.ts | 87 + src/conversation/run-persona.test.ts | 100 +- src/conversation/run-persona.ts | 46 +- src/durable/spawn-journal.ts | 1 + src/improvement/agentic-generator.ts | 651 +- src/improvement/improve.test.ts | 760 +- src/improvement/improve.ts | 477 +- src/improvement/improvement-driver.ts | 159 +- src/improvement/index.ts | 26 +- src/improvement/mcp-serve-verifier.ts | 6 +- src/improvement/profile-diff-proposer.test.ts | 147 + src/improvement/profile-diff-proposer.ts | 108 + src/improvement/raw-trace-distiller.test.ts | 27 + src/improvement/raw-trace-distiller.ts | 37 +- src/index.ts | 7 + src/intelligence/capability.test.ts | 18 + src/intelligence/delivery.test.ts | 154 +- src/intelligence/delivery.ts | 207 +- src/intelligence/improvement-cycle.ts | 547 + src/intelligence/index.ts | 425 +- src/intelligence/intelligence.test.ts | 205 +- src/intelligence/resolver.ts | 4 - src/intelligence/with-intelligence.test.ts | 397 + src/intelligence/with-intelligence.ts | 289 + src/mcp/codex-diagnostics.ts | 188 + src/mcp/index.ts | 16 +- src/mcp/local-harness.ts | 1250 +- src/mcp/worktree-harness.ts | 297 +- src/mcp/worktree.ts | 122 +- src/otel-export.ts | 143 +- src/primeintellect/index.ts | 35 + src/primeintellect/package.ts | 413 + src/primeintellect/primeintellect.test.ts | 369 + src/primeintellect/runner.ts | 165 + src/primeintellect/traces.ts | 377 + src/primeintellect/types.ts | 124 + src/primeintellect/validation.ts | 156 + src/runtime/define-leaderboard.test.ts | 2 +- src/runtime/define-leaderboard.ts | 35 +- src/runtime/index.ts | 30 + src/runtime/loop-dispatch.ts | 105 +- src/runtime/personify/trajectory.ts | 3 + src/runtime/sandbox-events.ts | 9 +- src/runtime/strategy.ts | 28 +- src/runtime/structural-rollout.test.ts | 399 + src/runtime/structural-rollout.ts | 683 + src/runtime/supervise/budget.ts | 10 + src/runtime/supervise/driver-executor.ts | 2 + src/runtime/supervise/runtime.ts | 6 + src/runtime/supervise/scope.ts | 1 + src/runtime/supervise/supervisor.ts | 2 + src/runtime/supervise/trajectory-recorder.ts | 42 +- src/runtime/supervise/types.ts | 3 + .../supervise/worktree-cli-executor.ts | 80 +- src/runtime/tool-loop.ts | 9 +- src/runtime/types.ts | 11 +- tests/agent.test.ts | 75 + tests/agentic-generator.test.ts | 704 +- tests/candidate-bundle-builder.test.ts | 390 + tests/candidate-execution-claim.test.ts | 893 ++ tests/candidate-execution-cleanup.test.ts | 69 + tests/candidate-execution-core.test.ts | 244 + tests/candidate-execution-dispose.test.ts | 108 + tests/candidate-execution-execute.test.ts | 1649 +++ tests/candidate-execution-finalize.test.ts | 461 + tests/candidate-execution-model-port.test.ts | 611 + ...ndidate-execution-outcome-evidence.test.ts | 366 + ...ndidate-execution-output-artifacts.test.ts | 75 + tests/candidate-execution-prepare.test.ts | 702 + tests/candidate-execution-recover.test.ts | 342 + tests/candidate-execution-redaction.test.ts | 16 + .../candidate-execution-task-outcome.test.ts | 115 + ...didate-execution-workspace-archive.test.ts | 496 + tests/helpers/candidate-execution-fixture.ts | 402 + tests/improve.test.ts | 21 +- tests/improvement-cycle.test.ts | 573 + tests/improvement-driver.test.ts | 230 +- tests/loops/loop-dispatch.test.ts | 180 +- tests/loops/supervise.test.ts | 15 + tests/loops/trajectory-recorder.test.ts | 12 + tests/mcp-serve-verifier.test.ts | 27 +- tests/mcp/codex-diagnostics.test.ts | 51 + tests/mcp/local-harness.test.ts | 835 +- tests/mcp/worktree-harness.test.ts | 620 + tests/otel-export.test.ts | 77 +- tests/profile-improvement-stack.test.ts | 18 +- tests/runtime/worktree-cli-executor.test.ts | 214 + tsup.config.ts | 2 + typedoc.json | 2 + 219 files changed, 57563 insertions(+), 6161 deletions(-) create mode 100644 bench/CHANGELOG.md create mode 100644 bench/fixtures/pier-agent/no-model-task/environment/Dockerfile create mode 100644 bench/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt create mode 100644 bench/fixtures/pier-agent/no-model-task/instruction.md create mode 100755 bench/fixtures/pier-agent/no-model-task/pre_artifacts.sh create mode 100644 bench/fixtures/pier-agent/no-model-task/task.toml create mode 100644 bench/fixtures/pier-agent/no-model-task/tests/Dockerfile create mode 100644 bench/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt create mode 100755 bench/fixtures/pier-agent/no-model-task/tests/test.sh create mode 100644 bench/pier_agents/__init__.py create mode 100644 bench/pier_agents/candidate_contract.py create mode 100644 bench/pier_agents/process_boundary.py create mode 100644 bench/pier_agents/tangle_candidate.py create mode 100644 bench/pier_agents/tangle_candidate_test.py create mode 100644 bench/pier_agents/workspace_boundary.py create mode 100644 bench/scripts/run-package-tests.mjs create mode 100644 bench/scripts/terminate-pier-trial.mts create mode 100644 bench/scripts/verify-packed-consumer.mjs create mode 100644 bench/scripts/verify-pier-agent.mts create mode 100644 bench/scripts/verify-pier-pair.mts create mode 100644 bench/scripts/verify-pier-recovery.mts create mode 100644 bench/src/benchmarks/_harness.test.mts create mode 100644 bench/src/benchmarks/swe-bench.test.mts create mode 100644 bench/src/hev-eval.mts create mode 100644 bench/src/hev-improve.mts create mode 100644 bench/src/hev-structural.mts create mode 100644 bench/src/mbpp-structural.mts create mode 100644 bench/src/pier-agent.test-fixtures.mts create mode 100644 bench/src/pier-agent.test.mts create mode 100644 bench/src/pier-agent.ts create mode 100644 bench/src/pier-result-grader.mjs create mode 100644 bench/src/pier-result-grader.test.mts create mode 100644 bench/src/pier-result-grader.ts create mode 100644 bench/src/pier-task-outcome.test.mts create mode 100644 bench/src/pier-task-outcome.ts create mode 100644 bench/src/pier-trial-controller.test.mts create mode 100644 bench/src/pier-trial-controller.ts create mode 100644 bench/src/pier-trial-supervisor.mjs create mode 100644 bench/src/smoke-structural-rollout.mts create mode 100644 bench/tsconfig.public.json delete mode 100644 docs/agent-optimization-map.md create mode 100644 docs/api/candidate-execution.md create mode 100644 docs/api/primeintellect.md create mode 100644 docs/design/structural-rollout-integration.md create mode 100644 examples/ablation-suite/aisdk-env.ts create mode 100644 examples/ablation-suite/extract-contracts.mjs create mode 100644 examples/ablation-suite/fixtures/multi-contract/contracts.all.json create mode 100644 examples/ablation-suite/fixtures/multi-contract/contracts.json create mode 100644 examples/ablation-suite/multi-contract-env.ts create mode 100644 examples/ablation-suite/run-aisdk.ts create mode 100644 examples/ablation-suite/run-multi-contract.ts create mode 100644 scripts/verify-primeintellect-v1.mjs create mode 100644 src/candidate-execution/artifacts.ts create mode 100644 src/candidate-execution/benchmark-grader.ts create mode 100644 src/candidate-execution/builder.ts create mode 100644 src/candidate-execution/bundle.ts create mode 100644 src/candidate-execution/claim-file-formats.ts create mode 100644 src/candidate-execution/claim-file-store.ts create mode 100644 src/candidate-execution/claim-plan.ts create mode 100644 src/candidate-execution/claim-terminal.ts create mode 100644 src/candidate-execution/claim.ts create mode 100644 src/candidate-execution/cleanup.ts create mode 100644 src/candidate-execution/digest.ts create mode 100644 src/candidate-execution/dispose.ts create mode 100644 src/candidate-execution/exact-object.ts create mode 100644 src/candidate-execution/execute.ts create mode 100644 src/candidate-execution/execution-window.ts create mode 100644 src/candidate-execution/executor-capture.ts create mode 100644 src/candidate-execution/finalize.ts create mode 100644 src/candidate-execution/git-materialize.ts create mode 100644 src/candidate-execution/index.ts create mode 100644 src/candidate-execution/model-settlement.ts create mode 100644 src/candidate-execution/outcome-evidence.ts create mode 100644 src/candidate-execution/output-artifacts.ts create mode 100644 src/candidate-execution/prepare.ts create mode 100644 src/candidate-execution/prepared-state.ts create mode 100644 src/candidate-execution/profile.ts create mode 100644 src/candidate-execution/protected-model-port.ts create mode 100644 src/candidate-execution/protected-redaction.ts create mode 100644 src/candidate-execution/protected-trace-store.ts create mode 100644 src/candidate-execution/recover.ts create mode 100644 src/candidate-execution/types.ts create mode 100644 src/candidate-execution/verify.ts create mode 100644 src/candidate-execution/workspace-archive.ts create mode 100644 src/candidate-execution/workspace-snapshot.ts create mode 100644 src/improvement/profile-diff-proposer.test.ts create mode 100644 src/improvement/profile-diff-proposer.ts create mode 100644 src/intelligence/improvement-cycle.ts create mode 100644 src/intelligence/with-intelligence.test.ts create mode 100644 src/intelligence/with-intelligence.ts create mode 100644 src/mcp/codex-diagnostics.ts create mode 100644 src/primeintellect/index.ts create mode 100644 src/primeintellect/package.ts create mode 100644 src/primeintellect/primeintellect.test.ts create mode 100644 src/primeintellect/runner.ts create mode 100644 src/primeintellect/traces.ts create mode 100644 src/primeintellect/types.ts create mode 100644 src/primeintellect/validation.ts create mode 100644 src/runtime/structural-rollout.test.ts create mode 100644 src/runtime/structural-rollout.ts create mode 100644 tests/candidate-bundle-builder.test.ts create mode 100644 tests/candidate-execution-claim.test.ts create mode 100644 tests/candidate-execution-cleanup.test.ts create mode 100644 tests/candidate-execution-core.test.ts create mode 100644 tests/candidate-execution-dispose.test.ts create mode 100644 tests/candidate-execution-execute.test.ts create mode 100644 tests/candidate-execution-finalize.test.ts create mode 100644 tests/candidate-execution-model-port.test.ts create mode 100644 tests/candidate-execution-outcome-evidence.test.ts create mode 100644 tests/candidate-execution-output-artifacts.test.ts create mode 100644 tests/candidate-execution-prepare.test.ts create mode 100644 tests/candidate-execution-recover.test.ts create mode 100644 tests/candidate-execution-redaction.test.ts create mode 100644 tests/candidate-execution-task-outcome.test.ts create mode 100644 tests/candidate-execution-workspace-archive.test.ts create mode 100644 tests/helpers/candidate-execution-fixture.ts create mode 100644 tests/improvement-cycle.test.ts create mode 100644 tests/mcp/codex-diagnostics.test.ts create mode 100644 tests/mcp/worktree-harness.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c0dcc1f..b90e790e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,3 +39,31 @@ jobs: - name: Docs (regenerate API reference + freshness gate) run: pnpm run docs:check + + agent-bench: + runs-on: ubuntu-latest + defaults: + run: + working-directory: bench + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: bench/pnpm-lock.yaml + + - name: Install published dependencies + run: pnpm install --frozen-lockfile + + - name: Typecheck public entrypoint against installed packages + run: pnpm run typecheck:public + + - name: Test deterministic package paths + run: pnpm test + + - name: Verify packed package in a clean consumer + run: pnpm run verify:package diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 44fd144b..416bccf8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,10 +4,12 @@ on: push: tags: - 'v*' + - 'agent-bench-v*' workflow_dispatch: jobs: verify: + if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -80,7 +82,86 @@ jobs: # agent-runtime, workflow publish.yml. - name: Publish to npm (OIDC trusted publishing) run: | - npm install -g npm@latest + # npm 12.0.0 ships a broken provenance publish (libnpmpublish requires + # sigstore without bundling it); OIDC needs npm >= 11.5.1, so pin major 11. + npm install -g npm@11 + NAME=$(node -p "require('./package.json').name") + VERSION=$(node -p "require('./package.json').version") + if npm view "$NAME@$VERSION" version >/dev/null 2>&1; then + echo "$NAME@$VERSION already on registry; skipping publish" + else + npm publish --provenance --access public + fi + + verify-agent-bench: + # Manual dispatch remains root-only. Re-run a failed bench tag workflow so + # tag/version locking stays intact and the root package is never co-published. + if: startsWith(github.ref, 'refs/tags/agent-bench-v') + runs-on: ubuntu-latest + defaults: + run: + working-directory: bench + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: bench/pnpm-lock.yaml + + - name: Install deps + run: pnpm install --frozen-lockfile + + - name: Typecheck public entrypoint against installed packages + run: pnpm run typecheck:public + + - name: Test deterministic package paths + run: pnpm test + + - name: Verify packed package in a clean consumer + run: pnpm run verify:package + + - name: Verify tag/version lock + run: | + NPM_VERSION=$(node -p "require('./package.json').version") + TAG_VERSION="${GITHUB_REF#refs/tags/agent-bench-v}" + if [ "$TAG_VERSION" != "$NPM_VERSION" ]; then + echo "::error::Tag/version mismatch: tag=$TAG_VERSION package=$NPM_VERSION." + exit 1 + fi + echo "Version locked: $NPM_VERSION" + + publish-agent-bench: + needs: verify-agent-bench + if: startsWith(github.ref, 'refs/tags/agent-bench-v') + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + defaults: + run: + working-directory: bench + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: bench/pnpm-lock.yaml + + - run: pnpm install --frozen-lockfile + + - name: Publish to npm (OIDC trusted publishing) + run: | + # Requires the npmjs Trusted Publisher on @tangle-network/agent-bench: + # org tangle-network, repo agent-runtime, workflow publish.yml. + npm install -g npm@11 NAME=$(node -p "require('./package.json').name") VERSION=$(node -p "require('./package.json').version") if npm view "$NAME@$VERSION" version >/dev/null 2>&1; then diff --git a/README.md b/README.md index 1223ec21..709ab998 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ pnpm add @tangle-network/agent-runtime @tangle-network/agent-eval @tangle-networ - [Supervise a team of agents](#supervise-a-team-of-agents) - [Improve an agent](#improve-an-agent) - [Improve a knowledge base](#improve-a-knowledge-base) +- [Run on PrimeIntellect](#run-on-primeintellect) - [How it works](#how-it-works-the-short-version) - [Examples](#examples) - [Where to go next](#where-to-go-next) @@ -33,6 +34,7 @@ pnpm tsx examples/driver-loop/driver-loop.ts | Have one agent **supervise a team of agents** toward a goal | `supervise(profile, task, opts)` | | **Improve** an agent and prove the gain on fresh tasks | `improve(profile, findings, opts)` | | **Improve** a knowledge base with agents, checks, and safe promotion | `runKnowledgeImprovementJob(...)` | +| Evaluate or train the same agent on **PrimeIntellect** | `createPrimeIntellectPackage(...)` | ### Run a chat turn @@ -68,21 +70,34 @@ const result = await supervise( ### Improve an agent -`improve` optimizes one part of an agent (its prompt, skills, or code) and **only ships a change if it beats the current agent on tasks it never practiced on**. Registering an agent for self-improvement cannot ship a worse candidate unless the caller supplies a bad measurement. +`improve` optimizes one part of an agent and **only ships a change if it beats the current agent on tasks it never practiced on**. +It accepts prompt, skill document, curated memory, tool, MCP, hook, subagent, whole-profile, and code surfaces through one call. +Prompt, skill-document, and memory optimization have built-in generators; structured profile surfaces take an explicit generator, and code runs from isolated incumbent and candidate checkouts. +Workflow and rollout-policy files use the code surface so the measured winner is an exact patch that can be sealed and executed; JSON parameter sweeps use agent-eval's `parameterSweepProposer` instead of a runtime-specific optimizer. ```ts import { improve } from '@tangle-network/agent-runtime' const { profile, shipped, lift } = await improve(baseProfile, findings, { - surface: 'prompt', // what to optimize: prompt | skills | code + surface: 'prompt', // or skills/memory/tools/mcp/hooks/subagents/agent-profile/code gate: 'holdout', // certified on a held-back exam, never the practice set scenarios, judge, agent, // how to measure a candidate }) ``` +Curated memory is an external lesson document, not a knowledge store. Supply its current text and persist only the promoted winner: + +```ts +await improve(baseProfile, findings, { + surface: 'memory', + memory: { document: currentLessons, writeBack: saveLessons }, + scenarios, judge, agent, +}) +``` + ### Improve a knowledge base -`runKnowledgeImprovementJob` is the runtime-owned front door for KB, wiki, memory-backed, and RAG improvement jobs. It creates a candidate copy, runs supervised agents against it, checks readiness through `@tangle-network/agent-knowledge`, measures spend and timing, and promotes only when the candidate passes. +`runKnowledgeImprovementJob` is the runtime-owned front door for KB, wiki, memory-backed, and RAG improvement jobs. It creates a candidate copy, runs supervised agents against it, checks readiness through `@tangle-network/agent-knowledge`, measures spend and timing, and promotes only when the candidate passes. Use `improve(..., { surface: 'memory' })` for the agent's curated lesson document; use this job for source, retrieval, and knowledge-store changes. ```ts import { runKnowledgeImprovementJob } from '@tangle-network/agent-runtime/knowledge' @@ -100,6 +115,67 @@ console.log(result.promoted, result.measurement.supervisedSpent) Use it when the product needs one knob for "make this knowledge base better" instead of wiring `improveKnowledgeBase`, a runtime supervisor, candidate workspaces, readiness checks, and promotion tracking by hand. +### Run on PrimeIntellect + +`@tangle-network/agent-runtime/primeintellect` packages typed train and eval tasks as a Verifiers v1 environment. +Prime launches your actual runtime program against an intercepted model endpoint, so `runPersonified`, `runAgentic`, product agents, tool calls, and multiple rounds stay intact. +Reference answers remain in Prime's task process and never enter the agent workspace. +The runner file must be one executable bundle containing the app and its runtime dependencies. + +```ts +import { readFile } from 'node:fs/promises' +import { + createPrimeIntellectPackage, + writePrimeIntellectPackage, +} from '@tangle-network/agent-runtime/primeintellect' + +const bundledRunner = await readFile('./dist/prime-runner.mjs', 'utf8') +const bundle = createPrimeIntellectPackage({ + name: 'support-agent-v1', + version: '1.0.0', + tasks: [ + { + id: 'train-refund-policy', + split: 'train', + prompt: 'Can a subscription renewal be refunded?', + answer: 'No', + }, + { + id: 'eval-final-sale', + split: 'eval', + prompt: 'Can a final-sale order be refunded?', + answer: 'No', + }, + ], + scoring: { kind: 'exact', normalization: 'trim-casefold' }, + runner: { + image: 'node:22-bookworm-slim', + files: { 'runner.mjs': bundledRunner }, + command: ['node', 'runner.mjs'], + }, +}) + +await writePrimeIntellectPackage(bundle, './prime/support-agent-v1') +``` + +The runner reads the episode and uses the normal runtime APIs: +Here, `runProductAgent` is the application's existing entry point, not another loop supplied by this adapter. + +```ts +import { + createPrimeIntellectBackend, + runPrimeIntellectProgram, +} from '@tangle-network/agent-runtime/primeintellect' + +await runPrimeIntellectProgram(async (episode) => { + const backend = createPrimeIntellectBackend(episode) + return runProductAgent({ task: episode.task, backend }) +}) +``` + +Prime writes complete `traces.jsonl` rows. +Use `importPrimeIntellectTraces(...)` to convert them to agent-eval `RunRecord`s for the existing reports and release checks. + ## How it works (the short version) - **One agent, run two ways.** The same agent runs at "do the task" speed and at "get better at the task" speed. "Driver", "worker", and "coordinator" are roles one agent plays, not separate types. @@ -121,6 +197,7 @@ Runnable, grouped by what they show. Copy the one nearest your task: | Trace + bill + effort-gate the WebCode benchmark (the Intelligence SDK) | [`intelligence-webcode`](./examples/intelligence-webcode) | | Self-improve an agent, gated on a held-out set | [`improve`](./examples/improve) · [`self-improving-coder`](./examples/self-improving-coder) | | Improve a KB, wiki, or RAG corpus with runtime agents | [`docs/canonical-api.md`](./docs/canonical-api.md) | +| Evaluate or train a runtime program on PrimeIntellect | `@tangle-network/agent-runtime/primeintellect` | | Study coordination vs raw compute | [`ablation-suite`](./examples/ablation-suite) | All 29 live in [`examples/`](./examples). @@ -130,7 +207,7 @@ All 29 live in [`examples/`](./examples). - New here? [`docs/concepts.md`](./docs/concepts.md), the mental model in plain terms. - [`docs/canonical-api.md`](./docs/canonical-api.md), find the primitive: "I want to ___ → use ___". - [`docs/api/primitive-catalog.md`](./docs/api/primitive-catalog.md), every export in one generated, never-stale list with its import path. Check it before building anything new. -- Import subpaths: the root export is the product surface (`handleChatTurn`, `improve`); deeper capabilities ship as subpaths: `/loops` (multi-agent + the loop kernel), `/knowledge` (KB improvement), `/mcp` (tool servers), `/intelligence` (observability drop-in), `/lifecycle`, `/agent`, `/profiles`, `/platform`, `/analyst-loop`, `/environment-provider`. +- Import subpaths: the root export is the product surface (`handleChatTurn`, `improve`); deeper capabilities ship as subpaths: `/loops` (multi-agent + the loop kernel), `/knowledge` (KB improvement), `/primeintellect` (Prime task, runtime, and trace adapter), `/mcp` (tool servers), `/intelligence` (observability drop-in), `/lifecycle`, `/agent`, `/profiles`, `/platform`, `/analyst-loop`, `/environment-provider`. - [`docs/architecture.md`](./docs/architecture.md), the design, end to end. - [`bench/HARNESS.md`](./bench/HARNESS.md), the experiment harness and how to run a benchmark. diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md new file mode 100644 index 00000000..6a1f9781 --- /dev/null +++ b/bench/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +## 0.3.3 + +- Consume `@tangle-network/agent-runtime@0.94.9` so large candidate task-outcome and isolated-memory archives are validated before persistence, then recorded as artifact references instead of oversized embedded payloads. + +## 0.3.2 + +- Run large Pier workspace identity checks from a hash-verified evaluator file, avoiding Linux command-size limits while preserving exact path, type, mode, length, hardlink, and content checks. diff --git a/bench/HARNESS.md b/bench/HARNESS.md index 681ce992..f7a7dc62 100644 --- a/bench/HARNESS.md +++ b/bench/HARNESS.md @@ -3,7 +3,7 @@ If you're an agent picking this up: read this page, then run `pnpm help` + `pnpm gate` — do NOT re-derive the harness from source. This map is SHORT on purpose; if it disagrees with the code, the code wins — fix this page in the same turn (the anti-rediscovery law). -Verified against source 2026-07-07 · agent-eval pinned `^0.106.1`. The CANONICAL surface is now +Verified against source 2026-07-11 · agent-eval pinned `^0.114.0`. The CANONICAL surface is now the published optimization suite (`@tangle-network/agent-runtime/loops`): `Environment` + `Strategy`/`defineStrategy` + `runBenchmark` — see the section below FIRST. The recursive diverse-vs-blind gate runs through the keystone (`gate-cli.mts` → `runGate`); @@ -134,6 +134,7 @@ the gate + measurement tools: run-benchmarks-cli.mts runBenchmarks: any subset of the ADAPTERS registry × model/harness cells, one combined ranked report (#420) commit0-env-run.mts the HARD domain through `runBenchmark` (the optimization suite) terminal-compare.ts Terminal-Bench compare (own main) + pnpm verify:pier zero-model failure/pass Pier controls through a separate verifier unit tests (the only fully-green, cred-free runnable surface besides offline replay): node --test --import tsx src/{selector,refine-loop}.test.mts tsx src/gate.test.mts # offline plumbing test (no creds) @@ -209,12 +210,81 @@ The code-benches share `benchmarks/_harness.ts` (stage artifact → run the benc in a `.venv`/Docker subprocess → parse its JSON report → `{resolved,score}`). No per-adapter copy of the process/venv/Docker/temp/report plumbing; commit0+appworld also share its stdin-piping runner (`runVenvScriptStdin`). +Published-package consumers set `AGENT_BENCH_PYTHON` to the absolute path of an interpreter containing the benchmark dependencies. +If unset, source checkouts keep using `bench/.venv/bin/python`. +SWE-bench callers that need a hard evaluation deadline construct `createSweBenchAdapter({ timeoutMs })`; evaluator errors and incomplete or ambiguous reports throw instead of scoring zero. +Callers that must inspect the exact per-task image after scoring pass `cacheLevel: 'instance'`; the default remains `'env'`. +Callers retain the complete official evaluator tree plus raw process output with `captureEvaluatorArtifacts: ({ taskId, attemptSequence }) => ({ destination })`; the returned score (or `StagedJudgeError`) carries a receipt with every file hash and a whole-tree hash. - **Real, runnable with ZERO extra deps:** finsearchcomp (GitHub dataset + fixtures + LLM judge — the gate bench), hotpotqa + simpleqa + frames (HF/web QA + F1/LLM judge; `*_FIXTURES=1` offline), **ragbench**, **crag**, **nomiracl**, **open-rag-bench**, **t2-ragbench** (SOTA RAG/knowledge benchmarks with committed fixtures and deterministic answer/relevance judges; live mode reads explicit `*_DATA_FILE` JSON/JSONL exports), **aec-bench** (real GitHub task tree + fixtures; judge = the task's own `tests/verify.py` over python3 stdlib — **deterministic, graded per-field partial credit, no Docker, no LLM** → the candidate non-oracle correctable-middle-band bench for the open gate). - **Real code, needs an external harness/tools to run (fail loud with the exact install/Docker fix; never a fabricated score):** swe-bench + terminal-bench (`bench/.venv` + Docker), **commit0** (ISOLATED `bench/.venv-commit0` via `python3 -m venv bench/.venv-commit0 && bench/.venv-commit0/bin/pip install commit0 datasets` — its deps conflict with the shared `.venv`; override dir with `COMMIT0_VENV` — plus Docker; judge = official pytest harness, graded (passed+xfail)/total; the rollout prompt stages in-box (clones `commit-0/` @ `base_commit`, emits `git diff`); `COMMIT0_FIXTURES=1` for offline listing), **programbench** (`pip install programbench` + Docker on linux/amd64 + HF blobs; judge = official cleanroom eval, graded passed/total; `PROGRAMBENCH_FIXTURES=1` offline), **appworld** (`pip install appworld` + `appworld install` + `appworld download data`; judge = AppWorld's own `world.evaluate()`, graded passes/num_tests — NO committed fixture: task data exists only after `download data`, so loadTasks fails loud rather than fabricate a task), **dabstep** (`DABSTEP_DIR=/path/to/EnvCommons/DABStep` with the released `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`; judge delegates to official `grade.py`; `DABSTEP_FIXTURES=1` only tests adapter plumbing and does not fabricate benchmark scores), **webarena-verified** (`WEBARENA_VERIFIED_DIR=/path/to/webarena-verified`; judge delegates to official `eval-tasks` over a run output directory), **tau2-bench** (`TAU2_BENCH_DIR=/path/to/tau2-bench`; judge recomputes tau2 trajectory rewards), **tau3-banking** (`TAU3_BENCH_DIR=/path/to/tau2-bench`; default domain `banking_knowledge`; judge recomputes tau trajectory rewards through the upstream tau3 package), **agentbench** DBBench subset (`AGENTBENCH_DIR=/path/to/AgentBench`; exact-match deterministic label judge), **bfcl** deterministic function-call subset (`BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard`; loads official BFCL JSONL + `possible_answer`; score = structured call/argument match, not the full BFCL leaderboard evaluator), **toollm** API-selection subset (`TOOLBENCH_DIR=/path/to/ToolBench`; score = recall of ToolBench `relevant APIs` labels, resolved only when the worker emits the requested structured JSON call list; official ToolEval pass rate remains LLM-judged/stochastic), **finresearchbench** (`FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl`; rows must carry official `judge_system_prompt` + `judge_prompt_template`; no self-authored live judge), mind2web, cad-design + cadbench + cadgenbench (openscad/blender/build123d). - **goldArtifact:** aec-bench returns the task's real `golden_pass.md` (verify-judge works fully offline). commit0 / programbench / appworld return `undefined` — the oracle is a git ref / stripped source / engine-bundled solution, not a portable string; judge correctness is proven by a real solve through the harness, not a synthetic gold (documented + fail-loud, not a fake). - **Absent (not built):** swe-gym, swe-bench-multimodal, and the rest of the survey set. Every unbuilt/scaffold adapter fails LOUD (throws with the integration step) rather than faking a score — no silent zeros in any corpus. Offline fixture tests: `benchmarks/{aec-bench,commit0,programbench,appworld,rag-benchmarks}.test.mts` (`tsx --test`). +## Pier candidate bridge + +`pier_agents.tangle_candidate:TangleCandidateAgent` is the reusable Pier custom-agent path for frozen Tangle candidates. +`executePreparedPierCandidate()` is the only public entry point; its private staging step writes the runtime's execution-plan and materialization-receipt bytes verbatim. +The Python bridge rechecks those bytes and their signed task, candidate, profile, repository, instruction, and workspace identities before launch. +It rejects a raw candidate bundle, never projects an `AgentProfile`, and leaves task isolation, patch transfer, verification, retries, and result storage to Pier. + +The adapter fails the trial when any prepared identity drifts, when the task checkout or immutable OCI image differs from the signed identity, or when the candidate exits nonzero or exceeds its signed deadline. +Candidate code runs as an unprivileged numeric user, while evaluator inputs and timeout evidence remain root-owned. +It never accepts candidate-authored token, cost, or trace receipts; `executePreparedPierCandidate()` uses the runtime's atomic execution path and reconciles the protected `TraceStore` with the model-gateway ledger before returning a gradable receipt. +The prepared object contains no credentials. +The runtime passes model and trace bindings only to the trusted executor request. +The Pier launcher inherits their values through its protected process environment and passes only `${NAME}` references on the command line, so credentials never enter prepared bytes, CLI arguments, or job files. +The executor builds fresh task, candidate, and profile trees from the request's exact verified file bytes; prepared staging directories are never launch authority. +Isolated memory and knowledge-bearing candidates currently fail closed until Pier has executor-owned mount and after-state capture. +The signed wall deadline is a hard stop: the runtime aborts, Pier kills the process tree, and the executor acknowledges process and container death. +The signed tool-step count is a post-run validity check over protected traces, not a pre-tool stop; generic black-box Pier processes cannot honestly prevent step N+1. +The executable zero-model fixture is `fixtures/pier-agent/`; run it against the R360 Pier checkout with `PIER_REPO=/path/to/pier pnpm verify:pier`. +That command runs a no-change candidate that must score 0/1, proves a fresh evaluator process can kill a persisted child and remove its real Docker container, and runs a known-good candidate that must score 1/1. +It then checks that each official result and exact task patch is bound into its own runtime receipt with zero model usage. + +For a real frozen candidate, use `FilePierCandidateTrialController`, append `agentArgs` and `attemptArgs`, and pass each executor-only `evaluatorEnv` entry through Pier's evaluator-owned environment mechanism. +The launch callback's second argument carries the signed runtime `request`, protected `traceStore`, cancellation `signal`, and absolute `deadlineAtMs`; production launchers must use that context rather than reconstructing it. +The controller sends secrets to its supervisor over a pipe, while its durable files contain only process and Docker-project identities: +The supervisor never inherits `process.env`; non-default Docker connections use a stable `dockerConnection.id` plus the exact environment injected into both Pier and cleanup. +Fresh recovery workers must reconstruct that same named connection; `terminate-pier-trial.mts` selects only the comma-delimited variables named by `PIER_DOCKER_ENV_NAMES` when `PIER_DOCKER_CONNECTION_ID` is set. +`jobName` must be unique per prepared execution; the controller atomically reserves that job directory so recovery can remove only containers owned by that execution. + +```ts +const controller = new FilePierCandidateTrialController({ + directory: '/var/lib/tangle/pier-control', + launch: (staged, { request }) => { + const jobName = request.executionId + return { + command: 'uv', + args: ['run', 'pier', 'run', ...staged.agentArgs, ...staged.attemptArgs], + cwd: pierCheckout, + env: { ...evaluatorEnvironment, ...staged.evaluatorEnv }, + jobsDirectory, + jobName, + readResult: () => readOfficialPierResult(jobsDirectory, jobName), + } + }, +}) + +const result = await executePreparedPierCandidate({ + prepared, + directory: '/sealed/candidate', + pierVersion: '0.3.0', + traceStore, + claimStore, + outputArtifacts, + grader, + controller, +}) +``` + +The controller's result resolves only after normal process/container cleanup. +On a signed deadline, external abort, or recovery by another evaluator process, the adapter waits for `terminateAndWait()` to acknowledge both process exit and container removal before it returns control to the runtime. + +One prepared execution always maps to one Pier attempt (`--n-attempts 1 --max-retries 0`). +Production callers pass a long-lived `FileAgentCandidateExecutionClaimStore`; an in-memory claim store is test-only and cannot prevent a second process from replaying the same attempt. +Any allowed pre-model infrastructure retry is a new prepared execution with its own counted attempt identity. + ## Is it runnable RIGHT NOW? (verify the map, don't trust it blindly) ``` ls src/*.mts src/*.ts # the real tool list (each its own main — source of truth) diff --git a/bench/README.md b/bench/README.md index 522525fc..b3c1d2d3 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,6 +1,6 @@ # agent-runtime-bench -Private experiment workspace nested in agent-runtime; decoupled from its build/lint/release (the package builds `src/`, lints `src tests examples` — `bench/` is none of those). +Published as `@tangle-network/agent-bench`, with independent CI and release checks for its TypeScript and Python surfaces. **Read [`bench/HARNESS.md`](./HARNESS.md) FIRST.** It is the one maintained map: the commands, the `rollout → corpus → selector → CI → gate` data flow, the canonical-suite table, the wired/needs-creds/scaffolded matrix, and the gate one-liners — kept verified against source. @@ -13,3 +13,28 @@ pnpm install # tsx + link parent ``` The judge needs only Docker; workers need a model key (Tangle router `TANGLE_API_KEY`, or a direct provider). + +Retain every official per-test log and report before the temporary evaluator directory is removed: + +```ts +const adapter = createSweBenchAdapter({ + captureEvaluatorArtifacts: ({ taskId, attemptSequence }) => ({ + destination: path.join(runDirectory, taskId, String(attemptSequence)), + }), +}) +const score = await adapter.judge(task, patch) +console.log(score.judgeArtifacts?.manifestPath) +``` + +Each destination contains the untouched evaluator tree under `evaluator/`, raw `stdout.bin` and `stderr.bin` under `process/`, and `receipt.json` with per-file SHA-256 values plus a whole-tree SHA-256. +The destination must be unique and absent; an existing path fails loud instead of overwriting evidence. +Failed evaluators throw `StagedJudgeError` with the same `judgeArtifacts` receipt after retaining partial logs. + +## Pier custom candidates + +The package executes a branded `PreparedAgentCandidateExecution` from `@tangle-network/agent-runtime` through one atomic API and ships `pier_agents.tangle_candidate:TangleCandidateAgent` as its thin Pier transport. +The executor recreates every input from runtime-verified file bytes and reveals model credentials only inside the claimed execution callback. +Pier owns the task container and verifier; protected model usage and traces stay in `@tangle-network/agent-eval` and are finalized by the shared runtime. +`FilePierCandidateTrialController` atomically reserves a unique Pier job, then persists the supervisor PID, process-session identity, and that job's exact Docker projects so a fresh evaluator process can stop and remove an abandoned trial. +Run `PIER_REPO=/path/to/pier pnpm verify:pier` for the zero-model failure/pass and fresh-process recovery proof, and see `HARNESS.md` for the exact invocation and failure contract. +From an installed npm package, expose the shipped Python module with `export PYTHONPATH="$(npm root)/@tangle-network/agent-bench${PYTHONPATH:+:$PYTHONPATH}"` before invoking Pier. diff --git a/bench/fixtures/pier-agent/no-model-task/environment/Dockerfile b/bench/fixtures/pier-agent/no-model-task/environment/Dockerfile new file mode 100644 index 00000000..775ff6ce --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/environment/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:24.04 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY seed/ /app/ +RUN git init \ + && git config user.email fixture@tangle.tools \ + && git config user.name "Tangle Fixture" \ + && git add -A \ + && GIT_AUTHOR_DATE=2000-01-01T00:00:00Z \ + GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + git commit -m baseline \ + && git tag benchmark-base diff --git a/bench/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt b/bench/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt new file mode 100644 index 00000000..aee83dc4 --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt @@ -0,0 +1 @@ +not-ready diff --git a/bench/fixtures/pier-agent/no-model-task/instruction.md b/bench/fixtures/pier-agent/no-model-task/instruction.md new file mode 100644 index 00000000..1ab44251 --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/instruction.md @@ -0,0 +1,6 @@ +Change src/status.txt so it contains exactly: + +ready +owner=tangle + +Commit the change to git when finished. diff --git a/bench/fixtures/pier-agent/no-model-task/pre_artifacts.sh b/bench/fixtures/pier-agent/no-model-task/pre_artifacts.sh new file mode 100755 index 00000000..b9cc6aec --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/pre_artifacts.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +mkdir -p /logs/artifacts +cd /app +git -c safe.directory=/app diff --binary benchmark-base..HEAD > /logs/artifacts/model.patch diff --git a/bench/fixtures/pier-agent/no-model-task/task.toml b/bench/fixtures/pier-agent/no-model-task/task.toml new file mode 100644 index 00000000..2080342b --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/task.toml @@ -0,0 +1,35 @@ +version = "1.0" + +[task] +name = "agent-bench/pier-candidate-no-model" +authors = [] +keywords = ["pier", "candidate", "no-model"] + +[metadata] +author_name = "Tangle Research" +author_email = "research@tangle.tools" +difficulty = "synthetic" +category = "software-engineering" +tags = ["no-model", "separate-verifier"] + +[agent] +timeout_sec = 60.0 + +[verifier] +timeout_sec = 60.0 +environment_mode = "separate" + +[verifier.environment] + +[environment] +build_timeout_sec = 300.0 +cpus = 1 +memory_mb = 1024 +storage_mb = 2048 +gpus = 0 +allow_internet = false +mcp_servers = [] + +[verifier.env] + +[solution.env] diff --git a/bench/fixtures/pier-agent/no-model-task/tests/Dockerfile b/bench/fixtures/pier-agent/no-model-task/tests/Dockerfile new file mode 100644 index 00000000..323cf830 --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/tests/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:24.04 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY seed/ /app/ +RUN git init \ + && git config user.email fixture@tangle.tools \ + && git config user.name "Tangle Fixture" \ + && git add -A \ + && GIT_AUTHOR_DATE=2000-01-01T00:00:00Z \ + GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + git commit -m baseline \ + && git tag benchmark-base +COPY test.sh /tests/test.sh diff --git a/bench/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt b/bench/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt new file mode 100644 index 00000000..aee83dc4 --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt @@ -0,0 +1 @@ +not-ready diff --git a/bench/fixtures/pier-agent/no-model-task/tests/test.sh b/bench/fixtures/pier-agent/no-model-task/tests/test.sh new file mode 100755 index 00000000..bdd162f6 --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/tests/test.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +reward=0 +patch_applied=0 +patch=/logs/artifacts/model.patch + +if [ -s "$patch" ] \ + && git -c safe.directory=/app apply --check "$patch" \ + && git -c safe.directory=/app apply "$patch"; then + patch_applied=1 +fi +if [ "$(cat /app/src/status.txt 2>/dev/null || true)" = $'ready\nowner=tangle' ] \ + && [ ! -e /app/AGENTS.md ]; then + reward=1 +fi + +printf '{"reward":%d,"patch_applied":%d}\n' "$reward" "$patch_applied" \ + > /logs/verifier/reward.json diff --git a/bench/package.json b/bench/package.json index 8d4aad86..c29a8aeb 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,8 +1,13 @@ { "name": "@tangle-network/agent-bench", - "version": "0.1.0", + "version": "0.3.3", "type": "module", "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.", + "repository": { + "type": "git", + "url": "git+https://github.com/tangle-network/agent-runtime.git", + "directory": "bench" + }, "main": "src/index.ts", "types": "src/index.ts", "exports": { @@ -15,24 +20,42 @@ "gate-cli": "tsx src/gate-cli.mts", "run-benchmarks": "tsx src/run-benchmarks-cli.mts", "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl", - "terminal-compare": "tsx src/terminal-compare.ts" + "terminal-compare": "tsx src/terminal-compare.ts", + "test": "node scripts/run-package-tests.mjs", + "typecheck:public": "tsc -p tsconfig.public.json", + "verify:package": "node scripts/verify-packed-consumer.mjs", + "verify:pier": "tsx scripts/verify-pier-pair.mts" }, "dependencies": { - "@tangle-network/agent-eval": "^0.106.3", - "@tangle-network/agent-runtime": "^0.89.0", - "@tangle-network/sandbox": "^0.9.7" + "@tangle-network/agent-eval": "^0.117.1", + "@tangle-network/agent-interface": "^0.25.0", + "@tangle-network/agent-runtime": "0.94.9", + "@tangle-network/sandbox": "^0.10.3" }, "devDependencies": { + "@types/node": "^25.0.0", "tsx": "^4.19.0", "typescript": "^6.0.3" }, + "pnpm": { + "overrides": { + "@tangle-network/agent-eval": "0.117.1" + } + }, "files": [ "src", "fixtures", "scripts", + "pier_agents/__init__.py", + "pier_agents/candidate_contract.py", + "pier_agents/process_boundary.py", + "pier_agents/tangle_candidate.py", + "pier_agents/workspace_boundary.py", "tb_agents/*.py", "steerers", - "README.md" + "CHANGELOG.md", + "README.md", + "HARNESS.md" ], "publishConfig": { "access": "public" diff --git a/bench/pier_agents/__init__.py b/bench/pier_agents/__init__.py new file mode 100644 index 00000000..a27997bc --- /dev/null +++ b/bench/pier_agents/__init__.py @@ -0,0 +1,18 @@ +"""Pier custom agents shipped by ``@tangle-network/agent-bench``.""" + +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from .tangle_candidate import TangleCandidateAgent + +__all__ = ["TangleCandidateAgent"] + + +def __getattr__(name: str) -> Any: + """Load Pier only when callers request the custom agent.""" + if name != "TangleCandidateAgent": + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from .tangle_candidate import TangleCandidateAgent + + return TangleCandidateAgent diff --git a/bench/pier_agents/candidate_contract.py b/bench/pier_agents/candidate_contract.py new file mode 100644 index 00000000..5d667163 --- /dev/null +++ b/bench/pier_agents/candidate_contract.py @@ -0,0 +1,747 @@ +"""Strict reader for runtime-prepared candidate execution artifacts. + +The TypeScript runtime is the authority that verifies bundles and creates the +canonical execution plan and materialization receipt. This module performs the +small, security-critical replay checks needed before Pier copies those prepared +bytes into a task container. It deliberately does not define another plan. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import math +import re +import stat +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + + +_SHA256_RE = re.compile(r"^sha256:[a-f0-9]{64}$") +_GIT_OBJECT_RE = re.compile(r"^(?:[a-f0-9]{40}|[a-f0-9]{64})$") +_ENV_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_EXECUTABLE_RE = re.compile(r"^[A-Za-z0-9._+/-]+$") +_DNS_LABEL_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") +_SHELLS = {"sh", "bash", "zsh", "fish", "cmd", "cmd.exe", "powershell", "pwsh"} +_BLOCKED_GATEWAY_DOMAINS = { + "localhost", + "metadata.google.internal", + "metadata.google", +} +_RESERVED_PATH_PARTS = {".git", ".sidecar"} +_PLAN_KIND = "agent-candidate-execution-plan-material" +_RECEIPT_KIND = "agent-candidate-materialization" +_PROFILE_PLAN_KIND = "agent-profile-workspace-plan" +_EXECUTION_EVIDENCE_KIND = "agent-candidate-execution-plan" +_STDIN_TASK_PATH = "/tangle/input/stdin.txt" + + +class CandidateContractError(ValueError): + """Prepared execution bytes are missing, malformed, or inconsistent.""" + + +@dataclass(frozen=True) +class InstructionEvidence: + sha256: str + byte_length: int + delivery_kind: str + delivery_env: str | None = None + delivery_path: str | None = None + + +@dataclass(frozen=True) +class WorkspaceFile: + path: str + mode: int + sha256: str + byte_length: int + + +@dataclass(frozen=True) +class ProfileFile: + path: str + mode: int + sha256: str + + +@dataclass(frozen=True) +class PreparedCandidateContract: + plan: dict[str, Any] + plan_raw: bytes + receipt: dict[str, Any] + receipt_raw: bytes + bundle_digest: str + execution_id: str + execution_plan_digest: str + materialization_receipt_digest: str + instruction: InstructionEvidence + repository_base_commit: str + repository_base_tree: str + task_root: str + candidate_root: str | None + task_files: tuple[WorkspaceFile, ...] + candidate_files: tuple[WorkspaceFile, ...] + profile_target: str + profile_files: tuple[ProfileFile, ...] + executable: str + args: tuple[str, ...] + env: dict[str, str] + cwd_root: str + cwd_path: str + timeout_ms: int + requested_model: str + resolved_model: dict[str, Any] + model_network_domains: tuple[str, ...] + container: dict[str, Any] + + +def sha256_bytes(data: bytes) -> str: + return f"sha256:{hashlib.sha256(data).hexdigest()}" + + +def _object(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise CandidateContractError(f"{label} must be a JSON object") + return value + + +def _string(value: Any, label: str) -> str: + if not isinstance(value, str) or not value or "\0" in value: + raise CandidateContractError(f"{label} must be a non-empty string without NUL") + return value + + +def _integer(value: Any, label: str, *, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise CandidateContractError(f"{label} must be an integer >= {minimum}") + return value + + +def _number(value: Any, label: str, *, minimum: float = 0) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < minimum + ): + raise CandidateContractError(f"{label} must be a finite number >= {minimum}") + return float(value) + + +def _digest(value: Any, label: str) -> str: + digest = _string(value, label) + if not _SHA256_RE.fullmatch(digest): + raise CandidateContractError(f"{label} must be sha256:<64 lowercase hex>") + return digest + + +def _git_object(value: Any, label: str) -> str: + object_id = _string(value, label) + if not _GIT_OBJECT_RE.fullmatch(object_id): + raise CandidateContractError(f"{label} must be a full Git object id") + return object_id + + +def _safe_relative(value: Any, label: str, *, allow_dot: bool = False) -> str: + path = _string(value, label) + if any(ord(char) < 32 or ord(char) == 127 for char in path) or "\\" in path: + raise CandidateContractError(f"{label} contains a forbidden character") + parsed = PurePosixPath(path) + if parsed.is_absolute(): + raise CandidateContractError(f"{label} must be relative") + if path == ".": + if allow_dot: + return path + raise CandidateContractError(f"{label} must identify a child path") + parts = path.split("/") + if any( + not part or part in {".", ".."} or part in _RESERVED_PATH_PARTS + for part in parts + ): + raise CandidateContractError(f"{label} must be a canonical safe relative path") + return path + + +def _absolute(value: Any, label: str) -> str: + path = _string(value, label) + if any(ord(char) < 32 or ord(char) == 127 for char in path) or "\\" in path: + raise CandidateContractError(f"{label} contains a forbidden character") + parsed = PurePosixPath(path) + if ( + not parsed.is_absolute() + or path == "/" + or ".." in parsed.parts + or str(parsed) != path + ): + raise CandidateContractError(f"{label} must be a canonical absolute POSIX path") + return path + + +def _public(value: Any, label: str) -> str: + obj = _object(value, label) + if set(obj) != {"kind", "value"} or obj.get("kind") != "public": + raise CandidateContractError(f"{label} must be one explicit public value") + return _string(obj.get("value"), f"{label}.value") + + +def _gateway_domain(value: Any, label: str) -> str: + domain = _string(value, label) + labels = domain.split(".") + if ( + len(domain) > 253 + or domain != domain.lower() + or domain.endswith(".") + or ":" in domain + or domain in _BLOCKED_GATEWAY_DOMAINS + or domain.endswith(".localhost") + or len(labels) < 2 + or not all(1 <= len(part) <= 63 and _DNS_LABEL_RE.fullmatch(part) for part in labels) + or not re.search(r"[a-z]", labels[-1]) + ): + raise CandidateContractError( + f"{label} must be an exact lowercase public DNS name" + ) + return domain + + +def _read_json(path: Path, label: str) -> tuple[dict[str, Any], bytes]: + try: + raw = path.read_bytes() + except OSError as exc: + raise CandidateContractError(f"cannot read {label} {path}: {exc}") from exc + try: + value = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise CandidateContractError(f"{label} is not valid UTF-8 JSON: {exc}") from exc + return _object(value, label), raw + + +def _embedded_bytes(value: Any, label: str) -> bytes: + artifact = _object(value, label) + if artifact.get("encoding") != "base64" or not isinstance( + artifact.get("content"), str + ): + raise CandidateContractError(f"{label} must embed exact base64 bytes") + try: + raw = base64.b64decode(artifact["content"], validate=True) + except ValueError as exc: + raise CandidateContractError(f"{label} contains invalid base64") from exc + expected_digest = _digest(artifact.get("sha256"), f"{label}.sha256") + expected_length = _integer(artifact.get("byteLength"), f"{label}.byteLength") + if len(raw) != expected_length or sha256_bytes(raw) != expected_digest: + raise CandidateContractError(f"{label} bytes do not match their identity") + return raw + + +def _workspace_files(value: Any, label: str) -> tuple[WorkspaceFile, ...]: + snapshot = _object(value, label) + if ( + snapshot.get("schemaVersion") != 1 + or snapshot.get("kind") != "agent-candidate-workspace-snapshot" + ): + raise CandidateContractError(f"{label} has the wrong snapshot kind") + material = _object(snapshot.get("material"), f"{label}.material") + if ( + material.get("schemaVersion") != 1 + or material.get("kind") != "agent-candidate-workspace-manifest" + ): + raise CandidateContractError(f"{label}.material has the wrong manifest kind") + values = material.get("files") + if not isinstance(values, list): + raise CandidateContractError(f"{label}.material.files must be an array") + files: list[WorkspaceFile] = [] + for index, value in enumerate(values): + obj = _object(value, f"{label}.material.files[{index}]") + mode = _integer(obj.get("mode"), f"{label}.material.files[{index}].mode") + if mode not in {0o644, 0o755}: + raise CandidateContractError( + f"{label} contains unsupported file mode {mode:o}" + ) + files.append( + WorkspaceFile( + path=_safe_relative( + obj.get("path"), f"{label}.material.files[{index}].path" + ), + mode=mode, + sha256=_digest( + obj.get("sha256"), f"{label}.material.files[{index}].sha256" + ), + byte_length=_integer( + obj.get("byteLength"), f"{label}.material.files[{index}].byteLength" + ), + ) + ) + paths = [file.path for file in files] + if paths != sorted(set(paths)): + raise CandidateContractError(f"{label} file paths must be unique and sorted") + return tuple(files) + + +def _profile_files(value: Any) -> tuple[ProfileFile, ...]: + values = value.get("files") if isinstance(value, dict) else None + if not isinstance(values, list): + raise CandidateContractError("profilePlan.material.files must be an array") + files: list[ProfileFile] = [] + for index, value in enumerate(values): + obj = _object(value, f"profilePlan.material.files[{index}]") + mode = _integer(obj.get("mode"), f"profilePlan.material.files[{index}].mode") + if mode not in {0o644, 0o755}: + raise CandidateContractError( + f"profile plan contains unsupported mode {mode:o}" + ) + files.append( + ProfileFile( + path=_safe_relative( + obj.get("relPath"), f"profilePlan.material.files[{index}].relPath" + ), + mode=mode, + sha256=_digest( + obj.get("contentSha256"), + f"profilePlan.material.files[{index}].contentSha256", + ), + ) + ) + paths = [file.path for file in files] + if paths != sorted(set(paths)): + raise CandidateContractError( + "profile plan file paths must be unique and sorted" + ) + return tuple(files) + + +def _instruction(value: Any) -> InstructionEvidence: + obj = _object(value, "task.instruction") + if obj.get("encoding") != "utf8": + raise CandidateContractError("task.instruction.encoding must equal utf8") + delivery = _object(obj.get("delivery"), "task.instruction.delivery") + kind = delivery.get("kind") + if kind == "argv-append" or kind == "stdin-utf8": + if set(delivery) != {"kind"}: + raise CandidateContractError(f"{kind} delivery has unexpected fields") + env = path = None + elif kind == "utf8-file": + if ( + set(delivery) != {"kind", "env", "path"} + or delivery.get("env") != "TANGLE_CANDIDATE_TASK_PATH" + or delivery.get("path") != "/tangle/input/task.txt" + ): + raise CandidateContractError( + "utf8-file delivery has unexpected fields or does not use the fixed env and path" + ) + env = "TANGLE_CANDIDATE_TASK_PATH" + path = "/tangle/input/task.txt" + else: + raise CandidateContractError("unsupported task instruction delivery") + return InstructionEvidence( + sha256=_digest(obj.get("sha256"), "task.instruction.sha256"), + byte_length=_integer( + obj.get("byteLength"), "task.instruction.byteLength", minimum=1 + ), + delivery_kind=kind, + delivery_env=env, + delivery_path=path, + ) + + +def load_prepared_candidate_contract( + plan_path: Path, + receipt_path: Path, + expected_receipt_digest: str, +) -> PreparedCandidateContract: + """Load exact runtime bytes and prove the receipt binds the plan.""" + + plan, plan_raw = _read_json(plan_path, "execution plan") + receipt, receipt_raw = _read_json(receipt_path, "materialization receipt") + receipt_digest = _digest(expected_receipt_digest, "expected receipt digest") + if sha256_bytes(receipt_raw) != receipt_digest: + raise CandidateContractError( + "materialization receipt bytes do not match the expected digest" + ) + if plan.get("schemaVersion") != 1 or plan.get("kind") != _PLAN_KIND: + raise CandidateContractError("execution plan has the wrong schema or kind") + if receipt.get("schemaVersion") != 1 or receipt.get("kind") != _RECEIPT_KIND: + raise CandidateContractError( + "materialization receipt has the wrong schema or kind" + ) + if receipt.get("digestAlgorithm") != "rfc8785-sha256" or "digest" in receipt: + raise CandidateContractError( + "materialization receipt must be exact digest-free canonical material" + ) + + execution_evidence = _object(receipt.get("executionPlan"), "receipt.executionPlan") + plan_digest = _digest( + execution_evidence.get("digest"), "receipt.executionPlan.digest" + ) + if ( + execution_evidence.get("schemaVersion") != 1 + or execution_evidence.get("kind") != _EXECUTION_EVIDENCE_KIND + ): + raise CandidateContractError("receipt execution evidence has the wrong kind") + if ( + sha256_bytes(plan_raw) != plan_digest + or execution_evidence.get("material") != plan + ): + raise CandidateContractError( + "receipt does not bind the exact execution plan material" + ) + if ( + _embedded_bytes( + execution_evidence.get("artifact"), "receipt.executionPlan.artifact" + ) + != plan_raw + ): + raise CandidateContractError( + "receipt execution-plan artifact differs from the executed bytes" + ) + + bundle_digest = _digest(plan.get("bundleDigest"), "plan.bundleDigest") + if receipt.get("bundleDigest") != bundle_digest: + raise CandidateContractError("receipt and plan bundle digests differ") + execution_id = _string(plan.get("executionId"), "plan.executionId") + + task = _object(plan.get("task"), "plan.task") + repository = _object(task.get("repository"), "plan.task.repository") + base_commit = _git_object( + repository.get("baseCommit"), "plan.task.repository.baseCommit" + ) + base_tree = _git_object(repository.get("baseTree"), "plan.task.repository.baseTree") + if len(base_commit) != len(base_tree): + raise CandidateContractError("task Git objects use different hash formats") + instruction = _instruction(task.get("instruction")) + task_files = _workspace_files(task.get("workspace"), "plan.task.workspace") + if not task_files: + raise CandidateContractError("task workspace cannot be empty") + + roots = _object(plan.get("workspaces"), "plan.workspaces") + task_root = _absolute(roots.get("taskRoot"), "plan.workspaces.taskRoot") + candidate_root_value = roots.get("candidateRoot") + candidate_root = ( + _absolute(candidate_root_value, "plan.workspaces.candidateRoot") + if candidate_root_value is not None + else None + ) + if candidate_root is not None and ( + candidate_root == task_root + or candidate_root.startswith(f"{task_root}/") + or task_root.startswith(f"{candidate_root}/") + ): + raise CandidateContractError("task and candidate workspace roots overlap") + protected_instruction_path = ( + instruction.delivery_path + if instruction.delivery_path is not None + else _STDIN_TASK_PATH + if instruction.delivery_kind == "stdin-utf8" + else None + ) + if protected_instruction_path is not None and ( + protected_instruction_path == task_root + or protected_instruction_path.startswith(f"{task_root}/") + or task_root.startswith(f"{protected_instruction_path}/") + or ( + candidate_root is not None + and ( + protected_instruction_path == candidate_root + or protected_instruction_path.startswith(f"{candidate_root}/") + or candidate_root.startswith(f"{protected_instruction_path}/") + ) + ) + ): + raise CandidateContractError("instruction path overlaps an execution workspace") + + code_kind = plan.get("codeKind") + candidate_snapshot = plan.get("candidateWorkspace") + active_code = code_kind in {"no-op", "git-patch"} + if code_kind not in {"disabled", "no-op", "git-patch"}: + raise CandidateContractError("plan.codeKind is unsupported") + if active_code != (candidate_snapshot is not None) or active_code != ( + candidate_root is not None + ): + raise CandidateContractError( + "active code and candidate workspace presence disagree" + ) + candidate_files = ( + _workspace_files(candidate_snapshot, "plan.candidateWorkspace") + if candidate_snapshot is not None + else () + ) + if active_code and not candidate_files: + raise CandidateContractError("active candidate workspace cannot be empty") + if receipt.get("candidateWorkspace") != candidate_snapshot: + raise CandidateContractError("receipt and plan candidate workspaces differ") + + profile_evidence = _object(receipt.get("profilePlan"), "receipt.profilePlan") + if ( + profile_evidence.get("schemaVersion") != 1 + or profile_evidence.get("kind") != _PROFILE_PLAN_KIND + ): + raise CandidateContractError("receipt profile evidence has the wrong kind") + profile_digest = _digest( + profile_evidence.get("digest"), "receipt.profilePlan.digest" + ) + profile_raw = _embedded_bytes( + profile_evidence.get("artifact"), "receipt.profilePlan.artifact" + ) + if sha256_bytes(profile_raw) != profile_digest: + raise CandidateContractError("profile artifact does not match its digest") + try: + profile_artifact_material = json.loads(profile_raw) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise CandidateContractError("profile artifact is not UTF-8 JSON") from exc + profile_material = _object( + profile_evidence.get("material"), "receipt.profilePlan.material" + ) + if profile_artifact_material != profile_material: + raise CandidateContractError("profile artifact and material differ") + profile_files = _profile_files(profile_material) + profile_application = _object(plan.get("profile"), "plan.profile") + if profile_application.get("planDigest") != profile_digest: + raise CandidateContractError("plan does not bind the profile digest") + profile_target = profile_application.get("targetWorkspace") + if profile_target not in {"task", "candidate"}: + raise CandidateContractError("profile target workspace is unsupported") + if profile_target == "candidate" and candidate_root is None: + raise CandidateContractError( + "candidate-targeted profile lacks a candidate workspace" + ) + if profile_application.get("mountPaths") != [file.path for file in profile_files]: + raise CandidateContractError( + "profile mount paths differ from the exact profile files" + ) + + launch = _object(plan.get("launch"), "plan.launch") + executable = _string(launch.get("executable"), "plan.launch.executable") + executable_parts = executable.split("/") + if executable.startswith("/"): + executable_parts = executable_parts[1:] + if ( + not _EXECUTABLE_RE.fullmatch(executable) + or not executable_parts + or any(part in {"", ".", ".."} for part in executable_parts) + or executable_parts[-1].lower() in _SHELLS + ): + raise CandidateContractError("plan launch executable is unsafe") + args_value = launch.get("args") + if not isinstance(args_value, list): + raise CandidateContractError("plan.launch.args must be an array") + args = tuple( + _public(value, f"plan.launch.args[{index}]") + for index, value in enumerate(args_value) + ) + env_value = _object(launch.get("env"), "plan.launch.env") + env: dict[str, str] = {} + for name, value in env_value.items(): + if not _ENV_RE.fullmatch(name): + raise CandidateContractError(f"invalid launch environment name: {name}") + env[name] = _public(value, f"plan.launch.env.{name}") + if instruction.delivery_kind == "utf8-file": + if env.get("TANGLE_CANDIDATE_TASK_PATH") != instruction.delivery_path: + raise CandidateContractError( + "launch env does not bind the fixed instruction path" + ) + elif "TANGLE_CANDIDATE_TASK_PATH" in env: + raise CandidateContractError( + "non-file instruction delivery exposes a task path" + ) + + cwd = _object(launch.get("cwd"), "plan.launch.cwd") + cwd_workspace = cwd.get("workspace") + if cwd_workspace == "task": + cwd_root = task_root + elif cwd_workspace == "candidate" and candidate_root is not None: + cwd_root = candidate_root + else: + raise CandidateContractError("launch cwd names an unavailable workspace") + cwd_path = _safe_relative(cwd.get("path"), "plan.launch.cwd.path", allow_dot=True) + limits = _object(plan.get("limits"), "plan.limits") + timeout_ms = _integer(limits.get("timeoutMs"), "plan.limits.timeoutMs", minimum=1) + _integer(limits.get("maxSteps"), "plan.limits.maxSteps", minimum=1) + max_model_calls = _integer( + limits.get("maxModelCalls"), "plan.limits.maxModelCalls" + ) + _integer(limits.get("maxInputTokens"), "plan.limits.maxInputTokens") + _integer(limits.get("maxOutputTokens"), "plan.limits.maxOutputTokens") + _number(limits.get("maxCostUsd"), "plan.limits.maxCostUsd") + attempt = _object(plan.get("attempt"), "plan.attempt") + _integer(attempt.get("number"), "plan.attempt.number", minimum=1) + _integer(attempt.get("maxAttempts"), "plan.attempt.maxAttempts", minimum=1) + if attempt.get("retryPolicy") != "none": + raise CandidateContractError("Pier execution requires runtime-owned retries") + container = _object(plan.get("container"), "plan.container") + if receipt.get("container") != container: + raise CandidateContractError("receipt and plan container identities differ") + _digest(container.get("indexDigest"), "plan.container.indexDigest") + _digest(container.get("manifestDigest"), "plan.container.manifestDigest") + image = _string(container.get("image"), "plan.container.image") + if "@" in image or "://" in image or any(char.isspace() for char in image): + raise CandidateContractError( + "container image must be an unpinned OCI reference without credentials" + ) + if container.get("source") not in { + "pinned-container", + "evaluator-task-container", + }: + raise CandidateContractError("plan container source is unsupported") + platform = _object(container.get("platform"), "plan.container.platform") + if platform.get("os") != "linux" or platform.get("architecture") not in { + "amd64", + "arm64", + }: + raise CandidateContractError("Pier adapter requires a supported Linux platform") + model = _object(plan.get("model"), "plan.model") + resolved_model = _object(model.get("resolved"), "plan.model.resolved") + requested_model = _string( + resolved_model.get("requested"), "plan.model.resolved.requested" + ) + _string(resolved_model.get("provider"), "plan.model.resolved.provider") + _string(resolved_model.get("model"), "plan.model.resolved.model") + _string(resolved_model.get("snapshot"), "plan.model.resolved.snapshot") + if receipt.get("resolvedModel") != resolved_model: + raise CandidateContractError("receipt and plan resolved models differ") + model_access = _object(model.get("access"), "plan.model.access") + if set(model_access) != {"kind", "grantDigest", "network"}: + raise CandidateContractError("plan.model.access has unexpected fields") + if model_access.get("kind") != "evaluator-mediated": + raise CandidateContractError( + "plan.model.access.kind must equal evaluator-mediated" + ) + _digest(model_access.get("grantDigest"), "plan.model.access.grantDigest") + model_network = _object( + model_access.get("network"), "plan.model.access.network" + ) + if max_model_calls == 0: + if model_network != {"mode": "disabled"}: + raise CandidateContractError( + "zero-call plans cannot expose a model gateway" + ) + model_network_domains: tuple[str, ...] = () + else: + if set(model_network) != {"mode", "domains"} or model_network.get( + "mode" + ) != "gateway-only": + raise CandidateContractError( + "model-calling plans require one frozen gateway allowlist" + ) + domain_values = model_network.get("domains") + if not isinstance(domain_values, list) or not domain_values: + raise CandidateContractError( + "plan.model.access.network.domains must be a non-empty array" + ) + model_network_domains = tuple( + _gateway_domain( + value, f"plan.model.access.network.domains[{index}]" + ) + for index, value in enumerate(domain_values) + ) + if list(model_network_domains) != sorted(set(model_network_domains)): + raise CandidateContractError( + "model gateway domains must be unique and sorted" + ) + if receipt.get("harness") != plan.get("harness") or receipt.get( + "harnessVersion" + ) != plan.get("harnessVersion"): + raise CandidateContractError("receipt and plan harness identities differ") + if receipt.get("codeKind") != code_kind: + raise CandidateContractError("receipt and plan code kinds differ") + memory = _object(plan.get("memory"), "plan.memory") + if memory != {"mode": "disabled"}: + raise CandidateContractError( + "Pier adapter does not yet support isolated candidate memory" + ) + if plan.get("knowledgeManifestDigest") is not None: + raise CandidateContractError( + "Pier adapter does not yet support candidate knowledge snapshots" + ) + network = _object(plan.get("network"), "plan.network") + if network != {"mode": "disabled"}: + raise CandidateContractError( + "Pier candidate execution requires disabled network" + ) + + return PreparedCandidateContract( + plan=plan, + plan_raw=plan_raw, + receipt=receipt, + receipt_raw=receipt_raw, + bundle_digest=bundle_digest, + execution_id=execution_id, + execution_plan_digest=plan_digest, + materialization_receipt_digest=receipt_digest, + instruction=instruction, + repository_base_commit=base_commit, + repository_base_tree=base_tree, + task_root=task_root, + candidate_root=candidate_root, + task_files=task_files, + candidate_files=candidate_files, + profile_target=profile_target, + profile_files=profile_files, + executable=executable, + args=args, + env=env, + cwd_root=cwd_root, + cwd_path=cwd_path, + timeout_ms=timeout_ms, + requested_model=requested_model, + resolved_model=resolved_model, + model_network_domains=model_network_domains, + container=container, + ) + + +def verify_local_files( + root: Path, + expected: tuple[WorkspaceFile, ...] | tuple[ProfileFile, ...], + label: str, +) -> None: + """Require exact regular files, modes, and bytes below one staging root.""" + + if root.is_symlink() or not root.is_dir(): + raise CandidateContractError(f"{label} must be a real directory: {root}") + if root.resolve(strict=True) != root.absolute(): + raise CandidateContractError(f"{label} has a symlinked path component: {root}") + observed: dict[str, tuple[int, str, int]] = {} + for entry in root.rglob("*"): + relative = entry.relative_to(root).as_posix() + info = entry.lstat() + if stat.S_ISLNK(info.st_mode): + raise CandidateContractError(f"{label} contains a symlink: {relative}") + if stat.S_ISDIR(info.st_mode): + continue + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise CandidateContractError( + f"{label} contains a non-regular or hard-linked file: {relative}" + ) + raw = entry.read_bytes() + observed[relative] = (stat.S_IMODE(info.st_mode), sha256_bytes(raw), len(raw)) + expected_map = {file.path: file for file in expected} + if set(observed) != set(expected_map): + raise CandidateContractError( + f"{label} file set differs from the signed manifest" + ) + for path, expected_file in expected_map.items(): + actual = observed[path] + if isinstance(expected_file, ProfileFile): + if actual[:2] != (expected_file.mode, expected_file.sha256): + raise CandidateContractError(f"{label} bytes or mode differ: {path}") + elif actual != ( + expected_file.mode, + expected_file.sha256, + expected_file.byte_length, + ): + raise CandidateContractError( + f"{label} bytes, length, or mode differ: {path}" + ) + + +def immutable_snapshot( + source: Path, destination: Path, expected: tuple[Any, ...], label: str +) -> None: + """Copy a verified staging tree and prove it did not change during capture.""" + + import shutil + + verify_local_files(source, expected, label) + shutil.copytree(source, destination, symlinks=True, copy_function=shutil.copy2) + verify_local_files(destination, expected, f"{label} snapshot") diff --git a/bench/pier_agents/process_boundary.py b/bench/pier_agents/process_boundary.py new file mode 100644 index 00000000..ec91cc07 --- /dev/null +++ b/bench/pier_agents/process_boundary.py @@ -0,0 +1,321 @@ +"""Evaluator-owned process isolation for one Pier candidate invocation.""" + +from __future__ import annotations + +import secrets +import shlex +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + +from pier.environments.base import BaseEnvironment + +from .candidate_contract import sha256_bytes + + +class ProcessBoundaryHost(Protocol): + async def _exec( + self, environment: BaseEnvironment, command: str, **kwargs: Any + ) -> Any: ... + + async def _ensure_real_directory( + self, environment: BaseEnvironment, path: str, *, anchor: str, mode: int = 0o755 + ) -> None: ... + + async def _assert_real_directory( + self, + environment: BaseEnvironment, + path: str, + *, + label: str, + user: str | int | None = "root", + ) -> None: ... + + async def _write_verified_file( + self, + environment: BaseEnvironment, + source: Path, + target: str, + *, + anchor: str, + mode: int, + digest: str, + byte_length: int, + ) -> None: ... + + +@dataclass(frozen=True) +class ProcessBoundaryConfig: + uid: int + gid: int + evaluator_root: str + control_root: str + home: str + temporary: str + isolate_user: bool = True + + +class CandidateProcessBoundary: + def __init__(self, host: ProcessBoundaryHost, config: ProcessBoundaryConfig): + self._host = host + self.config = config + + async def prepare_identity( + self, environment: BaseEnvironment, workspace_roots: tuple[str, ...] + ) -> None: + config = self.config + await self._host._ensure_real_directory( + environment, config.evaluator_root, anchor="/", mode=0o755 + ) + await self._host._exec( + environment, + f"chmod 0755 -- {shlex.quote(config.evaluator_root)}", + user="root", + ) + for path, mode in ( + (config.control_root, 0o700), + (config.home, 0o700), + (config.temporary, 0o700), + ): + await self._host._ensure_real_directory( + environment, path, anchor="/", mode=mode + ) + await self._host._exec( + environment, + f"chmod {mode:o} -- {shlex.quote(path)}", + user="root", + ) + if not config.isolate_user: + return + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + "command -v setsid >/dev/null", + "command -v setpriv >/dev/null", + f"! grep -l '^Uid:[[:space:]]*{config.uid}[[:space:]]' " + "/proc/[0-9]*/status >/dev/null 2>&1", + f"chown {config.uid}:{config.gid} -- " + f"{shlex.quote(config.home)} {shlex.quote(config.temporary)}", + f"chown -R {config.uid}:{config.gid} -- " + + " ".join(shlex.quote(root) for root in workspace_roots), + ] + ), + user="root", + ) + + async def stage_wrapper(self, environment: BaseEnvironment) -> tuple[str, str]: + token = secrets.token_hex(24) + wrapper = f"{self.config.control_root}/run-{token}.sh" + marker = f"{self.config.control_root}/timeout-{token}" + raw = self._runner_script() + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(raw) + local_path = Path(handle.name) + try: + await self._host._write_verified_file( + environment, + local_path, + wrapper, + anchor=self.config.control_root, + mode=0o700, + digest=sha256_bytes(raw), + byte_length=len(raw), + ) + finally: + local_path.unlink(missing_ok=True) + return wrapper, marker + + async def kill_and_wait(self, environment: BaseEnvironment) -> None: + """Kill every process in the candidate identity and prove none remain.""" + if not self.config.isolate_user: + return + command = """set -eu +candidate_pids() { + for status in /proc/[0-9]*/status; do + test -r "$status" || continue + pid=${status#/proc/} + pid=${pid%/status} + while IFS=: read -r key value; do + if test "$key" = Uid; then + set -- $value + if test "${1:-}" = "$TARGET_UID"; then + printf '%s\n' "$pid" + fi + break + fi + done < "$status" + done +} + +for attempt in $(seq 1 100); do + pids=$(candidate_pids) + test -n "$pids" || exit 0 + for pid in $pids; do + kill -KILL "$pid" 2>/dev/null || true + done + sleep 0.02 +done +remaining=$(candidate_pids) +test -z "$remaining" || { + printf 'candidate processes survived SIGKILL: %s\n' "$remaining" >&2 + exit 1 +} +""" + await self._host._exec( + environment, + command, + env={"TARGET_UID": str(self.config.uid)}, + user="root", + timeout_sec=5, + ) + + async def remove_control_files( + self, + environment: BaseEnvironment, + wrapper: str | None, + marker: str | None, + ) -> None: + paths = [path for path in (wrapper, marker) if path is not None] + if not paths: + return + await self._host._assert_real_directory( + environment, + self.config.control_root, + label="candidate control directory", + ) + await self._host._exec( + environment, + "rm -f -- " + " ".join(shlex.quote(path) for path in paths), + user="root", + ) + + async def timeout_marker_exists( + self, environment: BaseEnvironment, marker: str + ) -> bool: + quoted = shlex.quote(marker) + result = await self._host._exec( + environment, + f'test -f {quoted} && test ! -L {quoted} && test "$(cat -- {quoted})" = timeout', + user="root", + check=False, + ) + return result.return_code == 0 + + def _runner_script(self) -> bytes: + if not self.config.isolate_user: + return b"""#!/bin/sh +set -eu +marker=$1 +uid=$2 +gid=$3 +deadline=$4 +grace=$5 +stdin_path=$6 +shift 6 +rm -f -- "$marker" "$marker.tmp" +set +e +if test "$stdin_path" = -; then + timeout --signal=TERM --kill-after="${grace}s" "${deadline}s" env "$@" +else + timeout --signal=TERM --kill-after="${grace}s" "${deadline}s" env "$@" < "$stdin_path" +fi +status=$? +set -e +if test "$status" = 124; then + printf timeout > "$marker" +fi +exit "$status" +""" + return b"""#!/bin/sh +set -eu +marker=$1 +uid=$2 +gid=$3 +deadline=$4 +grace=$5 +stdin_path=$6 +shift 6 +rm -f -- "$marker" "$marker.tmp" + +kill_candidate_uid() { + signal=$1 + for status in /proc/[0-9]*/status; do + test -r "$status" || continue + pid=${status#/proc/} + pid=${pid%/status} + while IFS=: read -r key value; do + if test "$key" = Uid; then + set -- $value + if test "${1:-}" = "$uid"; then + kill "-$signal" "$pid" 2>/dev/null || true + fi + break + fi + done < "$status" + done +} + +candidate_uid_is_dead() { + for status in /proc/[0-9]*/status; do + test -r "$status" || continue + while IFS=: read -r key value; do + if test "$key" = Uid; then + set -- $value + test "${1:-}" = "$uid" && return 1 + break + fi + done < "$status" + done + return 0 +} + +kill_and_wait_candidate_uid() { + kill_candidate_uid KILL + attempt=0 + while ! candidate_uid_is_dead; do + attempt=$((attempt + 1)) + test "$attempt" -lt 100 || { + printf 'candidate processes survived SIGKILL\n' >&2 + return 1 + } + kill_candidate_uid KILL + sleep 0.02 + done +} + +if test "$stdin_path" = -; then + setsid setpriv --reuid="$uid" --regid="$gid" --clear-groups --no-new-privs \ + --bounding-set=-all --inh-caps=-all --ambient-caps=-all --pdeathsig=KILL -- env "$@" & +else + setsid setpriv --reuid="$uid" --regid="$gid" --clear-groups --no-new-privs \ + --bounding-set=-all --inh-caps=-all --ambient-caps=-all --pdeathsig=KILL -- env "$@" < "$stdin_path" & +fi +candidate_pid=$! +( + sleep "$deadline" + if kill -0 "$candidate_pid" 2>/dev/null; then + umask 077 + printf timeout > "$marker.tmp" + mv -fT -- "$marker.tmp" "$marker" + kill_candidate_uid TERM + sleep "$grace" + kill_and_wait_candidate_uid + fi +) & +watchdog_pid=$! + +set +e +wait "$candidate_pid" +candidate_status=$? +set -e +kill "$watchdog_pid" 2>/dev/null || true +wait "$watchdog_pid" 2>/dev/null || true +kill_and_wait_candidate_uid +if test -f "$marker"; then + exit 124 +fi +exit "$candidate_status" +""" diff --git a/bench/pier_agents/tangle_candidate.py b/bench/pier_agents/tangle_candidate.py new file mode 100644 index 00000000..ae27d2cb --- /dev/null +++ b/bench/pier_agents/tangle_candidate.py @@ -0,0 +1,907 @@ +"""Execute one runtime-prepared candidate inside a Pier task container. + +Pier owns the task container, patch transfer, and verifier. Agent Runtime owns +candidate verification, exact execution bytes, model access, and trace usage. +This bridge only rechecks the prepared bytes, copies their signed workspaces, +delivers the exact task instruction, and reports truthful process termination. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import math +import os +import re +import secrets +import shlex +import tempfile +import time +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version as package_version +from pathlib import Path, PurePosixPath +from typing import Any + +from pier.agents.base import BaseAgent +from pier.agents.installed.base import NonZeroAgentExitCodeError +from pier.environments.base import BaseEnvironment +from pier.models.agent.context import AgentContext +from pier.models.agent.network import NetworkAllowlist + +from .candidate_contract import ( + CandidateContractError, + PreparedCandidateContract, + immutable_snapshot, + load_prepared_candidate_contract, + sha256_bytes, +) +from .process_boundary import CandidateProcessBoundary, ProcessBoundaryConfig +from .workspace_boundary import ( + CandidateWorkspaceBoundary, + WorkspaceBoundaryConfig, + candidate_git_env, +) + + +_ADAPTER_VERSION = "2.0.0" +_ENV_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_PROTECTED_TASK_PATH = "/tangle/input/stdin.txt" +_PIER_PROTECTED_ROOTS = ("/logs", "/tests", "/solution", "/tangle") +_CANDIDATE_UID = 65532 +_CANDIDATE_GID = 65532 +_EVALUATOR_ROOT = "/tangle" +_CANDIDATE_HOME = "/tangle/home" +_CANDIDATE_TMP = "/tangle/tmp" +_CONTROL_ROOT = "/tangle/control" +_FIXED_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +_RESERVED_PROCESS_ENV = { + "GIT_CONFIG_COUNT", + "GIT_CONFIG_KEY_0", + "GIT_CONFIG_VALUE_0", + "GIT_CONFIG_NOSYSTEM", + "GIT_CONFIG_GLOBAL", + "GIT_CONFIG_SYSTEM", + "GIT_NO_REPLACE_OBJECTS", + "GIT_TERMINAL_PROMPT", + "HOME", + "LC_ALL", + "PATH", + "TMPDIR", +} +_TRACE_ENV_NAMES = { + "TANGLE_CANDIDATE_EXECUTION_ID", + "TANGLE_CANDIDATE_BUNDLE_DIGEST", + "TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST", + "TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST", + "TANGLE_TRACE_RUN_ID", +} + + +@dataclass(frozen=True) +class OriginalProfileFile: + path: str + existed: bool + tracked: bool + mode: int | None + content: bytes | None + + +class PierCandidateError(CandidateContractError): + """The prepared candidate cannot be replayed safely by Pier.""" + + +def _runtime_pier_version() -> str: + try: + return package_version("datacurve-pier") + except PackageNotFoundError as exc: + raise PierCandidateError( + "datacurve-pier package metadata is unavailable" + ) from exc + + +def _same_or_descendant(path: str, root: str) -> bool: + return path == root or path.startswith(f"{root}/") + + +def _join_workspace(root: str, relative: str) -> str: + root_path = PurePosixPath(root) + joined = root_path if relative == "." else root_path / PurePosixPath(relative) + if joined != root_path and root_path not in joined.parents: + raise PierCandidateError(f"cwd escapes its workspace: {relative}") + return str(joined) + + +def _trace_env( + contract: PreparedCandidateContract, trace_run_id: str +) -> dict[str, str]: + return { + "TANGLE_CANDIDATE_EXECUTION_ID": contract.execution_id, + "TANGLE_CANDIDATE_BUNDLE_DIGEST": contract.bundle_digest, + "TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST": contract.execution_plan_digest, + "TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST": contract.materialization_receipt_digest, + "TANGLE_TRACE_RUN_ID": trace_run_id, + } + + +class TangleCandidateAgent(BaseAgent): + """Pier adapter for a branded ``PreparedAgentCandidateExecution``.""" + + SUPPORTS_ATIF = False + SUPPORTS_WINDOWS = False + ISOLATE_CANDIDATE_USER = True + + def __init__( + self, + logs_dir: Path, + model_name: str | None = None, + plan_path: str | None = None, + receipt_path: str | None = None, + expected_receipt_digest: str | None = None, + trace_run_id: str | None = None, + task_dir: str | None = None, + profile_dir: str | None = None, + candidate_dir: str | None = None, + pier_version: str | None = None, + extra_env: dict[str, str] | None = None, + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__(logs_dir=logs_dir, model_name=model_name, *args, **kwargs) + if ( + not plan_path + or not receipt_path + or not expected_receipt_digest + or not trace_run_id + ): + raise PierCandidateError( + "plan_path, receipt_path, expected_receipt_digest, and trace_run_id are required" + ) + if not task_dir or not profile_dir or not pier_version: + raise PierCandidateError("task_dir, profile_dir, and pier_version are required") + + self._expected_pier_version = pier_version + self._contract = load_prepared_candidate_contract( + Path(plan_path).expanduser().resolve(), + Path(receipt_path).expanduser().resolve(), + expected_receipt_digest, + ) + if model_name != self._contract.requested_model: + raise PierCandidateError( + "Pier model does not match the runtime-resolved request: " + f"expected={self._contract.requested_model} observed={model_name}" + ) + self._extra_env = self._validate_protected_env( + extra_env or {}, trace_run_id + ) + self._validate_execution_roots() + + self._snapshots = tempfile.TemporaryDirectory( + prefix="agent-bench-pier-candidate-" + ) + snapshot_root = Path(self._snapshots.name) + self._task_snapshot = snapshot_root / "task" + self._profile_snapshot = snapshot_root / "profile" + self._candidate_snapshot: Path | None = None + try: + immutable_snapshot( + Path(os.path.abspath(Path(task_dir).expanduser())), + self._task_snapshot, + self._contract.task_files, + "prepared task directory", + ) + immutable_snapshot( + Path(os.path.abspath(Path(profile_dir).expanduser())), + self._profile_snapshot, + self._contract.profile_files, + "prepared profile directory", + ) + if self._contract.candidate_root is not None: + if not candidate_dir: + raise PierCandidateError("active code requires candidate_dir") + self._candidate_snapshot = snapshot_root / "candidate" + immutable_snapshot( + Path(os.path.abspath(Path(candidate_dir).expanduser())), + self._candidate_snapshot, + self._contract.candidate_files, + "prepared candidate directory", + ) + elif candidate_dir: + raise PierCandidateError("disabled code cannot receive candidate_dir") + except Exception: + self._snapshots.cleanup() + raise + + self._original_profile_files: tuple[OriginalProfileFile, ...] = () + self._process_boundary = CandidateProcessBoundary( + self, + ProcessBoundaryConfig( + uid=_CANDIDATE_UID, + gid=_CANDIDATE_GID, + evaluator_root=_EVALUATOR_ROOT, + control_root=_CONTROL_ROOT, + home=_CANDIDATE_HOME, + temporary=_CANDIDATE_TMP, + isolate_user=self.ISOLATE_CANDIDATE_USER, + ), + ) + self._workspace_boundary = CandidateWorkspaceBoundary( + self, + WorkspaceBoundaryConfig( + contract=self._contract, + task_snapshot=self._task_snapshot, + control_root=_CONTROL_ROOT, + protected_roots=_PIER_PROTECTED_ROOTS, + error_type=PierCandidateError, + ), + ) + + def _validate_protected_env( + self, values: dict[str, str], trace_run_id: str + ) -> dict[str, str]: + protected: dict[str, str] = {} + public_names = set(self._contract.env) + reserved_public = sorted(public_names & _RESERVED_PROCESS_ENV) + if reserved_public: + raise PierCandidateError( + "signed public env collides with evaluator Git safety fields: " + + ", ".join(reserved_public) + ) + expected_trace = _trace_env(self._contract, trace_run_id) + for name, expected in expected_trace.items(): + if values.get(name) != expected: + raise PierCandidateError( + f"evaluator trace environment is missing or mismatched: {name}" + ) + for name, value in values.items(): + if not _ENV_RE.fullmatch(name) or name in _RESERVED_PROCESS_ENV: + raise PierCandidateError( + f"invalid protected model environment name: {name}" + ) + if name in public_names: + raise PierCandidateError( + f"protected model environment overlaps signed public env: {name}" + ) + if name.startswith("TANGLE_CANDIDATE_") and name not in _TRACE_ENV_NAMES: + raise PierCandidateError( + f"unknown evaluator candidate environment: {name}" + ) + if not isinstance(value, str) or not value or "\0" in value: + raise PierCandidateError( + f"protected model environment value is empty or invalid: {name}" + ) + protected[name] = value + return protected + + def _validate_execution_roots(self) -> None: + roots = [self._contract.task_root] + if self._contract.candidate_root is not None: + roots.append(self._contract.candidate_root) + for root in roots: + protected = next( + ( + path + for path in _PIER_PROTECTED_ROOTS + if _same_or_descendant(root, path) + ), + None, + ) + if protected is not None: + raise PierCandidateError( + f"execution workspace cannot be below Pier-owned path {protected}" + ) + + def name(self) -> str: + short = self._contract.bundle_digest.removeprefix("sha256:")[:16] + return f"tangle-candidate-{short}" + + def version(self) -> str: + return _ADAPTER_VERSION + + def network_allowlist(self) -> NetworkAllowlist: + return NetworkAllowlist(domains=list(self._contract.model_network_domains)) + + async def _exec( + self, + environment: BaseEnvironment, + command: str, + *, + cwd: str | None = None, + env: dict[str, str] | None = None, + user: str | int | None = None, + timeout_sec: float | None = None, + check: bool = True, + ) -> Any: + result = await environment.exec( + command=command, + cwd=cwd, + env=environment.agent_process_env(env), + user=user, + timeout_sec=timeout_sec, + ) + if check and result.return_code != 0: + raise PierCandidateError( + f"candidate bridge command failed ({result.return_code}): {command}; " + f"stderr={(result.stderr or '')[:500]}" + ) + return result + + async def _assert_real_directory( + self, + environment: BaseEnvironment, + path: str, + *, + label: str, + user: str | int | None = "root", + ) -> None: + quoted = shlex.quote(path) + await self._exec( + environment, + "\n".join( + [ + "set -eu", + f"test -d {quoted}", + f"test ! -L {quoted}", + f'test "$(realpath -e -- {quoted})" = {quoted}', + ] + ), + user=user, + ) + + async def _ensure_real_directory( + self, + environment: BaseEnvironment, + path: str, + *, + anchor: str, + mode: int = 0o755, + ) -> None: + parsed = PurePosixPath(path) + parsed_anchor = PurePosixPath(anchor) + if parsed != parsed_anchor and parsed_anchor not in parsed.parents: + raise PierCandidateError(f"directory escapes its trusted anchor: {path}") + await self._assert_real_directory( + environment, anchor, label="trusted directory anchor" + ) + current = parsed_anchor + relative_parts = parsed.relative_to(parsed_anchor).parts + for part in relative_parts: + current /= part + value = str(current) + quoted = shlex.quote(value) + await self._exec( + environment, + "\n".join( + [ + "set -eu", + f"if test -e {quoted} || test -L {quoted}; then", + f" test -d {quoted}", + f" test ! -L {quoted}", + "else", + f" mkdir -- {quoted}", + f" chmod {mode:o} -- {quoted}", + "fi", + f'test "$(realpath -e -- {quoted})" = {quoted}', + ] + ), + user="root", + ) + + async def _write_verified_file( + self, + environment: BaseEnvironment, + source: Path, + target: str, + *, + anchor: str, + mode: int, + digest: str, + byte_length: int, + ) -> None: + parent = str(PurePosixPath(target).parent) + await self._ensure_real_directory(environment, parent, anchor=anchor) + temporary = f"{parent}/.tangle-{secrets.token_hex(24)}" + quoted_temporary = shlex.quote(temporary) + quoted_target = shlex.quote(target) + expected_hash = digest.removeprefix("sha256:") + await self._exec( + environment, + f"test ! -e {quoted_temporary} && test ! -L {quoted_temporary}", + user="root", + ) + try: + await environment.upload_file(source, temporary) + await self._exec( + environment, + "\n".join( + [ + "set -eu", + f"test -f {quoted_temporary}", + f"test ! -L {quoted_temporary}", + f'test "$(stat -c %h -- {quoted_temporary})" = 1', + f'test "$(wc -c < {quoted_temporary})" = {byte_length}', + f"test \"$(sha256sum {quoted_temporary} | cut -d' ' -f1)\" = {expected_hash}", + f"chmod {mode:o} -- {quoted_temporary}", + f'test "$(stat -c %a -- {quoted_temporary})" = {mode:o}', + f"rm -rf -- {quoted_target}", + f"mv -fT -- {quoted_temporary} {quoted_target}", + f"test -f {quoted_target}", + f"test ! -L {quoted_target}", + f'test "$(stat -c %h -- {quoted_target})" = 1', + f'test "$(stat -c %a -- {quoted_target})" = {mode:o}', + f'test "$(wc -c < {quoted_target})" = {byte_length}', + f"test \"$(sha256sum {quoted_target} | cut -d' ' -f1)\" = {expected_hash}", + ] + ), + user="root", + ) + finally: + await self._exec( + environment, + f"rm -f -- {quoted_temporary}", + user="root", + check=False, + ) + + async def _verify_pier_execution_identity( + self, environment: BaseEnvironment + ) -> None: + await self._workspace_boundary.verify_pier_execution_identity(environment) + + async def _capture_original_profile_files( + self, environment: BaseEnvironment + ) -> tuple[OriginalProfileFile, ...]: + if self._contract.profile_target != "task": + return () + originals: list[OriginalProfileFile] = [] + root = self._contract.task_root + git = f"git -c core.hooksPath=/dev/null -C {shlex.quote(root)}" + for file in self._contract.profile_files: + target = f"{root}/{file.path}" + quoted = shlex.quote(target) + parent = shlex.quote(str(PurePosixPath(target).parent)) + presence = await self._exec( + environment, + "\n".join( + [ + "set -eu", + f'test "$(realpath -m -- {parent})" = {parent}', + f"if test -e {quoted} || test -L {quoted}; then", + f" test -f {quoted}", + f" test ! -L {quoted}", + f' test "$(stat -c %h -- {quoted})" = 1', + "else", + " exit 2", + "fi", + ] + ), + user="root", + check=False, + ) + tracked = ( + await self._exec( + environment, + f"{git} ls-files --error-unmatch -- {shlex.quote(file.path)}", + env=candidate_git_env(root), + user="root", + check=False, + ) + ).return_code == 0 + if presence.return_code == 2: + originals.append( + OriginalProfileFile(file.path, False, tracked, None, None) + ) + continue + if presence.return_code != 0: + raise PierCandidateError( + f"profile mount target is not one regular file: {file.path}" + ) + encoded = await self._exec( + environment, f"base64 -w0 -- {quoted}", user="root" + ) + mode = await self._exec(environment, f"stat -c %a -- {quoted}", user="root") + try: + content = base64.b64decode( + (encoded.stdout or "").strip(), validate=True + ) + parsed_mode = int((mode.stdout or "").strip(), 8) + except (ValueError, TypeError) as exc: + raise PierCandidateError( + f"could not capture original profile target: {file.path}" + ) from exc + originals.append( + OriginalProfileFile(file.path, True, tracked, parsed_mode, content) + ) + return tuple(originals) + + async def _apply_profile(self, environment: BaseEnvironment) -> None: + target_root = ( + self._contract.task_root + if self._contract.profile_target == "task" + else self._contract.candidate_root + ) + if target_root is None: + raise PierCandidateError("profile target workspace is unavailable") + self._original_profile_files = await self._capture_original_profile_files( + environment + ) + for file in self._contract.profile_files: + target = f"{target_root}/{file.path}" + source = self._profile_snapshot / file.path + await self._write_verified_file( + environment, + source, + target, + anchor=target_root, + mode=file.mode, + digest=file.sha256, + byte_length=source.stat().st_size, + ) + + async def setup(self, environment: BaseEnvironment) -> None: + try: + observed_pier = _runtime_pier_version() + if observed_pier != self._expected_pier_version: + raise PierCandidateError( + f"Pier version mismatch: expected={self._expected_pier_version} " + f"installed={observed_pier}" + ) + await self._verify_pier_execution_identity(environment) + await self._workspace_boundary.materialize_task_workspace(environment) + await self._workspace_boundary.verify_repository(environment) + await self._workspace_boundary.verify_container_workspace( + environment, + self._contract.task_root, + self._contract.task_files, + allow_task_metadata=True, + ) + if self._contract.candidate_root is not None: + if self._candidate_snapshot is None: + raise PierCandidateError("candidate snapshot is unavailable") + quoted = shlex.quote(self._contract.candidate_root) + candidate_parent = str( + PurePosixPath(self._contract.candidate_root).parent + ) + await self._ensure_real_directory( + environment, candidate_parent, anchor="/" + ) + await self._exec( + environment, + "\n".join( + [ + "set -eu", + f"test ! -e {quoted}", + f"test ! -L {quoted}", + f"mkdir -- {quoted}", + f'test "$(realpath -e -- {quoted})" = {quoted}', + ] + ), + user="root", + ) + await self._workspace_boundary.verify_resolved_root_isolation( + environment + ) + await environment.upload_dir( + self._candidate_snapshot, self._contract.candidate_root + ) + await self._workspace_boundary.verify_container_workspace( + environment, + self._contract.candidate_root, + self._contract.candidate_files, + allow_task_metadata=False, + ) + else: + await self._workspace_boundary.verify_resolved_root_isolation( + environment + ) + workspace_roots = (self._contract.task_root,) + ( + (self._contract.candidate_root,) + if self._contract.candidate_root is not None + else () + ) + await self._process_boundary.prepare_identity(environment, workspace_roots) + await self._apply_profile(environment) + + self.logs_dir.mkdir(parents=True, exist_ok=True) + (self.logs_dir / "execution-plan.json").write_bytes(self._contract.plan_raw) + (self.logs_dir / "materialization-receipt.json").write_bytes( + self._contract.receipt_raw + ) + finally: + self._snapshots.cleanup() + + def _identity_metadata(self) -> dict[str, Any]: + return { + "executionId": self._contract.execution_id, + "bundleDigest": self._contract.bundle_digest, + "executionPlanDigest": self._contract.execution_plan_digest, + "materializationReceiptDigest": self._contract.materialization_receipt_digest, + "usageSource": "protected-agent-eval-trace", + } + + async def _upload_instruction( + self, environment: BaseEnvironment, instruction: str + ) -> tuple[list[str], str | None, str | None]: + try: + raw = instruction.encode("utf-8") + except UnicodeEncodeError as exc: + raise PierCandidateError("Pier instruction is not valid UTF-8") from exc + evidence = self._contract.instruction + if len(raw) != evidence.byte_length or sha256_bytes(raw) != evidence.sha256: + raise PierCandidateError( + "Pier instruction bytes do not match the signed task instruction" + ) + + argv = [self._contract.executable, *self._contract.args] + if evidence.delivery_kind == "argv-append": + argv.append(instruction) + return argv, None, None + + target = evidence.delivery_path or _PROTECTED_TASK_PATH + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(raw) + local_path = Path(handle.name) + try: + await self._write_verified_file( + environment, + local_path, + target, + anchor="/", + mode=0o644, + digest=evidence.sha256, + byte_length=len(raw), + ) + finally: + local_path.unlink(missing_ok=True) + return ( + argv, + target if evidence.delivery_kind == "stdin-utf8" else None, + target, + ) + + async def _remove_instruction_file( + self, environment: BaseEnvironment, path: str | None + ) -> None: + if path is None: + return + parent = str(PurePosixPath(path).parent) + await self._assert_real_directory( + environment, parent, label="protected instruction directory" + ) + await self._exec( + environment, + f"rm -f -- {shlex.quote(path)}", + user="root", + ) + + async def _restore_profile_and_capture_solution( + self, environment: BaseEnvironment + ) -> None: + root = self._contract.task_root + quoted_root = shlex.quote(root) + git = f"git -c core.hooksPath=/dev/null -C {quoted_root}" + git_env = candidate_git_env(root) + + # Ignore candidate-authored commits and derive one evaluator-owned final + # tree from the actual working files relative to the signed base commit. + await self._exec( + environment, + f"{git} reset --mixed {shlex.quote(self._contract.repository_base_commit)}", + env=git_env, + user="root", + ) + for original in self._original_profile_files: + target = f"{root}/{original.path}" + quoted = shlex.quote(target) + if original.existed: + if original.content is None or original.mode is None: + raise PierCandidateError( + f"profile backup is incomplete: {original.path}" + ) + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(original.content) + local_path = Path(handle.name) + try: + await self._write_verified_file( + environment, + local_path, + target, + anchor=root, + mode=original.mode, + digest=sha256_bytes(original.content), + byte_length=len(original.content), + ) + finally: + local_path.unlink(missing_ok=True) + else: + parent = str(PurePosixPath(target).parent) + await self._ensure_real_directory(environment, parent, anchor=root) + await self._exec(environment, f"rm -rf -- {quoted}", user="root") + + await self._exec(environment, f"{git} add -A -- .", env=git_env, user="root") + for original in self._original_profile_files: + if original.tracked: + continue + await self._exec( + environment, + f"{git} rm -r -f --cached --ignore-unmatch -- {shlex.quote(original.path)}", + env=git_env, + user="root", + ) + staged = await self._exec( + environment, + f"{git} diff --cached --quiet", + env=git_env, + user="root", + check=False, + ) + if staged.return_code not in {0, 1}: + raise PierCandidateError("could not inspect the sanitized solution tree") + if staged.return_code == 1: + commit_env = { + **git_env, + "GIT_AUTHOR_NAME": "Tangle Evaluator", + "GIT_AUTHOR_EMAIL": "evaluator@tangle.tools", + "GIT_COMMITTER_NAME": "Tangle Evaluator", + "GIT_COMMITTER_EMAIL": "evaluator@tangle.tools", + } + await self._exec( + environment, + f"{git} commit -m 'capture candidate solution'", + env=commit_env, + user="root", + ) + + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + argv, stdin_path, instruction_path = await self._upload_instruction( + environment, instruction + ) + cwd = _join_workspace(self._contract.cwd_root, self._contract.cwd_path) + await self._exec(environment, f"test -d {shlex.quote(cwd)}") + + process_env = { + **self._extra_env, + **candidate_git_env(self._contract.task_root), + "HOME": _CANDIDATE_HOME, + "TMPDIR": _CANDIDATE_TMP, + "PATH": _FIXED_PATH, + } + wrapper, timeout_marker = await self._process_boundary.stage_wrapper( + environment + ) + timeout_seconds = self._contract.timeout_ms / 1000 + command = shlex.join( + [ + wrapper, + timeout_marker, + str(_CANDIDATE_UID), + str(_CANDIDATE_GID), + f"{timeout_seconds:.3f}", + "2", + stdin_path or "-", + *(f"{name}={value}" for name, value in self._contract.env.items()), + *argv, + ] + ) + context.metadata = { + **self._identity_metadata(), + "termination": {"kind": "running"}, + } + + started = time.monotonic() + try: + result = await environment.exec( + command=command, + cwd=cwd, + env=environment.agent_process_env(process_env), + user="root", + timeout_sec=math.ceil(timeout_seconds) + 12, + ) + except (asyncio.CancelledError, asyncio.TimeoutError): + elapsed = int((time.monotonic() - started) * 1000) + cleanup_errors: list[str] = [] + try: + await asyncio.shield( + self._process_boundary.kill_and_wait(environment) + ) + except Exception as cleanup_exc: + cleanup_errors.append(str(cleanup_exc)) + try: + await asyncio.shield( + self._remove_instruction_file(environment, instruction_path) + ) + except Exception as cleanup_exc: + cleanup_errors.append(str(cleanup_exc)) + try: + await asyncio.shield( + self._restore_profile_and_capture_solution(environment) + ) + except ( + Exception + ) as cleanup_exc: # cleanup failure is evidence, never hidden + cleanup_errors.append(str(cleanup_exc)) + try: + await asyncio.shield( + self._process_boundary.remove_control_files( + environment, wrapper, timeout_marker + ) + ) + except Exception as cleanup_exc: + cleanup_errors.append(str(cleanup_exc)) + context.metadata = { + **self._identity_metadata(), + "termination": {"kind": "cancelled", "observedElapsedMs": elapsed}, + **( + {"cleanupError": "; ".join(cleanup_errors)} + if cleanup_errors + else {} + ), + } + raise + except Exception: + await asyncio.shield(self._process_boundary.kill_and_wait(environment)) + await asyncio.shield( + self._remove_instruction_file(environment, instruction_path) + ) + await asyncio.shield( + self._restore_profile_and_capture_solution(environment) + ) + await asyncio.shield( + self._process_boundary.remove_control_files( + environment, wrapper, timeout_marker + ) + ) + raise + + elapsed = int((time.monotonic() - started) * 1000) + timed_out = await self._process_boundary.timeout_marker_exists( + environment, timeout_marker + ) + await self._process_boundary.kill_and_wait(environment) + await self._remove_instruction_file(environment, instruction_path) + await self._restore_profile_and_capture_solution(environment) + await self._process_boundary.remove_control_files( + environment, wrapper, timeout_marker + ) + self.logs_dir.mkdir(parents=True, exist_ok=True) + (self.logs_dir / "stdout.txt").write_text(result.stdout or "", encoding="utf-8") + (self.logs_dir / "stderr.txt").write_text(result.stderr or "", encoding="utf-8") + (self.logs_dir / "process.json").write_text( + json.dumps( + { + "executionId": self._contract.execution_id, + "exitCode": result.return_code, + "elapsedMs": elapsed, + "cwd": cwd, + "argv": argv, + "instructionDelivery": self._contract.instruction.delivery_kind, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + context.metadata = { + **self._identity_metadata(), + "termination": ( + {"kind": "timeout", "timeoutMs": self._contract.timeout_ms} + if timed_out + else {"kind": "exit", "exitCode": result.return_code} + ), + "observedElapsedMs": elapsed, + } + if timed_out: + raise asyncio.TimeoutError( + f"candidate exceeded signed timeout {self._contract.timeout_ms}ms" + ) + if result.return_code != 0: + raise NonZeroAgentExitCodeError( + f"candidate exited {result.return_code}: " + f"stdout={(result.stdout or '')[:500]} " + f"stderr={(result.stderr or '')[:500]}" + ) diff --git a/bench/pier_agents/tangle_candidate_test.py b/bench/pier_agents/tangle_candidate_test.py new file mode 100644 index 00000000..5d7de7bd --- /dev/null +++ b/bench/pier_agents/tangle_candidate_test.py @@ -0,0 +1,1022 @@ +import asyncio +import base64 +import hashlib +import importlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import types +import unittest +from pathlib import Path + + +class _BaseAgent: + def __init__(self, logs_dir, model_name=None, *args, **kwargs): + self.logs_dir = Path(logs_dir) + self.model_name = model_name + + +class _NonZeroAgentExitCodeError(RuntimeError): + pass + + +class _AgentContext: + def __init__(self): + self.n_input_tokens = None + self.n_cache_tokens = None + self.n_output_tokens = None + self.cost_usd = None + self.metadata = None + + +class _NetworkAllowlist: + def __init__(self, domains=None): + self.domains = list(domains or []) + + +def _install_pier_stubs(): + modules = { + name: types.ModuleType(name) + for name in ( + "pier", + "pier.agents", + "pier.agents.base", + "pier.agents.installed", + "pier.agents.installed.base", + "pier.environments", + "pier.environments.base", + "pier.models", + "pier.models.agent", + "pier.models.agent.context", + "pier.models.agent.network", + ) + } + modules["pier.agents.base"].BaseAgent = _BaseAgent + modules[ + "pier.agents.installed.base" + ].NonZeroAgentExitCodeError = _NonZeroAgentExitCodeError + modules["pier.environments.base"].BaseEnvironment = type("BaseEnvironment", (), {}) + modules["pier.models.agent.context"].AgentContext = _AgentContext + modules["pier.models.agent.network"].NetworkAllowlist = _NetworkAllowlist + sys.modules.update(modules) + + +_install_pier_stubs() +candidate = importlib.import_module("pier_agents.tangle_candidate") +contract_module = importlib.import_module("pier_agents.candidate_contract") +process_boundary_module = importlib.import_module("pier_agents.process_boundary") +workspace_boundary_module = importlib.import_module("pier_agents.workspace_boundary") +candidate._runtime_pier_version = lambda: "0.3.0" + + +class _LocalTangleCandidateAgent(candidate.TangleCandidateAgent): + ISOLATE_CANDIDATE_USER = False + + async def _verify_pier_execution_identity(self, environment): + del environment + + +class _Result: + def __init__(self, return_code=0, stdout="", stderr=""): + self.return_code = return_code + self.stdout = stdout + self.stderr = stderr + + +class _LocalEnvironment: + def __init__(self): + self.commands = [] + + def agent_process_env(self, env): + return {**os.environ, **(env or {})} + + async def upload_dir(self, source_dir, target_dir): + shutil.copytree(source_dir, target_dir, dirs_exist_ok=True) + + async def upload_file(self, source_path, target_path): + target = Path(target_path) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_path, target) + + async def exec(self, command, cwd=None, env=None, user=None, timeout_sec=None): + del user + self.commands.append(command) + completed = subprocess.run( + ["bash", "-c", command], + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=False, + timeout=timeout_sec, + ) + return _Result(completed.returncode, completed.stdout, completed.stderr) + + +class _CorruptingUploadEnvironment(_LocalEnvironment): + async def upload_file(self, source_path, target_path): + await super().upload_file(source_path, target_path) + with Path(target_path).open("ab") as handle: + handle.write(b"corrupted") + + +class _FailingWorkspaceCleanupEnvironment(_LocalEnvironment): + async def exec(self, command, cwd=None, env=None, user=None, timeout_sec=None): + if command.startswith("rm -f -- ") and "workspace-check-" in command: + self.commands.append(command) + return _Result(73, stderr="forced workspace cleanup failure") + return await super().exec(command, cwd, env, user, timeout_sec) + + +class _LocalBoundaryHost: + async def _exec(self, environment, command, **kwargs): + check = kwargs.pop("check", True) + result = await environment.exec(command=command, **kwargs) + if check and result.return_code != 0: + raise RuntimeError(result.stderr or f"boundary command exited {result.return_code}") + return result + + +def _sha(raw): + return f"sha256:{hashlib.sha256(raw).hexdigest()}" + + +def _canonical(value): + return json.dumps( + value, ensure_ascii=False, separators=(",", ":"), sort_keys=True + ).encode() + + +def _embedded(raw): + return { + "encoding": "base64", + "content": base64.b64encode(raw).decode(), + "sha256": _sha(raw), + "byteLength": len(raw), + } + + +def _workspace(files): + material = { + "schemaVersion": 1, + "kind": "agent-candidate-workspace-manifest", + "files": [ + { + "path": path, + "mode": mode, + "sha256": _sha(raw), + "byteLength": len(raw), + } + for path, mode, raw in sorted(files) + ], + } + manifest = _canonical(material) + return { + "schemaVersion": 1, + "kind": "agent-candidate-workspace-snapshot", + "digest": _sha(manifest), + "material": material, + "manifest": _embedded(manifest), + "archive": _embedded(b"fixture-archive"), + } + + +def _git(root, *args): + return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip() + + +def _fixture(root: Path): + task_root = root / "execution-task" + task_staging = root / "task-staging" + candidate_root = root / "execution-candidate" + candidate_staging = root / "candidate-staging" + profile_staging = root / "profile-staging" + sealed = root / "sealed" + logs = root / "logs" + task_root.mkdir() + (task_staging / "src").mkdir(parents=True) + candidate_staging.mkdir() + profile_staging.mkdir() + sealed.mkdir() + + status = b"not-ready\n" + (task_root / "src").mkdir() + (task_root / "src/status.txt").write_bytes(status) + os.chmod(task_root / "src/status.txt", 0o644) + (task_staging / "src/status.txt").write_bytes(status) + os.chmod(task_staging / "src/status.txt", 0o644) + subprocess.run( + ["git", "init", "-b", "main", str(task_root)], check=True, capture_output=True + ) + _git(task_root, "config", "user.email", "fixture@example.com") + _git(task_root, "config", "user.name", "Fixture") + _git(task_root, "add", "-A") + _git(task_root, "commit", "-m", "base") + base_commit = _git(task_root, "rev-parse", "HEAD") + base_tree = _git(task_root, "rev-parse", "HEAD^{tree}") + + runner = ( + "import os, pathlib, sys, time\n" + "root = pathlib.Path.cwd()\n" + "profile = root / 'AGENTS.md'\n" + "if not profile.exists():\n" + " profile = pathlib.Path(__file__).parent / 'AGENTS.md'\n" + "assert profile.read_text() == 'fixture-profile\\n'\n" + "assert sys.argv[-1] == 'make the task ready'\n" + "if os.environ.get('FIXTURE_SLEEP'):\n" + " time.sleep(float(os.environ['FIXTURE_SLEEP']))\n" + "attack = os.environ.get('FIXTURE_ATTACK')\n" + "if attack == 'symlink':\n" + " profile.unlink()\n" + " profile.symlink_to('src/status.txt')\n" + "elif attack == 'hardlink':\n" + " profile.unlink()\n" + " os.link(root / 'src/status.txt', profile)\n" + "(root / 'src/status.txt').write_text('ready\\n')\n" + ).encode() + (candidate_staging / "runner.py").write_bytes(runner) + os.chmod(candidate_staging / "runner.py", 0o755) + profile = b"fixture-profile\n" + (profile_staging / "AGENTS.md").write_bytes(profile) + os.chmod(profile_staging / "AGENTS.md", 0o644) + + instruction = "make the task ready".encode() + task_snapshot = _workspace([("src/status.txt", 0o644, status)]) + candidate_snapshot = _workspace([("runner.py", 0o755, runner)]) + profile_material = { + "version": 1, + "harness": "codex", + "files": [ + { + "relPath": "AGENTS.md", + "mode": 0o644, + "contentSha256": _sha(profile), + } + ], + "env": {}, + "flags": [], + "unsupported": [], + } + profile_raw = _canonical(profile_material) + profile_digest = _sha(profile_raw) + bundle_digest = f"sha256:{'b' * 64}" + plan = { + "schemaVersion": 1, + "kind": "agent-candidate-execution-plan-material", + "bundleDigest": bundle_digest, + "executionId": "pier-fixture-execution", + "attempt": {"number": 1, "maxAttempts": 1, "retryPolicy": "none"}, + "task": { + "benchmark": "pier-fixture", + "benchmarkVersion": "1", + "taskId": "fixture-1", + "splitDigest": f"sha256:{'1' * 64}", + "instruction": { + "encoding": "utf8", + "sha256": _sha(instruction), + "byteLength": len(instruction), + "delivery": {"kind": "argv-append"}, + }, + "repository": { + "identity": "fixture/repository", + "rootIdentity": "fixture/repository", + "baseCommit": base_commit, + "baseTree": base_tree, + }, + "workspace": task_snapshot, + }, + "workspaces": { + "taskRoot": str(task_root), + "candidateRoot": str(candidate_root), + }, + "codeKind": "no-op", + "candidateWorkspace": candidate_snapshot, + "profile": { + "planDigest": profile_digest, + "targetWorkspace": "task", + "mountPaths": ["AGENTS.md"], + }, + "harness": "codex", + "harnessVersion": "fixture", + "container": { + "source": "evaluator-task-container", + "image": "ghcr.io/tangle-network/fixture:latest", + "indexDigest": f"sha256:{'2' * 64}", + "manifestDigest": f"sha256:{'3' * 64}", + "platform": {"os": "linux", "architecture": "amd64"}, + }, + "model": { + "policy": "single", + "resolved": { + "requested": "openai/gpt-5.4", + "provider": "openai", + "model": "gpt-5.4", + "snapshot": "fixture", + "reasoningEffort": "xhigh", + }, + "access": { + "kind": "evaluator-mediated", + "grantDigest": f"sha256:{'4' * 64}", + "network": {"mode": "disabled"}, + }, + "routes": [{"kind": "primary", "requested": "openai/gpt-5.4"}], + }, + "launch": { + "executable": "python3", + "args": [{"kind": "public", "value": str(candidate_root / "runner.py")}], + "env": {}, + "cwd": {"workspace": "task", "path": "."}, + }, + "memory": {"mode": "disabled"}, + "limits": { + "timeoutMs": 60_000, + "maxSteps": 8, + "maxModelCalls": 0, + "maxInputTokens": 0, + "maxOutputTokens": 0, + "maxCostUsd": 0, + }, + "network": {"mode": "disabled"}, + } + plan_raw = _canonical(plan) + plan_digest = _sha(plan_raw) + execution_evidence = { + "schemaVersion": 1, + "kind": "agent-candidate-execution-plan", + "digest": plan_digest, + "material": plan, + "artifact": _embedded(plan_raw), + } + profile_evidence = { + "schemaVersion": 1, + "kind": "agent-profile-workspace-plan", + "digest": profile_digest, + "material": profile_material, + "artifact": _embedded(profile_raw), + } + receipt = { + "schemaVersion": 1, + "kind": "agent-candidate-materialization", + "digestAlgorithm": "rfc8785-sha256", + "bundleDigest": bundle_digest, + "profilePlan": profile_evidence, + "executionPlan": execution_evidence, + "candidateWorkspace": candidate_snapshot, + "codeKind": "no-op", + "materializedTree": base_tree, + "harness": "codex", + "harnessVersion": "fixture", + "container": plan["container"], + "resolvedModel": plan["model"]["resolved"], + "entrypoint": { + "path": "runner.py", + "sha256": _sha(runner), + "byteLength": len(runner), + }, + } + receipt_raw = _canonical(receipt) + plan_path = sealed / "execution-plan.json" + receipt_path = sealed / "materialization-receipt.json" + plan_path.write_bytes(plan_raw) + receipt_path.write_bytes(receipt_raw) + return { + "plan_path": plan_path, + "receipt_path": receipt_path, + "receipt_digest": _sha(receipt_raw), + "candidate_staging": candidate_staging, + "task_staging": task_staging, + "profile_staging": profile_staging, + "candidate_root": candidate_root, + "task_root": task_root, + "logs": logs, + "base_commit": base_commit, + "plan": plan, + } + + +def _rewrite_signed_plan(fixture): + plan_raw = _canonical(fixture["plan"]) + plan_digest = _sha(plan_raw) + receipt = json.loads(fixture["receipt_path"].read_text()) + receipt["executionPlan"] = { + "schemaVersion": 1, + "kind": "agent-candidate-execution-plan", + "digest": plan_digest, + "material": fixture["plan"], + "artifact": _embedded(plan_raw), + } + receipt["container"] = fixture["plan"]["container"] + receipt["resolvedModel"] = fixture["plan"]["model"]["resolved"] + receipt_raw = _canonical(receipt) + fixture["plan_path"].write_bytes(plan_raw) + fixture["receipt_path"].write_bytes(receipt_raw) + fixture["receipt_digest"] = _sha(receipt_raw) + + +def _agent(fixture): + evaluator_root = fixture["plan_path"].parent / "evaluator" + candidate._CANDIDATE_UID = os.getuid() + candidate._CANDIDATE_GID = os.getgid() + candidate._EVALUATOR_ROOT = str(evaluator_root) + candidate._CONTROL_ROOT = str(evaluator_root / "control") + candidate._CANDIDATE_HOME = str(evaluator_root / "home") + candidate._CANDIDATE_TMP = str(evaluator_root / "tmp") + plan_digest = _sha(fixture["plan_path"].read_bytes()) + trace_env = { + "TANGLE_CANDIDATE_EXECUTION_ID": fixture["plan"]["executionId"], + "TANGLE_CANDIDATE_BUNDLE_DIGEST": fixture["plan"]["bundleDigest"], + "TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST": plan_digest, + "TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST": fixture["receipt_digest"], + "TANGLE_TRACE_RUN_ID": fixture["plan"]["executionId"], + } + return _LocalTangleCandidateAgent( + logs_dir=fixture["logs"], + model_name=fixture["plan"]["model"]["resolved"]["requested"], + plan_path=str(fixture["plan_path"]), + receipt_path=str(fixture["receipt_path"]), + expected_receipt_digest=fixture["receipt_digest"], + trace_run_id=fixture["plan"]["executionId"], + task_dir=str(fixture["task_staging"]), + candidate_dir=str(fixture["candidate_staging"]), + profile_dir=str(fixture["profile_staging"]), + pier_version="0.3.0", + extra_env=trace_env, + ) + + +class WorkspaceBoundaryTest(unittest.TestCase): + def test_large_workspace_identity_is_not_serialized_into_one_command(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + agent = _agent(fixture) + environment = _LocalEnvironment() + workspace = Path(directory) / "large-workspace" + long_parent = Path("a" * 200) / ("b" * 200) + (workspace / long_parent).mkdir(parents=True) + files = [] + for index in range(326): + relative = str(long_parent / f"file-{index:03d}.txt") + raw = f"{index}\n".encode() + (workspace / relative).write_bytes(raw) + os.chmod(workspace / relative, 0o644) + files.append( + contract_module.WorkspaceFile( + path=relative, + mode=0o644, + sha256=_sha(raw), + byte_length=len(raw), + ) + ) + + identity_script = workspace_boundary_module._workspace_check_script( + str(workspace), + tuple(files), + allow_task_metadata=False, + ) + self.assertGreater(len(identity_script), 131_072) + try: + asyncio.run( + agent._workspace_boundary.verify_container_workspace( + environment, + str(workspace), + tuple(files), + allow_task_metadata=False, + ) + ) + finally: + agent._snapshots.cleanup() + + self.assertTrue(environment.commands) + self.assertLess( + max(len(command.encode("utf-8")) for command in environment.commands), + 32 * 1024, + ) + self.assertTrue( + any(command.startswith("/bin/sh ") for command in environment.commands) + ) + control = fixture["plan_path"].parent / "evaluator" / "control" + self.assertEqual(list(control.glob("workspace-check-*.sh")), []) + self.assertEqual(list(control.glob(".tangle-*")), []) + + def test_workspace_identity_rejects_every_file_identity_mutation(self): + for mutation in ( + "content", + "length", + "mode", + "missing", + "extra", + "symlink", + "hardlink", + "fifo", + ): + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + fixture = _fixture(Path(directory)) + agent = _agent(fixture) + environment = _LocalEnvironment() + workspace = Path(directory) / "workspace" + workspace.mkdir() + original = b"signed\n" + target = workspace / "signed.txt" + target.write_bytes(original) + os.chmod(target, 0o644) + files = ( + contract_module.WorkspaceFile( + path="signed.txt", + mode=0o644, + sha256=_sha(original), + byte_length=len(original), + ), + ) + if mutation == "content": + target.write_bytes(b"tigned\n") + elif mutation == "length": + target.write_bytes(b"signed-longer\n") + elif mutation == "mode": + os.chmod(target, 0o755) + elif mutation == "missing": + target.unlink() + elif mutation == "extra": + (workspace / "unsigned.txt").write_text("unsigned\n") + elif mutation == "symlink": + outside = Path(directory) / "outside.txt" + outside.write_bytes(original) + target.unlink() + target.symlink_to(outside) + elif mutation == "hardlink": + os.link(target, Path(directory) / "second-link.txt") + else: + target.unlink() + os.mkfifo(target) + + try: + with self.assertRaises(candidate.PierCandidateError): + asyncio.run( + agent._workspace_boundary.verify_container_workspace( + environment, + str(workspace), + files, + allow_task_metadata=False, + ) + ) + finally: + agent._snapshots.cleanup() + control = fixture["plan_path"].parent / "evaluator" / "control" + self.assertEqual(list(control.glob("workspace-check-*.sh")), []) + self.assertEqual(list(control.glob(".tangle-*")), []) + + def test_failed_staged_workspace_check_cleans_up(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + agent = _agent(fixture) + environment = _LocalEnvironment() + workspace = Path(directory) / "workspace" + workspace.mkdir() + original = b"signed\n" + target = workspace / "signed.txt" + target.write_bytes(original) + os.chmod(target, 0o644) + files = ( + contract_module.WorkspaceFile( + path="signed.txt", + mode=0o644, + sha256=_sha(original), + byte_length=len(original), + ), + ) + target.write_bytes(b"tigned\n") + try: + with self.assertRaises(candidate.PierCandidateError): + asyncio.run( + agent._workspace_boundary.verify_container_workspace( + environment, + str(workspace), + files, + allow_task_metadata=False, + ) + ) + finally: + agent._snapshots.cleanup() + self.assertTrue( + any(command.startswith("/bin/sh ") for command in environment.commands) + ) + control = fixture["plan_path"].parent / "evaluator" / "control" + self.assertEqual(list(control.glob("workspace-check-*.sh")), []) + self.assertEqual(list(control.glob(".tangle-*")), []) + + def test_workspace_identity_quotes_shell_metacharacters(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + agent = _agent(fixture) + environment = _LocalEnvironment() + workspace = Path(directory) / "quoted-workspace" + workspace.mkdir() + names = ( + "-leading.txt", + "$(printf injected).txt", + "*.glob", + "apostrophe's.txt", + "space name.txt", + ) + files = [] + for name in names: + raw = f"{name}\n".encode() + target = workspace / name + target.write_bytes(raw) + os.chmod(target, 0o644) + files.append( + contract_module.WorkspaceFile( + path=name, + mode=0o644, + sha256=_sha(raw), + byte_length=len(raw), + ) + ) + try: + asyncio.run( + agent._workspace_boundary.verify_container_workspace( + environment, + str(workspace), + tuple(sorted(files, key=lambda file: file.path)), + allow_task_metadata=False, + ) + ) + finally: + agent._snapshots.cleanup() + self.assertEqual( + sorted(path.name for path in workspace.iterdir()), sorted(names) + ) + + def test_workspace_identity_rejects_corrupted_staged_program_and_cleans_up(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + agent = _agent(fixture) + environment = _CorruptingUploadEnvironment() + workspace = Path(directory) / "workspace" + workspace.mkdir() + raw = b"signed\n" + target = workspace / "signed.txt" + target.write_bytes(raw) + os.chmod(target, 0o644) + files = ( + contract_module.WorkspaceFile( + path="signed.txt", + mode=0o644, + sha256=_sha(raw), + byte_length=len(raw), + ), + ) + try: + with self.assertRaises(candidate.PierCandidateError): + asyncio.run( + agent._workspace_boundary.verify_container_workspace( + environment, + str(workspace), + files, + allow_task_metadata=False, + ) + ) + finally: + agent._snapshots.cleanup() + control = fixture["plan_path"].parent / "evaluator" / "control" + self.assertEqual(list(control.glob("workspace-check-*.sh")), []) + self.assertEqual(list(control.glob(".tangle-*")), []) + + def test_workspace_check_preserves_primary_failure_when_cleanup_also_fails(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + agent = _agent(fixture) + environment = _FailingWorkspaceCleanupEnvironment() + workspace = Path(directory) / "workspace" + workspace.mkdir() + original = b"signed\n" + target = workspace / "signed.txt" + target.write_bytes(b"tigned\n") + os.chmod(target, 0o644) + files = ( + contract_module.WorkspaceFile( + path="signed.txt", + mode=0o644, + sha256=_sha(original), + byte_length=len(original), + ), + ) + try: + with self.assertRaisesRegex( + candidate.PierCandidateError, + r"candidate bridge command failed \(1\): /bin/sh", + ) as raised: + asyncio.run( + agent._workspace_boundary.verify_container_workspace( + environment, + str(workspace), + files, + allow_task_metadata=False, + ) + ) + finally: + agent._snapshots.cleanup() + self.assertTrue( + any( + "workspace check cleanup failed" in note + and "forced workspace cleanup failure" in note + for note in raised.exception.__notes__ + ) + ) + + +class CandidateContractTest(unittest.TestCase): + def test_loads_exact_runtime_artifacts_and_rejects_plan_substitution(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + loaded = contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + self.assertEqual(loaded.execution_id, "pier-fixture-execution") + self.assertEqual(loaded.instruction.delivery_kind, "argv-append") + self.assertEqual(loaded.model_network_domains, ()) + + fixture["plan"]["executionId"] = "substituted" + fixture["plan_path"].write_bytes(_canonical(fixture["plan"])) + with self.assertRaisesRegex( + contract_module.CandidateContractError, "does not bind" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_rejects_extra_utf8_file_delivery_fields(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["task"]["instruction"]["delivery"] = { + "kind": "utf8-file", + "env": "TANGLE_CANDIDATE_TASK_PATH", + "path": "/tangle/input/task.txt", + "ignored": "unsigned-semantics", + } + fixture["plan"]["launch"]["env"] = { + "TANGLE_CANDIDATE_TASK_PATH": { + "kind": "public", + "value": "/tangle/input/task.txt", + } + } + _rewrite_signed_plan(fixture) + + with self.assertRaisesRegex( + contract_module.CandidateContractError, "unexpected fields" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_accepts_only_frozen_public_model_gateway_domains(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["limits"]["maxModelCalls"] = 1 + fixture["plan"]["model"]["access"]["network"] = { + "mode": "gateway-only", + "domains": ["router.tangle.tools"], + } + _rewrite_signed_plan(fixture) + loaded = contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + self.assertEqual( + loaded.model_network_domains, ("router.tangle.tools",) + ) + agent = _agent(fixture) + try: + self.assertEqual( + agent.network_allowlist().domains, + ["router.tangle.tools"], + ) + finally: + agent._snapshots.cleanup() + + def test_rejects_wildcard_or_unsorted_model_gateway_domains(self): + for domains in ( + ["*.tangle.tools"], + ["router.tangle.tools", "api.openai.com"], + ): + with ( + self.subTest(domains=domains), + tempfile.TemporaryDirectory() as directory, + ): + fixture = _fixture(Path(directory)) + fixture["plan"]["limits"]["maxModelCalls"] = 1 + fixture["plan"]["model"]["access"]["network"] = { + "mode": "gateway-only", + "domains": domains, + } + _rewrite_signed_plan(fixture) + with self.assertRaises(contract_module.CandidateContractError): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_rejects_model_gateway_when_call_budget_is_zero(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["model"]["access"]["network"] = { + "mode": "gateway-only", + "domains": ["router.tangle.tools"], + } + _rewrite_signed_plan(fixture) + with self.assertRaisesRegex( + contract_module.CandidateContractError, "zero-call plans" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_rejects_unsigned_candidate_workspace_files(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + (fixture["candidate_staging"] / "unsigned.py").write_text("bad\n") + with self.assertRaisesRegex( + contract_module.CandidateContractError, "file set differs" + ): + _agent(fixture) + + def test_rejects_container_image_with_embedded_digest(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["container"]["image"] += f"@sha256:{'9' * 64}" + _rewrite_signed_plan(fixture) + with self.assertRaisesRegex( + contract_module.CandidateContractError, "unpinned OCI reference" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + +class TangleCandidateAgentTest(unittest.TestCase): + def test_process_stop_acknowledges_candidate_identity_is_empty(self): + boundary = process_boundary_module.CandidateProcessBoundary( + _LocalBoundaryHost(), + process_boundary_module.ProcessBoundaryConfig( + uid=2_000_000_000, + gid=2_000_000_000, + evaluator_root="/tmp/unused-evaluator", + control_root="/tmp/unused-control", + home="/tmp/unused-home", + temporary="/tmp/unused-tmp", + ), + ) + asyncio.run(boundary.kill_and_wait(_LocalEnvironment())) + + def test_executes_exact_instruction_and_excludes_profile_from_patch(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + agent = _agent(fixture) + environment = _LocalEnvironment() + context = _AgentContext() + asyncio.run(agent.setup(environment)) + self.assertEqual( + (fixture["task_root"] / "AGENTS.md").read_text(), + "fixture-profile\n", + ) + asyncio.run(agent.run("make the task ready", environment, context)) + + self.assertEqual( + (fixture["task_root"] / "src/status.txt").read_text(), "ready\n" + ) + self.assertFalse((fixture["task_root"] / "AGENTS.md").exists()) + changed = _git( + fixture["task_root"], + "diff", + "--name-only", + f"{fixture['base_commit']}..HEAD", + ).splitlines() + self.assertEqual(changed, ["src/status.txt"]) + self.assertEqual(context.n_input_tokens, None) + self.assertEqual(context.cost_usd, None) + self.assertEqual(context.metadata["executionId"], "pier-fixture-execution") + self.assertEqual( + context.metadata["termination"], {"kind": "exit", "exitCode": 0} + ) + self.assertEqual( + context.metadata["usageSource"], "protected-agent-eval-trace" + ) + + def test_rejects_instruction_substitution_before_launch(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + agent = _agent(fixture) + environment = _LocalEnvironment() + asyncio.run(agent.setup(environment)) + with self.assertRaisesRegex( + candidate.PierCandidateError, "instruction bytes do not match" + ): + asyncio.run(agent.run("different task", environment, _AgentContext())) + + def test_candidate_target_profile_still_captures_task_solution(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["profile"]["targetWorkspace"] = "candidate" + _rewrite_signed_plan(fixture) + agent = _agent(fixture) + environment = _LocalEnvironment() + context = _AgentContext() + asyncio.run(agent.setup(environment)) + asyncio.run(agent.run("make the task ready", environment, context)) + + changed = _git( + fixture["task_root"], + "diff", + "--name-only", + f"{fixture['base_commit']}..HEAD", + ).splitlines() + self.assertEqual(changed, ["src/status.txt"]) + self.assertEqual( + (fixture["task_root"] / "src/status.txt").read_text(), "ready\n" + ) + + def test_enforces_exact_signed_timeout(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["launch"]["env"] = { + "FIXTURE_SLEEP": {"kind": "public", "value": "0.25"} + } + fixture["plan"]["limits"]["timeoutMs"] = 10 + _rewrite_signed_plan(fixture) + agent = _agent(fixture) + environment = _LocalEnvironment() + context = _AgentContext() + asyncio.run(agent.setup(environment)) + started = time.monotonic() + with self.assertRaises(asyncio.TimeoutError): + asyncio.run(agent.run("make the task ready", environment, context)) + self.assertLess(time.monotonic() - started, 0.2) + self.assertEqual(context.metadata["termination"]["kind"], "timeout") + self.assertEqual(context.metadata["termination"]["timeoutMs"], 10) + + def test_profile_cleanup_does_not_follow_candidate_links(self): + for attack in ("symlink", "hardlink"): + with ( + self.subTest(attack=attack), + tempfile.TemporaryDirectory() as directory, + ): + fixture = _fixture(Path(directory)) + fixture["plan"]["launch"]["env"] = { + "FIXTURE_ATTACK": {"kind": "public", "value": attack} + } + _rewrite_signed_plan(fixture) + agent = _agent(fixture) + environment = _LocalEnvironment() + context = _AgentContext() + asyncio.run(agent.setup(environment)) + asyncio.run(agent.run("make the task ready", environment, context)) + + self.assertFalse((fixture["task_root"] / "AGENTS.md").exists()) + self.assertEqual( + (fixture["task_root"] / "src/status.txt").read_text(), + "ready\n", + ) + changed = _git( + fixture["task_root"], + "diff", + "--name-only", + f"{fixture['base_commit']}..HEAD", + ).splitlines() + self.assertEqual(changed, ["src/status.txt"]) + + def test_rejects_symlinked_candidate_root_ancestor(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + alias = Path(directory) / "aliased-parent" + alias.symlink_to(fixture["task_root"], target_is_directory=True) + candidate_root = alias / "candidate" + fixture["plan"]["workspaces"]["candidateRoot"] = str(candidate_root) + fixture["plan"]["launch"]["args"] = [ + {"kind": "public", "value": str(candidate_root / "runner.py")} + ] + _rewrite_signed_plan(fixture) + agent = _agent(fixture) + with self.assertRaises(candidate.PierCandidateError): + asyncio.run(agent.setup(_LocalEnvironment())) + + +if __name__ == "__main__": + unittest.main() diff --git a/bench/pier_agents/workspace_boundary.py b/bench/pier_agents/workspace_boundary.py new file mode 100644 index 00000000..dd946c10 --- /dev/null +++ b/bench/pier_agents/workspace_boundary.py @@ -0,0 +1,368 @@ +"""Verify and materialize runtime-signed workspaces inside a Pier container.""" + +from __future__ import annotations + +import secrets +import shlex +import tempfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Protocol + +from pier.environments.base import BaseEnvironment + +from .candidate_contract import ( + PreparedCandidateContract, + WorkspaceFile, + sha256_bytes, +) + + +def _workspace_check_script( + root: str, + files: tuple[WorkspaceFile, ...], + *, + allow_task_metadata: bool, +) -> bytes: + quoted_root = shlex.quote(root) + prune = ( + f"\\( -path {shlex.quote(root + '/.git')} -o " + f"-path {shlex.quote(root + '/.sidecar')} \\) -prune -o " + if allow_task_metadata + else "" + ) + expected_args = " ".join(shlex.quote(file.path) for file in files) + expected = ( + f"$(printf '%s\\n' {expected_args} | LC_ALL=C sort)" if files else "''" + ) + checks = [ + "#!/bin/sh", + "set -eu", + f"test -d {quoted_root}", + f"test ! -L {quoted_root}", + f'test "$(realpath -e -- {quoted_root})" = {quoted_root}', + f'test -z "$(find {quoted_root} {prune}-type l -print -quit)"', + f'test -z "$(find {quoted_root} {prune}! -type d ! -type f -print -quit)"', + f"observed=$(find {quoted_root} {prune}-type f -printf '%P\\n' | LC_ALL=C sort)", + f"expected={expected}", + 'test "$observed" = "$expected"', + ] + for file in files: + path = shlex.quote(f"{root}/{file.path}") + checks.extend( + [ + f"test -f {path}", + f"test ! -L {path}", + f'test "$(stat -c %h -- {path})" = 1', + f'test "$(stat -c %a -- {path})" = {file.mode:o}', + f'test "$(wc -c < {path})" = {file.byte_length}', + f"test \"$(sha256sum -- {path} | cut -d' ' -f1)\" = " + f"{file.sha256.removeprefix('sha256:')}", + ] + ) + return ("\n".join(checks) + "\n").encode("utf-8") + + +class WorkspaceBoundaryHost(Protocol): + async def _exec( + self, environment: BaseEnvironment, command: str, **kwargs: Any + ) -> Any: ... + + async def _ensure_real_directory( + self, + environment: BaseEnvironment, + path: str, + *, + anchor: str, + mode: int = 0o755, + ) -> None: ... + + async def _assert_real_directory( + self, + environment: BaseEnvironment, + path: str, + *, + label: str, + user: str | int | None = "root", + ) -> None: ... + + async def _write_verified_file( + self, + environment: BaseEnvironment, + source: Path, + target: str, + *, + anchor: str, + mode: int, + digest: str, + byte_length: int, + ) -> None: ... + + +@dataclass(frozen=True) +class WorkspaceBoundaryConfig: + contract: PreparedCandidateContract + task_snapshot: Path + control_root: str + protected_roots: tuple[str, ...] + error_type: type[Exception] + + +def candidate_git_env(task_root: str) -> dict[str, str]: + return { + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "safe.directory", + "GIT_CONFIG_VALUE_0": task_root, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": "/dev/null", + "GIT_CONFIG_SYSTEM": "/dev/null", + "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_TERMINAL_PROMPT": "0", + "LC_ALL": "C", + } + + +class CandidateWorkspaceBoundary: + def __init__(self, host: WorkspaceBoundaryHost, config: WorkspaceBoundaryConfig): + self._host = host + self._config = config + + @property + def _contract(self) -> PreparedCandidateContract: + return self._config.contract + + def _error(self, message: str) -> Exception: + return self._config.error_type(message) + + async def verify_repository(self, environment: BaseEnvironment) -> None: + root = shlex.quote(self._contract.task_root) + git = f"git -c core.hooksPath=/dev/null -C {root}" + env = candidate_git_env(self._contract.task_root) + head = await self._host._exec(environment, f"{git} rev-parse HEAD", env=env) + tree = await self._host._exec( + environment, f"{git} rev-parse 'HEAD^{{tree}}'", env=env + ) + replacements = await self._host._exec( + environment, + f"{git} for-each-ref --format='%(refname)' refs/replace", + env=env, + ) + if (head.stdout or "").strip() != self._contract.repository_base_commit: + raise self._error("task checkout HEAD does not match the signed base commit") + if (tree.stdout or "").strip() != self._contract.repository_base_tree: + raise self._error("task checkout tree does not match the signed base tree") + if (replacements.stdout or "").strip(): + raise self._error("task checkout contains forbidden Git replace refs") + + async def materialize_task_workspace(self, environment: BaseEnvironment) -> None: + root = self._contract.task_root + quoted = shlex.quote(root) + exists = await self._host._exec( + environment, + f"test -e {quoted} || test -L {quoted}", + user="root", + check=False, + ) + if exists.return_code == 0: + return + if exists.return_code != 1: + raise self._error("could not inspect the signed task root") + + parent = str(PurePosixPath(root).parent) + await self._host._ensure_real_directory(environment, parent, anchor="/") + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + f"test ! -e {quoted}", + f"test ! -L {quoted}", + f"mkdir -- {quoted}", + f'test "$(realpath -e -- {quoted})" = {quoted}', + ] + ), + user="root", + ) + await environment.upload_dir(self._config.task_snapshot, root) + git = f"git -c core.hooksPath=/dev/null -C {quoted}" + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + f"{git} init -b main", + f"{git} config user.email fixture@tangle.tools", + f'{git} config user.name "Tangle Fixture"', + f"{git} add -A", + f"GIT_AUTHOR_DATE=2000-01-01T00:00:00Z " + f"GIT_COMMITTER_DATE=2000-01-01T00:00:00Z " + f"{git} commit -m baseline", + f"{git} tag benchmark-base", + ] + ), + env=candidate_git_env(root), + user="root", + ) + + async def verify_pier_execution_identity( + self, environment: BaseEnvironment + ) -> None: + expected = self._contract.container + configured_image = getattr(environment.task_env_config, "docker_image", None) + pinned_image = f"{expected['image']}@{expected['indexDigest']}" + if configured_image != pinned_image: + raise self._error( + "Pier task image does not match the signed immutable image: " + f"expected={pinned_image} observed={configured_image}" + ) + configured_os = getattr(environment.task_env_config, "os", None) + configured_os = getattr(configured_os, "value", configured_os) + if configured_os is not None and str(configured_os).lower() != "linux": + raise self._error( + f"Pier task OS does not match signed Linux platform: {configured_os}" + ) + platform = expected["platform"] + observed = await self._host._exec( + environment, + 'printf \'%s %s\' "$(uname -s)" "$(uname -m)"', + user="root", + ) + kernel, _, machine = (observed.stdout or "").strip().partition(" ") + architectures = {"x86_64": "amd64", "aarch64": "arm64"} + if kernel != "Linux" or architectures.get(machine) != platform["architecture"]: + raise self._error( + "running Pier platform does not match the signed container platform: " + f"observed={kernel}/{machine} expected=linux/{platform['architecture']}" + ) + + async def verify_resolved_root_isolation( + self, environment: BaseEnvironment + ) -> None: + roots = [self._contract.task_root] + if self._contract.candidate_root is not None: + roots.append(self._contract.candidate_root) + for root in roots: + await self._host._assert_real_directory( + environment, root, label="execution workspace" + ) + if self._contract.candidate_root is not None: + task = shlex.quote(self._contract.task_root) + candidate = shlex.quote(self._contract.candidate_root) + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + f"task=$(realpath -e -- {task})", + f"candidate=$(realpath -e -- {candidate})", + 'test "$task" != "$candidate"', + 'case "$task" in "$candidate"/*) exit 1;; esac', + 'case "$candidate" in "$task"/*) exit 1;; esac', + ] + ), + user="root", + ) + for protected in self._config.protected_roots: + quoted_protected = shlex.quote(protected) + for root in roots: + quoted_root = shlex.quote(root) + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + f"if test -e {quoted_protected}; then", + f" protected=$(realpath -e -- {quoted_protected})", + f" root=$(realpath -e -- {quoted_root})", + ' test "$root" != "$protected"', + ' case "$root" in "$protected"/*) exit 1;; esac', + ' case "$protected" in "$root"/*) exit 1;; esac', + "fi", + ] + ), + user="root", + ) + + async def verify_container_workspace( + self, + environment: BaseEnvironment, + root: str, + files: tuple[WorkspaceFile, ...], + *, + allow_task_metadata: bool, + ) -> None: + script = _workspace_check_script( + root, + files, + allow_task_metadata=allow_task_metadata, + ) + control_root = self._config.control_root + target = f"{control_root}/workspace-check-{secrets.token_hex(24)}.sh" + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(script) + local_path = Path(handle.name) + control_ready = False + + async def cleanup() -> None: + local_path.unlink(missing_ok=True) + if not control_ready: + return + await self._host._assert_real_directory( + environment, + control_root, + label="candidate control directory", + ) + await self._host._exec( + environment, + f"rm -f -- {shlex.quote(target)}", + user="root", + ) + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + f"test ! -e {shlex.quote(target)}", + f"test ! -L {shlex.quote(target)}", + f'test -z "$(find {shlex.quote(control_root)} -maxdepth 1 ' + "-name '.tangle-*' -print -quit)\"", + ] + ), + user="root", + ) + + try: + await self._host._ensure_real_directory( + environment, + control_root, + anchor="/", + mode=0o700, + ) + control_ready = True + await self._host._exec( + environment, + f"chmod 0700 -- {shlex.quote(control_root)}", + user="root", + ) + await self._host._write_verified_file( + environment, + local_path, + target, + anchor=control_root, + mode=0o600, + digest=sha256_bytes(script), + byte_length=len(script), + ) + await self._host._exec( + environment, + f"/bin/sh {shlex.quote(target)}", + user="root", + ) + except BaseException as primary_error: + try: + await cleanup() + except BaseException as cleanup_error: + primary_error.add_note(f"workspace check cleanup failed: {cleanup_error}") + raise + else: + await cleanup() diff --git a/bench/pnpm-lock.yaml b/bench/pnpm-lock.yaml index 186f3f70..94d8d185 100644 --- a/bench/pnpm-lock.yaml +++ b/bench/pnpm-lock.yaml @@ -4,20 +4,29 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + '@tangle-network/agent-eval': 0.117.1 + importers: .: dependencies: '@tangle-network/agent-eval': - specifier: ^0.106.3 - version: 0.106.3(typescript@6.0.3) + specifier: 0.117.1 + version: 0.117.1(typescript@6.0.3) + '@tangle-network/agent-interface': + specifier: ^0.25.0 + version: 0.25.0 '@tangle-network/agent-runtime': - specifier: ^0.89.0 - version: 0.89.0(@tangle-network/agent-eval@0.106.3(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))) + specifier: 0.94.9 + version: 0.94.9(@tangle-network/agent-eval@0.117.1(typescript@6.0.3))(@tangle-network/agent-interface@0.25.0)(@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) '@tangle-network/sandbox': - specifier: ^0.9.7 - version: 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) + specifier: ^0.10.3 + version: 0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) devDependencies: + '@types/node': + specifier: ^25.0.0 + version: 25.9.5 tsx: specifier: ^4.19.0 version: 4.22.4 @@ -251,37 +260,75 @@ packages: '@tangle-network/agent-core@0.3.4': resolution: {integrity: sha512-Hvz3ABRouNtBmRvGqPxifAO2yuILneJMylWH5jW/jeS2F03RvqkGYuXyGXWWLqosYbb3hVAvSEe4Ykm2FMGEDQ==} - '@tangle-network/agent-eval@0.106.3': - resolution: {integrity: sha512-5mLpTNoR4YcAbHueCrYcdoBRo+I9zmjXipzWaYfJ2FGj4byA6L4wrTPdTneOLjvxOGlrq+H/CRmHB738fP8R/Q==} + '@tangle-network/agent-core@0.4.9': + resolution: {integrity: sha512-BFU4WV12Y6D28PX+o8LIVkIMD+cXwrL1uyV182l12Bn9zEu7VHt2oVMEEzbsN8L+X0xM9baEfMCHTFKFNJv7Aw==} + + '@tangle-network/agent-eval@0.117.1': + resolution: {integrity: sha512-mtBdGhGIC+nrPZbseLQo0Iako1LR66ZeIr2UjVBJcNj/lHYfib2jbhdWy3RmbQcHR0Cxdqqydn7pLJ5dSXVYtQ==} engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-interface@0.10.1': - resolution: {integrity: sha512-yehY/0EgKvu8lG6jIVoZCtMPLkj8VEWwasuAtuph2RaB9MKE5wuxRF647O6jw8KufNZ3aQ2UVVWpZ19dGCbs6w==} - '@tangle-network/agent-interface@0.13.0': resolution: {integrity: sha512-CeTPGRLoXqpt0h+BCyFgZPkfU1zyRpWmqfD+85i/uk+uvbqxkfI+JprfKVf3tBsQuCgJPSjPt5qjdW8n3h2BVg==} '@tangle-network/agent-interface@0.14.0': resolution: {integrity: sha512-9CyGhIpl90E7v4MTm3b1ti3Bp7BfPigk2Nafgi21Lg0U+QxlNB656F2JmVpUuSbOo9aGZPtg5nXu5EBTlV5a1g==} - '@tangle-network/agent-runtime@0.89.0': - resolution: {integrity: sha512-tjQ/uJORuLOAE/E99w0NxWb5O0uP/nwescMRyIqxsqLXzry7WhNh9DK/BHKTEfb6XMvcIaT92J/3T1ZJTtXZ9Q==} + '@tangle-network/agent-interface@0.21.0': + resolution: {integrity: sha512-jDxhVJgxymrvU1RLWxWKueuaWQIpBAfrW8BuVTB5m2Y4eFMLo1SawDBEMDLdZN4/Gf34xrFrsRk+PAj9brGKMQ==} + + '@tangle-network/agent-interface@0.22.0': + resolution: {integrity: sha512-7fsJhNdvTmOB1X9E2owl06jzyrqaN+jMkOPVKbK7dvNqQvf627PowCtL/edhUqEEv+K0FWtkwG3R3xqjX7VNZg==} + + '@tangle-network/agent-interface@0.24.0': + resolution: {integrity: sha512-iaHWNTYne49cBYXwb72NjGDw5bjT2KVJlfawXygbvkqnfUsQG57BS++Yjf/XYvwJJdDeEaGs+Xvy8SWYkvu0qA==} + + '@tangle-network/agent-interface@0.25.0': + resolution: {integrity: sha512-bMKjyjk8bk2651HleiTEUOTPgNE2bWxCEl9VBVUyub/UkP87Y09byXOxeDEsNGc4TIT19Y5Uo0b+k1kRWXkMVQ==} + + '@tangle-network/agent-knowledge@1.12.1': + resolution: {integrity: sha512-j5riyIvz5C+ZBELTtNVtmUbcKAMpgTHMNmDQ12MhOIPJv5Y7AeX2oCWuF6c8aPqg1hOrsLqgqXSr1zkV0ePsrA==} + engines: {node: '>=20'} + hasBin: true + + '@tangle-network/agent-profile-materialize@0.3.2': + resolution: {integrity: sha512-jCj1Hc/brPQ5eS7or9A+Klv0FAZcGtyFr7kx2cKfk+wCj0/4Di3k1pMjayrmRNgI/7Oc47gt6/t+qvdtZxBkAw==} + + '@tangle-network/agent-runtime@0.94.9': + resolution: {integrity: sha512-p0mKK4DbzmrVVQPhNTH+Si7mpIV5+/0avhGVNzJcq2YdcH0YWYW2+OUy86mhXkxv/vVNkdR70Sco91hZuoC6ig==} engines: {node: '>=20'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.101.0 <1.0.0' - '@tangle-network/agent-interface': '>=0.14.0 <1.0.0' + '@tangle-network/agent-eval': 0.117.1 + '@tangle-network/agent-interface': '>=0.25.0 <0.26.0' '@tangle-network/sandbox': '>=0.8.0 <1.0.0' playwright: ^1.40.0 peerDependenciesMeta: - '@tangle-network/agent-interface': - optional: true '@tangle-network/sandbox': optional: true playwright: optional: true + '@tangle-network/sandbox@0.10.3': + resolution: {integrity: sha512-3nZnIaXc/vCH3Lb6SCZA3cYEJT4+ThGXLPkGo5z2kh434n3eJryk/PdnSXSwPoA6xuTQToJAPbGGIdMEHFU/Vw==} + peerDependencies: + '@mastra/core': ^1.36.0 + '@modelcontextprotocol/sdk': ^1.29.0 + ai: ^6.0.175 + openai: ^6.36.0 + viem: ^2.0.0 + peerDependenciesMeta: + '@mastra/core': + optional: true + '@modelcontextprotocol/sdk': + optional: true + ai: + optional: true + openai: + optional: true + viem: + optional: true + '@tangle-network/sandbox@0.9.7': resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} peerDependencies: @@ -311,6 +358,9 @@ packages: engines: {node: '>=18'} hasBin: true + '@types/node@25.9.5': + resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} + abitype@1.2.3: resolution: {integrity: sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==} peerDependencies: @@ -322,6 +372,51 @@ packages: zod: optional: true + b4a@1.8.1: + resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + + bare-events@2.9.1: + resolution: {integrity: sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==} + peerDependencies: + bare-abort-controller: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + + bare-fs@4.7.4: + resolution: {integrity: sha512-y1kC+ffIx/tPLdTE693uNjHfzTfr+ravR5tvWlMXe25nELbkqV400S71qHDwbkAQ1FVEZobB1NFRzFbCCcyBCQ==} + engines: {bare: '>=1.16.0'} + peerDependencies: + bare-buffer: '*' + peerDependenciesMeta: + bare-buffer: + optional: true + + bare-path@3.1.1: + resolution: {integrity: sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==} + + bare-stream@2.13.3: + resolution: {integrity: sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==} + peerDependencies: + bare-abort-controller: '*' + bare-buffer: '*' + bare-events: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + bare-buffer: + optional: true + bare-events: + optional: true + + bare-url@2.4.5: + resolution: {integrity: sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==} + commander@14.0.3: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} @@ -337,13 +432,19 @@ packages: eventemitter3@5.0.1: resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] - hono@4.12.23: - resolution: {integrity: sha512-eIaZ9qDgu7XV0pxOCrg7/WhnQ6Ivm22UcxhXx/A3dcbqbbYgBEkc6e/J/s7j2tS96zoB0S9VBdLwQNCWwUo4LA==} + hono@4.12.30: + resolution: {integrity: sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==} engines: {node: '>=16.9.0'} isows@1.0.7: @@ -362,6 +463,18 @@ packages: typescript: optional: true + streamx@2.28.0: + resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==} + + tar-stream@3.2.0: + resolution: {integrity: sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==} + + teex@1.0.1: + resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==} + + text-decoder@1.2.7: + resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} + tsx@4.22.4: resolution: {integrity: sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==} engines: {node: '>=18.0.0'} @@ -372,6 +485,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + viem@2.52.0: resolution: {integrity: sha512-py2QPYe9e1f4DmPJCsXF7zHmyZ0PkJrBxdQZ5dvNXvzy3UzWkUn7dNfC0TMeNm6Qv1tKw3b6qXXExpx6L0oMbw==} peerDependencies: @@ -494,9 +610,9 @@ snapshots: '@esbuild/win32-x64@0.28.0': optional: true - '@hono/node-server@2.0.4(hono@4.12.23)': + '@hono/node-server@2.0.4(hono@4.12.30)': dependencies: - hono: 4.12.23 + hono: 4.12.30 '@noble/ciphers@1.3.0': {} @@ -545,14 +661,19 @@ snapshots: '@tangle-network/agent-interface': 0.14.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.106.3(typescript@6.0.3)': + '@tangle-network/agent-core@0.4.9': + dependencies: + '@tangle-network/agent-interface': 0.24.0 + zod: 4.4.3 + + '@tangle-network/agent-eval@0.117.1(typescript@6.0.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 19.0.45(zod@4.4.3) - '@hono/node-server': 2.0.4(hono@4.12.23) - '@tangle-network/agent-interface': 0.10.1 + '@hono/node-server': 2.0.4(hono@4.12.30) + '@tangle-network/agent-interface': 0.22.0 '@tangle-network/tcloud': 0.4.14(typescript@6.0.3)(zod@4.4.3) - hono: 4.12.23 + hono: 4.12.30 zod: 4.4.3 transitivePeerDependencies: - '@mastra/core' @@ -563,24 +684,74 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-interface@0.10.1': + '@tangle-network/agent-interface@0.13.0': dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.13.0': + '@tangle-network/agent-interface@0.14.0': dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.14.0': + '@tangle-network/agent-interface@0.21.0': dependencies: zod: 4.4.3 - '@tangle-network/agent-runtime@0.89.0(@tangle-network/agent-eval@0.106.3(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))': + '@tangle-network/agent-interface@0.22.0': dependencies: - '@tangle-network/agent-eval': 0.106.3(typescript@6.0.3) + zod: 4.4.3 + + '@tangle-network/agent-interface@0.24.0': + dependencies: + zod: 4.4.3 + + '@tangle-network/agent-interface@0.25.0': + dependencies: + zod: 4.4.3 + + '@tangle-network/agent-knowledge@1.12.1(typescript@6.0.3)': + dependencies: + '@tangle-network/agent-eval': 0.117.1(typescript@6.0.3) + zod: 4.4.3 + transitivePeerDependencies: + - '@mastra/core' + - '@modelcontextprotocol/sdk' + - ai + - bufferutil + - openai + - typescript + - utf-8-validate + + '@tangle-network/agent-profile-materialize@0.3.2': + dependencies: + '@tangle-network/agent-interface': 0.25.0 + + '@tangle-network/agent-runtime@0.94.9(@tangle-network/agent-eval@0.117.1(typescript@6.0.3))(@tangle-network/agent-interface@0.25.0)(@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': + dependencies: + '@tangle-network/agent-eval': 0.117.1(typescript@6.0.3) + '@tangle-network/agent-interface': 0.25.0 + '@tangle-network/agent-knowledge': 1.12.1(typescript@6.0.3) + '@tangle-network/agent-profile-materialize': 0.3.2 + tar-stream: 3.2.0 optionalDependencies: - '@tangle-network/agent-interface': 0.14.0 - '@tangle-network/sandbox': 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) + '@tangle-network/sandbox': 0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) + transitivePeerDependencies: + - '@mastra/core' + - '@modelcontextprotocol/sdk' + - ai + - bare-abort-controller + - bare-buffer + - bufferutil + - openai + - react-native-b4a + - typescript + - utf-8-validate + + '@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))': + dependencies: + '@tangle-network/agent-core': 0.4.9 + '@tangle-network/agent-interface': 0.21.0 + optionalDependencies: + viem: 2.52.0(typescript@6.0.3)(zod@4.4.3) '@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))': dependencies: @@ -609,11 +780,46 @@ snapshots: - utf-8-validate - zod + '@types/node@25.9.5': + dependencies: + undici-types: 7.24.6 + abitype@1.2.3(typescript@6.0.3)(zod@4.4.3): optionalDependencies: typescript: 6.0.3 zod: 4.4.3 + b4a@1.8.1: {} + + bare-events@2.9.1: {} + + bare-fs@4.7.4: + dependencies: + bare-events: 2.9.1 + bare-path: 3.1.1 + bare-stream: 2.13.3(bare-events@2.9.1) + bare-url: 2.4.5 + fast-fifo: 1.3.2 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + bare-path@3.1.1: {} + + bare-stream@2.13.3(bare-events@2.9.1): + dependencies: + b4a: 1.8.1 + streamx: 2.28.0 + teex: 1.0.1 + optionalDependencies: + bare-events: 2.9.1 + transitivePeerDependencies: + - react-native-b4a + + bare-url@2.4.5: + dependencies: + bare-path: 3.1.1 + commander@14.0.3: {} dayjs@1.11.21: {} @@ -649,10 +855,18 @@ snapshots: eventemitter3@5.0.1: {} + events-universal@1.0.1: + dependencies: + bare-events: 2.9.1 + transitivePeerDependencies: + - bare-abort-controller + + fast-fifo@1.3.2: {} + fsevents@2.3.3: optional: true - hono@4.12.23: {} + hono@4.12.30: {} isows@1.0.7(ws@8.20.1): dependencies: @@ -677,6 +891,39 @@ snapshots: transitivePeerDependencies: - zod + streamx@2.28.0: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.7 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + tar-stream@3.2.0: + dependencies: + b4a: 1.8.1 + bare-fs: 4.7.4 + fast-fifo: 1.3.2 + streamx: 2.28.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + + teex@1.0.1: + dependencies: + streamx: 2.28.0 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + text-decoder@1.2.7: + dependencies: + b4a: 1.8.1 + transitivePeerDependencies: + - react-native-b4a + tsx@4.22.4: dependencies: esbuild: 0.28.0 @@ -685,6 +932,8 @@ snapshots: typescript@6.0.3: {} + undici-types@7.24.6: {} + viem@2.52.0(typescript@6.0.3)(zod@4.4.3): dependencies: '@noble/curves': 1.9.1 diff --git a/bench/scripts/run-package-tests.mjs b/bench/scripts/run-package-tests.mjs new file mode 100644 index 00000000..2a3514da --- /dev/null +++ b/bench/scripts/run-package-tests.mjs @@ -0,0 +1,56 @@ +import { execFile } from 'node:child_process' +import { access, readdir } from 'node:fs/promises' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const sourceDir = path.join(benchDir, 'src') + +async function collectTests(dir) { + const files = [] + for (const entry of await readdir(dir, { withFileTypes: true })) { + const absolute = path.join(dir, entry.name) + if (entry.isDirectory()) files.push(...(await collectTests(absolute))) + else if (entry.isFile() && /\.test\.(?:mts|ts)$/.test(entry.name)) files.push(absolute) + } + return files.sort() +} + +async function run(command, args, env = process.env) { + try { + await execFileAsync(command, args, { + cwd: benchDir, + env, + maxBuffer: 10 * 1024 * 1024, + timeout: 120_000, + }) + } catch (error) { + if (error?.stdout) process.stdout.write(error.stdout) + if (error?.stderr) process.stderr.write(error.stderr) + const invocation = [command, ...args].join(' ') + const message = error instanceof Error ? error.message : String(error) + throw new Error(`${invocation} failed: ${message}`, { cause: error }) + } +} + +const python = path.join(benchDir, '.venv', 'bin', 'python') +try { + await access(python) +} catch { + await run('python3', ['-m', 'venv', '.venv']) +} + +const tests = await collectTests(sourceDir) +const relativeTests = tests.map((file) => path.relative(benchDir, file)) +if (relativeTests.length === 0) throw new Error('no package tests found under src/') + +await run(process.execPath, ['--test', '--import', 'tsx', ...relativeTests], { + ...process.env, + TSX_TSCONFIG_PATH: 'tsconfig.public.json', +}) + +await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py']) + +console.log(`package tests passed: ${tests.length}/${tests.length} TypeScript files + Pier bridge`) diff --git a/bench/scripts/terminate-pier-trial.mts b/bench/scripts/terminate-pier-trial.mts new file mode 100644 index 00000000..4ea9d489 --- /dev/null +++ b/bench/scripts/terminate-pier-trial.mts @@ -0,0 +1,66 @@ +import path from 'node:path' + +import { InMemoryTraceStore } from '@tangle-network/agent-eval' + +import { createPierCandidateRecoveryExecutor } from '../src/pier-agent' +import { FilePierCandidateTrialController } from '../src/pier-trial-controller' + +const [directoryArg, executionId, executionPlanDigest, ...extra] = process.argv.slice(2) +if (!directoryArg || !executionId || !executionPlanDigest || extra.length > 0) { + throw new Error( + 'usage: terminate-pier-trial.mts ', + ) +} +if (!/^sha256:[a-f0-9]{64}$/.test(executionPlanDigest)) { + throw new Error('execution-plan-digest must be a SHA-256 digest') +} + +const dockerConnectionId = process.env.PIER_DOCKER_CONNECTION_ID +const dockerEnvironmentNames = process.env.PIER_DOCKER_ENV_NAMES +if ((dockerConnectionId === undefined) !== (dockerEnvironmentNames === undefined)) { + throw new Error( + 'PIER_DOCKER_CONNECTION_ID and PIER_DOCKER_ENV_NAMES must be set together', + ) +} +const dockerConnection = (() => { + if (dockerConnectionId === undefined || dockerEnvironmentNames === undefined) return undefined + const names = dockerEnvironmentNames + .split(',') + .map((name) => name.trim()) + .filter(Boolean) + if (new Set(names).size !== names.length) { + throw new Error('PIER_DOCKER_ENV_NAMES must not contain duplicates') + } + const env: Record = {} + for (const name of names) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) { + throw new Error('PIER_DOCKER_ENV_NAMES contains an invalid environment name') + } + const value = process.env[name] + if (value === undefined) throw new Error(`required Docker environment variable ${name} is unset`) + env[name] = value + } + return { id: dockerConnectionId, env } +})() + +const controller = new FilePierCandidateTrialController({ + directory: path.resolve(directoryArg), + ...(dockerConnection ? { dockerConnection } : {}), +}) +const executor = createPierCandidateRecoveryExecutor(controller) +const recovered = await executor.stopAndCapture( + { + executionId, + executionPlanDigest: executionPlanDigest as `sha256:${string}`, + }, + { + traceStore: new InMemoryTraceStore(), + reason: 'failed', + signal: new AbortController().signal, + deadlineAtMs: Date.now() + 30_000, + }, +) +if (recovered.stopped !== true) throw new Error('Pier recovery executor did not stop the trial') +process.stdout.write( + `${JSON.stringify({ processExited: true, containersRemoved: true })}\n`, +) diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs new file mode 100644 index 00000000..209261f2 --- /dev/null +++ b/bench/scripts/verify-packed-consumer.mjs @@ -0,0 +1,175 @@ +import { execFile } from 'node:child_process' +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const scratch = await mkdtemp(path.join(tmpdir(), 'agent-bench-consumer-')) +const runtimePackage = process.env.AGENT_RUNTIME_PACKAGE + ? path.resolve(process.env.AGENT_RUNTIME_PACKAGE) + : undefined + +async function run(command, args, cwd, env = process.env) { + try { + return await execFileAsync(command, args, { + cwd, + env, + maxBuffer: 10 * 1024 * 1024, + timeout: 120_000, + }) + } catch (error) { + if (error?.stdout) process.stdout.write(error.stdout) + if (error?.stderr) process.stderr.write(error.stderr) + const invocation = [command, ...args].join(' ') + const message = error instanceof Error ? error.message : String(error) + throw new Error(`${invocation} failed: ${message}`, { cause: error }) + } +} + +try { + const packDir = path.join(scratch, 'pack') + const consumerDir = path.join(scratch, 'consumer') + await mkdir(packDir) + await mkdir(consumerDir) + + const packed = await run('npm', ['pack', '--json', '--pack-destination', packDir], benchDir) + const [{ filename }] = JSON.parse(packed.stdout) + const tarball = path.join(packDir, filename) + const manifest = JSON.parse(await readFile(path.join(benchDir, 'package.json'), 'utf8')) + const devDependencies = manifest.devDependencies + if ( + typeof devDependencies?.['@types/node'] !== 'string' || + typeof devDependencies.typescript !== 'string' || + typeof devDependencies.tsx !== 'string' || + !devDependencies['@types/node'] || + !devDependencies.typescript || + !devDependencies.tsx + ) { + throw new Error( + 'package verification requires @types/node, typescript, and tsx devDependencies', + ) + } + const publicTsconfig = JSON.parse( + await readFile(path.join(benchDir, 'tsconfig.public.json'), 'utf8'), + ) + if (!publicTsconfig.compilerOptions) + throw new Error('tsconfig.public.json must define compilerOptions') + + await writeFile( + path.join(consumerDir, 'package.json'), + `${JSON.stringify( + { + name: 'agent-bench-package-verifier', + private: true, + type: 'module', + dependencies: { + '@tangle-network/agent-bench': `file:${tarball}`, + ...(runtimePackage + ? { '@tangle-network/agent-runtime': `file:${runtimePackage}` } + : {}), + }, + devDependencies: { + '@types/node': devDependencies['@types/node'], + typescript: devDependencies.typescript, + tsx: devDependencies.tsx, + }, + }, + null, + 2, + )}\n`, + ) + await writeFile( + path.join(consumerDir, 'index.ts'), + "import { createSweBenchAdapter, executePreparedPierCandidate, FilePierCandidateTrialController, resolveAdapter, runBenchmarks, runStagedJudge, StagedJudgeError, type BenchmarkAdapter, type JudgeArtifactReceipt, type PierCandidateTrialController, type PierCandidateTrialHandle, type PierDockerConnection, type StagedPierCandidateExecution } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nconst captureAdapter: BenchmarkAdapter = createSweBenchAdapter({ captureEvaluatorArtifacts: ({ taskId, attemptSequence }) => ({ destination: `/tmp/${taskId}/${attemptSequence}` }) })\nconst receipt = undefined as JudgeArtifactReceipt | undefined\nconst staged = undefined as StagedPierCandidateExecution | undefined\nconst trial = undefined as PierCandidateTrialHandle | undefined\nconst controller = undefined as PierCandidateTrialController | undefined\nconst dockerConnection = undefined as PierDockerConnection | undefined\nvoid adapter\nvoid captureAdapter\nvoid receipt\nvoid staged\nvoid trial\nvoid controller\nvoid dockerConnection\nvoid executePreparedPierCandidate\nvoid FilePierCandidateTrialController\nvoid runBenchmarks\nvoid runStagedJudge\nvoid StagedJudgeError\n", + ) + await writeFile( + path.join(consumerDir, 'tsconfig.json'), + `${JSON.stringify( + { + compilerOptions: publicTsconfig.compilerOptions, + files: ['index.ts'], + }, + null, + 2, + )}\n`, + ) + await writeFile( + path.join(consumerDir, 'verify_pier_payload.py'), + `import py_compile +from pathlib import Path + +import pier_agents +from pier_agents import candidate_contract + +root = Path(pier_agents.__file__).parent +expected = { + "__init__.py", + "candidate_contract.py", + "process_boundary.py", + "tangle_candidate.py", + "workspace_boundary.py", +} +observed = {path.name for path in root.glob("*.py")} +assert observed == expected, (observed, expected) +assert candidate_contract.PreparedCandidateContract.__module__ == "pier_agents.candidate_contract" +for name in sorted(expected): + py_compile.compile(root / name, doraise=True) +`, + ) + + // Intentionally resolve the declared ranges like a brand-new registry consumer. + // The preceding frozen install + public typecheck cover the exact bench lockfile. + await run( + 'npm', + ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--package-lock=false'], + consumerDir, + ) + await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir) + await run('npm', ['exec', '--', 'tsx', 'index.ts'], consumerDir) + const installedPackage = path.join(consumerDir, 'node_modules', '@tangle-network', 'agent-bench') + const prepared = await run( + 'npm', + [ + 'exec', + '--', + 'tsx', + path.join(installedPackage, 'scripts', 'verify-pier-agent.mts'), + ], + consumerDir, + { ...process.env, PIER_PREPARE_ONLY: '1', PIER_PROOF_ARM: 'failure' }, + ) + const prepareProof = JSON.parse(prepared.stdout) + if ( + prepareProof.prepared !== true || + prepareProof.disposed !== true || + !/^sha256:[a-f0-9]{64}$/.test(prepareProof.executionPlanDigest) || + !/^sha256:[a-f0-9]{64}$/.test(prepareProof.graderDigest) + ) { + throw new Error(`packed consumer did not prepare a real candidate: ${prepared.stdout}`) + } + await run('python3', ['verify_pier_payload.py'], consumerDir, { + ...process.env, + PYTHONPATH: installedPackage, + }) + let runtimeManifest + try { + runtimeManifest = JSON.parse( + await readFile( + path.join(consumerDir, 'node_modules/@tangle-network/agent-runtime/package.json'), + 'utf8', + ), + ) + } catch (error) { + throw new Error('packed consumer did not install @tangle-network/agent-runtime', { + cause: error, + }) + } + console.log( + `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}; prepared ${prepareProof.executionPlanDigest}`, + ) +} finally { + await rm(scratch, { recursive: true, force: true }) +} diff --git a/bench/scripts/verify-pier-agent.mts b/bench/scripts/verify-pier-agent.mts new file mode 100644 index 00000000..cd5ae119 --- /dev/null +++ b/bench/scripts/verify-pier-agent.mts @@ -0,0 +1,679 @@ +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + chmodSync, + cpSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { canonicalJson, InMemoryTraceStore } from '@tangle-network/agent-eval' +import type { + AgentCandidateArtifactRef, + AgentCandidateBundle, + AgentCandidateWorkspaceSnapshotEvidence, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { + createAgentCandidateWorkspacePort, + disposePreparedAgentCandidateExecution, + FileAgentCandidateExecutionClaimStore, + prepareAgentCandidateExecution, + type AgentCandidateExecutionPorts, + type AgentCandidateOutputArtifactPort, + type AgentCandidateTaskExecution, + type ResolvedAgentCandidateContainer, + verifyAgentCandidateBundle, +} from '@tangle-network/agent-runtime' + +import { executePreparedPierCandidate } from '../src/pier-agent' +import { createPierResultGrader } from '../src/pier-result-grader' +import { FilePierCandidateTrialController } from '../src/pier-trial-controller' + +const pinnedPierCommit = 'e69a20e4e0ac073ec71fde0274bab3d9f40bac87' +const pinnedPierVersion = '0.3.0' +const modelRequest = 'openai/gpt-5.4' +const fixtureImage = 'ghcr.io/tangle-network/devcontainers/universal:latest' +const prepareOnly = process.env.PIER_PREPARE_ONLY === '1' +const proofArm = process.env.PIER_PROOF_ARM +if (proofArm !== 'failure' && proofArm !== 'success') { + throw new Error('PIER_PROOF_ARM must be explicitly set to failure or success') +} +const expectedReward = proofArm === 'success' ? 1 : 0 +const expectedPatchApplied = proofArm === 'success' ? 1 : 0 +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const pierRepo = path.resolve(process.env.PIER_REPO ?? path.join(benchDir, '..', '..', 'pier')) +const fixtureSource = path.join(benchDir, 'fixtures', 'pier-agent', 'no-model-task') +const scratch = mkdtempSync(path.join(tmpdir(), 'agent-bench-pier-runtime-')) +const taskDir = path.join(scratch, 'task') +const taskRoot = path.join(scratch, 'task-workspace') +const candidateRoot = path.join(scratch, 'candidate-workspace') +const profileRoot = path.join(scratch, 'profile-workspace') +const jobsDir = path.join(scratch, 'jobs') + +function output( + command: string, + args: string[], + cwd = benchDir, + env: NodeJS.ProcessEnv = process.env, +): string { + return execFileSync(command, args, { + cwd, + env, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10 * 60_000, + maxBuffer: 20 * 1024 * 1024, + }).trim() +} + +function sha256(bytes: Uint8Array): Sha256Digest { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + +function canonicalBytes(value: unknown): Buffer { + return Buffer.from(canonicalJson(value), 'utf8') +} + +function embedded(bytes: Uint8Array) { + return { + encoding: 'base64' as const, + content: Buffer.from(bytes).toString('base64'), + sha256: sha256(bytes), + byteLength: bytes.byteLength, + } +} + +function workspaceSnapshot( + root: string, + paths: readonly string[], +): AgentCandidateWorkspaceSnapshotEvidence { + const files = paths + .map((relative) => { + const absolute = path.join(root, relative) + const bytes = readFileSync(absolute) + const mode = lstatSync(absolute).mode & 0o777 + if (mode !== 0o644 && mode !== 0o755) { + throw new Error(`unsupported fixture mode ${mode.toString(8)}: ${relative}`) + } + return { + path: relative, + mode: mode as 0o644 | 0o755, + sha256: sha256(bytes), + byteLength: bytes.byteLength, + } + }) + .sort((left, right) => left.path.localeCompare(right.path)) + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files, + } + const manifest = canonicalBytes(material) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: sha256(manifest), + material, + manifest: embedded(manifest), + archive: embedded(Buffer.from(`pre-materialized:${sha256(manifest)}`, 'utf8')), + } +} + +function deterministicRepository(root: string, seed: string): { commit: string; tree: string } { + mkdirSync(path.join(root, 'src'), { recursive: true }) + cpSync(seed, path.join(root, 'src', 'status.txt')) + chmodSync(path.join(root, 'src', 'status.txt'), 0o644) + output('git', ['init', '-b', 'main', root]) + output('git', ['-C', root, 'config', 'user.email', 'fixture@tangle.tools']) + output('git', ['-C', root, 'config', 'user.name', 'Tangle Fixture']) + output('git', [ + '-C', + root, + 'remote', + 'add', + 'origin', + 'git@github.com:tangle-network/agent-bench-pier-fixture.git', + ]) + output('git', ['-c', 'core.hooksPath=/dev/null', '-C', root, 'add', '-A']) + output( + 'git', + ['-c', 'core.hooksPath=/dev/null', '-C', root, 'commit', '-m', 'baseline'], + benchDir, + { + ...process.env, + GIT_AUTHOR_DATE: '2000-01-01T00:00:00Z', + GIT_COMMITTER_DATE: '2000-01-01T00:00:00Z', + }, + ) + return { + commit: output('git', ['-C', root, 'rev-parse', 'HEAD']), + tree: output('git', ['-C', root, 'rev-parse', 'HEAD^{tree}']), + } +} + +function publicOciIdentity( + image: string, +): { indexDigest: Sha256Digest; manifestDigest: Sha256Digest } { + const material = JSON.parse( + output('docker', ['buildx', 'imagetools', 'inspect', image, '--format', '{{json .Manifest}}']), + ) as { + digest?: string + manifests?: Array<{ + digest?: string + platform?: { os?: string; architecture?: string } + }> + } + const topDigest = material.digest + if (!topDigest?.match(/^sha256:[a-f0-9]{64}$/)) { + throw new Error(`public image omitted a valid index digest: ${topDigest}`) + } + const selected = material.manifests?.find( + (entry) => entry.platform?.os === 'linux' && entry.platform.architecture === 'amd64', + ) + const manifestDigest = selected?.digest + if (!manifestDigest) throw new Error('public image has no linux/amd64 manifest') + if (!manifestDigest.match(/^sha256:[a-f0-9]{64}$/)) { + throw new Error(`public image returned an invalid platform manifest: ${manifestDigest}`) + } + return { + indexDigest: topDigest as Sha256Digest, + manifestDigest: manifestDigest as Sha256Digest, + } +} + +function findTrialResult(root: string): { path: string; value: Record } | undefined { + for (const entry of readdirSync(root, { withFileTypes: true })) { + if (!entry.isDirectory()) continue + const child = path.join(root, entry.name) + const candidate = path.join(child, 'result.json') + try { + const value = JSON.parse(readFileSync(candidate, 'utf8')) as Record + if (typeof value.trial_name === 'string') return { path: candidate, value } + } catch {} + const nested = findTrialResult(child) + if (nested) return nested + } + return undefined +} + +function assertTreeOmits(root: string, forbidden: string): void { + for (const entry of readdirSync(root, { withFileTypes: true })) { + const absolute = path.join(root, entry.name) + if (entry.isDirectory()) { + assertTreeOmits(absolute, forbidden) + } else if (entry.isFile() && readFileSync(absolute).includes(Buffer.from(forbidden))) { + throw new Error(`protected value persisted in ${path.relative(root, absolute)}`) + } + } +} + +function outputArtifactStore(graderBytes: Uint8Array): { + outputArtifacts: AgentCandidateOutputArtifactPort + graderArtifact: AgentCandidateArtifactRef +} { + const stored = new Map() + const put = (bytes: Uint8Array, purpose: string): AgentCandidateArtifactRef => { + const detached = Uint8Array.from(bytes) + const digest = sha256(detached) + const artifact: AgentCandidateArtifactRef = { + locator: { + kind: 's3', + bucket: 'agent-bench-pier-proof', + key: `${purpose}/${digest.slice('sha256:'.length)}`, + }, + sha256: digest, + byteLength: detached.byteLength, + } + stored.set(digest, detached) + return artifact + } + const graderArtifact = put(graderBytes, 'graders') + return { + graderArtifact, + outputArtifacts: { + put: async ({ bytes, purpose, signal }) => { + signal?.throwIfAborted() + return put(bytes, purpose) + }, + read: async (artifact) => { + const bytes = stored.get(artifact.sha256) + if (!bytes) throw new Error(`missing Pier proof artifact ${artifact.sha256}`) + return Uint8Array.from(bytes) + }, + }, + } +} + +try { + if (!prepareOnly) { + const pierHead = output('git', ['rev-parse', 'HEAD'], pierRepo) + if (pierHead !== pinnedPierCommit) { + throw new Error(`Pier checkout mismatch: expected ${pinnedPierCommit}, got ${pierHead}`) + } + const pierStatus = output('git', ['status', '--porcelain'], pierRepo) + if (pierStatus !== '') throw new Error(`Pier checkout must be clean: ${pierStatus}`) + const pierVersion = output('uv', ['run', 'pier', '--version'], pierRepo) + if (pierVersion !== pinnedPierVersion) { + throw new Error(`Pier version mismatch: expected ${pinnedPierVersion}, got ${pierVersion}`) + } + } + + cpSync(fixtureSource, taskDir, { recursive: true }) + for (const relative of ['environment/seed/src/status.txt', 'tests/seed/src/status.txt']) { + chmodSync(path.join(taskDir, relative), 0o644) + } + for (const relative of ['pre_artifacts.sh', 'tests/test.sh']) { + chmodSync(path.join(taskDir, relative), 0o755) + } + + const contextDigest = sha256( + Buffer.concat([ + readFileSync(path.join(taskDir, 'environment', 'Dockerfile')), + readFileSync(path.join(taskDir, 'environment', 'seed', 'src', 'status.txt')), + ]), + ).slice(7, 23) + const identity = prepareOnly + ? { + indexDigest: `sha256:${'1'.repeat(64)}` as Sha256Digest, + manifestDigest: `sha256:${'2'.repeat(64)}` as Sha256Digest, + } + : publicOciIdentity(fixtureImage) + const pinnedImage = `${fixtureImage}@${identity.indexDigest}` + if (!prepareOnly) output('docker', ['pull', '--platform', 'linux/amd64', pinnedImage]) + const platform = prepareOnly + ? 'linux/amd64' + : output('docker', [ + 'image', + 'inspect', + '--format', + '{{.Os}}/{{.Architecture}}', + pinnedImage, + ]) + if (platform !== 'linux/amd64') throw new Error(`fixture image platform drifted: ${platform}`) + + const configPath = path.join(taskDir, 'task.toml') + const config = readFileSync(configPath, 'utf8') + writeFileSync( + configPath, + config.replace( + '[environment]\n', + `[environment]\ndocker_image = "${pinnedImage}"\nos = "linux"\n`, + ), + ) + + mkdirSync(candidateRoot) + mkdirSync(profileRoot) + const instruction = readFileSync(path.join(taskDir, 'instruction.md'), 'utf8') + const runner = `import pathlib, sys +task = pathlib.Path.cwd() +profile = task / 'AGENTS.md' +assert profile.is_file() and profile.read_text().strip() +assert sys.argv[-1] == ${JSON.stringify(instruction)} +${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=tangle\\n')" : ''} +` + writeFileSync(path.join(candidateRoot, 'runner.py'), runner) + chmodSync(path.join(candidateRoot, 'runner.py'), 0o755) + const repositoryState = deterministicRepository( + taskRoot, + path.join(taskDir, 'environment', 'seed', 'src', 'status.txt'), + ) + + const taskWorkspace = workspaceSnapshot(taskRoot, ['src/status.txt']) + const candidateWorkspace = workspaceSnapshot(candidateRoot, ['runner.py']) + const bundleWithoutDigest = { + schemaVersion: 1 as const, + kind: 'agent-candidate-bundle' as const, + digestAlgorithm: 'rfc8785-sha256' as const, + profile: { + name: `pier-no-model-runtime-${proofArm}`, + prompt: { + instructions: [ + 'Edit only the task repository. Follow the exact user instruction and verify the result.', + ], + }, + model: { default: modelRequest, reasoningEffort: 'xhigh' as const }, + harness: 'codex' as const, + resources: { failOnError: true as const }, + }, + code: { + kind: 'no-op' as const, + reason: 'proposer-no-change' as const, + repository: { + kind: 'github' as const, + owner: 'tangle-network', + repo: 'agent-bench-pier-fixture', + }, + baseCommit: repositoryState.commit, + baseTree: repositoryState.tree, + }, + execution: { + harness: 'codex' as const, + harnessVersion: 'agent-bench-pier/2.0.0', + launch: { + kind: 'candidate-entrypoint' as const, + entrypoint: 'runner.py', + interpreter: 'python3' as const, + }, + instructionDelivery: { kind: 'argv-append' as const }, + cwd: { workspace: 'task' as const, path: '.' }, + environment: { kind: 'evaluator-task-container' as const }, + workspace: candidateWorkspace, + isolation: { + network: 'disabled' as const, + remoteIntegrations: 'disabled' as const, + candidateSecrets: 'disabled' as const, + }, + }, + memory: { mode: 'disabled' as const }, + lineage: { + source: 'optimizer' as const, + parentDigests: [`sha256:${'a'.repeat(64)}` as Sha256Digest], + runIds: [`pier-no-model-proposer-${proofArm}`], + benchmark: { + name: 'pier-runtime-proof', + version: '1', + splitDigest: `sha256:${'b'.repeat(64)}` as Sha256Digest, + }, + spend: { + proposal: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + evaluation: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + }, + }, + } + const bundle = { + ...bundleWithoutDigest, + digest: sha256(canonicalBytes(bundleWithoutDigest)), + } as AgentCandidateBundle + const container: ResolvedAgentCandidateContainer = { + source: 'evaluator-task-container', + image: fixtureImage, + indexDigest: identity.indexDigest, + manifestDigest: identity.manifestDigest, + platform: { os: 'linux', architecture: 'amd64' }, + } + const graderBytes = readFileSync(new URL('../src/pier-result-grader.mjs', import.meta.url)) + const { outputArtifacts, graderArtifact } = outputArtifactStore(graderBytes) + const grader = createPierResultGrader({ + name: 'pier-official-result', + version: '1.0.0', + artifact: graderArtifact, + }) + const executionId = `pier-no-model-${proofArm}-${contextDigest}` + const task: AgentCandidateTaskExecution = { + executionId, + benchmark: 'pier-runtime-proof', + benchmarkVersion: '1', + taskId: `agent-bench/pier-candidate-no-model-${proofArm}`, + splitDigest: `sha256:${'b'.repeat(64)}`, + instruction, + repository: { + identity: 'github.com/tangle-network/agent-bench-pier-fixture', + rootIdentity: 'tangle-network/agent-bench-pier-fixture', + baseCommit: repositoryState.commit, + baseTree: repositoryState.tree, + }, + attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, + model: { requested: modelRequest, reasoningEffort: 'xhigh' }, + grader: { + name: grader.name, + version: grader.version, + artifact: grader.artifact, + }, + executionRoots: { taskRoot: '/app', candidateRoot: '/opt/tangle-candidate' }, + stagingRoots: { taskRoot, candidateRoot, profileRoot }, + workspace: taskWorkspace, + evaluatorTaskContainer: container, + limits: { + timeoutMs: 60_000, + maxSteps: 8, + maxModelCalls: 0, + maxInputTokens: 0, + maxOutputTokens: 0, + maxCostUsd: 0, + }, + } + const workspaces = createAgentCandidateWorkspacePort() + const ports: AgentCandidateExecutionPorts = { + artifacts: { + read: async () => { + throw new Error('the proof uses only embedded artifacts') + }, + }, + repositories: { resolve: async () => taskRoot }, + workspaces: { + materialize: async ({ role, snapshot, archive, destination }) => { + if (destination === taskRoot || destination === candidateRoot) return + await workspaces.materialize({ + role, + snapshot, + archive, + destination, + }) + }, + }, + containers: { resolve: async () => container }, + models: { + resolve: async ({ requested, reasoningEffort }) => { + if (!reasoningEffort) throw new Error('resolved model requires reasoning effort') + return { + requested, + provider: 'openai', + model: 'gpt-5.4', + snapshot: 'gpt-5.4-no-model-proof', + reasoningEffort, + } + }, + reserveGrant: async ({ limits, preparationId, expiresAtMs }) => ({ + preparationId, + digest: `sha256:${'c'.repeat(64)}`, + expiresAtMs, + enforcedLimits: limits, + network: { mode: 'disabled' }, + }), + activateGrant: async () => ({ + env: { MODEL_GATEWAY_TOKEN: 'zero-model-proof' }, + }), + settleGrant: async ({ preparationId }) => ({ + preparationId, + grantDigest: `sha256:${'c'.repeat(64)}`, + closed: true, + calls: [], + }), + }, + memory: { + reset: async () => { + throw new Error('disabled memory must not be reset') + }, + activate: async () => { + throw new Error('disabled memory must not be activated') + }, + close: async () => { + throw new Error('disabled memory must not be closed') + }, + }, + } + + const verified = await verifyAgentCandidateBundle(bundle, ports) + const prepared = await prepareAgentCandidateExecution(verified, task, ports) + if (prepareOnly) { + const disposal = await disposePreparedAgentCandidateExecution(prepared) + if (disposal.disposed !== true) throw new Error('prepared candidate was not disposed') + process.stdout.write( + `${JSON.stringify({ + prepared: true, + disposed: true, + executionPlanDigest: prepared.executionPlan.value.digest, + graderDigest: task.grader.artifact.sha256, + })}\n`, + ) + } else { + const traceStore = new InMemoryTraceStore() + const claimStore = new FileAgentCandidateExecutionClaimStore({ + directory: path.join(scratch, 'claims'), + }) + let acceptedRewards: { reward: number; patch_applied: number } | undefined + let acceptedTrialPath: string | undefined + const jobName = `tangle-runtime-candidate-no-model-${proofArm}` + const controller = new FilePierCandidateTrialController({ + directory: path.join(scratch, 'trial-control'), + launch: (staged, { request }) => { + const evaluatorArgs = Object.keys(staged.evaluatorEnv).flatMap((name) => [ + '--agent-env', + `${name}=\${${name}}`, + ]) + return { + command: 'uv', + args: [ + 'run', + 'pier', + 'run', + '--path', + taskDir, + ...staged.agentArgs, + ...staged.attemptArgs, + ...evaluatorArgs, + '--env', + 'docker', + '--job-name', + jobName, + '--jobs-dir', + jobsDir, + '--n-concurrent', + '1', + '--agent-timeout-multiplier', + '2', + '--quiet', + ], + cwd: pierRepo, + env: { ...process.env, PYTHONPATH: benchDir, ...staged.evaluatorEnv }, + jobsDirectory: jobsDir, + jobName, + readResult: async () => { + const trialResult = findTrialResult(jobsDir) + if (!trialResult) throw new Error(`Pier emitted no trial result under ${jobsDir}`) + const result = trialResult.value + if (result.exception_info !== null) { + throw new Error( + `Pier trial captured an exception: ${JSON.stringify(result.exception_info)}`, + ) + } + const rewards = result.verifier_result?.rewards + if ( + rewards?.reward !== expectedReward || + rewards?.patch_applied !== expectedPatchApplied + ) { + throw new Error( + `Pier ${proofArm} control returned unexpected rewards: ${JSON.stringify(rewards)}`, + ) + } + const agentResult = result.agent_result + for (const name of ['n_input_tokens', 'n_cache_tokens', 'n_output_tokens', 'cost_usd']) { + if (agentResult?.[name] !== null) { + throw new Error( + `Pier must not author protected usage ${name}: ${agentResult?.[name]}`, + ) + } + } + const observedElapsedMs = agentResult?.metadata?.observedElapsedMs + if (!Number.isInteger(observedElapsedMs) || observedElapsedMs < 0) { + throw new Error(`Pier omitted protected elapsed time: ${observedElapsedMs}`) + } + const endedAt = Date.now() + await traceStore.appendRun({ + runId: request.trace.runId, + scenarioId: task.taskId, + startedAt: endedAt - observedElapsedMs, + endedAt, + status: 'completed', + tags: { ...request.trace.tags }, + }) + acceptedRewards = rewards + acceptedTrialPath = path.relative(jobsDir, trialResult.path) + return { + value: result, + resultBytes: readFileSync(trialResult.path), + taskPatch: readFileSync( + path.join(path.dirname(trialResult.path), 'artifacts', 'model.patch'), + ), + } + }, + } + }, + }) + const finalized = await executePreparedPierCandidate({ + prepared, + directory: path.join(scratch, 'sealed'), + pierVersion: pinnedPierVersion, + traceStore, + claimStore, + outputArtifacts, + grader, + controller, + }) + if (!finalized.succeeded) { + throw new Error(`runtime rejected the protected Pier capture: ${finalized.reason}`) + } + if (!acceptedRewards || !acceptedTrialPath) { + throw new Error('atomic Pier executor returned without accepted verifier evidence') + } + if ( + finalized.receipt.value.benchmarkResult.material.score !== expectedReward || + finalized.receipt.value.benchmarkResult.material.passed !== (expectedReward === 1) + ) { + throw new Error( + `runtime receipt rejected the official Pier result: ${JSON.stringify(finalized.receipt.value.benchmarkResult.material)}`, + ) + } + assertTreeOmits(scratch, 'zero-model-proof') + const usage = finalized.receipt.value.usage + if ( + usage.modelCalls !== 0 || + usage.inputTokens !== 0 || + usage.outputTokens !== 0 || + usage.costUsd !== 0 || + finalized.receipt.value.trace.modelCallCount !== 0 + ) { + throw new Error(`runtime minted nonzero usage for a zero-model run: ${JSON.stringify(usage)}`) + } + + console.log( + JSON.stringify( + { + arm: proofArm, + reward: acceptedRewards.reward, + patchApplied: acceptedRewards.patch_applied, + modelCalls: usage.modelCalls, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + costUsd: usage.costUsd, + pierContextUsage: null, + profileExcludedByVerifier: true, + container: { + image: fixtureImage, + indexDigest: identity.indexDigest, + manifestDigest: identity.manifestDigest, + platform, + }, + executionPlanDigest: prepared.executionPlan.value.digest, + materializationReceiptDigest: prepared.materializationReceipt.digest, + runReceiptDigest: finalized.receipt.digest, + trial: acceptedTrialPath, + }, + null, + 2, + ), + ) + } +} finally { + if (process.env.KEEP_PIER_FIXTURE !== '1') rmSync(scratch, { recursive: true, force: true }) + else console.error(`Pier runtime fixture retained at ${scratch}`) +} diff --git a/bench/scripts/verify-pier-pair.mts b/bench/scripts/verify-pier-pair.mts new file mode 100644 index 00000000..e5d60152 --- /dev/null +++ b/bench/scripts/verify-pier-pair.mts @@ -0,0 +1,74 @@ +import { execFileSync } from 'node:child_process' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +interface PierProofResult { + readonly arm: 'failure' | 'success' + readonly reward: number + readonly patchApplied: number + readonly modelCalls: number + readonly inputTokens: number + readonly outputTokens: number + readonly costUsd: number + readonly runReceiptDigest: string +} + +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const script = path.join(benchDir, 'scripts', 'verify-pier-agent.mts') +const recoveryScript = path.join(benchDir, 'scripts', 'verify-pier-recovery.mts') + +function runArm(arm: PierProofResult['arm']): PierProofResult { + const stdout = execFileSync(process.execPath, ['--import', 'tsx', script], { + cwd: benchDir, + env: { ...process.env, PIER_PROOF_ARM: arm }, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'inherit'], + timeout: 20 * 60_000, + maxBuffer: 20 * 1024 * 1024, + }) + return JSON.parse(stdout) as PierProofResult +} + +const failure = runArm('failure') +const recovery = JSON.parse( + execFileSync(process.execPath, ['--import', 'tsx', recoveryScript], { + cwd: benchDir, + env: { ...process.env }, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'inherit'], + timeout: 120_000, + }), +) as { freshProcess?: boolean; processExited?: boolean; containersRemoved?: boolean } +if ( + recovery.freshProcess !== true || + recovery.processExited !== true || + recovery.containersRemoved !== true +) { + throw new Error(`Pier fresh-process recovery failed: ${JSON.stringify(recovery)}`) +} +const success = runArm('success') +if ( + failure.arm !== 'failure' || + failure.reward !== 0 || + failure.patchApplied !== 0 || + success.arm !== 'success' || + success.reward !== 1 || + success.patchApplied !== 1 +) { + throw new Error(`Pier controls did not separate: ${JSON.stringify({ failure, success })}`) +} +for (const result of [failure, success]) { + if ( + result.modelCalls !== 0 || + result.inputTokens !== 0 || + result.outputTokens !== 0 || + result.costUsd !== 0 + ) { + throw new Error(`Pier ${result.arm} control spent model budget: ${JSON.stringify(result)}`) + } + if (!/^sha256:[a-f0-9]{64}$/.test(result.runReceiptDigest)) { + throw new Error(`Pier ${result.arm} control omitted its run receipt digest`) + } +} + +console.log(JSON.stringify({ recovery, controls: [failure, success] }, null, 2)) diff --git a/bench/scripts/verify-pier-recovery.mts b/bench/scripts/verify-pier-recovery.mts new file mode 100644 index 00000000..0aa25645 --- /dev/null +++ b/bench/scripts/verify-pier-recovery.mts @@ -0,0 +1,139 @@ +import { execFileSync } from 'node:child_process' +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime' +import { InMemoryTraceStore } from '@tangle-network/agent-eval' + +import { createStagedPierCandidateExecutionFixture } from '../src/pier-agent.test-fixtures.mts' +import { FilePierCandidateTrialController } from '../src/pier-trial-controller' + +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const root = mkdtempSync(path.join(tmpdir(), 'pier-fresh-recovery-')) +const controlRoot = path.join(root, 'control') +const jobsDirectory = path.join(root, 'jobs') +const jobName = 'fresh-recovery' +const trialName = 'fresh-recovery-trial' +const executionId = 'pier-fresh-process-recovery' +const executionPlanDigest = `sha256:${'e'.repeat(64)}` as const +const image = 'ghcr.io/tangle-network/devcontainers/universal:latest' +let containerId: string | undefined +let handle: ReturnType | undefined +let resultSettled: Promise | undefined + +try { + containerId = execFileSync( + 'docker', + [ + 'run', + '-d', + '--label', + `com.docker.compose.project=${trialName}`, + image, + 'sleep', + 'infinity', + ], + { encoding: 'utf8', timeout: 120_000 }, + ).trim() + if (!/^[a-f0-9]{12,64}$/.test(containerId)) { + throw new Error(`Docker returned an invalid recovery container id: ${containerId}`) + } + + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + launch: () => ({ + command: process.execPath, + args: [ + '-e', + `require('node:fs').mkdirSync(${JSON.stringify(path.join(jobsDirectory, jobName, trialName))}, { recursive: true }); setInterval(() => undefined, 1_000)`, + ], + cwd: root, + env: { ...process.env }, + jobsDirectory, + jobName, + readResult: async () => { + throw new Error('recovery probe must not complete normally') + }, + }), + }) + handle = controller.start( + createStagedPierCandidateExecutionFixture(executionId), + { + request: { + executionId, + executionPlan: { value: { digest: executionPlanDigest } }, + } as AgentCandidateExecutorRequest, + traceStore: new InMemoryTraceStore(), + signal: new AbortController().signal, + deadlineAtMs: Date.now() + 30_000, + }, + ) + resultSettled = handle.result.catch(() => undefined) + + const [slot] = readdirSync(controlRoot) + if (!slot) throw new Error('Pier controller omitted durable state') + const identityPath = path.join(controlRoot, slot, 'identity.json') + let pierPid: number | undefined + for (let attempt = 0; attempt < 200; attempt++) { + const identity = JSON.parse(readFileSync(identityPath, 'utf8')) + if (identity.state === 'running' && Number.isSafeInteger(identity.pier?.pid)) { + pierPid = identity.pier.pid + break + } + await new Promise((resolveWait) => setTimeout(resolveWait, 10)) + } + if (!pierPid) throw new Error('Pier controller never persisted the child process identity') + const trialDirectory = path.join(jobsDirectory, jobName, trialName) + for (let attempt = 0; attempt < 200 && !existsSync(trialDirectory); attempt++) { + await new Promise((resolveWait) => setTimeout(resolveWait, 10)) + } + if (!existsSync(trialDirectory)) { + throw new Error('recovery probe never created its evaluator-owned trial directory') + } + + const recovery = JSON.parse( + execFileSync( + process.execPath, + [ + '--import', + 'tsx', + path.join(benchDir, 'scripts', 'terminate-pier-trial.mts'), + controlRoot, + executionId, + executionPlanDigest, + ], + { cwd: benchDir, encoding: 'utf8', timeout: 20_000 }, + ), + ) + if (recovery.processExited !== true || recovery.containersRemoved !== true) { + throw new Error(`fresh recovery returned an incomplete acknowledgement: ${JSON.stringify(recovery)}`) + } + try { + process.kill(pierPid, 0) + throw new Error(`recovered Pier process ${pierPid} is still alive`) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error + } + let containerPresent = true + try { + execFileSync('docker', ['inspect', containerId], { stdio: 'ignore', timeout: 30_000 }) + } catch { + containerPresent = false + } + if (containerPresent) throw new Error(`recovered Pier container ${containerId} is still present`) + await resultSettled + process.stdout.write( + `${JSON.stringify({ freshProcess: true, processExited: true, containersRemoved: true })}\n`, + ) +} finally { + if (handle) await handle.terminateAndWait().catch(() => undefined) + if (resultSettled) await resultSettled + if (containerId) { + try { + execFileSync('docker', ['rm', '-f', containerId], { stdio: 'ignore', timeout: 30_000 }) + } catch {} + } + rmSync(root, { recursive: true, force: true }) +} diff --git a/bench/src/benchmarks/_harness.test.mts b/bench/src/benchmarks/_harness.test.mts new file mode 100644 index 00000000..a0101dc5 --- /dev/null +++ b/bench/src/benchmarks/_harness.test.mts @@ -0,0 +1,178 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import test from 'node:test' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { resolveBenchPython, runStagedJudge } from './_harness' + +const digest = (bytes: Uint8Array): `sha256:${string}` => + `sha256:${createHash('sha256').update(bytes).digest('hex')}` + +test('resolveBenchPython defaults to the bench-local virtual environment', () => { + assert.equal(resolveBenchPython({}, '/opt/agent-bench'), join('/opt/agent-bench', '.venv', 'bin', 'python')) +}) + +test('resolveBenchPython accepts an absolute consumer-managed interpreter', () => { + assert.equal( + resolveBenchPython({ AGENT_BENCH_PYTHON: '/srv/bench-venv/bin/python' }, '/opt/agent-bench'), + '/srv/bench-venv/bin/python', + ) +}) + +test('resolveBenchPython rejects relative and empty interpreter paths', () => { + assert.throws( + () => resolveBenchPython({ AGENT_BENCH_PYTHON: '.venv/bin/python' }, '/opt/agent-bench'), + /must be an absolute path/, + ) + assert.throws( + () => resolveBenchPython({ AGENT_BENCH_PYTHON: '' }, '/opt/agent-bench'), + /must be an absolute path/, + ) +}) + +test('runStagedJudge terminates a hung evaluator at its explicit timeout', async () => { + const started = Date.now() + await assert.rejects( + () => runStagedJudge({ + tmpPrefix: 'timeout-test-', + async stage() {}, + bin: process.execPath, + argv: () => ['-e', 'setInterval(() => {}, 1_000)'], + timeoutMs: 100, + async parseReport() { + throw new Error('timed-out evaluator must not reach report parsing') + }, + }), + /evaluator failed/, + ) + assert.ok(Date.now() - started < 5_000) +}) + +test('runStagedJudge durably captures exact evaluator and process bytes with hashes', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'staged-judge-capture-test-')) + const destination = join(scratch, 'attempt-001') + const perTestBytes = Buffer.from([0x00, 0xff, 0x80, 0x0a, 0x41]) + const stdoutBytes = Buffer.from([0x53, 0x00, 0xff, 0x0a]) + const stderrBytes = Buffer.from([0x45, 0x80, 0x0a]) + try { + const score = await runStagedJudge({ + tmpPrefix: 'capture-test-', + async stage(dir) { + await writeFile(join(dir, 'preds.json'), Buffer.from([0x7b, 0x7d])) + }, + bin: process.execPath, + argv: () => [ + '-e', + [ + "const fs = require('node:fs')", + "fs.mkdirSync('logs/run_evaluation/run/model/task', { recursive: true })", + `fs.writeFileSync('logs/run_evaluation/run/model/task/test_output.txt', Buffer.from(${JSON.stringify([...perTestBytes])}))`, + "fs.writeFileSync('official.report.json', Buffer.from('{\"resolved\":true}'))", + `process.stdout.write(Buffer.from(${JSON.stringify([...stdoutBytes])}))`, + `process.stderr.write(Buffer.from(${JSON.stringify([...stderrBytes])}))`, + ].join(';'), + ], + capture: { destination }, + async parseReport(dir) { + assert.deepEqual( + await readFile(join(dir, 'logs/run_evaluation/run/model/task/test_output.txt')), + perTestBytes, + ) + return { resolved: true, score: 1 } + }, + }) + + const receipt = score.judgeArtifacts + assert.ok(receipt) + assert.equal(receipt.schema, 'agent-bench/judge-artifacts/v1') + assert.equal(receipt.evaluatorSucceeded, true) + assert.equal(receipt.directory, destination) + assert.deepEqual( + await readFile(join(destination, 'evaluator/logs/run_evaluation/run/model/task/test_output.txt')), + perTestBytes, + ) + assert.deepEqual(await readFile(join(destination, 'process/stdout.bin')), stdoutBytes) + assert.deepEqual(await readFile(join(destination, 'process/stderr.bin')), stderrBytes) + assert.deepEqual( + JSON.parse(await readFile(receipt.manifestPath, 'utf8')), + receipt, + ) + + const expected = new Map([ + ['evaluator/logs/run_evaluation/run/model/task/test_output.txt', perTestBytes], + ['evaluator/official.report.json', Buffer.from('{"resolved":true}')], + ['evaluator/preds.json', Buffer.from('{}')], + ['process/stderr.bin', stderrBytes], + ['process/stdout.bin', stdoutBytes], + ]) + assert.deepEqual(receipt.files.map((file) => file.path), [...expected.keys()]) + for (const file of receipt.files) { + const bytes = expected.get(file.path) + assert.ok(bytes) + assert.equal(file.kind, 'file') + assert.equal(file.byteLength, bytes.byteLength) + assert.equal(file.sha256, digest(bytes)) + } + const treeBytes = Buffer.from( + receipt.files + .map((file) => `${file.path}\0${file.kind}\0${file.byteLength}\0${file.sha256}\n`) + .join(''), + ) + assert.equal(receipt.treeSha256, digest(treeBytes)) + assert.equal(receipt.fileCount, expected.size) + assert.equal( + receipt.byteLength, + [...expected.values()].reduce((total, bytes) => total + bytes.byteLength, 0), + ) + } finally { + await rm(scratch, { recursive: true, force: true }) + } +}) + +test('runStagedJudge captures partial evaluator evidence before throwing', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'staged-judge-failure-capture-test-')) + const destination = join(scratch, 'attempt-failed') + try { + let caught: unknown + try { + await runStagedJudge({ + tmpPrefix: 'capture-failure-test-', + async stage() {}, + bin: process.execPath, + argv: () => [ + '-e', + [ + "const fs = require('node:fs')", + "fs.mkdirSync('logs/run_evaluation/run/model/task', { recursive: true })", + "fs.writeFileSync('logs/run_evaluation/run/model/task/run_instance.log', Buffer.from([1, 2, 255]))", + 'process.stderr.write(Buffer.from([69, 82, 82, 0, 255]))', + 'process.exit(9)', + ].join(';'), + ], + capture: { destination }, + async parseReport() { + throw new Error('failed evaluator must not reach report parsing') + }, + }) + } catch (error) { + caught = error + } + assert.ok(caught instanceof Error) + assert.match(caught.message, /evaluator failed/) + const receipt = (caught as Error & { judgeArtifacts?: { evaluatorSucceeded: boolean } }) + .judgeArtifacts + assert.ok(receipt) + assert.equal(receipt.evaluatorSucceeded, false) + assert.deepEqual( + await readFile(join(destination, 'evaluator/logs/run_evaluation/run/model/task/run_instance.log')), + Buffer.from([1, 2, 255]), + ) + assert.deepEqual( + await readFile(join(destination, 'process/stderr.bin')), + Buffer.from([69, 82, 82, 0, 255]), + ) + } finally { + await rm(scratch, { recursive: true, force: true }) + } +}) diff --git a/bench/src/benchmarks/_harness.ts b/bench/src/benchmarks/_harness.ts index 8b1a8695..7f82c53c 100644 --- a/bench/src/benchmarks/_harness.ts +++ b/bench/src/benchmarks/_harness.ts @@ -17,19 +17,49 @@ */ import { execFile, spawn } from 'node:child_process' -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { createHash } from 'node:crypto' +import { + cp, + lstat, + mkdir, + mkdtemp, + readFile, + readlink, + readdir, + rename, + rm, + writeFile, +} from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' -import type { BenchScore } from './types' +import type { + BenchScore, + JudgeArtifactFileReceipt, + JudgeArtifactReceipt, +} from './types' const execFileAsync = promisify(execFile) /** Repo root for the bench package (…/bench), so `.venv` and `fixtures` resolve. */ export const benchRoot = fileURLToPath(new URL('../..', import.meta.url)) -/** The bench venv interpreter every python-backed evaluator runs through. */ -export const venvPython = join(benchRoot, '.venv', 'bin', 'python') + +/** Resolve the shared interpreter without requiring an installed package to contain a venv. */ +export function resolveBenchPython( + env: Readonly<{ AGENT_BENCH_PYTHON?: string }> = process.env, + root: string = benchRoot, +): string { + const configured = env.AGENT_BENCH_PYTHON + if (configured === undefined) return join(root, '.venv', 'bin', 'python') + if (!isAbsolute(configured)) { + throw new Error('AGENT_BENCH_PYTHON must be an absolute path') + } + return configured +} + +/** The shared interpreter every Python-backed evaluator runs through. */ +export const venvPython = resolveBenchPython() /** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip * deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy) @@ -149,10 +179,160 @@ export interface StagedRunSpec { * if the expected report is absent/malformed (fail loud — no default score). */ parseReport(dir: string): Promise + /** + * Copy the complete evaluator directory plus raw process stdout/stderr to this + * caller-owned directory before cleanup. The destination must not exist. + */ + capture?: StagedRunCaptureSpec /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */ keepTmp?: boolean } +export interface StagedRunCaptureSpec { + /** Destination for `evaluator/`, `process/`, and the hashed `receipt.json`. */ + destination: string +} + +/** A staged run failed after any requested evidence was durably retained. */ +export class StagedJudgeError extends Error { + readonly judgeArtifacts?: JudgeArtifactReceipt + + constructor(message: string, judgeArtifacts?: JudgeArtifactReceipt, options?: ErrorOptions) { + super(message, options) + this.name = 'StagedJudgeError' + this.judgeArtifacts = judgeArtifacts + } +} + +function sha256(bytes: Uint8Array): `sha256:${string}` { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + +function portablePath(path: string): string { + return path.split(sep).join('/') +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0 +} + +async function collectArtifactFiles( + root: string, + current: string, +): Promise { + const absolute = join(root, current) + const entries = await readdir(absolute, { withFileTypes: true }) + const files: JudgeArtifactFileReceipt[] = [] + for (const entry of entries.sort((left, right) => compareText(left.name, right.name))) { + const relativePath = join(current, entry.name) + const path = join(root, relativePath) + if (entry.isDirectory()) { + files.push(...await collectArtifactFiles(root, relativePath)) + continue + } + if (entry.isFile()) { + const bytes = await readFile(path) + files.push({ + path: portablePath(relativePath), + byteLength: bytes.byteLength, + sha256: sha256(bytes), + kind: 'file', + }) + continue + } + if (entry.isSymbolicLink()) { + const targetBytes = await readlink(path, { encoding: 'buffer' }) + files.push({ + path: portablePath(relativePath), + byteLength: targetBytes.byteLength, + sha256: sha256(targetBytes), + kind: 'symlink', + }) + continue + } + throw new Error(`staged judge capture does not support ${relativePath}`) + } + return files +} + +function isWithin(parent: string, candidate: string): boolean { + const path = relative(parent, candidate) + return path === '' || (!path.startsWith(`..${sep}`) && path !== '..' && !isAbsolute(path)) +} + +async function assertDestinationAbsent(destination: string): Promise { + try { + await lstat(destination) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return + throw error + } + throw new Error(`staged judge capture destination already exists: ${destination}`) +} + +async function captureStagedRun( + sourceDirectory: string, + spec: StagedRunCaptureSpec, + processOutput: Readonly<{ stdout: Buffer; stderr: Buffer }>, + evaluatorSucceeded: boolean, +): Promise { + const source = resolve(sourceDirectory) + const destination = resolve(spec.destination) + if (isWithin(source, destination)) { + throw new Error('staged judge capture destination must be outside the evaluator directory') + } + await mkdir(dirname(destination), { recursive: true }) + await assertDestinationAbsent(destination) + const staging = await mkdtemp(join(dirname(destination), `.${basename(destination)}-capture-`)) + try { + await cp(source, join(staging, 'evaluator'), { + recursive: true, + errorOnExist: true, + force: false, + preserveTimestamps: true, + verbatimSymlinks: true, + }) + await mkdir(join(staging, 'process')) + await writeFile(join(staging, 'process', 'stdout.bin'), processOutput.stdout) + await writeFile(join(staging, 'process', 'stderr.bin'), processOutput.stderr) + + const files = [ + ...await collectArtifactFiles(staging, 'evaluator'), + ...await collectArtifactFiles(staging, 'process'), + ].sort((left, right) => compareText(left.path, right.path)) + const byteLength = files.reduce((total, file) => total + file.byteLength, 0) + const treeBytes = Buffer.from( + files + .map((file) => `${file.path}\0${file.kind}\0${file.byteLength}\0${file.sha256}\n`) + .join(''), + 'utf8', + ) + const receipt: JudgeArtifactReceipt = { + schema: 'agent-bench/judge-artifacts/v1', + directory: destination, + evaluatorDirectory: join(destination, 'evaluator'), + manifestPath: join(destination, 'receipt.json'), + evaluatorSucceeded, + files, + fileCount: files.length, + byteLength, + treeSha256: sha256(treeBytes), + } + await writeFile(join(staging, 'receipt.json'), `${JSON.stringify(receipt, null, 2)}\n`) + await rename(staging, destination) + return receipt + } catch (error) { + await rm(staging, { recursive: true, force: true }).catch(() => {}) + throw error + } +} + +function processBytes(value: unknown): Buffer { + if (Buffer.isBuffer(value)) return value + if (value === undefined || value === null) return Buffer.alloc(0) + return Buffer.from(String(value), 'utf8') +} + /** * The shared judge body: mkdtemp → stage → spawn evaluator → parseReport → * cleanup. The evaluator's stdout/stderr is surfaced on failure; the temp dir is @@ -160,23 +340,66 @@ export interface StagedRunSpec { */ export async function runStagedJudge(spec: StagedRunSpec): Promise { const dir = await mkdtemp(join(tmpdir(), spec.tmpPrefix)) + let stdout: Buffer = Buffer.alloc(0) + let stderr: Buffer = Buffer.alloc(0) + let evaluatorSucceeded = false + let score: BenchScore | undefined + let failure: unknown try { - await spec.stage(dir) - const bin = spec.bin ?? venvPython try { - await execFileAsync(bin, spec.argv(dir), { - cwd: spec.cwd ? spec.cwd(dir) : dir, - maxBuffer: bigBuffer, - ...(spec.timeoutMs ? { timeout: spec.timeoutMs } : {}), - }) + await spec.stage(dir) + const bin = spec.bin ?? venvPython + const argv = spec.argv(dir) + try { + const output = await execFileAsync(bin, argv, { + cwd: spec.cwd ? spec.cwd(dir) : dir, + encoding: 'buffer', + maxBuffer: bigBuffer, + ...(spec.timeoutMs ? { timeout: spec.timeoutMs } : {}), + }) + stdout = processBytes(output.stdout) + stderr = processBytes(output.stderr) + evaluatorSucceeded = true + } catch (err) { + const e = err as { stderr?: unknown; stdout?: unknown; message?: string } + stdout = processBytes(e.stdout) + stderr = processBytes(e.stderr) + const detail = processBytes(e.stderr ?? e.stdout ?? e.message ?? String(err)) + .toString('utf8') + .slice(0, 2000) + throw new Error(`${spec.tmpPrefix.replace(/-$/, '')} evaluator failed (${bin} ${argv.join(' ')}):\n${detail}`) + } + score = await spec.parseReport(dir) } catch (err) { - const e = err as { stderr?: string; stdout?: string; message?: string } - const detail = (e.stderr || e.stdout || e.message || String(err)).slice(0, 2000) - throw new Error(`${spec.tmpPrefix.replace(/-$/, '')} evaluator failed (${bin} ${spec.argv(dir).join(' ')}):\n${detail}`) + failure = err } - return await spec.parseReport(dir) } finally { + let judgeArtifacts: JudgeArtifactReceipt | undefined + if (spec.capture) { + try { + judgeArtifacts = await captureStagedRun( + dir, + spec.capture, + { stdout, stderr }, + evaluatorSucceeded, + ) + } catch (captureError) { + failure = new Error( + `staged judge failed to capture evaluator artifacts: ${captureError instanceof Error ? captureError.message : captureError}`, + { cause: failure ?? captureError }, + ) + } + } if (!spec.keepTmp) await rm(dir, { recursive: true, force: true }).catch(() => {}) + if (failure) { + throw new StagedJudgeError( + failure instanceof Error ? failure.message : String(failure), + judgeArtifacts, + { cause: failure }, + ) + } + if (!score) throw new StagedJudgeError('staged judge completed without a score', judgeArtifacts) + return judgeArtifacts ? { ...score, judgeArtifacts } : score } } diff --git a/bench/src/benchmarks/humaneval.ts b/bench/src/benchmarks/humaneval.ts index 619d11f7..7a03bad7 100644 --- a/bench/src/benchmarks/humaneval.ts +++ b/bench/src/benchmarks/humaneval.ts @@ -17,7 +17,7 @@ */ import { execFile } from 'node:child_process' -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { gunzipSync } from 'node:zlib' @@ -41,9 +41,17 @@ export interface HumanEvalTask { * selects a deeper slice (the later tasks are harder) so the worker has a * correctable middle band rather than a saturated easy prefix. */ export async function loadHumanEval(limit: number, offset = 0): Promise { - const res = await fetch(humanevalUrl) - if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`) - const gz = Buffer.from(await res.arrayBuffer()) + // Prefer a locally-cached .jsonl.gz (HUMANEVAL_GZ) — the GitHub raw URL rate-limits + // (429) under repeated runs. Fall back to the network fetch when unset. + const localGz = process.env.HUMANEVAL_GZ + let gz: Buffer + if (localGz) { + gz = readFileSync(localGz) + } else { + const res = await fetch(humanevalUrl) + if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`) + gz = Buffer.from(await res.arrayBuffer()) + } const text = gunzipSync(gz).toString('utf8') const tasks: HumanEvalTask[] = [] for (const line of text.split('\n')) { diff --git a/bench/src/benchmarks/swe-bench.test.mts b/bench/src/benchmarks/swe-bench.test.mts new file mode 100644 index 00000000..703807cd --- /dev/null +++ b/bench/src/benchmarks/swe-bench.test.mts @@ -0,0 +1,58 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { createSweBenchAdapter, scoreSweReport, sweEvaluationArgv } from './swe-bench' + +const taskId = 'django__django-12345' + +test('scoreSweReport preserves official resolved, unresolved, and empty-patch outcomes', () => { + assert.equal( + scoreSweReport(taskId, { submitted_ids: [taskId], completed_ids: [taskId], resolved_ids: [taskId] }).score, + 1, + ) + assert.equal( + scoreSweReport(taskId, { submitted_ids: [taskId], completed_ids: [taskId], unresolved_ids: [taskId] }).score, + 0, + ) + assert.equal(scoreSweReport(taskId, { submitted_ids: [taskId], empty_patch_ids: [taskId] }).score, 0) +}) + +test('scoreSweReport rejects evaluator failures instead of scoring them as agent failures', () => { + assert.throws(() => scoreSweReport(taskId, { submitted_ids: [taskId], error_ids: [taskId] }), /evaluator failed/) + assert.throws(() => scoreSweReport(taskId, { submitted_ids: [taskId], incomplete_ids: [taskId] }), /evaluator failed/) +}) + +test('scoreSweReport rejects missing, ambiguous, malformed, and mismatched outcomes', () => { + assert.throws(() => scoreSweReport(taskId, { submitted_ids: [taskId] }), /no unique outcome/) + assert.throws( + () => scoreSweReport(taskId, { resolved_ids: [taskId], unresolved_ids: [taskId] }), + /no unique outcome/, + ) + assert.throws(() => scoreSweReport(taskId, { resolved_ids: taskId }), /malformed resolved_ids/) + assert.throws(() => scoreSweReport(taskId, { resolved_ids: ['other__repo-1'] }), /identity mismatch/) + assert.throws(() => scoreSweReport(taskId, { resolved_ids: [taskId] }), /lacks a completed evaluation/) + assert.throws( + () => scoreSweReport(taskId, { submitted_ids: [taskId, 'other__repo-1'], empty_patch_ids: [taskId] }), + /identity mismatch/, + ) +}) + +test('createSweBenchAdapter accepts only positive integer evaluation timeouts', () => { + assert.doesNotThrow(() => createSweBenchAdapter({ timeoutMs: 1_200_000 })) + assert.throws(() => createSweBenchAdapter({ timeoutMs: 0 }), /positive integer/) + assert.throws(() => createSweBenchAdapter({ timeoutMs: 1.5 }), /positive integer/) +}) + +test('SWE evaluation command preserves the requested instance image', () => { + const argv = sweEvaluationArgv({ + predictionsPath: '/tmp/preds.json', + runId: 'r364', + instanceId: taskId, + cacheLevel: 'instance', + }) + const cacheIndex = argv.indexOf('--cache_level') + assert.equal(argv[cacheIndex + 1], 'instance') + assert.throws( + () => createSweBenchAdapter({ cacheLevel: 'invalid' as 'instance' }), + /invalid cacheLevel/, + ) +}) diff --git a/bench/src/benchmarks/swe-bench.ts b/bench/src/benchmarks/swe-bench.ts index 75371416..60a77580 100644 --- a/bench/src/benchmarks/swe-bench.ts +++ b/bench/src/benchmarks/swe-bench.ts @@ -21,6 +21,7 @@ import { runVenvPython, safeRunId, stageFile, + type StagedRunCaptureSpec, } from './_harness' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' @@ -58,6 +59,28 @@ export const swePatchOutput: OutputAdapter = { } const DATASET = 'princeton-nlp/SWE-bench_Verified' +export type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance' + +export interface SweBenchArtifactCaptureContext { + readonly taskId: string + readonly runId: string + /** One-based sequence unique within this adapter instance. */ + readonly attemptSequence: number +} + +export interface SweBenchAdapterOptions { + readonly timeoutMs?: number + readonly cacheLevel?: SweBenchCacheLevel + /** + * Return a unique destination for any attempt whose complete official + * evaluator directory and process logs should be retained. + */ + readonly captureEvaluatorArtifacts?: ( + context: SweBenchArtifactCaptureContext, + ) => StagedRunCaptureSpec | undefined +} + +const SWE_CACHE_LEVELS = new Set(['none', 'base', 'env', 'instance']) const TEST_FILE_EXCLUDES = [ "':(exclude,glob)**/tests/**'", "':(exclude,glob)**/test/**'", @@ -72,6 +95,78 @@ const TEST_FILE_EXCLUDES = [ interface SweReport { resolved_instances?: number resolved_ids?: string[] + unresolved_ids?: string[] + empty_patch_ids?: string[] + completed_ids?: string[] + incomplete_ids?: string[] + error_ids?: string[] + submitted_ids?: string[] +} + +function stringIds(report: Record, key: keyof SweReport): string[] { + const value = report[key] + if (value === undefined) return [] + if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) { + throw new Error(`swe-bench: malformed ${key}`) + } + return value +} + +/** Convert one official report into a score without turning evaluator failures into agent failures. */ +export function scoreSweReport(taskId: string, value: unknown): BenchScore { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('swe-bench: report must be an object') + } + const report = value as Record + const statusIds = { + resolved: stringIds(report, 'resolved_ids'), + unresolved: stringIds(report, 'unresolved_ids'), + emptyPatch: stringIds(report, 'empty_patch_ids'), + completed: stringIds(report, 'completed_ids'), + incomplete: stringIds(report, 'incomplete_ids'), + error: stringIds(report, 'error_ids'), + } + const submitted = stringIds(report, 'submitted_ids') + const mentioned = Object.values(statusIds).flat() + if ( + mentioned.some((id) => id !== taskId) + || (submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId)) + ) { + throw new Error(`swe-bench: report identity mismatch for ${taskId}`) + } + if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) { + throw new Error(`swe-bench: evaluator failed for ${taskId}`) + } + const outcomes = [ + statusIds.resolved.includes(taskId), + statusIds.unresolved.includes(taskId), + statusIds.emptyPatch.includes(taskId), + ] + if (outcomes.filter(Boolean).length !== 1) { + throw new Error(`swe-bench: report has no unique outcome for ${taskId}`) + } + if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) { + throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`) + } + const resolved = outcomes[0] + return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) } +} + +export function sweEvaluationArgv(args: { + readonly predictionsPath: string + readonly runId: string + readonly instanceId: string + readonly cacheLevel: SweBenchCacheLevel +}): string[] { + return [ + '-m', 'swebench.harness.run_evaluation', + '--dataset_name', DATASET, + '--predictions_path', args.predictionsPath, + '--run_id', args.runId, + '--instance_ids', args.instanceId, + '--max_workers', '1', + '--cache_level', args.cacheLevel, + ] } function shellQuote(value: string): string { @@ -90,7 +185,18 @@ function sweMetadata(task: BenchTask): { repo: string; base: string } { return { repo, base } } -export function createSweBenchAdapter(): BenchmarkAdapter { +export function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): BenchmarkAdapter { + if ( + options.timeoutMs !== undefined + && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0) + ) throw new Error('swe-bench: timeoutMs must be a positive integer') + const cacheLevel = options.cacheLevel ?? 'env' + if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error('swe-bench: invalid cacheLevel') + if ( + options.captureEvaluatorArtifacts !== undefined + && typeof options.captureEvaluatorArtifacts !== 'function' + ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function') + let attemptSequence = 0 return { name: 'swe-bench-verified', output: swePatchOutput, @@ -178,8 +284,15 @@ print(json.dumps(out)) async judge(task: BenchTask, artifact: string): Promise { const runId = safeRunId('bench', task.id) + const capture = options.captureEvaluatorArtifacts?.({ + taskId: task.id, + runId, + attemptSequence: ++attemptSequence, + }) return runStagedJudge({ tmpPrefix: 'swebench-', + ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), + ...(capture === undefined ? {} : { capture }), // Debug: retain the staged dir (holds swebench's per-instance apply/run // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default. ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}), @@ -193,20 +306,16 @@ print(json.dumps(out)) }, // The official evaluation harness. Pulls/builds the instance image, applies // the patch, runs the test spec, writes a per-run report JSON in cwd. - argv: (dir) => [ - '-m', 'swebench.harness.run_evaluation', - '--dataset_name', DATASET, - '--predictions_path', join(dir, 'preds.json'), - '--run_id', runId, - '--instance_ids', task.id, - '--max_workers', '1', - '--cache_level', 'env', - ], + argv: (dir) => sweEvaluationArgv({ + predictionsPath: join(dir, 'preds.json'), + runId, + instanceId: task.id, + cacheLevel, + }), async parseReport(dir) { // Report file: agent-runtime-bench..json const report = await readJsonReport(join(dir, `agent-runtime-bench.${runId}.json`)) - const resolved = (report.resolved_ids ?? []).includes(task.id) - return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) } + return scoreSweReport(task.id, report) }, }) }, diff --git a/bench/src/benchmarks/types.ts b/bench/src/benchmarks/types.ts index 2329c3e3..b02c16d0 100644 --- a/bench/src/benchmarks/types.ts +++ b/bench/src/benchmarks/types.ts @@ -20,12 +20,40 @@ export interface BenchTask { metadata?: Record } +export interface JudgeArtifactFileReceipt { + /** POSIX path relative to the capture directory. */ + path: string + /** Exact byte length of the retained file or symbolic-link target. */ + byteLength: number + /** SHA-256 over the retained file bytes or UTF-8 symbolic-link target. */ + sha256: `sha256:${string}` + kind: 'file' | 'symlink' +} + +/** Durable evidence written before a staged evaluator's temporary directory is removed. */ +export interface JudgeArtifactReceipt { + schema: 'agent-bench/judge-artifacts/v1' + /** Absolute directory containing `evaluator/`, `process/`, and `receipt.json`. */ + directory: string + /** Exact copy of the evaluator working directory. */ + evaluatorDirectory: string + manifestPath: string + evaluatorSucceeded: boolean + files: JudgeArtifactFileReceipt[] + fileCount: number + byteLength: number + /** SHA-256 over every sorted path, kind, byte length, and content hash. */ + treeSha256: `sha256:${string}` +} + export interface BenchScore { /** Did the deterministic judge pass (tests resolved / state correct)? */ resolved: boolean /** 0..1 — 1 = fully resolved; partial credit where the harness supports it. */ score: number detail?: string + /** Present only when the caller explicitly requested durable judge evidence. */ + judgeArtifacts?: JudgeArtifactReceipt } export interface LoadOptions { diff --git a/bench/src/hev-eval.mts b/bench/src/hev-eval.mts new file mode 100644 index 00000000..dfba923a --- /dev/null +++ b/bench/src/hev-eval.mts @@ -0,0 +1,69 @@ +/** + * Minimal HumanEval evaluator: given an INSTRUCTION (env) + a fixed task set, run the + * worker model on each task and print the pass rate + the per-task result. Used to + * measure a baseline instruction vs a proposer-supplied instruction on the SAME + * held-out set (the proposer proposes; this grades — kept separate for honesty). + * + * INSTRUCTION="..." IDS=HumanEval/55,... WORKER_MODEL=... ROUTER_BASE=... TANGLE_API_KEY=... \ + * HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz tsx src/hev-eval.mts + */ +import { readFileSync } from 'node:fs' +import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval' + +const SEED_INSTRUCTION = + 'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.' + +async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise { + const res = await fetch(`${base}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }), + }) + if (!res.ok) return '' + const d = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } + return d.choices?.[0]?.message?.content ?? '' +} + +async function main(): Promise { + const key = process.env.TANGLE_API_KEY + if (!key) throw new Error('TANGLE_API_KEY required') + const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' + const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' + const instruction = process.env.INSTRUCTION_FILE + ? readFileSync(process.env.INSTRUCTION_FILE, 'utf8') + : (process.env.INSTRUCTION ?? SEED_INSTRUCTION) + const maxTokens = Number(process.env.MAX_TOKENS ?? 2500) + const conc = Number(process.env.CONC ?? 6) + const offset = Number(process.env.OFFSET ?? 55) + const n = Number(process.env.N ?? 40) + const idsEnv = (process.env.IDS ?? '').split(',').map((s) => s.trim()).filter(Boolean) + + const all = await loadHumanEval(164, 0) + const byId = new Map(all.map((t) => [t.taskId, t])) + const tasks: HumanEvalTask[] = idsEnv.length + ? idsEnv.map((id) => byId.get(id)).filter((t): t is HumanEvalTask => !!t) + : all.slice(offset, offset + n) + + console.log(`eval model=${model} n=${tasks.length} instr_len=${instruction.length}`) + let pass = 0 + const fails: string[] = [] + // simple concurrency pool + let i = 0 + async function worker(): Promise { + while (i < tasks.length) { + const t = tasks[i++] + const reply = await complete(base, key, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens) + const { pass: p } = await runChecker(t, extractCode(reply)) + if (p === 1) pass += 1 + else fails.push(t.taskId) + } + } + await Promise.all(Array.from({ length: conc }, () => worker())) + console.log(`PASS ${pass}/${tasks.length} = ${((100 * pass) / tasks.length).toFixed(1)}%`) + console.log(`FAILED: ${fails.sort().join(', ')}`) +} + +main().catch((e) => { + console.error(e instanceof Error ? (e.stack ?? e.message) : String(e)) + process.exit(1) +}) diff --git a/bench/src/hev-improve.mts b/bench/src/hev-improve.mts new file mode 100644 index 00000000..fd86b8f3 --- /dev/null +++ b/bench/src/hev-improve.mts @@ -0,0 +1,168 @@ +/** + * SELF-IMPROVEMENT on HumanEval — the prompt-sensitive, VISIBLE-ORACLE counterpart + * to the SWE-bench run. Same machinery (improve(surface:'prompt') + gepaProposer + + * held-out gate), but the worker is a single chat completion and the judge is the + * deterministic Docker checker (run the function against its own hidden unit tests). + * + * WHY this exists: on SWE-bench the same GEPA loop was NULL because the grading test + * is withheld — the worker cannot verify, so prompt wording cannot move resolve. + * HumanEval hands the worker a well-specified function to complete and grades by + * running tests, so the instruction prompt DOES move pass-rate. This run measures + * whether self-improvement lifts a CHEAP model when the task is prompt-sensitive. + * + * Worker + reflect models call the zai coding endpoint directly (no tangle router, + * no WAF, no 503): TANGLE_API_KEY=$ZAI_API_KEY ROUTER_BASE=https://api.z.ai/api/coding/paas/v4 + */ +import { improve } from '@tangle-network/agent-runtime' +import type { AgentProfile } from '@tangle-network/agent-interface' +import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import { gepaProposer } from '@tangle-network/agent-eval/campaign' +import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval' + +// The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's +// solveInstruction so the baseline arm reproduces the plain-prompt denominator. +const SEED_INSTRUCTION = + 'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.' + +interface Completion { + text: string + usd: number + tokIn: number + tokOut: number +} + +async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise { + const res = await fetch(`${base}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }), + }) + if (!res.ok) throw new Error(`completion HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`) + const d = (await res.json()) as { + choices?: Array<{ message?: { content?: string } }> + usage?: { prompt_tokens?: number; completion_tokens?: number } + } + const text = d.choices?.[0]?.message?.content ?? '' + const tokIn = d.usage?.prompt_tokens ?? 0 + const tokOut = d.usage?.completion_tokens ?? 0 + // zai glm pricing is ~ $0.6/M in, $2.2/M out (coding plan); a rough cost tag so the + // stub-guard sees a real backend. Exact cost is not the metric (pass-rate is). + const usd = (tokIn * 0.6 + tokOut * 2.2) / 1_000_000 + return { text, usd, tokIn, tokOut } +} + +async function main(): Promise { + const key = process.env.TANGLE_API_KEY + if (!key) throw new Error('TANGLE_API_KEY required (worker + reflect completions)') + const base = process.env.ROUTER_BASE ?? 'https://api.z.ai/api/coding/paas/v4' + const workerModel = process.env.WORKER_MODEL ?? 'glm-4.5-air' + const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6' + // The GEPA reflector may live on a DIFFERENT endpoint than the (cheap) worker — + // e.g. a small worker on Together + a strong optimizer on zai. Defaults to the + // worker endpoint when unset. + const reflectBase = process.env.REFLECT_BASE ?? base + const reflectKey = process.env.REFLECT_KEY ?? key + const trainN = Number(process.env.TRAIN_N ?? 12) + const holdoutN = Number(process.env.HOLDOUT_N ?? 12) + const offset = Number(process.env.OFFSET ?? 80) + const generations = Number(process.env.GENERATIONS ?? 1) + const population = Number(process.env.POPULATION ?? 2) + const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000) + const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000) + const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4) + + // TRAIN and HOLDOUT are DISJOINT slices of the harder middle band (offset). + const train = await loadHumanEval(trainN, offset) + const holdout = await loadHumanEval(holdoutN, offset + trainN) + const byId = new Map([...train, ...holdout].map((t) => [t.taskId, t])) + const allIds = [...byId.keys()] + + console.log('═══ HumanEval self-improvement — VISIBLE oracle (deterministic Docker checker) ═══') + console.log(`worker=${workerModel} reflect=${reflectModel} base=${base}`) + console.log(`train=[${train.map((t) => t.taskId).join(', ')}]`) + console.log(`holdout=[${holdout.map((t) => t.taskId).join(', ')}]`) + console.log(`generations=${generations} population=${population} offset=${offset} maxTokens=${workerMaxTokens}`) + console.log(`≈ ${trainN * (1 + generations * population) + 2 * holdoutN} cells (each = 1 completion + 1 Docker check)\n`) + + const stats = { n: 0 } + const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise => { + const instr = String(surface) + const t = byId.get(scenario.id) + if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`) + const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\`` + const t0 = Date.now() + const r = await complete(base, key, workerModel, prompt, workerMaxTokens) + const zeroUsage = r.tokIn === 0 && r.tokOut === 0 + const hasText = r.text.trim().length > 0 + ctx.cost.observe(zeroUsage && hasText ? Math.max(r.usd, 0.0001) : r.usd, workerModel) + ctx.cost.observeTokens( + zeroUsage && hasText ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut }, + ) + stats.n += 1 + const codeLen = extractCode(r.text).length + console.log(` [agent] ${scenario.id} instr=${instr.length}c code=${codeLen}b tok=in:${r.tokIn}/out:${r.tokOut} ${Math.round((Date.now() - t0) / 1000)}s`) + return hasText ? r.text : null + } + + const judge: JudgeConfig = { + name: 'humaneval-docker', + dimensions: [{ key: 'pass', description: 'the completed function passes its hidden unit tests (deterministic Docker checker)' }], + async score({ artifact, scenario }) { + const t = byId.get(scenario.id) + if (!t) throw new Error(`judge: unknown scenario ${scenario.id}`) + const code = extractCode(String(artifact ?? '')) + if (!code.trim()) { + console.log(` [judge] ${scenario.id} pass=0 (empty)`) + return { dimensions: { pass: 0 }, composite: 0, notes: 'empty' } + } + const { pass } = await runChecker(t, code) + console.log(` [judge] ${scenario.id} pass=${pass}`) + return { dimensions: { pass }, composite: pass, notes: pass === 1 ? 'passed' : 'failed' } + }, + } + + const profile: AgentProfile = { name: 'hev-solver', prompt: { systemPrompt: SEED_INSTRUCTION } } + const proposer = gepaProposer({ + llm: { baseUrl: reflectBase, apiKey: reflectKey }, + model: reflectModel, + target: + 'the instruction/system prompt strategy for a SMALL model completing Python functions to pass hidden unit tests. ' + + 'Propose SUBSTANTIALLY different strategies, not wording tweaks: e.g. require the model to first reason step-by-step ' + + 'about the algorithm and edge cases (empty inputs, off-by-one, boundary values, types) in a brief plan or comments ' + + 'BEFORE writing the code; provide a short worked example; or add an explicit self-check against the docstring. ' + + 'Bold rewrites that change model BEHAVIOR beat cosmetic edits.', + maxTokens: reflectMaxTokens, + temperature: 0.7, + }) + + const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'humaneval' })) + const holdoutScenarios: Scenario[] = holdout.map((t) => ({ id: t.taskId, kind: 'humaneval' })) + + const out = await improve(profile, [], { + surface: 'prompt', + gate: 'holdout', + generator: proposer, + scenarios, + judge, + agent, + expectUsage: 'warn', + budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 }, + llm: { baseUrl: reflectBase, apiKey: reflectKey, model: reflectModel }, + }) + + console.log('\n═══ RESULT ═══') + console.log(`gateDecision=${out.gateDecision} shipped=${out.shipped} lift=${out.lift}`) + console.log(`baseline holdout pass-rate = ${out.raw.baseline.compositeMean}`) + console.log(`winner holdout pass-rate = ${out.raw.winner.compositeMean}`) + console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`) + console.log(`winner per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`) + if (out.raw.winner.label) console.log(`winner label : ${out.raw.winner.label}`) + if ((out.raw.winner as { surface?: unknown }).surface) { + console.log(`winner instruction:\n${String((out.raw.winner as { surface?: unknown }).surface).slice(0, 1200)}`) + } +} + +main().catch((e) => { + console.error(e instanceof Error ? (e.stack ?? e.message) : String(e)) + process.exit(1) +}) diff --git a/bench/src/hev-structural.mts b/bench/src/hev-structural.mts new file mode 100644 index 00000000..ed2d1585 --- /dev/null +++ b/bench/src/hev-structural.mts @@ -0,0 +1,688 @@ +/** + * HumanEval STRUCTURAL lever — best-of-k selection + self-repair grounded ONLY on the + * VISIBLE docstring `>>>` examples (the honest oracle), graded on the HIDDEN check() + * suite. This is the experiment the self-improvement push identified but never ran: + * the existing gates (`humaneval-gate.mts`, `humaneval-repair-gate.mts`) select/steer + * on the task's own grading test — defensible in a deployable-verifier framing, but + * NOT a benchmark-lift claim. Here the harness sees nothing the model can't already + * read in its prompt. + * + * Honesty by construction — two physically separated phases: + * Phase A (harness): k samples/task at one temperature → honest doctest score per + * sample (docker, --network=none) → argmax select → ≤R repair rounds steered by + * the doctest FAILURE OUTPUT → final artifact locked. No access to task.test. + * Phase B (grading): the hidden check() suite grades every sample and every locked + * final. Nothing from this phase flows back. + * + * Judge integrity (adversarially reviewed; both spoof channels closed): + * - both judges print a per-call random NONCE sentinel and the verdict is parsed + * from that exact nonce — a candidate printing a forged summary line cannot win; + * - the hidden judge requires the sentinel, not exit-0 — `sys.exit(0)` before + * check() is a FAIL, not a pass; + * - containers run under an in-container `timeout -s KILL` so a hung candidate + * cannot outlive a crashed harness; a process-exit reaper force-removes strays. + * + * Estimators (paired across the same tasks, same sample batch): + * blind1_mean — mean hidden-pass over ALL k samples = expected pass@1 at this + * temperature (a built-in k-rep baseline; the primary control) + * blind1_first — hidden-pass of sample 0 (single-rep reference only) + * selected@1 — hidden-pass of the honest-oracle argmax sample (selection value) + * repaired@1 — hidden-pass of the final after honest-grounded repair (full harness) + * oracle@k — any sample passes hidden (the pass@k ceiling) + * Every lift carries a 95% paired-bootstrap CI (B=10000, seeded) AND an exact + * two-sided sign test — the bootstrap alone is anticonservative when few tasks move. + * + * CALIBRATE=1 skips the model entirely: canonical solutions vs both judges → + * hidden-judge self-check (must be ~100%) + honest-oracle coverage & false-fail rate. + * + * TANGLE_API_KEY=… WORKER_MODEL=meta-llama/Meta-Llama-3-8B-Instruct-Lite \ + * ROUTER_BASE=https://api.together.xyz/v1 HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz \ + * N=164 K=5 REPAIRS=2 TEMPERATURE=0.8 OUT=/abs/rows.jsonl tsx src/hev-structural.mts + */ +import { execFile, execFileSync } from 'node:child_process' +import { randomBytes } from 'node:crypto' +import { appendFileSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { type HumanEvalTask, extractCode, loadHumanEval } from './benchmarks/humaneval' +import { composeStrategies } from './directives' +import { type PairedLift, pairedLift, pool } from './stats.mts' + +const dockerImage = 'python:3.12-slim' +const dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000) + +const solveInstruction = + 'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.' + +function must(name: string): string { + const v = process.env[name] + if (!v) throw new Error(`env ${name} is required`) + return v +} + +// ---------- docker semaphore + jailed runner (shared by BOTH judges) ---------- +// Phase A workers each run docker calls too, so the container count must be bounded +// by ONE global semaphore, not by whichever pool happens to wrap the caller. + +let dockerSlots = 6 +let dockerInFlight = 0 +const dockerWaiters: Array<() => void> = [] +async function withDockerSlot(fn: () => Promise): Promise { + if (dockerInFlight >= dockerSlots) await new Promise((r) => dockerWaiters.push(r)) + dockerInFlight += 1 + try { + return await fn() + } finally { + dockerInFlight -= 1 + dockerWaiters.shift()?.() + } +} + +const containerPrefix = `hevs-${process.pid}` +let containerSeq = 0 + +// Best-effort stray-container reap on any exit path (crash, SIGINT, clean end). +function reapContainers(): void { + try { + const ids = execFileSync('docker', ['ps', '-aq', '--filter', `name=${containerPrefix}`], { timeout: 10000 }).toString().trim() + if (ids) execFileSync('docker', ['rm', '-f', ...ids.split('\n')], { timeout: 15000 }) + } catch { + /* reaper is best-effort by design */ + } +} +process.on('SIGINT', () => { + reapContainers() + process.exit(130) +}) +process.on('SIGTERM', () => { + reapContainers() + process.exit(143) +}) + +interface JailResult { + exitCode: number + stdout: string + stderr: string +} + +/** Run one python program in the jail: --network=none, cpu/mem caps, an IN-CONTAINER + * `timeout -s KILL` (so a hung candidate's container self-terminates even if this + * process dies), a client timeout, and a backstop. Docker INFRA faults (daemon, + * image, permission) throw — a broken checker must fail loud, not score zeros. */ +function runJailed(program: string): Promise { + return withDockerSlot( + () => + new Promise((resolvePromise, reject) => { + const dir = mkdtempSync(join(tmpdir(), 'hevs-')) + writeFileSync(join(dir, 'p.py'), program) + const name = `${containerPrefix}-${containerSeq++}` + let settled = false + const cleanup = () => { + rmSync(dir, { recursive: true, force: true }) + execFile('docker', ['rm', '-f', name], () => {}) + } + const finish = (res: JailResult) => { + if (settled) return + settled = true + clearTimeout(backstop) + cleanup() + resolvePromise(res) + } + const fail = (e: Error) => { + if (settled) return + settled = true + clearTimeout(backstop) + cleanup() + reject(e) + } + const backstop = setTimeout(() => finish({ exitCode: 124, stdout: '', stderr: 'backstop timeout (no output)' }), dockerTimeoutMs + 5000) + const inContainerSecs = Math.ceil(dockerTimeoutMs / 1000) + 2 + execFile( + 'docker', + [ + 'run', '--rm', '--name', name, '--network=none', '--cpus=1', '--memory=512m', + '-v', `${dir}:/w:ro`, '-w', '/w', dockerImage, + 'timeout', '-s', 'KILL', String(inContainerSecs), 'python', '/w/p.py', + ], + { timeout: dockerTimeoutMs + 3000, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 }, + (err, stdout, stderr) => { + if (err) { + const e = err as NodeJS.ErrnoException & { code?: number | string } + if (e.code === 'ENOENT') return fail(new Error('docker binary not found on PATH')) + const se = stderr ?? '' + if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(se)) { + return fail(new Error(`docker daemon unreachable: ${se.slice(0, 200)}`)) + } + if (/(unable to find image|pull access denied|manifest unknown|error response from daemon).*(pull|repository|registry)/i.test(se)) { + return fail(new Error(`docker image ${dockerImage} unavailable: ${se.slice(0, 200)}`)) + } + const code = typeof e.code === 'number' ? e.code : 1 + return finish({ exitCode: code, stdout: stdout ?? '', stderr: se }) + } + finish({ exitCode: 0, stdout: stdout ?? '', stderr: stderr ?? '' }) + }, + ) + }), + ) +} + +// ---------- the honest oracle: doctest over the VISIBLE docstring examples ---------- + +export interface HonestResult { + /** total visible checks: doctest examples + model-generated asserts (0 = no + * signal, -1 = the candidate crashed before the oracle could run) */ + attempted: number + failed: number + /** doctest's failure report — the ONLY feedback the repair loop may see */ + failureOutput: string + /** attempted > 0 && failed === 0 */ + pass: boolean + /** split for post-hoc audit: doctest vs generated-assert counts */ + dAttempted?: number + dFailed?: number + gAttempted?: number + gFailed?: number +} + +/** The honest program: candidate executes (prompt header first, for its imports; + * candidate def shadows the stub), then doctest runs the examples taken from the + * STUB's `__doc__` (parsing the raw prompt text instead swallows the closing `"""` + * into the last example — caught by CALIBRATE=1). Verdict line carries a per-call + * NONCE so candidate-printed forgeries can't be parsed as the summary. task.test + * never appears here. */ +function buildHonestProgram(task: HumanEvalTask, candidate: string, nonce: string, genTests: string[] = []): string { + const promptB64 = Buffer.from(task.prompt, 'utf8').toString('base64') + const entryB64 = Buffer.from(task.entryPoint, 'utf8').toString('base64') + const genB64 = Buffer.from(JSON.stringify(genTests), 'utf8').toString('base64') + return `${task.prompt}\n${candidate}\n +import ast as _ast, base64 as _b64, doctest as _doctest, io as _io, json as _json, sys as _sys +_prompt_text = _b64.b64decode("${promptB64}").decode("utf8") +_entry = _b64.b64decode("${entryB64}").decode("utf8") +_gen_tests = _json.loads(_b64.b64decode("${genB64}").decode("utf8")) +_stub_ns = {} +exec(_prompt_text, _stub_ns) +_doc = getattr(_stub_ns.get(_entry), "__doc__", None) or "" +try: + _examples = _doctest.DocTestParser().get_examples(_doc) +except ValueError: + _examples = [] # malformed docstring indentation -> no usable signal, not a crash + +# Dataset-quirk normalizations, all decidable from VISIBLE output alone: +# assertion-style examples ("f(x) == 0" with no expected output) pass iff they print True; +# quote-style repr mismatches ("21" vs '21') compare by literal value. +class _Checker(_doctest.OutputChecker): + def check_output(self, want, got, optionflags): + if super().check_output(want, got, optionflags): + return True + if want.strip() == "" and got.strip() == "True": + return True + try: + return _ast.literal_eval(want.strip()) == _ast.literal_eval(got.strip()) + except Exception: + return False + +_test = _doctest.DocTest(_examples, globs=dict(globals()), name="visible", filename="p", lineno=0, docstring=_doc) +_runner = _doctest.DocTestRunner(checker=_Checker(), verbose=False, optionflags=_doctest.NORMALIZE_WHITESPACE | _doctest.IGNORE_EXCEPTION_DETAIL) +_buf = _io.StringIO() +_res = _runner.run(_test, out=_buf.write) + +# Model-generated asserts (CodeT-style; written from the prompt BEFORE any candidate +# existed). Each runs individually so one malformed assert doesn't zero the rest. +_g_att, _g_fail = 0, 0 +for _t in _gen_tests: + _g_att += 1 + try: + exec(_t, dict(globals())) + except Exception as _e: + _g_fail += 1 + _buf.write("GENTEST FAILED: %s -> %s: %s\\n" % (_t.strip()[:200], type(_e).__name__, str(_e)[:200])) + +_att = _res.attempted + _g_att +_fail = _res.failed + _g_fail +print("HONEST-${nonce} attempted=%d failed=%d datt=%d dfail=%d gatt=%d gfail=%d" % (_att, _fail, _res.attempted, _res.failed, _g_att, _g_fail)) +_sys.stdout.write(_buf.getvalue()[-1500:]) +_sys.exit(0 if _att > 0 and _fail == 0 else 1) +` +} + +export async function runHonestOracle(task: HumanEvalTask, candidate: string, genTests: string[] = []): Promise { + const nonce = randomBytes(8).toString('hex') + const r = await runJailed(buildHonestProgram(task, candidate, nonce, genTests)) + const summary = new RegExp(`HONEST-${nonce} attempted=(\\d+) failed=(\\d+) datt=(\\d+) dfail=(\\d+) gatt=(\\d+) gfail=(\\d+)`).exec(r.stdout) + if (!summary) { + // candidate crashed / hung before the oracle scaffold could report + const detail = (r.stderr || r.stdout).slice(-1500) || 'timed out (no output)' + return { attempted: -1, failed: -1, failureOutput: detail, pass: false } + } + const attempted = Number(summary[1]) + const failed = Number(summary[2]) + // Strip the sentinel line from the feedback shown to the repair loop — the model + // must never learn the summary format it could try to forge. + const failureOutput = r.stdout.replace(summary[0], '').slice(-1500) + return { + attempted, + failed, + failureOutput, + pass: attempted > 0 && failed === 0, + dAttempted: Number(summary[3]), + dFailed: Number(summary[4]), + gAttempted: Number(summary[5]), + gFailed: Number(summary[6]), + } +} + +// ---------- CodeT-style test generation (visible info only, BEFORE any candidate) ---------- + +const testGenInstruction = (count: number, entry: string) => + `Read the following Python function signature and docstring. Write exactly ${count} single-line assert statements that test the function \`${entry}\`, based ONLY on the behavior the docstring describes. Each assert must be one physical line of the form \`assert ${entry}(...) == expected\` (or a True/False check). Do NOT implement the function. Do NOT copy examples verbatim if you can test other cases too. Output ONLY the assert lines inside a single \`\`\`python code block.` + +/** One LLM call per task, before sampling. Keeps only single-line, paren-balanced + * asserts that reference the entry point — malformed lines are dropped here rather + * than poisoning every candidate's score identically. */ +async function generateTests(cfg: ClientCfg, task: HumanEvalTask, count: number): Promise<{ tests: string[]; completion: Completion }> { + const c = await complete(cfg, [ + { role: 'user', content: `${testGenInstruction(count, task.entryPoint)}\n\n\`\`\`python\n${task.prompt}\`\`\`` }, + ]) + const block = extractCode(c.content) + const balanced = (s: string) => { + let d = 0 + for (const ch of s) { + if (ch === '(' || ch === '[' || ch === '{') d += 1 + else if (ch === ')' || ch === ']' || ch === '}') d -= 1 + if (d < 0) return false + } + return d === 0 + } + const tests = block + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.startsWith('assert ') && l.includes(task.entryPoint) && balanced(l)) + .slice(0, count) + return { tests, completion: c } +} + +/** Honest score for ranking: fraction of visible examples passed; a candidate that + * crashed before doctest ran ranks below one that ran and failed everything. */ +function honestScore(h: HonestResult): number { + if (h.attempted <= 0) return h.attempted === 0 ? 0 : -1 + return (h.attempted - h.failed) / h.attempted +} + +// ---------- the hidden judge (Phase B / calibration ONLY) ---------- +// Rig-local rather than the shared runChecker: pass requires the nonce sentinel that +// check() prints AFTER succeeding — exit-0-before-check (sys.exit(0) in a candidate) +// is a fail here, where trusting the exit code alone would score it a pass. + +function buildHiddenProgram(task: HumanEvalTask, candidate: string, nonce: string): string { + return `${task.prompt}\n${candidate}\n\n${task.test}\n\ncheck(${task.entryPoint})\nprint("HIDDEN-${nonce} PASS")\n` +} + +async function runHiddenJudge(task: HumanEvalTask, candidate: string): Promise<{ pass: number; detail?: string }> { + const nonce = randomBytes(8).toString('hex') + const r = await runJailed(buildHiddenProgram(task, candidate, nonce)) + if (r.exitCode === 0 && r.stdout.includes(`HIDDEN-${nonce} PASS`)) return { pass: 1 } + return { pass: 0, detail: (r.stderr || r.stdout).slice(-600) || 'timed out (no output)' } +} + +// ---------- model client (plain fetch; retries on transient HTTP + empty content) ---------- + +interface ClientCfg { + base: string + key: string + model: string + maxTokens: number + temperature: number +} + +interface Completion { + content: string + attempts: number + tokensIn: number + tokensOut: number +} + +async function complete(cfg: ClientCfg, messages: Array<{ role: string; content: string }>): Promise { + let lastErr = '' + for (let attempt = 1; attempt <= 4; attempt += 1) { + if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) + const ctl = new AbortController() + const timer = setTimeout(() => ctl.abort(), 240_000) + try { + const res = await fetch(`${cfg.base}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }), + signal: ctl.signal, + }) + if (!res.ok) { + lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` + continue + } + const d = (await res.json()) as { + choices?: Array<{ message?: { content?: string } }> + usage?: { prompt_tokens?: number; completion_tokens?: number } + } + const content = d.choices?.[0]?.message?.content ?? '' + // Reasoning models starve `content` when reasoning exhausts max_tokens — an + // empty reply is a transient fault to retry, not a candidate to score. + if (content.trim() === '') { + lastErr = 'empty content' + continue + } + return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 } + } catch (e) { + lastErr = e instanceof Error ? e.message : String(e) + } finally { + clearTimeout(timer) + } + } + throw new Error(`completion failed after retries: ${lastErr}`) +} + +/** Repair replies often echo the failure report in a bare ``` block before the fixed + * code — first-fence extraction would grab the echo. Prefer the LAST fenced block + * that contains a `def`, else fall back to the shared extractor. Purely mechanical + * parsing of the model's own reply; no task information involved. */ +function extractRepairCode(reply: string): string { + const fences = [...reply.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/gi)].map((m) => (m[1] ?? '').trim()) + for (let i = fences.length - 1; i >= 0; i -= 1) { + if (/(^|\n)\s*def\s+\w+/.test(fences[i] as string)) return fences[i] as string + } + return extractCode(reply) +} + +// ---------- Phase A: the harness (sees ONLY visible information) ---------- + +interface HarnessOutcome { + taskId: string + samples: string[] + honest: HonestResult[] + selectedIdx: number + repairs: Array<{ code: string; honest: HonestResult }> + finalCode: string + /** 'already-passing' | 'no-signal' | 'repaired-pass' | 'rounds-exhausted' */ + repairStop: string + /** model-generated asserts used as extra oracle signal ([] when TESTGEN off) */ + genTests: string[] + llmCalls: number + llmAttempts: number + tokensIn: number + tokensOut: number +} + +async function runHarnessForTask(cfg: ClientCfg, task: HumanEvalTask, k: number, maxRepairs: number, testGen: number, diverse: boolean): Promise { + const basePrompt = `${solveInstruction}\n\n\`\`\`python\n${task.prompt}\`\`\`` + // DIVERSE mode: each sample slot gets a distinct strategy prefix — targets the + // all-k-samples-fail bucket, where iid resampling keeps drawing the same bug. + const slotPrompts = diverse ? composeStrategies(basePrompt, k) : Array.from({ length: k }, () => basePrompt) + let llmCalls = 0 + let llmAttempts = 0 + let tokensIn = 0 + let tokensOut = 0 + const track = (c: Completion) => { + llmCalls += 1 + llmAttempts += c.attempts + tokensIn += c.tokensIn + tokensOut += c.tokensOut + } + + // Generated tests come from the prompt alone, BEFORE any candidate exists, and + // are frozen for every sample and repair round of this task. + let genTests: string[] = [] + if (testGen > 0) { + const g = await generateTests(cfg, task, testGen) + track(g.completion) + genTests = g.tests + } + + const samples: string[] = [] + for (let i = 0; i < k; i += 1) { + const c = await complete(cfg, [{ role: 'user', content: slotPrompts[i] as string }]) + track(c) + samples.push(extractCode(c.content)) + } + const honest: HonestResult[] = [] + for (const s of samples) honest.push(await runHonestOracle(task, s, genTests)) + + // argmax by honest score, first index wins ties (deterministic; with zero + // coverage every sample ties at 0 → sample 0 = the blind pick) + let selectedIdx = 0 + for (let i = 1; i < k; i += 1) { + if (honestScore(honest[i] as HonestResult) > honestScore(honest[selectedIdx] as HonestResult)) selectedIdx = i + } + + const selHonest = honest[selectedIdx] as HonestResult + let best = { code: samples[selectedIdx] as string, honest: selHonest } + const repairs: HarnessOutcome['repairs'] = [] + let repairStop = 'already-passing' + if (!selHonest.pass) { + if (selHonest.attempted === 0) { + repairStop = 'no-signal' // no visible examples → nothing honest to steer on + } else { + repairStop = 'rounds-exhausted' + let current = best + for (let r = 0; r < maxRepairs; r += 1) { + const repairPrompt = [ + 'Your Python function failed some of the example checks shown in its own docstring.', + 'Here is the task again:', + '```python', + task.prompt.trimEnd(), + '```', + 'Your current attempt:', + '```python', + current.code, + '```', + 'Result of running the docstring examples against your attempt:', + '```', + current.honest.failureOutput.trim() || '(the code crashed before the examples could run)', + '```', + 'Fix the function so the docstring examples pass. Output the COMPLETE corrected function definition inside a single ```python code block. Do not write tests or example calls.', + ].join('\n') + const c = await complete(cfg, [{ role: 'user', content: repairPrompt }]) + track(c) + const code = extractRepairCode(c.content) + const h = await runHonestOracle(task, code, genTests) + repairs.push({ code, honest: h }) + if (honestScore(h) > honestScore(current.honest)) current = { code, honest: h } + if (honestScore(current.honest) > honestScore(best.honest)) best = current + if (h.pass) { + repairStop = 'repaired-pass' + break + } + } + } + } + + return { taskId: task.taskId, samples, honest, selectedIdx, repairs, finalCode: best.code, repairStop, genTests, llmCalls, llmAttempts, tokensIn, tokensOut } +} + +// ---------- statistics: exact sign test to pair with the bootstrap CI ---------- +// The percentile bootstrap is anticonservative when few tasks move (4 improved / 0 +// regressed at n=164 prints CI [+0.6, +4.9]pp while the exact test says p=0.125). +// The verdict requires BOTH. + +function signTestP(deltas: number[]): { pos: number; neg: number; p: number } { + const pos = deltas.filter((d) => d > 1e-9).length + const neg = deltas.filter((d) => d < -1e-9).length + const m = pos + neg + if (m === 0) return { pos, neg, p: 1 } + // two-sided exact binomial(m, 0.5) tail from the observed extreme + const logC: number[] = [0] + for (let i = 1; i <= m; i += 1) logC.push((logC[i - 1] as number) + Math.log(m - i + 1) - Math.log(i)) + const pmf = (x: number) => Math.exp((logC[x] as number) - m * Math.LN2) + const extreme = Math.max(pos, neg) + let p = 0 + for (let x = extreme; x <= m; x += 1) p += pmf(x) + p *= 2 + if (pos === neg) p = 1 + return { pos, neg, p: Math.min(1, p) } +} + +// ---------- calibration mode ---------- + +async function calibrate(tasks: HumanEvalTask[]): Promise { + console.log(`=== CALIBRATION · canonical solutions vs both judges · n=${tasks.length} ===`) + const usable = tasks.filter((t) => t.canonicalSolution) + if (usable.length !== tasks.length) console.log(` WARNING: ${tasks.length - usable.length} task(s) missing canonical_solution`) + const rows = await pool(usable, 16, async (t) => { + const full = `${t.prompt}${t.canonicalSolution}` + const hidden = await runHiddenJudge(t, full) + const honest = await runHonestOracle(t, full) + return { id: t.taskId, hidden: hidden.pass, attempted: honest.attempted, failed: honest.failed, honestPass: honest.pass } + }) + const hiddenPass = rows.filter((r) => r.hidden === 1) + const covered = rows.filter((r) => r.attempted > 0) + const falseFail = covered.filter((r) => !r.honestPass) + console.log(` hidden judge self-check: ${hiddenPass.length}/${rows.length} canonical solutions pass (must be ~100%)`) + if (hiddenPass.length < rows.length) console.log(` hidden FAILS: ${rows.filter((r) => r.hidden !== 1).map((r) => r.id).join(', ')}`) + console.log(` honest-oracle coverage: ${covered.length}/${rows.length} tasks have >=1 parseable docstring example`) + console.log(` zero-coverage tasks: ${rows.filter((r) => r.attempted === 0).map((r) => r.id).join(', ') || '(none)'}`) + console.log(` crashed-oracle tasks (attempted=-1): ${rows.filter((r) => r.attempted < 0).map((r) => r.id).join(', ') || '(none)'}`) + console.log(` honest-oracle false-fail on canonical: ${falseFail.length}/${covered.length} covered tasks`) + if (falseFail.length > 0) console.log(` false-fail ids: ${falseFail.map((r) => `${r.id}(${r.failed}/${r.attempted})`).join(', ')}`) + const examplesTotal = covered.reduce((s, r) => s + r.attempted, 0) + console.log(` examples per covered task: mean ${(examplesTotal / Math.max(1, covered.length)).toFixed(1)}`) +} + +// ---------- main ---------- + +const pct = (x: number) => `${(x * 100).toFixed(1)}%` +const pp = (x: number) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}pp` + +interface GradedRow extends HarnessOutcome { + hiddenSamples: number[] + hiddenFinal: number +} + +async function main(): Promise { + const n = Number(process.env.N ?? 164) + const k = Number(process.env.K ?? 5) + const maxRepairs = Number(process.env.REPAIRS ?? 2) + const offset = Number(process.env.OFFSET ?? 0) + const temperature = Number(process.env.TEMPERATURE ?? 0.8) + const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' + const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' + const solveConc = Number(process.env.CONCURRENCY ?? 6) + dockerSlots = Number(process.env.DOCKER_CONCURRENCY ?? 6) + const testGen = Number(process.env.TESTGEN ?? 0) + const diverse = process.env.DIVERSE === '1' + const out = process.env.OUT + + const tasks = await loadHumanEval(n, offset) + + if (process.env.CALIBRATE === '1') { + await calibrate(tasks) + return + } + + const cfg: ClientCfg = { base, key: must('TANGLE_API_KEY'), model, maxTokens: Number(process.env.MAX_TOKENS ?? 2500), temperature } + + console.log(`=== HumanEval STRUCTURAL lever · honest docstring oracle · n=${tasks.length} k=${k} repairs<=${maxRepairs} temp=${temperature} testgen=${testGen} diverse=${diverse ? 1 : 0} ===`) + console.log(` model=${model} base=${base} llm-conc=${solveConc} docker-conc=${dockerSlots} (global semaphore)`) + console.log(` Phase A (harness: sample->honest-select->honest-repair) then Phase B (hidden grading)`) + + // Phase A — all harness decisions locked before any hidden grading. A per-task + // fault becomes an error row (persisted, excluded from stats), not a lost run; + // >15% error rate aborts loud since that means the harness itself is broken. + let done = 0 + let errCount = 0 + const outcomes = await pool(tasks, solveConc, async (task): Promise => { + try { + const o = await runHarnessForTask(cfg, task, k, maxRepairs, testGen, diverse) + done += 1 + if (out) appendFileSync(`${out}.phaseA`, `${JSON.stringify({ model, temperature, k, maxRepairs, ...o })}\n`) + process.stderr.write( + ` [A ${done}/${tasks.length}] ${o.taskId}: sel=${o.selectedIdx} honest=${o.honest.map((h) => honestScore(h).toFixed(2)).join('/')} repairs=${o.repairs.length} stop=${o.repairStop}\n`, + ) + return o + } catch (e) { + errCount += 1 + const error = e instanceof Error ? e.message : String(e) + if (out) appendFileSync(`${out}.phaseA`, `${JSON.stringify({ model, taskId: task.taskId, error })}\n`) + process.stderr.write(` [A ERROR] ${task.taskId}: ${error.slice(0, 160)}\n`) + if (errCount > Math.max(3, 0.15 * tasks.length)) throw new Error(`aborting: ${errCount} task errors — harness-level fault, not task noise (last: ${error})`) + return { taskId: task.taskId, error } + } + }) + + const okOutcomes = outcomes.filter((o): o is HarnessOutcome => !('error' in o)) + const okTasks = okOutcomes.map((o) => tasks.find((t) => t.taskId === o.taskId) as HumanEvalTask) + if (errCount > 0) console.log(` WARNING: ${errCount}/${tasks.length} task(s) errored in Phase A — excluded from stats, recorded in ${out ?? '(no OUT set)'}.phaseA`) + + // Phase B — hidden grading of the locked artifacts. + console.log(`\n▶ Phase B: hidden grading (${okOutcomes.length} tasks × ${k} samples + finals)`) + const graded: GradedRow[] = await pool(okOutcomes, 16, async (o, ti) => { + const task = okTasks[ti] as HumanEvalTask + const hiddenSamples: number[] = [] + for (const s of o.samples) hiddenSamples.push((await runHiddenJudge(task, s)).pass) + const finalIsSelected = o.finalCode === o.samples[o.selectedIdx] + const hiddenFinal = finalIsSelected ? (hiddenSamples[o.selectedIdx] as number) : (await runHiddenJudge(task, o.finalCode)).pass + const g: GradedRow = { ...o, hiddenSamples, hiddenFinal } + if (out) appendFileSync(out, `${JSON.stringify({ model, temperature, k, maxRepairs, ...g })}\n`) + return g + }) + if (out) console.log(` raw rows appended to ${out} (phase-A rows incl. errors: ${out}.phaseA)`) + + // Estimators (all paired over the same graded tasks). + const blind1First = graded.map((g) => g.hiddenSamples[0] as number) + const blind1Mean = graded.map((g) => g.hiddenSamples.reduce((s, x) => s + x, 0) / g.hiddenSamples.length) + const selected = graded.map((g) => g.hiddenSamples[g.selectedIdx] as number) + const repaired = graded.map((g) => g.hiddenFinal) + const oracleK = graded.map((g) => (g.hiddenSamples.some((x) => x === 1) ? 1 : 0)) + // coverage is a task-level property; any non-crashed sample's oracle run proves it + const covered = graded.map((g) => (g.honest.some((h) => h.attempted > 0) ? 1 : 0)) + const rate = (xs: number[]) => xs.reduce((s, x) => s + x, 0) / xs.length + const llmCallsTotal = graded.reduce((s, g) => s + g.llmCalls, 0) + const llmAttemptsTotal = graded.reduce((s, g) => s + g.llmAttempts, 0) + const tokensInTotal = graded.reduce((s, g) => s + g.tokensIn, 0) + const tokensOutTotal = graded.reduce((s, g) => s + g.tokensOut, 0) + const repairFired = graded.filter((g) => g.repairs.length > 0) + + console.log(`\n${'='.repeat(78)}`) + console.log(`RESULTS · HumanEval structural lever · n=${graded.length} · k=${k} · repairs<=${maxRepairs} · temp=${temperature} · ${model}`) + console.log('='.repeat(78)) + console.log(` honest-oracle coverage ${pct(rate(covered))} of tasks (>=1 docstring example)`) + console.log(` blind pass@1 (mean of k) ${pct(rate(blind1Mean))} [PRIMARY baseline — ${k}-rep estimator]`) + console.log(` blind pass@1 (first sample) ${pct(rate(blind1First))} [single-rep reference]`) + console.log(` selected@1 (honest argmax) ${pct(rate(selected))}`) + console.log(` repaired@1 (full harness) ${pct(rate(repaired))}`) + console.log(` oracle pass@${k} (ceiling) ${pct(rate(oracleK))}`) + console.log( + ` compute: ${llmCallsTotal} llm calls (${llmAttemptsTotal} incl. retries) = ${(llmCallsTotal / graded.length).toFixed(2)}/task; tokens in/out ${tokensInTotal}/${tokensOutTotal} (blind@1 spends 1 call/task)`, + ) + console.log(` repair fired on ${repairFired.length}/${graded.length} tasks (stop: ${['already-passing', 'no-signal', 'repaired-pass', 'rounds-exhausted'].map((s) => `${s}=${graded.filter((g) => g.repairStop === s).length}`).join(', ')})`) + + const row = (label: string, baseline: number[], treatment: number[]) => { + const l = pairedLift(baseline, treatment) + const st = signTestP(baseline.map((b, i) => (treatment[i] as number) - b)) + console.log( + ` ${label.padEnd(36)} ${pp(l.point).padStart(7)} CI [${pp(l.low)}, ${pp(l.high)}] sign-test p=${st.p < 0.001 ? st.p.toExponential(1) : st.p.toFixed(3)} (+${st.pos}/−${st.neg}) (pairs ${l.pairs})`, + ) + return { l, st } + } + + console.log(`\n PAIRED LIFTS vs blind pass@1 (mean-of-${k}) · 95% bootstrap CI (B=10000) + exact sign test:`) + const sel = row('selected@1 − blind@1 (selection)', blind1Mean, selected) + const rep = row('repaired@1 − blind@1 (full harness)', blind1Mean, repaired) + row('repaired@1 − selected@1 (repair)', selected, repaired) + row(`oracle@${k} − repaired@1 (unrealized)`, repaired, oracleK) + + // Subgroup views (report-only; the primary claim is the unconditional lift): + const coveredIdx = graded.map((_, i) => i).filter((i) => covered[i] === 1) + if (coveredIdx.length > 0 && coveredIdx.length < graded.length) { + const pick = (xs: number[]) => coveredIdx.map((i) => xs[i] as number) + console.log(`\n COVERED-ONLY subgroup (n=${coveredIdx.length} tasks with visible examples; oracle can only act here):`) + row(' selected@1 − blind@1', pick(blind1Mean), pick(selected)) + row(' repaired@1 − blind@1', pick(blind1Mean), pick(repaired)) + } + + const verdict = (name: string, r: { l: PairedLift; st: { p: number } }) => + `${name}: ${pp(r.l.point)} — ${r.l.low > 0 && r.st.p < 0.05 ? 'POSITIVE (CI excludes 0 AND sign-test p<0.05)' : r.l.high < 0 && r.st.p < 0.05 ? 'NEGATIVE' : 'n.s.'}` + console.log(`\n VERDICT: ${verdict('full harness', rep)}; ${verdict('selection alone', sel)}`) +} + +main().catch((e) => { + reapContainers() + console.error(`hev-structural: ${e instanceof Error ? (e.stack ?? e.message) : String(e)}`) + process.exit(1) +}) diff --git a/bench/src/index.ts b/bench/src/index.ts index a3d522d2..aa2e32a7 100644 --- a/bench/src/index.ts +++ b/bench/src/index.ts @@ -30,10 +30,27 @@ export { type RagContext, } from './benchmarks/rag-shared' export { createT2RagBenchAdapter } from './benchmarks/t2-ragbench' +export { + createSweBenchAdapter, + scoreSweReport, + sweEvaluationArgv, + swePatchOutput, + type SweBenchAdapterOptions, + type SweBenchArtifactCaptureContext, + type SweBenchCacheLevel, +} from './benchmarks/swe-bench' +export { + runStagedJudge, + StagedJudgeError, + type StagedRunCaptureSpec, + type StagedRunSpec, +} from './benchmarks/_harness' export type { BenchmarkAdapter, BenchScore, BenchTask, + JudgeArtifactFileReceipt, + JudgeArtifactReceipt, LoadOptions, } from './benchmarks/types' @@ -53,3 +70,23 @@ export { type RunBenchmarksOptions, type RunBenchmarksReport, } from './run-benchmarks' +export { + createPierCandidateRecoveryExecutor, + executePreparedPierCandidate, + type ExecutePreparedPierCandidateOptions, + type PierCandidateGraderPort, + type PierCandidateOfficialResult, + type PierCandidateTerminationAcknowledgement, + type PierCandidateTrialController, + type PierCandidateTrialHandle, + type PierCandidateTrialIdentity, + type PierCandidateTrialResult, + type StagedPierCandidateExecution, +} from './pier-agent' +export { + FilePierCandidateTrialController, + type FilePierCandidateTrialControllerOptions, + type PierCandidateProcessSpec, + type PierDockerConnection, +} from './pier-trial-controller' +export { createPierResultGrader } from './pier-result-grader' diff --git a/bench/src/mbpp-structural.mts b/bench/src/mbpp-structural.mts new file mode 100644 index 00000000..abf5d33b --- /dev/null +++ b/bench/src/mbpp-structural.mts @@ -0,0 +1,662 @@ +/** + * MBPP STRUCTURAL lever — the transfer test for the HumanEval result in + * `hev-structural.mts`: best-of-k selection + self-repair grounded ONLY on + * VISIBLE checks, graded on HIDDEN tests the harness never shows the model. + * + * MBPP (sanitized, 427 tasks) has no docstring examples; the standard protocol + * shows the model test_list[0] (it pins the function name/signature). So: + * VISIBLE = test_list[0] (+ TESTGEN model-written asserts, generated from the + * description BEFORE any candidate exists) + * HIDDEN = test_list[1:] (with test_imports) — the grading suite + * A task with <2 asserts cannot split visible/hidden and is dropped at load. + * + * Architecture is hev-structural.mts verbatim (kept self-contained on purpose — + * the HumanEval rig is frozen post-verification): Phase A (harness: sample → + * visible-check select → visible-grounded repair, all decisions locked) then + * Phase B (hidden grading); per-call NONCE sentinels on both judges; global + * docker semaphore; in-container timeout + exit reaper; incremental OUT jsonl; + * per-task error rows with >15% abort; paired bootstrap + exact sign test. + * + * CALIBRATE=1: reference solutions through both judges (hidden self-check must + * be ~100%; failures listed — some MBPP references are known-defective). + * + * TANGLE_API_KEY=… WORKER_MODEL=meta-llama/Meta-Llama-3-8B-Instruct-Lite \ + * ROUTER_BASE=https://api.together.xyz/v1 MBPP_JSON=/abs/sanitized-mbpp.json \ + * N=427 K=5 REPAIRS=2 TESTGEN=6 TEMPERATURE=0.8 OUT=/abs/rows.jsonl \ + * tsx src/mbpp-structural.mts + */ +import { execFile, execFileSync } from 'node:child_process' +import { randomBytes } from 'node:crypto' +import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { extractCode } from './benchmarks/humaneval' +import { type PairedLift, pairedLift, pool } from './stats.mts' + +const dockerImage = 'python:3.12-slim' +const dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000) + +function must(name: string): string { + const v = process.env[name] + if (!v) throw new Error(`env ${name} is required`) + return v +} + +// ---------- dataset ---------- + +export interface MbppTask { + taskId: number + description: string + /** test_list[0] — shown to the model; pins the function name/signature */ + shownAssert: string + /** test_list[1:] — the hidden grading suite */ + hiddenAsserts: string[] + testImports: string[] + entryPoint: string + /** reference solution — judge self-check only, never shown to the model */ + referenceCode: string +} + +interface RawMbpp { + task_id: number + prompt: string + code: string + test_imports?: string[] + test_list: string[] +} + +/** Load sanitized MBPP, sorted by task_id. Drops (and counts) tasks that cannot + * split visible/hidden (<2 asserts) or whose entry point cannot be resolved. + * Entry point = the FIRST called name in test_list[0] that also has a + * `def ` in the reference code — `assert set(f(...)) == …` must resolve + * to f, not set. */ +export function loadMbpp(limit: number, offset = 0): { tasks: MbppTask[]; droppedShort: number[]; droppedEntry: number[] } { + const path = process.env.MBPP_JSON + if (!path) throw new Error('env MBPP_JSON is required (path to sanitized-mbpp.json)') + const raw = JSON.parse(readFileSync(path, 'utf8')) as RawMbpp[] + raw.sort((a, b) => a.task_id - b.task_id) + const tasks: MbppTask[] = [] + const droppedShort: number[] = [] + const droppedEntry: number[] = [] + for (const d of raw) { + if (!d.test_list || d.test_list.length < 2) { + droppedShort.push(d.task_id) + continue + } + const shown = d.test_list[0] as string + const calls = [...shown.matchAll(/(\w+)\s*\(/g)].map((m) => m[1] as string) + const entry = calls.find((c) => new RegExp(`def\\s+${c}\\s*\\(`).test(d.code)) + if (!entry) { + droppedEntry.push(d.task_id) + continue + } + tasks.push({ + taskId: d.task_id, + description: d.prompt, + shownAssert: shown, + hiddenAsserts: d.test_list.slice(1), + testImports: d.test_imports ?? [], + entryPoint: entry, + referenceCode: d.code, + }) + } + if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= usable dataset size ${tasks.length}`) + return { tasks: tasks.slice(offset, offset + limit), droppedShort, droppedEntry } +} + +const solveInstruction = + 'Write a Python function for the following task. Output the COMPLETE function definition (plus any imports it needs) inside a single ```python code block. Do not write tests or example calls.' + +function basePrompt(task: MbppTask): string { + return `${solveInstruction}\n\nTask: ${task.description}\nYour function must satisfy this example test:\n\`\`\`python\n${task.shownAssert}\n\`\`\`` +} + +// ---------- docker semaphore + jailed runner (mirrors hev-structural) ---------- + +let dockerSlots = 6 +let dockerInFlight = 0 +const dockerWaiters: Array<() => void> = [] +async function withDockerSlot(fn: () => Promise): Promise { + if (dockerInFlight >= dockerSlots) await new Promise((r) => dockerWaiters.push(r)) + dockerInFlight += 1 + try { + return await fn() + } finally { + dockerInFlight -= 1 + dockerWaiters.shift()?.() + } +} + +const containerPrefix = `mbpps-${process.pid}` +let containerSeq = 0 + +function reapContainers(): void { + try { + const ids = execFileSync('docker', ['ps', '-aq', '--filter', `name=${containerPrefix}`], { timeout: 10000 }).toString().trim() + if (ids) execFileSync('docker', ['rm', '-f', ...ids.split('\n')], { timeout: 15000 }) + } catch { + /* reaper is best-effort by design */ + } +} +process.on('SIGINT', () => { + reapContainers() + process.exit(130) +}) +process.on('SIGTERM', () => { + reapContainers() + process.exit(143) +}) + +interface JailResult { + exitCode: number + stdout: string + stderr: string +} + +function runJailed(program: string): Promise { + return withDockerSlot( + () => + new Promise((resolvePromise, reject) => { + const dir = mkdtempSync(join(tmpdir(), 'mbpps-')) + writeFileSync(join(dir, 'p.py'), program) + const name = `${containerPrefix}-${containerSeq++}` + let settled = false + const cleanup = () => { + rmSync(dir, { recursive: true, force: true }) + execFile('docker', ['rm', '-f', name], () => {}) + } + const finish = (res: JailResult) => { + if (settled) return + settled = true + clearTimeout(backstop) + cleanup() + resolvePromise(res) + } + const fail = (e: Error) => { + if (settled) return + settled = true + clearTimeout(backstop) + cleanup() + reject(e) + } + const backstop = setTimeout(() => finish({ exitCode: 124, stdout: '', stderr: 'backstop timeout (no output)' }), dockerTimeoutMs + 5000) + const inContainerSecs = Math.ceil(dockerTimeoutMs / 1000) + 2 + execFile( + 'docker', + [ + 'run', '--rm', '--name', name, '--network=none', '--cpus=1', '--memory=512m', + '-v', `${dir}:/w:ro`, '-w', '/w', dockerImage, + 'timeout', '-s', 'KILL', String(inContainerSecs), 'python', '/w/p.py', + ], + { timeout: dockerTimeoutMs + 3000, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 }, + (err, stdout, stderr) => { + if (err) { + const e = err as NodeJS.ErrnoException & { code?: number | string } + if (e.code === 'ENOENT') return fail(new Error('docker binary not found on PATH')) + const se = stderr ?? '' + if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(se)) { + return fail(new Error(`docker daemon unreachable: ${se.slice(0, 200)}`)) + } + if (/(unable to find image|pull access denied|manifest unknown|error response from daemon).*(pull|repository|registry)/i.test(se)) { + return fail(new Error(`docker image ${dockerImage} unavailable: ${se.slice(0, 200)}`)) + } + const code = typeof e.code === 'number' ? e.code : 1 + return finish({ exitCode: code, stdout: stdout ?? '', stderr: se }) + } + finish({ exitCode: 0, stdout: stdout ?? '', stderr: stderr ?? '' }) + }, + ) + }), + ) +} + +// ---------- the visible-check judge (Phase A's ONLY signal) ---------- + +export interface VisibleResult { + /** total visible checks run: shown assert + generated asserts (-1 = crashed) */ + attempted: number + failed: number + failureOutput: string + pass: boolean + sAttempted?: number + sFailed?: number + gAttempted?: number + gFailed?: number +} + +/** Each visible assert runs INDIVIDUALLY in try/except so one malformed line + * cannot zero the rest; per-assert failure text is the repair feedback. The + * hidden asserts (test_list[1:]) never appear here. */ +function buildVisibleProgram(task: MbppTask, candidate: string, nonce: string, genTests: string[]): string { + const shownB64 = Buffer.from(JSON.stringify([task.shownAssert]), 'utf8').toString('base64') + const genB64 = Buffer.from(JSON.stringify(genTests), 'utf8').toString('base64') + return `${task.testImports.join('\n')}\n${candidate}\n +import base64 as _b64, json as _json, sys as _sys +_shown = _json.loads(_b64.b64decode("${shownB64}").decode("utf8")) +_gen = _json.loads(_b64.b64decode("${genB64}").decode("utf8")) +_out = [] +def _run(_tests): + _att, _fail = 0, 0 + for _t in _tests: + _att += 1 + try: + exec(_t, dict(globals())) + except Exception as _e: + _fail += 1 + _out.append("CHECK FAILED: %s -> %s: %s" % (_t.strip()[:200], type(_e).__name__, str(_e)[:200])) + return _att, _fail +_s_att, _s_fail = _run(_shown) +_g_att, _g_fail = _run(_gen) +_att, _fail = _s_att + _g_att, _s_fail + _g_fail +print("VISIBLE-${nonce} attempted=%d failed=%d satt=%d sfail=%d gatt=%d gfail=%d" % (_att, _fail, _s_att, _s_fail, _g_att, _g_fail)) +_sys.stdout.write("\\n".join(_out)[-1500:]) +_sys.exit(0 if _att > 0 and _fail == 0 else 1) +` +} + +export async function runVisibleJudge(task: MbppTask, candidate: string, genTests: string[]): Promise { + const nonce = randomBytes(8).toString('hex') + const r = await runJailed(buildVisibleProgram(task, candidate, nonce, genTests)) + const summary = new RegExp(`VISIBLE-${nonce} attempted=(\\d+) failed=(\\d+) satt=(\\d+) sfail=(\\d+) gatt=(\\d+) gfail=(\\d+)`).exec(r.stdout) + if (!summary) { + const detail = (r.stderr || r.stdout).slice(-1500) || 'timed out (no output)' + return { attempted: -1, failed: -1, failureOutput: detail, pass: false } + } + const attempted = Number(summary[1]) + const failed = Number(summary[2]) + const failureOutput = r.stdout.replace(summary[0], '').slice(-1500) + return { + attempted, + failed, + failureOutput, + pass: attempted > 0 && failed === 0, + sAttempted: Number(summary[3]), + sFailed: Number(summary[4]), + gAttempted: Number(summary[5]), + gFailed: Number(summary[6]), + } +} + +function visibleScore(h: VisibleResult): number { + if (h.attempted <= 0) return h.attempted === 0 ? 0 : -1 + // The shown assert is OFFICIAL (printed in the model's prompt); generated asserts + // are the model's own guesses and run ~70% wrong on MBPP's one-sentence specs + // (measured on the pilot: 71/102 failed on officially-passing code). Rank by the + // official signal first; guesses only break ties — otherwise 6 noisy guesses + // outvote the one reliable check and selection goes NEGATIVE. + const sA = h.sAttempted ?? 0 + const gA = h.gAttempted ?? 0 + const sFrac = sA > 0 ? (sA - (h.sFailed ?? 0)) / sA : 0 + const gFrac = gA > 0 ? (gA - (h.gFailed ?? 0)) / gA : 0 + return sFrac + 0.001 * gFrac +} + +// ---------- the hidden judge (Phase B / calibration ONLY) ---------- + +function buildHiddenProgram(task: MbppTask, candidate: string, nonce: string): string { + return `${task.testImports.join('\n')}\n${candidate}\n\n${task.hiddenAsserts.join('\n')}\nprint("HIDDEN-${nonce} PASS")\n` +} + +async function runHiddenJudge(task: MbppTask, candidate: string): Promise<{ pass: number; detail?: string }> { + const nonce = randomBytes(8).toString('hex') + const r = await runJailed(buildHiddenProgram(task, candidate, nonce)) + if (r.exitCode === 0 && r.stdout.includes(`HIDDEN-${nonce} PASS`)) return { pass: 1 } + return { pass: 0, detail: (r.stderr || r.stdout).slice(-600) || 'timed out (no output)' } +} + +// ---------- model client (mirrors hev-structural) ---------- + +interface ClientCfg { + base: string + key: string + model: string + maxTokens: number + temperature: number +} + +interface Completion { + content: string + attempts: number + tokensIn: number + tokensOut: number +} + +async function complete(cfg: ClientCfg, messages: Array<{ role: string; content: string }>): Promise { + let lastErr = '' + for (let attempt = 1; attempt <= 4; attempt += 1) { + if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) + const ctl = new AbortController() + const timer = setTimeout(() => ctl.abort(), 240_000) + try { + const res = await fetch(`${cfg.base}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }), + signal: ctl.signal, + }) + if (!res.ok) { + lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` + continue + } + const d = (await res.json()) as { + choices?: Array<{ message?: { content?: string } }> + usage?: { prompt_tokens?: number; completion_tokens?: number } + } + const content = d.choices?.[0]?.message?.content ?? '' + if (content.trim() === '') { + lastErr = 'empty content' + continue + } + return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 } + } catch (e) { + lastErr = e instanceof Error ? e.message : String(e) + } finally { + clearTimeout(timer) + } + } + throw new Error(`completion failed after retries: ${lastErr}`) +} + +function extractRepairCode(reply: string): string { + const fences = [...reply.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/gi)].map((m) => (m[1] ?? '').trim()) + for (let i = fences.length - 1; i >= 0; i -= 1) { + if (/(^|\n)\s*def\s+\w+/.test(fences[i] as string)) return fences[i] as string + } + return extractCode(reply) +} + +// ---------- TESTGEN (mirrors hev-structural; description + shown assert only) ---------- + +const testGenInstruction = (count: number, entry: string) => + `Read the following task description and example test. Write exactly ${count} single-line assert statements that test the function \`${entry}\`, based ONLY on the described behavior. Match the EXACT output type and format the example test shows (if it expects a string, expect a string; if a tuple, a tuple). Each assert must be one physical line of the form \`assert ${entry}(...) == expected\` (or a True/False check). Do NOT implement the function. Do NOT repeat the example test verbatim if you can test other cases too. Output ONLY the assert lines inside a single \`\`\`python code block.` + +async function generateTests(cfg: ClientCfg, task: MbppTask, count: number): Promise<{ tests: string[]; completion: Completion }> { + const c = await complete(cfg, [ + { role: 'user', content: `${testGenInstruction(count, task.entryPoint)}\n\nTask: ${task.description}\nExample test:\n\`\`\`python\n${task.shownAssert}\n\`\`\`` }, + ]) + const block = extractCode(c.content) + const balanced = (s: string) => { + let d = 0 + for (const ch of s) { + if (ch === '(' || ch === '[' || ch === '{') d += 1 + else if (ch === ')' || ch === ']' || ch === '}') d -= 1 + if (d < 0) return false + } + return d === 0 + } + const tests = block + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.startsWith('assert ') && l.includes(task.entryPoint) && balanced(l)) + .slice(0, count) + return { tests, completion: c } +} + +// ---------- Phase A: the harness (sees ONLY visible information) ---------- + +interface HarnessOutcome { + taskId: number + samples: string[] + visible: VisibleResult[] + selectedIdx: number + repairs: Array<{ code: string; visible: VisibleResult }> + finalCode: string + repairStop: string + genTests: string[] + llmCalls: number + llmAttempts: number + tokensIn: number + tokensOut: number +} + +async function runHarnessForTask(cfg: ClientCfg, task: MbppTask, k: number, maxRepairs: number, testGen: number): Promise { + let llmCalls = 0 + let llmAttempts = 0 + let tokensIn = 0 + let tokensOut = 0 + const track = (c: Completion) => { + llmCalls += 1 + llmAttempts += c.attempts + tokensIn += c.tokensIn + tokensOut += c.tokensOut + } + + let genTests: string[] = [] + if (testGen > 0) { + const g = await generateTests(cfg, task, testGen) + track(g.completion) + genTests = g.tests + } + + const samples: string[] = [] + for (let i = 0; i < k; i += 1) { + const c = await complete(cfg, [{ role: 'user', content: basePrompt(task) }]) + track(c) + samples.push(extractCode(c.content)) + } + const visible: VisibleResult[] = [] + for (const s of samples) visible.push(await runVisibleJudge(task, s, genTests)) + + let selectedIdx = 0 + for (let i = 1; i < k; i += 1) { + if (visibleScore(visible[i] as VisibleResult) > visibleScore(visible[selectedIdx] as VisibleResult)) selectedIdx = i + } + + const selVisible = visible[selectedIdx] as VisibleResult + let best = { code: samples[selectedIdx] as string, visible: selVisible } + const repairs: HarnessOutcome['repairs'] = [] + let repairStop = 'already-passing' + if (!selVisible.pass) { + if (selVisible.attempted === 0) { + repairStop = 'no-signal' + } else { + repairStop = 'rounds-exhausted' + let current = best + for (let r = 0; r < maxRepairs; r += 1) { + const repairPrompt = [ + 'Your Python function failed some of its checks.', + 'The task:', + task.description, + 'It must satisfy this example test:', + '```python', + task.shownAssert, + '```', + 'Your current attempt:', + '```python', + current.code, + '```', + 'Result of running the checks against your attempt:', + '```', + current.visible.failureOutput.trim() || '(the code crashed before the checks could run)', + '```', + 'Fix the function so the checks pass. Output the COMPLETE corrected function definition inside a single ```python code block. Do not write tests or example calls.', + ].join('\n') + const c = await complete(cfg, [{ role: 'user', content: repairPrompt }]) + track(c) + const code = extractRepairCode(c.content) + const h = await runVisibleJudge(task, code, genTests) + repairs.push({ code, visible: h }) + if (visibleScore(h) > visibleScore(current.visible)) current = { code, visible: h } + if (visibleScore(current.visible) > visibleScore(best.visible)) best = current + if (h.pass) { + repairStop = 'repaired-pass' + break + } + } + } + } + + return { taskId: task.taskId, samples, visible, selectedIdx, repairs, finalCode: best.code, repairStop, genTests, llmCalls, llmAttempts, tokensIn, tokensOut } +} + +// ---------- statistics (mirrors hev-structural) ---------- + +function signTestP(deltas: number[]): { pos: number; neg: number; p: number } { + const pos = deltas.filter((d) => d > 1e-9).length + const neg = deltas.filter((d) => d < -1e-9).length + const m = pos + neg + if (m === 0) return { pos, neg, p: 1 } + const logC: number[] = [0] + for (let i = 1; i <= m; i += 1) logC.push((logC[i - 1] as number) + Math.log(m - i + 1) - Math.log(i)) + const pmf = (x: number) => Math.exp((logC[x] as number) - m * Math.LN2) + const extreme = Math.max(pos, neg) + let p = 0 + for (let x = extreme; x <= m; x += 1) p += pmf(x) + p *= 2 + if (pos === neg) p = 1 + return { pos, neg, p: Math.min(1, p) } +} + +// ---------- calibration mode ---------- + +async function calibrate(tasks: MbppTask[]): Promise { + console.log(`=== CALIBRATION · MBPP reference solutions vs both judges · n=${tasks.length} ===`) + const rows = await pool(tasks, 16, async (t) => { + const hidden = await runHiddenJudge(t, t.referenceCode) + const visible = await runVisibleJudge(t, t.referenceCode, []) + return { id: t.taskId, hidden: hidden.pass, hiddenDetail: hidden.detail, vAttempted: visible.attempted, vFailed: visible.failed, visiblePass: visible.pass } + }) + const hiddenPass = rows.filter((r) => r.hidden === 1) + const visibleFail = rows.filter((r) => !r.visiblePass) + console.log(` hidden judge self-check: ${hiddenPass.length}/${rows.length} reference solutions pass (must be ~100%)`) + if (hiddenPass.length < rows.length) { + for (const r of rows.filter((x) => x.hidden !== 1)) console.log(` hidden FAIL ${r.id}: ${(r.hiddenDetail ?? '').replace(/\n/g, ' | ').slice(0, 160)}`) + } + console.log(` visible-check false-fail on reference: ${visibleFail.length}/${rows.length}`) + if (visibleFail.length > 0) console.log(` visible false-fail ids: ${visibleFail.map((r) => `${r.id}(${r.vFailed}/${r.vAttempted})`).join(', ')}`) +} + +// ---------- main ---------- + +const pct = (x: number) => `${(x * 100).toFixed(1)}%` +const pp = (x: number) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}pp` + +interface GradedRow extends HarnessOutcome { + hiddenSamples: number[] + hiddenFinal: number +} + +async function main(): Promise { + const n = Number(process.env.N ?? 427) + const k = Number(process.env.K ?? 5) + const maxRepairs = Number(process.env.REPAIRS ?? 2) + const offset = Number(process.env.OFFSET ?? 0) + const temperature = Number(process.env.TEMPERATURE ?? 0.8) + const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' + const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' + const solveConc = Number(process.env.CONCURRENCY ?? 6) + dockerSlots = Number(process.env.DOCKER_CONCURRENCY ?? 6) + const testGen = Number(process.env.TESTGEN ?? 0) + const out = process.env.OUT + + const { tasks, droppedShort, droppedEntry } = loadMbpp(n, offset) + console.log(`loaded ${tasks.length} MBPP task(s); dropped ${droppedShort.length} (<2 asserts: ${droppedShort.join(',') || '-'}), ${droppedEntry.length} (entry unresolved: ${droppedEntry.join(',') || '-'})`) + + if (process.env.CALIBRATE === '1') { + await calibrate(tasks) + return + } + + const cfg: ClientCfg = { base, key: must('TANGLE_API_KEY'), model, maxTokens: Number(process.env.MAX_TOKENS ?? 2500), temperature } + + console.log(`=== MBPP STRUCTURAL lever · visible=test_list[0]+gen · hidden=test_list[1:] · n=${tasks.length} k=${k} repairs<=${maxRepairs} temp=${temperature} testgen=${testGen} ===`) + console.log(` model=${model} base=${base} llm-conc=${solveConc} docker-conc=${dockerSlots} (global semaphore)`) + + let done = 0 + let errCount = 0 + const outcomes = await pool(tasks, solveConc, async (task): Promise => { + try { + const o = await runHarnessForTask(cfg, task, k, maxRepairs, testGen) + done += 1 + if (out) appendFileSync(`${out}.phaseA`, `${JSON.stringify({ model, temperature, k, maxRepairs, ...o })}\n`) + process.stderr.write( + ` [A ${done}/${tasks.length}] mbpp/${o.taskId}: sel=${o.selectedIdx} visible=${o.visible.map((h) => visibleScore(h).toFixed(2)).join('/')} repairs=${o.repairs.length} stop=${o.repairStop}\n`, + ) + return o + } catch (e) { + errCount += 1 + const error = e instanceof Error ? e.message : String(e) + if (out) appendFileSync(`${out}.phaseA`, `${JSON.stringify({ model, taskId: task.taskId, error })}\n`) + process.stderr.write(` [A ERROR] mbpp/${task.taskId}: ${error.slice(0, 160)}\n`) + if (errCount > Math.max(3, 0.15 * tasks.length)) throw new Error(`aborting: ${errCount} task errors — harness-level fault, not task noise (last: ${error})`) + return { taskId: task.taskId, error } + } + }) + + const okOutcomes = outcomes.filter((o): o is HarnessOutcome => !('error' in o)) + const okTasks = okOutcomes.map((o) => tasks.find((t) => t.taskId === o.taskId) as MbppTask) + if (errCount > 0) console.log(` WARNING: ${errCount}/${tasks.length} task(s) errored in Phase A — excluded from stats, recorded in ${out ?? '(no OUT set)'}.phaseA`) + + console.log(`\n▶ Phase B: hidden grading (${okOutcomes.length} tasks × ${k} samples + finals)`) + const graded: GradedRow[] = await pool(okOutcomes, 16, async (o, ti) => { + const task = okTasks[ti] as MbppTask + const hiddenSamples: number[] = [] + for (const s of o.samples) hiddenSamples.push((await runHiddenJudge(task, s)).pass) + const finalIsSelected = o.finalCode === o.samples[o.selectedIdx] + const hiddenFinal = finalIsSelected ? (hiddenSamples[o.selectedIdx] as number) : (await runHiddenJudge(task, o.finalCode)).pass + const g: GradedRow = { ...o, hiddenSamples, hiddenFinal } + if (out) appendFileSync(out, `${JSON.stringify({ model, temperature, k, maxRepairs, ...g })}\n`) + return g + }) + if (out) console.log(` raw rows appended to ${out} (phase-A rows incl. errors: ${out}.phaseA)`) + + const blind1First = graded.map((g) => g.hiddenSamples[0] as number) + const blind1Mean = graded.map((g) => g.hiddenSamples.reduce((s, x) => s + x, 0) / g.hiddenSamples.length) + const selected = graded.map((g) => g.hiddenSamples[g.selectedIdx] as number) + const repaired = graded.map((g) => g.hiddenFinal) + const oracleK = graded.map((g) => (g.hiddenSamples.some((x) => x === 1) ? 1 : 0)) + const covered = graded.map((g) => (g.visible.some((h) => h.attempted > 0) ? 1 : 0)) + const rate = (xs: number[]) => xs.reduce((s, x) => s + x, 0) / xs.length + const llmCallsTotal = graded.reduce((s, g) => s + g.llmCalls, 0) + const llmAttemptsTotal = graded.reduce((s, g) => s + g.llmAttempts, 0) + const tokensInTotal = graded.reduce((s, g) => s + g.tokensIn, 0) + const tokensOutTotal = graded.reduce((s, g) => s + g.tokensOut, 0) + const repairFired = graded.filter((g) => g.repairs.length > 0) + + console.log(`\n${'='.repeat(78)}`) + console.log(`RESULTS · MBPP structural lever · n=${graded.length} · k=${k} · repairs<=${maxRepairs} · temp=${temperature} · ${model}`) + console.log('='.repeat(78)) + console.log(` visible-check coverage ${pct(rate(covered))} of tasks (shown assert always present)`) + console.log(` blind pass@1 (mean of k) ${pct(rate(blind1Mean))} [PRIMARY baseline — ${k}-rep estimator]`) + console.log(` blind pass@1 (first sample) ${pct(rate(blind1First))} [single-rep reference]`) + console.log(` selected@1 (visible argmax) ${pct(rate(selected))}`) + console.log(` repaired@1 (full harness) ${pct(rate(repaired))}`) + console.log(` oracle pass@${k} (ceiling) ${pct(rate(oracleK))}`) + console.log( + ` compute: ${llmCallsTotal} llm calls (${llmAttemptsTotal} incl. retries) = ${(llmCallsTotal / graded.length).toFixed(2)}/task; tokens in/out ${tokensInTotal}/${tokensOutTotal} (blind@1 spends 1 call/task)`, + ) + console.log(` repair fired on ${repairFired.length}/${graded.length} tasks (stop: ${['already-passing', 'no-signal', 'repaired-pass', 'rounds-exhausted'].map((s) => `${s}=${graded.filter((g) => g.repairStop === s).length}`).join(', ')})`) + + const row = (label: string, baseline: number[], treatment: number[]) => { + const l = pairedLift(baseline, treatment) + const st = signTestP(baseline.map((b, i) => (treatment[i] as number) - b)) + console.log( + ` ${label.padEnd(36)} ${pp(l.point).padStart(7)} CI [${pp(l.low)}, ${pp(l.high)}] sign-test p=${st.p < 0.001 ? st.p.toExponential(1) : st.p.toFixed(3)} (+${st.pos}/−${st.neg}) (pairs ${l.pairs})`, + ) + return { l, st } + } + + console.log(`\n PAIRED LIFTS vs blind pass@1 (mean-of-${k}) · 95% bootstrap CI (B=10000) + exact sign test:`) + const sel = row('selected@1 − blind@1 (selection)', blind1Mean, selected) + const rep = row('repaired@1 − blind@1 (full harness)', blind1Mean, repaired) + row('repaired@1 − selected@1 (repair)', selected, repaired) + row(`oracle@${k} − repaired@1 (unrealized)`, repaired, oracleK) + + const coveredIdx = graded.map((_, i) => i).filter((i) => covered[i] === 1) + if (coveredIdx.length > 0 && coveredIdx.length < graded.length) { + const pick = (xs: number[]) => coveredIdx.map((i) => xs[i] as number) + console.log(`\n COVERED-ONLY subgroup (n=${coveredIdx.length}):`) + row(' selected@1 − blind@1', pick(blind1Mean), pick(selected)) + row(' repaired@1 − blind@1', pick(blind1Mean), pick(repaired)) + } + + const verdict = (name: string, r: { l: PairedLift; st: { p: number } }) => + `${name}: ${pp(r.l.point)} — ${r.l.low > 0 && r.st.p < 0.05 ? 'POSITIVE (CI excludes 0 AND sign-test p<0.05)' : r.l.high < 0 && r.st.p < 0.05 ? 'NEGATIVE' : 'n.s.'}` + console.log(`\n VERDICT: ${verdict('full harness', rep)}; ${verdict('selection alone', sel)}`) +} + +main().catch((e) => { + reapContainers() + console.error(`mbpp-structural: ${e instanceof Error ? (e.stack ?? e.message) : String(e)}`) + process.exit(1) +}) diff --git a/bench/src/pier-agent.test-fixtures.mts b/bench/src/pier-agent.test-fixtures.mts new file mode 100644 index 00000000..d331e57a --- /dev/null +++ b/bench/src/pier-agent.test-fixtures.mts @@ -0,0 +1,19 @@ +import type { StagedPierCandidateExecution } from './pier-agent' + +export function createStagedPierCandidateExecutionFixture( + executionId: string, +): StagedPierCandidateExecution { + const directory = `/tmp/agent-bench-pier-test/${encodeURIComponent(executionId)}` + return { + executionId, + directory, + taskDirectory: `${directory}/task`, + candidateDirectory: `${directory}/candidate`, + profileDirectory: `${directory}/profile`, + planPath: `${directory}/execution-plan.json`, + receiptPath: `${directory}/materialization-receipt.json`, + agentArgs: ['--agent', 'pier_agents.tangle_candidate:TangleCandidateAgent'], + evaluatorEnv: {}, + attemptArgs: ['--n-attempts', '1', '--max-retries', '0'], + } +} diff --git a/bench/src/pier-agent.test.mts b/bench/src/pier-agent.test.mts new file mode 100644 index 00000000..61d3bae6 --- /dev/null +++ b/bench/src/pier-agent.test.mts @@ -0,0 +1,350 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' + +import { canonicalJson } from '@tangle-network/agent-eval' +import type { + AgentCandidateExecutorRequest, + PreparedAgentCandidateExecution, +} from '@tangle-network/agent-runtime' + +import { + awaitAbortableTrial, + protectedCaptureFromPierResult, + stagePreparedPierCandidateExecution, +} from './pier-agent' + +const digest = (bytes: Uint8Array): `sha256:${string}` => + `sha256:${createHash('sha256').update(bytes).digest('hex')}` + +function prepared(root: string): PreparedAgentCandidateExecution { + const bundleDigest = `sha256:${'b'.repeat(64)}` as const + const executionId = 'pier-test-execution' + const material = { + schemaVersion: 1, + kind: 'agent-candidate-execution-plan-material', + bundleDigest, + executionId, + limits: { + timeoutMs: 60_000, + maxSteps: 8, + maxModelCalls: 0, + maxInputTokens: 0, + maxOutputTokens: 0, + maxCostUsd: 0, + }, + } + const planBytes = Buffer.from(canonicalJson(material)) + const planDigest = digest(planBytes) + const receiptMaterial = { + schemaVersion: 1, + kind: 'agent-candidate-materialization', + digestAlgorithm: 'rfc8785-sha256', + bundleDigest, + executionPlan: { digest: planDigest, material }, + } + const receiptBytes = Buffer.from(canonicalJson(receiptMaterial)) + const receiptDigest = digest(receiptBytes) + const profileBytes = Buffer.from('fixture profile\n') + return { + bundle: { digest: bundleDigest }, + executionId, + roots: { + execution: { taskRoot: '/app' }, + staging: { + taskRoot: join(root, 'task'), + profileRoot: join(root, 'profile'), + }, + }, + executionPlan: { + value: { digest: planDigest, material }, + bytes: planBytes, + }, + profilePlan: { + value: { + material: { + files: [ + { + relPath: 'AGENTS.md', + mode: 0o644, + contentSha256: digest(profileBytes), + }, + ], + }, + }, + bytes: Buffer.from('{}'), + written: ['AGENTS.md'], + }, + materializationReceipt: { + value: { ...receiptMaterial, digest: receiptDigest }, + bytes: receiptBytes, + digest: receiptDigest, + }, + resolvedModel: { + requested: 'openai/gpt-5.4', + provider: 'openai', + model: 'gpt-5.4', + snapshot: 'gpt-5.4-2026-06-01', + reasoningEffort: 'xhigh', + }, + launch: { + executable: 'python3', + args: ['/opt/tangle-candidate/runner.py'], + env: { PUBLIC_MODE: 'fixture' }, + flags: [], + cwd: '/app', + }, + memory: { mode: 'disabled' }, + trace: { + runId: executionId, + tags: {}, + env: { + TANGLE_CANDIDATE_EXECUTION_ID: executionId, + TANGLE_CANDIDATE_BUNDLE_DIGEST: bundleDigest, + TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST: planDigest, + TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST: receiptDigest, + TANGLE_TRACE_RUN_ID: executionId, + }, + }, + } as unknown as PreparedAgentCandidateExecution +} + +function request(execution: PreparedAgentCandidateExecution): AgentCandidateExecutorRequest { + const taskBytes = Buffer.from('not ready\n') + const profileBytes = Buffer.from('fixture profile\n') + return { + executionId: execution.executionId, + inputs: { + task: { + snapshot: { + material: { + files: [ + { + path: 'src/status.txt', + mode: 0o644, + sha256: digest(taskBytes), + byteLength: taskBytes.byteLength, + }, + ], + }, + }, + files: [{ path: 'src/status.txt', mode: 0o644, bytes: taskBytes }], + }, + profile: { + files: [{ path: 'AGENTS.md', mode: 0o644, bytes: profileBytes }], + }, + }, + roots: execution.roots.execution, + profilePlan: execution.profilePlan, + executionPlan: execution.executionPlan, + materializationReceipt: execution.materializationReceipt, + launch: { + ...execution.launch, + env: { + ...execution.launch.env, + OPENAI_API_KEY: 'evaluator-only', + ...execution.trace.env, + }, + }, + instruction: execution.instruction, + resolvedModel: execution.resolvedModel, + hardLimits: { timeoutMs: execution.executionPlan.value.material.limits.timeoutMs }, + observedLimits: { maxSteps: execution.executionPlan.value.material.limits.maxSteps }, + trace: execution.trace, + memory: execution.memory, + } as unknown as AgentCandidateExecutorRequest +} + +test('stages the runtime exact bytes without defining another execution plan', async () => { + const root = await mkdtemp(join(tmpdir(), 'pier-stage-')) + try { + const execution = prepared(root) + const staged = await stagePreparedPierCandidateExecution( + { + prepared: execution, + directory: join(root, 'sealed'), + pierVersion: '0.3.0', + }, + request(execution), + ) + assert.deepEqual(await readFile(staged.planPath), Buffer.from(execution.executionPlan.bytes)) + assert.deepEqual( + await readFile(staged.receiptPath), + Buffer.from(execution.materializationReceipt.bytes), + ) + assert.equal( + await readFile(join(staged.taskDirectory, 'src/status.txt'), 'utf8'), + 'not ready\n', + ) + assert.equal( + await readFile(join(staged.profileDirectory, 'AGENTS.md'), 'utf8'), + 'fixture profile\n', + ) + assert.deepEqual(staged.evaluatorEnv, { + ...execution.trace.env, + OPENAI_API_KEY: 'evaluator-only', + }) + assert.doesNotMatch(JSON.stringify(execution), /evaluator-only|OPENAI_API_KEY/) + assert.doesNotMatch(staged.agentArgs.join(' '), /evaluator-only|OPENAI_API_KEY/) + assert.deepEqual(staged.attemptArgs, ['--n-attempts', '1', '--max-retries', '0']) + assert.ok(staged.agentArgs.includes('pier_agents.tangle_candidate:TangleCandidateAgent')) + assert.ok(staged.agentArgs.includes('openai/gpt-5.4')) + assert.ok( + staged.agentArgs.includes( + `expected_receipt_digest=${execution.materializationReceipt.digest}`, + ), + ) + assert.ok(staged.agentArgs.includes(`task_dir=${staged.taskDirectory}`)) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +test('reads only identity and termination from Pier, never candidate-authored usage', () => { + const root = '/tmp/pier-capture-test' + const execution = prepared(root) + const metadata = { + executionId: execution.executionId, + bundleDigest: execution.bundle.digest, + executionPlanDigest: execution.executionPlan.value.digest, + materializationReceiptDigest: execution.materializationReceipt.digest, + termination: { kind: 'exit', exitCode: 0 }, + } + const capture = protectedCaptureFromPierResult(request(execution), { + exception_info: null, + agent_result: { + n_input_tokens: 999_999, + cost_usd: 999_999, + metadata, + }, + }) + assert.deepEqual(capture, { + executionId: execution.executionId, + termination: { kind: 'exit', exitCode: 0 }, + }) + assert.deepEqual( + protectedCaptureFromPierResult(request(execution), { + exception_info: { exception_type: 'AgentTimeoutError' }, + agent_result: { metadata: { ...metadata, termination: { kind: 'cancelled' } } }, + }), + { + executionId: execution.executionId, + termination: { kind: 'timeout', timeoutMs: 60_000 }, + }, + ) +}) + +test('rejects substituted canonical bytes before writing Pier artifacts', async () => { + const root = await mkdtemp(join(tmpdir(), 'pier-stage-tamper-')) + try { + const execution = prepared(root) + const tampered = { + ...execution, + executionPlan: { ...execution.executionPlan, bytes: Buffer.from('{}') }, + } as PreparedAgentCandidateExecution + await assert.rejects( + stagePreparedPierCandidateExecution( + { + prepared: tampered, + directory: join(root, 'sealed'), + pierVersion: '0.3.0', + }, + request(tampered), + ), + /do not match/, + ) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +test('rejects an executor request that changes signed public environment', async () => { + const root = await mkdtemp(join(tmpdir(), 'pier-stage-env-')) + try { + const execution = prepared(root) + const changed = request(execution) + ;(changed.launch as { env: Record }).env.PUBLIC_MODE = 'changed' + await assert.rejects( + stagePreparedPierCandidateExecution( + { + prepared: execution, + directory: join(root, 'sealed'), + pierVersion: '0.3.0', + }, + changed, + ), + /changed signed public environment/, + ) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +test('rejects changed or escaping executor files before launch', async () => { + for (const attack of ['bytes', 'path'] as const) { + const root = await mkdtemp(join(tmpdir(), `pier-stage-${attack}-`)) + try { + const execution = prepared(root) + const changed = request(execution) + const file = changed.inputs.task.files[0] as { path: string; bytes: Uint8Array } + if (attack === 'bytes') file.bytes = Buffer.from('changed\n') + else file.path = '../escape' + await assert.rejects( + stagePreparedPierCandidateExecution( + { + prepared: execution, + directory: join(root, 'sealed'), + pierVersion: '0.3.0', + }, + changed, + ), + attack === 'bytes' ? /differs from signed evidence/ : /canonical relative POSIX path/, + ) + } finally { + await rm(root, { recursive: true, force: true }) + } + } +}) + +test('waits for process and container death acknowledgement after abort', async () => { + const controller = new AbortController() + let terminated = false + const waiting = awaitAbortableTrial( + { + identity: { + executionId: 'abort-test', + executionPlanDigest: `sha256:${'a'.repeat(64)}`, + }, + result: new Promise(() => undefined), + terminateAndWait: async () => { + await new Promise((resolve) => setTimeout(resolve, 5)) + terminated = true + return { processExited: true, containersRemoved: true } + }, + }, + controller.signal, + ) + controller.abort(new Error('signed deadline elapsed')) + await assert.rejects(waiting, /signed deadline elapsed/) + assert.equal(terminated, true) +}) + +test('fails closed when termination does not acknowledge container removal', async () => { + const controller = new AbortController() + const waiting = awaitAbortableTrial( + { + identity: { + executionId: 'bad-ack-test', + executionPlanDigest: `sha256:${'b'.repeat(64)}`, + }, + result: new Promise(() => undefined), + terminateAndWait: async () => ({ processExited: true, containersRemoved: false }) as never, + }, + controller.signal, + ) + controller.abort() + await assert.rejects(waiting, /did not acknowledge/) +}) diff --git a/bench/src/pier-agent.ts b/bench/src/pier-agent.ts new file mode 100644 index 00000000..efd48343 --- /dev/null +++ b/bench/src/pier-agent.ts @@ -0,0 +1,631 @@ +/** Pier transport for a runtime-prepared immutable candidate execution. */ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { chmod, lstat, mkdir, realpath, writeFile } from 'node:fs/promises' +import { dirname, isAbsolute, join, resolve } from 'node:path' + +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateExecutionClaimStore, + AgentCandidateExecutorRequest, + AgentCandidateExecutorStopRequest, + AgentCandidateExecutorPort, + AgentCandidateOutputArtifactPort, + AgentCandidateProtectedRunCapture, + AgentCandidateRunFinalization, + PreparedAgentCandidateExecution, +} from '@tangle-network/agent-runtime' +import { executePreparedAgentCandidate } from '@tangle-network/agent-runtime' +import type { TraceStore } from '@tangle-network/agent-eval' + +import { capturePierTaskOutcome } from './pier-task-outcome' + +const adapterImportPath = 'pier_agents.tangle_candidate:TangleCandidateAgent' +const sha256Pattern = /^sha256:[a-f0-9]{64}$/ + +interface StagePreparedPierCandidateOptions { + readonly prepared: PreparedAgentCandidateExecution + /** Evaluator-owned directory persisted with the Pier trial. */ + readonly directory: string + /** Exact Pier package version pinned by the experiment contract. */ + readonly pierVersion: string +} + +export interface StagedPierCandidateExecution { + readonly executionId: string + readonly directory: string + readonly taskDirectory: string + readonly candidateDirectory?: string + readonly profileDirectory: string + readonly planPath: string + readonly receiptPath: string + readonly agentArgs: readonly string[] + /** Executor-only model and trace bindings; never present on the prepared object or disk. */ + readonly evaluatorEnv: Readonly> + /** One prepared execution is exactly one Pier trial attempt. */ + readonly attemptArgs: readonly ['--n-attempts', '1', '--max-retries', '0'] +} + +export interface PierCandidateTerminationAcknowledgement { + /** The Pier process has exited and has been reaped. */ + readonly processExited: true + /** Every task container created for this one trial has been removed. */ + readonly containersRemoved: true +} + +export interface PierCandidateTrialHandle { + /** Non-secret durable identity shared with a fresh evaluator process. */ + readonly identity: PierCandidateTrialIdentity + /** Resolves only after the Pier process exits and its task container is gone. */ + readonly result: Promise + /** Idempotently kill/reap Pier and remove its task container, then acknowledge their death. */ + readonly terminateAndWait: () => Promise +} + +export type PierCandidateTrialIdentity = Readonly + +/** + * Evaluator-owned lifecycle whose stop path works without the process-local + * handle returned by `start`. + */ +export interface PierCandidateTrialController { + start( + staged: StagedPierCandidateExecution, + context: { + readonly request: AgentCandidateExecutorRequest + readonly traceStore: TraceStore + readonly signal: AbortSignal + readonly deadlineAtMs: number + }, + ): PierCandidateTrialHandle + terminateAndWait( + identity: PierCandidateTrialIdentity, + ): Promise +} + +/** Evaluator-owned bytes captured from one completed official Pier trial. */ +export interface PierCandidateTrialResult { + /** Parsed value of the exact `result.json` bytes. */ + readonly value: unknown + /** Exact official `result.json` bytes used as grader evidence. */ + readonly resultBytes: Uint8Array + /** Exact `/logs/artifacts/model.patch` bytes emitted before official tests. */ + readonly taskPatch: Uint8Array +} + +export interface PierCandidateOfficialResult { + readonly value: unknown + readonly bytes: Uint8Array +} + +type RuntimeGraderInput = Parameters[0] +type RuntimeGraderResult = Awaited> + +/** Executes the exact admitted grader bytes against the official Pier result. */ +export interface PierCandidateGraderPort { + readonly name: string + readonly version: string + readonly artifact: AgentCandidateBenchmarkGraderPort['artifact'] + run(input: RuntimeGraderInput & { + readonly officialResult: PierCandidateOfficialResult + }): Promise +} + +export interface ExecutePreparedPierCandidateOptions extends StagePreparedPierCandidateOptions { + readonly traceStore: TraceStore + /** Durable one-shot store shared by every process capable of running this benchmark. */ + readonly claimStore: AgentCandidateExecutionClaimStore + readonly outputArtifacts: AgentCandidateOutputArtifactPort + readonly grader: PierCandidateGraderPort + /** + * Starts exactly one Pier trial synchronously and persists its non-secret + * process/container identity before returning. + */ + readonly controller: PierCandidateTrialController +} + +/** Recovery-only runtime executor for an expired attempt owned by another process. */ +export function createPierCandidateRecoveryExecutor( + controller: PierCandidateTrialController, +): AgentCandidateExecutorPort { + return { + execute: async () => { + throw new Error('recovery-only Pier executor cannot start a candidate') + }, + stopAndCapture: async (request) => { + assertTerminationAcknowledged( + await controller.terminateAndWait({ + executionId: request.executionId, + executionPlanDigest: request.executionPlanDigest, + }), + ) + return { stopped: true } + }, + } +} + +interface PierResultLike { + exception_info?: unknown + agent_result?: unknown +} + +function sha256(bytes: Uint8Array): `sha256:${string}` { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + +function nonEmpty(value: string, label: string): string { + if (!value || value.includes('\0')) throw new Error(`${label} must be non-empty without NUL`) + return value +} + +function absoluteHostPath(value: string, label: string): string { + if (!isAbsolute(value)) throw new Error(`${label} must be an absolute host path`) + return resolve(value) +} + +function digest(value: string, label: string): string { + if (!sha256Pattern.test(value)) throw new Error(`${label} is not a SHA-256 digest`) + return value +} + +function record(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + return value as Record +} + +function parseExactJson(bytes: Uint8Array, label: string): unknown { + try { + return JSON.parse(Buffer.from(bytes).toString('utf8')) + } catch (error) { + throw new Error(`${label} bytes are not UTF-8 JSON`, { cause: error }) + } +} + +function safeRelativePath(value: string, label: string): string { + const parts = value.split('/') + if ( + !value || + value.includes('\0') || + value.includes('\\') || + isAbsolute(value) || + parts.some((part) => !part || part === '.' || part === '..') + ) { + throw new Error(`${label} must be a canonical relative POSIX path`) + } + return value +} + +async function assertRealDirectory(path: string, label: string): Promise { + const stats = await lstat(path) + if (!stats.isDirectory() || stats.isSymbolicLink()) { + throw new Error(`${label} must be a real directory`) + } + if ((await realpath(path)) !== path) { + throw new Error(`${label} has a symlinked path component`) + } +} + +interface ExpectedExecutorFile { + readonly path: string + readonly mode: number + readonly sha256: string + readonly byteLength?: number +} + +async function materializeExecutorFiles( + root: string, + files: AgentCandidateExecutorRequest['inputs']['profile']['files'], + expected: readonly ExpectedExecutorFile[], + label: string, +): Promise { + const expectedByPath = new Map(expected.map((file) => [file.path, file])) + if (expectedByPath.size !== expected.length || files.length !== expected.length) { + throw new Error(`${label} file set differs from signed evidence`) + } + + await mkdir(root, { mode: 0o700 }) + await assertRealDirectory(root, label) + for (const file of files) { + const relative = safeRelativePath(file.path, `${label} file path`) + const identity = expectedByPath.get(relative) + if (!identity) throw new Error(`${label} contains unsigned file ${relative}`) + if ( + file.mode !== identity.mode || + sha256(file.bytes) !== identity.sha256 || + (identity.byteLength !== undefined && file.bytes.byteLength !== identity.byteLength) + ) { + throw new Error(`${label} file ${relative} differs from signed evidence`) + } + + const target = join(root, relative) + const parent = dirname(target) + if (parent !== root) await mkdir(parent, { recursive: true, mode: 0o700 }) + await writeFile(target, file.bytes, { flag: 'wx', mode: 0o600 }) + await chmod(target, file.mode) + const stats = await lstat(target) + if ( + !stats.isFile() || + stats.isSymbolicLink() || + stats.nlink !== 1 || + (stats.mode & 0o777) !== file.mode + ) { + throw new Error(`${label} file ${relative} is not a singly-linked regular file`) + } + } +} + +function protectedEnvironment( + prepared: PreparedAgentCandidateExecution, + request: AgentCandidateExecutorRequest, +): Readonly> { + const protectedEntries: Array = [] + for (const [name, value] of Object.entries(request.launch.env)) { + const publicValue = prepared.launch.env[name] + if (publicValue === undefined) { + protectedEntries.push([name, value]) + } else if (publicValue !== value) { + throw new Error(`executor request changed signed public environment ${name}`) + } + } + for (const name of Object.keys(prepared.launch.env)) { + if (!(name in request.launch.env)) { + throw new Error(`executor request omitted signed public environment ${name}`) + } + } + return Object.freeze(Object.fromEntries(protectedEntries)) +} + +/** + * Persist the runtime's exact canonical bytes and return only Pier transport + * arguments. No benchmark-specific plan or candidate-authored telemetry exists. + */ +/** @internal Package-private transport seam exercised by adapter tests. */ +export async function stagePreparedPierCandidateExecution( + options: StagePreparedPierCandidateOptions, + request: AgentCandidateExecutorRequest, +): Promise { + const { prepared } = options + nonEmpty(options.pierVersion, 'pierVersion') + const directory = absoluteHostPath(options.directory, 'directory') + await mkdir(directory, { mode: 0o700 }) + await assertRealDirectory(directory, 'directory') + if (request.memory.mode !== 'disabled') { + throw new Error('Pier transport does not yet implement protected isolated memory') + } + if (request.knowledge !== undefined) { + throw new Error('Pier transport does not yet implement immutable knowledge mounts') + } + const taskDirectory = join(directory, 'task') + const candidateDirectory = request.inputs.candidate ? join(directory, 'candidate') : undefined + const profileDirectory = join(directory, 'profile') + await materializeExecutorFiles( + taskDirectory, + request.inputs.task.files, + request.inputs.task.snapshot.material.files, + 'task executor input', + ) + if (request.inputs.candidate && candidateDirectory) { + await materializeExecutorFiles( + candidateDirectory, + request.inputs.candidate.files, + request.inputs.candidate.snapshot.material.files, + 'candidate executor input', + ) + } + await materializeExecutorFiles( + profileDirectory, + request.inputs.profile.files, + request.profilePlan.value.material.files.map((file) => ({ + path: file.relPath, + mode: file.mode, + sha256: file.contentSha256, + })), + 'profile executor input', + ) + + const planDigest = digest(request.executionPlan.value.digest, 'execution plan digest') + if (sha256(request.executionPlan.bytes) !== planDigest) { + throw new Error('prepared execution-plan bytes do not match their runtime digest') + } + assert.deepEqual( + parseExactJson(request.executionPlan.bytes, 'execution plan'), + request.executionPlan.value.material, + 'prepared execution-plan bytes differ from runtime material', + ) + + const receiptDigest = digest( + request.materializationReceipt.digest, + 'materialization receipt digest', + ) + if (sha256(request.materializationReceipt.bytes) !== receiptDigest) { + throw new Error('prepared materialization-receipt bytes do not match their runtime digest') + } + const { digest: _receiptDigest, ...receiptMaterial } = request.materializationReceipt.value + assert.deepEqual( + parseExactJson(request.materializationReceipt.bytes, 'materialization receipt'), + receiptMaterial, + 'prepared receipt bytes differ from runtime material', + ) + + const planPath = join(directory, 'execution-plan.json') + const receiptPath = join(directory, 'materialization-receipt.json') + await Promise.all([ + writeFile(planPath, request.executionPlan.bytes, { mode: 0o600, flag: 'wx' }), + writeFile(receiptPath, request.materializationReceipt.bytes, { mode: 0o600, flag: 'wx' }), + ]) + + const agentArgs = [ + '--agent-import-path', + adapterImportPath, + '--model', + nonEmpty(request.resolvedModel.requested, 'resolved requested model'), + '--agent-kwarg', + `plan_path=${planPath}`, + '--agent-kwarg', + `receipt_path=${receiptPath}`, + '--agent-kwarg', + `expected_receipt_digest=${receiptDigest}`, + '--agent-kwarg', + `trace_run_id=${nonEmpty(request.trace.runId, 'trace run id')}`, + '--agent-kwarg', + `task_dir=${taskDirectory}`, + '--agent-kwarg', + `profile_dir=${profileDirectory}`, + '--agent-kwarg', + `pier_version=${options.pierVersion}`, + ] + if (candidateDirectory !== undefined) { + agentArgs.push('--agent-kwarg', `candidate_dir=${candidateDirectory}`) + } + + return { + executionId: request.executionId, + directory, + taskDirectory, + ...(candidateDirectory ? { candidateDirectory } : {}), + profileDirectory, + planPath, + receiptPath, + agentArgs, + evaluatorEnv: protectedEnvironment(prepared, request), + attemptArgs: ['--n-attempts', '1', '--max-retries', '0'], + } +} + +function exceptionType(value: unknown): string | undefined { + const info = value && typeof value === 'object' ? (value as Record) : undefined + for (const key of ['exception_type', 'type', 'name']) { + if (typeof info?.[key] === 'string') return info[key] + } + return undefined +} + +/** Read only identity and termination from Pier; usage always comes from agent-eval. */ +/** @internal Package-private result parser exercised by adapter tests. */ +export function protectedCaptureFromPierResult( + request: AgentCandidateExecutorRequest, + value: unknown, +): AgentCandidateProtectedRunCapture { + const result = record(value, 'Pier trial result') as PierResultLike + const agentResult = record(result.agent_result, 'Pier agent_result') + const metadata = record(agentResult.metadata, 'Pier agent_result.metadata') + const expected = { + executionId: request.executionId, + bundleDigest: request.executionPlan.value.material.bundleDigest, + executionPlanDigest: request.executionPlan.value.digest, + materializationReceiptDigest: request.materializationReceipt.digest, + } + for (const [name, identity] of Object.entries(expected)) { + if (metadata[name] !== identity) { + throw new Error(`Pier result ${name} does not match the prepared execution`) + } + } + + const reported = record(metadata.termination, 'Pier termination') + const type = exceptionType(result.exception_info) + if (type?.includes('AgentTimeout')) { + return { + executionId: request.executionId, + termination: { + kind: 'timeout', + timeoutMs: request.hardLimits.timeoutMs, + }, + } + } + if (type?.includes('Cancelled') || reported.kind === 'cancelled') { + return { executionId: request.executionId, termination: { kind: 'cancelled' } } + } + if (reported.kind === 'exit' && Number.isInteger(reported.exitCode)) { + return { + executionId: request.executionId, + termination: { kind: 'exit', exitCode: reported.exitCode as number }, + } + } + throw new Error(`Pier result has no recognized truthful termination${type ? ` (${type})` : ''}`) +} + +function sealPierTrialResult(value: PierCandidateTrialResult): PierCandidateTrialResult { + if (!(value.resultBytes instanceof Uint8Array) || !(value.taskPatch instanceof Uint8Array)) { + throw new Error('Pier trial result must contain raw result and patch bytes') + } + const parsed = parseExactJson(value.resultBytes, 'Pier result.json') + assert.deepEqual(parsed, value.value, 'Pier parsed result differs from result.json bytes') + const resultBytes = Uint8Array.from(value.resultBytes) + const taskPatch = Uint8Array.from(value.taskPatch) + return Object.freeze({ + value: parsed, + get resultBytes(): Uint8Array { + return Uint8Array.from(resultBytes) + }, + get taskPatch(): Uint8Array { + return Uint8Array.from(taskPatch) + }, + }) +} + +function abortReason(signal: AbortSignal): Error { + const reason = signal.reason + if (reason instanceof Error) return reason + return new Error(`Pier candidate execution aborted${reason ? `: ${String(reason)}` : ''}`) +} + +function assertTerminationAcknowledged( + acknowledgement: PierCandidateTerminationAcknowledgement, +): void { + if (acknowledgement.processExited !== true || acknowledgement.containersRemoved !== true) { + throw new Error('Pier termination did not acknowledge process and container death') + } +} + +/** @internal Package-private cancellation seam exercised by adapter tests. */ +export async function awaitAbortableTrial( + trial: Omit & { readonly result: Promise }, + signal: AbortSignal, +): Promise { + if (signal.aborted) { + assertTerminationAcknowledged(await trial.terminateAndWait()) + throw abortReason(signal) + } + + return await new Promise((resolveResult, reject) => { + let settled = false + let aborting = false + const settle = (callback: () => void): void => { + if (settled) return + settled = true + signal.removeEventListener('abort', onAbort) + callback() + } + const onAbort = (): void => { + aborting = true + void trial.terminateAndWait().then( + (acknowledgement) => { + try { + assertTerminationAcknowledged(acknowledgement) + settle(() => reject(abortReason(signal))) + } catch (error) { + settle(() => reject(error)) + } + }, + (error) => settle(() => reject(new Error('Pier termination failed', { cause: error }))), + ) + } + signal.addEventListener('abort', onAbort, { once: true }) + trial.result.then( + (result) => { + if (!aborting && !signal.aborted) settle(() => resolveResult(result)) + }, + (error) => { + if (!aborting && !signal.aborted) settle(() => reject(error)) + }, + ) + }) +} + +/** Execute and finalize through the runtime's only gradable candidate path. */ +export async function executePreparedPierCandidate( + options: ExecutePreparedPierCandidateOptions, +): Promise { + const trials = new Map< + string, + { readonly handle: PierCandidateTrialHandle; result?: PierCandidateTrialResult } + >() + const officialResults = new Map() + const trialIdentity = (executionId: string, executionPlanDigest: string): string => + JSON.stringify([executionId, executionPlanDigest]) + const executor: AgentCandidateExecutorPort = { + execute: async (request, context) => { + context.signal.throwIfAborted() + const staged = await stagePreparedPierCandidateExecution(options, request) + context.signal.throwIfAborted() + const identity = trialIdentity(request.executionId, request.executionPlan.value.digest) + if (trials.has(identity)) throw new Error('Pier trial identity is already active') + const trial = options.controller.start(staged, { + request, + traceStore: context.traceStore, + signal: context.signal, + deadlineAtMs: context.deadlineAtMs, + }) + if ( + trial.identity.executionId !== request.executionId || + trial.identity.executionPlanDigest !== request.executionPlan.value.digest + ) { + assertTerminationAcknowledged(await trial.terminateAndWait()) + throw new Error('Pier controller returned a different durable trial identity') + } + const active = { handle: trial } as { + readonly handle: PierCandidateTrialHandle + result?: PierCandidateTrialResult + } + trials.set(identity, active) + const result = sealPierTrialResult(await awaitAbortableTrial(trial, context.signal)) + active.result = result + officialResults.set(request.executionId, result) + return protectedCaptureFromPierResult(request, result.value) + }, + stopAndCapture: async (request) => { + const identity = trialIdentity(request.executionId, request.executionPlanDigest) + const active = trials.get(identity) + assertTerminationAcknowledged( + await options.controller.terminateAndWait({ + executionId: request.executionId, + executionPlanDigest: request.executionPlanDigest, + }), + ) + if (!active) return { stopped: true } + let result = active.result + if (!result) { + try { + result = sealPierTrialResult(await active.handle.result) + } catch { + result = undefined + } + } + trials.delete(identity) + if (!result) return { stopped: true } + officialResults.set(request.executionId, result) + const repository = options.prepared.executionPlan.value.material.task.repository + const taskOutcome = await capturePierTaskOutcome({ + repositoryRoot: options.prepared.roots.staging.taskRoot, + baseCommit: repository.baseCommit, + baseTree: repository.baseTree, + patch: result.taskPatch, + artifactPersistence: { + executionId: request.executionId, + outputArtifacts: options.outputArtifacts, + }, + }) + return { stopped: true, taskOutcome } + }, + } + const grader: AgentCandidateBenchmarkGraderPort = { + name: options.grader.name, + version: options.grader.version, + artifact: options.grader.artifact, + run: async (input) => { + const official = officialResults.get(input.executionId) + if (!official) throw new Error('Pier official result is missing for executable grading') + return await options.grader.run({ + ...input, + officialResult: { + value: parseExactJson(official.resultBytes, 'Pier result.json'), + bytes: Uint8Array.from(official.resultBytes), + }, + }) + }, + } + try { + return await executePreparedAgentCandidate(options.prepared, { + executor, + grader, + outputArtifacts: options.outputArtifacts, + traceStore: options.traceStore, + claimStore: options.claimStore, + }) + } finally { + officialResults.clear() + trials.clear() + } +} diff --git a/bench/src/pier-result-grader.mjs b/bench/src/pier-result-grader.mjs new file mode 100644 index 00000000..388cab59 --- /dev/null +++ b/bench/src/pier-result-grader.mjs @@ -0,0 +1,30 @@ +/** Executable parser for official Pier `result.json` evidence. Reads JSON on stdin. */ +const chunks = [] +for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk)) +const result = JSON.parse(Buffer.concat(chunks).toString('utf8')) +const rewards = result?.verifier_result?.rewards +if (!rewards || typeof rewards !== 'object' || Array.isArray(rewards)) { + throw new Error('official Pier result omitted verifier_result.rewards') +} +const dimensions = {} +for (const name of Object.keys(rewards).sort()) { + const value = rewards[name] + if (!/^[a-z0-9]+(?:[._-][a-z0-9]+)*$/.test(name)) { + throw new Error(`official Pier reward name is not normalized: ${name}`) + } + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`official Pier reward ${name} is outside [0, 1]`) + } + dimensions[name] = value +} +if (typeof dimensions.reward !== 'number') { + throw new Error('official Pier result omitted the reward dimension') +} +process.stdout.write( + JSON.stringify({ + score: dimensions.reward, + passed: dimensions.reward === 1, + dimensions, + raw: {}, + }), +) diff --git a/bench/src/pier-result-grader.test.mts b/bench/src/pier-result-grader.test.mts new file mode 100644 index 00000000..cd40afd4 --- /dev/null +++ b/bench/src/pier-result-grader.test.mts @@ -0,0 +1,62 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import test from 'node:test' + +import type { AgentCandidateArtifactRef } from '@tangle-network/agent-interface' + +import { createPierResultGrader } from './pier-result-grader' + +const digest = (bytes: Uint8Array): `sha256:${string}` => + `sha256:${createHash('sha256').update(bytes).digest('hex')}` + +const artifact: AgentCandidateArtifactRef = { + locator: { kind: 's3', bucket: 'pier-graders', key: 'pier-result-grader.mjs' }, + sha256: `sha256:${'a'.repeat(64)}`, + byteLength: 1, +} + +test('executes admitted Pier grader bytes over exact official result evidence', async () => { + const implementation = await readFile(new URL('./pier-result-grader.mjs', import.meta.url)) + const resultBytes = Buffer.from( + JSON.stringify({ verifier_result: { rewards: { reward: 1, patch_applied: 1 } } }), + ) + const grader = createPierResultGrader({ name: 'pier-result', version: '1', artifact }) + const output = await grader.run({ + executionId: 'execution-1', + termination: { kind: 'exit', exitCode: 0 }, + outcome: { evidence: { digest: `sha256:${'b'.repeat(64)}` } } as never, + implementation: { byteLength: implementation.byteLength, bytes: implementation }, + signal: new AbortController().signal, + officialResult: { value: JSON.parse(resultBytes.toString('utf8')), bytes: resultBytes }, + }) + + assert.deepEqual(output.evaluation, { + score: 1, + passed: true, + dimensions: { patch_applied: 1, reward: 1 }, + raw: {}, + }) + assert.deepEqual(Buffer.from(output.evidence), resultBytes) + assert.equal(output.binding.implementationDigest, digest(implementation)) + assert.equal(output.binding.outputDigest, digest(resultBytes)) +}) + +test('fails closed for missing official rewards and pre-aborted execution', async () => { + const implementation = await readFile(new URL('./pier-result-grader.mjs', import.meta.url)) + const grader = createPierResultGrader({ name: 'pier-result', version: '1', artifact }) + const base = { + executionId: 'execution-1', + termination: { kind: 'exit', exitCode: 0 } as const, + outcome: { evidence: { digest: `sha256:${'b'.repeat(64)}` } } as never, + implementation: { byteLength: implementation.byteLength, bytes: implementation }, + officialResult: { value: {}, bytes: Buffer.from('{}') }, + } + await assert.rejects( + grader.run({ ...base, signal: new AbortController().signal }), + /omitted verifier_result.rewards/, + ) + const controller = new AbortController() + controller.abort(new Error('deadline')) + await assert.rejects(grader.run({ ...base, signal: controller.signal }), /deadline/) +}) diff --git a/bench/src/pier-result-grader.ts b/bench/src/pier-result-grader.ts new file mode 100644 index 00000000..2ce52154 --- /dev/null +++ b/bench/src/pier-result-grader.ts @@ -0,0 +1,108 @@ +/** Execute the exact admitted Pier result parser bytes in a fresh Node process. */ +import { spawn } from 'node:child_process' +import { createHash } from 'node:crypto' +import { chmod, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { BenchmarkEvaluation } from '@tangle-network/agent-eval' + +import type { PierCandidateGraderPort } from './pier-agent' + +const sha256 = (bytes: Uint8Array): `sha256:${string}` => + `sha256:${createHash('sha256').update(bytes).digest('hex')}` + +export function createPierResultGrader( + descriptor: Pick, +): PierCandidateGraderPort { + return { + ...descriptor, + run: async (input) => { + input.signal.throwIfAborted() + const implementation = Uint8Array.from(input.implementation.bytes) + if (implementation.byteLength !== input.implementation.byteLength) { + throw new Error('Pier grader implementation length changed before execution') + } + const evidence = Uint8Array.from(input.officialResult.bytes) + const evaluation = await executeGrader(implementation, evidence, input.signal) + input.signal.throwIfAborted() + return { + evaluation, + evidence, + binding: { + implementationDigest: sha256(implementation), + taskOutcomeDigest: input.outcome.evidence.digest, + outputDigest: sha256(evidence), + }, + } + }, + } +} + +async function executeGrader( + implementation: Uint8Array, + resultBytes: Uint8Array, + signal: AbortSignal, +): Promise { + const directory = await mkdtemp(join(tmpdir(), 'pier-result-grader-')) + const script = join(directory, 'grader.mjs') + try { + await writeFile(script, implementation, { flag: 'wx', mode: 0o600 }) + await chmod(script, 0o500) + const stdout = await runNode(script, resultBytes, signal) + const parsed = JSON.parse(stdout.toString('utf8')) as BenchmarkEvaluation + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('Pier result grader returned a non-object evaluation') + } + return parsed + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +async function runNode( + script: string, + input: Uint8Array, + signal: AbortSignal, +): Promise { + signal.throwIfAborted() + return await new Promise((resolveResult, reject) => { + const child = spawn(process.execPath, ['--no-warnings', script], { + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + LANG: 'C', + LC_ALL: 'C', + }, + stdio: ['pipe', 'pipe', 'pipe'], + signal, + }) + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + let bytes = 0 + const collect = (target: Buffer[]) => (chunk: Buffer) => { + bytes += chunk.byteLength + if (bytes > 1024 * 1024) { + child.kill('SIGKILL') + reject(new Error('Pier result grader exceeded its output limit')) + return + } + target.push(Buffer.from(chunk)) + } + child.stdout.on('data', collect(stdout)) + child.stderr.on('data', collect(stderr)) + child.once('error', reject) + child.once('close', (code, childSignal) => { + if (code !== 0) { + reject( + new Error( + `Pier result grader failed (${childSignal ?? code}): ${Buffer.concat(stderr).toString('utf8').trim()}`, + ), + ) + return + } + resolveResult(Buffer.concat(stdout)) + }) + child.stdin.end(input) + }) +} diff --git a/bench/src/pier-task-outcome.test.mts b/bench/src/pier-task-outcome.test.mts new file mode 100644 index 00000000..2c6bcef6 --- /dev/null +++ b/bench/src/pier-task-outcome.test.mts @@ -0,0 +1,144 @@ +import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' + +import { + type AgentCandidateOutputArtifactPort, + captureAgentCandidateWorkspaceFiles, + createAgentCandidateWorkspacePort, +} from '@tangle-network/agent-runtime/candidate-execution' + +import { capturePierTaskOutcome } from './pier-task-outcome' + +test('Pier task outcome reconstructs the exact tree and deterministic archive from a patch', async () => { + const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-')) + try { + mkdirSync(join(root, 'src')) + writeFileSync(join(root, 'src/status.txt'), 'not-ready\n') + chmodSync(join(root, 'src/status.txt'), 0o644) + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'fixture@example.com']) + git(root, ['config', 'user.name', 'Fixture']) + git(root, ['add', '-A']) + git(root, ['commit', '-m', 'base']) + const baseCommit = git(root, ['rev-parse', 'HEAD']) + const baseTree = git(root, ['rev-parse', 'HEAD^{tree}']) + writeFileSync(join(root, 'src/status.txt'), 'ready\n') + const patch = execFileSync('git', ['-C', root, 'diff', '--binary']) + git(root, ['restore', '.']) + + const captured = await capturePierTaskOutcome({ repositoryRoot: root, baseCommit, baseTree, patch }) + assert.notEqual(captured.resultTree, baseTree) + assert.deepEqual(Buffer.from(captured.gitDiff), patch) + const expected = await captureAgentCandidateWorkspaceFiles([ + { path: 'src/status.txt', mode: 0o644, bytes: Buffer.from('ready\n') }, + ]) + assert.deepEqual(captured.afterState, expected.snapshot.material) + assert.deepEqual(Buffer.from(captured.archive), Buffer.from(expected.archive)) + const destination = join(root, 'materialized') + await createAgentCandidateWorkspacePort().materialize({ + role: 'candidate', + archive: captured.archive, + snapshot: expected.snapshot, + destination, + }) + assert.equal(readFileSync(join(destination, 'src/status.txt'), 'utf8'), 'ready\n') + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('Pier task outcome preserves the signed base for an empty patch', async () => { + const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-')) + try { + writeFileSync(join(root, 'README.md'), 'base\n') + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'fixture@example.com']) + git(root, ['config', 'user.name', 'Fixture']) + git(root, ['add', '-A']) + git(root, ['commit', '-m', 'base']) + const baseCommit = git(root, ['rev-parse', 'HEAD']) + const baseTree = git(root, ['rev-parse', 'HEAD^{tree}']) + + const captured = await capturePierTaskOutcome({ + repositoryRoot: root, + baseCommit, + baseTree, + patch: Buffer.alloc(0), + }) + assert.equal(captured.resultTree, baseTree) + assert.equal(captured.gitDiff.byteLength, 0) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('Pier task outcome externalizes workspace evidence above the embedded limit', async () => { + const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-')) + try { + const payload = Buffer.alloc(1024 * 1024 + 1, 7) + writeFileSync(join(root, 'large.bin'), payload) + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'fixture@example.com']) + git(root, ['config', 'user.name', 'Fixture']) + git(root, ['add', '-A']) + git(root, ['commit', '-m', 'base']) + const baseCommit = git(root, ['rev-parse', 'HEAD']) + const baseTree = git(root, ['rev-parse', 'HEAD^{tree}']) + const stored = new Map() + const outputArtifacts: AgentCandidateOutputArtifactPort = { + put: async ({ bytes, purpose }) => { + const detached = Uint8Array.from(bytes) + const digest = sha256(detached) + stored.set(digest, detached) + return { + locator: { + kind: 's3', + bucket: 'pier-test-artifacts', + key: `${purpose}/${digest}`, + }, + sha256: digest, + byteLength: detached.byteLength, + } + }, + read: async (ref) => Uint8Array.from(stored.get(ref.sha256) ?? []), + } + + const captured = await capturePierTaskOutcome({ + repositoryRoot: root, + baseCommit, + baseTree, + patch: Buffer.alloc(0), + artifactPersistence: { executionId: 'pier-large-1', outputArtifacts }, + }) + + assert.ok(captured.archive.byteLength > 1024 * 1024) + assert.equal(captured.afterState.files[0]?.byteLength, payload.byteLength) + assert.deepEqual(Buffer.from(stored.get(sha256(captured.archive)) ?? []), captured.archive) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +function sha256(bytes: Uint8Array): `sha256:${string}` { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + +function git(root: string, args: string[]): string { + return execFileSync('git', ['-C', root, ...args], { + encoding: 'utf8', + env: { + ...process.env, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_NO_REPLACE_OBJECTS: '1', + LC_ALL: 'C', + }, + }).trim() +} diff --git a/bench/src/pier-task-outcome.ts b/bench/src/pier-task-outcome.ts new file mode 100644 index 00000000..9ae8b4c1 --- /dev/null +++ b/bench/src/pier-task-outcome.ts @@ -0,0 +1,251 @@ +/** Reconstruct a runtime task outcome from Pier's evaluator-captured binary patch. */ +import { spawn } from 'node:child_process' +import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +import { + type AgentCandidateExecutorTaskOutcomeCapture, + type AgentCandidateOutputArtifactPort, + captureAgentCandidateWorkspaceFiles, +} from '@tangle-network/agent-runtime/candidate-execution' + +interface GitResult { + readonly stdout: Buffer + readonly stderr: Buffer +} + +/** Apply the official patch in an isolated Git object directory and capture its exact tree. */ +export async function capturePierTaskOutcome(input: { + readonly repositoryRoot: string + readonly baseCommit: string + readonly baseTree: string + readonly patch: Uint8Array + readonly signal?: AbortSignal + readonly artifactPersistence?: { + readonly executionId: string + readonly outputArtifacts: AgentCandidateOutputArtifactPort + } +}): Promise { + input.signal?.throwIfAborted() + const repositoryRoot = resolve(input.repositoryRoot) + const head = (await git(repositoryRoot, ['rev-parse', 'HEAD'], undefined, {}, input.signal)).stdout + .toString('utf8') + .trim() + const headTree = ( + await git(repositoryRoot, ['rev-parse', 'HEAD^{tree}'], undefined, {}, input.signal) + ).stdout + .toString('utf8') + .trim() + if (head !== input.baseCommit || headTree !== input.baseTree) { + throw new Error('Pier task outcome repository drifted from its signed base') + } + const gitDir = resolve( + ( + await git(repositoryRoot, ['rev-parse', '--absolute-git-dir'], undefined, {}, input.signal) + ).stdout + .toString('utf8') + .trim(), + ) + if ((await realpath(gitDir)) !== gitDir || gitDir.includes(':')) { + throw new Error('Pier task outcome Git directory is unsupported') + } + const replacements = ( + await git( + repositoryRoot, + ['for-each-ref', '--format=%(refname)', 'refs/replace'], + undefined, + {}, + input.signal, + ) + ).stdout + .toString('utf8') + .trim() + if (replacements) throw new Error('Pier task outcome repository contains Git replace refs') + + const temporary = await mkdtemp(join(tmpdir(), 'pier-task-outcome-')) + try { + const objectDirectory = join(temporary, 'objects') + const indexFile = join(temporary, 'index') + await mkdir(objectDirectory) + const environment = { + GIT_INDEX_FILE: indexFile, + GIT_OBJECT_DIRECTORY: objectDirectory, + GIT_ALTERNATE_OBJECT_DIRECTORIES: join(gitDir, 'objects'), + } + await git(repositoryRoot, ['read-tree', input.baseTree], undefined, environment, input.signal) + const patch = Uint8Array.from(input.patch) + if (patch.byteLength > 0) { + await git( + repositoryRoot, + ['apply', '--cached', '--binary', '--whitespace=nowarn', '-'], + patch, + environment, + input.signal, + ) + } + const resultTree = ( + await git(repositoryRoot, ['write-tree'], undefined, environment, input.signal) + ).stdout + .toString('utf8') + .trim() + const entries = parseTreeEntries( + ( + await git( + repositoryRoot, + ['ls-tree', '-rz', '--full-tree', resultTree], + undefined, + environment, + input.signal, + ) + ).stdout, + ) + if (entries.length === 0) throw new Error('Pier task outcome tree cannot be empty') + const files = await Promise.all( + entries.map(async (entry) => { + if (entry.type !== 'blob' || (entry.mode !== '100644' && entry.mode !== '100755')) { + throw new Error(`Pier task outcome contains a non-regular entry: ${entry.path}`) + } + return { + path: safeGitPath(entry.path), + mode: entry.mode === '100755' ? (0o755 as const) : (0o644 as const), + bytes: ( + await git( + repositoryRoot, + ['cat-file', 'blob', entry.object], + undefined, + environment, + input.signal, + ) + ).stdout, + } + }), + ) + // The executor outcome must still return exact bytes; the runtime verifies + // and persists its final task references after this capture completes. + const captured = await captureAgentCandidateWorkspaceFiles(files, { + ...(input.artifactPersistence + ? { + artifactPersistence: { + ...input.artifactPersistence, + ...(input.signal ? { signal: input.signal } : {}), + }, + } + : {}), + }) + return { + resultTree, + afterState: captured.snapshot.material, + archive: captured.archive, + gitDiff: patch, + } + } finally { + await rm(temporary, { recursive: true, force: true }) + } +} + +function parseTreeEntries(bytes: Uint8Array): Array<{ + readonly mode: string + readonly type: string + readonly object: string + readonly path: string +}> { + const raw = Buffer.from(bytes) + const decoded = raw.toString('utf8') + if (!Buffer.from(decoded, 'utf8').equals(raw)) { + throw new Error('Pier task outcome contains a non-UTF-8 path') + } + return decoded + .split('\0') + .filter(Boolean) + .map((row) => { + const tab = row.indexOf('\t') + const [mode, type, object] = row.slice(0, tab).split(' ') + const path = row.slice(tab + 1) + if (tab < 1 || !mode || !type || !object || !path) { + throw new Error('Pier task outcome contains a malformed Git entry') + } + return { mode, type, object, path } + }) +} + +function safeGitPath(path: string): string { + const parts = path.split('/') + if ( + !path || + path.startsWith('/') || + path.includes('\\') || + path.includes('\0') || + parts.some((part) => !part || part === '.' || part === '..') || + parts[0]?.toLowerCase() === '.git' + ) { + throw new Error(`Pier task outcome contains an unsafe Git path: ${path}`) + } + return path +} + +async function git( + repositoryRoot: string, + args: readonly string[], + input: Uint8Array | undefined, + extraEnvironment: Readonly>, + signal: AbortSignal | undefined, +): Promise { + signal?.throwIfAborted() + const environment = Object.fromEntries( + Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_')), + ) as Record + Object.assign(environment, { + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_NO_REPLACE_OBJECTS: '1', + LC_ALL: 'C', + ...extraEnvironment, + }) + return await new Promise((resolveResult, reject) => { + const child = spawn( + 'git', + [ + '-c', + 'core.hooksPath=/dev/null', + '-c', + 'protocol.file.allow=never', + '-C', + repositoryRoot, + ...args, + ], + { env: environment, stdio: ['pipe', 'pipe', 'pipe'], signal }, + ) + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + let outputBytes = 0 + const collect = (target: Buffer[]) => (chunk: Buffer) => { + outputBytes += chunk.byteLength + if (outputBytes > 64 * 1024 * 1024) { + child.kill('SIGKILL') + reject(new Error(`git ${args[0] ?? ''} exceeded the output limit`)) + return + } + target.push(Buffer.from(chunk)) + } + child.stdout.on('data', collect(stdout)) + child.stderr.on('data', collect(stderr)) + child.once('error', reject) + child.once('close', (code, childSignal) => { + const out = Buffer.concat(stdout) + const err = Buffer.concat(stderr) + if (code !== 0) { + reject( + new Error( + `git ${args[0] ?? ''} failed (${childSignal ?? code}): ${err.toString('utf8').trim()}`, + ), + ) + return + } + resolveResult({ stdout: out, stderr: err }) + }) + child.stdin.end(input) + }) +} diff --git a/bench/src/pier-trial-controller.test.mts b/bench/src/pier-trial-controller.test.mts new file mode 100644 index 00000000..8eeb036a --- /dev/null +++ b/bench/src/pier-trial-controller.test.mts @@ -0,0 +1,412 @@ +import assert from 'node:assert/strict' +import { execFile } from 'node:child_process' +import { chmod, mkdtemp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { promisify } from 'node:util' + +import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime' +import { InMemoryTraceStore } from '@tangle-network/agent-eval' + +import { createStagedPierCandidateExecutionFixture } from './pier-agent.test-fixtures.mts' +import { FilePierCandidateTrialController } from './pier-trial-controller' + +const execFileAsync = promisify(execFile) + +function testRequest(executionId: string, executionPlanDigest: `sha256:${string}`) { + return { + staged: createStagedPierCandidateExecutionFixture(executionId), + context: { + request: { + executionId, + executionPlan: { value: { digest: executionPlanDigest } }, + } as AgentCandidateExecutorRequest, + traceStore: new InMemoryTraceStore(), + signal: new AbortController().signal, + deadlineAtMs: Date.now() + 30_000, + }, + } +} + +test('an existing Pier job is rejected without deleting or starting it', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-scope-')) + const controlRoot = path.join(root, 'control') + const jobsDirectory = path.join(root, 'jobs') + const jobName = 'already-owned' + const marker = path.join(jobsDirectory, jobName, 'keep') + const fakeDocker = path.join(root, 'docker') + try { + await mkdir(path.dirname(marker), { recursive: true }) + await writeFile(marker, 'do not delete\n') + await writeFile(fakeDocker, '#!/bin/sh\nexit 0\n', { mode: 0o700 }) + await chmod(fakeDocker, 0o700) + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + launch: () => ({ + command: process.execPath, + args: ['-e', 'process.exit(0)'], + cwd: root, + env: { ...process.env }, + jobsDirectory, + jobName, + dockerCommand: fakeDocker, + readResult: async () => { + throw new Error('existing job must never launch') + }, + }), + }) + const { staged, context } = testRequest( + 'pier-job-scope', + `sha256:${'c'.repeat(64)}`, + ) + assert.throws( + () => controller.start(staged, context), + /job name must be unique/, + ) + assert.equal(await readFile(marker, 'utf8'), 'do not delete\n') + assert.deepEqual(await readdir(controlRoot), []) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +test('the Pier result wait does not add time beyond an expired deadline', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-deadline-')) + const controlRoot = path.join(root, 'control') + const jobsDirectory = path.join(root, 'jobs') + const fakeDocker = path.join(root, 'docker') + const readyPath = path.join(root, 'supervisor-ready') + const supervisorPath = path.join(root, 'supervisor.mjs') + let handle: ReturnType | undefined + let resultSettled: Promise | undefined + try { + await writeFile(fakeDocker, '#!/bin/sh\nexit 0\n', { mode: 0o700 }) + await chmod(fakeDocker, 0o700) + await writeFile( + supervisorPath, + `import { createReadStream } from 'node:fs' +import { writeFileSync } from 'node:fs' +import path from 'node:path' +const directory = process.argv[2] +process.on('SIGTERM', () => { + writeFileSync(path.join(directory, 'terminal.json'), JSON.stringify({ + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: 'stopped', + processExited: true, + containersRemoved: true, + })) + process.exit(0) +}) +for await (const _chunk of createReadStream('', { fd: 3 })) {} +writeFileSync(${JSON.stringify(readyPath)}, 'ready') +setInterval(() => undefined, 1_000) +`, + ) + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + supervisorPath, + pollIntervalMs: 5, + launch: () => ({ + command: process.execPath, + args: ['-e', 'process.exit(0)'], + cwd: root, + env: {}, + jobsDirectory, + jobName: 'expired-deadline', + dockerCommand: fakeDocker, + readResult: async () => { + throw new Error('an expired trial must not read a result') + }, + }), + }) + const { staged, context } = testRequest( + 'pier-expired-deadline', + `sha256:${'e'.repeat(64)}`, + ) + context.deadlineAtMs = Date.now() + handle = controller.start(staged, context) + resultSettled = handle.result.catch((error) => error) + + const outcome = await Promise.race([ + resultSettled, + new Promise<'pending'>((resolvePending) => setTimeout(() => resolvePending('pending'), 500)), + ]) + assert.notEqual(outcome, 'pending') + assert.match(String(outcome), /no terminal acknowledgement/) + } finally { + if (handle) await handle.terminateAndWait().catch(() => undefined) + if (resultSettled) await resultSettled + await rm(root, { recursive: true, force: true }) + } +}) + +test('the supervisor receives only launch data and no inherited evaluator environment', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-supervisor-env-')) + const controlRoot = path.join(root, 'control') + const jobsDirectory = path.join(root, 'jobs') + const fakeDocker = path.join(root, 'docker') + const supervisorPath = path.join(root, 'supervisor.mjs') + const previousSecret = process.env.PIER_PARENT_SECRET + let handle: ReturnType | undefined + try { + process.env.PIER_PARENT_SECRET = 'must-not-be-inherited' + await writeFile(fakeDocker, '#!/bin/sh\nexit 0\n', { mode: 0o700 }) + await chmod(fakeDocker, 0o700) + await writeFile( + supervisorPath, + `import { createReadStream, writeFileSync } from 'node:fs' +import path from 'node:path' +const chunks = [] +for await (const chunk of createReadStream('', { fd: 3 })) chunks.push(Buffer.from(chunk)) +const launch = JSON.parse(Buffer.concat(chunks).toString('utf8')) +const isolated = process.env.PIER_SUPERVISOR_MODE === 'explicit' && + process.env.PIER_PARENT_SECRET === undefined && + launch.env.PIER_PARENT_SECRET === undefined && + launch.env.PIER_CHILD_SECRET === 'child-only' +writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({ + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: isolated ? 'completed' : 'failed', + processExited: true, + containersRemoved: true, + ...(isolated ? {} : { error: 'supervisor inherited evaluator environment' }), +})) +`, + ) + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + supervisorPath, + pollIntervalMs: 5, + dockerConnection: { + id: 'explicit-supervisor-test', + env: { PIER_SUPERVISOR_MODE: 'explicit' }, + }, + launch: () => ({ + command: process.execPath, + args: ['-e', 'process.exit(0)'], + cwd: root, + env: { PIER_CHILD_SECRET: 'child-only' }, + jobsDirectory, + jobName: 'isolated-supervisor', + dockerCommand: fakeDocker, + readResult: async () => ({ + value: { output: 'isolated' }, + resultBytes: Uint8Array.from(Buffer.from('{}')), + taskPatch: new Uint8Array(), + }), + }), + }) + const { staged, context } = testRequest( + 'pier-isolated-supervisor', + `sha256:${'f'.repeat(64)}`, + ) + handle = controller.start(staged, context) + assert.deepEqual((await handle.result).value, { output: 'isolated' }) + } finally { + if (previousSecret === undefined) delete process.env.PIER_PARENT_SECRET + else process.env.PIER_PARENT_SECRET = previousSecret + if (handle) await handle.terminateAndWait().catch(() => undefined) + await rm(root, { recursive: true, force: true }) + } +}) + +test('terminal acknowledgements reject unknown fields', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-terminal-schema-')) + const controlRoot = path.join(root, 'control') + const jobsDirectory = path.join(root, 'jobs') + const fakeDocker = path.join(root, 'docker') + const supervisorPath = path.join(root, 'supervisor.mjs') + let handle: ReturnType | undefined + try { + await writeFile(fakeDocker, '#!/bin/sh\nexit 0\n', { mode: 0o700 }) + await chmod(fakeDocker, 0o700) + await writeFile( + supervisorPath, + `import { createReadStream, writeFileSync } from 'node:fs' +import path from 'node:path' +for await (const _chunk of createReadStream('', { fd: 3 })) {} +writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({ + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: 'completed', + processExited: true, + containersRemoved: true, + containerRemoved: true, +})) +`, + ) + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + supervisorPath, + pollIntervalMs: 5, + launch: () => ({ + command: process.execPath, + args: ['-e', 'process.exit(0)'], + cwd: root, + env: {}, + jobsDirectory, + jobName: 'strict-terminal', + dockerCommand: fakeDocker, + readResult: async () => { + throw new Error('a malformed terminal must not read a result') + }, + }), + }) + const { staged, context } = testRequest( + 'pier-strict-terminal', + `sha256:${'0'.repeat(64)}`, + ) + handle = controller.start(staged, context) + await assert.rejects(handle.result, /terminal is malformed/) + } finally { + if (handle) await handle.terminateAndWait().catch(() => undefined) + await rm(root, { recursive: true, force: true }) + } +}) + +test('a fresh evaluator process terminates the persisted process and container identity', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-recovery-')) + const controlRoot = path.join(root, 'control') + const jobsDirectory = path.join(root, 'jobs') + const jobName = 'recovery-job' + const trialName = 'trial-recovery' + const primaryContainerState = path.join(root, 'primary-container-live') + const otherContainerState = path.join(root, 'other-container-live') + const fakeDocker = path.join(root, 'docker') + const childPidPath = path.join(root, 'child.pid') + const executionId = 'pier-fresh-process-recovery' + const executionPlanDigest = `sha256:${'d'.repeat(64)}` as const + let handle: ReturnType | undefined + let resultSettled: Promise | undefined + + try { + await writeFile(primaryContainerState, 'live\n') + await writeFile(otherContainerState, 'live\n') + await writeFile( + fakeDocker, + `#!/bin/sh +set -eu +state="\${FAKE_DOCKER_STATE:?missing FAKE_DOCKER_STATE}" +if test "\${1:-}" = ps; then + test -f "$state" && printf 'fake-container\\t${trialName}\\n' + exit 0 +fi +if test "\${1:-}" = rm; then + rm -f -- "$state" + exit 0 +fi +printf 'unexpected fake docker invocation: %s\\n' "$*" >&2 +exit 2 +`, + { mode: 0o700 }, + ) + await chmod(fakeDocker, 0o700) + + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + dockerConnection: { + id: 'fake-primary', + env: { FAKE_DOCKER_STATE: primaryContainerState }, + }, + launch: () => ({ + command: process.execPath, + args: [ + '-e', + `const { spawn } = require('node:child_process'); const { mkdirSync, writeFileSync } = require('node:fs'); mkdirSync(${JSON.stringify(path.join(jobsDirectory, jobName, trialName))}, { recursive: true }); const child = spawn(process.execPath, ['-e', 'setInterval(() => undefined, 1_000)'], { stdio: 'ignore' }); writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid)); setInterval(() => undefined, 1_000)`, + ], + cwd: root, + env: {}, + jobsDirectory, + jobName, + dockerCommand: fakeDocker, + readResult: async () => { + throw new Error('recovery trial must not complete normally') + }, + }), + }) + const { staged, context } = testRequest(executionId, executionPlanDigest) + handle = controller.start(staged, context) + resultSettled = handle.result.catch(() => undefined) + + const [slot] = await readdir(controlRoot) + assert.ok(slot) + const identityPath = path.join(controlRoot, slot, 'identity.json') + let persisted: { state?: string; pier?: { pid: number } } = {} + for (let attempt = 0; attempt < 200; attempt++) { + persisted = JSON.parse(await readFile(identityPath, 'utf8')) + if (persisted.state === 'running' && persisted.pier) break + await new Promise((resolveWait) => setTimeout(resolveWait, 10)) + } + assert.equal(persisted.state, 'running') + assert.ok(persisted.pier) + let childPid: number | undefined + for (let attempt = 0; attempt < 200; attempt++) { + try { + childPid = Number(await readFile(childPidPath, 'utf8')) + if (Number.isSafeInteger(childPid)) break + } catch {} + await new Promise((resolveWait) => setTimeout(resolveWait, 10)) + } + assert.ok(childPid) + + const recoveryScript = path.resolve('scripts/terminate-pier-trial.mts') + const recoveryArgs = [ + '--import', + 'tsx', + recoveryScript, + controlRoot, + executionId, + executionPlanDigest, + ] + const recoveryEnvironment = { + ...process.env, + PIER_DOCKER_CONNECTION_ID: 'fake-primary', + PIER_DOCKER_ENV_NAMES: 'FAKE_DOCKER_STATE', + FAKE_DOCKER_STATE: primaryContainerState, + } + await assert.rejects( + execFileAsync(process.execPath, recoveryArgs, { + cwd: path.resolve('.'), + env: { + ...recoveryEnvironment, + PIER_DOCKER_CONNECTION_ID: 'fake-other', + FAKE_DOCKER_STATE: otherContainerState, + }, + timeout: 20_000, + }), + (error: unknown) => + String((error as { stderr?: string }).stderr).includes( + 'requires Docker connection fake-primary', + ), + ) + const recovered = await Promise.all([ + execFileAsync(process.execPath, recoveryArgs, { + cwd: path.resolve('.'), + env: recoveryEnvironment, + timeout: 20_000, + }), + execFileAsync(process.execPath, recoveryArgs, { + cwd: path.resolve('.'), + env: recoveryEnvironment, + timeout: 20_000, + }), + ]) + for (const worker of recovered) { + assert.deepEqual(JSON.parse(worker.stdout), { + processExited: true, + containersRemoved: true, + }) + } + await assert.rejects(readFile(primaryContainerState), /ENOENT/) + assert.equal(await readFile(otherContainerState, 'utf8'), 'live\n') + assert.throws(() => process.kill(persisted.pier!.pid, 0), /ESRCH/) + assert.throws(() => process.kill(childPid!, 0), /ESRCH/) + await resultSettled + } finally { + if (handle) await handle.terminateAndWait().catch(() => undefined) + if (resultSettled) await resultSettled + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/bench/src/pier-trial-controller.ts b/bench/src/pier-trial-controller.ts new file mode 100644 index 00000000..c818098a --- /dev/null +++ b/bench/src/pier-trial-controller.ts @@ -0,0 +1,825 @@ +/** Restart-safe evaluator ownership for one Pier process and its Docker projects. */ +import { execFileSync, spawn } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + accessSync, + closeSync, + constants, + existsSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + realpathSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { isAbsolute, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import type { Writable } from 'node:stream' + +import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime' +import type { TraceStore } from '@tangle-network/agent-eval' + +import type { + PierCandidateTerminationAcknowledgement, + PierCandidateTrialController, + PierCandidateTrialHandle, + PierCandidateTrialIdentity, + PierCandidateTrialResult, + StagedPierCandidateExecution, +} from './pier-agent' + +const identityFile = 'identity.json' +const terminalFile = 'terminal.json' +const stopFile = 'stop-requested' +const sha256Pattern = /^sha256:[a-f0-9]{64}$/ +const defaultDockerConnectionId = 'local-default' + +interface ProcessIdentity { + readonly pid: number + readonly processGroupId: number + readonly sessionId: number + readonly startTicks: string +} + +interface PersistedTrialIdentity { + readonly schemaVersion: 1 + readonly kind: 'pier-trial-identity' + readonly state: 'allocating' | 'starting' | 'running' + readonly executionId: string + readonly executionPlanDigest: `sha256:${string}` + readonly supervisor?: ProcessIdentity + readonly pier?: ProcessIdentity + readonly jobsDirectory: string + readonly jobName: string + readonly dockerCommand: string + readonly dockerConnectionId: string +} + +interface PersistedTrialTerminal { + readonly schemaVersion: 1 + readonly kind: 'pier-trial-terminal' + readonly status: 'completed' | 'stopped' | 'failed' + readonly processExited: boolean + readonly containersRemoved: boolean + readonly removedContainers?: number + readonly exitCode?: number | null + readonly signal?: NodeJS.Signals | null + readonly error?: string + readonly recoveredByPid?: number +} + +export interface PierCandidateProcessSpec { + /** Returns launch data only. The controller, not this callback, starts the process. */ + readonly command: string + readonly args: readonly string[] + readonly cwd: string + /** Exact Pier child environment. The controller adds its Docker connection variables. */ + readonly env: Readonly> + readonly jobsDirectory: string + /** Must be unique: the controller atomically reserves this Pier job directory. */ + readonly jobName: string + readonly dockerCommand?: string + /** Called only by the originating process after the supervisor reports clean exit. */ + readonly readResult: () => Promise +} + +export interface PierDockerConnection { + /** Stable, non-secret name that every recovery worker maps to the same Docker endpoint. */ + readonly id: string + /** Exact variables needed by both Pier and Docker cleanup; values are never persisted. */ + readonly env: Readonly> +} + +export interface FilePierCandidateTrialControllerOptions { + /** Shared evaluator-owned control root, available to crash-recovery workers. */ + readonly directory: string + readonly launch?: ( + staged: StagedPierCandidateExecution, + context: { + readonly request: AgentCandidateExecutorRequest + readonly traceStore: TraceStore + readonly signal: AbortSignal + readonly deadlineAtMs: number + }, + ) => PierCandidateProcessSpec + /** Omit only for the default local Docker socket with no environment variables. */ + readonly dockerConnection?: PierDockerConnection + readonly supervisorPath?: string + readonly pollIntervalMs?: number +} + +function nonEmpty(value: string, label: string): string { + if (!value || value.includes('\0')) throw new Error(`${label} must be non-empty without NUL`) + return value +} + +function stableDockerConnectionId(value: string): string { + if (!/^[A-Za-z0-9._:-]{1,200}$/.test(value)) { + throw new Error('Docker connection id must be a non-secret stable name') + } + return value +} + +function absolutePath(value: string, label: string): string { + if (!isAbsolute(value)) throw new Error(`${label} must be an absolute path`) + return resolve(value) +} + +function validateEnvironment( + environment: Readonly>, + label: string, +): void { + for (const [name, value] of Object.entries(environment)) { + if (!name || name.includes('\0') || value?.includes('\0')) { + throw new Error(`${label} contains an invalid name or value`) + } + } +} + +function resolveExecutable(command: string, searchPath: string | undefined, label: string): string { + nonEmpty(command, label) + const candidates = command.includes('/') + ? [absolutePath(command, label)] + : (searchPath ?? '') + .split(':') + .filter(Boolean) + .map((directory) => join(directory, command)) + for (const candidate of candidates) { + try { + accessSync(candidate, constants.X_OK) + return realpathSync(candidate) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT' && (error as NodeJS.ErrnoException).code !== 'EACCES') { + throw error + } + } + } + throw new Error(`${label} is not executable on the evaluator PATH`) +} + +function syncDirectory(directory: string): void { + const fd = openSync(directory, 'r') + try { + fsyncSync(fd) + } finally { + closeSync(fd) + } +} + +function writeJsonAtomic(directory: string, name: string, value: unknown): void { + const target = join(directory, name) + const temporary = join(directory, `.${name}.${process.pid}.${Date.now()}.tmp`) + const fd = openSync(temporary, 'wx', 0o600) + try { + writeFileSync(fd, `${JSON.stringify(value)}\n`, 'utf8') + fsyncSync(fd) + } finally { + closeSync(fd) + } + renameSync(temporary, target) + syncDirectory(directory) +} + +function touchDurable(directory: string, name: string): void { + try { + const fd = openSync(join(directory, name), 'wx', 0o600) + try { + fsyncSync(fd) + } finally { + closeSync(fd) + } + syncDirectory(directory) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error + } +} + +function readJson(path: string, label: string): Record { + let value: unknown + try { + value = JSON.parse(readFileSync(path, 'utf8')) + } catch (error) { + throw new Error(`${label} is not valid JSON`, { cause: error }) + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + return value as Record +} + +function parseProcessIdentity(value: unknown, label: string): ProcessIdentity { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + const record = value as Record + if ( + !Number.isSafeInteger(record.pid) || + !Number.isSafeInteger(record.processGroupId) || + !Number.isSafeInteger(record.sessionId) || + typeof record.startTicks !== 'string' || + !/^\d+$/.test(record.startTicks) + ) { + throw new Error(`${label} is malformed`) + } + return { + pid: record.pid as number, + processGroupId: record.processGroupId as number, + sessionId: record.sessionId as number, + startTicks: record.startTicks, + } +} + +function parsePersistedIdentity(path: string): PersistedTrialIdentity { + const record = readJson(path, 'Pier trial identity') + if ( + record.schemaVersion !== 1 || + record.kind !== 'pier-trial-identity' || + !['allocating', 'starting', 'running'].includes(record.state as string) || + typeof record.executionId !== 'string' || + typeof record.executionPlanDigest !== 'string' || + !sha256Pattern.test(record.executionPlanDigest) || + typeof record.jobsDirectory !== 'string' || + typeof record.jobName !== 'string' || + typeof record.dockerCommand !== 'string' || + typeof record.dockerConnectionId !== 'string' + ) { + throw new Error('Pier trial identity is malformed') + } + return { + schemaVersion: 1, + kind: 'pier-trial-identity', + state: record.state as PersistedTrialIdentity['state'], + executionId: record.executionId, + executionPlanDigest: record.executionPlanDigest as `sha256:${string}`, + ...(record.supervisor + ? { supervisor: parseProcessIdentity(record.supervisor, 'Pier supervisor identity') } + : {}), + ...(record.pier ? { pier: parseProcessIdentity(record.pier, 'Pier process identity') } : {}), + jobsDirectory: absolutePath(record.jobsDirectory, 'persisted jobs directory'), + jobName: nonEmpty(record.jobName, 'persisted job name'), + dockerCommand: nonEmpty(record.dockerCommand, 'persisted Docker command'), + dockerConnectionId: stableDockerConnectionId(record.dockerConnectionId), + } +} + +function parseTerminal(path: string): PersistedTrialTerminal { + const record = readJson(path, 'Pier trial terminal') + const allowedKeys = new Set([ + 'schemaVersion', + 'kind', + 'status', + 'processExited', + 'containersRemoved', + 'removedContainers', + 'exitCode', + 'signal', + 'error', + 'recoveredByPid', + ]) + const hasMalformedOptionalField = + (record.removedContainers !== undefined && + (!Number.isSafeInteger(record.removedContainers) || (record.removedContainers as number) < 0)) || + (record.exitCode !== undefined && + record.exitCode !== null && + !Number.isSafeInteger(record.exitCode)) || + (record.signal !== undefined && + record.signal !== null && + (typeof record.signal !== 'string' || !/^SIG[A-Z0-9]+$/.test(record.signal))) || + (record.error !== undefined && typeof record.error !== 'string') || + (record.recoveredByPid !== undefined && + (!Number.isSafeInteger(record.recoveredByPid) || (record.recoveredByPid as number) < 1)) + if ( + Object.keys(record).some((key) => !allowedKeys.has(key)) || + record.schemaVersion !== 1 || + record.kind !== 'pier-trial-terminal' || + !['completed', 'stopped', 'failed'].includes(record.status as string) || + typeof record.processExited !== 'boolean' || + typeof record.containersRemoved !== 'boolean' || + hasMalformedOptionalField + ) { + throw new Error('Pier trial terminal is malformed') + } + return { + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: record.status as PersistedTrialTerminal['status'], + processExited: record.processExited, + containersRemoved: record.containersRemoved, + ...(record.removedContainers !== undefined + ? { removedContainers: record.removedContainers as number } + : {}), + ...(record.exitCode !== undefined ? { exitCode: record.exitCode as number | null } : {}), + ...(record.signal !== undefined ? { signal: record.signal as NodeJS.Signals | null } : {}), + ...(record.error !== undefined ? { error: record.error as string } : {}), + ...(record.recoveredByPid !== undefined + ? { recoveredByPid: record.recoveredByPid as number } + : {}), + } +} + +function processIdentity(pid: number): ProcessIdentity { + const text = readFileSync(`/proc/${pid}/stat`, 'utf8') + const closing = text.lastIndexOf(')') + if (closing < 0) throw new Error(`cannot parse process identity for ${pid}`) + const fields = text.slice(closing + 2).trim().split(/\s+/) + const processGroupId = Number(fields[2]) + const sessionId = Number(fields[3]) + const startTicks = fields[19] + if ( + !Number.isSafeInteger(processGroupId) || + !Number.isSafeInteger(sessionId) || + !/^\d+$/.test(startTicks ?? '') + ) { + throw new Error(`cannot parse process identity for ${pid}`) + } + return { pid, processGroupId, sessionId, startTicks } +} + +function processMatches(identity: ProcessIdentity): boolean { + try { + const current = processIdentity(identity.pid) + return ( + current.startTicks === identity.startTicks && + current.processGroupId === identity.processGroupId && + current.sessionId === identity.sessionId + ) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false + throw error + } +} + +function signalProcess(identity: ProcessIdentity, signal: NodeJS.Signals, group: boolean): void { + if (!processMatches(identity)) return + try { + process.kill(group ? -identity.processGroupId : identity.pid, signal) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error + } +} + +function sessionMembers(sessionId: number): ProcessIdentity[] { + const members: ProcessIdentity[] = [] + for (const entry of readdirSync('/proc', { withFileTypes: true })) { + if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) continue + try { + const identity = processIdentity(Number(entry.name)) + if (identity.sessionId === sessionId) members.push(identity) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + } + } + return members +} + +async function signalSessionUntilEmpty( + identity: ProcessIdentity, + signal: NodeJS.Signals, + timeoutMs: number, + intervalMs: number, +): Promise { + const deadline = Date.now() + timeoutMs + while (true) { + const members = sessionMembers(identity.sessionId) + if (members.length === 0) return true + for (const member of members) signalProcess(member, signal, false) + if (Date.now() >= deadline) return false + await sleep(intervalMs) + } +} + +function sanitizeProject(value: string): string { + let project = value.toLowerCase() + if (!/^[a-z0-9]/.test(project)) project = `0${project}` + return project.replace(/[^a-z0-9_-]/g, '-') +} + +function trialProjects(identity: PersistedTrialIdentity): string[] { + const jobRoot = join(identity.jobsDirectory, identity.jobName) + if (!existsSync(jobRoot)) return [] + return readdirSync(jobRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()) + .map((entry) => sanitizeProject(entry.name)) +} + +function matchingContainers( + identity: PersistedTrialIdentity, + dockerEnvironment: Readonly>, +): string[] { + const projects = trialProjects(identity) + if (projects.length === 0) return [] + const output = execFileSync( + identity.dockerCommand, + ['ps', '-a', '--format', '{{.ID}}\t{{.Label "com.docker.compose.project"}}'], + { + encoding: 'utf8', + env: dockerEnvironment, + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }, + ) + return output + .split('\n') + .filter(Boolean) + .flatMap((line) => { + const tab = line.indexOf('\t') + if (tab < 1) return [] + const id = line.slice(0, tab) + const project = line.slice(tab + 1) + return projects.some( + (expected) => + project === expected || project.startsWith(`${expected}__verifier__`), + ) + ? [id] + : [] + }) +} + +function removeTrialContainers( + identity: PersistedTrialIdentity, + dockerEnvironment: Readonly>, +): number { + const containers = matchingContainers(identity, dockerEnvironment) + if (containers.length > 0) { + try { + execFileSync(identity.dockerCommand, ['rm', '-f', ...containers], { + encoding: 'utf8', + env: dockerEnvironment, + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }) + } catch (error) { + if (matchingContainers(identity, dockerEnvironment).length > 0) throw error + } + } + const remaining = matchingContainers(identity, dockerEnvironment) + if (remaining.length > 0) { + throw new Error(`Pier task containers survived removal: ${remaining.join(', ')}`) + } + return containers.length +} + +function sleep(milliseconds: number): Promise { + return new Promise((resolveSleep) => setTimeout(resolveSleep, milliseconds)) +} + +async function waitUntil(predicate: () => boolean, timeoutMs: number, intervalMs: number) { + const deadline = Date.now() + timeoutMs + while (!predicate()) { + if (Date.now() >= deadline) return false + await sleep(intervalMs) + } + return true +} + +function trialKey(identity: PierCandidateTrialIdentity): string { + if (!sha256Pattern.test(identity.executionPlanDigest)) { + throw new Error('executionPlanDigest must be a SHA-256 digest') + } + return createHash('sha256') + .update(JSON.stringify([nonEmpty(identity.executionId, 'executionId'), identity.executionPlanDigest])) + .digest('hex') +} + +function assertRealDirectory(path: string, label: string): void { + const stats = lstatSync(path) + if (!stats.isDirectory() || stats.isSymbolicLink() || realpathSync(path) !== path) { + throw new Error(`${label} must be a real canonical directory`) + } +} + +export class FilePierCandidateTrialController implements PierCandidateTrialController { + private readonly directory: string + private readonly launch?: NonNullable + private readonly dockerConnection: PierDockerConnection + private readonly supervisorPath: string + private readonly pollIntervalMs: number + + constructor(options: FilePierCandidateTrialControllerOptions) { + this.directory = absolutePath(options.directory, 'Pier controller directory') + this.launch = options.launch + const configuredDocker = options.dockerConnection + if (configuredDocker?.id === defaultDockerConnectionId) { + throw new Error(`${defaultDockerConnectionId} is reserved for the implicit local connection`) + } + const dockerConnection = configuredDocker ?? { + id: defaultDockerConnectionId, + env: {}, + } + const connectionId = stableDockerConnectionId(dockerConnection.id) + validateEnvironment(dockerConnection.env, 'Docker connection environment') + this.dockerConnection = Object.freeze({ + id: connectionId, + env: Object.freeze({ ...dockerConnection.env }), + }) + this.supervisorPath = absolutePath( + options.supervisorPath ?? + fileURLToPath(new URL('./pier-trial-supervisor.mjs', import.meta.url)), + 'Pier supervisor path', + ) + this.pollIntervalMs = options.pollIntervalMs ?? 25 + if (!Number.isSafeInteger(this.pollIntervalMs) || this.pollIntervalMs < 1) { + throw new Error('pollIntervalMs must be a positive integer') + } + mkdirSync(this.directory, { recursive: true, mode: 0o700 }) + assertRealDirectory(this.directory, 'Pier controller directory') + } + + start( + staged: StagedPierCandidateExecution, + context: { + readonly request: AgentCandidateExecutorRequest + readonly traceStore: TraceStore + readonly signal: AbortSignal + readonly deadlineAtMs: number + }, + ): PierCandidateTrialHandle { + if (!this.launch) throw new Error('this Pier controller is recovery-only') + const identity: PierCandidateTrialIdentity = { + executionId: context.request.executionId, + executionPlanDigest: context.request.executionPlan.value.digest, + } + if (staged.executionId !== identity.executionId) { + throw new Error('staged Pier execution does not match the runtime request') + } + + // The launch callback is a pure spec builder. Resolve every fallible input + // before reserving durable identities or starting a process. + const spec = this.launch(staged, context) + this.validateSpec(spec) + for (const [name, value] of Object.entries(this.dockerConnection.env)) { + if ( + Object.prototype.hasOwnProperty.call(spec.env, name) && + spec.env[name] !== value + ) { + throw new Error(`Pier child environment conflicts with Docker connection variable ${name}`) + } + } + const pierEnvironment = Object.freeze({ + ...spec.env, + ...this.dockerConnection.env, + }) + mkdirSync(spec.jobsDirectory, { recursive: true, mode: 0o700 }) + assertRealDirectory(spec.jobsDirectory, 'Pier jobs directory') + const dockerCommand = resolveExecutable( + spec.dockerCommand ?? 'docker', + process.env.PATH, + 'Docker command', + ) + const pierCommand = resolveExecutable(spec.command, pierEnvironment.PATH, 'Pier command') + + const controlDirectory = this.controlDirectory(identity) + try { + mkdirSync(controlDirectory, { mode: 0o700 }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') { + throw new Error('Pier trial identity already has durable control state') + } + throw error + } + assertRealDirectory(controlDirectory, 'Pier trial control directory') + + // Pier creates each trial directory before starting its environment. An + // atomically reserved, one-execution job directory therefore makes every + // Compose project discovered below exclusive to this execution. + const jobRoot = join(spec.jobsDirectory, spec.jobName) + let jobRootCreated = false + try { + mkdirSync(jobRoot, { mode: 0o700 }) + jobRootCreated = true + assertRealDirectory(jobRoot, 'Pier job directory') + } catch (error) { + // No identity or process exists yet; leave the execution reusable. + rmSync(controlDirectory, { recursive: true, force: true }) + if (jobRootCreated) rmSync(jobRoot, { recursive: true, force: true }) + if ((error as NodeJS.ErrnoException).code === 'EEXIST') { + throw new Error('Pier job name must be unique; its job directory already exists') + } + throw error + } + const allocating: PersistedTrialIdentity = { + schemaVersion: 1, + kind: 'pier-trial-identity', + state: 'allocating', + ...identity, + jobsDirectory: absolutePath(spec.jobsDirectory, 'Pier jobs directory'), + jobName: nonEmpty(spec.jobName, 'Pier job name'), + dockerCommand: nonEmpty(dockerCommand, 'Docker command'), + dockerConnectionId: this.dockerConnection.id, + } + try { + writeJsonAtomic(controlDirectory, identityFile, allocating) + } catch (error) { + // The supervisor can only start after this durable write succeeds. + rmSync(controlDirectory, { recursive: true, force: true }) + rmSync(jobRoot, { recursive: true, force: true }) + throw error + } + + const supervisor = spawn(process.execPath, [this.supervisorPath, controlDirectory], { + detached: true, + // Launch data arrives over fd 3. The trusted supervisor receives only + // explicitly declared process variables, never the evaluator environment. + env: { ...this.dockerConnection.env }, + stdio: ['ignore', 'ignore', 'ignore', 'pipe'], + }) + if (supervisor.pid === undefined) throw new Error('Pier supervisor started without a pid') + const controlPipe = supervisor.stdio[3] as (Writable & { unref?: () => void }) | null + let supervisorIdentity: ProcessIdentity | undefined + try { + supervisorIdentity = processIdentity(supervisor.pid) + const persisted: PersistedTrialIdentity = { + ...allocating, + state: 'starting', + supervisor: supervisorIdentity, + } + writeJsonAtomic(controlDirectory, identityFile, persisted) + if (!controlPipe) throw new Error('Pier supervisor has no protected control pipe') + controlPipe.on('error', () => undefined) + controlPipe.end( + JSON.stringify({ + command: pierCommand, + args: spec.args, + cwd: spec.cwd, + env: pierEnvironment, + jobsDirectory: spec.jobsDirectory, + jobName: spec.jobName, + dockerCommand, + }), + ) + } catch (error) { + controlPipe?.destroy() + if (supervisorIdentity) signalProcess(supervisorIdentity, 'SIGKILL', false) + else { + try { + process.kill(supervisor.pid, 'SIGKILL') + } catch (killError) { + if ((killError as NodeJS.ErrnoException).code !== 'ESRCH') throw killError + } + } + throw error + } + supervisor.unref() + controlPipe.unref?.() + + const result = this.waitForResult(controlDirectory, spec, context.deadlineAtMs) + return { + identity, + result, + terminateAndWait: async () => await this.terminateAndWait(identity), + } + } + + async terminateAndWait( + requested: PierCandidateTrialIdentity, + ): Promise { + const controlDirectory = this.controlDirectory(requested) + assertRealDirectory(controlDirectory, 'Pier trial control directory') + const persisted = parsePersistedIdentity(join(controlDirectory, identityFile)) + if ( + persisted.executionId !== requested.executionId || + persisted.executionPlanDigest !== requested.executionPlanDigest + ) { + throw new Error('persisted Pier trial identity differs from the stop request') + } + this.assertDockerConnection(persisted) + + touchDurable(controlDirectory, stopFile) + if (!existsSync(join(controlDirectory, terminalFile))) { + if (persisted.supervisor) signalProcess(persisted.supervisor, 'SIGTERM', false) + } + const terminalAppeared = await waitUntil( + () => existsSync(join(controlDirectory, terminalFile)), + persisted.supervisor ? 10_000 : 1, + this.pollIntervalMs, + ) + if (!terminalAppeared) await this.forceRecovery(controlDirectory) + + let terminal = parseTerminal(join(controlDirectory, terminalFile)) + const latest = parsePersistedIdentity(join(controlDirectory, identityFile)) + const liveProcesses = latest.pier ? sessionMembers(latest.pier.sessionId).length : 0 + this.assertDockerConnection(latest) + const liveContainers = matchingContainers(latest, this.dockerConnection.env).length + if (liveProcesses > 0 || liveContainers > 0) { + await this.forceRecovery(controlDirectory) + terminal = parseTerminal(join(controlDirectory, terminalFile)) + } + if (!terminal.processExited || !terminal.containersRemoved) { + throw new Error( + `Pier recovery did not prove process/container death${terminal.error ? `: ${terminal.error}` : ''}`, + ) + } + return { processExited: true, containersRemoved: true } + } + + private controlDirectory(identity: PierCandidateTrialIdentity): string { + return join(this.directory, trialKey(identity)) + } + + private assertDockerConnection(identity: PersistedTrialIdentity): void { + if (identity.dockerConnectionId !== this.dockerConnection.id) { + throw new Error( + `Pier trial requires Docker connection ${identity.dockerConnectionId}; configured ${this.dockerConnection.id}`, + ) + } + } + + private validateSpec(spec: PierCandidateProcessSpec): void { + nonEmpty(spec.command, 'Pier command') + if (!Array.isArray(spec.args) || spec.args.some((arg) => typeof arg !== 'string' || arg.includes('\0'))) { + throw new Error('Pier arguments must be strings without NUL') + } + absolutePath(spec.cwd, 'Pier cwd') + absolutePath(spec.jobsDirectory, 'Pier jobs directory') + if (!/^[A-Za-z0-9._-]{1,200}$/.test(spec.jobName)) { + throw new Error('Pier job name must be filesystem-neutral') + } + validateEnvironment(spec.env, 'Pier child environment') + } + + private async waitForResult( + controlDirectory: string, + spec: PierCandidateProcessSpec, + deadlineAtMs: number, + ): Promise { + const waitMs = Math.max(0, deadlineAtMs - Date.now()) + const appeared = await waitUntil( + () => existsSync(join(controlDirectory, terminalFile)), + waitMs, + this.pollIntervalMs, + ) + if (!appeared) throw new Error('Pier supervisor emitted no terminal acknowledgement') + const terminal = parseTerminal(join(controlDirectory, terminalFile)) + if (!terminal.processExited || !terminal.containersRemoved) { + throw new Error('Pier supervisor did not prove process and container death') + } + if (terminal.status !== 'completed') { + throw new Error(terminal.error ? `Pier process failed: ${terminal.error}` : 'Pier process stopped') + } + return await spec.readResult() + } + + private async forceRecovery( + controlDirectory: string, + ): Promise { + const latest = parsePersistedIdentity(join(controlDirectory, identityFile)) + this.assertDockerConnection(latest) + const errors: string[] = [] + if (latest.pier) { + try { + await signalSessionUntilEmpty(latest.pier, 'SIGTERM', 2_000, this.pollIntervalMs) + } catch (error) { + errors.push(`could not signal Pier session: ${error instanceof Error ? error.message : String(error)}`) + } + } + if (latest.supervisor) { + try { + signalProcess(latest.supervisor, 'SIGKILL', false) + if (!(await waitUntil(() => !processMatches(latest.supervisor!), 5_000, this.pollIntervalMs))) { + errors.push(`Pier supervisor ${latest.supervisor.pid} survived SIGKILL`) + } + } catch (error) { + errors.push(`could not kill Pier supervisor: ${error instanceof Error ? error.message : String(error)}`) + } + } + if (latest.pier) { + try { + if ( + !(await signalSessionUntilEmpty( + latest.pier, + 'SIGKILL', + 5_000, + this.pollIntervalMs, + )) + ) { + errors.push(`Pier process session ${latest.pier.sessionId} survived SIGKILL`) + } + } catch (error) { + errors.push(`could not kill Pier session: ${error instanceof Error ? error.message : String(error)}`) + } + } + let removedContainers = 0 + try { + removedContainers = removeTrialContainers(latest, this.dockerConnection.env) + } catch (error) { + errors.push(`could not remove Pier containers: ${error instanceof Error ? error.message : String(error)}`) + } + if (errors.length > 0) { + throw new Error(`Pier recovery incomplete: ${errors.join('; ')}`) + } + writeJsonAtomic(controlDirectory, terminalFile, { + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: 'stopped', + processExited: true, + containersRemoved: true, + removedContainers, + recoveredByPid: process.pid, + }) + } +} diff --git a/bench/src/pier-trial-supervisor.mjs b/bench/src/pier-trial-supervisor.mjs new file mode 100644 index 00000000..349b15ed --- /dev/null +++ b/bench/src/pier-trial-supervisor.mjs @@ -0,0 +1,352 @@ +import { execFileSync, spawn } from 'node:child_process' +import { + closeSync, + createReadStream, + existsSync, + fsyncSync, + openSync, + readFileSync, + readdirSync, + renameSync, + writeFileSync, +} from 'node:fs' +import { basename, isAbsolute, join, resolve } from 'node:path' + +const identityFile = 'identity.json' +const terminalFile = 'terminal.json' +const stopFile = 'stop-requested' + +function fail(message) { + throw new Error(message) +} + +function readJson(path, label) { + let value + try { + value = JSON.parse(readFileSync(path, 'utf8')) + } catch (error) { + throw new Error(`${label} is not valid JSON`, { cause: error }) + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + fail(`${label} must be an object`) + } + return value +} + +function syncDirectory(directory) { + const fd = openSync(directory, 'r') + try { + fsyncSync(fd) + } finally { + closeSync(fd) + } +} + +function writeJsonAtomic(directory, name, value) { + const target = join(directory, name) + const temporary = join(directory, `.${name}.${process.pid}.tmp`) + const fd = openSync(temporary, 'wx', 0o600) + try { + writeFileSync(fd, `${JSON.stringify(value)}\n`, 'utf8') + fsyncSync(fd) + } finally { + closeSync(fd) + } + renameSync(temporary, target) + syncDirectory(directory) +} + +async function readControlInput() { + const chunks = [] + for await (const chunk of createReadStream('', { fd: 3 })) { + chunks.push(Buffer.from(chunk)) + } + const bytes = Buffer.concat(chunks) + if (bytes.byteLength === 0) fail('Pier supervisor received no launch request') + return JSON.parse(bytes.toString('utf8')) +} + +function processIdentity(pid) { + const text = readFileSync(`/proc/${pid}/stat`, 'utf8') + const closing = text.lastIndexOf(')') + if (closing < 0) fail(`cannot parse process identity for ${pid}`) + const fields = text.slice(closing + 2).trim().split(/\s+/) + const processGroupId = Number(fields[2]) + const sessionId = Number(fields[3]) + const startTicks = fields[19] + if ( + !Number.isSafeInteger(processGroupId) || + !Number.isSafeInteger(sessionId) || + !/^\d+$/.test(startTicks ?? '') + ) { + fail(`cannot parse process identity for ${pid}`) + } + return { pid, processGroupId, sessionId, startTicks } +} + +function processMatches(identity) { + try { + const current = processIdentity(identity.pid) + return ( + current.startTicks === identity.startTicks && + current.processGroupId === identity.processGroupId && + current.sessionId === identity.sessionId + ) + } catch (error) { + if (error?.code === 'ENOENT' || error?.code === 'ESRCH') return false + throw error + } +} + +function sessionMembers(sessionId) { + const members = [] + for (const entry of readdirSync('/proc', { withFileTypes: true })) { + if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) continue + try { + const identity = processIdentity(Number(entry.name)) + if (identity.sessionId === sessionId) members.push(identity) + } catch (error) { + if (error?.code !== 'ENOENT') throw error + } + } + return members +} + +async function signalSessionUntilEmpty(identity, signal, timeoutMs) { + const deadline = Date.now() + timeoutMs + while (true) { + const members = sessionMembers(identity.sessionId) + if (members.length === 0) return true + for (const member of members) { + try { + process.kill(member.pid, signal) + } catch (error) { + if (error?.code !== 'ESRCH') throw error + } + } + if (Date.now() >= deadline) return false + await new Promise((resolveWait) => setTimeout(resolveWait, 20)) + } +} + +function sanitizeProject(value) { + let project = value.toLowerCase() + if (!/^[a-z0-9]/.test(project)) project = `0${project}` + return project.replace(/[^a-z0-9_-]/g, '-') +} + +function trialProjects(jobsDirectory, jobName) { + const jobRoot = join(jobsDirectory, jobName) + if (!existsSync(jobRoot)) return [] + return readdirSync(jobRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()) + .map((entry) => sanitizeProject(entry.name)) +} + +function matchingContainers(dockerCommand, projects) { + if (projects.length === 0) return [] + const output = execFileSync( + dockerCommand, + ['ps', '-a', '--format', '{{.ID}}\t{{.Label "com.docker.compose.project"}}'], + { encoding: 'utf8', timeout: 30_000, maxBuffer: 4 * 1024 * 1024 }, + ) + const matches = [] + for (const line of output.split('\n')) { + if (!line) continue + const tab = line.indexOf('\t') + if (tab < 1) continue + const id = line.slice(0, tab) + const project = line.slice(tab + 1) + if ( + projects.some( + (expected) => + project === expected || project.startsWith(`${expected}__verifier__`), + ) + ) { + matches.push(id) + } + } + return matches +} + +function removeTrialContainers(config) { + const projects = trialProjects(config.jobsDirectory, config.jobName) + const containers = matchingContainers(config.dockerCommand, projects) + if (containers.length > 0) { + try { + execFileSync(config.dockerCommand, ['rm', '-f', ...containers], { + encoding: 'utf8', + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }) + } catch (error) { + if (matchingContainers(config.dockerCommand, projects).length > 0) throw error + } + } + const remaining = matchingContainers(config.dockerCommand, projects) + if (remaining.length > 0) { + fail(`Pier task containers survived removal: ${remaining.join(', ')}`) + } + return containers.length +} + +async function waitForExit(exited, timeoutMs) { + return await Promise.race([ + exited.then(() => true), + new Promise((resolveTimeout) => setTimeout(() => resolveTimeout(false), timeoutMs)), + ]) +} + +const directory = resolve(process.argv[2] ?? '') +if (!isAbsolute(directory) || basename(directory) === '') { + fail('Pier supervisor control directory must be absolute') +} + +let config +let child +let childIdentity +let stopRequested = existsSync(join(directory, stopFile)) +let stopping + +async function stopChild() { + if (!child) return + if (!childIdentity) { + try { + child.process.kill('SIGKILL') + } catch (error) { + if (error?.code !== 'ESRCH') throw error + } + if (!(await waitForExit(child.exited, 5_000))) { + fail('Pier process with unreadable identity survived SIGKILL') + } + return + } + if (!(await signalSessionUntilEmpty(childIdentity, 'SIGTERM', 2_000))) { + if (!(await signalSessionUntilEmpty(childIdentity, 'SIGKILL', 5_000))) { + fail(`Pier process session ${childIdentity.sessionId} survived SIGKILL`) + } + } + if (!(await waitForExit(child.exited, 5_000))) { + fail(`Pier process session ${childIdentity.sessionId} retained child processes`) + } +} + +function requestStop() { + stopRequested = true + stopping ??= stopChild() +} + +process.on('SIGTERM', requestStop) +process.on('SIGINT', requestStop) + +try { + config = await readControlInput() + if ( + !config || + typeof config !== 'object' || + typeof config.command !== 'string' || + !Array.isArray(config.args) || + typeof config.cwd !== 'string' || + !config.env || + typeof config.env !== 'object' || + typeof config.jobsDirectory !== 'string' || + typeof config.jobName !== 'string' || + typeof config.dockerCommand !== 'string' + ) { + fail('Pier supervisor launch request is malformed') + } + + const identity = readJson(join(directory, identityFile), 'Pier trial identity') + if (stopRequested || existsSync(join(directory, stopFile))) { + const removedContainers = removeTrialContainers(config) + writeJsonAtomic(directory, terminalFile, { + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: 'stopped', + processExited: true, + containersRemoved: true, + removedContainers, + }) + process.exitCode = 0 + } else { + const spawned = spawn(config.command, config.args, { + cwd: config.cwd, + env: config.env, + detached: true, + stdio: 'ignore', + }) + if (spawned.pid === undefined) fail('Pier process started without a pid') + const exited = new Promise((resolveExit) => { + let settled = false + spawned.once('error', (error) => { + if (settled) return + settled = true + resolveExit({ code: null, signal: null, error }) + }) + spawned.once('close', (code, signal) => { + if (settled) return + settled = true + resolveExit({ code, signal }) + }) + }) + child = { process: spawned, exited } + childIdentity = processIdentity(spawned.pid) + writeJsonAtomic(directory, identityFile, { + ...identity, + state: 'running', + pier: childIdentity, + }) + + if (stopRequested || existsSync(join(directory, stopFile))) { + stopRequested = true + stopping = stopChild() + } + const exit = await exited + if (stopping) await stopping + else await stopChild() + const removedContainers = removeTrialContainers(config) + const status = stopRequested ? 'stopped' : exit.code === 0 ? 'completed' : 'failed' + writeJsonAtomic(directory, terminalFile, { + schemaVersion: 1, + kind: 'pier-trial-terminal', + status, + processExited: true, + containersRemoved: true, + removedContainers, + exitCode: exit.code, + signal: exit.signal, + ...(exit.error ? { error: 'Pier process failed to start' } : {}), + ...(status === 'failed' && !exit.error + ? { error: `Pier process exited ${exit.signal ?? exit.code ?? 'without status'}` } + : {}), + }) + } +} catch (error) { + const cleanupErrors = [] + try { + requestStop() + if (stopping) await stopping + } catch (cleanup) { + cleanupErrors.push(cleanup) + } + if (config) { + try { + removeTrialContainers(config) + } catch (cleanup) { + cleanupErrors.push(cleanup) + } + } + writeJsonAtomic(directory, terminalFile, { + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: 'failed', + processExited: !childIdentity || !processMatches(childIdentity), + containersRemoved: cleanupErrors.length === 0, + error: `${error instanceof Error ? error.message : String(error)}${ + cleanupErrors.length > 0 + ? `; cleanup failed: ${cleanupErrors.map((item) => item?.message ?? String(item)).join('; ')}` + : '' + }`, + }) + process.exitCode = 1 +} diff --git a/bench/src/smoke-structural-rollout.mts b/bench/src/smoke-structural-rollout.mts new file mode 100644 index 00000000..c2a42fc9 --- /dev/null +++ b/bench/src/smoke-structural-rollout.mts @@ -0,0 +1,393 @@ +/** + * smoke-structural-rollout — the ship gate for the PORTED structuralRollout strategy + * (src/runtime/structural-rollout.ts): prove the RUNTIME code path works against a REAL + * model on REAL HumanEval tasks. This runs `runAgentic` with the strategy over a + * `createVerifierEnvironment` surface — routerToolLoop, the conserved pool, the metered + * consult channel, `sandboxCheckRunner`, receipts — NOT the bench rig (hev-structural.mts). + * + * Honesty split (the rig's Phase A / Phase B, preserved): + * - The strategy sees ONLY task-visible information. The surface's check is INERT + * (always 0/1), so no hidden-test signal can reach selection or repair; the visible + * checks are the strategy's own default CheckSource (model-authored asserts, frozen + * before sampling), executed by the shipped `sandboxCheckRunner` over a docker + * `--network=none` exec channel (the thin adapter this script provides). + * - The task's own hidden check() suite grades every locked candidate HERE, in the + * script, AFTER the strategy has finished the task (`runChecker`, docker, + * --network=none). Nothing flows back. + * + * Per task we collect: the k per-sample hidden grades, the selected candidate's grade + * (argmax over samples by the exported `selectBestIndex` — the strategy's own order), + * the final (post-repair) grade (the receipt marked `selected`), authored-check counts, + * and the SelectionReceipts (cross-checked against the recorded outcomes). + * + * Run (key via dotenvx; never in the shell history): + * cd ~/company/devops/secrets && dotenvx run -f agent-state.env -- bash -c ' \ + * cd /home/drew/code/agent-runtime-swe && \ + * HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz N=20 OFFSET=0 npx tsx bench/src/smoke-structural-rollout.mts' + */ + +import { execFile } from 'node:child_process' +import { appendFileSync } from 'node:fs' +import { + type AgenticRunResult, + type CheckExecChannel, + type CheckOutcome, + type CheckRunner, + createVerifierEnvironment, + defaultStructuralRolloutPolicy, + runAgentic, + type StructuralRolloutResult, + sandboxCheckRunner, + selectBestIndex, + structuralRollout, + visibleCheckScore, +} from '../../src/runtime/index' +import { basePrompt, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval' + +function must(name: string): string { + const v = process.env[name] + if (!v) throw new Error(`env ${name} is required`) + return v +} + +const N = Number(process.env.N ?? 20) +const OFFSET = Number(process.env.OFFSET ?? 0) +const MODEL = process.env.MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' +const BASE = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' +const TEMP = Number(process.env.TEMPERATURE ?? 0.8) +const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 2500) +const CONCURRENCY = Number(process.env.CONCURRENCY ?? 3) +const OUT = process.env.OUT // optional JSONL of raw per-task rows + +const systemPrompt = 'You are an expert Python programmer.' +const dockerImage = 'python:3.12-slim' + +// ── The thin adapter: a docker --network=none exec channel for sandboxCheckRunner ──── +// The runner pipes its check program as `printf '%s' '' | base64 -d | python3 -`; +// this channel runs that command inside a jailed container. Infra faults (no docker, +// daemon down) fail loud; a candidate crash/hang is a real outcome (non-zero exit / no +// sentinel), returned, never thrown. Modeled on bench/src/benchmarks/humaneval.ts. +let containerSeq = 0 +function dockerExecChannel(): CheckExecChannel { + return { + exec(command, options) { + const timeoutMs = options?.timeoutMs ?? 20000 + const name = `srck-${process.pid}-${containerSeq++}` + return new Promise((resolve, reject) => { + let settled = false + const reap = () => execFile('docker', ['rm', '-f', name], () => {}) + const finish = (r: { exitCode: number; stdout: string; stderr: string }) => { + if (settled) return + settled = true + clearTimeout(backstop) + reap() + resolve(r) + } + const fail = (e: Error) => { + if (settled) return + settled = true + clearTimeout(backstop) + reap() + reject(e) + } + // execFile's timeout kills the docker CLIENT; a hung container could leave the + // callback unfired. The backstop guarantees resolution (no sentinel ⇒ the runner + // scores it crashed) and the named reap kills the stray container. + const backstop = setTimeout( + () => finish({ exitCode: 124, stdout: '', stderr: 'timed out (backstop)' }), + timeoutMs + 3000, + ) + execFile( + 'docker', + ['run', '--rm', '--name', name, '--network=none', '--cpus=1', '--memory=512m', dockerImage, 'sh', '-c', command], + { timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 }, + (err, stdout, stderr) => { + if (err) { + const e = err as NodeJS.ErrnoException & { code?: number | string } + if (e.code === 'ENOENT') { + fail(new Error('docker binary not found on PATH — cannot run visible checks')) + return + } + if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr ?? '')) { + fail(new Error(`docker daemon unreachable: ${(stderr ?? '').slice(0, 200)}`)) + return + } + const exitCode = typeof e.code === 'number' ? e.code : 1 + finish({ exitCode, stdout: stdout ?? '', stderr: stderr ?? '' }) + return + } + finish({ exitCode: 0, stdout: stdout ?? '', stderr: stderr ?? '' }) + }, + ) + }) + }, + } +} + +// ── Recording wrapper: capture each (candidate, outcome) the strategy scores, in order. +// Delegates verbatim to the shipped runner — the code under test stays the strategy. +interface ScoredCandidate { + candidate: string + outcome: CheckOutcome +} +function recordingRunner(inner: CheckRunner, log: ScoredCandidate[]): CheckRunner { + return { + async run(candidate, checks, ctx) { + const outcome = await inner.run(candidate, checks, ctx) + log.push({ candidate, outcome }) + return outcome + }, + } +} + +interface TaskRow { + taskId: string + error?: string + officialChecks: number + authoredChecks: number + repairStop: string + shots: number + sampleHidden: number[] + blindMean: number + selectedIdx: number + selectedHidden: number + finalIdx: number + finalHidden: number + selectedVisible: number + receipts: Array<{ candidateIndex: number; selected: boolean; score: number; reason: string }> + tokens: { input: number; output: number } + usd: number + ms: number +} + +async function runTask(t: HumanEvalTask): Promise { + const scored: ScoredCandidate[] = [] + const policy = { ...defaultStructuralRolloutPolicy, temperature: TEMP } + const strategy = structuralRollout({ + policy, + checkRunner: recordingRunner(sandboxCheckRunner({ box: dockerExecChannel() }), scored), + }) + // INERT check: the strategy's harness-verified score channel must carry no hidden + // signal — hidden grading is this script's job, after the rollout locks its artifacts. + const surface = createVerifierEnvironment({ + name: 'humaneval-inert', + check: () => ({ passes: 0, total: 1, errored: 0 }), + }) + const result = (await runAgentic({ + surface, + task: { + id: t.taskId, + systemPrompt, + userPrompt: basePrompt(t), + meta: { entryPoint: t.entryPoint }, + }, + routerBaseUrl: BASE, + routerKey: must('TOGETHER_API_KEY'), + model: MODEL, + temperature: TEMP, + maxTokens: MAX_TOKENS, + innerTurns: 2, + strategy, + // The strategy's documented sizing: k samples + repair rounds + the check-author consult. + budget: policy.k + policy.repairRounds + 1, + })) as AgenticRunResult & StructuralRolloutResult + + // Receipts ↔ recorded outcomes must agree exactly (candidateIndex is the recording + // order: samples first, then repairs). A mismatch is an adapter or strategy defect. + if (result.selection.length !== scored.length) { + throw new Error( + `${t.taskId}: ${result.selection.length} receipts vs ${scored.length} scored candidates`, + ) + } + for (const r of result.selection) { + const rec = scored[r.candidateIndex] + if (!rec || Math.abs(r.score - visibleCheckScore(rec.outcome)) > 1e-9) { + throw new Error(`${t.taskId}: receipt #${r.candidateIndex} score ${r.score} does not match the recorded outcome`) + } + } + + const sampleCount = result.selection.filter((r) => r.reason.startsWith('sample')).length + const samples = scored.slice(0, sampleCount) + if (samples.length === 0) throw new Error(`${t.taskId}: no sample candidates settled`) + + // Hidden grading (script-side, docker --network=none): every distinct candidate once. + const gradeCache = new Map>() + const grade = (candidate: string): Promise => { + let p = gradeCache.get(candidate) + if (!p) { + p = runChecker(t, candidate).then((r) => r.pass) + gradeCache.set(candidate, p) + } + return p + } + const sampleHidden = await Promise.all(samples.map((s) => grade(s.candidate))) + const selectedIdx = selectBestIndex(samples.map((s) => s.outcome)) + const selectedHidden = sampleHidden[selectedIdx] as number + const winner = result.selection.find((r) => r.selected) + if (!winner) throw new Error(`${t.taskId}: no receipt marked selected`) + const finalIdx = winner.candidateIndex + const finalHidden = await grade((scored[finalIdx] as ScoredCandidate).candidate) + + return { + taskId: t.taskId, + officialChecks: result.officialChecks, + authoredChecks: result.authoredChecks, + repairStop: result.repairStop, + shots: result.shots, + sampleHidden, + blindMean: sampleHidden.reduce((a, b) => a + b, 0) / sampleHidden.length, + selectedIdx, + selectedHidden, + finalIdx, + finalHidden, + selectedVisible: visibleCheckScore((samples[selectedIdx] as ScoredCandidate).outcome), + receipts: result.selection.map((r) => ({ + candidateIndex: r.candidateIndex, + selected: r.selected, + score: r.score, + reason: r.reason, + })), + tokens: result.tokens, + usd: result.usd, + ms: result.ms, + } +} + +async function pooled(items: T[], limit: number, fn: (item: T) => Promise): Promise { + const out: R[] = new Array(items.length) + let next = 0 + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const i = next + next += 1 + out[i] = await fn(items[i] as T) + } + }) + await Promise.all(workers) + return out +} + +let unhandled = 0 +process.on('unhandledRejection', (e) => { + unhandled += 1 + console.error('UNHANDLED REJECTION:', e) +}) + +const pct = (x: number) => `${(100 * x).toFixed(1)}%` + +async function main(): Promise { + must('TOGETHER_API_KEY') + const tasks = await loadHumanEval(N, OFFSET) + const { k, repairRounds, testgen } = defaultStructuralRolloutPolicy + console.log( + `=== structuralRollout RUNTIME smoke · n=${tasks.length} offset=${OFFSET} · k=${k} repairs<=${repairRounds} testgen=${testgen} temp=${TEMP} ===`, + ) + console.log(` model=${MODEL} base=${BASE} maxTokens=${MAX_TOKENS} task-concurrency=${CONCURRENCY}`) + console.log(` path: runAgentic → structuralRollout(default policy) → createVerifierEnvironment(inert) → sandboxCheckRunner(docker --network=none)`) + + const started = Date.now() + const rows = await pooled(tasks, CONCURRENCY, async (t): Promise => { + try { + const row = await runTask(t) + const hid = row.sampleHidden.join(' ') + console.log( + ` ${t.taskId.padEnd(14)} auth=${row.authoredChecks} samples=[${hid}] sel=#${row.selectedIdx}:${row.selectedHidden ? 'PASS' : 'fail'} final=#${row.finalIdx}:${row.finalHidden ? 'PASS' : 'fail'} ${row.repairStop}`, + ) + if (OUT) appendFileSync(OUT, `${JSON.stringify(row)}\n`) + return row + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + console.error(` ${t.taskId.padEnd(14)} ERROR: ${msg}`) + if (OUT) appendFileSync(OUT, `${JSON.stringify({ taskId: t.taskId, error: msg })}\n`) + return { + taskId: t.taskId, + error: msg, + officialChecks: 0, + authoredChecks: 0, + repairStop: 'error', + shots: 0, + sampleHidden: [], + blindMean: 0, + selectedIdx: -1, + selectedHidden: 0, + finalIdx: -1, + finalHidden: 0, + selectedVisible: 0, + receipts: [], + tokens: { input: 0, output: 0 }, + usd: 0, + ms: 0, + } + } + }) + + const ok = rows.filter((r) => !r.error) + const errored = rows.filter((r) => r.error) + + console.log('\n── per-task ──') + console.log( + 'task auth samples(hidden) blind% sel final visSel repairStop shots tok(in/out) ms', + ) + for (const r of rows) { + if (r.error) { + console.log(`${r.taskId.padEnd(15)} ERROR ${r.error.slice(0, 90)}`) + continue + } + console.log( + [ + r.taskId.padEnd(15), + String(r.authoredChecks).padEnd(5), + r.sampleHidden.join(' ').padEnd(16), + pct(r.blindMean).padEnd(8), + `#${r.selectedIdx}:${r.selectedHidden ? 'PASS' : 'fail'}`.padEnd(8), + `#${r.finalIdx}:${r.finalHidden ? 'PASS' : 'fail'}`.padEnd(8), + r.selectedVisible.toFixed(3).padEnd(7), + r.repairStop.padEnd(17), + String(r.shots).padEnd(6), + `${r.tokens.input}/${r.tokens.output}`.padEnd(15), + String(r.ms), + ].join(' '), + ) + } + + const blind = ok.reduce((a, r) => a + r.blindMean, 0) / Math.max(1, ok.length) + const selected = ok.reduce((a, r) => a + r.selectedHidden, 0) / Math.max(1, ok.length) + const final = ok.reduce((a, r) => a + r.finalHidden, 0) / Math.max(1, ok.length) + const sample0 = ok.reduce((a, r) => a + (r.sampleHidden[0] ?? 0), 0) / Math.max(1, ok.length) + const oracleAtK = ok.reduce((a, r) => a + (r.sampleHidden.some((x) => x === 1) ? 1 : 0), 0) / Math.max(1, ok.length) + const tokens = ok.reduce((a, r) => ({ input: a.input + r.tokens.input, output: a.output + r.tokens.output }), { input: 0, output: 0 }) + + console.log('\n── summary ──') + console.log(`tasks: ${ok.length} scored, ${errored.length} errored (of ${rows.length})`) + console.log(`blind mean-of-k : ${pct(blind)} (sample-0 only: ${pct(sample0)}; oracle@k ceiling: ${pct(oracleAtK)})`) + console.log(`selected@1 : ${pct(selected)} (lift over blind: ${(100 * (selected - blind)).toFixed(1)}pp)`) + console.log(`final (repaired): ${pct(final)} (lift over blind: ${(100 * (final - blind)).toFixed(1)}pp)`) + console.log(`spend: tokens ${tokens.input} in / ${tokens.output} out · wall ${((Date.now() - started) / 1000).toFixed(0)}s`) + + // ── Acceptance (mechanism, not significance — n is small) ── + const authoredTasks = ok.filter((r) => r.authoredChecks > 0).length + const receiptsSane = ok.every((r) => r.receipts.length > 0 && r.receipts.length === r.shots) + const ordering = final >= selected && selected >= blind + const liftPp = 100 * (final - blind) + const rescued = ok.filter((r) => r.finalHidden === 1 && (r.sampleHidden[0] ?? 0) === 0) + const checks: Array<[string, boolean, string]> = [ + ['authored checks on >=17/20 tasks', authoredTasks >= 17, `${authoredTasks}/${rows.length} tasks`], + ['selection receipts present, scores match recorded outcomes', receiptsSane && ok.length > 0, `verified on ${ok.length} tasks (hard-checked per receipt)`], + ['final >= selected >= blind and final-blind >= +5pp', ordering && liftPp >= 5, `blind ${pct(blind)} → selected ${pct(selected)} → final ${pct(final)} (+${liftPp.toFixed(1)}pp)`], + ['>=1 task rescued (final passes, sample 0 fails)', rescued.length >= 1, rescued.map((r) => r.taskId).join(', ') || 'none'], + ['zero crashes / unhandled rejections', errored.length === 0 && unhandled === 0, `${errored.length} task errors, ${unhandled} unhandled rejections`], + ] + console.log('\n── acceptance ──') + let allPass = true + for (const [label, pass, detail] of checks) { + if (!pass) allPass = false + console.log(` [${pass ? 'PASS' : 'FAIL'}] ${label} — ${detail}`) + } + console.log(allPass ? '\nSMOKE: PASS' : '\nSMOKE: FAIL') + process.exit(allPass ? 0 : 1) +} + +main().catch((e) => { + console.error(e) + process.exit(1) +}) diff --git a/bench/src/swe-bench-env.test.ts b/bench/src/swe-bench-env.test.ts index 30dbdc02..a2a93d78 100644 --- a/bench/src/swe-bench-env.test.ts +++ b/bench/src/swe-bench-env.test.ts @@ -1,27 +1,28 @@ +import assert from 'node:assert/strict' import { mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterAll, describe, expect, it } from 'vitest' +import { after, describe, it } from 'node:test' import { isInsideJail, isTestPath, jailPath } from './swe-bench-env' describe('isTestPath', () => { it('flags test directories and test-named python files', () => { - expect(isTestPath('tests/test_models.py')).toBe(true) - expect(isTestPath('pkg/test/helpers.py')).toBe(true) - expect(isTestPath('pkg/tests/helpers.py')).toBe(true) - expect(isTestPath('test_models.py')).toBe(true) - expect(isTestPath('models_test.py')).toBe(true) - expect(isTestPath('conftest.py')).toBe(true) - expect(isTestPath('pkg/conftest.py')).toBe(true) + assert.equal(isTestPath('tests/test_models.py'), true) + assert.equal(isTestPath('pkg/test/helpers.py'), true) + assert.equal(isTestPath('pkg/tests/helpers.py'), true) + assert.equal(isTestPath('test_models.py'), true) + assert.equal(isTestPath('models_test.py'), true) + assert.equal(isTestPath('conftest.py'), true) + assert.equal(isTestPath('pkg/conftest.py'), true) }) it('does not flag ordinary source files', () => { - expect(isTestPath('src/foo.py')).toBe(false) - expect(isTestPath('pkg/models.py')).toBe(false) + assert.equal(isTestPath('src/foo.py'), false) + assert.equal(isTestPath('pkg/models.py'), false) // `testing.py` is not a test file by the test_/_test/conftest rules. - expect(isTestPath('pkg/testing.py')).toBe(false) + assert.equal(isTestPath('pkg/testing.py'), false) // A `latest/` segment must not trip the `tests?/` directory rule. - expect(isTestPath('latest/foo.py')).toBe(false) + assert.equal(isTestPath('latest/foo.py'), false) }) }) @@ -29,15 +30,15 @@ describe('jailPath', () => { const root = '/work/repo' it('rejects `..` traversal and absolute paths', () => { - expect(jailPath(root, '../x')).toBeNull() - expect(jailPath(root, 'a/../../etc/passwd')).toBeNull() - expect(jailPath(root, '/etc/passwd')).toBeNull() + assert.equal(jailPath(root, '../x'), null) + assert.equal(jailPath(root, 'a/../../etc/passwd'), null) + assert.equal(jailPath(root, '/etc/passwd'), null) }) it('accepts in-repo relative paths and strips a leading `./`', () => { - expect(jailPath(root, 'src/a.py')).toBe('src/a.py') - expect(jailPath(root, './a.py')).toBe('a.py') - expect(jailPath(root, 'a.py')).toBe('a.py') + assert.equal(jailPath(root, 'src/a.py'), 'src/a.py') + assert.equal(jailPath(root, './a.py'), 'a.py') + assert.equal(jailPath(root, 'a.py'), 'a.py') }) }) @@ -46,13 +47,13 @@ describe('isInsideJail (realpath containment)', () => { // assert containment. Offline — operates on a throwaway temp dir, no git clone, no network. const dir = mkdtempSync(join(tmpdir(), 'swe-jail-')) const jailRoot = realpathSync(dir) - afterAll(() => rmSync(dir, { recursive: true, force: true })) + after(() => rmSync(dir, { recursive: true, force: true })) it('admits a real file inside the jail', () => { const inside = join(dir, 'a.py') writeFileSync(inside, 'x = 1\n') - expect(isInsideJail(jailRoot, realpathSync(inside))).toBe(true) - expect(isInsideJail(jailRoot, jailRoot)).toBe(true) + assert.equal(isInsideJail(jailRoot, realpathSync(inside)), true) + assert.equal(isInsideJail(jailRoot, jailRoot), true) }) it('rejects reading through a symlink that escapes the jail', () => { @@ -61,11 +62,11 @@ describe('isInsideJail (realpath containment)', () => { symlinkSync('/etc', link) // `resolveInJail` does `realpathSync(join(ws.dir, relPath))` then this containment check. const real = realpathSync(join(dir, 'escape/passwd')) - expect(real).toBe('/etc/passwd') - expect(isInsideJail(jailRoot, real)).toBe(false) + assert.equal(real, '/etc/passwd') + assert.equal(isInsideJail(jailRoot, real), false) }) it('rejects a sibling dir that shares the jail-root prefix', () => { - expect(isInsideJail('/tmp/swe-x', '/tmp/swe-x-evil/secret')).toBe(false) + assert.equal(isInsideJail('/tmp/swe-x', '/tmp/swe-x-evil/secret'), false) }) }) diff --git a/bench/src/tb-container-executor.test.mts b/bench/src/tb-container-executor.test.mts index 180f07d2..8afc3569 100644 --- a/bench/src/tb-container-executor.test.mts +++ b/bench/src/tb-container-executor.test.mts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict' import { chmod, mkdtemp, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' -import type { AgentSpec, ExecutorContext } from '../../src/runtime/index' +import type { AgentSpec, ExecutorContext } from '@tangle-network/agent-runtime/loops' import { buildTbDockerExecArgs, createTbContainerExecutor } from './tb-container-executor.mts' const spec: AgentSpec = { profile: { name: 'tb-test-worker' }, harness: null } diff --git a/bench/tsconfig.public.json b/bench/tsconfig.public.json new file mode 100644 index 00000000..e688a3f6 --- /dev/null +++ b/bench/tsconfig.public.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "module": "esnext", + "target": "es2023", + "lib": ["es2023"], + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"], + "esModuleInterop": true, + "resolveJsonModule": true + }, + "files": ["src/index.ts"] +} diff --git a/docs/agent-optimization-map.md b/docs/agent-optimization-map.md deleted file mode 100644 index 09c1b29c..00000000 --- a/docs/agent-optimization-map.md +++ /dev/null @@ -1,91 +0,0 @@ -# The Agent Optimization Map - -**What this document is.** -The one-page truth about optimizing agents on this substrate: every lever an `AgentProfile` exposes, what machinery exists to optimize or create each one, what is actually wired to the paved path, and what has been proven on live infrastructure. -Written to be read by humans and by agents configuring an improvement run. -Audited 2026-07-06 against `agent-interface`, `agent-eval`, `agent-runtime`, `agent-knowledge`, and the supervisor-lab live campaigns (runs 1–6). - -## The mental model in three sentences - -An **AgentProfile is the complete specification of an agent**: its brain-instructions, capabilities, and starting world. -**`improve()` is the one verb** that optimizes any single lever of that profile against an executable judge, with a statistical held-out gate deciding promotion. -Everything else in the stack is either a **proposer** (something that generates candidate lever-values), a **judge** (something that scores results), or a **loop** (something that runs proposers against judges until the gate ships or holds). - -## Lever inventory — what an AgentProfile actually carries - -| Lever (your words) | Profile field | In the type? | -|---|---|---| -| system prompt | `prompt.systemPrompt` (+ per-mode `modes`) | yes | -| skills | `resources.skills` (SKILL.md packages) | yes | -| tools | `tools` (enable/disable map) + `resources.tools` (file-based tool defs) | yes | -| MCP | `mcp` (server map) + `connections` | yes | -| knowledge base | no first-class field — lives as `resources.files` content or app-side (agent-knowledge KB store) | partial | -| resources | `resources.{instructions,agents,commands}` | yes | -| starting files | `resources.files` (materialized into the workspace before execution) | yes | -| full sandbox/VM state | backend/image choice + `extensions` (non-portable) + `resources.files` | partial — files yes, image/toolchain is the tcloud backend's, not the profile's | - -Also in the type and often forgotten: `subagents` (native sub-agents), `permissions`, `model` hints, `hooks`. - -## Optimization matrix — lever × machinery × wiring × proof - -| Surface | Baseline/apply plumbing | Proposer that exists | Reachable from `improve()`? | Proven live? | -|---|---|---|---|---| -| `prompt` | yes | `gepaProposer` (Pareto frontier + crossover-merge + reflective mutation) | **yes (default)** | **yes — supervisor-lab runs 3–6, ~1,700 sandbox cells** | -| `skills` | yes | `skillOptProposer` (skill-document patching) | **yes (default)** | no live run anywhere yet | -| `tools` | yes (JSON string surface) | none wired; `parameterSweepProposer` is shape-compatible for toggle/config sweeps | no — fails loud, needs `generator` | no | -| `mcp` | yes (JSON string surface) | none | no — fails loud | no | -| `hooks` | yes (JSON string surface) | none | no — fails loud | no | -| `code` (tools/harness/anything in a repo) | yes | `improvementDriver` + `agenticGenerator` (real coding harness per candidate worktree, verify-gated) | **yes — `code: { repoRoot }` facade (#480)** | offline test only; live milestone pending | -| `resources.files` (starting world) | **no surface** | none | no | no | -| knowledge base | **no surface** | agent-knowledge loops exist (below) but nothing writes back into a profile | no | no | - -**The one-sentence verdict: every lever you believe in is representable; two are optimizable-and-proven (prompt today, code as of #480); skills is one command away; tools/mcp/hooks are string surfaces awaiting a config proposer; files/knowledge aren't surfaces at all yet.** - -## The proposer zoo — what each one is for (plain words) - -| Proposer | What it does | Wired to paved path? | -|---|---|---| -| `gepaProposer` | evolves prompt text: keeps a frontier of variants that each win somewhere, merges them, mutates with evidence from best/worst trials | yes (prompt default) | -| `skillOptProposer` | same idea for skill documents (structured patches) | yes (skills default) | -| `fapoProposer` | **the meta-policy**: attribute each failure to a level, propose ONE scoped change, escalate prompt → parameters → structure only when the cheaper level is exhausted — with pluggable per-level generators | **no** — exported, zero consumers | -| `traceAnalystProposer` | turns trace-analyst findings (an Ax agent that reads execution traces with tools) into candidates | **no** — exported, zero consumers | -| `aceProposer` | append-only "playbook" curator: accumulates provenance-tagged lessons without ever summarizing old ones away (anti context-collapse) | no | -| `memoryCurationProposer` | the dedup-and-replace contrast to ACE | no | -| `parameterSweepProposer` | sweeps numeric/config parameters | no | -| `haloProposer` | hierarchical analyst-driven optimization | no | - -Since #480/#310/#311 the paved path also has: per-generation **failure distiller** (the proposer automatically sees each generation's worst cells + judge reasons), **power preflight** (minimum detectable lift computed from baseline cells; structurally-hopeless budgets warn), durable `runDir`, and the statistical holdout gate everywhere (the point-estimate gate was folded away). - -## Creation loops — making new things, not just mutating strings - -What you asked for → what exists: - -- **"Create new skills by mining traces"** — components all exist, composition does not: `trace-analyst` (reads traces, emits findings) → `skillOptProposer`/`aceProposer` (turn findings into skill-document content) → `improve(surface: 'skills')` (prove them). Nobody has connected the three. One composition, high value. -- **"Auto-research loops that write code"** — exists as of #480: `improve(surface: 'code', code: { repoRoot, verify })` runs a real coding harness per candidate in a worktree, gated by any executable judge. "Create a new tool" = point it at the repo where tools live. -- **"Knowledge discovery / find relevant OSS libraries / find MCPs"** — the research machinery exists in `agent-knowledge` (`runVerifiedResearchLoop`: two-agent verified research with claim-grounding; `discovery`; collection drivers) and is consumed by physim/legal/tax products — but **no bridge writes research results into an AgentProfile lever**. The missing primitive is small: a `knowledge → resources.files/skills` materializer, after which "research the ecosystem, propose an MCP server for the profile, prove it on the benchmark" is a normal `improve()` run over the `mcp` surface. -- **"Full sandbox initial state as a lever"** — `resources.files` covers workspace seeding; making it an `ImproveSurface` (candidate = a file-set, judge = task performance) is unbuilt but fits the existing surface contract exactly. - -## The three gaps that matter (everything else is garnish) - -1. **Reachability**: the zoo (FAPO, trace-analyst, ACE, sweep) needs one dial — `improve(..., { proposer: 'fapo' })` — instead of hand-assembly. FAPO especially: it *is* the "optimize everything, escalate sensibly" policy this document describes, already grounded in the paper, already pluggable. -2. **The knowledge bridge**: research loops produce verified knowledge; profiles can carry files/skills; nothing connects them. -3. **This document didn't exist.** Now it does; keep it honest — every row's "proven live" column only flips with a linked run. - -## Recipes (copy-paste truth, today) - -```ts -// Optimize a prompt against any executable judge (proven, runs 3–6): -await improve(profile, findings, { surface: 'prompt', scenarios, judge, agent, runDir }) - -// Optimize the skill documents: -await improve(profile, findings, { surface: 'skills', scenarios, judge, agent, runDir }) - -// Evolve CODE (tools, harnesses, algorithms) with a real coding agent per candidate: -await improve(profile, findings, { - surface: 'code', - code: { repoRoot, verify: commandVerifier({ command: 'pnpm', args: ['test'] }) }, - scenarios, judge, agent, runDir, -}) - -// Every result now carries result.power — read it BEFORE buying a bigger search. -``` diff --git a/docs/api/README.md b/docs/api/README.md index 4a62f568..a5ebd062 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -8,12 +8,14 @@ - [agent](agent.md) - [analyst-loop](analyst-loop.md) +- [candidate-execution](candidate-execution.md) - [index](index.md) - [intelligence](intelligence.md) - [knowledge](knowledge.md) - [lifecycle](lifecycle.md) - [mcp](mcp.md) - [platform](platform.md) +- [primeintellect](primeintellect.md) - [profiles](profiles.md) - [runtime/environment-provider](runtime/environment-provider.md) - [runtime](runtime.md) diff --git a/docs/api/agent.md b/docs/api/agent.md index 745c55c7..8f3eff75 100644 --- a/docs/api/agent.md +++ b/docs/api/agent.md @@ -865,7 +865,7 @@ Required for `mode === 'open-pr'` — the GH owner/repo (`tangle-network/tax-age ##### allowCreateForKinds? -> `optional` **allowCreateForKinds?**: readonly (`"knowledge.wiki"` \| `"knowledge.claim"` \| `"knowledge.raw"` \| `"knowledge.stale"` \| `"system-prompt"` \| `"tool-doc"` \| `"new-tool"` \| `"rag"` \| `"memory"` \| `"scaffolding"` \| `"output-schema"` \| `"websearch.outdated"` \| `"prior-run-summary"` \| `"cluster"`)[] +> `optional` **allowCreateForKinds?**: readonly (`"code"` \| `"mcp"` \| `"memory"` \| `"agent-profile"` \| `"skill"` \| `"hook"` \| `"subagent"` \| `"knowledge.wiki"` \| `"knowledge.claim"` \| `"knowledge.raw"` \| `"knowledge.stale"` \| `"system-prompt"` \| `"tool-doc"` \| `"new-tool"` \| `"workflow"` \| `"rollout-policy"` \| `"rag"` \| `"scaffolding"` \| `"output-schema"` \| `"websearch.outdated"` \| `"prior-run-summary"` \| `"cluster"`)[] Defined in: [agent/improvement-adapter.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/improvement-adapter.ts#L100) @@ -1544,11 +1544,75 @@ Defined in: [agent/surfaces.ts:55](https://github.com/tangle-network/agent-runti Optional: single file defining the output schema (Zod / JSON Schema). +##### skills? + +> `optional` **skills?**: `string` + +Defined in: [agent/surfaces.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L57) + +Optional: directory containing Agent Skill packages. + +##### mcp? + +> `optional` **mcp?**: `string` + +Defined in: [agent/surfaces.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L59) + +Optional: directory containing MCP server/tool configuration. + +##### hooks? + +> `optional` **hooks?**: `string` + +Defined in: [agent/surfaces.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L61) + +Optional: directory containing hook definitions. + +##### subagents? + +> `optional` **subagents?**: `string` + +Defined in: [agent/surfaces.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L63) + +Optional: directory containing subagent definitions. + +##### workflows? + +> `optional` **workflows?**: `string` + +Defined in: [agent/surfaces.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L65) + +Optional: directory containing orchestration/workflow policies. + +##### rolloutPolicy? + +> `optional` **rolloutPolicy?**: `string` + +Defined in: [agent/surfaces.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L67) + +Optional: single file containing rollout-policy settings. + +##### agentProfile? + +> `optional` **agentProfile?**: `string` + +Defined in: [agent/surfaces.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L69) + +Optional: single canonical AgentProfile file. + +##### code? + +> `optional` **code?**: `string` + +Defined in: [agent/surfaces.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L71) + +Optional: source root for code findings. + *** ### ResolvedSurface -Defined in: [agent/surfaces.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L58) +Defined in: [agent/surfaces.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L74) #### Properties @@ -1556,7 +1620,7 @@ Defined in: [agent/surfaces.ts:58](https://github.com/tangle-network/agent-runti > **absolutePath**: `string` -Defined in: [agent/surfaces.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L60) +Defined in: [agent/surfaces.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L76) Absolute filesystem path the operator can `cat` / `vim`. @@ -1564,7 +1628,7 @@ Absolute filesystem path the operator can `cat` / `vim`. > **repoRelativePath**: `string` -Defined in: [agent/surfaces.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L62) +Defined in: [agent/surfaces.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L78) Repo-relative path for PR descriptions, diffs, audit logs. @@ -1572,7 +1636,7 @@ Repo-relative path for PR descriptions, diffs, audit logs. > **exists**: `boolean` -Defined in: [agent/surfaces.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L64) +Defined in: [agent/surfaces.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L80) Whether the path currently exists on disk. @@ -1580,7 +1644,7 @@ Whether the path currently exists on disk. > **intent**: `"edit-existing"` \| `"create-new"` -Defined in: [agent/surfaces.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L66) +Defined in: [agent/surfaces.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L82) The substrate's intent: edit an existing file or create a new one. @@ -1588,7 +1652,7 @@ The substrate's intent: edit an existing file or create a new one. ### SurfaceValidationIssue -Defined in: [agent/surfaces.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L191) +Defined in: [agent/surfaces.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L264) Validate that every declared surface exists on disk under `repoRoot`. @@ -1603,19 +1667,19 @@ the loop produces 20 minutes later). > **surface**: keyof [`AgentSurfaces`](#agentsurfaces) -Defined in: [agent/surfaces.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L192) +Defined in: [agent/surfaces.ts:265](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L265) ##### path > **path**: `string` -Defined in: [agent/surfaces.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L193) +Defined in: [agent/surfaces.ts:266](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L266) ##### reason > **reason**: `"missing"` \| `"not-directory"` \| `"not-file"` -Defined in: [agent/surfaces.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L194) +Defined in: [agent/surfaces.ts:267](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L267) ## Type Aliases @@ -1993,7 +2057,7 @@ resolves only after the iterator drains. > **resolveSubjectPath**(`subject`, `surfaces`, `repoRoot`): [`ResolvedSurface`](#resolvedsurface) \| `null` -Defined in: [agent/surfaces.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L86) +Defined in: [agent/surfaces.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L102) Resolve a parsed `FindingSubject` to the file path the substrate should edit (or create) on disk. @@ -2035,7 +2099,7 @@ it's the whole point. > **validateSurfaces**(`surfaces`, `repoRoot`): readonly [`SurfaceValidationIssue`](#surfacevalidationissue)[] -Defined in: [agent/surfaces.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L198) +Defined in: [agent/surfaces.ts:271](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L271) Validate an `AgentSurfaces` map on disk — missing paths fail loud at `defineAgent` time instead of silently skipping self-improvement edits. @@ -2059,7 +2123,7 @@ readonly [`SurfaceValidationIssue`](#surfacevalidationissue)[] > **renderSurfaceIssues**(`issues`, `repoRoot`): `string` -Defined in: [agent/surfaces.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L247) +Defined in: [agent/surfaces.ts:345](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/surfaces.ts#L345) Format a list of surface validation issues into a human-readable error string. diff --git a/docs/api/candidate-execution.md b/docs/api/candidate-execution.md new file mode 100644 index 00000000..80323719 --- /dev/null +++ b/docs/api/candidate-execution.md @@ -0,0 +1,577 @@ +[**@tangle-network/agent-runtime**](README.md) + +*** + +[@tangle-network/agent-runtime](README.md) / candidate-execution + +# candidate-execution + +## References + +### AgentCandidateCodeSource + +Re-exports [AgentCandidateCodeSource](index.md#agentcandidatecodesource) + +*** + +### AgentCandidateCodeSurfaceSource + +Re-exports [AgentCandidateCodeSurfaceSource](index.md#agentcandidatecodesurfacesource) + +*** + +### AgentCandidateProfileSource + +Re-exports [AgentCandidateProfileSource](index.md#agentcandidateprofilesource) + +*** + +### BuildAgentCandidateBundleInput + +Re-exports [BuildAgentCandidateBundleInput](index.md#buildagentcandidatebundleinput) + +*** + +### buildAgentCandidateBundle + +Re-exports [buildAgentCandidateBundle](index.md#buildagentcandidatebundle) + +*** + +### AgentCandidateBundleInput + +Re-exports [AgentCandidateBundleInput](index.md#agentcandidatebundleinput) + +*** + +### sealAgentCandidateBundle + +Re-exports [sealAgentCandidateBundle](index.md#sealagentcandidatebundle) + +*** + +### AgentCandidateExecutionAttemptRecord + +Re-exports [AgentCandidateExecutionAttemptRecord](index.md#agentcandidateexecutionattemptrecord) + +*** + +### AgentCandidateExecutionAttemptRef + +Re-exports [AgentCandidateExecutionAttemptRef](index.md#agentcandidateexecutionattemptref) + +*** + +### AgentCandidateExecutionClaim + +Re-exports [AgentCandidateExecutionClaim](index.md#agentcandidateexecutionclaim) + +*** + +### AgentCandidateExecutionClaimResult + +Re-exports [AgentCandidateExecutionClaimResult](index.md#agentcandidateexecutionclaimresult) + +*** + +### AgentCandidateExecutionClaimStore + +Re-exports [AgentCandidateExecutionClaimStore](index.md#agentcandidateexecutionclaimstore) + +*** + +### AgentCandidateExecutionCleanupHandles + +Re-exports [AgentCandidateExecutionCleanupHandles](index.md#agentcandidateexecutioncleanuphandles) + +*** + +### AgentCandidateExecutionFailureClass + +Re-exports [AgentCandidateExecutionFailureClass](index.md#agentcandidateexecutionfailureclass) + +*** + +### AgentCandidateExecutionFinishResult + +Re-exports [AgentCandidateExecutionFinishResult](index.md#agentcandidateexecutionfinishresult) + +*** + +### AgentCandidateExecutionLease + +Re-exports [AgentCandidateExecutionLease](index.md#agentcandidateexecutionlease) + +*** + +### AgentCandidateExecutionPhase + +Re-exports [AgentCandidateExecutionPhase](index.md#agentcandidateexecutionphase) + +*** + +### AgentCandidateExecutionPhaseResult + +Re-exports [AgentCandidateExecutionPhaseResult](index.md#agentcandidateexecutionphaseresult) + +*** + +### AgentCandidateExecutionRecoveryEvidence + +Re-exports [AgentCandidateExecutionRecoveryEvidence](index.md#agentcandidateexecutionrecoveryevidence) + +*** + +### AgentCandidateExecutionStageResult + +Re-exports [AgentCandidateExecutionStageResult](index.md#agentcandidateexecutionstageresult) + +*** + +### AgentCandidateExecutionTerminalRecord + +Re-exports [AgentCandidateExecutionTerminalRecord](index.md#agentcandidateexecutionterminalrecord) + +*** + +### AgentCandidateExecutionTerminalResult + +Re-exports [AgentCandidateExecutionTerminalResult](index.md#agentcandidateexecutionterminalresult) + +*** + +### AgentCandidateExecutionUsage + +Re-exports [AgentCandidateExecutionUsage](index.md#agentcandidateexecutionusage) + +*** + +### AgentCandidateRetryRejection + +Re-exports [AgentCandidateRetryRejection](index.md#agentcandidateretryrejection) + +*** + +### InMemoryAgentCandidateExecutionClaimStore + +Re-exports [InMemoryAgentCandidateExecutionClaimStore](index.md#inmemoryagentcandidateexecutionclaimstore) + +*** + +### FileAgentCandidateExecutionClaimStore + +Re-exports [FileAgentCandidateExecutionClaimStore](index.md#fileagentcandidateexecutionclaimstore) + +*** + +### FileAgentCandidateExecutionClaimStoreOptions + +Re-exports [FileAgentCandidateExecutionClaimStoreOptions](index.md#fileagentcandidateexecutionclaimstoreoptions) + +*** + +### candidateExecutionClaim + +Re-exports [candidateExecutionClaim](index.md#candidateexecutionclaim) + +*** + +### DisposePreparedAgentCandidateOptions + +Re-exports [DisposePreparedAgentCandidateOptions](index.md#disposepreparedagentcandidateoptions) + +*** + +### disposePreparedAgentCandidateExecution + +Re-exports [disposePreparedAgentCandidateExecution](index.md#disposepreparedagentcandidateexecution) + +*** + +### ExecutePreparedAgentCandidateOptions + +Re-exports [ExecutePreparedAgentCandidateOptions](index.md#executepreparedagentcandidateoptions) + +*** + +### executePreparedAgentCandidate + +Re-exports [executePreparedAgentCandidate](index.md#executepreparedagentcandidate) + +*** + +### persistCandidateOutputArtifact + +Re-exports [persistCandidateOutputArtifact](index.md#persistcandidateoutputartifact) + +*** + +### PrepareAgentCandidateExecutionOptions + +Re-exports [PrepareAgentCandidateExecutionOptions](index.md#prepareagentcandidateexecutionoptions) + +*** + +### prepareAgentCandidateExecution + +Re-exports [prepareAgentCandidateExecution](index.md#prepareagentcandidateexecution) + +*** + +### applyExactAgentProfileDiff + +Re-exports [applyExactAgentProfileDiff](index.md#applyexactagentprofilediff) + +*** + +### parseExactAgentProfile + +Re-exports [parseExactAgentProfile](index.md#parseexactagentprofile) + +*** + +### parseExactAgentProfileDiff + +Re-exports [parseExactAgentProfileDiff](index.md#parseexactagentprofilediff) + +*** + +### AgentCandidateModelGrantActivateInput + +Re-exports [AgentCandidateModelGrantActivateInput](index.md#agentcandidatemodelgrantactivateinput) + +*** + +### AgentCandidateModelGrantClient + +Re-exports [AgentCandidateModelGrantClient](index.md#agentcandidatemodelgrantclient) + +*** + +### AgentCandidateModelGrantReservation + +Re-exports [AgentCandidateModelGrantReservation](index.md#agentcandidatemodelgrantreservation) + +*** + +### AgentCandidateModelGrantReserveInput + +Re-exports [AgentCandidateModelGrantReserveInput](index.md#agentcandidatemodelgrantreserveinput) + +*** + +### AgentCandidateModelGrantSettleInput + +Re-exports [AgentCandidateModelGrantSettleInput](index.md#agentcandidatemodelgrantsettleinput) + +*** + +### CreateProtectedAgentCandidateModelPortOptions + +Re-exports [CreateProtectedAgentCandidateModelPortOptions](index.md#createprotectedagentcandidatemodelportoptions) + +*** + +### createProtectedAgentCandidateModelPort + +Re-exports [createProtectedAgentCandidateModelPort](index.md#createprotectedagentcandidatemodelport) + +*** + +### RecoverExpiredAgentCandidateOptions + +Re-exports [RecoverExpiredAgentCandidateOptions](index.md#recoverexpiredagentcandidateoptions) + +*** + +### recoverExpiredAgentCandidateExecution + +Re-exports [recoverExpiredAgentCandidateExecution](index.md#recoverexpiredagentcandidateexecution) + +*** + +### AgentCandidateArtifactPort + +Re-exports [AgentCandidateArtifactPort](index.md#agentcandidateartifactport) + +*** + +### AgentCandidateBenchmarkGraderIdentity + +Re-exports [AgentCandidateBenchmarkGraderIdentity](index.md#agentcandidatebenchmarkgraderidentity) + +*** + +### AgentCandidateBenchmarkGraderPort + +Re-exports [AgentCandidateBenchmarkGraderPort](index.md#agentcandidatebenchmarkgraderport) + +*** + +### AgentCandidateContainerPort + +Re-exports [AgentCandidateContainerPort](index.md#agentcandidatecontainerport) + +*** + +### AgentCandidateExecutionPorts + +Re-exports [AgentCandidateExecutionPorts](index.md#agentcandidateexecutionports) + +*** + +### AgentCandidateExecutorFinalCapture + +Re-exports [AgentCandidateExecutorFinalCapture](index.md#agentcandidateexecutorfinalcapture) + +*** + +### AgentCandidateExecutorMemoryCapture + +Re-exports [AgentCandidateExecutorMemoryCapture](index.md#agentcandidateexecutormemorycapture) + +*** + +### AgentCandidateExecutorPort + +Re-exports [AgentCandidateExecutorPort](index.md#agentcandidateexecutorport) + +*** + +### AgentCandidateExecutorProfileFile + +Re-exports [AgentCandidateExecutorProfileFile](index.md#agentcandidateexecutorprofilefile) + +*** + +### AgentCandidateExecutorRequest + +Re-exports [AgentCandidateExecutorRequest](index.md#agentcandidateexecutorrequest) + +*** + +### AgentCandidateExecutorStopRequest + +Re-exports [AgentCandidateExecutorStopRequest](index.md#agentcandidateexecutorstoprequest) + +*** + +### AgentCandidateExecutorTaskOutcomeCapture + +Re-exports [AgentCandidateExecutorTaskOutcomeCapture](index.md#agentcandidateexecutortaskoutcomecapture) + +*** + +### AgentCandidateExecutorWorkspaceFile + +Re-exports [AgentCandidateExecutorWorkspaceFile](index.md#agentcandidateexecutorworkspacefile) + +*** + +### AgentCandidateExecutorWorkspaceInput + +Re-exports [AgentCandidateExecutorWorkspaceInput](index.md#agentcandidateexecutorworkspaceinput) + +*** + +### AgentCandidateMemoryPort + +Re-exports [AgentCandidateMemoryPort](index.md#agentcandidatememoryport) + +*** + +### AgentCandidateMemoryResetResult + +Re-exports [AgentCandidateMemoryResetResult](index.md#agentcandidatememoryresetresult) + +*** + +### AgentCandidateModelLimits + +Re-exports [AgentCandidateModelLimits](index.md#agentcandidatemodellimits) + +*** + +### AgentCandidateModelPort + +Re-exports [AgentCandidateModelPort](index.md#agentcandidatemodelport) + +*** + +### AgentCandidateOutputArtifactPort + +Re-exports [AgentCandidateOutputArtifactPort](index.md#agentcandidateoutputartifactport) + +*** + +### AgentCandidateOutputPurpose + +Re-exports [AgentCandidateOutputPurpose](index.md#agentcandidateoutputpurpose) + +*** + +### AgentCandidateProtectedModelActivation + +Re-exports [AgentCandidateProtectedModelActivation](index.md#agentcandidateprotectedmodelactivation) + +*** + +### AgentCandidateProtectedModelCall + +Re-exports [AgentCandidateProtectedModelCall](index.md#agentcandidateprotectedmodelcall) + +*** + +### AgentCandidateProtectedModelReservation + +Re-exports [AgentCandidateProtectedModelReservation](index.md#agentcandidateprotectedmodelreservation) + +*** + +### AgentCandidateProtectedModelSettlement + +Re-exports [AgentCandidateProtectedModelSettlement](index.md#agentcandidateprotectedmodelsettlement) + +*** + +### AgentCandidateProtectedRunCapture + +Re-exports [AgentCandidateProtectedRunCapture](index.md#agentcandidateprotectedruncapture) + +*** + +### AgentCandidateRepositoryPort + +Re-exports [AgentCandidateRepositoryPort](index.md#agentcandidaterepositoryport) + +*** + +### AgentCandidateRunFinalization + +Re-exports [AgentCandidateRunFinalization](index.md#agentcandidaterunfinalization) + +*** + +### AgentCandidateTaskExecution + +Re-exports [AgentCandidateTaskExecution](index.md#agentcandidatetaskexecution) + +*** + +### AgentCandidateVerificationPorts + +Re-exports [AgentCandidateVerificationPorts](index.md#agentcandidateverificationports) + +*** + +### AgentCandidateWorkspacePort + +Re-exports [AgentCandidateWorkspacePort](index.md#agentcandidateworkspaceport) + +*** + +### CANDIDATE\_TRACE\_ENV + +Re-exports [CANDIDATE_TRACE_ENV](index.md#candidate_trace_env) + +*** + +### CANDIDATE\_TRACE\_TAGS + +Re-exports [CANDIDATE_TRACE_TAGS](index.md#candidate_trace_tags) + +*** + +### CanonicalCandidateDocument + +Re-exports [CanonicalCandidateDocument](index.md#canonicalcandidatedocument) + +*** + +### PreparedAgentCandidateExecution + +Re-exports [PreparedAgentCandidateExecution](index.md#preparedagentcandidateexecution) + +*** + +### PreparedAgentCandidateInstruction + +Re-exports [PreparedAgentCandidateInstruction](index.md#preparedagentcandidateinstruction) + +*** + +### PreparedAgentCandidateLaunch + +Re-exports [PreparedAgentCandidateLaunch](index.md#preparedagentcandidatelaunch) + +*** + +### PreparedAgentCandidateTrace + +Re-exports [PreparedAgentCandidateTrace](index.md#preparedagentcandidatetrace) + +*** + +### ResolvedAgentCandidateContainer + +Re-exports [ResolvedAgentCandidateContainer](index.md#resolvedagentcandidatecontainer) + +*** + +### VerifiedAgentCandidate + +Re-exports [VerifiedAgentCandidate](index.md#verifiedagentcandidate) + +*** + +### VerifiedAgentCandidateTaskOutcome + +Re-exports [VerifiedAgentCandidateTaskOutcome](index.md#verifiedagentcandidatetaskoutcome) + +*** + +### verifyAgentCandidateBundle + +Re-exports [verifyAgentCandidateBundle](index.md#verifyagentcandidatebundle) + +*** + +### AgentCandidateWorkspaceArchiveLimits + +Re-exports [AgentCandidateWorkspaceArchiveLimits](index.md#agentcandidateworkspacearchivelimits) + +*** + +### CaptureAgentCandidateWorkspaceOptions + +Re-exports [CaptureAgentCandidateWorkspaceOptions](index.md#captureagentcandidateworkspaceoptions) + +*** + +### CapturedAgentCandidateWorkspace + +Re-exports [CapturedAgentCandidateWorkspace](index.md#capturedagentcandidateworkspace) + +*** + +### CreateAgentCandidateWorkspacePortOptions + +Re-exports [CreateAgentCandidateWorkspacePortOptions](index.md#createagentcandidateworkspaceportoptions) + +*** + +### captureAgentCandidateWorkspace + +Re-exports [captureAgentCandidateWorkspace](index.md#captureagentcandidateworkspace) + +*** + +### captureAgentCandidateWorkspaceFiles + +Re-exports [captureAgentCandidateWorkspaceFiles](index.md#captureagentcandidateworkspacefiles) + +*** + +### createAgentCandidateWorkspacePort + +Re-exports [createAgentCandidateWorkspacePort](index.md#createagentcandidateworkspaceport) diff --git a/docs/api/index.md b/docs/api/index.md index 4d93ec8d..6129081b 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -8,6 +8,352 @@ ## Classes +### FileAgentCandidateExecutionClaimStore + +Defined in: [candidate-execution/claim-file-store.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L74) + +Cross-process lifecycle implemented as fsynced, create-if-absent records. + +#### Implements + +- [`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore) + +#### Constructors + +##### Constructor + +> **new FileAgentCandidateExecutionClaimStore**(`options`): [`FileAgentCandidateExecutionClaimStore`](#fileagentcandidateexecutionclaimstore) + +Defined in: [candidate-execution/claim-file-store.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L78) + +###### Parameters + +###### options + +[`FileAgentCandidateExecutionClaimStoreOptions`](#fileagentcandidateexecutionclaimstoreoptions) + +###### Returns + +[`FileAgentCandidateExecutionClaimStore`](#fileagentcandidateexecutionclaimstore) + +#### Methods + +##### tryClaim() + +> **tryClaim**(`requested`): `Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> + +Defined in: [candidate-execution/claim-file-store.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L86) + +###### Parameters + +###### requested + +[`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`tryClaim`](#tryclaim-1) + +##### getAttempt() + +> **getAttempt**(`requestedAttempt`): `Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> + +Defined in: [candidate-execution/claim-file-store.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L115) + +###### Parameters + +###### requestedAttempt + +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`getAttempt`](#getattempt-1) + +##### markCandidateMayRun() + +> **markCandidateMayRun**(`requestedLease`): `Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> + +Defined in: [candidate-execution/claim-file-store.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L121) + +Persist the point after which candidate code may have run. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`markCandidateMayRun`](#markcandidatemayrun-1) + +##### stageTerminal() + +> **stageTerminal**(`requestedLease`, `result`): `Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> + +Defined in: [candidate-execution/claim-file-store.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L158) + +Fsync the complete terminal record into the durable outbox. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### result + +[`AgentCandidateExecutionTerminalResult`](#agentcandidateexecutionterminalresult) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`stageTerminal`](#stageterminal-1) + +##### finish() + +> **finish**(`requestedLease`, `requestedTerminalDigest`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/claim-file-store.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L191) + +Publish exactly the staged terminal identified by `terminalDigest`. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### requestedTerminalDigest + +`` `sha256:${string}` `` + +###### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`finish`](#finish-1) + +##### recoverExpired() + +> **recoverExpired**(`requestedAttempt`, `evidence`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/claim-file-store.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L228) + +Write a failed terminal only after the lease expired and a trusted worker +independently proved process death plus model and memory closure. + +###### Parameters + +###### requestedAttempt + +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +###### evidence + +[`AgentCandidateExecutionRecoveryEvidence`](#agentcandidateexecutionrecoveryevidence) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`recoverExpired`](#recoverexpired-1) + +*** + +### InMemoryAgentCandidateExecutionClaimStore + +Defined in: [candidate-execution/claim.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L290) + +Single-process lifecycle implementation. + +#### Implements + +- [`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore) + +#### Constructors + +##### Constructor + +> **new InMemoryAgentCandidateExecutionClaimStore**(`options?`): [`InMemoryAgentCandidateExecutionClaimStore`](#inmemoryagentcandidateexecutionclaimstore) + +Defined in: [candidate-execution/claim.ts:296](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L296) + +###### Parameters + +###### options? + +`InMemoryAgentCandidateExecutionClaimStoreOptions` = `{}` + +###### Returns + +[`InMemoryAgentCandidateExecutionClaimStore`](#inmemoryagentcandidateexecutionclaimstore) + +#### Methods + +##### tryClaim() + +> **tryClaim**(`requested`): `Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> + +Defined in: [candidate-execution/claim.ts:300](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L300) + +###### Parameters + +###### requested + +[`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`tryClaim`](#tryclaim-1) + +##### getAttempt() + +> **getAttempt**(`requestedAttempt`): `Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> + +Defined in: [candidate-execution/claim.ts:323](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L323) + +###### Parameters + +###### requestedAttempt + +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`getAttempt`](#getattempt-1) + +##### markCandidateMayRun() + +> **markCandidateMayRun**(`requestedLease`): `Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> + +Defined in: [candidate-execution/claim.ts:333](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L333) + +Persist the point after which candidate code may have run. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`markCandidateMayRun`](#markcandidatemayrun-1) + +##### stageTerminal() + +> **stageTerminal**(`requestedLease`, `result`): `Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> + +Defined in: [candidate-execution/claim.ts:350](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L350) + +Fsync the complete terminal record into the durable outbox. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### result + +[`AgentCandidateExecutionTerminalResult`](#agentcandidateexecutionterminalresult) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`stageTerminal`](#stageterminal-1) + +##### finish() + +> **finish**(`requestedLease`, `requestedTerminalDigest`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/claim.ts:365](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L365) + +Publish exactly the staged terminal identified by `terminalDigest`. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### requestedTerminalDigest + +`` `sha256:${string}` `` + +###### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`finish`](#finish-1) + +##### recoverExpired() + +> **recoverExpired**(`requestedAttempt`, `evidence`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/claim.ts:383](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L383) + +Write a failed terminal only after the lease expired and a trusted worker +independently proved process death plus model and memory closure. + +###### Parameters + +###### requestedAttempt + +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +###### evidence + +[`AgentCandidateExecutionRecoveryEvidence`](#agentcandidateexecutionrecoveryevidence) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`recoverExpired`](#recoverexpired-1) + +*** + ### CircuitOpenError Defined in: [conversation/call-policy.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L43) @@ -849,7 +1195,7 @@ Defined in: [sessions.ts:49](https://github.com/tangle-network/agent-runtime/blo ###### Implementation of -[`RuntimeSessionStore`](#runtimesessionstore).[`put`](#put-1) +[`RuntimeSessionStore`](#runtimesessionstore).[`put`](#put-2) ##### appendEvent() @@ -897,7057 +1243,11463 @@ Defined in: [sessions.ts:59](https://github.com/tangle-network/agent-runtime/blo ## Interfaces -### CircuitBreakerConfig +### AgentCandidateCodeSurfaceSource -Defined in: [conversation/call-policy.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L24) +Defined in: [candidate-execution/builder.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L44) -Circuit-breaker tuning. `failuresToOpen` consecutive failures opens it; closed only after `cooldownMs`. +The only accepted path from an agent-eval code candidate to executable bytes. #### Properties -##### failuresToOpen +##### kind -> **failuresToOpen**: `number` +> **kind**: `"code-surface"` -Defined in: [conversation/call-policy.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L25) +Defined in: [candidate-execution/builder.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L45) -##### cooldownMs +##### surface -> **cooldownMs**: `number` +> **surface**: `CodeSurface` -Defined in: [conversation/call-policy.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L26) +Defined in: [candidate-execution/builder.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L46) -*** +##### repository -### BackendCallPolicy +> **repository**: `AgentCandidateGitHubRepository` -Defined in: [conversation/call-policy.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L29) +Defined in: [candidate-execution/builder.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L47) -#### Properties +##### worktreeDir? -##### perAttemptDeadlineMs? +> `optional` **worktreeDir?**: `string` -> `optional` **perAttemptDeadlineMs?**: `number` +Defined in: [candidate-execution/builder.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L49) -Defined in: [conversation/call-policy.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L31) +Optional parent directory used to resolve a relative `surface.worktreeRef`. -Per-attempt wall clock limit. Exceeding fires an AbortSignal and is treated as a retryable failure. +*** -##### maxRetries? +### BuildAgentCandidateBundleInput -> `optional` **maxRetries?**: `number` +Defined in: [candidate-execution/builder.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L59) -Defined in: [conversation/call-policy.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L33) +Complete measured surfaces and execution policy compiled into one candidate bundle. -Number of retries after the first attempt; total attempts = 1 + maxRetries. Default 0. +#### Properties -##### retryBackoffMs? +##### profile -> `optional` **retryBackoffMs?**: [`RetryBackoff`](#retrybackoff) +> **profile**: [`AgentCandidateProfileSource`](#agentcandidateprofilesource) -Defined in: [conversation/call-policy.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L35) +Defined in: [candidate-execution/builder.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L60) -Backoff between attempts. Default 250ms with jitter. +##### code -##### isRetryable? +> **code**: [`AgentCandidateCodeSource`](#agentcandidatecodesource) -> `optional` **isRetryable?**: [`RetryableErrorPredicate`](#retryableerrorpredicate) +Defined in: [candidate-execution/builder.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L61) -Defined in: [conversation/call-policy.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L37) +##### execution -Custom retry classifier. Defaults to [defaultIsRetryable](#defaultisretryable). +> **execution**: `AgentCandidateExecution` -##### circuitBreaker? +Defined in: [candidate-execution/builder.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L62) -> `optional` **circuitBreaker?**: [`CircuitBreakerConfig`](#circuitbreakerconfig) +##### knowledge? -Defined in: [conversation/call-policy.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L39) +> `optional` **knowledge?**: `AgentCandidateKnowledge` -Circuit breaker that opens after N consecutive failures per participant. +Defined in: [candidate-execution/builder.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L63) -*** +##### memory -### SqlAdapter +> **memory**: `AgentCandidateMemoryPolicy` -Defined in: [conversation/journal-sql.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L49) +Defined in: [candidate-execution/builder.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L64) -Minimal SQL driver shape. Implementations forward to whichever client the -deployment already uses; agent-runtime takes no opinion on which. +##### lineage -Parameter placeholders MUST be `?` (positional). All adapters listed in the -file header accept this convention. +> **lineage**: `Omit`\<`AgentCandidateLineage`, `"profileDiffIds"`\> -#### Methods +Defined in: [candidate-execution/builder.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L66) -##### exec() +`profileDiffIds` is derived from `profile`; callers cannot contradict it. -> **exec**(`sql`, `params?`): `Promise`\<\{ `rowsAffected`: `number`; \}\> +*** -Defined in: [conversation/journal-sql.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L51) +### FileAgentCandidateExecutionClaimStoreOptions -Execute a write statement (INSERT/UPDATE/DELETE/DDL). +Defined in: [candidate-execution/claim-file-store.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L66) -###### Parameters +#### Properties -###### sql +##### directory -`string` +> **directory**: `string` -###### params? +Defined in: [candidate-execution/claim-file-store.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L68) -readonly `unknown`[] +Evaluator-owned directory shared by every process allowed to execute candidates. + +##### now? + +> `optional` **now?**: () => `number` + +Defined in: [candidate-execution/claim-file-store.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L70) + +Testable evaluator clock; defaults to `Date.now`. ###### Returns -`Promise`\<\{ `rowsAffected`: `number`; \}\> +`number` -##### query() +*** -> **query**\<`TRow`\>(`sql`, `params?`): `Promise`\<`TRow`[]\> +### AgentCandidateExecutionCleanupHandles -Defined in: [conversation/journal-sql.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L53) +Defined in: [candidate-execution/claim.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L39) -Execute a read statement (SELECT). Returns rows as plain objects. +Non-secret identities a trusted recovery worker needs to close an abandoned attempt. -###### Type Parameters +#### Properties -###### TRow +##### preparationId -`TRow` = `Record`\<`string`, `unknown`\> +> `readonly` **preparationId**: `string` -###### Parameters +Defined in: [candidate-execution/claim.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L40) -###### sql +##### modelGrantDigest -`string` +> `readonly` **modelGrantDigest**: `` `sha256:${string}` `` -###### params? +Defined in: [candidate-execution/claim.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L41) -readonly `unknown`[] +##### resolvedModel -###### Returns +> `readonly` **resolvedModel**: `AgentCandidateResolvedModel` -`Promise`\<`TRow`[]\> +Defined in: [candidate-execution/claim.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L42) -*** +##### traceRunId -### D1DatabaseLike +> `readonly` **traceRunId**: `string` -Defined in: [conversation/journal-sql.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L84) +Defined in: [candidate-execution/claim.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L43) -Structural type matching the surface of `D1Database` we depend on, so the -SDK never imports `@cloudflare/workers-types`. Consumers pass their real -`D1Database` from `env.DB` and TS structural compatibility lines it up. +##### cleanupTimeoutMs -#### Methods +> `readonly` **cleanupTimeoutMs**: `number` -##### prepare() +Defined in: [candidate-execution/claim.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L44) -> **prepare**(`sql`): [`D1StmtLike`](#d1stmtlike) +##### memory? -Defined in: [conversation/journal-sql.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L85) +> `readonly` `optional` **memory?**: `object` -###### Parameters +Defined in: [candidate-execution/claim.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L45) -###### sql +###### accessDigest -`string` +> `readonly` **accessDigest**: `` `sha256:${string}` `` -###### Returns +###### effectiveNamespace -[`D1StmtLike`](#d1stmtlike) +> `readonly` **effectiveNamespace**: `string` *** -### D1StmtLike +### AgentCandidateExecutionClaim -Defined in: [conversation/journal-sql.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L87) +Defined in: [candidate-execution/claim.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L52) -#### Methods +Immutable signed identity stored for one execution attempt. -##### bind() +#### Properties -> **bind**(...`params`): [`D1StmtLike`](#d1stmtlike) +##### executionId -Defined in: [conversation/journal-sql.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L88) +> `readonly` **executionId**: `string` -###### Parameters +Defined in: [candidate-execution/claim.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L53) -###### params +##### attempt -...`unknown`[] +> `readonly` **attempt**: `number` -###### Returns +Defined in: [candidate-execution/claim.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L54) -[`D1StmtLike`](#d1stmtlike) +##### maxAttempts -##### run() +> `readonly` **maxAttempts**: `number` -> **run**(): `Promise`\<`unknown`\> +Defined in: [candidate-execution/claim.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L55) -Defined in: [conversation/journal-sql.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L89) +##### retryPolicy -###### Returns +> `readonly` **retryPolicy**: `"none"` \| `"pre-model-infrastructure-only"` -`Promise`\<`unknown`\> +Defined in: [candidate-execution/claim.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L56) -##### all() +##### bundleDigest -> **all**\<`TRow`\>(): `Promise`\<\{ `results?`: `TRow`[]; \}\> +> `readonly` **bundleDigest**: `` `sha256:${string}` `` -Defined in: [conversation/journal-sql.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L90) +Defined in: [candidate-execution/claim.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L57) -###### Type Parameters +##### executionPlanDigest -###### TRow +> `readonly` **executionPlanDigest**: `` `sha256:${string}` `` -`TRow` = `unknown` +Defined in: [candidate-execution/claim.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L58) -###### Returns +##### retryLineageDigest -`Promise`\<\{ `results?`: `TRow`[]; \}\> +> `readonly` **retryLineageDigest**: `` `sha256:${string}` `` -*** +Defined in: [candidate-execution/claim.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L60) -### ConversationJournalEntry +Frozen plan identity with only attempt number and per-attempt grant identity normalized. -Defined in: [conversation/journal.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L20) +##### leaseExpiresAtMs -#### Properties +> `readonly` **leaseExpiresAtMs**: `number` -##### runId +Defined in: [candidate-execution/claim.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L62) -> **runId**: `string` +The winning lease stops authorizing a new terminal write at this instant. -Defined in: [conversation/journal.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L21) +##### resultTimeoutMs -##### startedAt +> `readonly` **resultTimeoutMs**: `number` -> **startedAt**: `string` +Defined in: [candidate-execution/claim.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L64) -Defined in: [conversation/journal.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L22) +Frozen budget for task verification, executable grading, and receipt construction. -##### halted? +##### cleanup -> `optional` **halted?**: [`HaltReason`](#haltreason) +> `readonly` **cleanup**: [`AgentCandidateExecutionCleanupHandles`](#agentcandidateexecutioncleanuphandles) -Defined in: [conversation/journal.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L24) +Defined in: [candidate-execution/claim.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L66) -Set when the run reaches a terminal state. +Non-secret handles retained so an expired attempt can be closed and reconciled. -##### endedAt? +*** -> `optional` **endedAt?**: `string` +### AgentCandidateExecutionLease -Defined in: [conversation/journal.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L25) +Defined in: [candidate-execution/claim.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L70) -##### turns +Secret capability required to finish the acquired attempt. -> **turns**: [`ConversationTurn`](#conversationturn)[] +#### Properties -Defined in: [conversation/journal.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L26) +##### executionId -*** +> `readonly` **executionId**: `string` -### ConversationJournal +Defined in: [candidate-execution/claim.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L71) -Defined in: [conversation/journal.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L29) +##### attempt -#### Methods +> `readonly` **attempt**: `number` -##### loadRun() +Defined in: [candidate-execution/claim.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L72) -> **loadRun**(`runId`): `Promise`\<[`ConversationJournalEntry`](#conversationjournalentry) \| `undefined`\> +##### token -Defined in: [conversation/journal.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L36) +> `readonly` **token**: `string` -Load any prior state for `runId`. Returns `undefined` for a fresh run. -Implementations MUST NOT mutate the returned object — the runner clones -before continuing — but the runtime treats absence and emptiness -identically, so a journal with zero turns is equivalent to "fresh." +Defined in: [candidate-execution/claim.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L73) -###### Parameters +##### expiresAtMs -###### runId +> `readonly` **expiresAtMs**: `number` -`string` +Defined in: [candidate-execution/claim.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L74) -###### Returns +*** -`Promise`\<[`ConversationJournalEntry`](#conversationjournalentry) \| `undefined`\> +### AgentCandidateExecutionUsage -##### beginRun() +Defined in: [candidate-execution/claim.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L85) -> **beginRun**(`runId`, `startedAt`): `Promise`\<`void`\> +Exact fixed-point usage proven by the closed evaluator model ledger. -Defined in: [conversation/journal.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L43) +#### Properties -Initialise journal state for a fresh run. Called once per run, before any -`appendTurn`. Idempotent: calling with an existing runId is a no-op if -the entry already exists with the same `startedAt`. +##### costUsdNanos -###### Parameters +> `readonly` **costUsdNanos**: `number` -###### runId +Defined in: [candidate-execution/claim.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L86) -`string` +##### inputTokens -###### startedAt +> `readonly` **inputTokens**: `number` -`string` +Defined in: [candidate-execution/claim.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L87) -###### Returns +##### outputTokens -`Promise`\<`void`\> +> `readonly` **outputTokens**: `number` -##### appendTurn() +Defined in: [candidate-execution/claim.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L88) -> **appendTurn**(`runId`, `turn`): `Promise`\<`void`\> +##### cachedInputTokens -Defined in: [conversation/journal.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L50) +> `readonly` **cachedInputTokens**: `number` -Append a committed turn. The runner only calls this AFTER the turn's -backend stream completed and the credit total has been updated, so an -appended turn is observed-committed and never speculative. +Defined in: [candidate-execution/claim.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L89) -###### Parameters +##### reasoningTokens -###### runId +> `readonly` **reasoningTokens**: `number` -`string` +Defined in: [candidate-execution/claim.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L90) -###### turn +##### modelCalls -[`ConversationTurn`](#conversationturn) +> `readonly` **modelCalls**: `number` -###### Returns +Defined in: [candidate-execution/claim.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L91) -`Promise`\<`void`\> +*** -##### recordHalt() +### AgentCandidateExecutionRecoveryEvidence -> **recordHalt**(`runId`, `halt`, `endedAt`): `Promise`\<`void`\> +Defined in: [candidate-execution/claim.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L128) -Defined in: [conversation/journal.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L56) +Trusted, independently observed closure facts for one expired winning lease. -Record the run's terminal halt reason + end time. Once called, the run -is observed-final; subsequent `loadRun` returns the same halt. +#### Properties -###### Parameters +##### failureClass -###### runId +> `readonly` **failureClass**: [`AgentCandidateExecutionFailureClass`](#agentcandidateexecutionfailureclass) -`string` +Defined in: [candidate-execution/claim.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L129) -###### halt +##### usage -[`HaltReason`](#haltreason) +> `readonly` **usage**: [`AgentCandidateExecutionUsage`](#agentcandidateexecutionusage) -###### endedAt +Defined in: [candidate-execution/claim.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L130) -`string` +##### modelSettlement -###### Returns +> `readonly` **modelSettlement**: `AgentCandidateArtifactRef` -`Promise`\<`void`\> +Defined in: [candidate-execution/claim.ts:131](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L131) -*** +##### failureEvidence? -### RunPersonaConversationOptions +> `readonly` `optional` **failureEvidence?**: `AgentCandidateArtifactRef` -Defined in: [conversation/run-persona.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L36) +Defined in: [candidate-execution/claim.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L132) -#### Properties +##### process -##### worker +> `readonly` **process**: `object` -> **worker**: `AgentProfile` +Defined in: [candidate-execution/claim.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L133) -Defined in: [conversation/run-persona.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L38) +###### stopped -The agent under test. Metered; its rendered prompt leads its turns. +> `readonly` **stopped**: `true` -##### persona +###### executionPlanDigest -> **persona**: [`PersonaDriver`](#personadriver) +> `readonly` **executionPlanDigest**: `` `sha256:${string}` `` -Defined in: [conversation/run-persona.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L40) +##### model -The simulated user driving the dialogue. +> `readonly` **model**: `object` -##### backendFor +Defined in: [candidate-execution/claim.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L137) -> **backendFor**: (`profile`, `role`) => [`AgentExecutionBackend`](#agentexecutionbackend) +###### closed -Defined in: [conversation/run-persona.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L43) +> `readonly` **closed**: `true` -Turn an `AgentProfile` into a runnable backend (router / sandbox / fake). - Applied to the worker and to a `profile`-kind persona. +###### preparationId -###### Parameters +> `readonly` **preparationId**: `string` -###### profile +###### grantDigest -`AgentProfile` +> `readonly` **grantDigest**: `` `sha256:${string}` `` -###### role +##### memory? -`"worker"` \| `"persona"` +> `readonly` `optional` **memory?**: `object` -###### Returns +Defined in: [candidate-execution/claim.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L142) -[`AgentExecutionBackend`](#agentexecutionbackend) +###### closed -##### systemPromptOf +> `readonly` **closed**: `true` -> **systemPromptOf**: (`profile`) => `string` +###### preparationId -Defined in: [conversation/run-persona.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L45) +> `readonly` **preparationId**: `string` -Render a profile's system prompt — prepended to that profile's messages. +###### accessDigest -###### Parameters +> `readonly` **accessDigest**: `` `sha256:${string}` `` -###### profile +###### effectiveNamespace -`AgentProfile` +> `readonly` **effectiveNamespace**: `string` -###### Returns +*** -`string` +### AgentCandidateExecutionAttemptRef -##### maxTurns? +Defined in: [candidate-execution/claim.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L150) -> `optional` **maxTurns?**: `number` +#### Properties -Defined in: [conversation/run-persona.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L48) +##### executionId -Speaker-turn cap. Default for a scripted persona = `2 * turns.length` - (worker answers each user turn). REQUIRED for a `profile` persona. +> `readonly` **executionId**: `string` -##### seed? +Defined in: [candidate-execution/claim.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L151) -> `optional` **seed?**: `string` +##### attempt -Defined in: [conversation/run-persona.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L50) +> `readonly` **attempt**: `number` -Kickoff message routed to the first speaker (the persona). Default 'Begin.' +Defined in: [candidate-execution/claim.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L152) -##### haltOn? +*** -> `optional` **haltOn?**: [`HaltPredicate`](#haltpredicate) +### AgentCandidateExecutionAttemptRecord -Defined in: [conversation/run-persona.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L53) +Defined in: [candidate-execution/claim.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L156) -Content-based "until satisfied" halt, called after every turn. `maxTurns` is the - hard ceiling; this is the early stop (the persona declares the goal met / unreachable). +Persisted state available to a fresh trusted recovery worker after a crash. -##### signal? +#### Properties -> `optional` **signal?**: `AbortSignal` +##### claim -Defined in: [conversation/run-persona.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L54) +> `readonly` **claim**: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) -##### workerName? +Defined in: [candidate-execution/claim.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L157) -> `optional` **workerName?**: `string` +##### phase -Defined in: [conversation/run-persona.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L56) +> `readonly` **phase**: [`AgentCandidateExecutionPhase`](#agentcandidateexecutionphase) -Worker participant / transcript speaker label. Default 'agent'. +Defined in: [candidate-execution/claim.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L158) -*** +##### staged? -### PersonaConversationResult +> `readonly` `optional` **staged?**: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord) -Defined in: [conversation/run-persona.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L59) +Defined in: [candidate-execution/claim.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L160) -#### Properties +Durable outbox content written before the terminal compare-and-set. -##### transcript +##### terminal? -> **transcript**: [`ConversationTurn`](#conversationturn)[] +> `readonly` `optional` **terminal?**: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord) -Defined in: [conversation/run-persona.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L60) +Defined in: [candidate-execution/claim.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L161) -##### turns +*** -> **turns**: `number` +### AgentCandidateExecutionClaimStore -Defined in: [conversation/run-persona.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L61) +Defined in: [candidate-execution/claim.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L233) -##### halted +Atomic one-shot store for candidate execution attempts. -> **halted**: [`HaltReason`](#haltreason) +Implementations must linearize both methods across every process sharing the +store. Terminal publication is deliberately two-step: `stageTerminal` +fsyncs the complete immutable outbox record, then `finish` publishes exactly +those staged bytes by digest. A crash between the two leaves recoverable +evidence rather than an ambiguous completed run. -Defined in: [conversation/run-persona.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L62) +#### Methods -##### costUsd +##### tryClaim() -> **costUsd**: `number` +> **tryClaim**(`claim`): `Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> -Defined in: [conversation/run-persona.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L64) +Defined in: [candidate-execution/claim.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L234) -Worker-only spend (the side under test). +###### Parameters -##### tokensIn +###### claim -> **tokensIn**: `number` - -Defined in: [conversation/run-persona.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L65) - -##### tokensOut - -> **tokensOut**: `number` +[`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) -Defined in: [conversation/run-persona.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L66) +###### Returns -*** +`Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> -### RunPersonaConfig +##### getAttempt() -Defined in: [conversation/run-persona.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L198) +> **getAttempt**(`attempt`): `Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> -#### Type Parameters +Defined in: [candidate-execution/claim.ts:235](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L235) -##### TScenario +###### Parameters -`TScenario` *extends* `Scenario` +###### attempt -##### TArtifact +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) -`TArtifact` +###### Returns -#### Properties +`Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> -##### backendFor +##### markCandidateMayRun() -> **backendFor**: (`profile`, `role`) => [`AgentExecutionBackend`](#agentexecutionbackend) +> **markCandidateMayRun**(`lease`): `Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> -Defined in: [conversation/run-persona.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L200) +Defined in: [candidate-execution/claim.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L239) -Turn an `AgentProfile` into a runnable backend (router / sandbox / fake). +Persist the point after which candidate code may have run. ###### Parameters -###### profile - -`AgentProfile` - -###### role +###### lease -`"worker"` \| `"persona"` +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) ###### Returns -[`AgentExecutionBackend`](#agentexecutionbackend) +`Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> -##### systemPromptOf +##### stageTerminal() -> **systemPromptOf**: (`profile`) => `string` +> **stageTerminal**(`lease`, `result`): `Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> -Defined in: [conversation/run-persona.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L202) +Defined in: [candidate-execution/claim.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L243) -Render a profile's system prompt. +Fsync the complete terminal record into the durable outbox. ###### Parameters -###### profile +###### lease -`AgentProfile` +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### result + +[`AgentCandidateExecutionTerminalResult`](#agentcandidateexecutionterminalresult) ###### Returns -`string` +`Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> -##### personaOf +##### finish() -> **personaOf**: (`scenario`) => [`PersonaDriver`](#personadriver) +> **finish**(`lease`, `terminalDigest`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -Defined in: [conversation/run-persona.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L204) +Defined in: [candidate-execution/claim.ts:248](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L248) -The persona driving each scenario — a driver profile or scripted turns. +Publish exactly the staged terminal identified by `terminalDigest`. ###### Parameters -###### scenario +###### lease -`TScenario` +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### terminalDigest + +`` `sha256:${string}` `` ###### Returns -[`PersonaDriver`](#personadriver) +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -##### artifactOf +##### recoverExpired() -> **artifactOf**: (`transcript`, `scenario`) => `TArtifact` +> **recoverExpired**(`attempt`, `evidence`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -Defined in: [conversation/run-persona.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L206) +Defined in: [candidate-execution/claim.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L256) -Build the scored artifact from the finished transcript. +Write a failed terminal only after the lease expired and a trusted worker +independently proved process death plus model and memory closure. ###### Parameters -###### transcript +###### attempt -[`ConversationTurn`](#conversationturn)[] +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) -###### scenario +###### evidence -`TScenario` +[`AgentCandidateExecutionRecoveryEvidence`](#agentcandidateexecutionrecoveryevidence) ###### Returns -`TArtifact` - -##### maxTurns? +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -> `optional` **maxTurns?**: (`scenario`) => `number` +*** -Defined in: [conversation/run-persona.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L208) +### DisposePreparedAgentCandidateOptions -Speaker-turn cap (required when a persona is profile-driven). +Defined in: [candidate-execution/dispose.ts:10](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/dispose.ts#L10) -###### Parameters +#### Properties -###### scenario +##### cleanupTimeoutMs? -`TScenario` +> `optional` **cleanupTimeoutMs?**: `number` -###### Returns +Defined in: [candidate-execution/dispose.ts:11](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/dispose.ts#L11) -`number` +*** -##### seed? +### ExecutePreparedAgentCandidateOptions -> `optional` **seed?**: (`scenario`) => `string` +Defined in: [candidate-execution/execute.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L61) -Defined in: [conversation/run-persona.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L209) +#### Properties -###### Parameters +##### executor -###### scenario +> **executor**: [`AgentCandidateExecutorPort`](#agentcandidateexecutorport) -`TScenario` +Defined in: [candidate-execution/execute.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L62) -###### Returns +##### grader -`string` +> **grader**: [`AgentCandidateBenchmarkGraderPort`](#agentcandidatebenchmarkgraderport) -##### workerName? +Defined in: [candidate-execution/execute.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L63) -> `optional` **workerName?**: `string` +##### outputArtifacts -Defined in: [conversation/run-persona.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L210) +> **outputArtifacts**: [`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) -*** +Defined in: [candidate-execution/execute.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L64) -### ConversationParticipant +##### traceStore -Defined in: [conversation/types.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L21) +> **traceStore**: `TraceStore` -#### Stable +Defined in: [candidate-execution/execute.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L65) -#### Properties +##### claimStore -##### name +> **claimStore**: [`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore) -> **name**: `string` +Defined in: [candidate-execution/execute.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L67) -Defined in: [conversation/types.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L26) +Long-lived evaluator-owned store shared by every process that can run this benchmark. -Stable name used as the speaker label in the transcript. Must be unique -within a `Conversation`. +##### cleanupTimeoutMs? -##### backend +> `optional` **cleanupTimeoutMs?**: `number` -> **backend**: [`AgentExecutionBackend`](#agentexecutionbackend) +Defined in: [candidate-execution/execute.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L69) -Defined in: [conversation/types.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L33) +Maximum time to prove process death and revoke protected access after a run ends. -Backend that runs this participant's turn. Reuses the existing -`AgentExecutionBackend` contract from `runAgentTaskStream`, so any -registered backend (iterable, sandbox, OpenAI-compatible) works without -adaptation. +##### resultTimeoutMs? -##### label? +> `optional` **resultTimeoutMs?**: `number` -> `optional` **label?**: `string` +Defined in: [candidate-execution/execute.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L71) -Defined in: [conversation/types.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L38) +Maximum time for task verification, executable grading, and receipt construction. -Optional human label for traces / dashboards. Distinct from `name`, which -is the addressing key. +*** -##### callPolicy? +### PrepareAgentCandidateExecutionOptions -> `optional` **callPolicy?**: [`BackendCallPolicy`](#backendcallpolicy) +Defined in: [candidate-execution/prepare.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L91) -Defined in: [conversation/types.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L44) +#### Properties -Optional per-participant override of the conversation's default -`callPolicy`. Use to tighten the deadline or raise the retry budget for -a participant known to be slow or flaky. +##### cleanupTimeoutMs? -##### authSource? +> `optional` **cleanupTimeoutMs?**: `number` -> `optional` **authSource?**: [`AuthSource`](#authsource-1) +Defined in: [candidate-execution/prepare.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L92) -Defined in: [conversation/types.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L64) +##### resultTimeoutMs? -Who pays for THIS participant's outbound calls? +> `optional` **resultTimeoutMs?**: `number` -- `'forward-user'` (default) — propagate the caller's - `X-Tangle-Forwarded-Authorization` so the downstream gateway bills the - original user. Right for pass-through agents that aggregate/route - without taking economic risk. -- `'agent-owned'` — DO NOT forward the user's auth; the participant's - backend uses its own credentials (typically a sk-tan-AGENT or x402 - wallet baked into the backend at construction). Downstream charges - land on the agent, not the user. Right for resold-bundle agents that - take margin between their inbound price and their sub-agent costs. -- `(state) => AuthSource` — per-turn / per-condition decision, e.g. base - sub-services are agent-owned but premium add-ons forward the user. +Defined in: [candidate-execution/prepare.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L94) -The agent's own credentials live on the backend (set at construction -time, e.g. `createOpenAICompatibleBackend({ apiKey })`); this field is -purely about *whether to also forward the user's identity downstream*. +Maximum time for task verification, executable grading, and receipt construction. *** -### ConversationDriveState +### AgentCandidateModelGrantClient -Defined in: [conversation/types.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L77) +Defined in: [candidate-execution/protected-model-port.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L39) -#### Stable +Narrow transport contract for a service that owns scoped model credentials +and the authoritative per-call usage ledger. -#### Extended by +An HTTP client can bind these methods to control-plane endpoints. Keeping +transport out of the runtime prevents parent credentials, endpoint paths, +and retry policy from becoming part of the portable candidate contract. -- [`HaltContext`](#haltcontext) +#### Methods -#### Properties +##### reserve() -##### transcript +> **reserve**(`input`): `Promise`\<[`AgentCandidateProtectedModelReservation`](#agentcandidateprotectedmodelreservation)\> -> **transcript**: readonly [`ConversationTurn`](#conversationturn)[] +Defined in: [candidate-execution/protected-model-port.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L40) -Defined in: [conversation/types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L78) +###### Parameters -##### turnIndex +###### input -> **turnIndex**: `number` +###### executionId -Defined in: [conversation/types.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L79) +`string` -##### spentCreditsCents +###### preparationId -> **spentCreditsCents**: `number` +`string` -Defined in: [conversation/types.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L80) +###### expiresAtMs -*** +`number` -### HaltContext +###### attempt -Defined in: [conversation/types.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L84) +`AgentCandidateAttemptPolicy` -#### Stable +###### bundleDigest -#### Extends +`` `sha256:${string}` `` -- [`ConversationDriveState`](#conversationdrivestate) +###### resolved -#### Properties +`AgentCandidateResolvedModel` -##### transcript +###### limits -> **transcript**: readonly [`ConversationTurn`](#conversationturn)[] +[`AgentCandidateModelLimits`](#agentcandidatemodellimits) -Defined in: [conversation/types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L78) +###### Returns -###### Inherited from +`Promise`\<[`AgentCandidateProtectedModelReservation`](#agentcandidateprotectedmodelreservation)\> -[`ConversationDriveState`](#conversationdrivestate).[`transcript`](#transcript-1) +##### activate() -##### turnIndex +> **activate**(`input`): `Promise`\<[`AgentCandidateProtectedModelActivation`](#agentcandidateprotectedmodelactivation)\> -> **turnIndex**: `number` +Defined in: [candidate-execution/protected-model-port.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L41) -Defined in: [conversation/types.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L79) +###### Parameters -###### Inherited from +###### input -[`ConversationDriveState`](#conversationdrivestate).[`turnIndex`](#turnindex) +###### executionId -##### spentCreditsCents +`string` -> **spentCreditsCents**: `number` +###### preparationId -Defined in: [conversation/types.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L80) +`string` -###### Inherited from +###### grantDigest -[`ConversationDriveState`](#conversationdrivestate).[`spentCreditsCents`](#spentcreditscents) +`` `sha256:${string}` `` -##### lastTurn +###### resolved -> **lastTurn**: [`ConversationTurn`](#conversationturn) +`AgentCandidateResolvedModel` -Defined in: [conversation/types.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L85) +###### deadlineAtMs -*** +`number` -### HaltSignal +###### Returns -Defined in: [conversation/types.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L89) +`Promise`\<[`AgentCandidateProtectedModelActivation`](#agentcandidateprotectedmodelactivation)\> -#### Stable +##### settle() -#### Properties +> **settle**(`input`): `Promise`\<[`AgentCandidateProtectedModelSettlement`](#agentcandidateprotectedmodelsettlement)\> -##### halted +Defined in: [candidate-execution/protected-model-port.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L44) -> **halted**: `true` +###### Parameters -Defined in: [conversation/types.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L90) +###### input -##### reason +###### executionId -> **reason**: `string` +`string` -Defined in: [conversation/types.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L91) +###### preparationId -*** +`string` -### ConversationPolicy +###### grantDigest -Defined in: [conversation/types.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L108) +`` `sha256:${string}` `` -#### Stable +###### resolved -#### Properties +`AgentCandidateResolvedModel` -##### maxTurns +###### reason -> **maxTurns**: `number` +`"completed"` \| `"failed"` \| `"timeout"` \| `"replayed"` \| `"preparation-failed"` \| `"abandoned"` -Defined in: [conversation/types.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L110) +###### Returns -Hard cap on speaker-turns. Each call into a participant's backend counts as 1. +`Promise`\<[`AgentCandidateProtectedModelSettlement`](#agentcandidateprotectedmodelsettlement)\> -##### maxCreditsCents? +*** -> `optional` **maxCreditsCents?**: `number` +### CreateProtectedAgentCandidateModelPortOptions -Defined in: [conversation/types.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L117) +Defined in: [candidate-execution/protected-model-port.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L49) -Hard cap on aggregate credit spend across all participants, in cents. -Computed by summing `llm_call.costUsd` from every participant's stream. -Unset (`undefined`) means no credit ceiling — the run is bounded only by -`maxTurns` and `haltOn`. +#### Properties -##### turnOrder? +##### client -> `optional` **turnOrder?**: [`TurnOrder`](#turnorder) +> **client**: [`AgentCandidateModelGrantClient`](#agentcandidatemodelgrantclient) -Defined in: [conversation/types.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L122) +Defined in: [candidate-execution/protected-model-port.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L50) -Speaker selection. Defaults to `'alternate'` for two-participant -conversations and `'round-robin'` for any other arity. +##### resolveModel -##### haltOn? +> **resolveModel**: (`input`) => `Promise`\<`AgentCandidateResolvedModel`\> -> `optional` **haltOn?**: [`HaltPredicate`](#haltpredicate) +Defined in: [candidate-execution/protected-model-port.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L52) -Defined in: [conversation/types.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L127) +Catalog/snapshot resolution stays separate from credential issuance. -Optional convergence / content-based halt. Called after every turn ends; -returning truthy stops the loop with `{ kind: 'predicate', ... }`. +###### Parameters -##### defaultCallPolicy? +###### input -> `optional` **defaultCallPolicy?**: [`BackendCallPolicy`](#backendcallpolicy) +###### requested -Defined in: [conversation/types.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L133) +`string` -Default per-turn resilience policy applied to every participant call -(deadline, retries, circuit breaker). Individual participants may -override via `ConversationParticipant.callPolicy`. +###### harness -*** +`HarnessType` -### ConversationTurn +###### reasoningEffort -Defined in: [conversation/types.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L137) +`ReasoningEffort` \| `undefined` -#### Stable +###### Returns -#### Properties +`Promise`\<`AgentCandidateResolvedModel`\> -##### index +##### gatewayDomain -> **index**: `number` +> **gatewayDomain**: `string` -Defined in: [conversation/types.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L138) +Defined in: [candidate-execution/protected-model-port.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L54) -##### speaker +The only public DNS name candidate processes may reach for inference. -> **speaker**: `string` +##### activationEnvNames -Defined in: [conversation/types.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L139) +> **activationEnvNames**: readonly `string`[] -##### turnId +Defined in: [candidate-execution/protected-model-port.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L56) -> **turnId**: `string` +Exact environment names the activation endpoint must return, no more or fewer. -Defined in: [conversation/types.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L145) +*** -Deterministic turn identifier — stable across retries of the same logical -turn so caching gateways and trace backends can dedupe. Shape: -`${runId}.t${index}.${speakerSlug}`. +### RecoverExpiredAgentCandidateOptions -##### text +Defined in: [candidate-execution/recover.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L23) -> **text**: `string` +#### Properties -Defined in: [conversation/types.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L146) +##### attempt -##### usage? +> **attempt**: [`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) -> `optional` **usage?**: `object` +Defined in: [candidate-execution/recover.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L24) -Defined in: [conversation/types.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L152) +##### claimStore -Aggregated backend usage for this turn alone. Populated from any -`llm_call` stream events the backend emitted; `undefined` when the -backend reports no usage. +> **claimStore**: [`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore) -###### tokensIn? +Defined in: [candidate-execution/recover.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L25) -> `optional` **tokensIn?**: `number` +##### executor -###### tokensOut? +> **executor**: [`AgentCandidateExecutorPort`](#agentcandidateexecutorport) -> `optional` **tokensOut?**: `number` +Defined in: [candidate-execution/recover.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L26) -###### costUsd? +##### traceStore -> `optional` **costUsd?**: `number` +> **traceStore**: `TraceStore` -###### latencyMs? +Defined in: [candidate-execution/recover.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L27) -> `optional` **latencyMs?**: `number` +##### ports -###### model? +> **ports**: `Pick`\<[`AgentCandidateExecutionPorts`](#agentcandidateexecutionports), `"models"` \| `"memory"`\> -> `optional` **model?**: `string` +Defined in: [candidate-execution/recover.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L28) -##### attempts +##### outputArtifacts -> **attempts**: `number` +> **outputArtifacts**: [`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) -Defined in: [conversation/types.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L164) +Defined in: [candidate-execution/recover.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L29) -Number of attempts that ran before this turn committed. `1` is the -common case; higher means the call policy retried after transient -failures. +##### cleanupTimeoutMs? -##### startedAt +> `optional` **cleanupTimeoutMs?**: `number` -> **startedAt**: `string` +Defined in: [candidate-execution/recover.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L30) -Defined in: [conversation/types.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L165) +##### now? -##### endedAt +> `optional` **now?**: () => `number` -> **endedAt**: `string` +Defined in: [candidate-execution/recover.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L32) -Defined in: [conversation/types.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L166) +Evaluator clock; must be the same clock used by the claim store. + +###### Returns + +`number` *** -### Conversation +### AgentCandidateArtifactPort -Defined in: [conversation/types.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L170) +Defined in: [candidate-execution/types.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L34) -#### Stable +Reads one content-addressed object from the closed S3/IPFS locator set. -#### Properties +#### Extended by -##### participants +- [`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) -> **participants**: readonly [`ConversationParticipant`](#conversationparticipant)[] +#### Methods -Defined in: [conversation/types.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L171) +##### read() -##### policy +> **read**(`ref`): `Promise`\<`Uint8Array`\<`ArrayBufferLike`\>\> -> **policy**: [`ConversationPolicy`](#conversationpolicy) +Defined in: [candidate-execution/types.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L35) -Defined in: [conversation/types.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L172) +###### Parameters -*** +###### ref -### RunConversationOptions +`AgentCandidateArtifactRef` -Defined in: [conversation/types.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L176) +###### Returns -#### Stable +`Promise`\<`Uint8Array`\<`ArrayBufferLike`\>\> -#### Properties +*** -##### seed +### AgentCandidateOutputArtifactPort -> **seed**: `string` +Defined in: [candidate-execution/types.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L55) -Defined in: [conversation/types.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L178) +Durable content-addressed evidence store controlled only by the evaluator. -First message kicking off the conversation. Routes to the first speaker. +#### Extends -##### runId? +- [`AgentCandidateArtifactPort`](#agentcandidateartifactport) -> `optional` **runId?**: `string` +#### Methods -Defined in: [conversation/types.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L185) +##### read() -Optional run identifier for cross-participant trace correlation. Auto- -generated when omitted. Reusing a runId against the same `journal` -resumes the prior run — the runner replays the persisted transcript and -continues from the first un-recorded turn. +> **read**(`ref`): `Promise`\<`Uint8Array`\<`ArrayBufferLike`\>\> -##### signal? +Defined in: [candidate-execution/types.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L35) -> `optional` **signal?**: `AbortSignal` +###### Parameters -Defined in: [conversation/types.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L187) +###### ref -Cancellation signal — aborts mid-stream and halts with `{ kind: 'abort' }`. +`AgentCandidateArtifactRef` -##### onEvent? +###### Returns -> `optional` **onEvent?**: (`event`) => `void` \| `Promise`\<`void`\> +`Promise`\<`Uint8Array`\<`ArrayBufferLike`\>\> -Defined in: [conversation/types.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L194) +###### Inherited from -Event sink for per-turn micro-events. Distinct from the result transcript: -the sink fires for every text-delta, every turn-start/end, and the -conversation-start/end markers. Used to drive SSE / dashboard updates -without waiting for the conversation to finish. +[`AgentCandidateArtifactPort`](#agentcandidateartifactport).[`read`](#read) -###### Parameters +##### put() -###### event +> **put**(`input`): `Promise`\<`AgentCandidateArtifactRef`\> -[`ConversationStreamEvent`](#conversationstreamevent) +Defined in: [candidate-execution/types.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L57) -###### Returns +Must be idempotent for identical bytes and return only a durable S3/IPFS locator. -`void` \| `Promise`\<`void`\> +###### Parameters -##### journal? +###### input -> `optional` **journal?**: [`ConversationJournal`](#conversationjournal) +###### executionId -Defined in: [conversation/types.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L201) +`string` -Optional durable transcript. When set, the runner persists every -committed turn before yielding `turn_end`. Reusing the same `runId` -against the same journal resumes from the last committed turn — so a -driver process crash mid-run loses zero acknowledged turns. +###### purpose -##### propagatedHeaders? +[`AgentCandidateOutputPurpose`](#agentcandidateoutputpurpose) -> `optional` **propagatedHeaders?**: `Readonly`\<`Record`\<`string`, `string`\>\> +###### bytes -Defined in: [conversation/types.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L208) +`Uint8Array` -Headers to forward verbatim to every participant backend call (gateway -propagation: `X-Tangle-Forwarded-Authorization`, run/turn correlation, -depth counter). Backends opt in by reading `propagatedHeaders` from -their `AgentBackendContext`; backends that ignore the field still work. +###### signal? -##### inboundDepth? +`AbortSignal` -> `optional` **inboundDepth?**: `number` +Abort must prevent durable publication when it happens before resolution. -Defined in: [conversation/types.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L214) +###### Returns -Inbound depth at the point this driver was invoked. The runner -increments it on every outbound participant call; gateways refuse at -`DEFAULT_MAX_DEPTH`. Default 0 (origin caller). +`Promise`\<`AgentCandidateArtifactRef`\> -##### parentTurnId? +*** -> `optional` **parentTurnId?**: `string` +### AgentCandidateRepositoryPort -Defined in: [conversation/types.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L221) +Defined in: [candidate-execution/types.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L67) -Parent turn id when this conversation is *inside* another turn (i.e. the -driver is itself a participant via `createConversationBackend`). The -runner stamps each outbound call with this as `X-Tangle-Parent-TurnId` -so trace stitching survives nested orchestration. +Resolves a declared GitHub repository to an already-present local Git object store. -*** +#### Methods -### ConversationResult +##### resolve() -Defined in: [conversation/types.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L225) +> **resolve**(`repository`): `Promise`\<`string`\> -#### Stable +Defined in: [candidate-execution/types.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L68) -#### Properties +###### Parameters -##### runId +###### repository -> **runId**: `string` +`AgentCandidateGitHubRepository` -Defined in: [conversation/types.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L226) +###### Returns -##### transcript +`Promise`\<`string`\> -> **transcript**: [`ConversationTurn`](#conversationturn)[] +*** -Defined in: [conversation/types.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L227) +### AgentCandidateVerificationPorts -##### turns +Defined in: [candidate-execution/types.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L71) -> **turns**: `number` +#### Extended by -Defined in: [conversation/types.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L228) +- [`AgentCandidateExecutionPorts`](#agentcandidateexecutionports) -##### spentCreditsCents +#### Properties -> **spentCreditsCents**: `number` +##### artifacts -Defined in: [conversation/types.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L229) +> **artifacts**: [`AgentCandidateArtifactPort`](#agentcandidateartifactport) -##### halted +Defined in: [candidate-execution/types.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L72) -> **halted**: [`HaltReason`](#haltreason) +##### repositories -Defined in: [conversation/types.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L230) +> **repositories**: [`AgentCandidateRepositoryPort`](#agentcandidaterepositoryport) -##### durationMs +Defined in: [candidate-execution/types.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L73) -> **durationMs**: `number` +*** -Defined in: [conversation/types.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L231) +### AgentCandidateWorkspacePort -##### startedAt +Defined in: [candidate-execution/types.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L83) -> **startedAt**: `string` +Materializes an already-verified workspace archive. -Defined in: [conversation/types.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L232) +The runtime independently scans every resulting byte, mode, and path against +the signed manifest after this returns. Implementations may therefore unpack +any archive encoding, or no-op when the exact workspace is already present. -##### endedAt +#### Methods -> **endedAt**: `string` +##### materialize() -Defined in: [conversation/types.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L233) +> **materialize**(`input`): `Promise`\<`void`\> -*** +Defined in: [candidate-execution/types.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L84) -### ChatStreamEvent +###### Parameters -Defined in: [durable/chat-engine.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L28) +###### input -The NDJSON line protocol every product chat client already speaks. +###### role -#### Properties +`"task"` \| `"memory"` \| `"candidate"` -##### type +###### snapshot -> **type**: `string` +`AgentCandidateWorkspaceSnapshotEvidence` -Defined in: [durable/chat-engine.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L29) +###### archive -##### data? +`Uint8Array` -> `optional` **data?**: `Record`\<`string`, `unknown`\> +###### destination -Defined in: [durable/chat-engine.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L30) +`string` -*** +###### Returns -### ChatTurnIdentity +`Promise`\<`void`\> -Defined in: [durable/chat-engine.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L35) +*** -Identity of a chat turn. `tenantId` is the workspace id for workspace- - scoped products and the user id for session-scoped products. +### ResolvedAgentCandidateContainer + +Defined in: [candidate-execution/types.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L92) #### Properties -##### tenantId +##### source -> **tenantId**: `string` +> **source**: `"pinned-container"` \| `"evaluator-task-container"` -Defined in: [durable/chat-engine.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L36) +Defined in: [candidate-execution/types.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L93) -##### sessionId +##### image -> **sessionId**: `string` +> **image**: `string` -Defined in: [durable/chat-engine.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L38) +Defined in: [candidate-execution/types.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L94) -Thread / session id. +##### indexDigest -##### userId +> **indexDigest**: `` `sha256:${string}` `` -> **userId**: `string` +Defined in: [candidate-execution/types.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L95) -Defined in: [durable/chat-engine.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L39) +##### manifestDigest -##### turnIndex +> **manifestDigest**: `` `sha256:${string}` `` -> **turnIndex**: `number` +Defined in: [candidate-execution/types.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L96) -Defined in: [durable/chat-engine.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L41) +##### platform -Monotonic 0-based turn index within the session. +> **platform**: `AgentCandidateOciPlatform` + +Defined in: [candidate-execution/types.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L97) *** -### ChatTurnProducer +### AgentCandidateContainerPort -Defined in: [durable/chat-engine.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L45) +Defined in: [candidate-execution/types.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L100) -The live side of a turn — what the product's `produce` hook returns. +#### Methods -#### Type Parameters +##### resolve() -##### TEvent +> **resolve**(`input`): `Promise`\<[`ResolvedAgentCandidateContainer`](#resolvedagentcandidatecontainer)\> -`TEvent` *extends* [`ChatStreamEvent`](#chatstreamevent) = [`ChatStreamEvent`](#chatstreamevent) +Defined in: [candidate-execution/types.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L101) -#### Properties +###### Parameters -##### stream +###### input -> **stream**: `AsyncGenerator`\<`TEvent`, `void`, `unknown`\> +###### candidate -Defined in: [durable/chat-engine.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L47) +`AgentCandidateContainer` \| `undefined` -The turn's event stream. Forwarded verbatim to the caller. +###### evaluatorTaskContainer -#### Methods +[`ResolvedAgentCandidateContainer`](#resolvedagentcandidatecontainer) \| `undefined` -##### finalText() +###### Returns -> **finalText**(): `string` +`Promise`\<[`ResolvedAgentCandidateContainer`](#resolvedagentcandidatecontainer)\> -Defined in: [durable/chat-engine.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L49) +*** -The turn's final assistant text. Read once, after `stream` drains. +### AgentCandidateModelPort -###### Returns +Defined in: [candidate-execution/types.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L107) -`string` +#### Methods -*** +##### resolve() -### ChatTurnHooks +> **resolve**(`input`): `Promise`\<`AgentCandidateResolvedModel`\> -Defined in: [durable/chat-engine.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L52) +Defined in: [candidate-execution/types.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L108) -Turn-lifecycle helpers for `@tangle-network/agent-runtime`. +###### Parameters -Execution state — long-running execution, reconnect, replay, dedup — -lives in the substrate (`@tangle-network/sandbox` + orchestrator). -agent-runtime owns: +###### input - - `handleChatTurn` — framework-neutral turn lifecycle: NDJSON framing, - `session.run.*` envelope, persist / post-process / trace-flush - hook ordering. - - `deriveExecutionId` — convention helper for the stable id products - persist so a retry of the same turn lands on the same execution. +###### requested -#### Methods +`string` -##### produce() +###### harness -> **produce**(): [`ChatTurnProducer`](#chatturnproducer) +`HarnessType` -Defined in: [durable/chat-engine.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L55) +###### reasoningEffort -Build the backend stream. The engine forwards events verbatim and - reads `finalText()` once the stream drains. +`ReasoningEffort` \| `undefined` ###### Returns -[`ChatTurnProducer`](#chatturnproducer) +`Promise`\<`AgentCandidateResolvedModel`\> -##### persistAssistantMessage() +##### reserveGrant() -> **persistAssistantMessage**(`input`): `Promise`\<`void`\> +> **reserveGrant**(`input`): `Promise`\<[`AgentCandidateProtectedModelReservation`](#agentcandidateprotectedmodelreservation)\> -Defined in: [durable/chat-engine.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L58) +Defined in: [candidate-execution/types.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L118) -Persist the assistant message to the product's own store. Called - once, after drain, with the assembled (transform-applied) text. +Reserve a stable access identity without creating a live credential. +The reservation is scoped to `preparationId` and must automatically expire +at `expiresAtMs`, even if this call returns ambiguously to the runtime. ###### Parameters ###### input -###### identity - -[`ChatTurnIdentity`](#chatturnidentity) - -###### finalText +###### executionId `string` -###### Returns +###### preparationId -`Promise`\<`void`\> +`string` -##### onTurnComplete()? +###### expiresAtMs -> `optional` **onTurnComplete**(`input`): `Promise`\<`void`\> +`number` -Defined in: [durable/chat-engine.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L62) +###### attempt -Optional post-processing (proposals, citations, credit metering …). - Errors are swallowed + logged — post-process must never fail a turn - that already streamed successfully. +`AgentCandidateAttemptPolicy` -###### Parameters +###### bundleDigest -###### input +`` `sha256:${string}` `` -###### identity +###### resolved -[`ChatTurnIdentity`](#chatturnidentity) +`AgentCandidateResolvedModel` -###### finalText +###### limits -`string` +[`AgentCandidateModelLimits`](#agentcandidatemodellimits) ###### Returns -`Promise`\<`void`\> +`Promise`\<[`AgentCandidateProtectedModelReservation`](#agentcandidateprotectedmodelreservation)\> -##### onEvent()? +##### activateGrant() -> `optional` **onEvent**(`event`): `void` \| `Promise`\<`void`\> +> **activateGrant**(`input`): `Promise`\<[`AgentCandidateProtectedModelActivation`](#agentcandidateprotectedmodelactivation)\> -Defined in: [durable/chat-engine.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L66) +Defined in: [candidate-execution/types.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L128) -Optional per-event side channel (e.g. DO broadcast). Runs for every - emitted event, lifecycle envelope included. Errors swallowed — a - broadcast failure must not break the chat stream. +Create the live scoped credential only after the execution attempt is durably claimed. ###### Parameters -###### event +###### input -[`ChatStreamEvent`](#chatstreamevent) +###### executionId -###### Returns +`string` -`void` \| `Promise`\<`void`\> +###### preparationId -##### transformFinalText()? +`string` -> `optional` **transformFinalText**(`text`): `string` \| `Promise`\<`string`\> +###### grantDigest -Defined in: [durable/chat-engine.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L70) +`` `sha256:${string}` `` -Optional pre-persist transform of the final text (e.g. PII - redaction). Affects only what is persisted; the live stream is - never altered. +###### resolved -###### Parameters +`AgentCandidateResolvedModel` -###### text +###### deadlineAtMs -`string` +`number` ###### Returns -`string` \| `Promise`\<`string`\> - -##### traceFlush()? +`Promise`\<[`AgentCandidateProtectedModelActivation`](#agentcandidateprotectedmodelactivation)\> -> `optional` **traceFlush**(): `Promise`\<`void`\> +##### settleGrant() -Defined in: [durable/chat-engine.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L73) +> **settleGrant**(`input`): `Promise`\<[`AgentCandidateProtectedModelSettlement`](#agentcandidateprotectedmodelsettlement)\> -Optional trace flush — resolves when OTLP export completes. Handed - to `waitUntil` so the worker isolate stays alive for the POST. +Defined in: [candidate-execution/types.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L141) -###### Returns +Atomically revoke the grant, drain in-flight calls, and return its immutable final ledger. +This operation must be idempotent for the exact preparation and must also +settle a reservation that was never activated. It must never affect a +different preparation, even when both reservations report the same digest. -`Promise`\<`void`\> +###### Parameters -*** +###### input -### RunChatTurnInput +###### executionId -Defined in: [durable/chat-engine.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L76) +`string` -Turn-lifecycle helpers for `@tangle-network/agent-runtime`. +###### preparationId -Execution state — long-running execution, reconnect, replay, dedup — -lives in the substrate (`@tangle-network/sandbox` + orchestrator). -agent-runtime owns: +`string` - - `handleChatTurn` — framework-neutral turn lifecycle: NDJSON framing, - `session.run.*` envelope, persist / post-process / trace-flush - hook ordering. - - `deriveExecutionId` — convention helper for the stable id products - persist so a retry of the same turn lands on the same execution. +###### grantDigest -#### Properties +`` `sha256:${string}` `` -##### identity +###### resolved -> **identity**: [`ChatTurnIdentity`](#chatturnidentity) +`AgentCandidateResolvedModel` -Defined in: [durable/chat-engine.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L77) +###### reason -##### hooks +`"completed"` \| `"failed"` \| `"timeout"` \| `"replayed"` \| `"preparation-failed"` \| `"abandoned"` -> **hooks**: [`ChatTurnHooks`](#chatturnhooks) +###### Returns -Defined in: [durable/chat-engine.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L78) +`Promise`\<[`AgentCandidateProtectedModelSettlement`](#agentcandidateprotectedmodelsettlement)\> -##### waitUntil? +*** -> `optional` **waitUntil?**: (`p`) => `void` +### AgentCandidateBenchmarkGraderIdentity -Defined in: [durable/chat-engine.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L81) +Defined in: [candidate-execution/types.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L156) -Worker liveness hook. When omitted, trace flush is awaited inline - before the stream closes. +#### Properties -###### Parameters +##### name -###### p +> **name**: `string` -`Promise`\<`unknown`\> +Defined in: [candidate-execution/types.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L157) -###### Returns +##### version -`void` +> **version**: `string` -##### log? +Defined in: [candidate-execution/types.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L158) -> `optional` **log?**: (`message`, `meta?`) => `void` +##### artifact -Defined in: [durable/chat-engine.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L84) +> **artifact**: `AgentCandidateArtifactRef` -Structured logger for swallowed hook errors. Defaults to - `console.error` so failures surface without product wiring. +Defined in: [candidate-execution/types.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L159) -###### Parameters +*** -###### message +### AgentCandidateProtectedModelReservation -`string` +Defined in: [candidate-execution/types.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L162) -###### meta? +#### Properties -`Record`\<`string`, `unknown`\> +##### preparationId -###### Returns +> **preparationId**: `string` -`void` +Defined in: [candidate-execution/types.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L163) -*** +##### digest -### ChatTurnResult +> **digest**: `` `sha256:${string}` `` -Defined in: [durable/chat-engine.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L87) +Defined in: [candidate-execution/types.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L164) -Turn-lifecycle helpers for `@tangle-network/agent-runtime`. +##### expiresAtMs -Execution state — long-running execution, reconnect, replay, dedup — -lives in the substrate (`@tangle-network/sandbox` + orchestrator). -agent-runtime owns: +> **expiresAtMs**: `number` - - `handleChatTurn` — framework-neutral turn lifecycle: NDJSON framing, - `session.run.*` envelope, persist / post-process / trace-flush - hook ordering. - - `deriveExecutionId` — convention helper for the stable id products - persist so a retry of the same turn lands on the same execution. +Defined in: [candidate-execution/types.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L166) -#### Properties +Evaluator service must expire and revoke this reservation at this epoch millisecond. -##### body +##### enforcedLimits -> **body**: `ReadableStream`\<`Uint8Array`\<`ArrayBufferLike`\>\> +> **enforcedLimits**: [`AgentCandidateModelLimits`](#agentcandidatemodellimits) -Defined in: [durable/chat-engine.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L89) +Defined in: [candidate-execution/types.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L168) -NDJSON body — return this as the platform `Response` body. +The gateway must stop calls before any one of these limits is exceeded. -##### contentType +##### network -> **contentType**: `"application/x-ndjson"` +> **network**: `AgentCandidateModelAccessNetwork` -Defined in: [durable/chat-engine.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L91) +Defined in: [candidate-execution/types.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L170) -Content type for the response. +Exact public endpoint exception; every other candidate destination stays blocked. *** -### VerifyResult - -Defined in: [improvement/agentic-generator.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L48) +### AgentCandidateProtectedModelActivation -Outcome of verifying a candidate worktree. `feedback` (compiler errors, - failing test output) is fed into the next shot when `ok` is false. +Defined in: [candidate-execution/types.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L173) #### Properties -##### ok +##### env -> **ok**: `boolean` +> **env**: `Readonly`\<`Record`\<`string`, `string`\>\> -Defined in: [improvement/agentic-generator.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L49) +Defined in: [candidate-execution/types.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L175) -##### feedback? +Injected only into the trusted executor after all pre-launch checks pass. -> `optional` **feedback?**: `string` +*** -Defined in: [improvement/agentic-generator.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L50) +### AgentCandidateProtectedModelCall -*** +Defined in: [candidate-execution/types.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L179) -### AgenticGeneratorOptions +One evaluator-gateway call in the final, revoked model-access ledger. -Defined in: [improvement/agentic-generator.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L58) +#### Properties -`@tangle-network/agent-runtime` improvement — the CODE-surface proposer for -agent-eval's improvement loop. +##### callId -The ONE entry point for optimization is agent-eval's `selfImprove` -(`@tangle-network/agent-eval/contract`) — text/config surfaces, held-out gated, -with `analyzeGeneration` for analyst-fed reflection and `analyzeRuns` / -`fromOtelSpans` / `partitionRunsByAuthoringModel` for production intake + -cohorting. This module supplies only the one genuinely runtime-specific piece: -a CODE-surface `SurfaceProposer` you pass to `selfImprove` as `proposer`, which -mutates a git worktree via a pluggable `CandidateGenerator`: - - `reflectiveGenerator` — cheap, no sandbox, applies pre-drafted patches - - `agenticGenerator` — full coding harness in the worktree, multi-shot +> **callId**: `string` -#### Properties +Defined in: [candidate-execution/types.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L180) -##### harness? +##### generationId -> `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) +> **generationId**: `string` -Defined in: [improvement/agentic-generator.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L60) +Defined in: [candidate-execution/types.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L182) -Local coding harness to run in the worktree. Default `claude`. +Router-generated public response identity. -##### timeoutMs? +##### traceSpanId -> `optional` **timeoutMs?**: `number` +> **traceSpanId**: `string` -Defined in: [improvement/agentic-generator.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L62) +Defined in: [candidate-execution/types.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L184) -Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). +Exact protected agent-eval LLM span produced from the router ledger. -##### buildPrompt? +##### status -> `optional` **buildPrompt?**: (`args`) => `string` +> **status**: `"failed"` \| `"succeeded"` -Defined in: [improvement/agentic-generator.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L65) +Defined in: [candidate-execution/types.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L185) -Build the harness task prompt from the report + findings. Override for - domain phrasing; the default turns findings into a concrete coder task. +##### model -###### Parameters +> **model**: `string` -###### args +Defined in: [candidate-execution/types.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L186) -###### report +##### startedAtMs -`unknown` +> **startedAtMs**: `number` -###### findings +Defined in: [candidate-execution/types.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L187) -`AnalystFinding`[] +##### endedAtMs -###### Returns +> **endedAtMs**: `number` -`string` +Defined in: [candidate-execution/types.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L188) -##### verify? +##### inputTokens -> `optional` **verify?**: [`Verifier`](#verifier) +> **inputTokens**: `number` -Defined in: [improvement/agentic-generator.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L71) +Defined in: [candidate-execution/types.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L189) -Verify the worktree after each dirtying shot. When set, a candidate that - fails verification is NOT returned — the failure feeds the next shot - (verify-in-session), up to `maxShots`; a candidate that never verifies is - discarded (`applied:false`), never shipped. Omitted ⇒ legacy behavior: - the first dirty shot is the candidate. See `commandVerifier`. +##### outputTokens -##### runHarness? +> **outputTokens**: `number` -> `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> +Defined in: [candidate-execution/types.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L190) -Defined in: [improvement/agentic-generator.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L73) +##### cachedInputTokens -Test seam — inject the harness runner (defaults to `runLocalHarness`). +> **cachedInputTokens**: `number` -**`Experimental`** +Defined in: [candidate-execution/types.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L191) -Spawn a local coding harness CLI as a subprocess + collect its output. +##### reasoningTokens -NOT responsible for parsing the harness's output or extracting a diff — -the in-process executor's `streamPrompt` orchestrates `git diff` against -the worktree after this resolves. This function is intentionally narrow: -spawn, wait, capture, return. +> **reasoningTokens**: `number` -Fails loud — throws when: - - `cwd` doesn't exist (subprocess emits ENOENT; surfaced as Error) - - the harness binary is not on PATH (ENOENT) +Defined in: [candidate-execution/types.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L192) -Does NOT throw when: - - the subprocess exits non-zero (`result.exitCode` carries the code) - - the subprocess is aborted / timed out (`result.killedBySignal` / - `result.timedOut` carries the reason) +##### costUsdNanos -###### Parameters +> **costUsdNanos**: `number` -###### options +Defined in: [candidate-execution/types.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L194) -[`RunLocalHarnessOptions`](mcp.md#runlocalharnessoptions) +Integer billionths of one US dollar; avoids floating-point ledger drift. -###### Returns +*** -`Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> +### AgentCandidateProtectedModelSettlement -##### isDirty? +Defined in: [candidate-execution/types.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L197) -> `optional` **isDirty?**: (`worktreePath`) => `boolean` +#### Properties -Defined in: [improvement/agentic-generator.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L75) +##### preparationId -Test seam — inject the worktree-dirty check (defaults to `git status`). +> **preparationId**: `string` -###### Parameters +Defined in: [candidate-execution/types.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L198) -###### worktreePath +##### grantDigest -`string` +> **grantDigest**: `` `sha256:${string}` `` -###### Returns +Defined in: [candidate-execution/types.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L199) -`boolean` +##### closed -*** +> **closed**: `true` -### ImproveOptions +Defined in: [candidate-execution/types.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L200) -Defined in: [improvement/improve.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L57) +##### calls -#### Type Parameters +> **calls**: readonly [`AgentCandidateProtectedModelCall`](#agentcandidateprotectedmodelcall)[] -##### TScenario +Defined in: [candidate-execution/types.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L201) -`TScenario` *extends* `Scenario` +*** -##### TArtifact +### AgentCandidateMemoryResetResult -`TArtifact` +Defined in: [candidate-execution/types.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L204) #### Properties -##### surface? +##### preparationId -> `optional` **surface?**: [`ImproveSurface`](#improvesurface) +> **preparationId**: `string` -Defined in: [improvement/improve.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L60) +Defined in: [candidate-execution/types.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L205) -Which profile lever to optimize. Default `'prompt'`. Selects the default - generator + the baseline-surface extraction shape. +##### accessDigest -##### generator? +> **accessDigest**: `` `sha256:${string}` `` -> `optional` **generator?**: `SurfaceProposer`\<`unknown`\> +Defined in: [candidate-execution/types.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L206) -Defined in: [improvement/improve.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L64) +##### expiresAtMs -The `SurfaceProposer` that mutates the surface. When unset, the facade - picks the default for `surface` (`gepaProposer` for prompt, `skillOptProposer` - for skills); surfaces with no default REQUIRE this (fail-loud otherwise). +> **expiresAtMs**: `number` -##### gate? +Defined in: [candidate-execution/types.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L207) -> `optional` **gate?**: `"none"` \| `"holdout"` +##### evidence -Defined in: [improvement/improve.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L67) +> **evidence**: `AgentCandidateCapturedArtifact` -Gate mode. `'holdout'` (default) runs the held-out promotion gate; - `'none'` is a baseline-only run (`budget.generations = 0`). +Defined in: [candidate-execution/types.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L208) -##### scenarios +##### emptyStateDigest -> **scenarios**: `TScenario`[] +> **emptyStateDigest**: `` `sha256:${string}` `` -Defined in: [improvement/improve.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L69) +Defined in: [candidate-execution/types.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L209) + +##### beforeState -Scenarios to evaluate against. Passthrough to `selfImprove`. +> **beforeState**: `AgentCandidateWorkspaceSnapshotEvidence` -##### judge +Defined in: [candidate-execution/types.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L210) + +*** -> **judge**: `JudgeConfig`\<`TArtifact`, `TScenario`\> +### AgentCandidateMemoryPort -Defined in: [improvement/improve.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L71) +Defined in: [candidate-execution/types.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L213) -Judge that scores artifacts. Passthrough to `selfImprove`. +#### Methods -##### agent +##### reset() -> **agent**: (`surface`, `scenario`, `ctx`) => `Promise`\<`TArtifact`\> +> **reset**(`input`): `Promise`\<[`AgentCandidateMemoryResetResult`](#agentcandidatememoryresetresult)\> -Defined in: [improvement/improve.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L74) +Defined in: [candidate-execution/types.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L219) -The agent under improvement — same shape as `selfImprove.agent`: it takes - the current surface + scenario + ctx and returns the artifact to judge. +Reset and reserve exact task memory without returning live access. +The service must scope the reservation to `preparationId`, automatically +revoke it at `expiresAtMs`, and never reuse it for another preparation. ###### Parameters -###### surface +###### input -`MutableSurface` +###### executionId -###### scenario +`string` -`TScenario` +###### preparationId -###### ctx +`string` -`DispatchContext` +###### expiresAtMs -###### Returns +`number` -`Promise`\<`TArtifact`\> +###### effectiveNamespace -##### budget? +`string` -> `optional` **budget?**: `SelfImproveBudget` +###### seed? -Defined in: [improvement/improve.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L76) +`Uint8Array`\<`ArrayBufferLike`\> -Budget + loop shape. Passthrough; `gate: 'none'` forces `generations = 0`. +###### seedDigest? -##### llm? +`` `sha256:${string}` `` -> `optional` **llm?**: `SelfImproveLlm` +###### Returns -Defined in: [improvement/improve.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L79) +`Promise`\<[`AgentCandidateMemoryResetResult`](#agentcandidatememoryresetresult)\> -LLM config. Passthrough to `selfImprove` AND used to construct the default - reflective proposer (`gepaProposer`/`skillOptProposer`) when `generator` is unset. +##### activate() -##### allowedModels? +> **activate**(`input`): `Promise`\<\{ `env`: `Readonly`\<`Record`\<`string`, `string`\>\>; \}\> -> `optional` **allowedModels?**: readonly `string`[] +Defined in: [candidate-execution/types.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L231) -Defined in: [improvement/improve.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L83) +Create live scoped access only after the execution attempt is durably claimed. +Activation must match the exact preparation/access pair and may not extend expiry. -Restrict the run to this subset of models. When set, the reflection model - (`llm.model`, or the default when unset) must be a member, or `improve()` throws - a `ConfigError` before the generator is built. Unset = unrestricted. +###### Parameters -##### runDir? +###### input -> `optional` **runDir?**: `string` +###### executionId -Defined in: [improvement/improve.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L89) +`string` -Run directory passthrough to `selfImprove`. Pass a REAL path to make the loop - durable: campaign cells + the loop provenance record land on the filesystem as - they complete, so a multi-hour search survives a process/infra death instead of - losing every generation with it (the default `mem://` run keeps everything - in-process). +###### preparationId -##### analyzeGeneration? +`string` -> `optional` **analyzeGeneration?**: ((`input`) => `Promise`\<`unknown`[]\>) \| `null` +###### accessDigest -Defined in: [improvement/improve.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L97) +`` `sha256:${string}` `` -Per-generation findings producer passthrough (see selfImprove.analyzeGeneration). - DEFAULT: the built-in failure distiller — after each generation it turns the - worst-scoring/errored cells into structured findings ({ scenario, composite, - notes, error }) for the NEXT proposal round, so the proposer reasons over what - actually failed instead of a static seed. Pass your own producer (e.g. a - trace-analyst over the runDir's traces) to replace it; pass `null` to disable - and keep the static `findings` all the way through. +###### effectiveNamespace -##### rawTraceContext? - -> `optional` **rawTraceContext?**: `boolean` +`string` -Defined in: [improvement/improve.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L107) +###### deadlineAtMs -META-HARNESS mode: instead of the ~400-char distilled findings, feed the - proposer RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's - real run traces under `runDir` (per-cell `spans.jsonl` event logs + - `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose - instruction — so the coding agent reads the actual failures itself rather than - a pre-summary. Requires a REAL `runDir` (that is where the traces live). - Ignored when `analyzeGeneration` is set explicitly (that wins) or is `null` - (disabled). Equivalent to `analyzeGeneration: rawTraceDistiller()`; this flag - is the one-line enable. Default `false` (the distiller stays the default). +`number` -##### code? +###### Returns -> `optional` **code?**: `ImproveCodeOptions` +`Promise`\<\{ `env`: `Readonly`\<`Record`\<`string`, `string`\>\>; \}\> -Defined in: [improvement/improve.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L115) +##### close() -CODE-surface wiring with prompt-parity DX: name `surface: 'code'`, point at a - repo, and the facade assembles the whole candidate pipeline — git worktrees - (`gitWorktreeAdapter`) driven by `improvementDriver` with the full agentic - generator (a real coding harness edits each candidate worktree; a `verify` - hook gates candidates before they are ever measured). Ignored when - `opts.generator` is supplied. Without either, `surface: 'code'` still fails - loud — there is no safe zero-config repo to invent. +> **close**(`input`): `Promise`\<\{ `closed`: `true`; \}\> -##### skills? +Defined in: [candidate-execution/types.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L243) -> `optional` **skills?**: `ImproveSkillsOptions` +Revoke evaluator-owned access after process death or a failed preparation. +Must be idempotent and concurrency-safe for the exact preparation/access +pair and must never close a different preparation. -Defined in: [improvement/improve.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L122) +###### Parameters -SKILLS-surface wiring for real skill-DOCUMENT optimization. Without this, - `surface: 'skills'` optimizes the profile's skills REFS array (file pointers) - — which `skillOptProposer` (a document patcher) cannot meaningfully edit. - Provide the document CONTENT to optimize + a `writeBack` to persist the - shipped winner (the profile ref points at a file the caller owns). This is - what makes skillOpt reachable through improve(). +###### input -##### storage? +###### executionId -> `optional` **storage?**: `CampaignStorage` +`string` -Defined in: [improvement/improve.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L124) +###### preparationId -Storage passthrough to `selfImprove`; overrides the default chosen from `runDir`. +`string` -*** +###### accessDigest -### ImproveResult +`` `sha256:${string}` `` -Defined in: [improvement/improve.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L155) +###### effectiveNamespace -#### Type Parameters +`string` -##### TScenario +###### reason -`TScenario` *extends* `Scenario` +`"completed"` \| `"failed"` \| `"timeout"` \| `"replayed"` \| `"preparation-failed"` \| `"abandoned"` -##### TArtifact +###### Returns -`TArtifact` +`Promise`\<\{ `closed`: `true`; \}\> -#### Properties +*** -##### profile +### AgentCandidateExecutionPorts -> **profile**: `AgentProfile` +Defined in: [candidate-execution/types.ts:252](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L252) -Defined in: [improvement/improve.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L158) +#### Extends -The profile after improvement: the winner surface applied back into the - matching field when the gate shipped, else the input profile unchanged. +- [`AgentCandidateVerificationPorts`](#agentcandidateverificationports) -##### shipped +#### Properties -> **shipped**: `boolean` +##### artifacts -Defined in: [improvement/improve.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L160) +> **artifacts**: [`AgentCandidateArtifactPort`](#agentcandidateartifactport) -True when `gateDecision === 'ship'`. +Defined in: [candidate-execution/types.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L72) -##### lift +###### Inherited from -> **lift**: `number` +[`AgentCandidateVerificationPorts`](#agentcandidateverificationports).[`artifacts`](#artifacts) -Defined in: [improvement/improve.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L162) +##### repositories -Held-out lift (`winner − baseline` composite). +> **repositories**: [`AgentCandidateRepositoryPort`](#agentcandidaterepositoryport) -##### gateDecision +Defined in: [candidate-execution/types.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L73) -> **gateDecision**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` +###### Inherited from -Defined in: [improvement/improve.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L164) +[`AgentCandidateVerificationPorts`](#agentcandidateverificationports).[`repositories`](#repositories) -The five-valued gate verdict from `selfImprove`. +##### workspaces -##### raw +> **workspaces**: [`AgentCandidateWorkspacePort`](#agentcandidateworkspaceport) -> **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> +Defined in: [candidate-execution/types.ts:253](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L253) -Defined in: [improvement/improve.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L166) +##### containers -Full `selfImprove` result for advanced inspection. +> **containers**: [`AgentCandidateContainerPort`](#agentcandidatecontainerport) -*** +Defined in: [candidate-execution/types.ts:254](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L254) -### CandidateGenerator +##### models -Defined in: [improvement/improvement-driver.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L36) +> **models**: [`AgentCandidateModelPort`](#agentcandidatemodelport) -The byte-producing seam — the ONE thing that differs between the cheap - reflective path and the full agentic path. A generator makes (uncommitted) - changes inside `worktreePath`; the driver commits them via the worktree - adapter's `finalize`. +Defined in: [candidate-execution/types.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L255) -#### Properties +##### memory -##### kind +> **memory**: [`AgentCandidateMemoryPort`](#agentcandidatememoryport) -> **kind**: `string` +Defined in: [candidate-execution/types.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L256) -Defined in: [improvement/improvement-driver.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L37) +*** -#### Methods +### AgentCandidateTaskExecution -##### generate() +Defined in: [candidate-execution/types.ts:259](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L259) -> **generate**(`args`): `Promise`\<\{ `applied`: `boolean`; `summary`: `string`; \}\> +#### Properties -Defined in: [improvement/improvement-driver.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L38) +##### executionId -###### Parameters +> **executionId**: `string` -###### args +Defined in: [candidate-execution/types.ts:260](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L260) -###### worktreePath +##### benchmark -`string` +> **benchmark**: `string` -The candidate worktree — a fresh checkout of baseRef. Write changes here. +Defined in: [candidate-execution/types.ts:261](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L261) -###### report +##### benchmarkVersion -`unknown` +> **benchmarkVersion**: `string` -Phase-2 research report (analyst findings + diff), opaque. +Defined in: [candidate-execution/types.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L262) -###### findings +##### taskId -`AnalystFinding`[] +> **taskId**: `string` -Findings resolved from the report or the loop context. +Defined in: [candidate-execution/types.ts:263](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L263) -###### dataset? +##### splitDigest -`LabeledScenarioStore` +> **splitDigest**: `` `sha256:${string}` `` -Handle to all captured data, to ground the change. +Defined in: [candidate-execution/types.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L264) -###### maxShots +##### instruction -`number` +> **instruction**: `string` -DEPTH: max iterations the generator may take (agentic uses this; the - reflective generator ignores it). +Defined in: [candidate-execution/types.ts:266](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L266) -###### signal +Exact agent-visible task instruction. The runtime rejects malformed Unicode. -`AbortSignal` +##### repository -###### Returns +> **repository**: `object` -`Promise`\<\{ `applied`: `boolean`; `summary`: `string`; \}\> +Defined in: [candidate-execution/types.ts:267](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L267) -*** +###### identity -### ImprovementDriverOptions +> **identity**: `string` -Defined in: [improvement/improvement-driver.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L54) +###### rootIdentity -#### Properties +> **rootIdentity**: `string` -##### worktree +###### baseCommit -> **worktree**: `WorktreeAdapter` +> **baseCommit**: `string` -Defined in: [improvement/improvement-driver.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L55) +###### baseTree -##### generator +> **baseTree**: `string` -> **generator**: [`CandidateGenerator`](#candidategenerator) +##### attempt -Defined in: [improvement/improvement-driver.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L56) +> **attempt**: `AgentCandidateAttemptPolicy` -##### baseRef? +Defined in: [candidate-execution/types.ts:273](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L273) -> `optional` **baseRef?**: `string` +##### model -Defined in: [improvement/improvement-driver.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L58) +> **model**: `object` -Base ref candidate worktrees fork from. Default `main`. +Defined in: [candidate-execution/types.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L274) -*** +###### requested -### McpServeSpec +> **requested**: `string` -Defined in: [improvement/mcp-serve-verifier.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L24) +###### reasoningEffort -#### Properties +> **reasoningEffort**: `ReasoningEffort` -##### command +##### grader -> **command**: `string` +> **grader**: [`AgentCandidateBenchmarkGraderIdentity`](#agentcandidatebenchmarkgraderidentity) -Defined in: [improvement/mcp-serve-verifier.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L26) +Defined in: [candidate-execution/types.ts:278](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L278) -Command that starts the built MCP server in the worktree (stdio transport). +##### executionRoots -##### args? +> **executionRoots**: `object` -> `optional` **args?**: `string`[] +Defined in: [candidate-execution/types.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L280) -Defined in: [improvement/mcp-serve-verifier.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L27) +Absolute paths inside the evaluator-owned execution environment. -##### env? +###### taskRoot -> `optional` **env?**: `Record`\<`string`, `string`\> +> **taskRoot**: `string` -Defined in: [improvement/mcp-serve-verifier.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L29) +###### candidateRoot? -Extra env for the server process (merged over `process.env`). +> `optional` **candidateRoot?**: `string` -##### timeoutMs? +##### stagingRoots -> `optional` **timeoutMs?**: `number` +> **stagingRoots**: `object` -Defined in: [improvement/mcp-serve-verifier.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L31) +Defined in: [candidate-execution/types.ts:285](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L285) -Handshake timeout (ms). Default 30s. +Host-side staging roots. These are verified but never signed as container paths. -##### minTools? +###### taskRoot -> `optional` **minTools?**: `number` +> **taskRoot**: `string` -Defined in: [improvement/mcp-serve-verifier.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L33) +###### candidateRoot? -Minimum tools the server must expose to pass. Default 1. +> `optional` **candidateRoot?**: `string` -*** +###### profileRoot -### RawTraceDistillerOptions +> **profileRoot**: `string` -Defined in: [improvement/raw-trace-distiller.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L44) +##### workspace -#### Properties +> **workspace**: `AgentCandidateWorkspaceSnapshotEvidence` -##### runDir? +Defined in: [candidate-execution/types.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L290) -> `optional` **runDir?**: `string` +##### evaluatorTaskContainer? -Defined in: [improvement/raw-trace-distiller.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L49) +> `optional` **evaluatorTaskContainer?**: [`ResolvedAgentCandidateContainer`](#resolvedagentcandidatecontainer) -Anchor the emitted paths at this run root instead of the generation `runDir` - the loop passes in. Normally unset — each call points at that generation's - own directory (`input.runDir`). Pass an absolute path when you construct the - producer ahead of the loop and want a fixed anchor (e.g. a test fixture). +Defined in: [candidate-execution/types.ts:291](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L291) -##### maxCandidates? +##### limits -> `optional` **maxCandidates?**: `number` +> **limits**: `AgentCandidateExecutionLimits` -Defined in: [improvement/raw-trace-distiller.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L51) +Defined in: [candidate-execution/types.ts:292](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L292) -Max candidates to surface trace paths for, worst-scoring first. Default 12. +*** -##### maxCellsPerCandidate? +### VerifiedAgentCandidate -> `optional` **maxCellsPerCandidate?**: `number` +Defined in: [candidate-execution/types.ts:295](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L295) -Defined in: [improvement/raw-trace-distiller.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L54) +#### Properties -Max failing cells to enumerate per candidate before collapsing the rest into - an "ls the candidate dir" pointer. Default 8. +##### bundle -##### maxFilesPerCell? +> `readonly` **bundle**: `AgentCandidateBundleV1` -> `optional` **maxFilesPerCell?**: `number` +Defined in: [candidate-execution/types.ts:296](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L296) -Defined in: [improvement/raw-trace-distiller.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L57) +##### materializedTree? -Max concrete file paths to list per cell (the agent can always `ls` the dir - for the rest). Default 24. +> `readonly` `optional` **materializedTree?**: `string` -##### fallbackFindings? +Defined in: [candidate-execution/types.ts:297](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L297) -> `optional` **fallbackFindings?**: `unknown`[] +##### \[verifiedCandidateBrand\] -Defined in: [improvement/raw-trace-distiller.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L61) +> `readonly` **\[verifiedCandidateBrand\]**: `true` -Findings to fall back to when the generation had NO failing cells, so a - clean round never wipes the proposer's steering context. Mirrors the default - distiller's static-seed fallback. Default: a single instruction finding. +Defined in: [candidate-execution/types.ts:298](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L298) *** -### ReflectiveGeneratorOptions +### CanonicalCandidateDocument -Defined in: [improvement/reflective-generator.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L21) +Defined in: [candidate-execution/types.ts:301](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L301) -#### Properties +#### Type Parameters -##### improvementAdapter +##### T -> **improvementAdapter**: [`ImprovementAdapter`](analyst-loop.md#improvementadapter)\<[`SurfaceImprovementEdit`](agent.md#surfaceimprovementedit)\> +`T` -Defined in: [improvement/reflective-generator.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L22) +#### Properties -*** +##### value -### RunKnowledgeImprovementJobOptions +> `readonly` **value**: `T` -Defined in: [knowledge/improvement-job.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L21) +Defined in: [candidate-execution/types.ts:302](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L302) -#### Extends +##### bytes -- `Omit`\<`KnowledgeImprovementOptions`, `"updateKnowledge"`\> +> `readonly` **bytes**: `Uint8Array` -#### Properties +Defined in: [candidate-execution/types.ts:304](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L304) -##### budget +Canonical UTF-8 bytes of `value` with its top-level digest omitted. -> **budget**: [`Budget`](runtime.md#budget-12) +##### digest -Defined in: [knowledge/improvement-job.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L23) +> `readonly` **digest**: `` `sha256:${string}` `` -##### readinessCheck? +Defined in: [candidate-execution/types.ts:305](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L305) -> `optional` **readinessCheck?**: [`KnowledgeReadinessCheck`](#knowledgereadinesscheck) +*** -Defined in: [knowledge/improvement-job.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L24) +### PreparedAgentCandidateLaunch -##### backend? +Defined in: [candidate-execution/types.ts:308](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L308) -> `optional` **backend?**: [`ExecutorConfig`](runtime.md#executorconfig) +#### Properties -Defined in: [knowledge/improvement-job.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L25) +##### executable -##### makeWorkerAgent? +> **executable**: `string` -> `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) +Defined in: [candidate-execution/types.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L309) -Defined in: [knowledge/improvement-job.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L26) +##### args -##### harness? +> **args**: readonly `string`[] -> `optional` **harness?**: `string` +Defined in: [candidate-execution/types.ts:311](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L311) -Defined in: [knowledge/improvement-job.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L27) +Complete fixed argv, including profile materializer flags but excluding task delivery. -##### supervisorModel? +##### env -> `optional` **supervisorModel?**: `string` +> **env**: `Readonly`\<`Record`\<`string`, `string`\>\> -Defined in: [knowledge/improvement-job.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L28) +Defined in: [candidate-execution/types.ts:312](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L312) -##### supervisorSystemPrompt? +##### flags -> `optional` **supervisorSystemPrompt?**: `string` +> **flags**: readonly `string`[] -Defined in: [knowledge/improvement-job.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L29) +Defined in: [candidate-execution/types.ts:314](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L314) -##### superviseOptions? +Informational subset already present at the tail of `args`; executors must not append twice. -> `optional` **superviseOptions?**: `Partial`\<`Omit`\<[`SuperviseOptions`](runtime.md#superviseoptions), `"backend"` \| `"budget"` \| `"makeWorkerAgent"` \| `"deliverable"` \| `"allowedModels"`\>\> +##### cwd -Defined in: [knowledge/improvement-job.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L30) +> **cwd**: `string` -##### allowedModels? +Defined in: [candidate-execution/types.ts:315](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L315) -> `optional` **allowedModels?**: readonly `string`[] +*** -Defined in: [knowledge/improvement-job.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L36) +### PreparedAgentCandidateInstruction -##### runSupervised? +Defined in: [candidate-execution/types.ts:318](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L318) -> `optional` **runSupervised?**: (`profile`, `task`, `opts`) => `Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> +#### Properties -Defined in: [knowledge/improvement-job.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L37) +##### bytes -###### Parameters +> **bytes**: `Uint8Array` -###### profile +Defined in: [candidate-execution/types.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L319) -[`SupervisorProfile`](runtime.md#supervisorprofile) +##### delivery -###### task +> **delivery**: `AgentCandidateInstructionDelivery` -`unknown` +Defined in: [candidate-execution/types.ts:320](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L320) -###### opts +*** -[`SuperviseOptions`](runtime.md#superviseoptions) +### PreparedAgentCandidateTrace -###### Returns +Defined in: [candidate-execution/types.ts:323](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L323) -`Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> +#### Properties -##### onMeasurement? +##### runId -> `optional` **onMeasurement?**: (`measurement`) => `void` \| `Promise`\<`void`\> +> **runId**: `string` -Defined in: [knowledge/improvement-job.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L42) +Defined in: [candidate-execution/types.ts:324](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L324) -###### Parameters +##### tags -###### measurement +> **tags**: `Readonly`\<`Record`\<`string`, `string`\>\> -[`KnowledgeImprovementJobMeasurement`](#knowledgeimprovementjobmeasurement) +Defined in: [candidate-execution/types.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L325) -###### Returns +##### env -`void` \| `Promise`\<`void`\> +> **env**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [candidate-execution/types.ts:326](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L326) *** -### KnowledgeImprovementJobMeasurement +### PreparedAgentCandidateExecution -Defined in: [knowledge/improvement-job.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L45) +Defined in: [candidate-execution/types.ts:329](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L329) #### Properties -##### startedAt +##### bundle -> **startedAt**: `string` +> `readonly` **bundle**: `AgentCandidateBundleV1` -Defined in: [knowledge/improvement-job.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L46) +Defined in: [candidate-execution/types.ts:330](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L330) -##### finishedAt +##### executionId -> **finishedAt**: `string` +> `readonly` **executionId**: `string` -Defined in: [knowledge/improvement-job.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L47) +Defined in: [candidate-execution/types.ts:331](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L331) -##### durationMs +##### roots -> **durationMs**: `number` +> `readonly` **roots**: `object` -Defined in: [knowledge/improvement-job.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L48) +Defined in: [candidate-execution/types.ts:332](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L332) -##### updateCalls +###### execution -> **updateCalls**: `number` +> **execution**: `object` -Defined in: [knowledge/improvement-job.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L49) +###### execution.taskRoot -##### updateDurationMs +> **taskRoot**: `string` -> **updateDurationMs**: `number` +###### execution.candidateRoot? -Defined in: [knowledge/improvement-job.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L50) +> `optional` **candidateRoot?**: `string` -##### supervisedSpent +###### staging -> **supervisedSpent**: `object` +> **staging**: `object` -Defined in: [knowledge/improvement-job.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L51) +###### staging.taskRoot -###### iterations +> **taskRoot**: `string` -> **iterations**: `number` +###### staging.candidateRoot? -###### inputTokens +> `optional` **candidateRoot?**: `string` -> **inputTokens**: `number` +###### staging.profileRoot -###### outputTokens +> **profileRoot**: `string` -> **outputTokens**: `number` +##### profilePlan -###### usd +> `readonly` **profilePlan**: `object` -> **usd**: `number` +Defined in: [candidate-execution/types.ts:343](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L343) -###### ms +###### value -> **ms**: `number` +> **value**: `AgentCandidateProfilePlanEvidence` -*** +###### bytes -### KnowledgeImprovementJobResult +> **bytes**: `Uint8Array` -Defined in: [knowledge/improvement-job.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L60) +###### written -#### Properties +> **written**: readonly `string`[] -##### improvement +##### executionPlan -> **improvement**: `KnowledgeImprovementResult` +> `readonly` **executionPlan**: `object` -Defined in: [knowledge/improvement-job.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L61) +Defined in: [candidate-execution/types.ts:348](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L348) -##### measurement +###### value -> **measurement**: [`KnowledgeImprovementJobMeasurement`](#knowledgeimprovementjobmeasurement) +> **value**: `AgentCandidateExecutionPlanEvidence` -Defined in: [knowledge/improvement-job.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L62) +###### bytes -##### promoted +> **bytes**: `Uint8Array` -> **promoted**: `boolean` +##### materializationReceipt -Defined in: [knowledge/improvement-job.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L63) +> `readonly` **materializationReceipt**: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateMaterializationReceiptV1`\> -##### blocked +Defined in: [candidate-execution/types.ts:352](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L352) -> **blocked**: `boolean` +##### launch -Defined in: [knowledge/improvement-job.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L64) +> `readonly` **launch**: [`PreparedAgentCandidateLaunch`](#preparedagentcandidatelaunch) -*** +Defined in: [candidate-execution/types.ts:353](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L353) -### AgentKnowledgeReadinessCheckOptions +##### instruction -Defined in: [knowledge/improvement-job.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L67) +> `readonly` **instruction**: [`PreparedAgentCandidateInstruction`](#preparedagentcandidateinstruction) -#### Properties +Defined in: [candidate-execution/types.ts:354](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L354) -##### goal +##### resolvedModel -> **goal**: `string` +> `readonly` **resolvedModel**: `AgentCandidateResolvedModel` -Defined in: [knowledge/improvement-job.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L68) +Defined in: [candidate-execution/types.ts:355](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L355) -##### readinessSpecs? +##### knowledge? -> `optional` **readinessSpecs?**: readonly `KnowledgeReadinessSpec`[] +> `readonly` `optional` **knowledge?**: `object` -Defined in: [knowledge/improvement-job.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L69) +Defined in: [candidate-execution/types.ts:356](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L356) -##### readinessTaskId? +###### snapshotId -> `optional` **readinessTaskId?**: `string` +> **snapshotId**: `string` -Defined in: [knowledge/improvement-job.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L70) +###### manifestDigest -##### readiness? +> **manifestDigest**: `` `sha256:${string}` `` -> `optional` **readiness?**: `Omit`\<`BuildEvalKnowledgeBundleOptions`, `"taskId"` \| `"index"` \| `"specs"`\> +###### manifest -Defined in: [knowledge/improvement-job.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L71) +> **manifest**: `Uint8Array` -##### strict? +##### trace -> `optional` **strict?**: `boolean` +> `readonly` **trace**: [`PreparedAgentCandidateTrace`](#preparedagentcandidatetrace) -Defined in: [knowledge/improvement-job.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L72) +Defined in: [candidate-execution/types.ts:361](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L361) -##### kbQuality? +##### memory -> `optional` **kbQuality?**: `KnowledgeBaseQualityOptions` +> `readonly` **memory**: `AgentCandidateEffectiveMemory` -Defined in: [knowledge/improvement-job.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L73) +Defined in: [candidate-execution/types.ts:362](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L362) + +##### \[preparedCandidateBrand\] + +> `readonly` **\[preparedCandidateBrand\]**: `true` + +Defined in: [candidate-execution/types.ts:363](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L363) *** -### KnowledgeReadinessCheckInput +### AgentCandidateProtectedRunCapture -Defined in: [knowledge/supervised-update.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L22) +Defined in: [candidate-execution/types.ts:366](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L366) #### Properties -##### root +##### executionId -> **root**: `string` +> **executionId**: `string` -Defined in: [knowledge/supervised-update.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L23) +Defined in: [candidate-execution/types.ts:367](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L367) -##### goal +##### termination -> **goal**: `string` +> **termination**: `AgentCandidateTermination` -Defined in: [knowledge/supervised-update.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L24) +Defined in: [candidate-execution/types.ts:368](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L368) -##### readinessSpecs? +*** -> `optional` **readinessSpecs?**: readonly `unknown`[] +### AgentCandidateExecutorTaskOutcomeCapture -Defined in: [knowledge/supervised-update.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L25) +Defined in: [candidate-execution/types.ts:372](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L372) -##### readinessTaskId? +Raw evaluator capture made only after the candidate process is dead. -> `optional` **readinessTaskId?**: `string` +#### Properties -Defined in: [knowledge/supervised-update.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L26) +##### resultTree -##### readiness? +> **resultTree**: `string` -> `optional` **readiness?**: `unknown` +Defined in: [candidate-execution/types.ts:374](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L374) -Defined in: [knowledge/supervised-update.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L27) +Claimed final tree. The runtime recomputes it independently from `gitDiff`. -*** +##### afterState -### SupervisedKnowledgeUpdateInput +> **afterState**: `AgentCandidateWorkspaceManifestMaterialV1` -Defined in: [knowledge/supervised-update.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L42) +Defined in: [candidate-execution/types.ts:376](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L376) -#### Properties +Complete evaluator-captured workspace description after candidate execution. -##### goal? +##### archive -> `optional` **goal?**: `string` +> **archive**: `Uint8Array` -Defined in: [knowledge/supervised-update.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L43) +Defined in: [candidate-execution/types.ts:378](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L378) -##### root? +Reproducible workspace archive corresponding to `afterState`. -> `optional` **root?**: `string` +##### gitDiff -Defined in: [knowledge/supervised-update.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L44) +> **gitDiff**: `Uint8Array` -##### candidateRoot? +Defined in: [candidate-execution/types.ts:380](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L380) -> `optional` **candidateRoot?**: `string` +Exact binary patch from the signed task base to `afterState`. -Defined in: [knowledge/supervised-update.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L45) +*** -##### findings? +### AgentCandidateExecutorMemoryCapture -> `optional` **findings?**: readonly `unknown`[] +Defined in: [candidate-execution/types.ts:384](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L384) -Defined in: [knowledge/supervised-update.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L46) +Raw isolated-memory capture made only after access has been revoked. -##### metadata? +#### Properties -> `optional` **metadata?**: `Record`\<`string`, `unknown`\> +##### afterState -Defined in: [knowledge/supervised-update.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L47) +> `readonly` **afterState**: `AgentCandidateWorkspaceManifestMaterialV1` -*** +Defined in: [candidate-execution/types.ts:385](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L385) -### SupervisedKnowledgeUpdateResult +##### archive -Defined in: [knowledge/supervised-update.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L50) +> `readonly` **archive**: `Uint8Array` -#### Properties +Defined in: [candidate-execution/types.ts:386](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L386) -##### applied +*** -> **applied**: `boolean` +### AgentCandidateExecutorFinalCapture -Defined in: [knowledge/supervised-update.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L51) +Defined in: [candidate-execution/types.ts:390](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L390) -##### summary +Idempotent executor result after process death and trace drain. -> **summary**: `string` +#### Properties -Defined in: [knowledge/supervised-update.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L52) +##### stopped -##### supervised +> `readonly` **stopped**: `true` -> **supervised**: [`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\> +Defined in: [candidate-execution/types.ts:391](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L391) -Defined in: [knowledge/supervised-update.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L53) +##### taskOutcome? -##### metadata +> `readonly` `optional` **taskOutcome?**: [`AgentCandidateExecutorTaskOutcomeCapture`](#agentcandidateexecutortaskoutcomecapture) -> **metadata**: `Record`\<`string`, `unknown`\> +Defined in: [candidate-execution/types.ts:392](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L392) -Defined in: [knowledge/supervised-update.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L54) +##### memoryAfter? + +> `readonly` `optional` **memoryAfter?**: [`AgentCandidateExecutorMemoryCapture`](#agentcandidateexecutormemorycapture) + +Defined in: [candidate-execution/types.ts:394](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L394) + +Required only when the prepared candidate uses isolated task memory. *** -### SupervisedKnowledgeUpdateOptions +### VerifiedAgentCandidateTaskOutcome -Defined in: [knowledge/supervised-update.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L57) +Defined in: [candidate-execution/types.ts:398](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L398) + +Branded task outcome that has survived independent patch and tree verification. #### Properties -##### root +##### evidence -> **root**: `string` +> `readonly` **evidence**: `AgentCandidateTaskOutcomeEvidence` & `object` -Defined in: [knowledge/supervised-update.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L58) +Defined in: [candidate-execution/types.ts:399](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L399) -##### goal +###### Type Declaration -> **goal**: `string` +###### artifact -Defined in: [knowledge/supervised-update.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L59) +> `readonly` **artifact**: `AgentCandidateArtifactRef` -##### readiness +##### patch -> **readiness**: [`KnowledgeReadinessCheck`](#knowledgereadinesscheck) +> `readonly` **patch**: `Uint8Array` -Defined in: [knowledge/supervised-update.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L60) +Defined in: [candidate-execution/types.ts:402](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L402) -##### readinessSpecs? +##### \[verifiedTaskOutcomeBrand\] -> `optional` **readinessSpecs?**: readonly `unknown`[] +> `readonly` **\[verifiedTaskOutcomeBrand\]**: `true` -Defined in: [knowledge/supervised-update.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L61) +Defined in: [candidate-execution/types.ts:403](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L403) -##### readinessTaskId? +*** -> `optional` **readinessTaskId?**: `string` +### AgentCandidateBenchmarkGraderPort -Defined in: [knowledge/supervised-update.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L62) +Defined in: [candidate-execution/types.ts:415](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L415) -##### readinessOptions? +Evaluator-owned executable grader, pinned by immutable implementation bytes. -> `optional` **readinessOptions?**: `unknown` +`run` is an isolation boundary, not an arbitrary scoring callback. The +implementation admitted to that boundary is supplied by the runtime after +artifact verification. Implementations must derive every returned binding +digest from the bytes and task outcome they actually admitted, rather than +copying an expected digest from ambient configuration. -Defined in: [knowledge/supervised-update.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L63) +#### Properties -##### findings? +##### name -> `optional` **findings?**: readonly `unknown`[] +> `readonly` **name**: `string` -Defined in: [knowledge/supervised-update.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L64) +Defined in: [candidate-execution/types.ts:416](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L416) -##### metadata? +##### version -> `optional` **metadata?**: `Record`\<`string`, `unknown`\> +> `readonly` **version**: `string` -Defined in: [knowledge/supervised-update.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L65) +Defined in: [candidate-execution/types.ts:417](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L417) -##### budget +##### artifact -> **budget**: [`Budget`](runtime.md#budget-12) +> `readonly` **artifact**: `AgentCandidateArtifactRef` -Defined in: [knowledge/supervised-update.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L66) +Defined in: [candidate-execution/types.ts:418](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L418) -##### backend? +#### Methods -> `optional` **backend?**: [`ExecutorConfig`](runtime.md#executorconfig) +##### run() -Defined in: [knowledge/supervised-update.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L67) +> **run**(`input`): `Promise`\<\{ `evaluation`: `BenchmarkEvaluation`; `evidence`: `Uint8Array`; `binding`: \{ `implementationDigest`: `` `sha256:${string}` ``; `taskOutcomeDigest`: `` `sha256:${string}` ``; `outputDigest`: `` `sha256:${string}` ``; \}; \}\> -##### makeWorkerAgent? +Defined in: [candidate-execution/types.ts:419](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L419) -> `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) +###### Parameters -Defined in: [knowledge/supervised-update.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L68) +###### input -##### harness? +###### executionId -> `optional` **harness?**: `string` +`string` -Defined in: [knowledge/supervised-update.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L69) +###### termination -##### supervisorModel? +`AgentCandidateTermination` -> `optional` **supervisorModel?**: `string` +###### outcome -Defined in: [knowledge/supervised-update.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L70) +[`VerifiedAgentCandidateTaskOutcome`](#verifiedagentcandidatetaskoutcome) -##### supervisorSystemPrompt? +###### implementation -> `optional` **supervisorSystemPrompt?**: `string` +\{ `byteLength`: `number`; `bytes`: `Uint8Array`; \} -Defined in: [knowledge/supervised-update.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L71) +Exact verified artifact bytes. Each read returns a detached copy. -##### superviseOptions? +###### implementation.byteLength -> `optional` **superviseOptions?**: `Partial`\<`Omit`\<[`SuperviseOptions`](runtime.md#superviseoptions), `"backend"` \| `"budget"` \| `"makeWorkerAgent"` \| `"deliverable"` \| `"allowedModels"`\>\> +`number` -Defined in: [knowledge/supervised-update.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L72) +###### implementation.bytes -##### allowedModels? +`Uint8Array` -> `optional` **allowedModels?**: readonly `string`[] +###### signal -Defined in: [knowledge/supervised-update.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L78) +`AbortSignal` -##### runSupervised? +Frozen result deadline; runners must stop work and side effects when aborted. -> `optional` **runSupervised?**: (`profile`, `task`, `opts`) => `Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> +###### Returns -Defined in: [knowledge/supervised-update.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L79) +`Promise`\<\{ `evaluation`: `BenchmarkEvaluation`; `evidence`: `Uint8Array`; `binding`: \{ `implementationDigest`: `` `sha256:${string}` ``; `taskOutcomeDigest`: `` `sha256:${string}` ``; `outputDigest`: `` `sha256:${string}` ``; \}; \}\> -###### Parameters +*** -###### profile +### AgentCandidateExecutorRequest -[`SupervisorProfile`](runtime.md#supervisorprofile) +Defined in: [candidate-execution/types.ts:447](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L447) -###### task +One detached request passed to the trusted environment-specific executor. -`unknown` +#### Properties -###### opts +##### executionId -[`SuperviseOptions`](runtime.md#superviseoptions) +> `readonly` **executionId**: `string` -###### Returns +Defined in: [candidate-execution/types.ts:448](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L448) -`Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> +##### inputs -*** +> `readonly` **inputs**: `object` -### DelegatedLoopResult +Defined in: [candidate-execution/types.ts:450](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L450) -Defined in: [loop-runner.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L67) +Immutable bytes from which the executor creates fresh isolated workspaces. -**`Experimental`** +###### task -Uniform result — never throws from a registered runner; a - thrown engine becomes `{ ok: false, error }` so a routine can record + move on. +> `readonly` **task**: [`AgentCandidateExecutorWorkspaceInput`](#agentcandidateexecutorworkspaceinput) -#### Type Parameters +###### candidate? -##### T +> `readonly` `optional` **candidate?**: [`AgentCandidateExecutorWorkspaceInput`](#agentcandidateexecutorworkspaceinput) -`T` = `unknown` +###### profile -#### Properties +> `readonly` **profile**: `object` -##### mode +###### profile.files -> **mode**: `"code"` \| `"review"` \| `"research"` \| `"audit"` \| `"self-improve"` +> `readonly` **files**: readonly [`AgentCandidateExecutorProfileFile`](#agentcandidateexecutorprofilefile)[] -Defined in: [loop-runner.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L68) +##### roots -**`Experimental`** +> `readonly` **roots**: `object` -##### ok +Defined in: [candidate-execution/types.ts:457](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L457) -> **ok**: `boolean` +###### taskRoot -Defined in: [loop-runner.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L69) +> **taskRoot**: `string` -**`Experimental`** +###### candidateRoot? -##### output? +> `optional` **candidateRoot?**: `string` -> `optional` **output?**: `T` +##### profilePlan -Defined in: [loop-runner.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L70) +> `readonly` **profilePlan**: `object` -**`Experimental`** +Defined in: [candidate-execution/types.ts:458](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L458) -##### error? +###### value -> `optional` **error?**: `string` +> **value**: `AgentCandidateProfilePlanEvidence` -Defined in: [loop-runner.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L71) +###### bytes -**`Experimental`** +> **bytes**: `Uint8Array` -##### durationMs +###### written -> **durationMs**: `number` +> **written**: readonly `string`[] -Defined in: [loop-runner.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L72) +##### executionPlan -**`Experimental`** +> `readonly` **executionPlan**: `object` -*** +Defined in: [candidate-execution/types.ts:459](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L459) -### RunDelegatedLoopOptions +###### value -Defined in: [loop-runner.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L76) +> **value**: `AgentCandidateExecutionPlanEvidence` -**`Experimental`** +###### bytes -#### Properties +> **bytes**: `Uint8Array` -##### signal? +##### materializationReceipt -> `optional` **signal?**: `AbortSignal` +> `readonly` **materializationReceipt**: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateMaterializationReceiptV1`\> -Defined in: [loop-runner.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L77) +Defined in: [candidate-execution/types.ts:460](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L460) -**`Experimental`** +##### launch -##### now? +> `readonly` **launch**: [`PreparedAgentCandidateLaunch`](#preparedagentcandidatelaunch) -> `optional` **now?**: () => `number` +Defined in: [candidate-execution/types.ts:461](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L461) -Defined in: [loop-runner.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L79) +##### instruction -**`Experimental`** +> `readonly` **instruction**: [`PreparedAgentCandidateInstruction`](#preparedagentcandidateinstruction) -Clock override for deterministic tests. +Defined in: [candidate-execution/types.ts:462](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L462) -###### Returns +##### resolvedModel -`number` +> `readonly` **resolvedModel**: `AgentCandidateResolvedModel` -*** +Defined in: [candidate-execution/types.ts:463](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L463) -### WorktreeLoopRunnerOptions +##### hardLimits -Defined in: [loop-runner.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L121) +> `readonly` **hardLimits**: `Pick`\<`AgentCandidateExecutionLimits`, `"timeoutMs"`\> -**`Experimental`** +Defined in: [candidate-execution/types.ts:465](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L465) -Options for the local-repo `code` runner over the GENERIC recursive path. - -#### Properties +Mechanically enforced by the runtime plus executor process-death acknowledgement. -##### repoRoot +##### observedLimits -> **repoRoot**: `string` +> `readonly` **observedLimits**: `Pick`\<`AgentCandidateExecutionLimits`, `"maxSteps"`\> -Defined in: [loop-runner.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L123) +Defined in: [candidate-execution/types.ts:467](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L467) -**`Experimental`** +Validity bound checked against protected traces; generic black-box executors cannot preempt it. -Absolute path to the local git checkout each worktree is cut from. +##### knowledge? -##### taskPrompt +> `readonly` `optional` **knowledge?**: `object` -> **taskPrompt**: `string` +Defined in: [candidate-execution/types.ts:468](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L468) -Defined in: [loop-runner.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L125) +###### snapshotId -**`Experimental`** +> **snapshotId**: `string` -The instruction handed to every authored harness (composed under each profile's systemPrompt). +###### manifestDigest -##### harnesses +> **manifestDigest**: `` `sha256:${string}` `` -> **harnesses**: readonly [`AuthoredHarness`](runtime.md#authoredharness)[] +###### manifest -Defined in: [loop-runner.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L127) +> **manifest**: `Uint8Array` -**`Experimental`** +##### trace -The supervisor-authored harness profiles — one fanout item (one worktree-CLI leaf) each. +> `readonly` **trace**: [`PreparedAgentCandidateTrace`](#preparedagentcandidatetrace) -##### budget +Defined in: [candidate-execution/types.ts:469](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L469) -> **budget**: [`Budget`](runtime.md#budget-12) +##### memory -Defined in: [loop-runner.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L129) +> `readonly` **memory**: `AgentCandidateEffectiveMemory` -**`Experimental`** +Defined in: [candidate-execution/types.ts:470](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L470) -Conserved budget pool bounding the fanout (equal-k holds by construction). +*** -##### testCmd? +### AgentCandidateExecutorPort -> `optional` **testCmd?**: `string` +Defined in: [candidate-execution/types.ts:481](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L481) -Defined in: [loop-runner.ts:131](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L131) +Executes one prepared request inside an evaluator-owned isolation boundary. -**`Experimental`** +`request.launch.env` is the complete allowlisted environment, including +protected model, memory, and trace bindings. Implementations must not merge +ambient host variables into it. The returned capture deliberately contains +no candidate-authored usage or score fields. -Shell command run in each worktree to derive the tests-PASS signal. +#### Methods -##### typecheckCmd? +##### execute() -> `optional` **typecheckCmd?**: `string` +> **execute**(`request`, `context`): `Promise`\<[`AgentCandidateProtectedRunCapture`](#agentcandidateprotectedruncapture)\> -Defined in: [loop-runner.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L133) +Defined in: [candidate-execution/types.ts:482](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L482) -**`Experimental`** +###### Parameters -Shell command run in each worktree to derive the typecheck-PASS signal. +###### request -##### require? +[`AgentCandidateExecutorRequest`](#agentcandidateexecutorrequest) -> `optional` **require?**: readonly (`"tests"` \| `"typecheck"`)[] +###### context -Defined in: [loop-runner.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L135) +###### traceStore -**`Experimental`** +`TraceStore` -Which verification signals the deliverable REQUIRES present-and-passing (default none). +###### signal -##### maxDiffLines? +`AbortSignal` -> `optional` **maxDiffLines?**: `number` +Aborted by the runtime at the exact frozen wall-time deadline. -Defined in: [loop-runner.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L137) +###### deadlineAtMs -**`Experimental`** +`number` -Diff-size cap (lines). +Absolute epoch-millisecond deadline owned by the runtime. -##### forbiddenPaths? +###### Returns -> `optional` **forbiddenPaths?**: `string`[] +`Promise`\<[`AgentCandidateProtectedRunCapture`](#agentcandidateprotectedruncapture)\> -Defined in: [loop-runner.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L139) +##### stopAndCapture() -**`Experimental`** +> **stopAndCapture**(`request`, `context`): `Promise`\<[`AgentCandidateExecutorFinalCapture`](#agentcandidateexecutorfinalcapture)\> -Literal path prefixes the patch must not touch (the secret-floor is always on regardless). +Defined in: [candidate-execution/types.ts:499](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L499) -##### winnerStrategy? +Kill any process/container still associated with the request, drain trace +writes, and capture the final task workspace before teardown. +The runtime calls this on success, failure, and timeout before model settlement. +Implementations must be idempotent and concurrency-safe for this exact +execution/plan pair because a fresh worker may repeat crash recovery. -> `optional` **winnerStrategy?**: [`WinnerStrategy`](runtime.md#winnerstrategy) +###### Parameters -Defined in: [loop-runner.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L141) +###### request -**`Experimental`** +[`AgentCandidateExecutorStopRequest`](#agentcandidateexecutorstoprequest) -Winner-selection strategy among gated candidates. Default `highest-score`. +###### context -##### runGit? +###### traceStore -> `optional` **runGit?**: [`GitRunner`](mcp.md#gitrunner) +`TraceStore` -Defined in: [loop-runner.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L143) +###### reason -**`Experimental`** +`"completed"` \| `"failed"` \| `"timeout"` -Test seams forwarded to the worktree-CLI leaves so the runner drives offline. +###### signal -##### runHarness? +`AbortSignal` -> `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> +Aborted at the frozen execution deadline or evaluator cleanup deadline. -Defined in: [loop-runner.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L144) +###### deadlineAtMs -**`Experimental`** +`number` -**`Experimental`** +Absolute execution deadline; a later stop acknowledgement cannot produce success. -Spawn a local coding harness CLI as a subprocess + collect its output. +###### Returns -NOT responsible for parsing the harness's output or extracting a diff — -the in-process executor's `streamPrompt` orchestrates `git diff` against -the worktree after this resolves. This function is intentionally narrow: -spawn, wait, capture, return. +`Promise`\<[`AgentCandidateExecutorFinalCapture`](#agentcandidateexecutorfinalcapture)\> -Fails loud — throws when: - - `cwd` doesn't exist (subprocess emits ENOENT; surfaced as Error) - - the harness binary is not on PATH (ENOENT) +*** -Does NOT throw when: - - the subprocess exits non-zero (`result.exitCode` carries the code) - - the subprocess is aborted / timed out (`result.killedBySignal` / - `result.timedOut` carries the reason) +### AgentCandidateExecutorStopRequest -###### Parameters +Defined in: [candidate-execution/types.ts:513](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L513) -###### options +Opaque process identity used for termination without re-exposing launch credentials. -[`RunLocalHarnessOptions`](mcp.md#runlocalharnessoptions) +#### Properties -###### Returns +##### executionId -`Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> +> `readonly` **executionId**: `string` -##### runCommand? +Defined in: [candidate-execution/types.ts:514](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L514) -> `optional` **runCommand?**: `WorktreeCheckRunner` +##### executionPlanDigest -Defined in: [loop-runner.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L145) +> `readonly` **executionPlanDigest**: `` `sha256:${string}` `` -**`Experimental`** +Defined in: [candidate-execution/types.ts:515](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L515) *** -### VetoedFact - -Defined in: [loop-runner.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L208) - -**`Experimental`** +### AgentCandidateExecutorWorkspaceInput -A fact rejected at the KB gate — surfaced, never dropped. +Defined in: [candidate-execution/types.ts:518](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L518) #### Properties -##### candidate +##### snapshot -> **candidate**: [`FactCandidate`](mcp.md#factcandidate) +> `readonly` **snapshot**: `AgentCandidateWorkspaceSnapshotEvidence` -Defined in: [loop-runner.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L209) +Defined in: [candidate-execution/types.ts:519](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L519) -**`Experimental`** +##### files -##### vetoedBy? +> `readonly` **files**: readonly [`AgentCandidateExecutorWorkspaceFile`](#agentcandidateexecutorworkspacefile)[] -> `optional` **vetoedBy?**: `string` +Defined in: [candidate-execution/types.ts:520](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L520) -Defined in: [loop-runner.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L210) +*** -**`Experimental`** +### AgentCandidateExecutorWorkspaceFile -##### reason? +Defined in: [candidate-execution/types.ts:523](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L523) -> `optional` **reason?**: `string` +#### Properties -Defined in: [loop-runner.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L211) +##### path -**`Experimental`** +> `readonly` **path**: `string` -*** +Defined in: [candidate-execution/types.ts:524](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L524) -### ResearchLoopResult +##### mode -Defined in: [loop-runner.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L215) +> `readonly` **mode**: `420` \| `493` -**`Experimental`** +Defined in: [candidate-execution/types.ts:525](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L525) -#### Properties +##### bytes -##### accepted +> `readonly` **bytes**: `Uint8Array` -> **accepted**: [`FactCandidate`](mcp.md#factcandidate)[] +Defined in: [candidate-execution/types.ts:526](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L526) -Defined in: [loop-runner.ts:217](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L217) +*** -**`Experimental`** +### AgentCandidateExecutorProfileFile -Facts that passed the fail-closed gate — safe to write to the KB. +Defined in: [candidate-execution/types.ts:529](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L529) -##### vetoed +#### Properties -> **vetoed**: [`VetoedFact`](#vetoedfact)[] +##### path -Defined in: [loop-runner.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L219) +> `readonly` **path**: `string` -**`Experimental`** +Defined in: [candidate-execution/types.ts:530](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L530) -Facts the gate vetoed in the final round — escalate, do not silently drop. +##### mode -##### rounds +> `readonly` **mode**: `420` \| `493` -> **rounds**: `number` +Defined in: [candidate-execution/types.ts:531](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L531) -Defined in: [loop-runner.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L221) +##### bytes -**`Experimental`** +> `readonly` **bytes**: `Uint8Array` -Research rounds actually run. +Defined in: [candidate-execution/types.ts:532](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L532) *** -### ResearchLoopRunnerOptions - -Defined in: [loop-runner.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L225) - -**`Experimental`** +### AgentCandidateWorkspaceArchiveLimits -Options for the default `research` runner. +Defined in: [candidate-execution/workspace-archive.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L52) #### Properties -##### research - -> **research**: (`round`, `vetoed`) => `Promise`\<[`FactCandidate`](mcp.md#factcandidate)[]\> +##### maxArchiveBytes -Defined in: [loop-runner.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L232) +> **maxArchiveBytes**: `number` -**`Experimental`** +Defined in: [candidate-execution/workspace-archive.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L53) -The research engine (the consumer's web/doc searcher + extractor). Called -each round with the prior round's vetoes so it can re-research the gaps. -Returns fact candidates carrying their grounding (`verbatimPassage` + -`sourceText`). +##### maxEmbeddedArtifactBytes -###### Parameters +> **maxEmbeddedArtifactBytes**: `number` -###### round +Defined in: [candidate-execution/workspace-archive.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L54) -`number` +##### maxFiles -###### vetoed +> **maxFiles**: `number` -[`VetoedFact`](#vetoedfact)[] +Defined in: [candidate-execution/workspace-archive.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L55) -###### Returns +##### maxFileBytes -`Promise`\<[`FactCandidate`](mcp.md#factcandidate)[]\> +> **maxFileBytes**: `number` -##### gate? +Defined in: [candidate-execution/workspace-archive.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L56) -> `optional` **gate?**: [`CreateKbGateOptions`](mcp.md#createkbgateoptions) +##### maxTotalFileBytes -Defined in: [loop-runner.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L234) +> **maxTotalFileBytes**: `number` -**`Experimental`** +Defined in: [candidate-execution/workspace-archive.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L57) -Gate config (extra judges, self-artifact kinds, …). The floor is always on. +##### maxPathBytes -##### maxRounds? +> **maxPathBytes**: `number` -> `optional` **maxRounds?**: `number` +Defined in: [candidate-execution/workspace-archive.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L58) -Defined in: [loop-runner.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L236) +##### maxRepositoryBundleBytes -**`Experimental`** +> **maxRepositoryBundleBytes**: `number` -Max research rounds (correct-on-veto remediation). Default 1. +Defined in: [candidate-execution/workspace-archive.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L59) *** -### ModelInfo - -Defined in: [model-resolution.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L22) +### CaptureAgentCandidateWorkspaceOptions -A model entry as returned by the Tangle Router `/v1/models` endpoint. -Intentionally minimal — only the fields resolution + validation read. +Defined in: [candidate-execution/workspace-archive.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L98) #### Properties -##### id - -> **id**: `string` +##### includeRepository? -Defined in: [model-resolution.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L23) +> `optional` **includeRepository?**: `boolean` -##### name? +Defined in: [candidate-execution/workspace-archive.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L100) -> `optional` **name?**: `string` +Include Git HEAD so task preparation can prove its exact commit and tree. -Defined in: [model-resolution.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L24) +##### limits? -##### description? +> `optional` **limits?**: `Partial`\<[`AgentCandidateWorkspaceArchiveLimits`](#agentcandidateworkspacearchivelimits)\> -> `optional` **description?**: `string` +Defined in: [candidate-execution/workspace-archive.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L101) -Defined in: [model-resolution.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L25) +##### artifactPersistence? -##### provider? +> `optional` **artifactPersistence?**: `object` -> `optional` **provider?**: `string` +Defined in: [candidate-execution/workspace-archive.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L103) -Defined in: [model-resolution.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L27) +Use the evaluator-owned artifact store when manifest or archive bytes should not be embedded. -Provider slug, when the router exposes it (`provider` or `_provider`). +###### executionId -##### \_provider? +> **executionId**: `string` -> `optional` **\_provider?**: `string` +###### outputArtifacts -Defined in: [model-resolution.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L28) +> **outputArtifacts**: [`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) -##### architecture? +###### signal? -> `optional` **architecture?**: `object` +> `optional` **signal?**: `AbortSignal` -Defined in: [model-resolution.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L29) +*** -###### modality? +### CreateAgentCandidateWorkspacePortOptions -> `optional` **modality?**: `string` +Defined in: [candidate-execution/workspace-archive.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L110) -###### input\_modalities? +#### Properties -> `optional` **input\_modalities?**: `string`[] +##### limits? -###### output\_modalities? +> `optional` **limits?**: `Partial`\<[`AgentCandidateWorkspaceArchiveLimits`](#agentcandidateworkspacearchivelimits)\> -> `optional` **output\_modalities?**: `string`[] +Defined in: [candidate-execution/workspace-archive.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L111) *** -### RouterEnv - -Defined in: [model-resolution.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L37) +### CapturedAgentCandidateWorkspace -Env keys the router base URL is resolved from. +Defined in: [candidate-execution/workspace-archive.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L114) #### Properties -##### TANGLE\_ROUTER\_URL? +##### snapshot -> `optional` **TANGLE\_ROUTER\_URL?**: `string` +> `readonly` **snapshot**: `AgentCandidateWorkspaceSnapshotEvidence` -Defined in: [model-resolution.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L38) +Defined in: [candidate-execution/workspace-archive.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L115) -##### TANGLE\_ROUTER\_BASE\_URL? +##### archive -> `optional` **TANGLE\_ROUTER\_BASE\_URL?**: `string` +> `readonly` **archive**: `Uint8Array` -Defined in: [model-resolution.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L39) +Defined in: [candidate-execution/workspace-archive.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L117) + +Caller-owned bytes accepted by createAgentCandidateWorkspacePort. *** -### ResolvedChatModel +### CircuitBreakerConfig -Defined in: [model-resolution.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L80) +Defined in: [conversation/call-policy.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L24) + +Circuit-breaker tuning. `failuresToOpen` consecutive failures opens it; closed only after `cooldownMs`. #### Properties -##### source +##### failuresToOpen -> **source**: `string` +> **failuresToOpen**: `number` -Defined in: [model-resolution.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L81) +Defined in: [conversation/call-policy.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L25) -##### model +##### cooldownMs -> **model**: `string` +> **cooldownMs**: `number` -Defined in: [model-resolution.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L82) +Defined in: [conversation/call-policy.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L26) *** -### OtelExportConfig +### BackendCallPolicy -Defined in: [otel-export.ts:12](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L12) +Defined in: [conversation/call-policy.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L29) -OTEL span exporter — streams LoopTraceEvents to an OTLP/HTTP collector. +#### Properties -Reads OTEL_EXPORTER_OTLP_ENDPOINT + OTEL_EXPORTER_OTLP_HEADERS from env -when no explicit config is given. Keeps the runtime dep-free from -@opentelemetry/sdk-trace-base — minimal OTLP/JSON serializer. +##### perAttemptDeadlineMs? -The exporter accepts both raw OtelSpan objects and LoopTraceEvents -(which get converted to OTLP spans automatically). +> `optional` **perAttemptDeadlineMs?**: `number` -#### Properties +Defined in: [conversation/call-policy.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L31) -##### endpoint? +Per-attempt wall clock limit. Exceeding fires an AbortSignal and is treated as a retryable failure. -> `optional` **endpoint?**: `string` +##### maxRetries? -Defined in: [otel-export.ts:14](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L14) +> `optional` **maxRetries?**: `number` -OTLP endpoint. Reads OTEL_EXPORTER_OTLP_ENDPOINT env by default. +Defined in: [conversation/call-policy.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L33) -##### headers? - -> `optional` **headers?**: `Record`\<`string`, `string`\> +Number of retries after the first attempt; total attempts = 1 + maxRetries. Default 0. -Defined in: [otel-export.ts:16](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L16) +##### retryBackoffMs? -OTLP headers. Reads OTEL_EXPORTER_OTLP_HEADERS env by default. +> `optional` **retryBackoffMs?**: [`RetryBackoff`](#retrybackoff) -##### batchSize? +Defined in: [conversation/call-policy.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L35) -> `optional` **batchSize?**: `number` +Backoff between attempts. Default 250ms with jitter. -Defined in: [otel-export.ts:18](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L18) +##### isRetryable? -Batch size before flush. Default 64. +> `optional` **isRetryable?**: [`RetryableErrorPredicate`](#retryableerrorpredicate) -##### flushIntervalMs? +Defined in: [conversation/call-policy.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L37) -> `optional` **flushIntervalMs?**: `number` +Custom retry classifier. Defaults to [defaultIsRetryable](#defaultisretryable). -Defined in: [otel-export.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L20) +##### circuitBreaker? -Flush interval ms. Default 5000. +> `optional` **circuitBreaker?**: [`CircuitBreakerConfig`](#circuitbreakerconfig) -##### resourceAttributes? +Defined in: [conversation/call-policy.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L39) -> `optional` **resourceAttributes?**: `Record`\<`string`, `string` \| `number` \| `boolean`\> +Circuit breaker that opens after N consecutive failures per participant. -Defined in: [otel-export.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L22) +*** -Resource attributes stamped on every export. +### SqlAdapter -##### serviceName? +Defined in: [conversation/journal-sql.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L49) -> `optional` **serviceName?**: `string` +Minimal SQL driver shape. Implementations forward to whichever client the +deployment already uses; agent-runtime takes no opinion on which. -Defined in: [otel-export.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L24) +Parameter placeholders MUST be `?` (positional). All adapters listed in the +file header accept this convention. -Service name. Default 'agent-runtime'. +#### Methods -*** +##### exec() -### OtelExporter +> **exec**(`sql`, `params?`): `Promise`\<\{ `rowsAffected`: `number`; \}\> -Defined in: [otel-export.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L27) +Defined in: [conversation/journal-sql.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L51) -#### Methods +Execute a write statement (INSERT/UPDATE/DELETE/DDL). -##### exportSpan() +###### Parameters -> **exportSpan**(`span`): `void` +###### sql -Defined in: [otel-export.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L29) +`string` -Export a span. +###### params? -###### Parameters +readonly `unknown`[] -###### span +###### Returns -[`OtelSpan`](#otelspan) +`Promise`\<\{ `rowsAffected`: `number`; \}\> -###### Returns +##### query() -`void` +> **query**\<`TRow`\>(`sql`, `params?`): `Promise`\<`TRow`[]\> -##### flush() +Defined in: [conversation/journal-sql.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L53) -> **flush**(): `Promise`\<`void`\> +Execute a read statement (SELECT). Returns rows as plain objects. -Defined in: [otel-export.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L31) +###### Type Parameters -Force flush pending spans. +###### TRow -###### Returns +`TRow` = `Record`\<`string`, `unknown`\> -`Promise`\<`void`\> +###### Parameters -##### shutdown() +###### sql -> **shutdown**(): `Promise`\<`void`\> +`string` -Defined in: [otel-export.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L33) +###### params? -Shutdown cleanly. +readonly `unknown`[] ###### Returns -`Promise`\<`void`\> +`Promise`\<`TRow`[]\> *** -### OtelSpan +### D1DatabaseLike -Defined in: [otel-export.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L36) +Defined in: [conversation/journal-sql.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L84) -#### Properties +Structural type matching the surface of `D1Database` we depend on, so the +SDK never imports `@cloudflare/workers-types`. Consumers pass their real +`D1Database` from `env.DB` and TS structural compatibility lines it up. -##### traceId +#### Methods -> **traceId**: `string` +##### prepare() -Defined in: [otel-export.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L37) +> **prepare**(`sql`): [`D1StmtLike`](#d1stmtlike) -##### spanId +Defined in: [conversation/journal-sql.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L85) -> **spanId**: `string` +###### Parameters -Defined in: [otel-export.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L38) +###### sql -##### parentSpanId? +`string` -> `optional` **parentSpanId?**: `string` +###### Returns -Defined in: [otel-export.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L39) +[`D1StmtLike`](#d1stmtlike) -##### name +*** -> **name**: `string` +### D1StmtLike -Defined in: [otel-export.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L40) +Defined in: [conversation/journal-sql.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L87) -##### kind? +#### Methods -> `optional` **kind?**: `number` +##### bind() -Defined in: [otel-export.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L41) +> **bind**(...`params`): [`D1StmtLike`](#d1stmtlike) -##### startTimeUnixNano +Defined in: [conversation/journal-sql.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L88) -> **startTimeUnixNano**: `string` +###### Parameters -Defined in: [otel-export.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L42) +###### params -##### endTimeUnixNano +...`unknown`[] -> **endTimeUnixNano**: `string` +###### Returns -Defined in: [otel-export.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L43) +[`D1StmtLike`](#d1stmtlike) -##### attributes? +##### run() -> `optional` **attributes?**: [`OtelAttribute`](#otelattribute)[] +> **run**(): `Promise`\<`unknown`\> -Defined in: [otel-export.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L44) +Defined in: [conversation/journal-sql.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L89) -##### status? +###### Returns -> `optional` **status?**: `object` +`Promise`\<`unknown`\> -Defined in: [otel-export.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L45) +##### all() -###### code +> **all**\<`TRow`\>(): `Promise`\<\{ `results?`: `TRow`[]; \}\> -> **code**: `number` +Defined in: [conversation/journal-sql.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal-sql.ts#L90) -###### message? +###### Type Parameters -> `optional` **message?**: `string` +###### TRow + +`TRow` = `unknown` + +###### Returns + +`Promise`\<\{ `results?`: `TRow`[]; \}\> *** -### OtelAttribute +### ConversationJournalEntry -Defined in: [otel-export.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L48) +Defined in: [conversation/journal.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L20) #### Properties -##### key +##### runId -> **key**: `string` +> **runId**: `string` -Defined in: [otel-export.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L49) +Defined in: [conversation/journal.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L21) -##### value +##### startedAt -> **value**: `object` +> **startedAt**: `string` -Defined in: [otel-export.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L50) +Defined in: [conversation/journal.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L22) -###### stringValue? +##### halted? -> `optional` **stringValue?**: `string` +> `optional` **halted?**: [`HaltReason`](#haltreason) -###### intValue? +Defined in: [conversation/journal.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L24) -> `optional` **intValue?**: `string` +Set when the run reaches a terminal state. -###### doubleValue? +##### endedAt? -> `optional` **doubleValue?**: `number` +> `optional` **endedAt?**: `string` -###### boolValue? +Defined in: [conversation/journal.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L25) -> `optional` **boolValue?**: `boolean` +##### turns -*** +> **turns**: [`ConversationTurn`](#conversationturn)[] -### LoopSpanNode +Defined in: [conversation/journal.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L26) -Defined in: [otel-export.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L231) +*** -Sink-neutral node in a reconstructed loop span tree. The root node's -`parentSpanId` is `undefined` — sinks decide how to parent it (the OTEL -mapper attaches the inherited delegation span; the delegation journal -leaves it as the tree root). +### ConversationJournal -#### Properties +Defined in: [conversation/journal.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L29) -##### spanId +#### Methods -> **spanId**: `string` +##### loadRun() -Defined in: [otel-export.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L232) +> **loadRun**(`runId`): `Promise`\<[`ConversationJournalEntry`](#conversationjournalentry) \| `undefined`\> -##### parentSpanId? +Defined in: [conversation/journal.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L36) -> `optional` **parentSpanId?**: `string` +Load any prior state for `runId`. Returns `undefined` for a fresh run. +Implementations MUST NOT mutate the returned object — the runner clones +before continuing — but the runtime treats absence and emptiness +identically, so a journal with zero turns is equivalent to "fresh." -Defined in: [otel-export.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L233) +###### Parameters -##### name +###### runId -> **name**: `string` +`string` -Defined in: [otel-export.ts:235](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L235) +###### Returns -`'loop'` | `'loop.round'` | `'loop.iteration'`. +`Promise`\<[`ConversationJournalEntry`](#conversationjournalentry) \| `undefined`\> -##### kind +##### beginRun() -> **kind**: `"loop"` \| `"round"` \| `"branch"` +> **beginRun**(`runId`, `startedAt`): `Promise`\<`void`\> -Defined in: [otel-export.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L237) +Defined in: [conversation/journal.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L43) -Topology level: loop root, plan round, or iteration branch. +Initialise journal state for a fresh run. Called once per run, before any +`appendTurn`. Idempotent: calling with an existing runId is a no-op if +the entry already exists with the same `startedAt`. -##### startMs +###### Parameters -> **startMs**: `number` +###### runId -Defined in: [otel-export.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L238) +`string` -##### endMs +###### startedAt -> **endMs**: `number` +`string` -Defined in: [otel-export.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L239) +###### Returns -##### attrs +`Promise`\<`void`\> -> **attrs**: `Record`\<`string`, `string` \| `number` \| `boolean`\> +##### appendTurn() -Defined in: [otel-export.ts:240](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L240) +> **appendTurn**(`runId`, `turn`): `Promise`\<`void`\> -##### error +Defined in: [conversation/journal.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L50) -> **error**: `boolean` +Append a committed turn. The runner only calls this AFTER the turn's +backend stream completed and the credit total has been updated, so an +appended turn is observed-committed and never speculative. -Defined in: [otel-export.ts:242](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L242) +###### Parameters -True when the iteration carried an error — maps to OTEL status code 2. +###### runId -*** +`string` -### EvalRunGeneration +###### turn -Defined in: [otel-export.ts:558](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L558) +[`ConversationTurn`](#conversationturn) -#### Properties - -##### index - -> **index**: `number` +###### Returns -Defined in: [otel-export.ts:560](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L560) +`Promise`\<`void`\> -0-based ordinal of this generation within the run (required by ingest). +##### recordHalt() -##### surfaceHash +> **recordHalt**(`runId`, `halt`, `endedAt`): `Promise`\<`void`\> -> **surfaceHash**: `string` +Defined in: [conversation/journal.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/journal.ts#L56) -Defined in: [otel-export.ts:562](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L562) +Record the run's terminal halt reason + end time. Once called, the run +is observed-final; subsequent `loadRun` returns the same halt. -Identity of the proposed surface change (content-addressed hash). +###### Parameters -##### surface? +###### runId -> `optional` **surface?**: `unknown` +`string` -Defined in: [otel-export.ts:564](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L564) +###### halt -Arbitrary provenance for this generation (rationale, evidence, source). +[`HaltReason`](#haltreason) -##### cells? +###### endedAt -> `optional` **cells?**: `unknown`[] +`string` -Defined in: [otel-export.ts:566](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L566) +###### Returns -Per-scenario results; empty until the generation is measured. +`Promise`\<`void`\> -##### compositeMean +*** -> **compositeMean**: `number` +### RunPersonaConversationOptions -Defined in: [otel-export.ts:568](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L568) +Defined in: [conversation/run-persona.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L36) -Mean composite score (0 when unmeasured — pair with labels.measured). +#### Properties -##### costUsd +##### worker -> **costUsd**: `number` +> **worker**: `AgentProfile` -Defined in: [otel-export.ts:569](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L569) +Defined in: [conversation/run-persona.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L38) -##### durationMs +The agent under test. Metered; its rendered prompt leads its turns. -> **durationMs**: `number` +##### persona -Defined in: [otel-export.ts:570](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L570) +> **persona**: [`PersonaDriver`](#personadriver) -*** +Defined in: [conversation/run-persona.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L40) -### EvalRunEvent +The simulated user driving the dialogue. -Defined in: [otel-export.ts:573](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L573) +##### backendFor -#### Properties +> **backendFor**: (`profile`, `role`) => [`AgentExecutionBackend`](#agentexecutionbackend) -##### runId +Defined in: [conversation/run-persona.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L43) -> **runId**: `string` +Turn an `AgentProfile` into a runnable backend (router / sandbox / fake). + Applied to the worker and to a `profile`-kind persona. -Defined in: [otel-export.ts:574](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L574) +###### Parameters -##### runDir +###### profile -> **runDir**: `string` +`AgentProfile` -Defined in: [otel-export.ts:575](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L575) +###### role -##### timestamp +`"worker"` \| `"persona"` -> **timestamp**: `string` +###### Returns -Defined in: [otel-export.ts:577](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L577) +[`AgentExecutionBackend`](#agentexecutionbackend) -ISO timestamp. +##### systemPromptOf -##### status +> **systemPromptOf**: (`profile`) => `string` -> **status**: `"started"` \| `"baseline-complete"` \| `"generation-complete"` \| `"gate-decided"` \| `"finished"` \| `"errored"` +Defined in: [conversation/run-persona.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L45) -Defined in: [otel-export.ts:578](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L578) +Render a profile's system prompt — prepended to that profile's messages. -##### labels? +###### Parameters -> `optional` **labels?**: `Record`\<`string`, `string`\> +###### profile -Defined in: [otel-export.ts:585](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L585) +`AgentProfile` -##### baseline? +###### Returns -> `optional` **baseline?**: [`EvalRunGeneration`](#evalrungeneration) +`string` -Defined in: [otel-export.ts:586](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L586) +##### maxTurns? -##### generations? +> `optional` **maxTurns?**: `number` -> `optional` **generations?**: [`EvalRunGeneration`](#evalrungeneration)[] +Defined in: [conversation/run-persona.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L48) -Defined in: [otel-export.ts:587](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L587) +Speaker-turn cap. Default for a scripted persona = `2 * turns.length` + (worker answers each user turn). REQUIRED for a `profile` persona. -##### gateDecision? +##### seed? -> `optional` **gateDecision?**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` +> `optional` **seed?**: `string` -Defined in: [otel-export.ts:588](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L588) +Defined in: [conversation/run-persona.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L50) -##### holdoutLift? +Kickoff message routed to the first speaker (the persona). Default 'Begin.' -> `optional` **holdoutLift?**: `number` +##### haltOn? -Defined in: [otel-export.ts:589](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L589) +> `optional` **haltOn?**: [`HaltPredicate`](#haltpredicate) -##### totalCostUsd +Defined in: [conversation/run-persona.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L53) -> **totalCostUsd**: `number` +Content-based "until satisfied" halt, called after every turn. `maxTurns` is the + hard ceiling; this is the early stop (the persona declares the goal met / unreachable). -Defined in: [otel-export.ts:590](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L590) +##### signal? -##### totalDurationMs +> `optional` **signal?**: `AbortSignal` -> **totalDurationMs**: `number` +Defined in: [conversation/run-persona.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L54) -Defined in: [otel-export.ts:591](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L591) +##### workerName? -##### errorMessage? +> `optional` **workerName?**: `string` -> `optional` **errorMessage?**: `string` +Defined in: [conversation/run-persona.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L56) -Defined in: [otel-export.ts:592](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L592) +Worker participant / transcript speaker label. Default 'agent'. *** -### EvalRunsExportConfig +### PersonaConversationResult -Defined in: [otel-export.ts:595](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L595) +Defined in: [conversation/run-persona.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L59) #### Properties -##### apiKey? +##### transcript -> `optional` **apiKey?**: `string` +> **transcript**: [`ConversationTurn`](#conversationturn)[] -Defined in: [otel-export.ts:597](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L597) +Defined in: [conversation/run-persona.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L60) -Bearer key — tenant is resolved server-side from it. Reads TANGLE_API_KEY. +##### turns -##### base? +> **turns**: `number` -> `optional` **base?**: `string` +Defined in: [conversation/run-persona.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L61) -Defined in: [otel-export.ts:599](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L599) +##### halted -Intelligence base. Reads INTELLIGENCE_BASE env, else prod. +> **halted**: [`HaltReason`](#haltreason) -##### idempotencyKey? +Defined in: [conversation/run-persona.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L62) -> `optional` **idempotencyKey?**: `string` +##### costUsd -Defined in: [otel-export.ts:601](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L601) +> **costUsd**: `number` -Idempotency-Key header (e.g. the runId) — safe retries + upsert. +Defined in: [conversation/run-persona.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L64) -*** +Worker-only spend (the side under test). -### EvalRunsExportResult +##### tokensIn -Defined in: [otel-export.ts:604](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L604) +> **tokensIn**: `number` -#### Properties +Defined in: [conversation/run-persona.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L65) -##### ok +##### tokensOut -> **ok**: `boolean` +> **tokensOut**: `number` -Defined in: [otel-export.ts:605](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L605) +Defined in: [conversation/run-persona.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L66) -##### status +*** -> **status**: `number` +### RunPersonaConfig -Defined in: [otel-export.ts:606](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L606) +Defined in: [conversation/run-persona.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L198) -##### accepted +#### Type Parameters -> **accepted**: `number` +##### TScenario -Defined in: [otel-export.ts:607](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L607) +`TScenario` *extends* `Scenario` -##### rejected +##### TArtifact -> **rejected**: `object`[] +`TArtifact` -Defined in: [otel-export.ts:608](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L608) +#### Properties -###### index +##### backendFor -> **index**: `number` +> **backendFor**: (`profile`, `role`) => [`AgentExecutionBackend`](#agentexecutionbackend) -###### reason +Defined in: [conversation/run-persona.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L200) -> **reason**: `string` +Turn an `AgentProfile` into a runnable backend (router / sandbox / fake). -*** +###### Parameters -### ResolveAgentBackendOptions +###### profile -Defined in: [resolve-agent-backend.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L51) +`AgentProfile` -#### Extends +###### role -- `OpenAICompatPassthrough` +`"worker"` \| `"persona"` -#### Type Parameters +###### Returns -##### TInput +[`AgentExecutionBackend`](#agentexecutionbackend) -`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) +##### systemPromptOf -#### Properties +> **systemPromptOf**: (`profile`) => `string` -##### tools? +Defined in: [conversation/run-persona.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L202) -> `optional` **tools?**: readonly [`OpenAIChatTool`](#openaichattool)[] +Render a profile's system prompt. -Defined in: [backends.ts:222](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L222) +###### Parameters -OpenAI Chat Completions `tools[]` definitions surfaced to the model on -every request. Omit to send a tool-free request (existing behavior). -The runtime makes no assumption about the dispatcher — calls stream out -as `tool_call` events and the caller is responsible for executing them -and feeding `tool_result` messages back on a follow-up turn. +###### profile -###### Inherited from +`AgentProfile` -`OpenAICompatPassthrough.tools` +###### Returns -##### toolChoice? +`string` -> `optional` **toolChoice?**: [`OpenAIChatToolChoice`](#openaichattoolchoice) +##### personaOf -Defined in: [backends.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L228) +> **personaOf**: (`scenario`) => [`PersonaDriver`](#personadriver) -OpenAI Chat Completions `tool_choice`. Default `undefined` (request -omits the field; provider falls back to its own default — typically -`'auto'`). +Defined in: [conversation/run-persona.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L204) -###### Inherited from +The persona driving each scenario — a driver profile or scripted turns. -`OpenAICompatPassthrough.toolChoice` +###### Parameters -##### responseFormat? +###### scenario -> `optional` **responseFormat?**: [`OpenAIChatResponseFormat`](#openaichatresponseformat) +`TScenario` -Defined in: [backends.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L232) +###### Returns -OpenAI Chat Completions `response_format`. Omit for provider default text. +[`PersonaDriver`](#personadriver) -###### Inherited from +##### artifactOf -`OpenAICompatPassthrough.responseFormat` +> **artifactOf**: (`transcript`, `scenario`) => `TArtifact` -##### temperature? +Defined in: [conversation/run-persona.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L206) -> `optional` **temperature?**: `number` +Build the scored artifact from the finished transcript. -Defined in: [backends.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L234) +###### Parameters -OpenAI Chat Completions `temperature`. Omit for provider default. +###### transcript -###### Inherited from +[`ConversationTurn`](#conversationturn)[] -`OpenAICompatPassthrough.temperature` +###### scenario -##### maxTokens? +`TScenario` -> `optional` **maxTokens?**: `number` +###### Returns -Defined in: [backends.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L236) +`TArtifact` -Maximum completion tokens, sent as OpenAI-compatible `max_tokens`. Omit for provider default. +##### maxTurns? -###### Inherited from +> `optional` **maxTurns?**: (`scenario`) => `number` -`OpenAICompatPassthrough.maxTokens` +Defined in: [conversation/run-persona.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L208) -##### fetchImpl? +Speaker-turn cap (required when a persona is profile-driven). -> `optional` **fetchImpl?**: (`input`, `init?`) => `Promise`\<`Response`\> +###### Parameters -Defined in: [backends.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L237) +###### scenario -###### Parameters +`TScenario` -###### input +###### Returns -`string` \| `URL` \| `Request` +`number` -###### init? +##### seed? -`RequestInit` +> `optional` **seed?**: (`scenario`) => `string` -###### Returns +Defined in: [conversation/run-persona.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L209) -`Promise`\<`Response`\> +###### Parameters -###### Inherited from +###### scenario -`OpenAICompatPassthrough.fetchImpl` +`TScenario` -##### retry? +###### Returns -> `optional` **retry?**: `BackendRetryPolicy` +`string` -Defined in: [backends.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L238) +##### workerName? -###### Inherited from +> `optional` **workerName?**: `string` -`OpenAICompatPassthrough.retry` +Defined in: [conversation/run-persona.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L210) -##### kind +##### maximumCharge? -> **kind**: [`AgentBackendKind`](#agentbackendkind) +> `optional` **maximumCharge?**: `MaximumCharge` \| ((`worker`, `scenario`) => MaximumCharge \| undefined) -Defined in: [resolve-agent-backend.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L54) +Defined in: [conversation/run-persona.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L213) -The chat transport to resolve. +Provider- or executor-enforced maximum for the whole worker conversation. +Required before execution when the enclosing campaign is cost-capped. -##### apiKey +*** -> **apiKey**: `string` +### ConversationParticipant -Defined in: [resolve-agent-backend.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L60) +Defined in: [conversation/types.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L21) -Bearer credential for the OpenAI-compat kinds. Empty string is valid for a -loopback-anonymous cli-bridge; a `router`/`tcloud` route with an empty key -is a caller bug the product surfaces before calling in. +#### Stable -##### baseUrl +#### Properties -> **baseUrl**: `string` +##### name -Defined in: [resolve-agent-backend.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L62) +> **name**: `string` -Base URL for the OpenAI-compat kinds. cli-bridge's is its `/v1`. +Defined in: [conversation/types.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L26) -##### model +Stable name used as the speaker label in the transcript. Must be unique +within a `Conversation`. -> **model**: `string` +##### backend -Defined in: [resolve-agent-backend.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L64) +> **backend**: [`AgentExecutionBackend`](#agentexecutionbackend) -Model id sent on every request. cli-bridge rejects a request without it. +Defined in: [conversation/types.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L33) + +Backend that runs this participant's turn. Reuses the existing +`AgentExecutionBackend` contract from `runAgentTaskStream`, so any +registered backend (iterable, sandbox, OpenAI-compatible) works without +adaptation. ##### label? > `optional` **label?**: `string` -Defined in: [resolve-agent-backend.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L66) +Defined in: [conversation/types.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L38) -`kind` label stamped on the resolved backend + its traces. Defaults to `kind`. +Optional human label for traces / dashboards. Distinct from `name`, which +is the addressing key. -##### sandboxBackend? +##### callPolicy? -> `optional` **sandboxBackend?**: () => [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> +> `optional` **callPolicy?**: [`BackendCallPolicy`](#backendcallpolicy) -Defined in: [resolve-agent-backend.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L72) +Defined in: [conversation/types.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L44) -`sandbox` kind: the product's own domain backend. Required for that kind — -the substrate owns no product sandbox shape, so a `sandbox` resolution with -no seam is a caller bug, not a silent fallback. +Optional per-participant override of the conversation's default +`callPolicy`. Use to tighten the deadline or raise the retry budget for +a participant known to be slow or flaky. -###### Returns +##### authSource? -[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> +> `optional` **authSource?**: [`AuthSource`](#authsource-1) -*** +Defined in: [conversation/types.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L64) -### RuntimeHookEvent +Who pays for THIS participant's outbound calls? -Defined in: [runtime-hooks.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L36) +- `'forward-user'` (default) — propagate the caller's + `X-Tangle-Forwarded-Authorization` so the downstream gateway bills the + original user. Right for pass-through agents that aggregate/route + without taking economic risk. +- `'agent-owned'` — DO NOT forward the user's auth; the participant's + backend uses its own credentials (typically a sk-tan-AGENT or x402 + wallet baked into the backend at construction). Downstream charges + land on the agent, not the user. Right for resold-bundle agents that + take margin between their inbound price and their sub-agent costs. +- `(state) => AuthSource` — per-turn / per-condition decision, e.g. base + sub-services are agent-owned but premium add-ons forward the user. -#### Type Parameters +The agent's own credentials live on the backend (set at construction +time, e.g. `createOpenAICompatibleBackend({ apiKey })`); this field is +purely about *whether to also forward the user's identity downstream*. -##### Payload +*** -`Payload` = `unknown` +### ConversationDriveState -#### Properties +Defined in: [conversation/types.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L77) -##### id +#### Stable -> **id**: `string` +#### Extended by -Defined in: [runtime-hooks.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L37) +- [`HaltContext`](#haltcontext) -##### runId +#### Properties -> **runId**: `string` +##### transcript -Defined in: [runtime-hooks.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L38) +> **transcript**: readonly [`ConversationTurn`](#conversationturn)[] -##### scenarioId? +Defined in: [conversation/types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L78) -> `optional` **scenarioId?**: `string` +##### turnIndex -Defined in: [runtime-hooks.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L39) +> **turnIndex**: `number` -##### target +Defined in: [conversation/types.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L79) -> **target**: [`RuntimeHookTarget`](#runtimehooktarget) +##### spentCreditsCents -Defined in: [runtime-hooks.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L40) +> **spentCreditsCents**: `number` -##### phase +Defined in: [conversation/types.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L80) -> **phase**: [`RuntimeHookPhase`](#runtimehookphase) +*** -Defined in: [runtime-hooks.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L41) +### HaltContext -##### timestamp +Defined in: [conversation/types.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L84) -> **timestamp**: `number` +#### Stable -Defined in: [runtime-hooks.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L42) +#### Extends -##### stepIndex? +- [`ConversationDriveState`](#conversationdrivestate) -> `optional` **stepIndex?**: `number` +#### Properties -Defined in: [runtime-hooks.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L43) +##### transcript -##### parentId? +> **transcript**: readonly [`ConversationTurn`](#conversationturn)[] -> `optional` **parentId?**: `string` +Defined in: [conversation/types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L78) -Defined in: [runtime-hooks.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L44) +###### Inherited from -##### payload? +[`ConversationDriveState`](#conversationdrivestate).[`transcript`](#transcript-1) -> `optional` **payload?**: `Payload` +##### turnIndex -Defined in: [runtime-hooks.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L45) +> **turnIndex**: `number` -##### metadata? +Defined in: [conversation/types.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L79) -> `optional` **metadata?**: `Record`\<`string`, `unknown`\> +###### Inherited from -Defined in: [runtime-hooks.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L46) +[`ConversationDriveState`](#conversationdrivestate).[`turnIndex`](#turnindex) -*** +##### spentCreditsCents -### RuntimeHookContext +> **spentCreditsCents**: `number` -Defined in: [runtime-hooks.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L49) +Defined in: [conversation/types.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L80) -#### Properties +###### Inherited from -##### signal? +[`ConversationDriveState`](#conversationdrivestate).[`spentCreditsCents`](#spentcreditscents) -> `optional` **signal?**: `AbortSignal` +##### lastTurn -Defined in: [runtime-hooks.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L50) +> **lastTurn**: [`ConversationTurn`](#conversationturn) + +Defined in: [conversation/types.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L85) *** -### RuntimeDecisionEvidenceRef +### HaltSignal -Defined in: [runtime-hooks.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L53) +Defined in: [conversation/types.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L89) + +#### Stable #### Properties -##### source +##### halted -> **source**: `string` +> **halted**: `true` -Defined in: [runtime-hooks.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L54) +Defined in: [conversation/types.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L90) -##### id +##### reason -> **id**: `string` +> **reason**: `string` -Defined in: [runtime-hooks.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L55) +Defined in: [conversation/types.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L91) -##### detail? +*** -> `optional` **detail?**: `string` +### ConversationPolicy -Defined in: [runtime-hooks.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L56) +Defined in: [conversation/types.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L108) -##### metadata? +#### Stable -> `optional` **metadata?**: `Record`\<`string`, `unknown`\> +#### Properties -Defined in: [runtime-hooks.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L57) +##### maxTurns -*** +> **maxTurns**: `number` -### RuntimeDecisionPoint +Defined in: [conversation/types.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L110) -Defined in: [runtime-hooks.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L60) +Hard cap on speaker-turns. Each call into a participant's backend counts as 1. -#### Properties +##### maxCreditsCents? -##### id +> `optional` **maxCreditsCents?**: `number` -> **id**: `string` +Defined in: [conversation/types.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L117) -Defined in: [runtime-hooks.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L61) +Hard cap on aggregate credit spend across all participants, in cents. +Computed by summing `llm_call.costUsd` from every participant's stream. +Unset (`undefined`) means no credit ceiling — the run is bounded only by +`maxTurns` and `haltOn`. -##### runId +##### turnOrder? -> **runId**: `string` +> `optional` **turnOrder?**: [`TurnOrder`](#turnorder) -Defined in: [runtime-hooks.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L62) +Defined in: [conversation/types.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L122) -##### scenarioId? +Speaker selection. Defaults to `'alternate'` for two-participant +conversations and `'round-robin'` for any other arity. -> `optional` **scenarioId?**: `string` +##### haltOn? -Defined in: [runtime-hooks.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L63) +> `optional` **haltOn?**: [`HaltPredicate`](#haltpredicate) -##### stepIndex +Defined in: [conversation/types.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L127) -> **stepIndex**: `number` +Optional convergence / content-based halt. Called after every turn ends; +returning truthy stops the loop with `{ kind: 'predicate', ... }`. -Defined in: [runtime-hooks.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L64) +##### defaultCallPolicy? -##### kind +> `optional` **defaultCallPolicy?**: [`BackendCallPolicy`](#backendcallpolicy) -> **kind**: [`RuntimeDecisionKind`](#runtimedecisionkind) +Defined in: [conversation/types.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L133) -Defined in: [runtime-hooks.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L65) +Default per-turn resilience policy applied to every participant call +(deadline, retries, circuit breaker). Individual participants may +override via `ConversationParticipant.callPolicy`. -##### candidateActions +*** -> **candidateActions**: `string`[] +### ConversationTurn -Defined in: [runtime-hooks.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L66) +Defined in: [conversation/types.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L137) -##### context? +#### Stable -> `optional` **context?**: `string` +#### Properties -Defined in: [runtime-hooks.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L67) +##### index -##### evidence +> **index**: `number` -> **evidence**: [`RuntimeDecisionEvidenceRef`](#runtimedecisionevidenceref)[] +Defined in: [conversation/types.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L138) -Defined in: [runtime-hooks.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L68) +##### speaker -##### metadata? +> **speaker**: `string` -> `optional` **metadata?**: `Record`\<`string`, `unknown`\> +Defined in: [conversation/types.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L139) -Defined in: [runtime-hooks.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L69) +##### turnId -*** +> **turnId**: `string` -### RuntimeHookErrorContext +Defined in: [conversation/types.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L145) -Defined in: [runtime-hooks.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L72) +Deterministic turn identifier — stable across retries of the same logical +turn so caching gateways and trace backends can dedupe. Shape: +`${runId}.t${index}.${speakerSlug}`. -#### Properties +##### text -##### hook +> **text**: `string` -> **hook**: `"onEvent"` \| `"onDecisionPoint"` +Defined in: [conversation/types.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L146) -Defined in: [runtime-hooks.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L73) +##### usage? -##### eventId? +> `optional` **usage?**: `object` -> `optional` **eventId?**: `string` +Defined in: [conversation/types.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L152) -Defined in: [runtime-hooks.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L74) +Aggregated backend usage for this turn alone. Populated from any +`llm_call` stream events the backend emitted; `undefined` when the +backend reports no usage. -##### target? +###### tokensIn? -> `optional` **target?**: [`RuntimeHookTarget`](#runtimehooktarget) +> `optional` **tokensIn?**: `number` -Defined in: [runtime-hooks.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L75) +###### tokensOut? -##### phase? +> `optional` **tokensOut?**: `number` -> `optional` **phase?**: [`RuntimeHookPhase`](#runtimehookphase) +###### costUsd? -Defined in: [runtime-hooks.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L76) +> `optional` **costUsd?**: `number` -##### decisionId? +###### latencyMs? -> `optional` **decisionId?**: `string` +> `optional` **latencyMs?**: `number` -Defined in: [runtime-hooks.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L77) +###### model? -##### decisionKind? +> `optional` **model?**: `string` -> `optional` **decisionKind?**: [`RuntimeDecisionKind`](#runtimedecisionkind) +##### attempts -Defined in: [runtime-hooks.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L78) +> **attempts**: `number` + +Defined in: [conversation/types.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L164) + +Number of attempts that ran before this turn committed. `1` is the +common case; higher means the call policy retried after transient +failures. + +##### startedAt + +> **startedAt**: `string` + +Defined in: [conversation/types.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L165) + +##### endedAt + +> **endedAt**: `string` + +Defined in: [conversation/types.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L166) *** -### RuntimeHooks +### Conversation -Defined in: [runtime-hooks.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L88) +Defined in: [conversation/types.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L170) -The observation seam attached to a running loop (never to the portable genome). -Implement the optional hooks to receive lifecycle events, semantic decision points, -and hook errors. Author with [defineRuntimeHooks](#defineruntimehooks) for inference, and attach N -observers at once with [composeRuntimeHooks](#composeruntimehooks) — there is ONE event stream, not a -callback-prop zoo. +#### Stable + +#### Properties + +##### participants + +> **participants**: readonly [`ConversationParticipant`](#conversationparticipant)[] + +Defined in: [conversation/types.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L171) + +##### policy + +> **policy**: [`ConversationPolicy`](#conversationpolicy) + +Defined in: [conversation/types.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L172) + +*** + +### RunConversationOptions + +Defined in: [conversation/types.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L176) + +#### Stable #### Properties +##### seed + +> **seed**: `string` + +Defined in: [conversation/types.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L178) + +First message kicking off the conversation. Routes to the first speaker. + +##### runId? + +> `optional` **runId?**: `string` + +Defined in: [conversation/types.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L185) + +Optional run identifier for cross-participant trace correlation. Auto- +generated when omitted. Reusing a runId against the same `journal` +resumes the prior run — the runner replays the persisted transcript and +continues from the first un-recorded turn. + +##### signal? + +> `optional` **signal?**: `AbortSignal` + +Defined in: [conversation/types.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L187) + +Cancellation signal — aborts mid-stream and halts with `{ kind: 'abort' }`. + ##### onEvent? -> `optional` **onEvent?**: (`event`, `context`) => `void` \| `Promise`\<`void`\> +> `optional` **onEvent?**: (`event`) => `void` \| `Promise`\<`void`\> -Defined in: [runtime-hooks.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L94) +Defined in: [conversation/types.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L194) -General before/after/event hook. Use this for telemetry, memory capture, -policy wrapping, child lifecycle observers, or product-specific extension -points. +Event sink for per-turn micro-events. Distinct from the result transcript: +the sink fires for every text-delta, every turn-start/end, and the +conversation-start/end markers. Used to drive SSE / dashboard updates +without waiting for the conversation to finish. ###### Parameters ###### event -[`RuntimeHookEvent`](#runtimehookevent) +[`ConversationStreamEvent`](#conversationstreamevent) -###### context +###### Returns -[`RuntimeHookContext`](#runtimehookcontext) +`void` \| `Promise`\<`void`\> + +##### journal? + +> `optional` **journal?**: [`ConversationJournal`](#conversationjournal) + +Defined in: [conversation/types.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L201) + +Optional durable transcript. When set, the runner persists every +committed turn before yielding `turn_end`. Reusing the same `runId` +against the same journal resumes from the last committed turn — so a +driver process crash mid-run loses zero acknowledged turns. + +##### propagatedHeaders? + +> `optional` **propagatedHeaders?**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [conversation/types.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L208) + +Headers to forward verbatim to every participant backend call (gateway +propagation: `X-Tangle-Forwarded-Authorization`, run/turn correlation, +depth counter). Backends opt in by reading `propagatedHeaders` from +their `AgentBackendContext`; backends that ignore the field still work. + +##### inboundDepth? + +> `optional` **inboundDepth?**: `number` + +Defined in: [conversation/types.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L214) + +Inbound depth at the point this driver was invoked. The runner +increments it on every outbound participant call; gateways refuse at +`DEFAULT_MAX_DEPTH`. Default 0 (origin caller). + +##### parentTurnId? + +> `optional` **parentTurnId?**: `string` + +Defined in: [conversation/types.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L221) + +Parent turn id when this conversation is *inside* another turn (i.e. the +driver is itself a participant via `createConversationBackend`). The +runner stamps each outbound call with this as `X-Tangle-Parent-TurnId` +so trace stitching survives nested orchestration. + +*** + +### ConversationResult + +Defined in: [conversation/types.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L225) + +#### Stable + +#### Properties + +##### runId + +> **runId**: `string` + +Defined in: [conversation/types.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L226) + +##### transcript + +> **transcript**: [`ConversationTurn`](#conversationturn)[] + +Defined in: [conversation/types.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L227) + +##### turns + +> **turns**: `number` + +Defined in: [conversation/types.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L228) + +##### spentCreditsCents + +> **spentCreditsCents**: `number` + +Defined in: [conversation/types.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L229) + +##### halted + +> **halted**: [`HaltReason`](#haltreason) + +Defined in: [conversation/types.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L230) + +##### durationMs + +> **durationMs**: `number` + +Defined in: [conversation/types.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L231) + +##### startedAt + +> **startedAt**: `string` + +Defined in: [conversation/types.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L232) + +##### endedAt + +> **endedAt**: `string` + +Defined in: [conversation/types.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L233) + +*** + +### ChatStreamEvent + +Defined in: [durable/chat-engine.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L28) + +The NDJSON line protocol every product chat client already speaks. + +#### Properties + +##### type + +> **type**: `string` + +Defined in: [durable/chat-engine.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L29) + +##### data? + +> `optional` **data?**: `Record`\<`string`, `unknown`\> + +Defined in: [durable/chat-engine.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L30) + +*** + +### ChatTurnIdentity + +Defined in: [durable/chat-engine.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L35) + +Identity of a chat turn. `tenantId` is the workspace id for workspace- + scoped products and the user id for session-scoped products. + +#### Properties + +##### tenantId + +> **tenantId**: `string` + +Defined in: [durable/chat-engine.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L36) + +##### sessionId + +> **sessionId**: `string` + +Defined in: [durable/chat-engine.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L38) + +Thread / session id. + +##### userId + +> **userId**: `string` + +Defined in: [durable/chat-engine.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L39) + +##### turnIndex + +> **turnIndex**: `number` + +Defined in: [durable/chat-engine.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L41) + +Monotonic 0-based turn index within the session. + +*** + +### ChatTurnProducer + +Defined in: [durable/chat-engine.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L45) + +The live side of a turn — what the product's `produce` hook returns. + +#### Type Parameters + +##### TEvent + +`TEvent` *extends* [`ChatStreamEvent`](#chatstreamevent) = [`ChatStreamEvent`](#chatstreamevent) + +#### Properties + +##### stream + +> **stream**: `AsyncGenerator`\<`TEvent`, `void`, `unknown`\> + +Defined in: [durable/chat-engine.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L47) + +The turn's event stream. Forwarded verbatim to the caller. + +#### Methods + +##### finalText() + +> **finalText**(): `string` + +Defined in: [durable/chat-engine.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L49) + +The turn's final assistant text. Read once, after `stream` drains. + +###### Returns + +`string` + +*** + +### ChatTurnHooks + +Defined in: [durable/chat-engine.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L52) + +Turn-lifecycle helpers for `@tangle-network/agent-runtime`. + +Execution state — long-running execution, reconnect, replay, dedup — +lives in the substrate (`@tangle-network/sandbox` + orchestrator). +agent-runtime owns: + + - `handleChatTurn` — framework-neutral turn lifecycle: NDJSON framing, + `session.run.*` envelope, persist / post-process / trace-flush + hook ordering. + - `deriveExecutionId` — convention helper for the stable id products + persist so a retry of the same turn lands on the same execution. + +#### Methods + +##### produce() + +> **produce**(): [`ChatTurnProducer`](#chatturnproducer) + +Defined in: [durable/chat-engine.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L55) + +Build the backend stream. The engine forwards events verbatim and + reads `finalText()` once the stream drains. + +###### Returns + +[`ChatTurnProducer`](#chatturnproducer) + +##### persistAssistantMessage() + +> **persistAssistantMessage**(`input`): `Promise`\<`void`\> + +Defined in: [durable/chat-engine.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L58) + +Persist the assistant message to the product's own store. Called + once, after drain, with the assembled (transform-applied) text. + +###### Parameters + +###### input + +###### identity + +[`ChatTurnIdentity`](#chatturnidentity) + +###### finalText + +`string` + +###### Returns + +`Promise`\<`void`\> + +##### onTurnComplete()? + +> `optional` **onTurnComplete**(`input`): `Promise`\<`void`\> + +Defined in: [durable/chat-engine.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L62) + +Optional post-processing (proposals, citations, credit metering …). + Errors are swallowed + logged — post-process must never fail a turn + that already streamed successfully. + +###### Parameters + +###### input + +###### identity + +[`ChatTurnIdentity`](#chatturnidentity) + +###### finalText + +`string` + +###### Returns + +`Promise`\<`void`\> + +##### onEvent()? + +> `optional` **onEvent**(`event`): `void` \| `Promise`\<`void`\> + +Defined in: [durable/chat-engine.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L66) + +Optional per-event side channel (e.g. DO broadcast). Runs for every + emitted event, lifecycle envelope included. Errors swallowed — a + broadcast failure must not break the chat stream. + +###### Parameters + +###### event + +[`ChatStreamEvent`](#chatstreamevent) + +###### Returns + +`void` \| `Promise`\<`void`\> + +##### transformFinalText()? + +> `optional` **transformFinalText**(`text`): `string` \| `Promise`\<`string`\> + +Defined in: [durable/chat-engine.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L70) + +Optional pre-persist transform of the final text (e.g. PII + redaction). Affects only what is persisted; the live stream is + never altered. + +###### Parameters + +###### text + +`string` + +###### Returns + +`string` \| `Promise`\<`string`\> + +##### traceFlush()? + +> `optional` **traceFlush**(): `Promise`\<`void`\> + +Defined in: [durable/chat-engine.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L73) + +Optional trace flush — resolves when OTLP export completes. Handed + to `waitUntil` so the worker isolate stays alive for the POST. + +###### Returns + +`Promise`\<`void`\> + +*** + +### RunChatTurnInput + +Defined in: [durable/chat-engine.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L76) + +Turn-lifecycle helpers for `@tangle-network/agent-runtime`. + +Execution state — long-running execution, reconnect, replay, dedup — +lives in the substrate (`@tangle-network/sandbox` + orchestrator). +agent-runtime owns: + + - `handleChatTurn` — framework-neutral turn lifecycle: NDJSON framing, + `session.run.*` envelope, persist / post-process / trace-flush + hook ordering. + - `deriveExecutionId` — convention helper for the stable id products + persist so a retry of the same turn lands on the same execution. + +#### Properties + +##### identity + +> **identity**: [`ChatTurnIdentity`](#chatturnidentity) + +Defined in: [durable/chat-engine.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L77) + +##### hooks + +> **hooks**: [`ChatTurnHooks`](#chatturnhooks) + +Defined in: [durable/chat-engine.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L78) + +##### waitUntil? + +> `optional` **waitUntil?**: (`p`) => `void` + +Defined in: [durable/chat-engine.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L81) + +Worker liveness hook. When omitted, trace flush is awaited inline + before the stream closes. + +###### Parameters + +###### p + +`Promise`\<`unknown`\> + +###### Returns + +`void` + +##### log? + +> `optional` **log?**: (`message`, `meta?`) => `void` + +Defined in: [durable/chat-engine.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L84) + +Structured logger for swallowed hook errors. Defaults to + `console.error` so failures surface without product wiring. + +###### Parameters + +###### message + +`string` + +###### meta? + +`Record`\<`string`, `unknown`\> + +###### Returns + +`void` + +*** + +### ChatTurnResult + +Defined in: [durable/chat-engine.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L87) + +Turn-lifecycle helpers for `@tangle-network/agent-runtime`. + +Execution state — long-running execution, reconnect, replay, dedup — +lives in the substrate (`@tangle-network/sandbox` + orchestrator). +agent-runtime owns: + + - `handleChatTurn` — framework-neutral turn lifecycle: NDJSON framing, + `session.run.*` envelope, persist / post-process / trace-flush + hook ordering. + - `deriveExecutionId` — convention helper for the stable id products + persist so a retry of the same turn lands on the same execution. + +#### Properties + +##### body + +> **body**: `ReadableStream`\<`Uint8Array`\<`ArrayBufferLike`\>\> + +Defined in: [durable/chat-engine.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L89) + +NDJSON body — return this as the platform `Response` body. + +##### contentType + +> **contentType**: `"application/x-ndjson"` + +Defined in: [durable/chat-engine.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/durable/chat-engine.ts#L91) + +Content type for the response. + +*** + +### VerifyResult + +Defined in: [improvement/agentic-generator.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L73) + +Outcome of verifying a candidate worktree. `feedback` (compiler errors, + failing test output) is fed into the next shot when `ok` is false. + +#### Properties + +##### ok + +> **ok**: `boolean` + +Defined in: [improvement/agentic-generator.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L74) + +##### feedback? + +> `optional` **feedback?**: `string` + +Defined in: [improvement/agentic-generator.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L75) + +*** + +### AgenticGeneratorShotReceipt + +Defined in: [improvement/agentic-generator.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L83) + +`@tangle-network/agent-runtime` improvement — the CODE-surface proposer for +agent-eval's improvement loop. + +The public entry point is `improve()`, a profile-aware facade over agent-eval's +`selfImprove`. This module also supplies the runtime-specific code candidate +producer, which mutates an isolated git worktree via a pluggable +`CandidateGenerator`: + - `reflectiveGenerator` — cheap, no sandbox, applies pre-drafted patches + - `agenticGenerator` — full coding harness in the worktree, multi-shot + +#### Properties + +##### schemaVersion + +> `readonly` **schemaVersion**: `1` + +Defined in: [improvement/agentic-generator.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L84) + +##### generation + +> `readonly` **generation**: `number` \| `null` + +Defined in: [improvement/agentic-generator.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L85) + +##### candidateIndex + +> `readonly` **candidateIndex**: `number` \| `null` + +Defined in: [improvement/agentic-generator.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L86) + +##### shot + +> `readonly` **shot**: `number` + +Defined in: [improvement/agentic-generator.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L88) + +One-based shot number within this candidate. + +##### maxShots + +> `readonly` **maxShots**: `number` + +Defined in: [improvement/agentic-generator.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L89) + +##### harness + +> `readonly` **harness**: [`LocalHarness`](mcp.md#localharness) + +Defined in: [improvement/agentic-generator.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L90) + +##### model + +> `readonly` **model**: `string` \| `null` + +Defined in: [improvement/agentic-generator.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L91) + +##### reasoningEffort + +> `readonly` **reasoningEffort**: `ReasoningEffort` \| `null` + +Defined in: [improvement/agentic-generator.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L92) + +##### promptSha256 + +> `readonly` **promptSha256**: `` `sha256:${string}` `` + +Defined in: [improvement/agentic-generator.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L93) + +##### startedAt + +> `readonly` **startedAt**: `string` + +Defined in: [improvement/agentic-generator.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L94) + +##### completedAt + +> `readonly` **completedAt**: `string` + +Defined in: [improvement/agentic-generator.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L95) + +##### durationMs + +> `readonly` **durationMs**: `number` + +Defined in: [improvement/agentic-generator.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L96) + +##### exitCode + +> `readonly` **exitCode**: `number` \| `null` + +Defined in: [improvement/agentic-generator.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L97) + +##### timedOut + +> `readonly` **timedOut**: `boolean` + +Defined in: [improvement/agentic-generator.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L98) + +##### killedBySignal + +> `readonly` **killedBySignal**: `Signals` \| `null` + +Defined in: [improvement/agentic-generator.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L99) + +##### stdoutBytes + +> `readonly` **stdoutBytes**: `number` \| `null` + +Defined in: [improvement/agentic-generator.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L100) + +##### stdoutSha256 + +> `readonly` **stdoutSha256**: `` `sha256:${string}` `` \| `null` + +Defined in: [improvement/agentic-generator.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L101) + +##### stderrBytes + +> `readonly` **stderrBytes**: `number` \| `null` + +Defined in: [improvement/agentic-generator.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L102) + +##### stderrSha256 + +> `readonly` **stderrSha256**: `` `sha256:${string}` `` \| `null` + +Defined in: [improvement/agentic-generator.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L103) + +##### usage + +> `readonly` **usage**: [`CodexTokenUsage`](mcp.md#codextokenusage) \| `null` + +Defined in: [improvement/agentic-generator.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L104) + +##### profileWorkspacePlanDigest + +> `readonly` **profileWorkspacePlanDigest**: `string` \| `null` + +Defined in: [improvement/agentic-generator.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L106) + +Digest of the exact profile-file workspace plan applied for this shot. + +##### profileWorkspaceFileCount + +> `readonly` **profileWorkspaceFileCount**: `number` + +Defined in: [improvement/agentic-generator.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L107) + +##### costCallId + +> `readonly` **costCallId**: `string` \| `null` + +Defined in: [improvement/agentic-generator.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L109) + +Shared run-ledger call id for this exact shot. + +##### costBasis + +> `readonly` **costBasis**: `"unknown"` \| `"provider-reported"` \| `"estimated-pricing"` + +Defined in: [improvement/agentic-generator.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L111) + +Whether dollars came from the provider, the pricing table, or are unknown. + +##### costUsd + +> `readonly` **costUsd**: `number` \| `null` + +Defined in: [improvement/agentic-generator.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L112) + +##### costUsdKnown + +> `readonly` **costUsdKnown**: `boolean` + +Defined in: [improvement/agentic-generator.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L114) + +True only for a provider-reported amount, never for a pricing estimate. + +##### evidence + +> `readonly` **evidence**: [`CodexExecutionEvidence`](mcp.md#codexexecutionevidence) \| `null` + +Defined in: [improvement/agentic-generator.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L115) + +##### error + +> `readonly` **error**: \{ `name`: `string`; `message`: `string`; \} \| `null` + +Defined in: [improvement/agentic-generator.ts:116](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L116) + +*** + +### AgenticGeneratorOptions + +Defined in: [improvement/agentic-generator.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L159) + +`@tangle-network/agent-runtime` improvement — the CODE-surface proposer for +agent-eval's improvement loop. + +The public entry point is `improve()`, a profile-aware facade over agent-eval's +`selfImprove`. This module also supplies the runtime-specific code candidate +producer, which mutates an isolated git worktree via a pluggable +`CandidateGenerator`: + - `reflectiveGenerator` — cheap, no sandbox, applies pre-drafted patches + - `agenticGenerator` — full coding harness in the worktree, multi-shot + +#### Properties + +##### harness? + +> `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) + +Defined in: [improvement/agentic-generator.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L161) + +Local coding harness to run in the worktree. Default `claude`. + +##### profile? + +> `optional` **profile?**: `AgentProfile` + +Defined in: [improvement/agentic-generator.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L164) + +Author profile rendered through the canonical harness mapper. Required + for reproducible Codex so model and reasoning settings are explicit. + +##### codexReproducible? + +> `optional` **codexReproducible?**: `boolean` + +Defined in: [improvement/agentic-generator.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L167) + +Run Codex with isolated configuration, exact prompt evidence, and required + terminal token usage. Requires `harness: 'codex'` and `profile`. + +##### codexReadDeniedPaths? + +> `optional` **codexReadDeniedPaths?**: readonly `string`[] \| ((`worktreePath`) => readonly `string`[]) + +Defined in: [improvement/agentic-generator.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L170) + +Absolute paths reproducible Codex must not read. A function can derive + candidate-specific paths after the driver creates its worktree. + +##### onShotCompleted? + +> `optional` **onShotCompleted?**: (`receipt`, `execution`) => `void` \| `Promise`\<`void`\> + +Defined in: [improvement/agentic-generator.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L175) + +Awaited once for every attempted author shot, including process failures. + The second argument preserves the exact harness result, including stdout + and stderr, before worktree inspection or verification can reject the + shot. Throwing aborts the candidate so evidence persistence fails closed. + +###### Parameters + +###### receipt + +[`AgenticGeneratorShotReceipt`](#agenticgeneratorshotreceipt) + +###### execution + +`Readonly`\<`Omit`\<[`LocalHarnessResult`](mcp.md#localharnessresult), `"usage"` \| `"evidence"`\> & `object`\> \| `null` + +###### Returns + +`void` \| `Promise`\<`void`\> + +##### onShotDisposition? + +> `optional` **onShotDisposition?**: (`receipt`, `disposition`) => `void` \| `Promise`\<`void`\> + +Defined in: [improvement/agentic-generator.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L181) + +Awaited after worktree inspection and before the shot is accepted, + retried, or discarded. Throwing aborts the candidate. + +###### Parameters + +###### receipt + +[`AgenticGeneratorShotReceipt`](#agenticgeneratorshotreceipt) + +###### disposition + +[`AgenticGeneratorShotDisposition`](#agenticgeneratorshotdisposition) + +###### Returns + +`void` \| `Promise`\<`void`\> + +##### maximumCharge? + +> `optional` **maximumCharge?**: `MaximumCharge` + +Defined in: [improvement/agentic-generator.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L189) + +Optional hard upper bound passed to the run-wide CostLedger before each + author shot. This MUST be enforced by the provider or executor; a planning + estimate is not an admissible bound. Omit for an uncapped ledger. A capped + ledger rejects before model dispatch when this is absent. + +##### timeoutMs? + +> `optional` **timeoutMs?**: `number` + +Defined in: [improvement/agentic-generator.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L191) + +Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). + +##### buildPrompt? + +> `optional` **buildPrompt?**: (`args`) => `string` + +Defined in: [improvement/agentic-generator.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L194) + +Build the harness task prompt from the report + findings. Override for + domain phrasing; the default turns findings into a concrete coder task. + +###### Parameters + +###### args + +###### report + +`unknown` + +###### findings + +`AnalystFinding`[] + +###### Returns + +`string` + +##### verify? + +> `optional` **verify?**: [`Verifier`](#verifier) + +Defined in: [improvement/agentic-generator.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L200) + +Verify the worktree after each dirtying shot. When set, a candidate that + fails verification is NOT returned — the failure feeds the next shot + (verify-in-session), up to `maxShots`; a candidate that never verifies is + discarded (`applied:false`), never shipped. Omitted ⇒ legacy behavior: + the first dirty shot is the candidate. See `commandVerifier`. + +##### runHarness? + +> `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> + +Defined in: [improvement/agentic-generator.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L202) + +Test seam — inject the harness runner (defaults to `runLocalHarness`). + +**`Experimental`** + +Spawn a local coding harness CLI as a subprocess + collect its output. + +NOT responsible for parsing the harness's output or extracting a diff — +the in-process executor's `streamPrompt` orchestrates `git diff` against +the worktree after this resolves. This function is intentionally narrow: +spawn, wait, capture, return. + +Fails loud — throws when: + - `cwd` doesn't exist (subprocess emits ENOENT; surfaced as Error) + - the harness binary is not on PATH (ENOENT) + +Does NOT throw when: + - the subprocess exits non-zero (`result.exitCode` carries the code) + - the subprocess is aborted / timed out (`result.killedBySignal` / + `result.timedOut` carries the reason) + +###### Parameters + +###### options + +[`RunLocalHarnessOptions`](mcp.md#runlocalharnessoptions) + +###### Returns + +`Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> + +##### isDirty? + +> `optional` **isDirty?**: (`worktreePath`) => `boolean` + +Defined in: [improvement/agentic-generator.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L204) + +Test seam — inject the worktree-dirty check (defaults to `git status`). + +###### Parameters + +###### worktreePath + +`string` + +###### Returns + +`boolean` + +*** + +### ImproveSkillsOptions + +Defined in: [improvement/improve.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L141) + +#### Properties + +##### document + +> **document**: `string` + +Defined in: [improvement/improve.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L143) + +The skill document's current text — the baseline `skillOptProposer` patches. + +##### writeBack? + +> `optional` **writeBack?**: (`winnerDocument`) => `void` \| `Promise`\<`void`\> + +Defined in: [improvement/improve.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L147) + +Persist the shipped winner document (write the file the profile ref points at). + Called only on a ship verdict. When omitted, the winner is still returned in + `result.raw.winner.surface` for the caller to materialize. + +###### Parameters + +###### winnerDocument + +`string` + +###### Returns + +`void` \| `Promise`\<`void`\> + +*** + +### ImproveMemoryOptions + +Defined in: [improvement/improve.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L150) + +#### Properties + +##### document + +> **document**: `string` + +Defined in: [improvement/improve.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L152) + +Current durable memory text used as the measured baseline. + +##### writeBack? + +> `optional` **writeBack?**: (`winnerDocument`) => `void` \| `Promise`\<`void`\> + +Defined in: [improvement/improve.ts:154](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L154) + +Persist the promoted memory document. Never called on hold or error. + +###### Parameters + +###### winnerDocument + +`string` + +###### Returns + +`void` \| `Promise`\<`void`\> + +*** + +### ImproveCodeOptions + +Defined in: [improvement/improve.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L157) + +#### Properties + +##### repoRoot + +> **repoRoot**: `string` + +Defined in: [improvement/improve.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L159) + +Repo root candidate worktrees fork from. + +##### baseRef? + +> `optional` **baseRef?**: `string` + +Defined in: [improvement/improve.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L161) + +Base ref candidates fork from. Default `main`. + +##### worktreeDir? + +> `optional` **worktreeDir?**: `string` + +Defined in: [improvement/improve.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L163) + +Directory worktrees are created under. Default `/.worktrees`. + +##### worktree? + +> `optional` **worktree?**: `WorktreeAdapter` + +Defined in: [improvement/improve.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L166) + +Git-compatible adapter override, primarily for tests. Candidate advancement + still requires normal Git worktree and commit semantics. + +##### harness? + +> `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) + +Defined in: [improvement/improve.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L168) + +Coding harness the agentic generator runs in each worktree. Default `claude`. + +##### verify? + +> `optional` **verify?**: [`Verifier`](#verifier) + +Defined in: [improvement/improve.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L171) + +Verify a candidate worktree before it becomes a measurable surface; failures + feed the next shot (see `agenticGenerator.verify` / `commandVerifier`). + +##### timeoutMs? + +> `optional` **timeoutMs?**: `number` + +Defined in: [improvement/improve.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L173) + +Per-shot wall-clock timeout for the harness (ms). + +##### generator? + +> `optional` **generator?**: [`CandidateGenerator`](#candidategenerator) + +Defined in: [improvement/improve.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L176) + +Byte-producer override — the test seam and the escape hatch for custom + candidate production. When set, `harness`/`verify`/`timeoutMs` are unused. + +*** + +### ImproveResult + +Defined in: [improvement/improve.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L179) + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +#### Properties + +##### profile + +> **profile**: `AgentProfile` + +Defined in: [improvement/improve.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L182) + +The profile after improvement: the winner surface applied back into the + matching field when the gate shipped, else the input profile unchanged. + +##### shipped + +> **shipped**: `boolean` + +Defined in: [improvement/improve.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L184) + +True when `gateDecision === 'ship'`. + +##### lift + +> **lift**: `number` + +Defined in: [improvement/improve.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L186) + +Held-out lift (`winner − baseline` composite). + +##### gateDecision + +> **gateDecision**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` + +Defined in: [improvement/improve.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L188) + +The five-valued gate verdict from `selfImprove`. + +##### raw + +> **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> + +Defined in: [improvement/improve.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L192) + +Full `selfImprove` result for advanced inspection. For code runs, + `raw.winner.surface.worktreeRef` remains live after return whether the + candidate shipped or held; call `dispose()` after consuming it. + +#### Methods + +##### dispose() + +> **dispose**(): `Promise`\<`void`\> + +Defined in: [improvement/improve.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L195) + +Release resources owned by this result. Idempotent; currently disposes + the returned code worktree and is a no-op for profile-only surfaces. + +###### Returns + +`Promise`\<`void`\> + +*** + +### CandidateGenerator + +Defined in: [improvement/improvement-driver.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L39) + +The byte-producing seam — the ONE thing that differs between the cheap + reflective path and the full agentic path. A generator makes (uncommitted) + changes inside `worktreePath`; the driver commits them via the worktree + adapter's `finalize`. + +#### Properties + +##### kind + +> **kind**: `string` + +Defined in: [improvement/improvement-driver.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L40) + +##### proposesWithoutFindings? + +> `optional` **proposesWithoutFindings?**: `boolean` + +Defined in: [improvement/improvement-driver.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L51) + +Whether this generator can produce a candidate from an EMPTY findings set + and no phase-2 report — i.e. it draws its change signal from the repo and + the raw-trace filesystem context on disk, not only from pre-summarized + findings. An agentic coder (`agenticGenerator`) sets this: the seed repo + + raw traces ARE the signal, so it must still run the full `populationSize` + when the distiller yielded nothing (this is the meta-harness contract — the + agent diagnoses from the raw traces itself). A patch-applier + (`reflectiveGenerator`) leaves it unset — with no findings there is no + patch to draft, so the driver short-circuits rather than spin up worktrees + for a guaranteed no-op. Default `false`. + +#### Methods + +##### generate() + +> **generate**(`args`): `Promise`\<\{ `applied`: `boolean`; `summary`: `string`; \}\> + +Defined in: [improvement/improvement-driver.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L52) + +###### Parameters + +###### args + +###### worktreePath + +`string` + +The candidate worktree — a clean checkout of the current incumbent. + +###### report + +`unknown` + +Phase-2 research report (analyst findings + diff), opaque. + +###### findings + +`AnalystFinding`[] + +Findings resolved from the report or the loop context. + +###### dataset? + +`LabeledScenarioStore` + +Handle to all captured data, to ground the change. + +###### maxShots + +`number` + +DEPTH: max iterations the generator may take (agentic uses this; the + reflective generator ignores it). + +###### signal + +`AbortSignal` + +###### generation? + +`number` + +Improvement-loop coordinates. Present when called through improvementDriver. + +###### candidateIndex? + +`number` + +###### costLedger? + +`CostLedger` + +Shared run-wide paid-call account supplied by agent-eval 0.117+. + +###### costPhase? + +`string` + +Receipt attribution phase supplied alongside `costLedger`. + +###### Returns + +`Promise`\<\{ `applied`: `boolean`; `summary`: `string`; \}\> + +*** + +### ImprovementDriverOptions + +Defined in: [improvement/improvement-driver.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L75) + +#### Properties + +##### worktree + +> **worktree**: `WorktreeAdapter` + +Defined in: [improvement/improvement-driver.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L76) + +##### generator + +> **generator**: [`CandidateGenerator`](#candidategenerator) + +Defined in: [improvement/improvement-driver.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L77) + +##### baseRef? + +> `optional` **baseRef?**: `string` + +Defined in: [improvement/improvement-driver.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L80) + +Root ref for first-generation/direct callers. Default `main`. + Later code generations retain the incumbent's original root. + +*** + +### ManagedImprovementDriver + +Defined in: [improvement/improvement-driver.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L83) + +#### Extends + +- `SurfaceProposer`\<`AnalystFinding`\> + +#### Methods + +##### cleanup() + +> **cleanup**(`retainWorktreeRefs?`): `Promise`\<`void`\> + +Defined in: [improvement/improvement-driver.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L85) + +Remove every owned candidate except explicitly retained finalized winners. + +###### Parameters + +###### retainWorktreeRefs? + +readonly `string`[] + +###### Returns + +`Promise`\<`void`\> + +*** + +### McpServeSpec + +Defined in: [improvement/mcp-serve-verifier.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L24) + +#### Properties + +##### command + +> **command**: `string` + +Defined in: [improvement/mcp-serve-verifier.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L26) + +Command that starts the built MCP server in the worktree (stdio transport). + +##### args? + +> `optional` **args?**: `string`[] + +Defined in: [improvement/mcp-serve-verifier.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L27) + +##### env? + +> `optional` **env?**: `Record`\<`string`, `string`\> + +Defined in: [improvement/mcp-serve-verifier.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L29) + +Extra env for the server process (merged over `process.env`). + +##### timeoutMs? + +> `optional` **timeoutMs?**: `number` + +Defined in: [improvement/mcp-serve-verifier.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L31) + +Handshake timeout (ms). Default 30s. + +##### minTools? + +> `optional` **minTools?**: `number` + +Defined in: [improvement/mcp-serve-verifier.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L33) + +Minimum tools the server must expose to pass. Default 1. + +*** + +### AgentProfileDiffProposal + +Defined in: [improvement/profile-diff-proposer.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L20) + +#### Properties + +##### diff + +> **diff**: `AgentProfileDiff` + +Defined in: [improvement/profile-diff-proposer.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L21) + +##### label? + +> `optional` **label?**: `string` + +Defined in: [improvement/profile-diff-proposer.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L22) + +##### rationale? + +> `optional` **rationale?**: `string` + +Defined in: [improvement/profile-diff-proposer.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L23) + +*** + +### ProfileDiffProposerOptions + +Defined in: [improvement/profile-diff-proposer.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L30) + +#### Type Parameters + +##### TFindings + +`TFindings` = `unknown` + +#### Methods + +##### proposeDiffs() + +> **proposeDiffs**(`context`): readonly [`AgentProfileDiffProposal`](#agentprofilediffproposal)[] \| `Promise`\ + +Defined in: [improvement/profile-diff-proposer.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L31) + +###### Parameters + +###### context + +[`ProfileDiffProposerContext`](#profilediffproposercontext)\<`TFindings`\> + +###### Returns + +readonly [`AgentProfileDiffProposal`](#agentprofilediffproposal)[] \| `Promise`\ + +*** + +### RawTraceDistillerOptions + +Defined in: [improvement/raw-trace-distiller.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L44) + +#### Properties + +##### runDir? + +> `optional` **runDir?**: `string` + +Defined in: [improvement/raw-trace-distiller.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L49) + +Anchor the emitted paths at this run root instead of the generation `runDir` + the loop passes in. Normally unset — each call points at that generation's + own directory (`input.runDir`). Pass an absolute path when you construct the + producer ahead of the loop and want a fixed anchor (e.g. a test fixture). + +##### maxCandidates? + +> `optional` **maxCandidates?**: `number` + +Defined in: [improvement/raw-trace-distiller.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L51) + +Max candidates to surface trace paths for, worst-scoring first. Default 12. + +##### maxCellsPerCandidate? + +> `optional` **maxCellsPerCandidate?**: `number` + +Defined in: [improvement/raw-trace-distiller.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L54) + +Max failing cells to enumerate per candidate before collapsing the rest into + an "ls the candidate dir" pointer. Default 8. + +##### maxFilesPerCell? + +> `optional` **maxFilesPerCell?**: `number` + +Defined in: [improvement/raw-trace-distiller.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L57) + +Max concrete file paths to list per cell (the agent can always `ls` the dir + for the rest). Default 24. + +##### fallbackFindings? + +> `optional` **fallbackFindings?**: `unknown`[] + +Defined in: [improvement/raw-trace-distiller.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L61) + +Findings to fall back to when the generation had NO failing cells, so a + clean round never wipes the proposer's steering context. Mirrors the default + distiller's static-seed fallback. Default: a single instruction finding. + +*** + +### ReflectiveGeneratorOptions + +Defined in: [improvement/reflective-generator.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L21) + +#### Properties + +##### improvementAdapter + +> **improvementAdapter**: [`ImprovementAdapter`](analyst-loop.md#improvementadapter)\<[`SurfaceImprovementEdit`](agent.md#surfaceimprovementedit)\> + +Defined in: [improvement/reflective-generator.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L22) + +*** + +### RunKnowledgeImprovementJobOptions + +Defined in: [knowledge/improvement-job.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L21) + +#### Extends + +- `Omit`\<`KnowledgeImprovementOptions`, `"updateKnowledge"`\> + +#### Properties + +##### budget + +> **budget**: [`Budget`](runtime.md#budget-12) + +Defined in: [knowledge/improvement-job.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L23) + +##### readinessCheck? + +> `optional` **readinessCheck?**: [`KnowledgeReadinessCheck`](#knowledgereadinesscheck) + +Defined in: [knowledge/improvement-job.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L24) + +##### backend? + +> `optional` **backend?**: [`ExecutorConfig`](runtime.md#executorconfig) + +Defined in: [knowledge/improvement-job.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L25) + +##### makeWorkerAgent? + +> `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) + +Defined in: [knowledge/improvement-job.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L26) + +##### harness? + +> `optional` **harness?**: `string` + +Defined in: [knowledge/improvement-job.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L27) + +##### supervisorModel? + +> `optional` **supervisorModel?**: `string` + +Defined in: [knowledge/improvement-job.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L28) + +##### supervisorSystemPrompt? + +> `optional` **supervisorSystemPrompt?**: `string` + +Defined in: [knowledge/improvement-job.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L29) + +##### superviseOptions? + +> `optional` **superviseOptions?**: `Partial`\<`Omit`\<[`SuperviseOptions`](runtime.md#superviseoptions), `"backend"` \| `"budget"` \| `"makeWorkerAgent"` \| `"deliverable"` \| `"allowedModels"`\>\> + +Defined in: [knowledge/improvement-job.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L30) + +##### allowedModels? + +> `optional` **allowedModels?**: readonly `string`[] + +Defined in: [knowledge/improvement-job.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L36) + +##### runSupervised? + +> `optional` **runSupervised?**: (`profile`, `task`, `opts`) => `Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> + +Defined in: [knowledge/improvement-job.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L37) + +###### Parameters + +###### profile + +[`SupervisorProfile`](runtime.md#supervisorprofile) + +###### task + +`unknown` + +###### opts + +[`SuperviseOptions`](runtime.md#superviseoptions) + +###### Returns + +`Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> + +##### onMeasurement? + +> `optional` **onMeasurement?**: (`measurement`) => `void` \| `Promise`\<`void`\> + +Defined in: [knowledge/improvement-job.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L42) + +###### Parameters + +###### measurement + +[`KnowledgeImprovementJobMeasurement`](#knowledgeimprovementjobmeasurement) + +###### Returns + +`void` \| `Promise`\<`void`\> + +*** + +### KnowledgeImprovementJobMeasurement + +Defined in: [knowledge/improvement-job.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L45) + +#### Properties + +##### startedAt + +> **startedAt**: `string` + +Defined in: [knowledge/improvement-job.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L46) + +##### finishedAt + +> **finishedAt**: `string` + +Defined in: [knowledge/improvement-job.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L47) + +##### durationMs + +> **durationMs**: `number` + +Defined in: [knowledge/improvement-job.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L48) + +##### updateCalls + +> **updateCalls**: `number` + +Defined in: [knowledge/improvement-job.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L49) + +##### updateDurationMs + +> **updateDurationMs**: `number` + +Defined in: [knowledge/improvement-job.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L50) + +##### supervisedSpent + +> **supervisedSpent**: `object` + +Defined in: [knowledge/improvement-job.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L51) + +###### iterations + +> **iterations**: `number` + +###### inputTokens + +> **inputTokens**: `number` + +###### outputTokens + +> **outputTokens**: `number` + +###### usd + +> **usd**: `number` + +###### ms + +> **ms**: `number` + +*** + +### KnowledgeImprovementJobResult + +Defined in: [knowledge/improvement-job.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L60) + +#### Properties + +##### improvement + +> **improvement**: `KnowledgeImprovementResult` + +Defined in: [knowledge/improvement-job.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L61) + +##### measurement + +> **measurement**: [`KnowledgeImprovementJobMeasurement`](#knowledgeimprovementjobmeasurement) + +Defined in: [knowledge/improvement-job.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L62) + +##### promoted + +> **promoted**: `boolean` + +Defined in: [knowledge/improvement-job.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L63) + +##### blocked + +> **blocked**: `boolean` + +Defined in: [knowledge/improvement-job.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L64) + +*** + +### AgentKnowledgeReadinessCheckOptions + +Defined in: [knowledge/improvement-job.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L67) + +#### Properties + +##### goal + +> **goal**: `string` + +Defined in: [knowledge/improvement-job.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L68) + +##### readinessSpecs? + +> `optional` **readinessSpecs?**: readonly `KnowledgeReadinessSpec`[] + +Defined in: [knowledge/improvement-job.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L69) + +##### readinessTaskId? + +> `optional` **readinessTaskId?**: `string` + +Defined in: [knowledge/improvement-job.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L70) + +##### readiness? + +> `optional` **readiness?**: `Omit`\<`BuildEvalKnowledgeBundleOptions`, `"taskId"` \| `"index"` \| `"specs"`\> + +Defined in: [knowledge/improvement-job.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L71) + +##### strict? + +> `optional` **strict?**: `boolean` + +Defined in: [knowledge/improvement-job.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L72) + +##### kbQuality? + +> `optional` **kbQuality?**: `KnowledgeBaseQualityOptions` + +Defined in: [knowledge/improvement-job.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L73) + +*** + +### KnowledgeReadinessCheckInput + +Defined in: [knowledge/supervised-update.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L22) + +#### Properties + +##### root + +> **root**: `string` + +Defined in: [knowledge/supervised-update.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L23) + +##### goal + +> **goal**: `string` + +Defined in: [knowledge/supervised-update.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L24) + +##### readinessSpecs? + +> `optional` **readinessSpecs?**: readonly `unknown`[] + +Defined in: [knowledge/supervised-update.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L25) + +##### readinessTaskId? + +> `optional` **readinessTaskId?**: `string` + +Defined in: [knowledge/supervised-update.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L26) + +##### readiness? + +> `optional` **readiness?**: `unknown` + +Defined in: [knowledge/supervised-update.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L27) + +*** + +### SupervisedKnowledgeUpdateInput + +Defined in: [knowledge/supervised-update.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L42) + +#### Properties + +##### goal? + +> `optional` **goal?**: `string` + +Defined in: [knowledge/supervised-update.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L43) + +##### root? + +> `optional` **root?**: `string` + +Defined in: [knowledge/supervised-update.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L44) + +##### candidateRoot? + +> `optional` **candidateRoot?**: `string` + +Defined in: [knowledge/supervised-update.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L45) + +##### findings? + +> `optional` **findings?**: readonly `unknown`[] + +Defined in: [knowledge/supervised-update.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L46) + +##### metadata? + +> `optional` **metadata?**: `Record`\<`string`, `unknown`\> + +Defined in: [knowledge/supervised-update.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L47) + +*** + +### SupervisedKnowledgeUpdateResult + +Defined in: [knowledge/supervised-update.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L50) + +#### Properties + +##### applied + +> **applied**: `boolean` + +Defined in: [knowledge/supervised-update.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L51) + +##### summary + +> **summary**: `string` + +Defined in: [knowledge/supervised-update.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L52) + +##### supervised + +> **supervised**: [`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\> + +Defined in: [knowledge/supervised-update.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L53) + +##### metadata + +> **metadata**: `Record`\<`string`, `unknown`\> + +Defined in: [knowledge/supervised-update.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L54) + +*** + +### SupervisedKnowledgeUpdateOptions + +Defined in: [knowledge/supervised-update.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L57) + +#### Properties + +##### root + +> **root**: `string` + +Defined in: [knowledge/supervised-update.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L58) + +##### goal + +> **goal**: `string` + +Defined in: [knowledge/supervised-update.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L59) + +##### readiness + +> **readiness**: [`KnowledgeReadinessCheck`](#knowledgereadinesscheck) + +Defined in: [knowledge/supervised-update.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L60) + +##### readinessSpecs? + +> `optional` **readinessSpecs?**: readonly `unknown`[] + +Defined in: [knowledge/supervised-update.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L61) + +##### readinessTaskId? + +> `optional` **readinessTaskId?**: `string` + +Defined in: [knowledge/supervised-update.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L62) + +##### readinessOptions? + +> `optional` **readinessOptions?**: `unknown` + +Defined in: [knowledge/supervised-update.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L63) + +##### findings? + +> `optional` **findings?**: readonly `unknown`[] + +Defined in: [knowledge/supervised-update.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L64) + +##### metadata? + +> `optional` **metadata?**: `Record`\<`string`, `unknown`\> + +Defined in: [knowledge/supervised-update.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L65) + +##### budget + +> **budget**: [`Budget`](runtime.md#budget-12) + +Defined in: [knowledge/supervised-update.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L66) + +##### backend? + +> `optional` **backend?**: [`ExecutorConfig`](runtime.md#executorconfig) + +Defined in: [knowledge/supervised-update.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L67) + +##### makeWorkerAgent? + +> `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) + +Defined in: [knowledge/supervised-update.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L68) + +##### harness? + +> `optional` **harness?**: `string` + +Defined in: [knowledge/supervised-update.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L69) + +##### supervisorModel? + +> `optional` **supervisorModel?**: `string` + +Defined in: [knowledge/supervised-update.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L70) + +##### supervisorSystemPrompt? + +> `optional` **supervisorSystemPrompt?**: `string` + +Defined in: [knowledge/supervised-update.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L71) + +##### superviseOptions? + +> `optional` **superviseOptions?**: `Partial`\<`Omit`\<[`SuperviseOptions`](runtime.md#superviseoptions), `"backend"` \| `"budget"` \| `"makeWorkerAgent"` \| `"deliverable"` \| `"allowedModels"`\>\> + +Defined in: [knowledge/supervised-update.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L72) + +##### allowedModels? + +> `optional` **allowedModels?**: readonly `string`[] + +Defined in: [knowledge/supervised-update.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L78) + +##### runSupervised? + +> `optional` **runSupervised?**: (`profile`, `task`, `opts`) => `Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> + +Defined in: [knowledge/supervised-update.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L79) + +###### Parameters + +###### profile + +[`SupervisorProfile`](runtime.md#supervisorprofile) + +###### task + +`unknown` + +###### opts + +[`SuperviseOptions`](runtime.md#superviseoptions) + +###### Returns + +`Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> + +*** + +### DelegatedLoopResult + +Defined in: [loop-runner.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L67) + +**`Experimental`** + +Uniform result — never throws from a registered runner; a + thrown engine becomes `{ ok: false, error }` so a routine can record + move on. + +#### Type Parameters + +##### T + +`T` = `unknown` + +#### Properties + +##### mode + +> **mode**: `"code"` \| `"review"` \| `"research"` \| `"audit"` \| `"self-improve"` + +Defined in: [loop-runner.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L68) + +**`Experimental`** + +##### ok + +> **ok**: `boolean` + +Defined in: [loop-runner.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L69) + +**`Experimental`** + +##### output? + +> `optional` **output?**: `T` + +Defined in: [loop-runner.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L70) + +**`Experimental`** + +##### error? + +> `optional` **error?**: `string` + +Defined in: [loop-runner.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L71) + +**`Experimental`** + +##### durationMs + +> **durationMs**: `number` + +Defined in: [loop-runner.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L72) + +**`Experimental`** + +*** + +### RunDelegatedLoopOptions + +Defined in: [loop-runner.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L76) + +**`Experimental`** + +#### Properties + +##### signal? + +> `optional` **signal?**: `AbortSignal` + +Defined in: [loop-runner.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L77) + +**`Experimental`** + +##### now? + +> `optional` **now?**: () => `number` + +Defined in: [loop-runner.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L79) + +**`Experimental`** + +Clock override for deterministic tests. + +###### Returns + +`number` + +*** + +### WorktreeLoopRunnerOptions + +Defined in: [loop-runner.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L121) + +**`Experimental`** + +Options for the local-repo `code` runner over the GENERIC recursive path. + +#### Properties + +##### repoRoot + +> **repoRoot**: `string` + +Defined in: [loop-runner.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L123) + +**`Experimental`** + +Absolute path to the local git checkout each worktree is cut from. + +##### taskPrompt + +> **taskPrompt**: `string` + +Defined in: [loop-runner.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L125) + +**`Experimental`** + +The instruction handed to every authored harness (composed under each profile's systemPrompt). + +##### harnesses + +> **harnesses**: readonly [`AuthoredHarness`](runtime.md#authoredharness)[] + +Defined in: [loop-runner.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L127) + +**`Experimental`** + +The supervisor-authored harness profiles — one fanout item (one worktree-CLI leaf) each. + +##### budget + +> **budget**: [`Budget`](runtime.md#budget-12) + +Defined in: [loop-runner.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L129) + +**`Experimental`** + +Conserved budget pool bounding the fanout (equal-k holds by construction). + +##### testCmd? + +> `optional` **testCmd?**: `string` + +Defined in: [loop-runner.ts:131](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L131) + +**`Experimental`** + +Shell command run in each worktree to derive the tests-PASS signal. + +##### typecheckCmd? + +> `optional` **typecheckCmd?**: `string` + +Defined in: [loop-runner.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L133) + +**`Experimental`** + +Shell command run in each worktree to derive the typecheck-PASS signal. + +##### require? + +> `optional` **require?**: readonly (`"tests"` \| `"typecheck"`)[] + +Defined in: [loop-runner.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L135) + +**`Experimental`** + +Which verification signals the deliverable REQUIRES present-and-passing (default none). + +##### maxDiffLines? + +> `optional` **maxDiffLines?**: `number` + +Defined in: [loop-runner.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L137) + +**`Experimental`** + +Diff-size cap (lines). + +##### forbiddenPaths? + +> `optional` **forbiddenPaths?**: `string`[] + +Defined in: [loop-runner.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L139) + +**`Experimental`** + +Literal path prefixes the patch must not touch (the secret-floor is always on regardless). + +##### winnerStrategy? + +> `optional` **winnerStrategy?**: [`WinnerStrategy`](runtime.md#winnerstrategy) + +Defined in: [loop-runner.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L141) + +**`Experimental`** + +Winner-selection strategy among gated candidates. Default `highest-score`. + +##### runGit? + +> `optional` **runGit?**: [`GitRunner`](mcp.md#gitrunner) + +Defined in: [loop-runner.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L143) + +**`Experimental`** + +Test seams forwarded to the worktree-CLI leaves so the runner drives offline. + +##### runHarness? + +> `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> + +Defined in: [loop-runner.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L144) + +**`Experimental`** + +**`Experimental`** + +Spawn a local coding harness CLI as a subprocess + collect its output. + +NOT responsible for parsing the harness's output or extracting a diff — +the in-process executor's `streamPrompt` orchestrates `git diff` against +the worktree after this resolves. This function is intentionally narrow: +spawn, wait, capture, return. + +Fails loud — throws when: + - `cwd` doesn't exist (subprocess emits ENOENT; surfaced as Error) + - the harness binary is not on PATH (ENOENT) + +Does NOT throw when: + - the subprocess exits non-zero (`result.exitCode` carries the code) + - the subprocess is aborted / timed out (`result.killedBySignal` / + `result.timedOut` carries the reason) + +###### Parameters + +###### options + +[`RunLocalHarnessOptions`](mcp.md#runlocalharnessoptions) + +###### Returns + +`Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> + +##### runCommand? + +> `optional` **runCommand?**: `WorktreeCheckRunner` + +Defined in: [loop-runner.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L145) + +**`Experimental`** + +*** + +### VetoedFact + +Defined in: [loop-runner.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L208) + +**`Experimental`** + +A fact rejected at the KB gate — surfaced, never dropped. + +#### Properties + +##### candidate + +> **candidate**: [`FactCandidate`](mcp.md#factcandidate) + +Defined in: [loop-runner.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L209) + +**`Experimental`** + +##### vetoedBy? + +> `optional` **vetoedBy?**: `string` + +Defined in: [loop-runner.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L210) + +**`Experimental`** + +##### reason? + +> `optional` **reason?**: `string` + +Defined in: [loop-runner.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L211) + +**`Experimental`** + +*** + +### ResearchLoopResult + +Defined in: [loop-runner.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L215) + +**`Experimental`** + +#### Properties + +##### accepted + +> **accepted**: [`FactCandidate`](mcp.md#factcandidate)[] + +Defined in: [loop-runner.ts:217](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L217) + +**`Experimental`** + +Facts that passed the fail-closed gate — safe to write to the KB. + +##### vetoed + +> **vetoed**: [`VetoedFact`](#vetoedfact)[] + +Defined in: [loop-runner.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L219) + +**`Experimental`** + +Facts the gate vetoed in the final round — escalate, do not silently drop. + +##### rounds + +> **rounds**: `number` + +Defined in: [loop-runner.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L221) + +**`Experimental`** + +Research rounds actually run. + +*** + +### ResearchLoopRunnerOptions + +Defined in: [loop-runner.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L225) + +**`Experimental`** + +Options for the default `research` runner. + +#### Properties + +##### research + +> **research**: (`round`, `vetoed`) => `Promise`\<[`FactCandidate`](mcp.md#factcandidate)[]\> + +Defined in: [loop-runner.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L232) + +**`Experimental`** + +The research engine (the consumer's web/doc searcher + extractor). Called +each round with the prior round's vetoes so it can re-research the gaps. +Returns fact candidates carrying their grounding (`verbatimPassage` + +`sourceText`). + +###### Parameters + +###### round + +`number` + +###### vetoed + +[`VetoedFact`](#vetoedfact)[] + +###### Returns + +`Promise`\<[`FactCandidate`](mcp.md#factcandidate)[]\> + +##### gate? + +> `optional` **gate?**: [`CreateKbGateOptions`](mcp.md#createkbgateoptions) + +Defined in: [loop-runner.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L234) + +**`Experimental`** + +Gate config (extra judges, self-artifact kinds, …). The floor is always on. + +##### maxRounds? + +> `optional` **maxRounds?**: `number` + +Defined in: [loop-runner.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L236) + +**`Experimental`** + +Max research rounds (correct-on-veto remediation). Default 1. + +*** + +### ModelInfo + +Defined in: [model-resolution.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L22) + +A model entry as returned by the Tangle Router `/v1/models` endpoint. +Intentionally minimal — only the fields resolution + validation read. + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [model-resolution.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L23) + +##### name? + +> `optional` **name?**: `string` + +Defined in: [model-resolution.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L24) + +##### description? + +> `optional` **description?**: `string` + +Defined in: [model-resolution.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L25) + +##### provider? + +> `optional` **provider?**: `string` + +Defined in: [model-resolution.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L27) + +Provider slug, when the router exposes it (`provider` or `_provider`). + +##### \_provider? + +> `optional` **\_provider?**: `string` + +Defined in: [model-resolution.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L28) + +##### architecture? + +> `optional` **architecture?**: `object` + +Defined in: [model-resolution.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L29) + +###### modality? + +> `optional` **modality?**: `string` + +###### input\_modalities? + +> `optional` **input\_modalities?**: `string`[] + +###### output\_modalities? + +> `optional` **output\_modalities?**: `string`[] + +*** + +### RouterEnv + +Defined in: [model-resolution.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L37) + +Env keys the router base URL is resolved from. + +#### Properties + +##### TANGLE\_ROUTER\_URL? + +> `optional` **TANGLE\_ROUTER\_URL?**: `string` + +Defined in: [model-resolution.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L38) + +##### TANGLE\_ROUTER\_BASE\_URL? + +> `optional` **TANGLE\_ROUTER\_BASE\_URL?**: `string` + +Defined in: [model-resolution.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L39) + +*** + +### ResolvedChatModel + +Defined in: [model-resolution.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L80) + +#### Properties + +##### source + +> **source**: `string` + +Defined in: [model-resolution.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L81) + +##### model + +> **model**: `string` + +Defined in: [model-resolution.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L82) + +*** + +### OtelExportConfig + +Defined in: [otel-export.ts:15](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L15) + +#### Properties + +##### endpoint? + +> `optional` **endpoint?**: `string` + +Defined in: [otel-export.ts:17](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L17) + +OTLP endpoint. Reads OTEL_EXPORTER_OTLP_ENDPOINT env by default. + +##### headers? + +> `optional` **headers?**: `Record`\<`string`, `string`\> + +Defined in: [otel-export.ts:19](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L19) + +OTLP headers. Reads OTEL_EXPORTER_OTLP_HEADERS env by default. + +##### batchSize? + +> `optional` **batchSize?**: `number` + +Defined in: [otel-export.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L21) + +Batch size before flush. Default 64. + +##### flushIntervalMs? + +> `optional` **flushIntervalMs?**: `number` + +Defined in: [otel-export.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L23) + +Flush interval ms. Default 5000. + +##### resourceAttributes? + +> `optional` **resourceAttributes?**: `Record`\<`string`, `string` \| `number` \| `boolean`\> + +Defined in: [otel-export.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L25) + +Resource attributes stamped on every export. + +##### serviceName? + +> `optional` **serviceName?**: `string` + +Defined in: [otel-export.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L27) + +Service name. Default 'agent-runtime'. + +*** + +### OtelExporter + +Defined in: [otel-export.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L30) + +#### Methods + +##### exportSpan() + +> **exportSpan**(`span`): `void` + +Defined in: [otel-export.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L32) + +Export a span. + +###### Parameters + +###### span + +[`OtelSpan`](#otelspan) + +###### Returns + +`void` + +##### flush() + +> **flush**(): `Promise`\<`void`\> + +Defined in: [otel-export.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L34) + +Force flush pending spans. + +###### Returns + +`Promise`\<`void`\> + +##### shutdown() + +> **shutdown**(): `Promise`\<`void`\> + +Defined in: [otel-export.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L36) + +Shutdown cleanly. + +###### Returns + +`Promise`\<`void`\> + +*** + +### OtelSpan + +Defined in: [otel-export.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L39) + +#### Properties + +##### traceId + +> **traceId**: `string` + +Defined in: [otel-export.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L40) + +##### spanId + +> **spanId**: `string` + +Defined in: [otel-export.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L41) + +##### parentSpanId? + +> `optional` **parentSpanId?**: `string` + +Defined in: [otel-export.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L42) + +##### name + +> **name**: `string` + +Defined in: [otel-export.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L43) + +##### kind? + +> `optional` **kind?**: `number` + +Defined in: [otel-export.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L44) + +##### startTimeUnixNano + +> **startTimeUnixNano**: `string` + +Defined in: [otel-export.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L45) + +##### endTimeUnixNano + +> **endTimeUnixNano**: `string` + +Defined in: [otel-export.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L46) + +##### attributes? + +> `optional` **attributes?**: [`OtelAttribute`](#otelattribute)[] + +Defined in: [otel-export.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L47) + +##### status? + +> `optional` **status?**: `object` + +Defined in: [otel-export.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L48) + +###### code + +> **code**: `number` + +###### message? + +> `optional` **message?**: `string` + +*** + +### OtelAttribute + +Defined in: [otel-export.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L51) + +#### Properties + +##### key + +> **key**: `string` + +Defined in: [otel-export.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L52) + +##### value + +> **value**: `object` + +Defined in: [otel-export.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L53) + +###### stringValue? + +> `optional` **stringValue?**: `string` + +###### intValue? + +> `optional` **intValue?**: `string` + +###### doubleValue? + +> `optional` **doubleValue?**: `number` + +###### boolValue? + +> `optional` **boolValue?**: `boolean` + +*** + +### RuntimeEventOtelOptions + +Defined in: [otel-export.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L230) + +#### Stable + +#### Extends + +- [`RuntimeTelemetryOptions`](#runtimetelemetryoptions) + +#### Properties + +##### redact? + +> `optional` **redact?**: (`value`) => `unknown` + +Defined in: [otel-export.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L232) + +Final customer redactor applied after the schema-aware runtime sanitizer. + +###### Parameters + +###### value + +`unknown` + +###### Returns + +`unknown` + +##### includeInputs? + +> `optional` **includeInputs?**: `boolean` + +Defined in: [sanitize.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L35) + +Include raw task inputs. Off by default because task inputs often contain +customer facts, credentials, source text, or internal IDs. + +###### Inherited from + +[`RuntimeTelemetryOptions`](#runtimetelemetryoptions).[`includeInputs`](#includeinputs-1) + +##### includeRequirementDescriptions? + +> `optional` **includeRequirementDescriptions?**: `boolean` + +Defined in: [sanitize.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L37) + +Include requirement descriptions. Secret requirements are always redacted. + +###### Inherited from + +[`RuntimeTelemetryOptions`](#runtimetelemetryoptions).[`includeRequirementDescriptions`](#includerequirementdescriptions-1) + +##### includeEvidenceIds? + +> `optional` **includeEvidenceIds?**: `boolean` + +Defined in: [sanitize.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L39) + +Include evidence IDs. Off by default; counts are safer for shared reports. + +###### Inherited from + +[`RuntimeTelemetryOptions`](#runtimetelemetryoptions).[`includeEvidenceIds`](#includeevidenceids-1) + +##### includeUserAnswers? + +> `optional` **includeUserAnswers?**: `boolean` + +Defined in: [sanitize.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L41) + +Include user answers from question preflight. Off by default. + +###### Inherited from + +[`RuntimeTelemetryOptions`](#runtimetelemetryoptions).[`includeUserAnswers`](#includeuseranswers-1) + +##### includeControlPayloads? + +> `optional` **includeControlPayloads?**: `boolean` + +Defined in: [sanitize.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L43) + +Include action payloads and action results for control steps. Off by default. + +###### Inherited from + +[`RuntimeTelemetryOptions`](#runtimetelemetryoptions).[`includeControlPayloads`](#includecontrolpayloads-1) + +##### includeMetadata? + +> `optional` **includeMetadata?**: `boolean` + +Defined in: [sanitize.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L45) + +Include task metadata. Off by default because metadata may carry IDs or policy internals. + +###### Inherited from + +[`RuntimeTelemetryOptions`](#runtimetelemetryoptions).[`includeMetadata`](#includemetadata-1) + +##### includeEvalDetails? + +> `optional` **includeEvalDetails?**: `boolean` + +Defined in: [sanitize.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L47) + +Include eval detail/evidence strings. Off by default because validators may echo private input. + +###### Inherited from + +[`RuntimeTelemetryOptions`](#runtimetelemetryoptions).[`includeEvalDetails`](#includeevaldetails-1) + +*** + +### LoopSpanNode + +Defined in: [otel-export.ts:334](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L334) + +Sink-neutral node in a reconstructed loop span tree. The root node's +`parentSpanId` is `undefined` — sinks decide how to parent it (the OTEL +mapper attaches the inherited delegation span; the delegation journal +leaves it as the tree root). + +#### Properties + +##### spanId + +> **spanId**: `string` + +Defined in: [otel-export.ts:335](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L335) + +##### parentSpanId? + +> `optional` **parentSpanId?**: `string` + +Defined in: [otel-export.ts:336](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L336) + +##### name + +> **name**: `string` + +Defined in: [otel-export.ts:338](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L338) + +`'loop'` | `'loop.round'` | `'loop.iteration'`. + +##### kind + +> **kind**: `"loop"` \| `"round"` \| `"branch"` + +Defined in: [otel-export.ts:340](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L340) + +Topology level: loop root, plan round, or iteration branch. + +##### startMs + +> **startMs**: `number` + +Defined in: [otel-export.ts:341](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L341) + +##### endMs + +> **endMs**: `number` + +Defined in: [otel-export.ts:342](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L342) + +##### attrs + +> **attrs**: `Record`\<`string`, `string` \| `number` \| `boolean`\> + +Defined in: [otel-export.ts:343](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L343) + +##### error + +> **error**: `boolean` + +Defined in: [otel-export.ts:345](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L345) + +True when the iteration carried an error — maps to OTEL status code 2. + +*** + +### EvalRunGeneration + +Defined in: [otel-export.ts:667](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L667) + +#### Properties + +##### index + +> **index**: `number` + +Defined in: [otel-export.ts:669](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L669) + +0-based ordinal of this generation within the run (required by ingest). + +##### surfaceHash + +> **surfaceHash**: `string` + +Defined in: [otel-export.ts:671](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L671) + +Identity of the proposed surface change (content-addressed hash). + +##### surface? + +> `optional` **surface?**: `unknown` + +Defined in: [otel-export.ts:673](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L673) + +Arbitrary provenance for this generation (rationale, evidence, source). + +##### cells? + +> `optional` **cells?**: `unknown`[] + +Defined in: [otel-export.ts:675](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L675) + +Per-scenario results; empty until the generation is measured. + +##### compositeMean + +> **compositeMean**: `number` + +Defined in: [otel-export.ts:677](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L677) + +Mean composite score (0 when unmeasured — pair with labels.measured). + +##### costUsd + +> **costUsd**: `number` + +Defined in: [otel-export.ts:678](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L678) + +##### durationMs + +> **durationMs**: `number` + +Defined in: [otel-export.ts:679](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L679) + +*** + +### EvalRunEvent + +Defined in: [otel-export.ts:682](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L682) + +#### Properties + +##### runId + +> **runId**: `string` + +Defined in: [otel-export.ts:683](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L683) + +##### runDir + +> **runDir**: `string` + +Defined in: [otel-export.ts:684](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L684) + +##### timestamp + +> **timestamp**: `string` + +Defined in: [otel-export.ts:686](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L686) + +ISO timestamp. + +##### status + +> **status**: `"started"` \| `"baseline-complete"` \| `"generation-complete"` \| `"gate-decided"` \| `"finished"` \| `"errored"` + +Defined in: [otel-export.ts:687](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L687) + +##### labels? + +> `optional` **labels?**: `Record`\<`string`, `string`\> + +Defined in: [otel-export.ts:694](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L694) + +##### baseline? + +> `optional` **baseline?**: [`EvalRunGeneration`](#evalrungeneration) + +Defined in: [otel-export.ts:695](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L695) + +##### generations? + +> `optional` **generations?**: [`EvalRunGeneration`](#evalrungeneration)[] + +Defined in: [otel-export.ts:696](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L696) + +##### gateDecision? + +> `optional` **gateDecision?**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` + +Defined in: [otel-export.ts:697](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L697) + +##### holdoutLift? + +> `optional` **holdoutLift?**: `number` + +Defined in: [otel-export.ts:698](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L698) + +##### totalCostUsd + +> **totalCostUsd**: `number` + +Defined in: [otel-export.ts:699](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L699) + +##### totalDurationMs + +> **totalDurationMs**: `number` + +Defined in: [otel-export.ts:700](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L700) + +##### errorMessage? + +> `optional` **errorMessage?**: `string` + +Defined in: [otel-export.ts:701](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L701) + +*** + +### EvalRunsExportConfig + +Defined in: [otel-export.ts:704](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L704) + +#### Properties + +##### apiKey? + +> `optional` **apiKey?**: `string` + +Defined in: [otel-export.ts:706](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L706) + +Bearer key — tenant is resolved server-side from it. Reads TANGLE_API_KEY. + +##### base? + +> `optional` **base?**: `string` + +Defined in: [otel-export.ts:708](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L708) + +Intelligence base. Reads TANGLE_INTELLIGENCE_URL env, else prod. + +##### idempotencyKey? + +> `optional` **idempotencyKey?**: `string` + +Defined in: [otel-export.ts:710](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L710) + +Idempotency-Key header (e.g. the runId) — safe retries + upsert. + +*** + +### EvalRunsExportResult + +Defined in: [otel-export.ts:713](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L713) + +#### Properties + +##### ok + +> **ok**: `boolean` + +Defined in: [otel-export.ts:714](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L714) + +##### status + +> **status**: `number` + +Defined in: [otel-export.ts:715](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L715) + +##### accepted + +> **accepted**: `number` + +Defined in: [otel-export.ts:716](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L716) + +##### rejected + +> **rejected**: `object`[] + +Defined in: [otel-export.ts:717](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L717) + +###### index + +> **index**: `number` + +###### reason + +> **reason**: `string` + +*** + +### ResolveAgentBackendOptions + +Defined in: [resolve-agent-backend.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L51) + +#### Extends + +- `OpenAICompatPassthrough` + +#### Type Parameters + +##### TInput + +`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) + +#### Properties + +##### tools? + +> `optional` **tools?**: readonly [`OpenAIChatTool`](#openaichattool)[] + +Defined in: [backends.ts:222](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L222) + +OpenAI Chat Completions `tools[]` definitions surfaced to the model on +every request. Omit to send a tool-free request (existing behavior). +The runtime makes no assumption about the dispatcher — calls stream out +as `tool_call` events and the caller is responsible for executing them +and feeding `tool_result` messages back on a follow-up turn. + +###### Inherited from + +`OpenAICompatPassthrough.tools` + +##### toolChoice? + +> `optional` **toolChoice?**: [`OpenAIChatToolChoice`](#openaichattoolchoice) + +Defined in: [backends.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L228) + +OpenAI Chat Completions `tool_choice`. Default `undefined` (request +omits the field; provider falls back to its own default — typically +`'auto'`). + +###### Inherited from + +`OpenAICompatPassthrough.toolChoice` + +##### responseFormat? + +> `optional` **responseFormat?**: [`OpenAIChatResponseFormat`](#openaichatresponseformat) + +Defined in: [backends.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L232) + +OpenAI Chat Completions `response_format`. Omit for provider default text. + +###### Inherited from + +`OpenAICompatPassthrough.responseFormat` + +##### temperature? + +> `optional` **temperature?**: `number` + +Defined in: [backends.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L234) + +OpenAI Chat Completions `temperature`. Omit for provider default. + +###### Inherited from + +`OpenAICompatPassthrough.temperature` + +##### maxTokens? + +> `optional` **maxTokens?**: `number` + +Defined in: [backends.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L236) + +Maximum completion tokens, sent as OpenAI-compatible `max_tokens`. Omit for provider default. + +###### Inherited from + +`OpenAICompatPassthrough.maxTokens` + +##### fetchImpl? + +> `optional` **fetchImpl?**: (`input`, `init?`) => `Promise`\<`Response`\> + +Defined in: [backends.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L237) + +###### Parameters + +###### input + +`string` \| `URL` \| `Request` + +###### init? + +`RequestInit` + +###### Returns + +`Promise`\<`Response`\> + +###### Inherited from + +`OpenAICompatPassthrough.fetchImpl` + +##### retry? + +> `optional` **retry?**: `BackendRetryPolicy` + +Defined in: [backends.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L238) + +###### Inherited from + +`OpenAICompatPassthrough.retry` + +##### kind + +> **kind**: [`AgentBackendKind`](#agentbackendkind) + +Defined in: [resolve-agent-backend.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L54) + +The chat transport to resolve. + +##### apiKey + +> **apiKey**: `string` + +Defined in: [resolve-agent-backend.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L60) + +Bearer credential for the OpenAI-compat kinds. Empty string is valid for a +loopback-anonymous cli-bridge; a `router`/`tcloud` route with an empty key +is a caller bug the product surfaces before calling in. + +##### baseUrl + +> **baseUrl**: `string` + +Defined in: [resolve-agent-backend.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L62) + +Base URL for the OpenAI-compat kinds. cli-bridge's is its `/v1`. + +##### model + +> **model**: `string` + +Defined in: [resolve-agent-backend.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L64) + +Model id sent on every request. cli-bridge rejects a request without it. + +##### label? + +> `optional` **label?**: `string` + +Defined in: [resolve-agent-backend.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L66) + +`kind` label stamped on the resolved backend + its traces. Defaults to `kind`. + +##### sandboxBackend? + +> `optional` **sandboxBackend?**: () => [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> + +Defined in: [resolve-agent-backend.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L72) + +`sandbox` kind: the product's own domain backend. Required for that kind — +the substrate owns no product sandbox shape, so a `sandbox` resolution with +no seam is a caller bug, not a silent fallback. + +###### Returns + +[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> + +*** + +### RuntimeHookEvent + +Defined in: [runtime-hooks.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L36) + +#### Type Parameters + +##### Payload + +`Payload` = `unknown` + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [runtime-hooks.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L37) + +##### runId + +> **runId**: `string` + +Defined in: [runtime-hooks.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L38) + +##### scenarioId? + +> `optional` **scenarioId?**: `string` + +Defined in: [runtime-hooks.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L39) + +##### target + +> **target**: [`RuntimeHookTarget`](#runtimehooktarget) + +Defined in: [runtime-hooks.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L40) + +##### phase + +> **phase**: [`RuntimeHookPhase`](#runtimehookphase) + +Defined in: [runtime-hooks.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L41) + +##### timestamp + +> **timestamp**: `number` + +Defined in: [runtime-hooks.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L42) + +##### stepIndex? + +> `optional` **stepIndex?**: `number` + +Defined in: [runtime-hooks.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L43) + +##### parentId? + +> `optional` **parentId?**: `string` + +Defined in: [runtime-hooks.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L44) + +##### payload? + +> `optional` **payload?**: `Payload` + +Defined in: [runtime-hooks.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L45) + +##### metadata? + +> `optional` **metadata?**: `Record`\<`string`, `unknown`\> + +Defined in: [runtime-hooks.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L46) + +*** + +### RuntimeHookContext + +Defined in: [runtime-hooks.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L49) + +#### Properties + +##### signal? + +> `optional` **signal?**: `AbortSignal` + +Defined in: [runtime-hooks.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L50) + +*** + +### RuntimeDecisionEvidenceRef + +Defined in: [runtime-hooks.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L53) + +#### Properties + +##### source + +> **source**: `string` + +Defined in: [runtime-hooks.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L54) + +##### id + +> **id**: `string` + +Defined in: [runtime-hooks.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L55) + +##### detail? + +> `optional` **detail?**: `string` + +Defined in: [runtime-hooks.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L56) + +##### metadata? + +> `optional` **metadata?**: `Record`\<`string`, `unknown`\> + +Defined in: [runtime-hooks.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L57) + +*** + +### RuntimeDecisionPoint + +Defined in: [runtime-hooks.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L60) + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [runtime-hooks.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L61) + +##### runId + +> **runId**: `string` + +Defined in: [runtime-hooks.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L62) + +##### scenarioId? + +> `optional` **scenarioId?**: `string` + +Defined in: [runtime-hooks.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L63) + +##### stepIndex + +> **stepIndex**: `number` + +Defined in: [runtime-hooks.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L64) + +##### kind + +> **kind**: [`RuntimeDecisionKind`](#runtimedecisionkind) + +Defined in: [runtime-hooks.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L65) + +##### candidateActions + +> **candidateActions**: `string`[] + +Defined in: [runtime-hooks.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L66) + +##### context? + +> `optional` **context?**: `string` + +Defined in: [runtime-hooks.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L67) + +##### evidence + +> **evidence**: [`RuntimeDecisionEvidenceRef`](#runtimedecisionevidenceref)[] + +Defined in: [runtime-hooks.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L68) + +##### metadata? + +> `optional` **metadata?**: `Record`\<`string`, `unknown`\> + +Defined in: [runtime-hooks.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L69) + +*** + +### RuntimeHookErrorContext + +Defined in: [runtime-hooks.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L72) + +#### Properties + +##### hook + +> **hook**: `"onEvent"` \| `"onDecisionPoint"` + +Defined in: [runtime-hooks.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L73) + +##### eventId? + +> `optional` **eventId?**: `string` + +Defined in: [runtime-hooks.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L74) + +##### target? + +> `optional` **target?**: [`RuntimeHookTarget`](#runtimehooktarget) + +Defined in: [runtime-hooks.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L75) + +##### phase? + +> `optional` **phase?**: [`RuntimeHookPhase`](#runtimehookphase) + +Defined in: [runtime-hooks.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L76) + +##### decisionId? + +> `optional` **decisionId?**: `string` + +Defined in: [runtime-hooks.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L77) + +##### decisionKind? + +> `optional` **decisionKind?**: [`RuntimeDecisionKind`](#runtimedecisionkind) + +Defined in: [runtime-hooks.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L78) + +*** + +### RuntimeHooks + +Defined in: [runtime-hooks.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L88) + +The observation seam attached to a running loop (never to the portable genome). +Implement the optional hooks to receive lifecycle events, semantic decision points, +and hook errors. Author with [defineRuntimeHooks](#defineruntimehooks) for inference, and attach N +observers at once with [composeRuntimeHooks](#composeruntimehooks) — there is ONE event stream, not a +callback-prop zoo. + +#### Properties + +##### onEvent? + +> `optional` **onEvent?**: (`event`, `context`) => `void` \| `Promise`\<`void`\> + +Defined in: [runtime-hooks.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L94) + +General before/after/event hook. Use this for telemetry, memory capture, +policy wrapping, child lifecycle observers, or product-specific extension +points. + +###### Parameters + +###### event + +[`RuntimeHookEvent`](#runtimehookevent) + +###### context + +[`RuntimeHookContext`](#runtimehookcontext) + +###### Returns + +`void` \| `Promise`\<`void`\> + +##### onDecisionPoint? + +> `optional` **onDecisionPoint?**: (`point`, `context`) => `void` \| `Promise`\<`void`\> + +Defined in: [runtime-hooks.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L99) + +Semantic decision hook. Belief-state evaluation consumes this, but runtime +code should keep emitting ordinary lifecycle events as the base layer. + +###### Parameters + +###### point + +[`RuntimeDecisionPoint`](#runtimedecisionpoint) + +###### context + +[`RuntimeHookContext`](#runtimehookcontext) + +###### Returns + +`void` \| `Promise`\<`void`\> + +##### onHookError? + +> `optional` **onHookError?**: (`error`, `context`) => `void` \| `Promise`\<`void`\> + +Defined in: [runtime-hooks.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L103) + +###### Parameters + +###### error + +`Error` + +###### context + +[`RuntimeHookErrorContext`](#runtimehookerrorcontext) + +###### Returns + +`void` \| `Promise`\<`void`\> + +*** + +### RuntimeRunRow + +Defined in: [runtime-run.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L61) + +#### Stable + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [runtime-run.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L63) + +Stable runtime-side identifier. Adapters may translate to their own primary key. + +##### workspaceId + +> **workspaceId**: `string` + +Defined in: [runtime-run.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L64) + +##### sessionId? + +> `optional` **sessionId?**: `string` + +Defined in: [runtime-run.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L65) + +##### agentId? + +> `optional` **agentId?**: `string` + +Defined in: [runtime-run.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L66) + +##### domain? + +> `optional` **domain?**: `string` + +Defined in: [runtime-run.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L67) + +##### taskId + +> **taskId**: `string` + +Defined in: [runtime-run.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L68) + +##### scenarioId? + +> `optional` **scenarioId?**: `string` + +Defined in: [runtime-run.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L69) + +##### status + +> **status**: `RuntimeRunStatus` + +Defined in: [runtime-run.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L70) + +##### resultSummary? + +> `optional` **resultSummary?**: `string` + +Defined in: [runtime-run.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L71) + +##### error? + +> `optional` **error?**: `string` + +Defined in: [runtime-run.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L72) + +##### cost + +> **cost**: `RuntimeRunCost` + +Defined in: [runtime-run.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L73) + +##### startedAt + +> **startedAt**: `string` + +Defined in: [runtime-run.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L74) + +##### completedAt? + +> `optional` **completedAt?**: `string` + +Defined in: [runtime-run.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L75) + +##### metadata? + +> `optional` **metadata?**: `Record`\<`string`, `unknown`\> + +Defined in: [runtime-run.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L76) + +*** + +### RuntimeRunPersistenceAdapter + +Defined in: [runtime-run.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L80) + +#### Stable + +#### Methods + +##### upsert() + +> **upsert**(`row`): `void` \| `Promise`\<`void`\> + +Defined in: [runtime-run.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L88) + +Called once when `handle.persist()` runs. Implementations write `row` to +their durable store (D1, postgres, KV) and return whatever the consumer +wants the caller to see (often the storage-side row id). Errors thrown +here propagate out of `persist()` so the caller can decide whether to +retry or log-and-continue. + +###### Parameters + +###### row + +[`RuntimeRunRow`](#runtimerunrow) + +###### Returns + +`void` \| `Promise`\<`void`\> + +*** + +### RuntimeRunHandle + +Defined in: [runtime-run.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L107) + +#### Stable + +#### Properties + +##### id + +> `readonly` **id**: `string` + +Defined in: [runtime-run.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L109) + +Stable id assigned at start. + +##### workspaceId + +> `readonly` **workspaceId**: `string` + +Defined in: [runtime-run.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L110) + +##### sessionId + +> `readonly` **sessionId**: `string` \| `undefined` + +Defined in: [runtime-run.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L111) + +##### taskSpec + +> `readonly` **taskSpec**: [`AgentTaskSpec`](#agenttaskspec) + +Defined in: [runtime-run.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L112) + +##### status + +> `readonly` **status**: `RuntimeRunStatus` + +Defined in: [runtime-run.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L113) + +#### Methods + +##### observe() + +> **observe**(`event`): `void` + +Defined in: [runtime-run.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L120) + +Observe a single `RuntimeStreamEvent`. The handle ignores non-cost events +(text deltas, tool calls) silently so consumers can pipe the whole stream +through `handle.observe`. `llm_call` events update the ledger. + +###### Parameters + +###### event + +[`RuntimeStreamEvent`](#runtimestreamevent) + +###### Returns + +`void` + +##### cost() + +> **cost**(): `RuntimeRunCost` + +Defined in: [runtime-run.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L123) + +Snapshot of the current cost ledger. Safe to call at any time. + +###### Returns + +`RuntimeRunCost` + +##### complete() + +> **complete**(`input`): `void` + +Defined in: [runtime-run.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L130) + +Transition to a terminal state. Idempotent for the same status; throws +`RuntimeRunStateError` for a different terminal status (state machines +don't time-travel). + +###### Parameters + +###### input + +`RuntimeRunCompleteInput` + +###### Returns + +`void` + +##### toRow() + +> **toRow**(`metadata?`): [`RuntimeRunRow`](#runtimerunrow) + +Defined in: [runtime-run.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L133) + +Build the current row without writing it. Useful for tests + dry runs. + +###### Parameters + +###### metadata? + +`Record`\<`string`, `unknown`\> + +###### Returns + +[`RuntimeRunRow`](#runtimerunrow) + +##### persist() + +> **persist**(`metadata?`): `Promise`\<`void`\> + +Defined in: [runtime-run.ts:140](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L140) + +Persist the current row via the configured adapter. Must be called after +`complete()`. Idempotent for the same terminal state (the adapter sees +the same row on retry). + +###### Parameters + +###### metadata? + +`Record`\<`string`, `unknown`\> + +###### Returns + +`Promise`\<`void`\> + +*** + +### RuntimeTelemetryOptions + +Defined in: [sanitize.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L30) + +#### Stable + +#### Extended by + +- [`RuntimeEventOtelOptions`](#runtimeeventoteloptions) + +#### Properties + +##### includeInputs? + +> `optional` **includeInputs?**: `boolean` + +Defined in: [sanitize.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L35) + +Include raw task inputs. Off by default because task inputs often contain +customer facts, credentials, source text, or internal IDs. + +##### includeRequirementDescriptions? + +> `optional` **includeRequirementDescriptions?**: `boolean` + +Defined in: [sanitize.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L37) + +Include requirement descriptions. Secret requirements are always redacted. + +##### includeEvidenceIds? + +> `optional` **includeEvidenceIds?**: `boolean` + +Defined in: [sanitize.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L39) + +Include evidence IDs. Off by default; counts are safer for shared reports. + +##### includeUserAnswers? + +> `optional` **includeUserAnswers?**: `boolean` + +Defined in: [sanitize.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L41) + +Include user answers from question preflight. Off by default. + +##### includeControlPayloads? + +> `optional` **includeControlPayloads?**: `boolean` + +Defined in: [sanitize.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L43) + +Include action payloads and action results for control steps. Off by default. + +##### includeMetadata? + +> `optional` **includeMetadata?**: `boolean` + +Defined in: [sanitize.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L45) + +Include task metadata. Off by default because metadata may carry IDs or policy internals. + +##### includeEvalDetails? + +> `optional` **includeEvalDetails?**: `boolean` + +Defined in: [sanitize.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L47) + +Include eval detail/evidence strings. Off by default because validators may echo private input. + +*** + +### SanitizedKnowledgeReadinessReport + +Defined in: [sanitize.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L68) + +#### Stable + +#### Properties + +##### taskId + +> **taskId**: `string` + +Defined in: [sanitize.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L69) + +##### readinessScore + +> **readinessScore**: `number` + +Defined in: [sanitize.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L70) + +##### recommendedAction + +> **recommendedAction**: `KnowledgeRecommendedAction` + +Defined in: [sanitize.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L71) + +##### severity + +> **severity**: `ControlSeverity` + +Defined in: [sanitize.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L72) + +##### reason + +> **reason**: `string` + +Defined in: [sanitize.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L73) + +##### blockingMissingRequirements + +> **blockingMissingRequirements**: `SanitizedKnowledgeRequirement`[] + +Defined in: [sanitize.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L74) + +##### nonBlockingGaps + +> **nonBlockingGaps**: `SanitizedKnowledgeRequirement`[] + +Defined in: [sanitize.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L75) + +##### evidenceCount + +> **evidenceCount**: `number` + +Defined in: [sanitize.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L76) + +##### evidenceIds? + +> `optional` **evidenceIds?**: `string`[] + +Defined in: [sanitize.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L77) + +##### missingRequirementIds + +> **missingRequirementIds**: `string`[] + +Defined in: [sanitize.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L78) + +*** + +### RuntimeEventCollector + +Defined in: [sanitize.ts:493](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L493) + +#### Stable + +#### Type Parameters + +##### TState + +`TState` = `unknown` + +##### TAction + +`TAction` = `unknown` + +##### TActionResult + +`TActionResult` = `unknown` + +##### TEval + +`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` + +#### Properties + +##### onEvent + +> **onEvent**: (`event`) => `void` + +Defined in: [sanitize.ts:499](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L499) + +###### Parameters + +###### event + +[`AgentRuntimeEvent`](#agentruntimeevent)\<`TState`, `TAction`, `TActionResult`, `TEval`\> + +###### Returns + +`void` + +##### events + +> **events**: `Record`\<`string`, `unknown`\>[] + +Defined in: [sanitize.ts:500](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L500) + +*** + +### RuntimeStreamEventCollector + +Defined in: [sanitize.ts:523](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L523) + +#### Stable + +#### Properties + +##### onEvent + +> **onEvent**: `RuntimeStreamEventSink` + +Defined in: [sanitize.ts:524](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L524) + +##### events + +> **events**: `Record`\<`string`, `unknown`\>[] + +Defined in: [sanitize.ts:525](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L525) + +#### Methods + +##### summary() + +> **summary**(): `RuntimeStreamEventSummary` + +Defined in: [sanitize.ts:527](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L527) + +Snapshot of a small streaming-flavored summary derived from collected events. + +###### Returns + +`RuntimeStreamEventSummary` + +*** + +### ToolLoopCall + +Defined in: [tool-loop.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L22) + +#### Properties + +##### toolCallId? + +> `optional` **toolCallId?**: `string` + +Defined in: [tool-loop.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L23) + +##### toolName + +> **toolName**: `string` + +Defined in: [tool-loop.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L24) + +##### args + +> **args**: `Record`\<`string`, `unknown`\> + +Defined in: [tool-loop.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L25) + +*** + +### ToolLoopAssistantToolCall + +Defined in: [tool-loop.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L45) + +One OpenAI-shaped tool-call entry carried on an assistant message. + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [tool-loop.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L46) + +##### type + +> **type**: `"function"` + +Defined in: [tool-loop.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L47) + +##### function + +> **function**: `object` + +Defined in: [tool-loop.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L48) + +###### name + +> **name**: `string` + +###### arguments + +> **arguments**: `string` + +*** + +### ToolLoopResult + +Defined in: [tool-loop.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L120) + +#### Properties + +##### finalText + +> **finalText**: `string` + +Defined in: [tool-loop.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L121) + +##### toolResults + +> **toolResults**: `object`[] + +Defined in: [tool-loop.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L122) + +###### call + +> **call**: [`ToolLoopCall`](#toolloopcall) + +###### label + +> **label**: `string` + +###### outcome + +> **outcome**: [`ToolCallOutcome`](#toolcalloutcome) + +##### turns + +> **turns**: `number` + +Defined in: [tool-loop.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L123) + +##### stopReason + +> **stopReason**: [`ToolLoopStopReason`](#toolloopstopreason) + +Defined in: [tool-loop.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L124) + +##### ~~cappedOut~~ + +> **cappedOut**: `boolean` + +Defined in: [tool-loop.ts:126](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L126) + +###### Deprecated + +Use `stopReason !== 'completed'` instead. + +*** + +### RunToolLoopOptions + +Defined in: [tool-loop.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L129) + +#### Properties + +##### systemPrompt + +> **systemPrompt**: `string` + +Defined in: [tool-loop.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L130) + +##### userMessage + +> **userMessage**: `string` + +Defined in: [tool-loop.ts:131](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L131) + +##### priorMessages? + +> `optional` **priorMessages?**: [`ToolLoopMessage`](#toolloopmessage)[] + +Defined in: [tool-loop.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L132) + +##### streamTurn + +> **streamTurn**: (`messages`) => `AsyncIterable`\<[`ToolLoopEvent`](#toolloopevent)\> + +Defined in: [tool-loop.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L133) + +###### Parameters + +###### messages + +[`ToolLoopMessage`](#toolloopmessage)[] + +###### Returns + +`AsyncIterable`\<[`ToolLoopEvent`](#toolloopevent)\> + +##### executeToolCall + +> **executeToolCall**: (`call`) => `Promise`\<[`ToolCallOutcome`](#toolcalloutcome)\> + +Defined in: [tool-loop.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L134) + +###### Parameters + +###### call + +[`ToolLoopCall`](#toolloopcall) + +###### Returns + +`Promise`\<[`ToolCallOutcome`](#toolcalloutcome)\> + +##### isExecutableTool + +> **isExecutableTool**: (`toolName`) => `boolean` + +Defined in: [tool-loop.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L135) + +###### Parameters + +###### toolName + +`string` + +###### Returns + +`boolean` + +##### maxToolTurns? + +> `optional` **maxToolTurns?**: `number` + +Defined in: [tool-loop.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L138) + +Runaway-backstop cap. Default 200 — set far above any legitimate workflow. + For per-workflow limits, use `maxCostUsd` or `deadlineMs` instead. + +##### deadlineMs? + +> `optional` **deadlineMs?**: `number` + +Defined in: [tool-loop.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L141) + +Wall-clock deadline in ms since epoch (Date.now()-based). When exceeded the + loop stops with stopReason `deadline`. + +##### maxCostUsd? + +> `optional` **maxCostUsd?**: `number` + +Defined in: [tool-loop.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L143) + +Maximum total cost in USD. Requires `costOf` to meter each tool call. + +##### costOf? + +> `optional` **costOf?**: (`call`, `outcome`) => `number` + +Defined in: [tool-loop.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L145) + +Return the USD cost of one outcome. Required for `maxCostUsd` to work. + +###### Parameters + +###### call + +[`ToolLoopCall`](#toolloopcall) + +###### outcome + +[`ToolCallOutcome`](#toolcalloutcome) + +###### Returns + +`number` + +##### renderResult? + +> `optional` **renderResult?**: (`label`, `outcome`) => `string` + +Defined in: [tool-loop.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L146) + +###### Parameters + +###### label + +`string` + +###### outcome + +[`ToolCallOutcome`](#toolcalloutcome) + +###### Returns + +`string` + +##### labelFor? + +> `optional` **labelFor?**: (`call`) => `string` + +Defined in: [tool-loop.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L147) + +###### Parameters + +###### call + +[`ToolLoopCall`](#toolloopcall) + +###### Returns + +`string` + +##### runId? + +> `optional` **runId?**: `string` + +Defined in: [tool-loop.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L148) + +##### scenarioId? + +> `optional` **scenarioId?**: `string` + +Defined in: [tool-loop.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L149) + +##### hooks? + +> `optional` **hooks?**: [`RuntimeHooks`](#runtimehooks) + +Defined in: [tool-loop.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L150) + +*** + +### StreamToolLoopOptions + +Defined in: [tool-loop.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L309) + +#### Type Parameters + +##### Raw + +`Raw` + +#### Properties + +##### systemPrompt + +> **systemPrompt**: `string` + +Defined in: [tool-loop.ts:310](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L310) + +##### userMessage + +> **userMessage**: `string` + +Defined in: [tool-loop.ts:311](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L311) + +##### priorMessages? + +> `optional` **priorMessages?**: [`ToolLoopMessage`](#toolloopmessage)[] + +Defined in: [tool-loop.ts:312](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L312) + +##### streamTurn + +> **streamTurn**: (`messages`) => `AsyncIterable`\<`Raw`\> + +Defined in: [tool-loop.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L313) + +###### Parameters + +###### messages + +[`ToolLoopMessage`](#toolloopmessage)[] + +###### Returns + +`AsyncIterable`\<`Raw`\> + +##### extractText + +> **extractText**: (`event`) => `string` + +Defined in: [tool-loop.ts:314](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L314) + +###### Parameters + +###### event + +`Raw` + +###### Returns + +`string` + +##### extractToolCall + +> **extractToolCall**: (`event`) => [`ToolLoopCall`](#toolloopcall) \| `null` + +Defined in: [tool-loop.ts:315](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L315) + +###### Parameters + +###### event + +`Raw` + +###### Returns + +[`ToolLoopCall`](#toolloopcall) \| `null` + +##### isExecutableTool + +> **isExecutableTool**: (`toolName`) => `boolean` + +Defined in: [tool-loop.ts:316](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L316) + +###### Parameters + +###### toolName + +`string` + +###### Returns + +`boolean` + +##### executeToolCall + +> **executeToolCall**: (`call`) => `Promise`\<[`ToolCallOutcome`](#toolcalloutcome)\> + +Defined in: [tool-loop.ts:317](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L317) + +###### Parameters + +###### call + +[`ToolLoopCall`](#toolloopcall) + +###### Returns + +`Promise`\<[`ToolCallOutcome`](#toolcalloutcome)\> + +##### maxToolTurns? + +> `optional` **maxToolTurns?**: `number` + +Defined in: [tool-loop.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L319) + +Runaway-backstop cap. Default 200 — set far above any legitimate workflow. + +##### deadlineMs? + +> `optional` **deadlineMs?**: `number` + +Defined in: [tool-loop.ts:321](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L321) + +Wall-clock deadline in ms since epoch (Date.now()-based). + +##### maxCostUsd? + +> `optional` **maxCostUsd?**: `number` -###### Returns +Defined in: [tool-loop.ts:323](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L323) -`void` \| `Promise`\<`void`\> +Maximum total cost in USD. Requires `costOf` to meter each tool call. -##### onDecisionPoint? +##### costOf? -> `optional` **onDecisionPoint?**: (`point`, `context`) => `void` \| `Promise`\<`void`\> +> `optional` **costOf?**: (`call`, `outcome`) => `number` -Defined in: [runtime-hooks.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L99) +Defined in: [tool-loop.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L325) -Semantic decision hook. Belief-state evaluation consumes this, but runtime -code should keep emitting ordinary lifecycle events as the base layer. +Return the USD cost of one outcome. Required for `maxCostUsd` to work. ###### Parameters -###### point +###### call -[`RuntimeDecisionPoint`](#runtimedecisionpoint) +[`ToolLoopCall`](#toolloopcall) -###### context +###### outcome -[`RuntimeHookContext`](#runtimehookcontext) +[`ToolCallOutcome`](#toolcalloutcome) ###### Returns -`void` \| `Promise`\<`void`\> +`number` -##### onHookError? +##### renderResult? -> `optional` **onHookError?**: (`error`, `context`) => `void` \| `Promise`\<`void`\> +> `optional` **renderResult?**: (`label`, `outcome`) => `string` -Defined in: [runtime-hooks.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L103) +Defined in: [tool-loop.ts:326](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L326) ###### Parameters -###### error +###### label -`Error` +`string` -###### context +###### outcome -[`RuntimeHookErrorContext`](#runtimehookerrorcontext) +[`ToolCallOutcome`](#toolcalloutcome) ###### Returns -`void` \| `Promise`\<`void`\> - -*** - -### RuntimeRunRow - -Defined in: [runtime-run.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L61) - -#### Stable +`string` -#### Properties +##### labelFor? -##### id +> `optional` **labelFor?**: (`call`) => `string` -> **id**: `string` +Defined in: [tool-loop.ts:327](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L327) -Defined in: [runtime-run.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L63) +###### Parameters -Stable runtime-side identifier. Adapters may translate to their own primary key. +###### call -##### workspaceId +[`ToolLoopCall`](#toolloopcall) -> **workspaceId**: `string` +###### Returns -Defined in: [runtime-run.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L64) +`string` -##### sessionId? +##### runId? -> `optional` **sessionId?**: `string` +> `optional` **runId?**: `string` -Defined in: [runtime-run.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L65) +Defined in: [tool-loop.ts:328](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L328) -##### agentId? +##### scenarioId? -> `optional` **agentId?**: `string` +> `optional` **scenarioId?**: `string` -Defined in: [runtime-run.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L66) +Defined in: [tool-loop.ts:329](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L329) -##### domain? +##### hooks? -> `optional` **domain?**: `string` +> `optional` **hooks?**: [`RuntimeHooks`](#runtimehooks) -Defined in: [runtime-run.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L67) +Defined in: [tool-loop.ts:330](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L330) -##### taskId +*** -> **taskId**: `string` +### AgentTaskSpec -Defined in: [runtime-run.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L68) +Defined in: [types.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L27) -##### scenarioId? +#### Stable -> `optional` **scenarioId?**: `string` +#### Properties -Defined in: [runtime-run.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L69) +##### id -##### status +> **id**: `string` -> **status**: `RuntimeRunStatus` +Defined in: [types.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L28) -Defined in: [runtime-run.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L70) +##### intent -##### resultSummary? +> **intent**: `string` -> `optional` **resultSummary?**: `string` +Defined in: [types.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L29) -Defined in: [runtime-run.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L71) +##### domain? -##### error? +> `optional` **domain?**: `string` -> `optional` **error?**: `string` +Defined in: [types.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L31) -Defined in: [runtime-run.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L72) +Domain is metadata, not an architectural boundary: tax, legal, gtm, creative, blueprint, redteam, etc. -##### cost +##### inputs? -> **cost**: `RuntimeRunCost` +> `optional` **inputs?**: `Record`\<`string`, `unknown`\> -Defined in: [runtime-run.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L73) +Defined in: [types.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L32) -##### startedAt +##### requiredKnowledge? -> **startedAt**: `string` +> `optional` **requiredKnowledge?**: `KnowledgeRequirement`[] -Defined in: [runtime-run.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L74) +Defined in: [types.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L33) -##### completedAt? +##### budget? -> `optional` **completedAt?**: `string` +> `optional` **budget?**: `Partial`\<`ControlBudget`\> -Defined in: [runtime-run.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L75) +Defined in: [types.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L34) ##### metadata? > `optional` **metadata?**: `Record`\<`string`, `unknown`\> -Defined in: [runtime-run.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L76) +Defined in: [types.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L35) *** -### RuntimeRunPersistenceAdapter +### AgentKnowledgeProvider -Defined in: [runtime-run.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L80) +Defined in: [types.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L39) #### Stable #### Methods -##### upsert() - -> **upsert**(`row`): `void` \| `Promise`\<`void`\> +##### buildReadiness()? -Defined in: [runtime-run.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L88) +> `optional` **buildReadiness**(`task`): `KnowledgeReadinessReport` \| `Promise`\<`KnowledgeReadinessReport`\> -Called once when `handle.persist()` runs. Implementations write `row` to -their durable store (D1, postgres, KV) and return whatever the consumer -wants the caller to see (often the storage-side row id). Errors thrown -here propagate out of `persist()` so the caller can decide whether to -retry or log-and-continue. +Defined in: [types.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L40) ###### Parameters -###### row +###### task -[`RuntimeRunRow`](#runtimerunrow) +[`AgentTaskSpec`](#agenttaskspec) ###### Returns -`void` \| `Promise`\<`void`\> +`KnowledgeReadinessReport` \| `Promise`\<`KnowledgeReadinessReport`\> -*** +##### answerQuestions()? -### RuntimeRunHandle +> `optional` **answerQuestions**(`questions`, `task`): `Record`\<`string`, `string`\> \| `Promise`\<`Record`\<`string`, `string`\>\> -Defined in: [runtime-run.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L107) +Defined in: [types.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L41) -#### Stable +###### Parameters -#### Properties +###### questions -##### id +`UserQuestion`[] -> `readonly` **id**: `string` +###### task -Defined in: [runtime-run.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L109) +[`AgentTaskSpec`](#agenttaskspec) -Stable id assigned at start. +###### Returns -##### workspaceId +`Record`\<`string`, `string`\> \| `Promise`\<`Record`\<`string`, `string`\>\> -> `readonly` **workspaceId**: `string` +##### executeAcquisitionPlans()? -Defined in: [runtime-run.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L110) +> `optional` **executeAcquisitionPlans**(`plans`, `task`): `string`[] \| `Promise`\<`string`[]\> -##### sessionId +Defined in: [types.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L45) -> `readonly` **sessionId**: `string` \| `undefined` +###### Parameters -Defined in: [runtime-run.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L111) +###### plans -##### taskSpec +`DataAcquisitionPlan`[] -> `readonly` **taskSpec**: [`AgentTaskSpec`](#agenttaskspec) +###### task -Defined in: [runtime-run.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L112) +[`AgentTaskSpec`](#agenttaskspec) -##### status +###### Returns -> `readonly` **status**: `RuntimeRunStatus` +`string`[] \| `Promise`\<`string`[]\> -Defined in: [runtime-run.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L113) +##### refreshReadiness()? -#### Methods +> `optional` **refreshReadiness**(`input`): `KnowledgeReadinessReport` \| `Promise`\<`KnowledgeReadinessReport`\> -##### observe() +Defined in: [types.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L49) -> **observe**(`event`): `void` +###### Parameters -Defined in: [runtime-run.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L120) +###### input -Observe a single `RuntimeStreamEvent`. The handle ignores non-cost events -(text deltas, tool calls) silently so consumers can pipe the whole stream -through `handle.observe`. `llm_call` events update the ledger. +###### task -###### Parameters +[`AgentTaskSpec`](#agenttaskspec) -###### event +###### previous -[`RuntimeStreamEvent`](#runtimestreamevent) +`KnowledgeReadinessReport` + +###### userAnswers + +`Record`\<`string`, `string`\> + +###### acquiredEvidenceIds + +`string`[] ###### Returns -`void` +`KnowledgeReadinessReport` \| `Promise`\<`KnowledgeReadinessReport`\> -##### cost() +*** -> **cost**(): `RuntimeRunCost` +### AgentTaskContext -Defined in: [runtime-run.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L123) +Defined in: [types.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L58) -Snapshot of the current cost ledger. Safe to call at any time. +#### Stable -###### Returns +#### Type Parameters -`RuntimeRunCost` +##### TState -##### complete() +`TState` -> **complete**(`input`): `void` +##### TAction -Defined in: [runtime-run.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L130) +`TAction` -Transition to a terminal state. Idempotent for the same status; throws -`RuntimeRunStateError` for a different terminal status (state machines -don't time-travel). +##### TActionResult -###### Parameters +`TActionResult` -###### input +##### TEval -`RuntimeRunCompleteInput` +`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` -###### Returns +#### Properties + +##### task + +> **task**: [`AgentTaskSpec`](#agenttaskspec) -`void` +Defined in: [types.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L64) -##### toRow() +##### knowledge -> **toRow**(`metadata?`): [`RuntimeRunRow`](#runtimerunrow) +> **knowledge**: `KnowledgeReadinessReport` -Defined in: [runtime-run.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L133) +Defined in: [types.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L65) -Build the current row without writing it. Useful for tests + dry runs. +##### state -###### Parameters +> **state**: `TState` -###### metadata? +Defined in: [types.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L66) -`Record`\<`string`, `unknown`\> +##### evals -###### Returns +> **evals**: `TEval`[] -[`RuntimeRunRow`](#runtimerunrow) +Defined in: [types.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L67) -##### persist() +##### history -> **persist**(`metadata?`): `Promise`\<`void`\> +> **history**: `ControlStep`\<`TState`, `TAction`, `TActionResult`, `TEval`\>[] -Defined in: [runtime-run.ts:140](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-run.ts#L140) +Defined in: [types.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L68) -Persist the current row via the configured adapter. Must be called after -`complete()`. Idempotent for the same terminal state (the adapter sees -the same row on retry). +##### budget -###### Parameters +> **budget**: `ControlBudget` -###### metadata? +Defined in: [types.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L69) -`Record`\<`string`, `unknown`\> +##### stepIndex -###### Returns +> **stepIndex**: `number` -`Promise`\<`void`\> +Defined in: [types.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L70) -*** +##### wallMs -### RuntimeTelemetryOptions +> **wallMs**: `number` -Defined in: [sanitize.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L30) +Defined in: [types.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L71) -#### Stable +##### spentCostUsd -#### Properties +> **spentCostUsd**: `number` -##### includeInputs? +Defined in: [types.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L72) -> `optional` **includeInputs?**: `boolean` +##### remainingCostUsd? -Defined in: [sanitize.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L35) +> `optional` **remainingCostUsd?**: `number` -Include raw task inputs. Off by default because task inputs often contain -customer facts, credentials, source text, or internal IDs. +Defined in: [types.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L73) -##### includeRequirementDescriptions? +##### abortSignal -> `optional` **includeRequirementDescriptions?**: `boolean` +> **abortSignal**: `AbortSignal` -Defined in: [sanitize.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L37) +Defined in: [types.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L74) -Include requirement descriptions. Secret requirements are always redacted. +*** -##### includeEvidenceIds? +### AgentAdapter -> `optional` **includeEvidenceIds?**: `boolean` +Defined in: [types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L78) -Defined in: [sanitize.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L39) +#### Stable -Include evidence IDs. Off by default; counts are safer for shared reports. +#### Type Parameters -##### includeUserAnswers? +##### TState -> `optional` **includeUserAnswers?**: `boolean` +`TState` -Defined in: [sanitize.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L41) +##### TAction -Include user answers from question preflight. Off by default. +`TAction` -##### includeControlPayloads? +##### TActionResult -> `optional` **includeControlPayloads?**: `boolean` +`TActionResult` -Defined in: [sanitize.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L43) +##### TEval -Include action payloads and action results for control steps. Off by default. +`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` -##### includeMetadata? +#### Methods -> `optional` **includeMetadata?**: `boolean` +##### observe() -Defined in: [sanitize.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L45) +> **observe**(`ctx`): `TState` \| `Promise`\<`TState`\> -Include task metadata. Off by default because metadata may carry IDs or policy internals. +Defined in: [types.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L84) -##### includeEvalDetails? +###### Parameters -> `optional` **includeEvalDetails?**: `boolean` +###### ctx -Defined in: [sanitize.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L47) +###### task -Include eval detail/evidence strings. Off by default because validators may echo private input. +[`AgentTaskSpec`](#agenttaskspec) -*** +###### knowledge -### SanitizedKnowledgeReadinessReport +`KnowledgeReadinessReport` -Defined in: [sanitize.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L68) +###### history -#### Stable +`ControlStep`\<`TState`, `TAction`, `TActionResult`, `TEval`\>[] -#### Properties +###### abortSignal -##### taskId +`AbortSignal` -> **taskId**: `string` +###### Returns -Defined in: [sanitize.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L69) +`TState` \| `Promise`\<`TState`\> -##### readinessScore +##### validate() -> **readinessScore**: `number` +> **validate**(`ctx`): `TEval`[] \| `Promise`\<`TEval`[]\> -Defined in: [sanitize.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L70) +Defined in: [types.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L91) -##### recommendedAction +###### Parameters -> **recommendedAction**: `KnowledgeRecommendedAction` +###### ctx -Defined in: [sanitize.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L71) +###### task -##### severity +[`AgentTaskSpec`](#agenttaskspec) -> **severity**: `ControlSeverity` +###### knowledge -Defined in: [sanitize.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L72) +`KnowledgeReadinessReport` -##### reason +###### state -> **reason**: `string` +`TState` -Defined in: [sanitize.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L73) +###### history -##### blockingMissingRequirements +`ControlStep`\<`TState`, `TAction`, `TActionResult`, `TEval`\>[] -> **blockingMissingRequirements**: `SanitizedKnowledgeRequirement`[] +###### abortSignal -Defined in: [sanitize.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L74) +`AbortSignal` -##### nonBlockingGaps +###### Returns -> **nonBlockingGaps**: `SanitizedKnowledgeRequirement`[] +`TEval`[] \| `Promise`\<`TEval`[]\> -Defined in: [sanitize.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L75) +##### decide() -##### evidenceCount +> **decide**(`ctx`): `ControlDecision`\<`TAction`\> \| `Promise`\<`ControlDecision`\<`TAction`\>\> -> **evidenceCount**: `number` +Defined in: [types.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L99) -Defined in: [sanitize.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L76) +###### Parameters -##### evidenceIds? +###### ctx -> `optional` **evidenceIds?**: `string`[] +[`AgentTaskContext`](#agenttaskcontext)\<`TState`, `TAction`, `TActionResult`, `TEval`\> -Defined in: [sanitize.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L77) +###### Returns -##### missingRequirementIds +`ControlDecision`\<`TAction`\> \| `Promise`\<`ControlDecision`\<`TAction`\>\> -> **missingRequirementIds**: `string`[] +##### act() -Defined in: [sanitize.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L78) +> **act**(`action`, `ctx`): `TActionResult` \| `Promise`\<`TActionResult`\> -*** +Defined in: [types.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L103) -### RuntimeEventCollector +###### Parameters -Defined in: [sanitize.ts:493](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L493) +###### action -#### Stable +`TAction` -#### Type Parameters +###### ctx -##### TState +[`AgentTaskContext`](#agenttaskcontext)\<`TState`, `TAction`, `TActionResult`, `TEval`\> -`TState` = `unknown` +###### Returns -##### TAction +`TActionResult` \| `Promise`\<`TActionResult`\> -`TAction` = `unknown` +##### shouldStop()? -##### TActionResult +> `optional` **shouldStop**(`ctx`): `Promise`\<\{ `stop`: `boolean`; `pass`: `boolean`; `reason`: `string`; `score?`: `number`; \}\> \| \{ `stop`: `boolean`; `pass`: `boolean`; `reason`: `string`; `score?`: `number`; \} -`TActionResult` = `unknown` +Defined in: [types.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L108) -##### TEval +###### Parameters -`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` +###### ctx -#### Properties +[`AgentTaskContext`](#agenttaskcontext)\<`TState`, `TAction`, `TActionResult`, `TEval`\> -##### onEvent +###### Returns -> **onEvent**: (`event`) => `void` +`Promise`\<\{ `stop`: `boolean`; `pass`: `boolean`; `reason`: `string`; `score?`: `number`; \}\> \| \{ `stop`: `boolean`; `pass`: `boolean`; `reason`: `string`; `score?`: `number`; \} -Defined in: [sanitize.ts:499](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L499) +##### onKnowledgeBlocked()? -###### Parameters +> `optional` **onKnowledgeBlocked**(`ctx`): `ControlDecision`\<`TAction`\> \| `Promise`\<`ControlDecision`\<`TAction`\>\> -###### event +Defined in: [types.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L122) -[`AgentRuntimeEvent`](#agentruntimeevent)\<`TState`, `TAction`, `TActionResult`, `TEval`\> +###### Parameters -###### Returns +###### ctx -`void` +###### task -##### events +[`AgentTaskSpec`](#agenttaskspec) -> **events**: `Record`\<`string`, `unknown`\>[] +###### knowledge -Defined in: [sanitize.ts:500](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L500) +`KnowledgeReadinessReport` -*** +###### questions -### RuntimeStreamEventCollector +`UserQuestion`[] -Defined in: [sanitize.ts:523](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L523) +###### acquisitionPlans -#### Stable +`DataAcquisitionPlan`[] -#### Properties +###### Returns -##### onEvent +`ControlDecision`\<`TAction`\> \| `Promise`\<`ControlDecision`\<`TAction`\>\> -> **onEvent**: `RuntimeStreamEventSink` +##### getActionCostUsd()? -Defined in: [sanitize.ts:524](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L524) +> `optional` **getActionCostUsd**(`ctx`): `number` \| `undefined` -##### events +Defined in: [types.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L129) -> **events**: `Record`\<`string`, `unknown`\>[] +###### Parameters -Defined in: [sanitize.ts:525](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L525) +###### ctx -#### Methods +###### action -##### summary() +`TAction` -> **summary**(): `RuntimeStreamEventSummary` +###### result -Defined in: [sanitize.ts:527](https://github.com/tangle-network/agent-runtime/blob/main/src/sanitize.ts#L527) +`TActionResult` -Snapshot of a small streaming-flavored summary derived from collected events. +###### task -###### Returns +[`AgentTaskSpec`](#agenttaskspec) -`RuntimeStreamEventSummary` +###### state -*** +`TState` -### ToolLoopCall +###### evals -Defined in: [tool-loop.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L22) +`TEval`[] -#### Properties +###### history -##### toolCallId? +`ControlStep`\<`TState`, `TAction`, `TActionResult`, `TEval`\>[] -> `optional` **toolCallId?**: `string` +###### Returns -Defined in: [tool-loop.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L23) +`number` \| `undefined` -##### toolName +##### projectRunRecords()? -> **toolName**: `string` +> `optional` **projectRunRecords**(`result`, `task`): `RunRecord`[] -Defined in: [tool-loop.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L24) +Defined in: [types.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L138) -##### args +###### Parameters -> **args**: `Record`\<`string`, `unknown`\> +###### result -Defined in: [tool-loop.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L25) +`ControlRunResult`\<`TState`, `TAction`, `TActionResult`, `TEval`\> -*** +###### task -### ToolLoopAssistantToolCall +[`AgentTaskSpec`](#agenttaskspec) -Defined in: [tool-loop.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L45) +###### Returns -One OpenAI-shaped tool-call entry carried on an assistant message. +`RunRecord`[] -#### Properties +*** -##### id +### BackendErrorDetail -> **id**: `string` +Defined in: [types.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L212) -Defined in: [tool-loop.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L46) +Typed transport / backend failure detail. Carried on `backend_error` and +`final` events when the backend's stream throws or the upstream HTTP call +returns a non-success status. Lets consumers (a) distinguish "stream +completed with no text" from "stream never reached the model" and +(b) reconstruct the precise upstream signal (status + truncated body) when +building a `RunRecord.error`. -##### type +`body` is truncated to 2 KiB by the backend so an HTML error page from a +misconfigured proxy never bloats event payloads or logs. Consumers needing +the full body should inspect the underlying `BackendTransportError.body` +via a custom `mapEvent` or backend wrapper. -> **type**: `"function"` +#### Stable -Defined in: [tool-loop.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L47) +#### Properties -##### function +##### kind -> **function**: `object` +> **kind**: `"backend"` \| `"transport"` -Defined in: [tool-loop.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L48) +Defined in: [types.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L218) -###### name +`'transport'` — upstream HTTP / network failure with optional status code. +`'backend'` — the backend's `stream()` generator threw for a non-transport +reason (e.g. a custom adapter error, sandbox crash). -> **name**: `string` +##### message -###### arguments +> **message**: `string` -> **arguments**: `string` +Defined in: [types.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L219) -*** +##### status? -### ToolLoopResult +> `optional` **status?**: `number` -Defined in: [tool-loop.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L120) +Defined in: [types.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L221) -#### Properties +Upstream HTTP status when known. `0` for connection / abort errors. -##### finalText +##### body? -> **finalText**: `string` +> `optional` **body?**: `string` -Defined in: [tool-loop.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L121) +Defined in: [types.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L223) -##### toolResults +Truncated response body (≤2 KiB). Diagnostic only — never machine-parsed. -> **toolResults**: `object`[] +*** -Defined in: [tool-loop.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L122) +### OpenAIChatTool -###### call +Defined in: [types.ts:242](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L242) -> **call**: [`ToolLoopCall`](#toolloopcall) +OpenAI Chat Completions tool descriptor. The shape mirrors the +`/v1/chat/completions` `tools[]` parameter so callers can pass tool +definitions through `createOpenAICompatibleBackend({ tools })` without any +runtime translation. The router proxies this shape verbatim to Anthropic +(translated server-side), DeepSeek, Groq, OpenAI, and Gemini — every model +that the eval surface targets. -###### label +Callers that build their tool list from MCP servers should run a one-shot +MCP `tools/list` at config time and project the result into this shape. The +runtime intentionally does NOT depend on `@modelcontextprotocol/sdk` — +keeping the backend transport thin lets domain repos own MCP plumbing. -> **label**: `string` +#### Stable -###### outcome +#### Properties -> **outcome**: [`ToolCallOutcome`](#toolcalloutcome) +##### type -##### turns +> **type**: `"function"` -> **turns**: `number` +Defined in: [types.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L243) -Defined in: [tool-loop.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L123) +##### function -##### stopReason +> **function**: `object` -> **stopReason**: [`ToolLoopStopReason`](#toolloopstopreason) +Defined in: [types.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L244) -Defined in: [tool-loop.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L124) +###### name -##### ~~cappedOut~~ +> **name**: `string` -> **cappedOut**: `boolean` +###### description? -Defined in: [tool-loop.ts:126](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L126) +> `optional` **description?**: `string` -###### Deprecated +###### parameters? -Use `stopReason !== 'completed'` instead. +> `optional` **parameters?**: `Record`\<`string`, `unknown`\> *** -### RunToolLoopOptions +### RuntimeSessionStore -Defined in: [tool-loop.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L129) +Defined in: [types.ts:461](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L461) -#### Properties +#### Stable -##### systemPrompt +#### Methods -> **systemPrompt**: `string` +##### get() -Defined in: [tool-loop.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L130) +> **get**(`sessionId`): `RuntimeSession` \| `Promise`\<`RuntimeSession` \| `undefined`\> \| `undefined` -##### userMessage +Defined in: [types.ts:462](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L462) -> **userMessage**: `string` +###### Parameters -Defined in: [tool-loop.ts:131](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L131) +###### sessionId -##### priorMessages? +`string` -> `optional` **priorMessages?**: [`ToolLoopMessage`](#toolloopmessage)[] +###### Returns -Defined in: [tool-loop.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L132) +`RuntimeSession` \| `Promise`\<`RuntimeSession` \| `undefined`\> \| `undefined` -##### streamTurn +##### put() -> **streamTurn**: (`messages`) => `AsyncIterable`\<[`ToolLoopEvent`](#toolloopevent)\> +> **put**(`session`): `void` \| `Promise`\<`void`\> -Defined in: [tool-loop.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L133) +Defined in: [types.ts:463](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L463) ###### Parameters -###### messages +###### session -[`ToolLoopMessage`](#toolloopmessage)[] +`RuntimeSession` ###### Returns -`AsyncIterable`\<[`ToolLoopEvent`](#toolloopevent)\> +`void` \| `Promise`\<`void`\> -##### executeToolCall +##### appendEvent()? -> **executeToolCall**: (`call`) => `Promise`\<[`ToolCallOutcome`](#toolcalloutcome)\> +> `optional` **appendEvent**(`sessionId`, `event`): `void` \| `Promise`\<`void`\> -Defined in: [tool-loop.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L134) +Defined in: [types.ts:464](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L464) ###### Parameters -###### call +###### sessionId -[`ToolLoopCall`](#toolloopcall) +`string` + +###### event + +[`RuntimeStreamEvent`](#runtimestreamevent) ###### Returns -`Promise`\<[`ToolCallOutcome`](#toolcalloutcome)\> +`void` \| `Promise`\<`void`\> -##### isExecutableTool +##### listEvents()? -> **isExecutableTool**: (`toolName`) => `boolean` +> `optional` **listEvents**(`sessionId`): [`RuntimeStreamEvent`](#runtimestreamevent)[] \| `Promise`\<[`RuntimeStreamEvent`](#runtimestreamevent)[]\> -Defined in: [tool-loop.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L135) +Defined in: [types.ts:465](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L465) ###### Parameters -###### toolName +###### sessionId `string` ###### Returns -`boolean` - -##### maxToolTurns? - -> `optional` **maxToolTurns?**: `number` - -Defined in: [tool-loop.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L138) +[`RuntimeStreamEvent`](#runtimestreamevent)[] \| `Promise`\<[`RuntimeStreamEvent`](#runtimestreamevent)[]\> -Runaway-backstop cap. Default 200 — set far above any legitimate workflow. - For per-workflow limits, use `maxCostUsd` or `deadlineMs` instead. +*** -##### deadlineMs? +### AgentBackendInput -> `optional` **deadlineMs?**: `number` +Defined in: [types.ts:469](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L469) -Defined in: [tool-loop.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L141) +#### Stable -Wall-clock deadline in ms since epoch (Date.now()-based). When exceeded the - loop stops with stopReason `deadline`. +#### Properties -##### maxCostUsd? +##### task -> `optional` **maxCostUsd?**: `number` +> **task**: [`AgentTaskSpec`](#agenttaskspec) -Defined in: [tool-loop.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L143) +Defined in: [types.ts:470](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L470) -Maximum total cost in USD. Requires `costOf` to meter each tool call. +##### message? -##### costOf? +> `optional` **message?**: `string` -> `optional` **costOf?**: (`call`, `outcome`) => `number` +Defined in: [types.ts:471](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L471) -Defined in: [tool-loop.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L145) +##### messages? -Return the USD cost of one outcome. Required for `maxCostUsd` to work. +> `optional` **messages?**: `object`[] -###### Parameters +Defined in: [types.ts:472](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L472) -###### call +###### role -[`ToolLoopCall`](#toolloopcall) +> **role**: `string` -###### outcome +###### content -[`ToolCallOutcome`](#toolcalloutcome) +> **content**: `string` -###### Returns +##### inputs? -`number` +> `optional` **inputs?**: `Record`\<`string`, `unknown`\> -##### renderResult? +Defined in: [types.ts:473](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L473) -> `optional` **renderResult?**: (`label`, `outcome`) => `string` +*** -Defined in: [tool-loop.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L146) +### AgentBackendContext -###### Parameters +Defined in: [types.ts:477](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L477) -###### label +#### Stable -`string` +#### Properties -###### outcome +##### task -[`ToolCallOutcome`](#toolcalloutcome) +> **task**: [`AgentTaskSpec`](#agenttaskspec) -###### Returns +Defined in: [types.ts:478](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L478) -`string` +##### knowledge -##### labelFor? +> **knowledge**: `KnowledgeReadinessReport` -> `optional` **labelFor?**: (`call`) => `string` +Defined in: [types.ts:479](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L479) -Defined in: [tool-loop.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L147) +##### session -###### Parameters +> **session**: `RuntimeSession` -###### call +Defined in: [types.ts:480](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L480) -[`ToolLoopCall`](#toolloopcall) +##### signal? -###### Returns +> `optional` **signal?**: `AbortSignal` -`string` +Defined in: [types.ts:481](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L481) ##### runId? > `optional` **runId?**: `string` -Defined in: [tool-loop.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L148) - -##### scenarioId? - -> `optional` **scenarioId?**: `string` - -Defined in: [tool-loop.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L149) +Defined in: [types.ts:487](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L487) -##### hooks? +Conversation/run identifier when this call is part of a multi-agent run. +Backends should stamp it into any trace/log emission so cross-participant +events correlate. Absent when the call is a stand-alone `runAgentTask`. -> `optional` **hooks?**: [`RuntimeHooks`](#runtimehooks) +##### turnId? -Defined in: [tool-loop.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L150) +> `optional` **turnId?**: `string` -*** +Defined in: [types.ts:492](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L492) -### StreamToolLoopOptions +Deterministic turn id for this single call. Stable across retries of the +same logical turn so a caching gateway / idempotent backend can dedupe. -Defined in: [tool-loop.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L309) +##### parentTurnId? -#### Type Parameters +> `optional` **parentTurnId?**: `string` -##### Raw +Defined in: [types.ts:498](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L498) -`Raw` +If this call is itself nested inside a higher-order conversation +(recursion via `createConversationBackend`), the enclosing turn's id. +Used for trace stitching across nested orchestration. -#### Properties +##### propagatedHeaders? -##### systemPrompt +> `optional` **propagatedHeaders?**: `Readonly`\<`Record`\<`string`, `string`\>\> -> **systemPrompt**: `string` +Defined in: [types.ts:505](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L505) -Defined in: [tool-loop.ts:310](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L310) +Headers to forward verbatim to any outbound HTTP the backend issues: +`X-Tangle-Forwarded-Authorization`, `X-Tangle-Forwarded-Depth`, +run/turn correlation. Backends that issue HTTP MUST merge these into +the outbound request; backends that don't issue HTTP may ignore them. -##### userMessage +*** -> **userMessage**: `string` +### AgentExecutionBackend -Defined in: [tool-loop.ts:311](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L311) +Defined in: [types.ts:509](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L509) -##### priorMessages? +#### Stable -> `optional` **priorMessages?**: [`ToolLoopMessage`](#toolloopmessage)[] +#### Type Parameters -Defined in: [tool-loop.ts:312](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L312) +##### TInput -##### streamTurn +`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) -> **streamTurn**: (`messages`) => `AsyncIterable`\<`Raw`\> +#### Properties -Defined in: [tool-loop.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L313) +##### kind -###### Parameters +> **kind**: `string` -###### messages +Defined in: [types.ts:510](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L510) -[`ToolLoopMessage`](#toolloopmessage)[] +#### Methods -###### Returns +##### start()? -`AsyncIterable`\<`Raw`\> +> `optional` **start**(`input`, `context`): `RuntimeSession` \| `Promise`\<`RuntimeSession`\> -##### extractText +Defined in: [types.ts:511](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L511) -> **extractText**: (`event`) => `string` +###### Parameters -Defined in: [tool-loop.ts:314](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L314) +###### input -###### Parameters +`TInput` -###### event +###### context -`Raw` +`Omit`\<[`AgentBackendContext`](#agentbackendcontext), `"session"`\> & `object` ###### Returns -`string` +`RuntimeSession` \| `Promise`\<`RuntimeSession`\> -##### extractToolCall +##### resume()? -> **extractToolCall**: (`event`) => [`ToolLoopCall`](#toolloopcall) \| `null` +> `optional` **resume**(`session`, `input`, `context`): `RuntimeSession` \| `Promise`\<`RuntimeSession`\> -Defined in: [tool-loop.ts:315](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L315) +Defined in: [types.ts:515](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L515) ###### Parameters -###### event - -`Raw` - -###### Returns - -[`ToolLoopCall`](#toolloopcall) \| `null` - -##### isExecutableTool +###### session -> **isExecutableTool**: (`toolName`) => `boolean` +`RuntimeSession` -Defined in: [tool-loop.ts:316](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L316) +###### input -###### Parameters +`TInput` -###### toolName +###### context -`string` +`Omit`\<[`AgentBackendContext`](#agentbackendcontext), `"session"`\> ###### Returns -`boolean` +`RuntimeSession` \| `Promise`\<`RuntimeSession`\> -##### executeToolCall +##### stream() -> **executeToolCall**: (`call`) => `Promise`\<[`ToolCallOutcome`](#toolcalloutcome)\> +> **stream**(`input`, `context`): `AsyncIterable`\<[`RuntimeStreamEvent`](#runtimestreamevent)\> -Defined in: [tool-loop.ts:317](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L317) +Defined in: [types.ts:520](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L520) ###### Parameters -###### call +###### input -[`ToolLoopCall`](#toolloopcall) +`TInput` + +###### context + +[`AgentBackendContext`](#agentbackendcontext) ###### Returns -`Promise`\<[`ToolCallOutcome`](#toolcalloutcome)\> +`AsyncIterable`\<[`RuntimeStreamEvent`](#runtimestreamevent)\> -##### maxToolTurns? +##### stop()? -> `optional` **maxToolTurns?**: `number` +> `optional` **stop**(`session`, `reason`): `void` \| `Promise`\<`void`\> -Defined in: [tool-loop.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L319) +Defined in: [types.ts:521](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L521) -Runaway-backstop cap. Default 200 — set far above any legitimate workflow. +###### Parameters -##### deadlineMs? +###### session -> `optional` **deadlineMs?**: `number` +`RuntimeSession` -Defined in: [tool-loop.ts:321](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L321) +###### reason -Wall-clock deadline in ms since epoch (Date.now()-based). +`string` -##### maxCostUsd? +###### Returns -> `optional` **maxCostUsd?**: `number` +`void` \| `Promise`\<`void`\> -Defined in: [tool-loop.ts:323](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L323) +*** -Maximum total cost in USD. Requires `costOf` to meter each tool call. +### AgentTaskRunResult -##### costOf? +Defined in: [types.ts:557](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L557) -> `optional` **costOf?**: (`call`, `outcome`) => `number` +#### Stable -Defined in: [tool-loop.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L325) +#### Type Parameters -Return the USD cost of one outcome. Required for `maxCostUsd` to work. +##### TState -###### Parameters +`TState` -###### call +##### TAction -[`ToolLoopCall`](#toolloopcall) +`TAction` -###### outcome +##### TActionResult -[`ToolCallOutcome`](#toolcalloutcome) +`TActionResult` -###### Returns +##### TEval -`number` +`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` -##### renderResult? +#### Properties -> `optional` **renderResult?**: (`label`, `outcome`) => `string` +##### task -Defined in: [tool-loop.ts:326](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L326) +> **task**: [`AgentTaskSpec`](#agenttaskspec) -###### Parameters +Defined in: [types.ts:563](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L563) -###### label +##### status -`string` +> **status**: [`AgentTaskStatus`](#agenttaskstatus) -###### outcome +Defined in: [types.ts:564](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L564) -[`ToolCallOutcome`](#toolcalloutcome) +##### knowledge -###### Returns +> **knowledge**: `KnowledgeReadinessReport` -`string` +Defined in: [types.ts:565](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L565) -##### labelFor? +##### questions -> `optional` **labelFor?**: (`call`) => `string` +> **questions**: `UserQuestion`[] -Defined in: [tool-loop.ts:327](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L327) +Defined in: [types.ts:566](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L566) -###### Parameters +##### acquisitionPlans -###### call +> **acquisitionPlans**: `DataAcquisitionPlan`[] -[`ToolLoopCall`](#toolloopcall) +Defined in: [types.ts:567](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L567) -###### Returns +##### userAnswers -`string` +> **userAnswers**: `Record`\<`string`, `string`\> -##### runId? +Defined in: [types.ts:568](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L568) -> `optional` **runId?**: `string` +##### acquiredEvidenceIds -Defined in: [tool-loop.ts:328](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L328) +> **acquiredEvidenceIds**: `string`[] -##### scenarioId? +Defined in: [types.ts:569](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L569) -> `optional` **scenarioId?**: `string` +##### control -Defined in: [tool-loop.ts:329](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L329) +> **control**: `ControlRunResult`\<`TState`, `TAction`, `TActionResult`, `TEval`\> -##### hooks? +Defined in: [types.ts:570](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L570) -> `optional` **hooks?**: [`RuntimeHooks`](#runtimehooks) +##### runRecords -Defined in: [tool-loop.ts:330](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L330) +> **runRecords**: `RunRecord`[] -*** +Defined in: [types.ts:571](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L571) -### AgentTaskSpec +## Type Aliases -Defined in: [types.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L27) +### AgentCandidateProfileSource -#### Stable +> **AgentCandidateProfileSource** = \{ `kind`: `"profile"`; `profile`: `AgentProfile`; \} \| \{ `kind`: `"profile-diffs"`; `base`: `AgentProfile`; `diffs`: readonly `AgentProfileDiff`[]; \} \| \{ `kind`: `"candidate-profile"`; `profile`: `AgentCandidateProfile`; \} -#### Properties +Defined in: [candidate-execution/builder.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L26) -##### id +A complete profile that can be frozen without losing behavior. -> **id**: `string` +#### Union Members -Defined in: [types.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L28) +##### Type Literal -##### intent +\{ `kind`: `"profile"`; `profile`: `AgentProfile`; \} -> **intent**: `string` +*** -Defined in: [types.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L29) +##### Type Literal -##### domain? +\{ `kind`: `"profile-diffs"`; `base`: `AgentProfile`; `diffs`: readonly `AgentProfileDiff`[]; \} -> `optional` **domain?**: `string` +###### kind -Defined in: [types.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L31) +> **kind**: `"profile-diffs"` -Domain is metadata, not an architectural boundary: tax, legal, gtm, creative, blueprint, redteam, etc. +###### base -##### inputs? +> **base**: `AgentProfile` -> `optional` **inputs?**: `Record`\<`string`, `unknown`\> +###### diffs -Defined in: [types.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L32) +> **diffs**: readonly `AgentProfileDiff`[] -##### requiredKnowledge? +Applied in order. Each exact diff is content-addressed into lineage. -> `optional` **requiredKnowledge?**: `KnowledgeRequirement`[] +*** -Defined in: [types.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L33) +##### Type Literal -##### budget? +\{ `kind`: `"candidate-profile"`; `profile`: `AgentCandidateProfile`; \} -> `optional` **budget?**: `Partial`\<`ControlBudget`\> +###### kind -Defined in: [types.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L34) +> **kind**: `"candidate-profile"` -##### metadata? +###### profile -> `optional` **metadata?**: `Record`\<`string`, `unknown`\> +> **profile**: `AgentCandidateProfile` -Defined in: [types.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L35) +Already converted to the closed, secret-free candidate profile contract. *** -### AgentKnowledgeProvider - -Defined in: [types.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L39) +### AgentCandidateCodeSource -#### Stable +> **AgentCandidateCodeSource** = `AgentCandidateCodeDisabled` \| `AgentCandidateCodeNoOp` \| [`AgentCandidateCodeSurfaceSource`](#agentcandidatecodesurfacesource) -#### Methods +Defined in: [candidate-execution/builder.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L53) -##### buildReadiness()? +Explicit control/no-op code or one finalized CodeSurface whose bytes must still verify. -> `optional` **buildReadiness**(`task`): `KnowledgeReadinessReport` \| `Promise`\<`KnowledgeReadinessReport`\> +*** -Defined in: [types.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L40) +### AgentCandidateBundleInput -###### Parameters +> **AgentCandidateBundleInput** = `Omit`\<`AgentCandidateBundle`, `"digest"`\> -###### task +Defined in: [candidate-execution/bundle.ts:7](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/bundle.ts#L7) -[`AgentTaskSpec`](#agenttaskspec) +Exact candidate wire shape before the runtime computes its canonical digest. -###### Returns +*** -`KnowledgeReadinessReport` \| `Promise`\<`KnowledgeReadinessReport`\> +### AgentCandidateExecutionFailureClass -##### answerQuestions()? +> **AgentCandidateExecutionFailureClass** = `"pre-model-infrastructure"` \| `"execution"` \| `"post-model-infrastructure"` \| `"unknown"` -> `optional` **answerQuestions**(`questions`, `task`): `Record`\<`string`, `string`\> \| `Promise`\<`Record`\<`string`, `string`\>\> +Defined in: [candidate-execution/claim.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L78) -Defined in: [types.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L41) +Only the first class is retryable, and only when the closed model ledger has zero calls. -###### Parameters +*** -###### questions +### AgentCandidateExecutionTerminalResult -`UserQuestion`[] +> **AgentCandidateExecutionTerminalResult** = \{ `schemaVersion`: `1`; `status`: `"succeeded"`; `usage`: [`AgentCandidateExecutionUsage`](#agentcandidateexecutionusage); `modelSettlement`: `AgentCandidateArtifactRef`; `taskOutcome`: `AgentCandidateArtifactRef`; `benchmarkResult`: `AgentCandidateArtifactRef`; `runReceipt`: `AgentCandidateArtifactRef`; \} \| \{ `schemaVersion`: `1`; `status`: `"failed"`; `failureClass`: [`AgentCandidateExecutionFailureClass`](#agentcandidateexecutionfailureclass); `usage`: [`AgentCandidateExecutionUsage`](#agentcandidateexecutionusage); `modelSettlement`: `AgentCandidateArtifactRef`; `failureEvidence?`: `AgentCandidateArtifactRef`; \} -###### task +Defined in: [candidate-execution/claim.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L95) -[`AgentTaskSpec`](#agenttaskspec) +Evaluator-owned terminal facts staged durably before the terminal CAS. -###### Returns +*** -`Record`\<`string`, `string`\> \| `Promise`\<`Record`\<`string`, `string`\>\> +### AgentCandidateExecutionTerminalRecord -##### executeAcquisitionPlans()? +> **AgentCandidateExecutionTerminalRecord** = [`AgentCandidateExecutionTerminalResult`](#agentcandidateexecutionterminalresult) & `object` -> `optional` **executeAcquisitionPlans**(`plans`, `task`): `string`[] \| `Promise`\<`string`[]\> +Defined in: [candidate-execution/claim.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L115) -Defined in: [types.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L45) +Durable terminal record for one acquired execution attempt. -###### Parameters +#### Type Declaration -###### plans +##### executionId -`DataAcquisitionPlan`[] +> `readonly` **executionId**: `string` -###### task +##### attempt -[`AgentTaskSpec`](#agenttaskspec) +> `readonly` **attempt**: `number` -###### Returns +##### bundleDigest -`string`[] \| `Promise`\<`string`[]\> +> `readonly` **bundleDigest**: `Sha256Digest` -##### refreshReadiness()? +##### executionPlanDigest -> `optional` **refreshReadiness**(`input`): `KnowledgeReadinessReport` \| `Promise`\<`KnowledgeReadinessReport`\> +> `readonly` **executionPlanDigest**: `Sha256Digest` -Defined in: [types.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L49) +##### terminalDigest -###### Parameters +> `readonly` **terminalDigest**: `Sha256Digest` -###### input +RFC 8785 SHA-256 of this record with `terminalDigest` omitted. -###### task +*** -[`AgentTaskSpec`](#agenttaskspec) +### AgentCandidateExecutionPhase -###### previous +> **AgentCandidateExecutionPhase** = `"claimed"` \| `"candidate-may-run"` -`KnowledgeReadinessReport` +Defined in: [candidate-execution/claim.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L125) -###### userAnswers +Monotonic durable phase: the second value means candidate code could have started. -`Record`\<`string`, `string`\> +*** -###### acquiredEvidenceIds +### AgentCandidateExecutionClaimResult -`string`[] +> **AgentCandidateExecutionClaimResult** = \{ `acquired`: `true`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `lease`: [`AgentCandidateExecutionLease`](#agentcandidateexecutionlease); \} \| \{ `acquired`: `false`; `reason`: `"already-claimed"`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `exactReplay`: `boolean`; \} \| \{ `acquired`: `false`; `reason`: `"retry-not-eligible"`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `detail`: [`AgentCandidateRetryRejection`](#agentcandidateretryrejection); \} -###### Returns +Defined in: [candidate-execution/claim.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L165) -`KnowledgeReadinessReport` \| `Promise`\<`KnowledgeReadinessReport`\> +Result of atomically claiming one execution attempt. -*** +#### Union Members -### AgentTaskContext +##### Type Literal -Defined in: [types.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L58) +\{ `acquired`: `true`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `lease`: [`AgentCandidateExecutionLease`](#agentcandidateexecutionlease); \} -#### Stable +*** -#### Type Parameters +##### Type Literal -##### TState +\{ `acquired`: `false`; `reason`: `"already-claimed"`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `exactReplay`: `boolean`; \} -`TState` +###### acquired -##### TAction +> `readonly` **acquired**: `false` -`TAction` +###### reason -##### TActionResult +> `readonly` **reason**: `"already-claimed"` -`TActionResult` +###### claim -##### TEval +> `readonly` **claim**: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) -`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` +The durable winner already occupying this execution-attempt slot. -#### Properties +###### exactReplay -##### task +> `readonly` **exactReplay**: `boolean` -> **task**: [`AgentTaskSpec`](#agenttaskspec) +True only when every signed claim field matches the durable winner. -Defined in: [types.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L64) +*** -##### knowledge +##### Type Literal -> **knowledge**: `KnowledgeReadinessReport` +\{ `acquired`: `false`; `reason`: `"retry-not-eligible"`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `detail`: [`AgentCandidateRetryRejection`](#agentcandidateretryrejection); \} -Defined in: [types.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L65) +*** -##### state +### AgentCandidateExecutionFinishResult -> **state**: `TState` +> **AgentCandidateExecutionFinishResult** = \{ `finished`: `true`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); \} \| \{ `finished`: `false`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); `exactReplay`: `boolean`; \} -Defined in: [types.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L66) +Defined in: [candidate-execution/claim.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L187) -##### evals +Result of atomically recording an attempt's terminal facts. -> **evals**: `TEval`[] +#### Union Members -Defined in: [types.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L67) +##### Type Literal -##### history +\{ `finished`: `true`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); \} -> **history**: `ControlStep`\<`TState`, `TAction`, `TActionResult`, `TEval`\>[] +*** -Defined in: [types.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L68) +##### Type Literal -##### budget +\{ `finished`: `false`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); `exactReplay`: `boolean`; \} -> **budget**: `ControlBudget` +###### finished -Defined in: [types.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L69) +> `readonly` **finished**: `false` -##### stepIndex +###### terminal -> **stepIndex**: `number` +> `readonly` **terminal**: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord) -Defined in: [types.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L70) +###### exactReplay -##### wallMs +> `readonly` **exactReplay**: `boolean` -> **wallMs**: `number` +True when a repeated finish supplied the same terminal digest. -Defined in: [types.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L71) +*** -##### spentCostUsd +### AgentCandidateExecutionStageResult -> **spentCostUsd**: `number` +> **AgentCandidateExecutionStageResult** = \{ `staged`: `true`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); \} \| \{ `staged`: `false`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); `exactReplay`: `boolean`; \} -Defined in: [types.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L72) +Defined in: [candidate-execution/claim.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L200) -##### remainingCostUsd? +Result of durably staging the one immutable terminal outbox entry. -> `optional` **remainingCostUsd?**: `number` +*** -Defined in: [types.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L73) +### AgentCandidateExecutionPhaseResult -##### abortSignal +> **AgentCandidateExecutionPhaseResult** = \{ `marked`: `true`; `phase`: `"candidate-may-run"`; \} \| \{ `marked`: `false`; `phase`: `"candidate-may-run"`; \} -> **abortSignal**: `AbortSignal` +Defined in: [candidate-execution/claim.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L212) -Defined in: [types.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L74) +Result of crossing the irreversible candidate-may-run boundary. *** -### AgentAdapter +### AgentCandidateRetryRejection -Defined in: [types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L78) +> **AgentCandidateRetryRejection** = `"prior-attempt-missing"` \| `"prior-attempt-running"` \| `"prior-attempt-succeeded"` \| `"prior-attempt-spent-model-calls"` \| `"prior-attempt-not-pre-model-infrastructure"` \| `"retry-lineage-mismatch"` -#### Stable +Defined in: [candidate-execution/claim.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L216) -#### Type Parameters +*** -##### TState +### AgentCandidateModelGrantReserveInput -`TState` +> **AgentCandidateModelGrantReserveInput** = `Parameters`\<[`AgentCandidateModelPort`](#agentcandidatemodelport)\[`"reserveGrant"`\]\>\[`0`\] -##### TAction +Defined in: [candidate-execution/protected-model-port.ts:18](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L18) -`TAction` +*** -##### TActionResult +### AgentCandidateModelGrantActivateInput -`TActionResult` +> **AgentCandidateModelGrantActivateInput** = `Parameters`\<[`AgentCandidateModelPort`](#agentcandidatemodelport)\[`"activateGrant"`\]\>\[`0`\] -##### TEval +Defined in: [candidate-execution/protected-model-port.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L21) -`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` +*** -#### Methods +### AgentCandidateModelGrantSettleInput -##### observe() +> **AgentCandidateModelGrantSettleInput** = `Parameters`\<[`AgentCandidateModelPort`](#agentcandidatemodelport)\[`"settleGrant"`\]\>\[`0`\] -> **observe**(`ctx`): `TState` \| `Promise`\<`TState`\> +Defined in: [candidate-execution/protected-model-port.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L24) -Defined in: [types.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L84) +*** -###### Parameters +### AgentCandidateModelGrantReservation -###### ctx +> **AgentCandidateModelGrantReservation** = [`AgentCandidateProtectedModelReservation`](#agentcandidateprotectedmodelreservation) -###### task +Defined in: [candidate-execution/protected-model-port.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L29) -[`AgentTaskSpec`](#agenttaskspec) +Secret-free response from the service's reservation endpoint. -###### knowledge +*** -`KnowledgeReadinessReport` +### AgentCandidateOutputPurpose -###### history +> **AgentCandidateOutputPurpose** = `"candidate-workspace-manifest"` \| `"candidate-workspace-archive"` \| `"task-manifest"` \| `"task-archive"` \| `"task-patch"` \| `"task-outcome"` \| `"memory-after-manifest"` \| `"memory-after-archive"` \| `"grader-evidence"` \| `"benchmark-result"` \| `"model-settlement"` \| `"trace"` \| `"run-receipt"` \| `"failure-evidence"` -`ControlStep`\<`TState`, `TAction`, `TActionResult`, `TEval`\>[] +Defined in: [candidate-execution/types.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L38) -###### abortSignal +*** -`AbortSignal` +### AgentCandidateModelLimits -###### Returns +> **AgentCandidateModelLimits** = `Pick`\<`AgentCandidateExecutionLimits`, `"maxModelCalls"` \| `"maxInputTokens"` \| `"maxOutputTokens"` \| `"maxCostUsd"`\> -`TState` \| `Promise`\<`TState`\> +Defined in: [candidate-execution/types.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L151) -##### validate() +Limits mechanically enforced by the evaluator-owned model gateway. -> **validate**(`ctx`): `TEval`[] \| `Promise`\<`TEval`[]\> +*** -Defined in: [types.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L91) +### AgentCandidateRunFinalization -###### Parameters +> **AgentCandidateRunFinalization** = \{ `succeeded`: `true`; `receipt`: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateRunReceiptV2`\>; `artifacts`: \{ `modelSettlement`: `AgentCandidateArtifactRef`; `taskOutcome`: `AgentCandidateArtifactRef`; `benchmarkResult`: `AgentCandidateArtifactRef`; `runReceipt`: `AgentCandidateArtifactRef`; \}; \} \| \{ `succeeded`: `false`; `reason`: `string`; `partial`: \{ `executionId`: `string`; `bundleDigest`: `Sha256Digest`; `executionPlanDigest`: `Sha256Digest`; `materializationReceiptDigest`: `Sha256Digest`; `termination?`: `AgentCandidateTermination`; \}; `usage`: `AgentCandidateSpend` \| `null`; \} -###### ctx +Defined in: [candidate-execution/types.ts:535](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L535) -###### task +#### Union Members -[`AgentTaskSpec`](#agenttaskspec) +##### Type Literal -###### knowledge +\{ `succeeded`: `true`; `receipt`: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateRunReceiptV2`\>; `artifacts`: \{ `modelSettlement`: `AgentCandidateArtifactRef`; `taskOutcome`: `AgentCandidateArtifactRef`; `benchmarkResult`: `AgentCandidateArtifactRef`; `runReceipt`: `AgentCandidateArtifactRef`; \}; \} -`KnowledgeReadinessReport` +*** -###### state +##### Type Literal -`TState` +\{ `succeeded`: `false`; `reason`: `string`; `partial`: \{ `executionId`: `string`; `bundleDigest`: `Sha256Digest`; `executionPlanDigest`: `Sha256Digest`; `materializationReceiptDigest`: `Sha256Digest`; `termination?`: `AgentCandidateTermination`; \}; `usage`: `AgentCandidateSpend` \| `null`; \} -###### history +###### succeeded -`ControlStep`\<`TState`, `TAction`, `TActionResult`, `TEval`\>[] +> **succeeded**: `false` -###### abortSignal +###### reason -`AbortSignal` +> **reason**: `string` -###### Returns +###### partial -`TEval`[] \| `Promise`\<`TEval`[]\> +> **partial**: `object` -##### decide() +###### partial.executionId -> **decide**(`ctx`): `ControlDecision`\<`TAction`\> \| `Promise`\<`ControlDecision`\<`TAction`\>\> +> **executionId**: `string` -Defined in: [types.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L99) +###### partial.bundleDigest -###### Parameters +> **bundleDigest**: `Sha256Digest` -###### ctx +###### partial.executionPlanDigest -[`AgentTaskContext`](#agenttaskcontext)\<`TState`, `TAction`, `TActionResult`, `TEval`\> +> **executionPlanDigest**: `Sha256Digest` -###### Returns +###### partial.materializationReceiptDigest -`ControlDecision`\<`TAction`\> \| `Promise`\<`ControlDecision`\<`TAction`\>\> +> **materializationReceiptDigest**: `Sha256Digest` -##### act() +###### partial.termination? -> **act**(`action`, `ctx`): `TActionResult` \| `Promise`\<`TActionResult`\> +> `optional` **termination?**: `AgentCandidateTermination` -Defined in: [types.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L103) +###### usage -###### Parameters +> **usage**: `AgentCandidateSpend` \| `null` -###### action +Independent evaluator-gateway usage, even when execution or trace capture failed. -`TAction` +*** -###### ctx +### RetryableErrorPredicate -[`AgentTaskContext`](#agenttaskcontext)\<`TState`, `TAction`, `TActionResult`, `TEval`\> +> **RetryableErrorPredicate** = (`err`) => `boolean` -###### Returns +Defined in: [conversation/call-policy.ts:18](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L18) -`TActionResult` \| `Promise`\<`TActionResult`\> +Pure judgment of whether an error is worth retrying. Defaults: TimeoutError, AbortError, fetch-level network errors. -##### shouldStop()? +#### Parameters -> `optional` **shouldStop**(`ctx`): `Promise`\<\{ `stop`: `boolean`; `pass`: `boolean`; `reason`: `string`; `score?`: `number`; \}\> \| \{ `stop`: `boolean`; `pass`: `boolean`; `reason`: `string`; `score?`: `number`; \} +##### err -Defined in: [types.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L108) +`unknown` -###### Parameters +#### Returns -###### ctx +`boolean` -[`AgentTaskContext`](#agenttaskcontext)\<`TState`, `TAction`, `TActionResult`, `TEval`\> +*** -###### Returns +### RetryBackoff -`Promise`\<\{ `stop`: `boolean`; `pass`: `boolean`; `reason`: `string`; `score?`: `number`; \}\> \| \{ `stop`: `boolean`; `pass`: `boolean`; `reason`: `string`; `score?`: `number`; \} +> **RetryBackoff** = `number` \| ((`attempt`) => `number`) -##### onKnowledgeBlocked()? +Defined in: [conversation/call-policy.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L21) -> `optional` **onKnowledgeBlocked**(`ctx`): `ControlDecision`\<`TAction`\> \| `Promise`\<`ControlDecision`\<`TAction`\>\> +Backoff between attempts. Constant ms, or `(attempt: 1-indexed) => ms`. -Defined in: [types.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L122) +*** -###### Parameters +### ForwardHeaderName -###### ctx +> **ForwardHeaderName** = *typeof* [`FORWARD_HEADERS`](#forward_headers)\[keyof *typeof* [`FORWARD_HEADERS`](#forward_headers)\] -###### task +Defined in: [conversation/headers.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/headers.ts#L35) -[`AgentTaskSpec`](#agenttaskspec) +*** -###### knowledge +### PropagatedHeaders -`KnowledgeReadinessReport` +> **PropagatedHeaders** = `Readonly`\<`Record`\<`string`, `string`\>\> -###### questions +Defined in: [conversation/headers.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/headers.ts#L111) -`UserQuestion`[] +Header bag carried through `AgentBackendContext.propagatedHeaders` so +backends that opt in can merge them into their outbound HTTP requests. +Distinct from `buildForwardHeaders` so callers can attach extra +non-protocol headers (e.g. tracing) without colliding. -###### acquisitionPlans +*** -`DataAcquisitionPlan`[] +### PersonaDriver -###### Returns +> **PersonaDriver** = \{ `kind`: `"profile"`; `profile`: `AgentProfile`; \} \| \{ `kind`: `"scripted"`; `turns`: `string`[]; \} -`ControlDecision`\<`TAction`\> \| `Promise`\<`ControlDecision`\<`TAction`\>\> +Defined in: [conversation/run-persona.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L32) -##### getActionCostUsd()? +A persona that drives the conversation: either a full driver `AgentProfile` + (an LLM user-sim) or a deterministic script of user turns (the fast-path). -> `optional` **getActionCostUsd**(`ctx`): `number` \| `undefined` +*** -Defined in: [types.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L129) +### AuthSource -###### Parameters +> **AuthSource** = `"forward-user"` \| `"agent-owned"` \| ((`state`) => `"forward-user"` \| `"agent-owned"`) -###### ctx +Defined in: [conversation/types.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L68) -###### action +#### Stable -`TAction` +*** -###### result +### TurnOrder -`TActionResult` +> **TurnOrder** = `"alternate"` \| `"round-robin"` \| ((`state`) => `number`) -###### task +Defined in: [conversation/types.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L74) -[`AgentTaskSpec`](#agenttaskspec) +#### Stable -###### state +*** -`TState` +### HaltPredicate -###### evals +> **HaltPredicate** = (`ctx`) => `boolean` \| [`HaltSignal`](#haltsignal) \| `Promise`\<`boolean` \| [`HaltSignal`](#haltsignal)\> -`TEval`[] +Defined in: [conversation/types.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L95) -###### history +#### Parameters -`ControlStep`\<`TState`, `TAction`, `TActionResult`, `TEval`\>[] +##### ctx -###### Returns +[`HaltContext`](#haltcontext) -`number` \| `undefined` +#### Returns -##### projectRunRecords()? +`boolean` \| [`HaltSignal`](#haltsignal) \| `Promise`\<`boolean` \| [`HaltSignal`](#haltsignal)\> -> `optional` **projectRunRecords**(`result`, `task`): `RunRecord`[] +#### Stable -Defined in: [types.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L138) +*** -###### Parameters +### HaltReason -###### result +> **HaltReason** = \{ `kind`: `"max_turns"`; `turns`: `number`; \} \| \{ `kind`: `"max_credits"`; `spentCents`: `number`; `capCents`: `number`; \} \| \{ `kind`: `"predicate"`; `reason`: `string`; \} \| \{ `kind`: `"abort"`; \} \| \{ `kind`: `"participant_error"`; `participant`: `string`; `message`: `string`; \} -`ControlRunResult`\<`TState`, `TAction`, `TActionResult`, `TEval`\> +Defined in: [conversation/types.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L100) -###### task +#### Stable -[`AgentTaskSpec`](#agenttaskspec) +*** -###### Returns +### ConversationStreamEvent -`RunRecord`[] +> **ConversationStreamEvent** = \{ `type`: `"conversation_start"`; `runId`: `string`; `participants`: readonly `string`[]; `seed`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"conversation_resumed"`; `runId`: `string`; `participants`: readonly `string`[]; `transcript`: readonly [`ConversationTurn`](#conversationturn)[]; `timestamp`: `string`; \} \| \{ `type`: `"turn_start"`; `runId`: `string`; `index`: `number`; `speaker`: `string`; `turnId`: `string`; `attempt`: `number`; `timestamp`: `string`; \} \| \{ `type`: `"turn_text_delta"`; `runId`: `string`; `index`: `number`; `speaker`: `string`; `turnId`: `string`; `text`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"turn_retry"`; `runId`: `string`; `index`: `number`; `speaker`: `string`; `turnId`: `string`; `attempt`: `number`; `reason`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"turn_end"`; `runId`: `string`; `turn`: [`ConversationTurn`](#conversationturn); `timestamp`: `string`; \} \| \{ `type`: `"conversation_end"`; `runId`: `string`; `result`: [`ConversationResult`](#conversationresult); `timestamp`: `string`; \} -*** +Defined in: [conversation/types.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L237) -### BackendErrorDetail +#### Stable -Defined in: [types.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L212) +*** -Typed transport / backend failure detail. Carried on `backend_error` and -`final` events when the backend's stream throws or the upstream HTTP call -returns a non-success status. Lets consumers (a) distinguish "stream -completed with no text" from "stream never reached the model" and -(b) reconstruct the precise upstream signal (status + truncated body) when -building a `RunRecord.error`. +### Verifier -`body` is truncated to 2 KiB by the backend so an HTML error page from a -misconfigured proxy never bloats event payloads or logs. Consumers needing -the full body should inspect the underlying `BackendTransportError.body` -via a custom `mapEvent` or backend wrapper. +> **Verifier** = (`worktreePath`) => `Promise`\<[`VerifyResult`](#verifyresult)\> \| [`VerifyResult`](#verifyresult) -#### Stable +Defined in: [improvement/agentic-generator.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L81) -#### Properties +Verifies the edited worktree. Sync or async; throws only on a setup fault + (a candidate that fails verification returns `{ok:false}`, it does not + throw). -##### kind +#### Parameters -> **kind**: `"backend"` \| `"transport"` +##### worktreePath -Defined in: [types.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L218) +`string` -`'transport'` — upstream HTTP / network failure with optional status code. -`'backend'` — the backend's `stream()` generator threw for a non-transport -reason (e.g. a custom adapter error, sandbox crash). +#### Returns -##### message +`Promise`\<[`VerifyResult`](#verifyresult)\> \| [`VerifyResult`](#verifyresult) -> **message**: `string` +*** -Defined in: [types.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L219) +### AgenticGeneratorShotExecution -##### status? +> **AgenticGeneratorShotExecution** = `Readonly`\<`Omit`\<[`LocalHarnessResult`](mcp.md#localharnessresult), `"usage"` \| `"evidence"`\> & `object`\> -> `optional` **status?**: `number` +Defined in: [improvement/agentic-generator.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L123) -Defined in: [types.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L221) +Frozen exact harness result for an author shot: full streams, process state, + token usage, and execution-policy evidence. + The `onShotCompleted` callback receives `null` when execution failed before + the harness returned. -Upstream HTTP status when known. `0` for connection / abort errors. +*** -##### body? +### AgenticGeneratorShotDisposition -> `optional` **body?**: `string` +> **AgenticGeneratorShotDisposition** = \{ `kind`: `"clean"`; `worktreePath`: `string`; \} \| \{ `kind`: `"rejected"`; `worktreePath`: `string`; `stage`: `"raw-trace-evidence"` \| `"verification"`; `feedback`: `string` \| `null`; \} \| \{ `kind`: `"accepted"`; `worktreePath`: `string`; `verified`: `boolean`; \} \| \{ `kind`: `"setup-error"`; `worktreePath`: `string`; `stage`: `"worktree-inspection"` \| `"raw-trace-evidence"` \| `"verification"`; `error`: \{ `name`: `string`; `message`: `string`; \}; \} -Defined in: [types.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L223) +Defined in: [improvement/agentic-generator.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L136) -Truncated response body (≤2 KiB). Diagnostic only — never machine-parsed. +Worktree decision emitted before a completed shot is retried, accepted, or + discarded. The callback runs while `worktreePath` is still available, so + callers can persist the exact diff. *** -### OpenAIChatTool +### ImproveSurface -Defined in: [types.ts:242](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L242) +> **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"subagents"` \| `"agent-profile"` \| `"memory"` \| `"code"` -OpenAI Chat Completions tool descriptor. The shape mirrors the -`/v1/chat/completions` `tools[]` parameter so callers can pass tool -definitions through `createOpenAICompatibleBackend({ tools })` without any -runtime translation. The router proxies this shape verbatim to Anthropic -(translated server-side), DeepSeek, Groq, OpenAI, and Gemini — every model -that the eval surface targets. +Defined in: [improvement/improve.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L69) -Callers that build their tool list from MCP servers should run a one-shot -MCP `tools/list` at config time and project the result into this shape. The -runtime intentionally does NOT depend on `@modelcontextprotocol/sdk` — -keeping the backend transport thin lets domain repos own MCP plumbing. +The executable agent lever `improve` optimizes. Profile fields remain + portable AgentProfile coordinates; implementation and orchestration files + use the code surface so a winner can be sealed into an exact candidate. -#### Stable +*** -#### Properties +### ImproveOptions -##### type +> **ImproveOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`SelfImproveOptions`\<`TScenario`, `TArtifact`\>, `"analyzeGeneration"` \| `"baselineSurface"` \| `"findings"` \| `"gate"` \| `"proposer"`\> & `object` -> **type**: `"function"` +Defined in: [improvement/improve.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L80) -Defined in: [types.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L243) +#### Type Declaration -##### function +##### surface? -> **function**: `object` +> `optional` **surface?**: [`ImproveSurface`](#improvesurface) -Defined in: [types.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L244) +Which profile lever to optimize. Default `'prompt'`. Selects the default + generator + the baseline-surface extraction shape. -###### name +##### generator? -> **name**: `string` +> `optional` **generator?**: `SurfaceProposer` -###### description? +The `SurfaceProposer` that mutates a profile surface. When unset, the facade + picks the default for prompt, skills, and memory; surfaces + with no default REQUIRE this (fail-loud otherwise). Forbidden for code; + use `code.generator` so the runtime owns candidate cleanup. -> `optional` **description?**: `string` +##### gate? -###### parameters? +> `optional` **gate?**: `"holdout"` \| `"none"` -> `optional` **parameters?**: `Record`\<`string`, `unknown`\> +Gate mode. `'holdout'` (default) runs the held-out promotion gate; + `'none'` is a baseline-only run (`budget.generations = 0`). -*** +##### allowedModels? -### RuntimeSessionStore +> `optional` **allowedModels?**: readonly `string`[] -Defined in: [types.ts:461](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L461) +Restrict the run to this subset of models. When set, the reflection model + (`llm.model`, or the default when unset) must be a member, or `improve()` throws + a `ConfigError` before the generator is built. Unset = unrestricted. -#### Stable +##### analyzeGeneration? -#### Methods +> `optional` **analyzeGeneration?**: `SelfImproveOptions`\<`TScenario`, `TArtifact`\>\[`"analyzeGeneration"`\] \| `null` -##### get() +Per-generation findings producer passthrough (see selfImprove.analyzeGeneration). + DEFAULT: the built-in failure distiller — after each generation it turns the + worst-scoring/errored cells into structured findings ({ scenario, composite, + notes, error }) for the NEXT proposal round, so the proposer reasons over what + actually failed instead of a static seed. Pass your own producer (e.g. a + trace-analyst over the runDir's traces) to replace it; pass `null` to disable + and keep the static `findings` all the way through. -> **get**(`sessionId`): `RuntimeSession` \| `Promise`\<`RuntimeSession` \| `undefined`\> \| `undefined` +##### rawTraceContext? -Defined in: [types.ts:462](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L462) +> `optional` **rawTraceContext?**: `boolean` -###### Parameters +META-HARNESS mode: instead of the ~400-char distilled findings, feed the + proposer RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's + real run traces under `runDir` (per-cell `spans.jsonl` event logs + + `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose + instruction — so the coding agent reads the actual failures itself rather than + a pre-summary. Requires a REAL `runDir` (that is where the traces live). + Ignored when `analyzeGeneration` is set explicitly (that wins) or is `null` + (disabled). Equivalent to `analyzeGeneration: rawTraceDistiller()`; this flag + is the one-line enable. Default `false` (the distiller stays the default). -###### sessionId +##### code? -`string` +> `optional` **code?**: [`ImproveCodeOptions`](#improvecodeoptions) -###### Returns +CODE-surface wiring: name `surface: 'code'`, point at a repo, and the + facade assembles the whole candidate pipeline — an isolated incumbent plus git worktrees + (`gitWorktreeAdapter`) driven by `improvementDriver` with the full agentic + generator (a real coding harness edits each candidate worktree; a `verify` + hook gates candidates before they are ever measured). Ignored when + `opts.generator` is supplied. Required for every code run because a real + repository and base ref are necessary to measure the incumbent. -`RuntimeSession` \| `Promise`\<`RuntimeSession` \| `undefined`\> \| `undefined` +##### skills? -##### put() +> `optional` **skills?**: [`ImproveSkillsOptions`](#improveskillsoptions) -> **put**(`session`): `void` \| `Promise`\<`void`\> +SKILLS-surface wiring for real skill-DOCUMENT optimization. Without this, + `surface: 'skills'` optimizes the profile's skills REFS array (file pointers) + — which `skillOptProposer` (a document patcher) cannot meaningfully edit. + Provide the document CONTENT to optimize + a `writeBack` to persist the + shipped winner (the profile ref points at a file the caller owns). This is + what makes skillOpt reachable through improve(). -Defined in: [types.ts:463](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L463) +##### memory? -###### Parameters +> `optional` **memory?**: [`ImproveMemoryOptions`](#improvememoryoptions) -###### session +MEMORY-surface wiring for a curated durable memory document. The default + deterministic proposer deduplicates and ranks lessons from findings, then + replaces its managed block instead of growing memory without bound. -`RuntimeSession` +##### promotionGate? -###### Returns +> `optional` **promotionGate?**: `SelfImproveOptions`\<`TScenario`, `TArtifact`\>\[`"gate"`\] -`void` \| `Promise`\<`void`\> +Custom held-back-exam decision. The string `gate` above controls whether + the exam runs; this callback controls how its evidence decides promotion. -##### appendEvent()? +#### Type Parameters -> `optional` **appendEvent**(`sessionId`, `event`): `void` \| `Promise`\<`void`\> +##### TScenario -Defined in: [types.ts:464](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L464) +`TScenario` *extends* `Scenario` -###### Parameters +##### TArtifact -###### sessionId +`TArtifact` -`string` +*** -###### event +### ProfileDiffProposerContext -[`RuntimeStreamEvent`](#runtimestreamevent) +> **ProfileDiffProposerContext**\<`TFindings`\> = `ProposeContext`\<`TFindings`\> & `object` -###### Returns +Defined in: [improvement/profile-diff-proposer.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L26) -`void` \| `Promise`\<`void`\> +#### Type Declaration -##### listEvents()? +##### profile -> `optional` **listEvents**(`sessionId`): [`RuntimeStreamEvent`](#runtimestreamevent)[] \| `Promise`\<[`RuntimeStreamEvent`](#runtimestreamevent)[]\> +> **profile**: `AgentProfile` + +#### Type Parameters -Defined in: [types.ts:465](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L465) +##### TFindings -###### Parameters +`TFindings` = `unknown` -###### sessionId +*** -`string` +### KnowledgeReadinessCheckResult -###### Returns +> **KnowledgeReadinessCheckResult** = `boolean` \| \{ `ready`: `boolean`; `summary?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; \} -[`RuntimeStreamEvent`](#runtimestreamevent)[] \| `Promise`\<[`RuntimeStreamEvent`](#runtimestreamevent)[]\> +Defined in: [knowledge/supervised-update.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L30) *** -### AgentBackendInput +### KnowledgeReadinessCheck -Defined in: [types.ts:469](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L469) +> **KnowledgeReadinessCheck** = (`input`) => `Promise`\<[`KnowledgeReadinessCheckResult`](#knowledgereadinesscheckresult)\> \| [`KnowledgeReadinessCheckResult`](#knowledgereadinesscheckresult) -#### Stable +Defined in: [knowledge/supervised-update.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L38) -#### Properties +#### Parameters -##### task +##### input -> **task**: [`AgentTaskSpec`](#agenttaskspec) +[`KnowledgeReadinessCheckInput`](#knowledgereadinesscheckinput) -Defined in: [types.ts:470](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L470) +#### Returns -##### message? +`Promise`\<[`KnowledgeReadinessCheckResult`](#knowledgereadinesscheckresult)\> \| [`KnowledgeReadinessCheckResult`](#knowledgereadinesscheckresult) -> `optional` **message?**: `string` +*** -Defined in: [types.ts:471](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L471) +### SupervisedKnowledgeUpdater -##### messages? +> **SupervisedKnowledgeUpdater** = (`input`) => `Promise`\<[`SupervisedKnowledgeUpdateResult`](#supervisedknowledgeupdateresult)\> -> `optional` **messages?**: `object`[] +Defined in: [knowledge/supervised-update.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L86) -Defined in: [types.ts:472](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L472) +#### Parameters -###### role +##### input -> **role**: `string` +[`SupervisedKnowledgeUpdateInput`](#supervisedknowledgeupdateinput) -###### content +#### Returns -> **content**: `string` +`Promise`\<[`SupervisedKnowledgeUpdateResult`](#supervisedknowledgeupdateresult)\> -##### inputs? +*** -> `optional` **inputs?**: `Record`\<`string`, `unknown`\> +### DelegatedLoopMode -Defined in: [types.ts:473](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L473) +> **DelegatedLoopMode** = *typeof* [`DELEGATED_LOOP_MODES`](#delegated_loop_modes)\[`number`\] -*** +Defined in: [loop-runner.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L50) -### AgentBackendContext +**`Experimental`** -Defined in: [types.ts:477](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L477) +*** -#### Stable +### DelegatedLoopRunner -#### Properties +> **DelegatedLoopRunner**\<`T`\> = (`signal`) => `Promise`\<`T`\> -##### task +Defined in: [loop-runner.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L59) -> **task**: [`AgentTaskSpec`](#agenttaskspec) +**`Experimental`** -Defined in: [types.ts:478](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L478) +A pre-configured loop for one mode. Returns the mode's raw + output; the dispatcher wraps it in a [DelegatedLoopResult](#delegatedloopresult). -##### knowledge +#### Type Parameters -> **knowledge**: `KnowledgeReadinessReport` +##### T -Defined in: [types.ts:479](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L479) +`T` = `unknown` -##### session +#### Parameters -> **session**: `RuntimeSession` +##### signal -Defined in: [types.ts:480](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L480) +`AbortSignal` -##### signal? +#### Returns -> `optional` **signal?**: `AbortSignal` +`Promise`\<`T`\> -Defined in: [types.ts:481](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L481) +*** -##### runId? +### DelegatedLoopRegistry -> `optional` **runId?**: `string` +> **DelegatedLoopRegistry** = `Partial`\<`Record`\<[`DelegatedLoopMode`](#delegatedloopmode), [`DelegatedLoopRunner`](#delegatedlooprunner)\>\> -Defined in: [types.ts:487](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L487) +Defined in: [loop-runner.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L63) -Conversation/run identifier when this call is part of a multi-agent run. -Backends should stamp it into any trace/log emission so cross-participant -events correlate. Absent when the call is a stand-alone `runAgentTask`. +**`Experimental`** -##### turnId? +Mode → configured runner. Partial: only register the modes a + given product/routine actually uses. -> `optional` **turnId?**: `string` +*** -Defined in: [types.ts:492](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L492) +### AgentBackendKind -Deterministic turn id for this single call. Stable across retries of the -same logical turn so a caching gateway / idempotent backend can dedupe. +> **AgentBackendKind** = `"router"` \| `"tcloud"` \| `"cli-bridge"` \| `"sandbox"` -##### parentTurnId? +Defined in: [resolve-agent-backend.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L38) -> `optional` **parentTurnId?**: `string` +The transport a chat backend runs on. -Defined in: [types.ts:498](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L498) +*** -If this call is itself nested inside a higher-order conversation -(recursion via `createConversationBackend`), the enclosing turn's id. -Used for trace stitching across nested orchestration. +### RuntimeHookPhase -##### propagatedHeaders? +> **RuntimeHookPhase** = `"before"` \| `"after"` \| `"error"` \| `"event"` -> `optional` **propagatedHeaders?**: `Readonly`\<`Record`\<`string`, `string`\>\> +Defined in: [runtime-hooks.ts:10](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L10) -Defined in: [types.ts:505](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L505) +**`Experimental`** -Headers to forward verbatim to any outbound HTTP the backend issues: -`X-Tangle-Forwarded-Authorization`, `X-Tangle-Forwarded-Depth`, -run/turn correlation. Backends that issue HTTP MUST merge these into -the outbound request; backends that don't issue HTTP may ignore them. +Runtime hook contracts. Hooks are execution-scoped observers, not part of an +`AgentProfile`: profiles stay portable agent recipes; hooks attach to the +loop or product harness that is running the profile. *** -### AgentExecutionBackend +### RuntimeHookTarget -Defined in: [types.ts:509](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L509) +> **RuntimeHookTarget** = `"agent.run"` \| `"agent.turn"` \| `"agent.tool_call"` \| `"agent.spawn"` \| `"agent.child"` \| `"agent.plan"` \| `"agent.decision"` \| `string` & `object` -#### Stable +Defined in: [runtime-hooks.ts:12](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L12) -#### Type Parameters +*** -##### TInput +### RuntimeDecisionKind -`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) +> **RuntimeDecisionKind** = `"continue"` \| `"verify"` \| `"ask"` \| `"retry"` \| `"stop"` \| `"memory-write"` \| `"memory-read"` \| `"tool-select"` \| `"skill-select"` \| `"workflow-select"` \| `"surface-promote"` \| `string` & `object` -#### Properties +Defined in: [runtime-hooks.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L22) -##### kind +*** -> **kind**: `string` +### ToolCallOutcome -Defined in: [types.ts:510](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L510) +> **ToolCallOutcome** = \{ `ok`: `true`; `result`: `unknown`; \} \| \{ `ok`: `false`; `code`: `string`; `message`: `string`; `status?`: `number`; \} -#### Methods +Defined in: [tool-loop.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L30) -##### start()? +Outcome of one tool dispatch — structurally compatible with a hub/integration + tool-outcome union, so callers can fold either through the loop. -> `optional` **start**(`input`, `context`): `RuntimeSession` \| `Promise`\<`RuntimeSession`\> +*** -Defined in: [types.ts:511](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L511) +### ToolLoopMessage -###### Parameters +> **ToolLoopMessage** = `object` -###### input +Defined in: [tool-loop.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L68) -`TInput` +A message in the running conversation the loop sends to `streamTurn`. -###### context +The base `{ role, content }` covers `system` / `user` / plain `assistant` +turns. Two optional fields carry the OpenAI function-calling contract so a +strict model (Claude, and any OpenAI-compatible provider that validates tool +history) reads its own tool use back instead of re-issuing the same call: -`Omit`\<[`AgentBackendContext`](#agentbackendcontext), `"session"`\> & `object` + - an assistant turn that emitted tool calls carries `tool_calls`, and its + `content` is `null` when the turn was tool-only; + - each tool result is its own `{ role: 'tool', tool_call_id, content }` + message keyed to the call that produced it. -###### Returns +Widening is additive: a `streamTurn` that reads only `role` + `content` still +works; one that forwards the whole message to an OpenAI-compatible endpoint +now sends correct tool history. -`RuntimeSession` \| `Promise`\<`RuntimeSession`\> +#### Properties -##### resume()? +##### role -> `optional` **resume**(`session`, `input`, `context`): `RuntimeSession` \| `Promise`\<`RuntimeSession`\> +> **role**: `string` -Defined in: [types.ts:515](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L515) +Defined in: [tool-loop.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L69) -###### Parameters +##### content -###### session +> **content**: `string` \| `null` -`RuntimeSession` +Defined in: [tool-loop.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L70) -###### input +##### tool\_calls? -`TInput` +> `optional` **tool\_calls?**: [`ToolLoopAssistantToolCall`](#toolloopassistanttoolcall)[] -###### context +Defined in: [tool-loop.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L71) -`Omit`\<[`AgentBackendContext`](#agentbackendcontext), `"session"`\> +##### tool\_call\_id? -###### Returns +> `optional` **tool\_call\_id?**: `string` -`RuntimeSession` \| `Promise`\<`RuntimeSession`\> +Defined in: [tool-loop.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L72) -##### stream() +*** -> **stream**(`input`, `context`): `AsyncIterable`\<[`RuntimeStreamEvent`](#runtimestreamevent)\> +### ToolLoopEvent -Defined in: [types.ts:520](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L520) +> **ToolLoopEvent** = \{ `type`: `"text"`; `text`: `string`; \} \| \{ `type`: `"tool_call"`; `call`: [`ToolLoopCall`](#toolloopcall); \} \| \{ `type`: `"other"`; `event`: `unknown`; \} -###### Parameters +Defined in: [tool-loop.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L108) -###### input +*** -`TInput` +### ToolLoopStopReason -###### context +> **ToolLoopStopReason** = `"completed"` \| `"stuck-loop"` \| `"backstop"` \| `"deadline"` \| `"budget"` -[`AgentBackendContext`](#agentbackendcontext) +Defined in: [tool-loop.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L118) -###### Returns +Why the loop stopped. `completed` = model finished naturally; `stuck-loop` = + ≥3 consecutive identical tool calls (same tool + args); `backstop` = hit the + runaway-backstop cap (200 by default); `deadline` = wall-clock deadlineMs + exceeded; `budget` = maxCostUsd exhausted. Non-`completed` stops are infra / + resource outcomes — eval scoring must distinguish them from capability failure. -`AsyncIterable`\<[`RuntimeStreamEvent`](#runtimestreamevent)\> +*** -##### stop()? +### StreamToolLoopYield -> `optional` **stop**(`session`, `reason`): `void` \| `Promise`\<`void`\> +> **StreamToolLoopYield**\<`Raw`\> = \{ `kind`: `"event"`; `event`: `Raw`; \} \| \{ `kind`: `"tool_result"`; `toolName`: `string`; `toolCallId?`: `string`; `label`: `string`; `outcome`: [`ToolCallOutcome`](#toolcalloutcome); \} \| \{ `kind`: `"capped"`; `pending`: `number`; `stopReason`: `Exclude`\<[`ToolLoopStopReason`](#toolloopstopreason), `"completed"`\>; \} -Defined in: [types.ts:521](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L521) +Defined in: [tool-loop.ts:298](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L298) -###### Parameters +#### Type Parameters -###### session +##### Raw -`RuntimeSession` +`Raw` -###### reason +*** -`string` +### AgentTaskStatus -###### Returns +> **AgentTaskStatus** = `"completed"` \| `"blocked"` \| `"failed"` \| `"aborted"` -`void` \| `Promise`\<`void`\> +Defined in: [types.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L145) + +#### Stable *** -### AgentTaskRunResult +### AgentRuntimeEvent -Defined in: [types.ts:557](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L557) +> **AgentRuntimeEvent**\<`TState`, `TAction`, `TActionResult`, `TEval`\> = \{ `type`: `"task_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); \} \| \{ `type`: `"readiness_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); \} \| \{ `type`: `"readiness_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `knowledge`: `KnowledgeReadinessReport`; \} \| \{ `type`: `"questions_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; \} \| \{ `type`: `"questions_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `userAnswers`: `Record`\<`string`, `string`\>; \} \| \{ `type`: `"acquisition_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; \} \| \{ `type`: `"acquisition_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `acquiredEvidenceIds`: `string`[]; \} \| \{ `type`: `"control_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `knowledge`: `KnowledgeReadinessReport`; \} \| \{ `type`: `"control_step"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `step`: `ControlStep`\<`TState`, `TAction`, `TActionResult`, `TEval`\>; \} \| \{ `type`: `"control_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `control`: `ControlRunResult`\<`TState`, `TAction`, `TActionResult`, `TEval`\>; \} \| \{ `type`: `"task_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; \} -#### Stable +Defined in: [types.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L148) #### Type Parameters ##### TState -`TState` +`TState` = `unknown` ##### TAction -`TAction` +`TAction` = `unknown` ##### TActionResult -`TActionResult` +`TActionResult` = `unknown` ##### TEval `TEval` *extends* `ControlEvalResult` = `ControlEvalResult` -#### Properties - -##### task +#### Stable -> **task**: [`AgentTaskSpec`](#agenttaskspec) +*** -Defined in: [types.ts:563](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L563) +### AgentRuntimeEventSink -##### status +> **AgentRuntimeEventSink**\<`TState`, `TAction`, `TActionResult`, `TEval`\> = (`event`) => `Promise`\<`void`\> \| `void` -> **status**: [`AgentTaskStatus`](#agenttaskstatus) +Defined in: [types.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L189) -Defined in: [types.ts:564](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L564) +#### Type Parameters -##### knowledge +##### TState -> **knowledge**: `KnowledgeReadinessReport` +`TState` = `unknown` -Defined in: [types.ts:565](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L565) +##### TAction -##### questions +`TAction` = `unknown` -> **questions**: `UserQuestion`[] +##### TActionResult -Defined in: [types.ts:566](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L566) +`TActionResult` = `unknown` -##### acquisitionPlans +##### TEval -> **acquisitionPlans**: `DataAcquisitionPlan`[] +`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` -Defined in: [types.ts:567](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L567) +#### Parameters -##### userAnswers +##### event -> **userAnswers**: `Record`\<`string`, `string`\> +[`AgentRuntimeEvent`](#agentruntimeevent)\<`TState`, `TAction`, `TActionResult`, `TEval`\> -Defined in: [types.ts:568](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L568) +#### Returns -##### acquiredEvidenceIds +`Promise`\<`void`\> \| `void` -> **acquiredEvidenceIds**: `string`[] +#### Stable -Defined in: [types.ts:569](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L569) +*** -##### control +### OpenAIChatToolChoice -> **control**: `ControlRunResult`\<`TState`, `TAction`, `TActionResult`, `TEval`\> +> **OpenAIChatToolChoice** = `"auto"` \| `"none"` \| `"required"` \| \{ `type`: `"function"`; `function`: \{ `name`: `string`; \}; \} -Defined in: [types.ts:570](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L570) +Defined in: [types.ts:260](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L260) -##### runRecords +`tool_choice` parameter for OpenAI-compat chat. Same shape as the OpenAI +spec: `'auto'` (default — model decides), `'none'` (disable tool calling +for this turn), `'required'` (force a tool call), or a specific function +pin `{ type: 'function', function: { name } }`. -> **runRecords**: `RunRecord`[] +#### Stable -Defined in: [types.ts:571](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L571) +*** -## Type Aliases +### OpenAIChatResponseFormat -### RetryableErrorPredicate +> **OpenAIChatResponseFormat** = \{ `type`: `"text"`; \} \| \{ `type`: `"json_object"`; \} \| \{ `type`: `"json_schema"`; `json_schema`: `Record`\<`string`, `unknown`\>; \} -> **RetryableErrorPredicate** = (`err`) => `boolean` +Defined in: [types.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L274) -Defined in: [conversation/call-policy.ts:18](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L18) +`response_format` parameter for OpenAI-compatible chat endpoints. Use +`json_object` when the caller needs syntactically valid JSON, or +`json_schema` when the upstream provider supports schema-constrained JSON. -Pure judgment of whether an error is worth retrying. Defaults: TimeoutError, AbortError, fetch-level network errors. +#### Stable -#### Parameters +*** -##### err +### RuntimeStreamEvent -`unknown` +> **RuntimeStreamEvent** = \{ `type`: `"task_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} \| \{ `type`: `"readiness_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} \| \{ `type`: `"readiness_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `knowledge`: `KnowledgeReadinessReport`; `decision`: `KnowledgeReadinessDecision`; `timestamp`: `string`; \} \| \{ `type`: `"questions_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `timestamp`: `string`; \} \| \{ `type`: `"questions_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `userAnswers`: `Record`\<`string`, `string`\>; `timestamp`: `string`; \} \| \{ `type`: `"acquisition_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `timestamp`: `string`; \} \| \{ `type`: `"acquisition_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `acquiredEvidenceIds`: `string`[]; `timestamp`: `string`; \} \| \{ `type`: `"session_created"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `timestamp`: `string`; \} \| \{ `type`: `"session_resumed"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `timestamp`: `string`; \} \| \{ `type`: `"backend_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `backend`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"text_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `text`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"reasoning_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `text`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"tool_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `toolName`: `string`; `toolCallId?`: `string`; `args?`: `unknown`; `timestamp?`: `string`; \} \| \{ `type`: `"tool_result"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `toolName`: `string`; `toolCallId?`: `string`; `result?`: `unknown`; `timestamp?`: `string`; \} \| \{ `type`: `"llm_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `model`: `string`; `tokensIn?`: `number`; `tokensOut?`: `number`; `costUsd?`: `number`; `latencyMs?`: `number`; `finishReason?`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"artifact"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `artifactId`: `string`; `name?`: `string`; `mimeType?`: `string`; `uri?`: `string`; `content?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `timestamp?`: `string`; \} \| \{ `type`: `"proposal_created"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `proposalId`: `string`; `title`: `string`; `status?`: `"pending"` \| `"approved"` \| `"rejected"`; `content?`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"backend_error"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `backend`: `string`; `message`: `string`; `recoverable`: `boolean`; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} \| \{ `type`: `"backend_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `backend`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"task_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"final"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `text?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} -#### Returns +Defined in: [types.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L280) -`boolean` +#### Union Members -*** +##### Type Literal -### RetryBackoff +\{ `type`: `"task_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} -> **RetryBackoff** = `number` \| ((`attempt`) => `number`) +*** -Defined in: [conversation/call-policy.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L21) +##### Type Literal -Backoff between attempts. Constant ms, or `(attempt: 1-indexed) => ms`. +\{ `type`: `"readiness_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} *** -### ForwardHeaderName - -> **ForwardHeaderName** = *typeof* [`FORWARD_HEADERS`](#forward_headers)\[keyof *typeof* [`FORWARD_HEADERS`](#forward_headers)\] +##### Type Literal -Defined in: [conversation/headers.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/headers.ts#L35) +\{ `type`: `"readiness_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `knowledge`: `KnowledgeReadinessReport`; `decision`: `KnowledgeReadinessDecision`; `timestamp`: `string`; \} *** -### PropagatedHeaders - -> **PropagatedHeaders** = `Readonly`\<`Record`\<`string`, `string`\>\> - -Defined in: [conversation/headers.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/headers.ts#L111) +##### Type Literal -Header bag carried through `AgentBackendContext.propagatedHeaders` so -backends that opt in can merge them into their outbound HTTP requests. -Distinct from `buildForwardHeaders` so callers can attach extra -non-protocol headers (e.g. tracing) without colliding. +\{ `type`: `"questions_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `timestamp`: `string`; \} *** -### PersonaDriver - -> **PersonaDriver** = \{ `kind`: `"profile"`; `profile`: `AgentProfile`; \} \| \{ `kind`: `"scripted"`; `turns`: `string`[]; \} - -Defined in: [conversation/run-persona.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L32) +##### Type Literal -A persona that drives the conversation: either a full driver `AgentProfile` - (an LLM user-sim) or a deterministic script of user turns (the fast-path). +\{ `type`: `"questions_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `userAnswers`: `Record`\<`string`, `string`\>; `timestamp`: `string`; \} *** -### AuthSource - -> **AuthSource** = `"forward-user"` \| `"agent-owned"` \| ((`state`) => `"forward-user"` \| `"agent-owned"`) - -Defined in: [conversation/types.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L68) +##### Type Literal -#### Stable +\{ `type`: `"acquisition_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `timestamp`: `string`; \} *** -### TurnOrder +##### Type Literal -> **TurnOrder** = `"alternate"` \| `"round-robin"` \| ((`state`) => `number`) +\{ `type`: `"acquisition_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `acquiredEvidenceIds`: `string`[]; `timestamp`: `string`; \} -Defined in: [conversation/types.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L74) +*** -#### Stable +##### Type Literal -*** +\{ `type`: `"session_created"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `timestamp`: `string`; \} -### HaltPredicate +*** -> **HaltPredicate** = (`ctx`) => `boolean` \| [`HaltSignal`](#haltsignal) \| `Promise`\<`boolean` \| [`HaltSignal`](#haltsignal)\> +##### Type Literal -Defined in: [conversation/types.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L95) +\{ `type`: `"session_resumed"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `timestamp`: `string`; \} -#### Parameters +*** -##### ctx +##### Type Literal -[`HaltContext`](#haltcontext) +\{ `type`: `"backend_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `backend`: `string`; `timestamp`: `string`; \} -#### Returns +*** -`boolean` \| [`HaltSignal`](#haltsignal) \| `Promise`\<`boolean` \| [`HaltSignal`](#haltsignal)\> +##### Type Literal -#### Stable +\{ `type`: `"text_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `text`: `string`; `timestamp?`: `string`; \} *** -### HaltReason +##### Type Literal -> **HaltReason** = \{ `kind`: `"max_turns"`; `turns`: `number`; \} \| \{ `kind`: `"max_credits"`; `spentCents`: `number`; `capCents`: `number`; \} \| \{ `kind`: `"predicate"`; `reason`: `string`; \} \| \{ `kind`: `"abort"`; \} \| \{ `kind`: `"participant_error"`; `participant`: `string`; `message`: `string`; \} +\{ `type`: `"reasoning_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `text`: `string`; `timestamp?`: `string`; \} -Defined in: [conversation/types.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L100) +*** -#### Stable +##### Type Literal + +\{ `type`: `"tool_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `toolName`: `string`; `toolCallId?`: `string`; `args?`: `unknown`; `timestamp?`: `string`; \} *** -### ConversationStreamEvent +##### Type Literal -> **ConversationStreamEvent** = \{ `type`: `"conversation_start"`; `runId`: `string`; `participants`: readonly `string`[]; `seed`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"conversation_resumed"`; `runId`: `string`; `participants`: readonly `string`[]; `transcript`: readonly [`ConversationTurn`](#conversationturn)[]; `timestamp`: `string`; \} \| \{ `type`: `"turn_start"`; `runId`: `string`; `index`: `number`; `speaker`: `string`; `turnId`: `string`; `attempt`: `number`; `timestamp`: `string`; \} \| \{ `type`: `"turn_text_delta"`; `runId`: `string`; `index`: `number`; `speaker`: `string`; `turnId`: `string`; `text`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"turn_retry"`; `runId`: `string`; `index`: `number`; `speaker`: `string`; `turnId`: `string`; `attempt`: `number`; `reason`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"turn_end"`; `runId`: `string`; `turn`: [`ConversationTurn`](#conversationturn); `timestamp`: `string`; \} \| \{ `type`: `"conversation_end"`; `runId`: `string`; `result`: [`ConversationResult`](#conversationresult); `timestamp`: `string`; \} +\{ `type`: `"tool_result"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `toolName`: `string`; `toolCallId?`: `string`; `result?`: `unknown`; `timestamp?`: `string`; \} -Defined in: [conversation/types.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/types.ts#L237) +*** -#### Stable +##### Type Literal + +\{ `type`: `"llm_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `model`: `string`; `tokensIn?`: `number`; `tokensOut?`: `number`; `costUsd?`: `number`; `latencyMs?`: `number`; `finishReason?`: `string`; `timestamp?`: `string`; \} *** -### Verifier +##### Type Literal -> **Verifier** = (`worktreePath`) => `Promise`\<[`VerifyResult`](#verifyresult)\> \| [`VerifyResult`](#verifyresult) +\{ `type`: `"artifact"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `artifactId`: `string`; `name?`: `string`; `mimeType?`: `string`; `uri?`: `string`; `content?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `timestamp?`: `string`; \} -Defined in: [improvement/agentic-generator.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L56) +*** -Verifies the edited worktree. Sync or async; throws only on a setup fault - (a candidate that fails verification returns `{ok:false}`, it does not - throw). +##### Type Literal -#### Parameters +\{ `type`: `"proposal_created"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `proposalId`: `string`; `title`: `string`; `status?`: `"pending"` \| `"approved"` \| `"rejected"`; `content?`: `string`; `timestamp?`: `string`; \} -##### worktreePath +*** -`string` +##### Type Literal -#### Returns +\{ `type`: `"backend_error"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `backend`: `string`; `message`: `string`; `recoverable`: `boolean`; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} -`Promise`\<[`VerifyResult`](#verifyresult)\> \| [`VerifyResult`](#verifyresult) +###### type -*** +> **type**: `"backend_error"` -### ImproveSurface +###### task -> **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"code"` +> **task**: [`AgentTaskSpec`](#agenttaskspec) -Defined in: [improvement/improve.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L55) +###### session? -The agent-profile lever `improve` optimizes. Mirrors the AgentProfile-law - profile levers; `code` is the implementation-tier surface. +> `optional` **session?**: `RuntimeSession` -*** +###### backend -### KnowledgeReadinessCheckResult +> **backend**: `string` -> **KnowledgeReadinessCheckResult** = `boolean` \| \{ `ready`: `boolean`; `summary?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; \} +###### message -Defined in: [knowledge/supervised-update.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L30) +> **message**: `string` -*** +###### recoverable -### KnowledgeReadinessCheck +> **recoverable**: `boolean` -> **KnowledgeReadinessCheck** = (`input`) => `Promise`\<[`KnowledgeReadinessCheckResult`](#knowledgereadinesscheckresult)\> \| [`KnowledgeReadinessCheckResult`](#knowledgereadinesscheckresult) +###### error? -Defined in: [knowledge/supervised-update.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L38) +> `optional` **error?**: [`BackendErrorDetail`](#backenderrordetail) -#### Parameters +Typed transport diagnostic. Present when the upstream returned a +non-success HTTP status or every retry attempt threw. Consumers MUST +surface this onto their `RunRecord.error` — silently treating a +`backend_error` as "no output" hides credit exhaustion, auth failure, +and upstream outages from operators. + - `kind: 'transport'` — HTTP / network failure with optional `status` + + truncated response `body`. + - `kind: 'backend'` — the backend's `stream()` generator threw for a + reason that isn't a recognized transport failure. -##### input +###### timestamp -[`KnowledgeReadinessCheckInput`](#knowledgereadinesscheckinput) +> **timestamp**: `string` -#### Returns +*** -`Promise`\<[`KnowledgeReadinessCheckResult`](#knowledgereadinesscheckresult)\> \| [`KnowledgeReadinessCheckResult`](#knowledgereadinesscheckresult) +##### Type Literal + +\{ `type`: `"backend_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `backend`: `string`; `timestamp`: `string`; \} *** -### SupervisedKnowledgeUpdater +##### Type Literal -> **SupervisedKnowledgeUpdater** = (`input`) => `Promise`\<[`SupervisedKnowledgeUpdateResult`](#supervisedknowledgeupdateresult)\> +\{ `type`: `"task_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `timestamp`: `string`; \} -Defined in: [knowledge/supervised-update.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L86) +*** -#### Parameters +##### Type Literal -##### input +\{ `type`: `"final"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `text?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} -[`SupervisedKnowledgeUpdateInput`](#supervisedknowledgeupdateinput) +###### type -#### Returns +> **type**: `"final"` + +###### task -`Promise`\<[`SupervisedKnowledgeUpdateResult`](#supervisedknowledgeupdateresult)\> +> **task**: [`AgentTaskSpec`](#agenttaskspec) -*** +###### session? -### DelegatedLoopMode +> `optional` **session?**: `RuntimeSession` -> **DelegatedLoopMode** = *typeof* [`DELEGATED_LOOP_MODES`](#delegated_loop_modes)\[`number`\] +###### status -Defined in: [loop-runner.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L50) +> **status**: [`AgentTaskStatus`](#agenttaskstatus) -**`Experimental`** +###### reason -*** +> **reason**: `string` -### DelegatedLoopRunner +###### text? -> **DelegatedLoopRunner**\<`T`\> = (`signal`) => `Promise`\<`T`\> +> `optional` **text?**: `string` -Defined in: [loop-runner.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L59) +###### metadata? -**`Experimental`** +> `optional` **metadata?**: `Record`\<`string`, `unknown`\> -A pre-configured loop for one mode. Returns the mode's raw - output; the dispatcher wraps it in a [DelegatedLoopResult](#delegatedloopresult). +###### error? -#### Type Parameters +> `optional` **error?**: [`BackendErrorDetail`](#backenderrordetail) -##### T +Typed terminal-error diagnostic. Mirrors the `backend_error.error` +shape so a consumer that only listens for `final` still receives a +loud, structured failure when the backend never produced output. Only +set when `status !== 'completed'`. Consumers building a `RunRecord` +MUST map this to `RunRecord.error` rather than recording silent +`error: null` with empty `finalText`. -`T` = `unknown` +###### timestamp -#### Parameters +> **timestamp**: `string` -##### signal +#### Stable -`AbortSignal` +## Variables -#### Returns +### CANDIDATE\_TRACE\_TAGS -`Promise`\<`T`\> +> `const` **CANDIDATE\_TRACE\_TAGS**: `object` -*** +Defined in: [candidate-execution/types.ts:561](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L561) -### DelegatedLoopRegistry +Protected trace tags that bind a run to one prepared candidate execution. -> **DelegatedLoopRegistry** = `Partial`\<`Record`\<[`DelegatedLoopMode`](#delegatedloopmode), [`DelegatedLoopRunner`](#delegatedlooprunner)\>\> +#### Type Declaration -Defined in: [loop-runner.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L63) +##### executionId -**`Experimental`** +> `readonly` **executionId**: `"tangle.candidate.execution_id"` = `'tangle.candidate.execution_id'` -Mode → configured runner. Partial: only register the modes a - given product/routine actually uses. +##### bundleDigest -*** +> `readonly` **bundleDigest**: `"tangle.candidate.bundle_digest"` = `'tangle.candidate.bundle_digest'` -### AgentBackendKind +##### executionPlanDigest -> **AgentBackendKind** = `"router"` \| `"tcloud"` \| `"cli-bridge"` \| `"sandbox"` +> `readonly` **executionPlanDigest**: `"tangle.candidate.execution_plan_digest"` = `'tangle.candidate.execution_plan_digest'` -Defined in: [resolve-agent-backend.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/resolve-agent-backend.ts#L38) +##### materializationReceiptDigest -The transport a chat backend runs on. +> `readonly` **materializationReceiptDigest**: `"tangle.candidate.materialization_receipt_digest"` = `'tangle.candidate.materialization_receipt_digest'` *** -### RuntimeHookPhase +### CANDIDATE\_TRACE\_ENV -> **RuntimeHookPhase** = `"before"` \| `"after"` \| `"error"` \| `"event"` +> `const` **CANDIDATE\_TRACE\_ENV**: `object` -Defined in: [runtime-hooks.ts:10](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L10) +Defined in: [candidate-execution/types.ts:569](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L569) -**`Experimental`** +Environment keys used to propagate immutable candidate trace identity. -Runtime hook contracts. Hooks are execution-scoped observers, not part of an -`AgentProfile`: profiles stay portable agent recipes; hooks attach to the -loop or product harness that is running the profile. +#### Type Declaration -*** +##### executionId -### RuntimeHookTarget +> `readonly` **executionId**: `"TANGLE_CANDIDATE_EXECUTION_ID"` = `'TANGLE_CANDIDATE_EXECUTION_ID'` -> **RuntimeHookTarget** = `"agent.run"` \| `"agent.turn"` \| `"agent.tool_call"` \| `"agent.spawn"` \| `"agent.child"` \| `"agent.plan"` \| `"agent.decision"` \| `string` & `object` +##### bundleDigest -Defined in: [runtime-hooks.ts:12](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L12) +> `readonly` **bundleDigest**: `"TANGLE_CANDIDATE_BUNDLE_DIGEST"` = `'TANGLE_CANDIDATE_BUNDLE_DIGEST'` -*** +##### executionPlanDigest -### RuntimeDecisionKind +> `readonly` **executionPlanDigest**: `"TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST"` = `'TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST'` -> **RuntimeDecisionKind** = `"continue"` \| `"verify"` \| `"ask"` \| `"retry"` \| `"stop"` \| `"memory-write"` \| `"memory-read"` \| `"tool-select"` \| `"skill-select"` \| `"workflow-select"` \| `"surface-promote"` \| `string` & `object` +##### materializationReceiptDigest -Defined in: [runtime-hooks.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime-hooks.ts#L22) +> `readonly` **materializationReceiptDigest**: `"TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST"` = `'TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST'` + +##### traceRunId + +> `readonly` **traceRunId**: `"TANGLE_TRACE_RUN_ID"` = `'TANGLE_TRACE_RUN_ID'` *** -### ToolCallOutcome +### defaultIsRetryable -> **ToolCallOutcome** = \{ `ok`: `true`; `result`: `unknown`; \} \| \{ `ok`: `false`; `code`: `string`; `message`: `string`; `status?`: `number`; \} +> `const` **defaultIsRetryable**: [`RetryableErrorPredicate`](#retryableerrorpredicate) -Defined in: [tool-loop.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L30) +Defined in: [conversation/call-policy.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L65) -Outcome of one tool dispatch — structurally compatible with a hub/integration - tool-outcome union, so callers can fold either through the loop. +Default retryable classification — network/timeout class errors. Errors +a model deliberately throws (validation, refusal, 4xx) are not retried; +those represent real outcomes, not transient infrastructure faults. *** -### ToolLoopMessage - -> **ToolLoopMessage** = `object` +### FORWARD\_HEADERS -Defined in: [tool-loop.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L68) +> `const` **FORWARD\_HEADERS**: `object` -A message in the running conversation the loop sends to `streamTurn`. +Defined in: [conversation/headers.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/headers.ts#L20) -The base `{ role, content }` covers `system` / `user` / plain `assistant` -turns. Two optional fields carry the OpenAI function-calling contract so a -strict model (Claude, and any OpenAI-compatible provider that validates tool -history) reads its own tool use back instead of re-issuing the same call: +Standard names — lowercased so Headers maps interop on every runtime. - - an assistant turn that emitted tool calls carries `tool_calls`, and its - `content` is `null` when the turn was tool-only; - - each tool result is its own `{ role: 'tool', tool_call_id, content }` - message keyed to the call that produced it. +#### Type Declaration -Widening is additive: a `streamTurn` that reads only `role` + `content` still -works; one that forwards the whole message to an OpenAI-compatible endpoint -now sends correct tool history. +##### authorization -#### Properties +> `readonly` **authorization**: `"x-tangle-forwarded-authorization"` = `'x-tangle-forwarded-authorization'` -##### role +Forwarded original-user identity (`Bearer sk-tan-`); downstream gateways bill against this. -> **role**: `string` +##### depth -Defined in: [tool-loop.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L69) +> `readonly` **depth**: `"x-tangle-forwarded-depth"` = `'x-tangle-forwarded-depth'` -##### content +Monotonically incremented on every gateway hop. Refused at MAX_DEPTH. -> **content**: `string` \| `null` +##### runId -Defined in: [tool-loop.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L70) +> `readonly` **runId**: `"x-tangle-runid"` = `'x-tangle-runid'` -##### tool\_calls? +Top-level conversation run identifier, propagated through every nested call. -> `optional` **tool\_calls?**: [`ToolLoopAssistantToolCall`](#toolloopassistanttoolcall)[] +##### turnId -Defined in: [tool-loop.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L71) +> `readonly` **turnId**: `"x-tangle-turnid"` = `'x-tangle-turnid'` -##### tool\_call\_id? +This call's turn within the run; deterministic + stable across retries. -> `optional` **tool\_call\_id?**: `string` +##### parentTurnId -Defined in: [tool-loop.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L72) +> `readonly` **parentTurnId**: `"x-tangle-parent-turnid"` = `'x-tangle-parent-turnid'` -*** +When the call is *inside* another turn (recursion), the parent turn's id. -### ToolLoopEvent +##### speaker -> **ToolLoopEvent** = \{ `type`: `"text"`; `text`: `string`; \} \| \{ `type`: `"tool_call"`; `call`: [`ToolLoopCall`](#toolloopcall); \} \| \{ `type`: `"other"`; `event`: `unknown`; \} +> `readonly` **speaker**: `"x-tangle-speaker"` = `'x-tangle-speaker'` -Defined in: [tool-loop.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L108) +Logical conversation peer label at the sending side, for trace stitching. *** -### ToolLoopStopReason +### DEFAULT\_MAX\_DEPTH -> **ToolLoopStopReason** = `"completed"` \| `"stuck-loop"` \| `"backstop"` \| `"deadline"` \| `"budget"` +> `const` **DEFAULT\_MAX\_DEPTH**: `4` = `4` -Defined in: [tool-loop.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L118) +Defined in: [conversation/headers.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/headers.ts#L38) -Why the loop stopped. `completed` = model finished naturally; `stuck-loop` = - ≥3 consecutive identical tool calls (same tool + args); `backstop` = hit the - runaway-backstop cap (200 by default); `deadline` = wall-clock deadlineMs - exceeded; `budget` = maxCostUsd exhausted. Non-`completed` stops are infra / - resource outcomes — eval scoring must distinguish them from capability failure. +Hard cap on chained gateway hops; refused beyond this. Default keeps recursion bounded. *** -### StreamToolLoopYield - -> **StreamToolLoopYield**\<`Raw`\> = \{ `kind`: `"event"`; `event`: `Raw`; \} \| \{ `kind`: `"tool_result"`; `toolName`: `string`; `toolCallId?`: `string`; `label`: `string`; `outcome`: [`ToolCallOutcome`](#toolcalloutcome); \} \| \{ `kind`: `"capped"`; `pending`: `number`; `stopReason`: `Exclude`\<[`ToolLoopStopReason`](#toolloopstopreason), `"completed"`\>; \} - -Defined in: [tool-loop.ts:298](https://github.com/tangle-network/agent-runtime/blob/main/src/tool-loop.ts#L298) +### AGENTIC\_PROFILE\_RESOURCE\_ROOT -#### Type Parameters +> `const` **AGENTIC\_PROFILE\_RESOURCE\_ROOT**: `".agent-runtime-profile-resources"` = `'.agent-runtime-profile-resources'` -##### Raw +Defined in: [improvement/agentic-generator.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L69) -`Raw` +Dedicated ephemeral root for generic author-profile files. Every declared +file must live below this root so cleanup cannot alter candidate-owned files. *** -### AgentTaskStatus +### RESEARCH\_SUPERVISOR\_SYSTEM\_PROMPT -> **AgentTaskStatus** = `"completed"` \| `"blocked"` \| `"failed"` \| `"aborted"` +> `const` **RESEARCH\_SUPERVISOR\_SYSTEM\_PROMPT**: `string` -Defined in: [types.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L145) +Defined in: [knowledge/supervised-update.ts:9](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L9) -#### Stable +Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. *** -### AgentRuntimeEvent +### DELEGATED\_LOOP\_MODES -> **AgentRuntimeEvent**\<`TState`, `TAction`, `TActionResult`, `TEval`\> = \{ `type`: `"task_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); \} \| \{ `type`: `"readiness_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); \} \| \{ `type`: `"readiness_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `knowledge`: `KnowledgeReadinessReport`; \} \| \{ `type`: `"questions_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; \} \| \{ `type`: `"questions_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `userAnswers`: `Record`\<`string`, `string`\>; \} \| \{ `type`: `"acquisition_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; \} \| \{ `type`: `"acquisition_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `acquiredEvidenceIds`: `string`[]; \} \| \{ `type`: `"control_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `knowledge`: `KnowledgeReadinessReport`; \} \| \{ `type`: `"control_step"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `step`: `ControlStep`\<`TState`, `TAction`, `TActionResult`, `TEval`\>; \} \| \{ `type`: `"control_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `control`: `ControlRunResult`\<`TState`, `TAction`, `TActionResult`, `TEval`\>; \} \| \{ `type`: `"task_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; \} +> `const` **DELEGATED\_LOOP\_MODES**: readonly \[`"code"`, `"review"`, `"research"`, `"audit"`, `"self-improve"`\] -Defined in: [types.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L148) +Defined in: [loop-runner.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L47) -#### Type Parameters +**`Experimental`** -##### TState +All valid delegated-loop mode names — used for validation and CLI surfaces. -`TState` = `unknown` +*** -##### TAction +### DEFAULT\_ROUTER\_BASE\_URL -`TAction` = `unknown` +> `const` **DEFAULT\_ROUTER\_BASE\_URL**: `"https://router.tangle.tools"` = `'https://router.tangle.tools'` -##### TActionResult +Defined in: [model-resolution.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L43) -`TActionResult` = `unknown` +Default Tangle Router base URL used when no env override is set. -##### TEval +*** -`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` +### INTELLIGENCE\_WIRE\_VERSION -#### Stable +> `const` **INTELLIGENCE\_WIRE\_VERSION**: `"2026-05-26.v1"` = `'2026-05-26.v1'` -*** +Defined in: [otel-export.ts:665](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L665) -### AgentRuntimeEventSink +Wire version the eval-runs ingest enforces (X-Tangle-Wire-Version + body). -> **AgentRuntimeEventSink**\<`TState`, `TAction`, `TActionResult`, `TEval`\> = (`event`) => `Promise`\<`void`\> \| `void` +## Functions -Defined in: [types.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L189) +### createIterableBackend() -#### Type Parameters +> **createIterableBackend**\<`TInput`\>(`options`): [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> -##### TState +Defined in: [backends.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L30) -`TState` = `unknown` +Wrap any custom async-iterable stream into a typed `AgentExecutionBackend`. -##### TAction +#### Type Parameters -`TAction` = `unknown` +##### TInput -##### TActionResult +`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) -`TActionResult` = `unknown` +#### Parameters -##### TEval +##### options -`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` +###### kind -#### Parameters +`string` -##### event +###### start? -[`AgentRuntimeEvent`](#agentruntimeevent)\<`TState`, `TAction`, `TActionResult`, `TEval`\> +(`input`, `context`) => `RuntimeSession` \| `Promise`\<`RuntimeSession`\> -#### Returns +###### resume? -`Promise`\<`void`\> \| `void` +(`session`, `input`, `context`) => `RuntimeSession` \| `Promise`\<`RuntimeSession`\> -#### Stable +###### stream -*** +(`input`, `context`) => `AsyncIterable`\<[`RuntimeStreamEvent`](#runtimestreamevent)\> -### OpenAIChatToolChoice +###### stop? -> **OpenAIChatToolChoice** = `"auto"` \| `"none"` \| `"required"` \| \{ `type`: `"function"`; `function`: \{ `name`: `string`; \}; \} +(`session`, `reason`) => `void` \| `Promise`\<`void`\> -Defined in: [types.ts:260](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L260) +#### Returns -`tool_choice` parameter for OpenAI-compat chat. Same shape as the OpenAI -spec: `'auto'` (default — model decides), `'none'` (disable tool calling -for this turn), `'required'` (force a tool call), or a specific function -pin `{ type: 'function', function: { name } }`. +[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> #### Stable *** -### OpenAIChatResponseFormat +### createSandboxPromptBackend() -> **OpenAIChatResponseFormat** = \{ `type`: `"text"`; \} \| \{ `type`: `"json_object"`; \} \| \{ `type`: `"json_schema"`; `json_schema`: `Record`\<`string`, `unknown`\>; \} +> **createSandboxPromptBackend**\<`TBox`, `TInput`\>(`options`): [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> -Defined in: [types.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L274) +Defined in: [backends.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L41) -`response_format` parameter for OpenAI-compatible chat endpoints. Use -`json_object` when the caller needs syntactically valid JSON, or -`json_schema` when the upstream provider supports schema-constrained JSON. +Build an `AgentExecutionBackend` backed by a sandbox/sidecar `streamPrompt` call. -#### Stable +#### Type Parameters -*** +##### TBox -### RuntimeStreamEvent +`TBox` -> **RuntimeStreamEvent** = \{ `type`: `"task_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} \| \{ `type`: `"readiness_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} \| \{ `type`: `"readiness_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `knowledge`: `KnowledgeReadinessReport`; `decision`: `KnowledgeReadinessDecision`; `timestamp`: `string`; \} \| \{ `type`: `"questions_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `timestamp`: `string`; \} \| \{ `type`: `"questions_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `userAnswers`: `Record`\<`string`, `string`\>; `timestamp`: `string`; \} \| \{ `type`: `"acquisition_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `timestamp`: `string`; \} \| \{ `type`: `"acquisition_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `acquiredEvidenceIds`: `string`[]; `timestamp`: `string`; \} \| \{ `type`: `"session_created"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `timestamp`: `string`; \} \| \{ `type`: `"session_resumed"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `timestamp`: `string`; \} \| \{ `type`: `"backend_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `backend`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"text_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `text`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"reasoning_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `text`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"tool_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `toolName`: `string`; `toolCallId?`: `string`; `args?`: `unknown`; `timestamp?`: `string`; \} \| \{ `type`: `"tool_result"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `toolName`: `string`; `toolCallId?`: `string`; `result?`: `unknown`; `timestamp?`: `string`; \} \| \{ `type`: `"llm_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `model`: `string`; `tokensIn?`: `number`; `tokensOut?`: `number`; `costUsd?`: `number`; `latencyMs?`: `number`; `finishReason?`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"artifact"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `artifactId`: `string`; `name?`: `string`; `mimeType?`: `string`; `uri?`: `string`; `content?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `timestamp?`: `string`; \} \| \{ `type`: `"proposal_created"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `proposalId`: `string`; `title`: `string`; `status?`: `"pending"` \| `"approved"` \| `"rejected"`; `content?`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"backend_error"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `backend`: `string`; `message`: `string`; `recoverable`: `boolean`; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} \| \{ `type`: `"backend_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `backend`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"task_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"final"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `text?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} +##### TInput -Defined in: [types.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L280) +`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) -#### Union Members +#### Parameters -##### Type Literal +##### options -\{ `type`: `"task_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} +###### kind? -*** +`string` -##### Type Literal +###### getBox -\{ `type`: `"readiness_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} +###### streamPrompt -*** +###### mapEvent? -##### Type Literal +(`event`, `context`) => [`RuntimeStreamEvent`](#runtimestreamevent) \| `undefined` -\{ `type`: `"readiness_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `knowledge`: `KnowledgeReadinessReport`; `decision`: `KnowledgeReadinessDecision`; `timestamp`: `string`; \} +###### getSessionId? -*** +(`box`, `input`) => `string` \| `undefined` -##### Type Literal +#### Returns -\{ `type`: `"questions_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `timestamp`: `string`; \} +[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> + +#### Stable *** -##### Type Literal +### createOpenAICompatibleBackend() -\{ `type`: `"questions_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `userAnswers`: `Record`\<`string`, `string`\>; `timestamp`: `string`; \} +> **createOpenAICompatibleBackend**\<`TInput`\>(`options`): [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> -*** +Defined in: [backends.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L208) -##### Type Literal +OpenAI-compat streaming backend. Routes `runAgentTaskStream` through any +`POST /chat/completions` endpoint that speaks OpenAI's SSE protocol — +Tangle Router, OpenAI direct, OpenRouter, Groq, DeepSeek, Together. The +router also fronts Anthropic models in Anthropic-native SSE shape; this +backend handles both. -\{ `type`: `"acquisition_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `timestamp`: `string`; \} +### Tool calls -*** +Pass `tools` (and optionally `toolChoice`) to forward an OpenAI Chat +Completions `tools[]` array on every request. Streamed `tool_call` chunks +are buffered until the model finalizes them (either `finish_reason: +'tool_calls'` for OpenAI shape or a `content_block_stop` for Anthropic +`tool_use` blocks proxied through the router), then emitted as a single +`tool_call` RuntimeStreamEvent with the assembled `args`. -##### Type Literal +The backend does NOT execute tools — it surfaces calls for the caller's +own dispatcher (typically the product's MCP / sandbox runtime) to fulfill +and feed back as a subsequent `messages` turn. This keeps the transport +thin and lets the agent host own tool dispatch policy. -\{ `type`: `"acquisition_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `acquiredEvidenceIds`: `string`[]; `timestamp`: `string`; \} +### Fail-loud errors -*** +Non-success HTTP responses (4xx/5xx) and exhausted retry budgets throw +`BackendTransportError` from inside the `stream()` generator. The runtime +catches the throw, yields a `backend_error` with a typed `error` field +(`kind`, `status`, truncated `body`) and a terminal `final` event with +`status: 'failed'` carrying the same detail. Consumers MUST map +`final.error` onto their `RunRecord.error` — silently treating an empty +`finalText` as "agent produced nothing" hides credit exhaustion, auth +failure, and upstream outages. -##### Type Literal +#### Type Parameters -\{ `type`: `"session_created"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `timestamp`: `string`; \} +##### TInput -*** +`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) -##### Type Literal +#### Parameters -\{ `type`: `"session_resumed"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `timestamp`: `string`; \} +##### options -*** +###### apiKey -##### Type Literal +`string` -\{ `type`: `"backend_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `backend`: `string`; `timestamp`: `string`; \} +###### baseUrl -*** +`string` -##### Type Literal +###### model -\{ `type`: `"text_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `text`: `string`; `timestamp?`: `string`; \} +`string` -*** +###### kind? -##### Type Literal +`string` -\{ `type`: `"reasoning_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `text`: `string`; `timestamp?`: `string`; \} +###### tools? -*** +readonly [`OpenAIChatTool`](#openaichattool)[] -##### Type Literal +OpenAI Chat Completions `tools[]` definitions surfaced to the model on +every request. Omit to send a tool-free request (existing behavior). +The runtime makes no assumption about the dispatcher — calls stream out +as `tool_call` events and the caller is responsible for executing them +and feeding `tool_result` messages back on a follow-up turn. -\{ `type`: `"tool_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `toolName`: `string`; `toolCallId?`: `string`; `args?`: `unknown`; `timestamp?`: `string`; \} +###### toolChoice? -*** +[`OpenAIChatToolChoice`](#openaichattoolchoice) -##### Type Literal +OpenAI Chat Completions `tool_choice`. Default `undefined` (request +omits the field; provider falls back to its own default — typically +`'auto'`). -\{ `type`: `"tool_result"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `toolName`: `string`; `toolCallId?`: `string`; `result?`: `unknown`; `timestamp?`: `string`; \} +###### responseFormat? -*** +[`OpenAIChatResponseFormat`](#openaichatresponseformat) -##### Type Literal +OpenAI Chat Completions `response_format`. Omit for provider default text. -\{ `type`: `"llm_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `model`: `string`; `tokensIn?`: `number`; `tokensOut?`: `number`; `costUsd?`: `number`; `latencyMs?`: `number`; `finishReason?`: `string`; `timestamp?`: `string`; \} +###### temperature? -*** +`number` -##### Type Literal +OpenAI Chat Completions `temperature`. Omit for provider default. -\{ `type`: `"artifact"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `artifactId`: `string`; `name?`: `string`; `mimeType?`: `string`; `uri?`: `string`; `content?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `timestamp?`: `string`; \} +###### maxTokens? -*** +`number` -##### Type Literal +Maximum completion tokens, sent as OpenAI-compatible `max_tokens`. Omit for provider default. -\{ `type`: `"proposal_created"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `proposalId`: `string`; `title`: `string`; `status?`: `"pending"` \| `"approved"` \| `"rejected"`; `content?`: `string`; `timestamp?`: `string`; \} +###### fetchImpl? -*** +(`input`, `init?`) => `Promise`\<`Response`\> -##### Type Literal +###### retry? -\{ `type`: `"backend_error"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `backend`: `string`; `message`: `string`; `recoverable`: `boolean`; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} +`BackendRetryPolicy` -###### type +#### Returns -> **type**: `"backend_error"` +[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> -###### task +#### Stable -> **task**: [`AgentTaskSpec`](#agenttaskspec) +*** -###### session? +### buildAgentCandidateBundle() -> `optional` **session?**: `RuntimeSession` +> **buildAgentCandidateBundle**(`input`): `AgentCandidateBundleV1` -###### backend +Defined in: [candidate-execution/builder.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L76) -> **backend**: `string` +Compile one measured profile/code candidate into the immutable execution +contract. Code bytes are re-read and verified by agent-eval before they are +embedded. The returned bundle is schema-validated, canonically digested, and +deeply immutable; call `verifyAgentCandidateBundle` at the execution boundary +to re-read external knowledge, memory, repository, and workspace artifacts. -###### message +#### Parameters -> **message**: `string` +##### input -###### recoverable +[`BuildAgentCandidateBundleInput`](#buildagentcandidatebundleinput) -> **recoverable**: `boolean` +#### Returns -###### error? +`AgentCandidateBundleV1` -> `optional` **error?**: [`BackendErrorDetail`](#backenderrordetail) +*** -Typed transport diagnostic. Present when the upstream returned a -non-success HTTP status or every retry attempt threw. Consumers MUST -surface this onto their `RunRecord.error` — silently treating a -`backend_error` as "no output" hides credit exhaustion, auth failure, -and upstream outages from operators. - - `kind: 'transport'` — HTTP / network failure with optional `status` - + truncated response `body`. - - `kind: 'backend'` — the backend's `stream()` generator threw for a - reason that isn't a recognized transport failure. +### sealAgentCandidateBundle() -###### timestamp +> **sealAgentCandidateBundle**(`input`): `AgentCandidateBundleV1` -> **timestamp**: `string` +Defined in: [candidate-execution/bundle.ts:10](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/bundle.ts#L10) -*** +Validate and content-address a candidate bundle before it crosses an approval boundary. -##### Type Literal +#### Parameters -\{ `type`: `"backend_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: `RuntimeSession`; `backend`: `string`; `timestamp`: `string`; \} +##### input -*** +[`AgentCandidateBundleInput`](#agentcandidatebundleinput) -##### Type Literal +#### Returns -\{ `type`: `"task_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `timestamp`: `string`; \} +`AgentCandidateBundleV1` *** -##### Type Literal +### candidateExecutionClaim() -\{ `type`: `"final"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: `RuntimeSession`; `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `text?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} +> **candidateExecutionClaim**(`prepared`): [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) -###### type +Defined in: [candidate-execution/claim-plan.ts:10](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-plan.ts#L10) -> **type**: `"final"` +Extract the complete durable claim from a prepared execution. -###### task +#### Parameters -> **task**: [`AgentTaskSpec`](#agenttaskspec) +##### prepared -###### session? +[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution) -> `optional` **session?**: `RuntimeSession` +#### Returns -###### status +[`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) -> **status**: [`AgentTaskStatus`](#agenttaskstatus) +*** -###### reason +### disposePreparedAgentCandidateExecution() -> **reason**: `string` +> **disposePreparedAgentCandidateExecution**(`prepared`, `options?`): `Promise`\<\{ `disposed`: `true`; \}\> -###### text? +Defined in: [candidate-execution/dispose.ts:15](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/dispose.ts#L15) -> `optional` **text?**: `string` +Revoke reservations held by a prepared candidate that will not be executed. -###### metadata? +#### Parameters -> `optional` **metadata?**: `Record`\<`string`, `unknown`\> +##### prepared -###### error? +[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution) -> `optional` **error?**: [`BackendErrorDetail`](#backenderrordetail) +##### options? -Typed terminal-error diagnostic. Mirrors the `backend_error.error` -shape so a consumer that only listens for `final` still receives a -loud, structured failure when the backend never produced output. Only -set when `status !== 'completed'`. Consumers building a `RunRecord` -MUST map this to `RunRecord.error` rather than recording silent -`error: null` with empty `finalText`. +[`DisposePreparedAgentCandidateOptions`](#disposepreparedagentcandidateoptions) = `{}` -###### timestamp +#### Returns -> **timestamp**: `string` +`Promise`\<\{ `disposed`: `true`; \}\> -#### Stable +*** -## Variables +### executePreparedAgentCandidate() -### defaultIsRetryable +> **executePreparedAgentCandidate**(`prepared`, `options`): `Promise`\<[`AgentCandidateRunFinalization`](#agentcandidaterunfinalization)\> -> `const` **defaultIsRetryable**: [`RetryableErrorPredicate`](#retryableerrorpredicate) +Defined in: [candidate-execution/execute.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L75) -Defined in: [conversation/call-policy.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L65) +Executes and finalizes one durably claimed candidate without exposing an unproven result. -Default retryable classification — network/timeout class errors. Errors -a model deliberately throws (validation, refusal, 4xx) are not retried; -those represent real outcomes, not transient infrastructure faults. +#### Parameters -*** +##### prepared -### FORWARD\_HEADERS +[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution) -> `const` **FORWARD\_HEADERS**: `object` +##### options -Defined in: [conversation/headers.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/headers.ts#L20) +[`ExecutePreparedAgentCandidateOptions`](#executepreparedagentcandidateoptions) -Standard names — lowercased so Headers maps interop on every runtime. +#### Returns -#### Type Declaration +`Promise`\<[`AgentCandidateRunFinalization`](#agentcandidaterunfinalization)\> -##### authorization +*** -> `readonly` **authorization**: `"x-tangle-forwarded-authorization"` = `'x-tangle-forwarded-authorization'` +### persistCandidateOutputArtifact() -Forwarded original-user identity (`Bearer sk-tan-`); downstream gateways bill against this. +> **persistCandidateOutputArtifact**(`port`, `input`): `Promise`\<`AgentCandidateArtifactRef`\> -##### depth +Defined in: [candidate-execution/output-artifacts.ts:11](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/output-artifacts.ts#L11) -> `readonly` **depth**: `"x-tangle-forwarded-depth"` = `'x-tangle-forwarded-depth'` +Persist evaluator evidence, read it back, and bind the returned locator to the exact bytes. -Monotonically incremented on every gateway hop. Refused at MAX_DEPTH. +#### Parameters -##### runId +##### port -> `readonly` **runId**: `"x-tangle-runid"` = `'x-tangle-runid'` +[`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) -Top-level conversation run identifier, propagated through every nested call. +##### input -##### turnId +###### executionId -> `readonly` **turnId**: `"x-tangle-turnid"` = `'x-tangle-turnid'` +`string` -This call's turn within the run; deterministic + stable across retries. +###### purpose -##### parentTurnId +[`AgentCandidateOutputPurpose`](#agentcandidateoutputpurpose) -> `readonly` **parentTurnId**: `"x-tangle-parent-turnid"` = `'x-tangle-parent-turnid'` +###### bytes -When the call is *inside* another turn (recursion), the parent turn's id. +`Uint8Array` + +###### signal? -##### speaker +`AbortSignal` -> `readonly` **speaker**: `"x-tangle-speaker"` = `'x-tangle-speaker'` +#### Returns -Logical conversation peer label at the sending side, for trace stitching. +`Promise`\<`AgentCandidateArtifactRef`\> *** -### DEFAULT\_MAX\_DEPTH +### prepareAgentCandidateExecution() -> `const` **DEFAULT\_MAX\_DEPTH**: `4` = `4` +> **prepareAgentCandidateExecution**(`candidate`, `task`, `ports`, `options?`): `Promise`\<[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution)\> -Defined in: [conversation/headers.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/headers.ts#L38) +Defined in: [candidate-execution/prepare.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L98) -Hard cap on chained gateway hops; refused beyond this. Default keeps recursion bounded. +Materializes a verified candidate into one immutable evaluator-owned execution plan. -*** +#### Parameters -### RESEARCH\_SUPERVISOR\_SYSTEM\_PROMPT +##### candidate -> `const` **RESEARCH\_SUPERVISOR\_SYSTEM\_PROMPT**: `string` +[`VerifiedAgentCandidate`](#verifiedagentcandidate) -Defined in: [knowledge/supervised-update.ts:9](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L9) +##### task -Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. +[`AgentCandidateTaskExecution`](#agentcandidatetaskexecution) -*** +##### ports -### DELEGATED\_LOOP\_MODES +[`AgentCandidateExecutionPorts`](#agentcandidateexecutionports) -> `const` **DELEGATED\_LOOP\_MODES**: readonly \[`"code"`, `"review"`, `"research"`, `"audit"`, `"self-improve"`\] +##### options? -Defined in: [loop-runner.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L47) +[`PrepareAgentCandidateExecutionOptions`](#prepareagentcandidateexecutionoptions) = `{}` -**`Experimental`** +#### Returns -All valid delegated-loop mode names — used for validation and CLI surfaces. +`Promise`\<[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution)\> *** -### DEFAULT\_ROUTER\_BASE\_URL - -> `const` **DEFAULT\_ROUTER\_BASE\_URL**: `"https://router.tangle.tools"` = `'https://router.tangle.tools'` +### parseExactAgentProfile() -Defined in: [model-resolution.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/model-resolution.ts#L43) +> **parseExactAgentProfile**(`input`, `label`): `AgentProfile` -Default Tangle Router base URL used when no env override is set. +Defined in: [candidate-execution/profile.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/profile.ts#L96) -*** +Parse a complete profile without silently discarding unsupported fields. -### INTELLIGENCE\_WIRE\_VERSION +#### Parameters -> `const` **INTELLIGENCE\_WIRE\_VERSION**: `"2026-05-26.v1"` = `'2026-05-26.v1'` +##### input -Defined in: [otel-export.ts:556](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L556) +`unknown` -Wire version the eval-runs ingest enforces (X-Tangle-Wire-Version + body). +##### label -## Functions +`string` -### createIterableBackend() +#### Returns -> **createIterableBackend**\<`TInput`\>(`options`): [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> +`AgentProfile` -Defined in: [backends.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L30) +*** -Wrap any custom async-iterable stream into a typed `AgentExecutionBackend`. +### parseExactAgentProfileDiff() -#### Type Parameters +> **parseExactAgentProfileDiff**(`input`, `label`): `AgentProfileDiff` -##### TInput +Defined in: [candidate-execution/profile.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/profile.ts#L103) -`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) +Parse a profile diff without silently discarding unsupported fields. #### Parameters -##### options +##### input -###### kind +`unknown` + +##### label `string` -###### start? +#### Returns -(`input`, `context`) => `RuntimeSession` \| `Promise`\<`RuntimeSession`\> +`AgentProfileDiff` -###### resume? +*** -(`session`, `input`, `context`) => `RuntimeSession` \| `Promise`\<`RuntimeSession`\> +### applyExactAgentProfileDiff() -###### stream +> **applyExactAgentProfileDiff**(`baseInput`, `diffInput`, `label`): `AgentProfile` -(`input`, `context`) => `AsyncIterable`\<[`RuntimeStreamEvent`](#runtimestreamevent)\> +Defined in: [candidate-execution/profile.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/profile.ts#L110) -###### stop? +Apply one exact diff and reject any value that cannot be preserved canonically. -(`session`, `reason`) => `void` \| `Promise`\<`void`\> +#### Parameters -#### Returns +##### baseInput -[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> +`unknown` -#### Stable +##### diffInput -*** +`unknown` -### createSandboxPromptBackend() +##### label -> **createSandboxPromptBackend**\<`TBox`, `TInput`\>(`options`): [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> +`string` -Defined in: [backends.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L41) +#### Returns -Build an `AgentExecutionBackend` backed by a sandbox/sidecar `streamPrompt` call. +`AgentProfile` -#### Type Parameters +*** -##### TBox +### createProtectedAgentCandidateModelPort() -`TBox` +> **createProtectedAgentCandidateModelPort**(`options`): [`AgentCandidateModelPort`](#agentcandidatemodelport) -##### TInput +Defined in: [candidate-execution/protected-model-port.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/protected-model-port.ts#L90) -`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) +Bind a protected model-grant service to the immutable candidate runtime. + +The service remains the authority for expiry, admission, revocation, and +metering. This adapter independently checks every response before allowing +it to cross into candidate execution or durable receipt finalization. #### Parameters ##### options -###### kind? +[`CreateProtectedAgentCandidateModelPortOptions`](#createprotectedagentcandidatemodelportoptions) -`string` +#### Returns -###### getBox +[`AgentCandidateModelPort`](#agentcandidatemodelport) -###### streamPrompt +*** -###### mapEvent? +### recoverExpiredAgentCandidateExecution() -(`event`, `context`) => [`RuntimeStreamEvent`](#runtimestreamevent) \| `undefined` +> **recoverExpiredAgentCandidateExecution**(`options`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -###### getSessionId? +Defined in: [candidate-execution/recover.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L36) -(`box`, `input`) => `string` \| `undefined` +Close an expired crashed attempt from persisted non-secret handles, then record failure. -#### Returns +#### Parameters -[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> +##### options -#### Stable +[`RecoverExpiredAgentCandidateOptions`](#recoverexpiredagentcandidateoptions) + +#### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> *** -### createOpenAICompatibleBackend() +### verifyAgentCandidateBundle() -> **createOpenAICompatibleBackend**\<`TInput`\>(`options`): [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> +> **verifyAgentCandidateBundle**(`input`, `ports`): `Promise`\<[`VerifiedAgentCandidate`](#verifiedagentcandidate)\> -Defined in: [backends.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/backends.ts#L208) +Defined in: [candidate-execution/verify.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/verify.ts#L37) -OpenAI-compat streaming backend. Routes `runAgentTaskStream` through any -`POST /chat/completions` endpoint that speaks OpenAI's SSE protocol — -Tangle Router, OpenAI direct, OpenRouter, Groq, DeepSeek, Together. The -router also fronts Anthropic models in Anthropic-native SSE shape; this -backend handles both. +Verifies every digest, resource, workspace, and Git object in a candidate bundle. -### Tool calls +#### Parameters -Pass `tools` (and optionally `toolChoice`) to forward an OpenAI Chat -Completions `tools[]` array on every request. Streamed `tool_call` chunks -are buffered until the model finalizes them (either `finish_reason: -'tool_calls'` for OpenAI shape or a `content_block_stop` for Anthropic -`tool_use` blocks proxied through the router), then emitted as a single -`tool_call` RuntimeStreamEvent with the assembled `args`. +##### input -The backend does NOT execute tools — it surfaces calls for the caller's -own dispatcher (typically the product's MCP / sandbox runtime) to fulfill -and feed back as a subsequent `messages` turn. This keeps the transport -thin and lets the agent host own tool dispatch policy. +`unknown` -### Fail-loud errors +##### ports -Non-success HTTP responses (4xx/5xx) and exhausted retry budgets throw -`BackendTransportError` from inside the `stream()` generator. The runtime -catches the throw, yields a `backend_error` with a typed `error` field -(`kind`, `status`, truncated `body`) and a terminal `final` event with -`status: 'failed'` carrying the same detail. Consumers MUST map -`final.error` onto their `RunRecord.error` — silently treating an empty -`finalText` as "agent produced nothing" hides credit exhaustion, auth -failure, and upstream outages. +[`AgentCandidateVerificationPorts`](#agentcandidateverificationports) -#### Type Parameters +#### Returns -##### TInput +`Promise`\<[`VerifiedAgentCandidate`](#verifiedagentcandidate)\> -`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) +*** -#### Parameters +### captureAgentCandidateWorkspace() -##### options +> **captureAgentCandidateWorkspace**(`rootInput`, `options?`): `Promise`\<[`CapturedAgentCandidateWorkspace`](#capturedagentcandidateworkspace)\> -###### apiKey +Defined in: [candidate-execution/workspace-archive.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L121) -`string` +Capture one exact regular-file workspace for immutable candidate execution. -###### baseUrl +#### Parameters + +##### rootInput `string` -###### model +##### options? -`string` +[`CaptureAgentCandidateWorkspaceOptions`](#captureagentcandidateworkspaceoptions) = `{}` -###### kind? +#### Returns -`string` +`Promise`\<[`CapturedAgentCandidateWorkspace`](#capturedagentcandidateworkspace)\> -###### tools? +*** -readonly [`OpenAIChatTool`](#openaichattool)[] +### captureAgentCandidateWorkspaceFiles() -OpenAI Chat Completions `tools[]` definitions surfaced to the model on -every request. Omit to send a tool-free request (existing behavior). -The runtime makes no assumption about the dispatcher — calls stream out -as `tool_call` events and the caller is responsible for executing them -and feeding `tool_result` messages back on a follow-up turn. +> **captureAgentCandidateWorkspaceFiles**(`input`, `options?`): `Promise`\<[`CapturedAgentCandidateWorkspace`](#capturedagentcandidateworkspace)\> -###### toolChoice? +Defined in: [candidate-execution/workspace-archive.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L142) -[`OpenAIChatToolChoice`](#openaichattoolchoice) +Capture detached files returned by a remote executor into the standard archive. -OpenAI Chat Completions `tool_choice`. Default `undefined` (request -omits the field; provider falls back to its own default — typically -`'auto'`). +#### Parameters -###### responseFormat? +##### input -[`OpenAIChatResponseFormat`](#openaichatresponseformat) +readonly [`AgentCandidateExecutorWorkspaceFile`](#agentcandidateexecutorworkspacefile)[] -OpenAI Chat Completions `response_format`. Omit for provider default text. +##### options? -###### temperature? +`Omit`\<[`CaptureAgentCandidateWorkspaceOptions`](#captureagentcandidateworkspaceoptions), `"includeRepository"`\> = `{}` -`number` +#### Returns -OpenAI Chat Completions `temperature`. Omit for provider default. +`Promise`\<[`CapturedAgentCandidateWorkspace`](#capturedagentcandidateworkspace)\> -###### maxTokens? +*** -`number` +### createAgentCandidateWorkspacePort() -Maximum completion tokens, sent as OpenAI-compatible `max_tokens`. Omit for provider default. +> **createAgentCandidateWorkspacePort**(`options?`): [`AgentCandidateWorkspacePort`](#agentcandidateworkspaceport) -###### fetchImpl? +Defined in: [candidate-execution/workspace-archive.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L204) -(`input`, `init?`) => `Promise`\<`Response`\> +Create the standard bounded materializer for candidate execution ports. -###### retry? +#### Parameters -`BackendRetryPolicy` +##### options? -#### Returns +[`CreateAgentCandidateWorkspacePortOptions`](#createagentcandidateworkspaceportoptions) = `{}` -[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> +#### Returns -#### Stable +[`AgentCandidateWorkspacePort`](#agentcandidateworkspaceport) *** @@ -8282,7 +13034,7 @@ until `maxTurns`. Returns the persistent transcript + worker-only usage. > **runPersonaDispatch**\<`TScenario`, `TArtifact`\>(`config`): `ProfileDispatchFn`\<`TScenario`, `TArtifact`\> -Defined in: [conversation/run-persona.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L219) +Defined in: [conversation/run-persona.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/run-persona.ts#L224) Wrap [runPersonaConversation](#runpersonaconversation) as a `ProfileDispatchFn` for `runProfileMatrix`: the profile axis is the worker-under-test, the scenario @@ -8440,7 +13192,7 @@ Wire integration: > **agenticGenerator**(`opts?`): [`CandidateGenerator`](#candidategenerator) -Defined in: [improvement/agentic-generator.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L79) +Defined in: [improvement/agentic-generator.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L208) Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. @@ -8460,7 +13212,7 @@ Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a rea > **commandVerifier**(`command`, `args?`, `timeoutMs?`): [`Verifier`](#verifier) -Defined in: [improvement/agentic-generator.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L237) +Defined in: [improvement/agentic-generator.ts:860](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L860) A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other exit ⇒ failed with stdout+stderr as feedback. The common case — verify by @@ -8529,11 +13281,40 @@ Build the starting instruction for a coder agent tasked with implementing a new *** +### applyImprovementWinnerToProfile() + +> **applyImprovementWinnerToProfile**(`profile`, `surface`, `winner`): `AgentProfile` + +Defined in: [improvement/improve.ts:466](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L466) + +Apply a promoted winner surface back into the profile field for `surface`. + Returns a shallow copy; never mutates the input profile. + +#### Parameters + +##### profile + +`AgentProfile` + +##### surface + +[`ImproveSurface`](#improvesurface) + +##### winner + +`MutableSurface` + +#### Returns + +`AgentProfile` + +*** + ### improve() > **improve**\<`TScenario`, `TArtifact`\>(`profile`, `findings`, `opts`): `Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> -Defined in: [improvement/improve.ts:358](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L358) +Defined in: [improvement/improve.ts:529](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L529) Run the held-out-gated self-improvement loop on ONE profile surface. @@ -8583,9 +13364,9 @@ Optimize the system prompt, default holdout gate: ### improvementDriver() -> **improvementDriver**(`opts`): `SurfaceProposer`\<`AnalystFinding`\> +> **improvementDriver**(`opts`): [`ManagedImprovementDriver`](#managedimprovementdriver) -Defined in: [improvement/improvement-driver.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L62) +Defined in: [improvement/improvement-driver.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L89) The one reflective/agentic improvement proposer (`SurfaceProposer`): owns the candidate worktree lifecycle and delegates HOW a change is produced to a pluggable `CandidateGenerator`. @@ -8597,7 +13378,7 @@ The one reflective/agentic improvement proposer (`SurfaceProposer`): owns the ca #### Returns -`SurfaceProposer`\<`AnalystFinding`\> +[`ManagedImprovementDriver`](#managedimprovementdriver) *** @@ -8621,6 +13402,34 @@ Build a `Verifier` that boots a generated MCP server over stdio and checks it ex *** +### profileDiffProposer() + +> **profileDiffProposer**\<`TFindings`\>(`options`): `SurfaceProposer`\<`TFindings`\> + +Defined in: [improvement/profile-diff-proposer.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L41) + +Turn exact AgentProfileDiffs from any source into full profile candidates for +the shared optimization loop. Research, catalogs, humans, and trace miners +differ only in `proposeDiffs`; measurement and promotion stay identical. + +#### Type Parameters + +##### TFindings + +`TFindings` = `unknown` + +#### Parameters + +##### options + +[`ProfileDiffProposerOptions`](#profilediffproposeroptions)\<`TFindings`\> + +#### Returns + +`SurfaceProposer`\<`TFindings`\> + +*** + ### rawTraceDistiller() > **rawTraceDistiller**\<`TScenario`, `TArtifact`\>(`options?`): (`input`) => `Promise`\<`unknown`[]\> @@ -9162,7 +13971,7 @@ Injectable catalog loader — overridden in tests. > **createOtelExporter**(`config?`): [`OtelExporter`](#otelexporter) \| `undefined` -Defined in: [otel-export.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L81) +Defined in: [otel-export.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L84) Create an OTEL exporter. Returns undefined when no endpoint is configured. @@ -9182,7 +13991,7 @@ Create an OTEL exporter. Returns undefined when no endpoint is configured. > **loopEventToOtelSpan**(`event`, `traceId`, `parentSpanId?`): [`OtelSpan`](#otelspan) -Defined in: [otel-export.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L162) +Defined in: [otel-export.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L165) Convert a LoopTraceEvent into an OtelSpan for export. @@ -9220,11 +14029,43 @@ Convert a LoopTraceEvent into an OtelSpan for export. *** +### buildRuntimeEventOtelSpans() + +> **buildRuntimeEventOtelSpans**(`events`, `traceId`, `parentSpanId?`, `options?`): [`OtelSpan`](#otelspan)[] + +Defined in: [otel-export.ts:261](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L261) + +Convert normalized runtime events into lossless, redacted child spans. + +#### Parameters + +##### events + +readonly [`RuntimeStreamEvent`](#runtimestreamevent)[] + +##### traceId + +`string` + +##### parentSpanId? + +`string` + +##### options? + +[`RuntimeEventOtelOptions`](#runtimeeventoteloptions) = `{}` + +#### Returns + +[`OtelSpan`](#otelspan)[] + +*** + ### buildLoopOtelSpans() > **buildLoopOtelSpans**(`events`, `traceId`, `rootParentSpanId?`): [`OtelSpan`](#otelspan)[] -Defined in: [otel-export.ts:261](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L261) +Defined in: [otel-export.ts:364](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L364) Build a nested, real-duration OTLP span tree for ONE loop run from its full ordered `LoopTraceEvent` stream. Unlike `loopEventToOtelSpan` (one flat, @@ -9265,7 +14106,7 @@ readonly `object`[] > **buildLoopSpanNodes**(`events`): [`LoopSpanNode`](#loopspannode)[] -Defined in: [otel-export.ts:292](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L292) +Defined in: [otel-export.ts:395](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L395) Sink-neutral core behind [buildLoopOtelSpans](#buildloopotelspans): reconstruct the loop → round → branch span tree from one run's ordered `LoopTraceEvent` @@ -9290,7 +14131,7 @@ readonly `object`[] > **exportEvalRuns**(`events`, `config?`): `Promise`\<[`EvalRunsExportResult`](#evalrunsexportresult)\> -Defined in: [otel-export.ts:619](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L619) +Defined in: [otel-export.ts:728](https://github.com/tangle-network/agent-runtime/blob/main/src/otel-export.ts#L728) Ship self-improvement eval-run events to Tangle Intelligence. Unlike the best-effort span exporter, this RESOLVES with the ingest verdict (accepted / diff --git a/docs/api/intelligence.md b/docs/api/intelligence.md index 5402fd5b..582ef37b 100644 --- a/docs/api/intelligence.md +++ b/docs/api/intelligence.md @@ -486,7 +486,7 @@ Tear down provisioned hosts (reverse dependency order). ### CertifiedArtifact -Defined in: [intelligence/delivery.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L34) +Defined in: [intelligence/delivery.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L35) A promoted, certified artifact (one entry in the composed profile). @@ -496,31 +496,31 @@ A promoted, certified artifact (one entry in the composed profile). > **path**: `string` \| `null` -Defined in: [intelligence/delivery.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L35) +Defined in: [intelligence/delivery.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L36) ##### content > **content**: `string` -Defined in: [intelligence/delivery.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L36) +Defined in: [intelligence/delivery.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L37) ##### contentHash > **contentHash**: `string` -Defined in: [intelligence/delivery.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L37) +Defined in: [intelligence/delivery.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L38) ##### version > **version**: `number` \| `null` -Defined in: [intelligence/delivery.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L38) +Defined in: [intelligence/delivery.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L39) ##### lift > **lift**: `string` \| `null` -Defined in: [intelligence/delivery.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L41) +Defined in: [intelligence/delivery.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L42) Held-out gate lift attached at certification, e.g. "+3.1pp" — never a within-run claim. `null` when the promotion carried no lift record. @@ -529,13 +529,13 @@ Held-out gate lift attached at certification, e.g. "+3.1pp" — never a > **promotedAt**: `string` -Defined in: [intelligence/delivery.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L42) +Defined in: [intelligence/delivery.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L43) *** ### CertifiedPromptSurface -Defined in: [intelligence/delivery.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L46) +Defined in: [intelligence/delivery.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L47) The active promoted prompt surface for a target. @@ -545,31 +545,140 @@ The active promoted prompt surface for a target. > **surface**: `string` -Defined in: [intelligence/delivery.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L47) +Defined in: [intelligence/delivery.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L48) ##### surfaceHash > **surfaceHash**: `string` -Defined in: [intelligence/delivery.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L48) +Defined in: [intelligence/delivery.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L49) ##### version > **version**: `number` \| `null` -Defined in: [intelligence/delivery.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L49) +Defined in: [intelligence/delivery.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L50) ##### lift > **lift**: `string` \| `null` -Defined in: [intelligence/delivery.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L50) +Defined in: [intelligence/delivery.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L51) + +*** + +### DiffProvenance + +Defined in: [intelligence/delivery.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L56) + +The held-out provenance the plane's certify step stamps on a promoted diff. + `lift` is the held-out gate lift (e.g. "+3.1pp"), never a within-run claim. + +#### Properties + +##### version + +> **version**: `number` \| `null` + +Defined in: [intelligence/delivery.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L57) + +##### lift + +> **lift**: `string` \| `null` + +Defined in: [intelligence/delivery.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L58) + +##### contentHash + +> **contentHash**: `string` + +Defined in: [intelligence/delivery.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L59) + +##### promotedAt + +> **promotedAt**: `string` + +Defined in: [intelligence/delivery.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L60) + +*** + +### ProposedProfileDiff + +Defined in: [intelligence/delivery.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L70) + +A gate-certified profile diff the plane has already promoted, plus the +held-out provenance it carries. This is the previously-DROPPED typed diff the +composed endpoint returns; `withIntelligence` deserializes it and surfaces it +as a PROPOSAL — a human, or the gated local `improve()` loop, turns a proposal +into a shipped profile. It is NEVER auto-applied at runtime. + +#### Properties + +##### diff + +> **diff**: `AgentProfileDiff` + +Defined in: [intelligence/delivery.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L71) + +##### provenance + +> **provenance**: [`DiffProvenance`](#diffprovenance) + +Defined in: [intelligence/delivery.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L72) + +*** + +### CertifiedCapabilitySummary + +Defined in: [intelligence/delivery.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L78) + +The composed endpoint's per-capability summary — the narrow shape on the + wire (id + surface + path/content + provenance). Distinct from the richer + `CertifiedCapability` the capability resolver lowers a manifest into. + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [intelligence/delivery.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L79) + +##### iface + +> **iface**: `object` + +Defined in: [intelligence/delivery.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L80) + +###### surface + +> **surface**: `string` + +##### binding + +> **binding**: `object` + +Defined in: [intelligence/delivery.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L81) + +###### path + +> **path**: `string` \| `null` + +###### content + +> **content**: `string` + +##### provenance + +> **provenance**: [`DiffProvenance`](#diffprovenance) + +Defined in: [intelligence/delivery.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L82) *** ### CertifiedProfile -Defined in: [intelligence/delivery.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L55) +Defined in: [intelligence/delivery.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L87) The composed certified profile — exactly the shape the plane's `GET /v1/profiles/:target/composed` returns. @@ -580,31 +689,57 @@ The composed certified profile — exactly the shape the plane's > **target**: `string` -Defined in: [intelligence/delivery.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L56) +Defined in: [intelligence/delivery.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L88) ##### generatedAt > **generatedAt**: `string` -Defined in: [intelligence/delivery.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L57) +Defined in: [intelligence/delivery.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L89) ##### promptSurface > **promptSurface**: [`CertifiedPromptSurface`](#certifiedpromptsurface) \| `null` -Defined in: [intelligence/delivery.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L58) +Defined in: [intelligence/delivery.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L90) ##### artifacts > **artifacts**: `Record`\<`string`, [`CertifiedArtifact`](#certifiedartifact)[]\> -Defined in: [intelligence/delivery.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L59) +Defined in: [intelligence/delivery.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L91) + +##### agentProfileDiffs + +> **agentProfileDiffs**: [`ProposedProfileDiff`](#proposedprofilediff)[] + +Defined in: [intelligence/delivery.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L94) + +The typed profile diffs the plane has promoted, each with held-out + provenance. Surfaced as proposals; never auto-applied. Empty when none. + +##### capabilities + +> **capabilities**: [`CertifiedCapabilitySummary`](#certifiedcapabilitysummary)[] + +Defined in: [intelligence/delivery.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L96) + +The composed capability summaries the plane returns. Empty when none. + +##### agentProfile + +> **agentProfile**: `AgentProfile` \| `null` + +Defined in: [intelligence/delivery.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L99) + +The composed profile the promoted diffs fold to, for inspection. `null` + when no diffs are promoted. *** ### PullCertifiedOptions -Defined in: [intelligence/delivery.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L68) +Defined in: [intelligence/delivery.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L108) #### Extended by @@ -616,7 +751,7 @@ Defined in: [intelligence/delivery.ts:68](https://github.com/tangle-network/agen > **target**: `string` -Defined in: [intelligence/delivery.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L70) +Defined in: [intelligence/delivery.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L110) The agent target certified artifacts are promoted under. @@ -624,7 +759,7 @@ The agent target certified artifacts are promoted under. > `optional` **apiKey?**: `string` -Defined in: [intelligence/delivery.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L72) +Defined in: [intelligence/delivery.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L112) Bearer key. Defaults to `process.env.TANGLE_API_KEY`. @@ -632,7 +767,7 @@ Bearer key. Defaults to `process.env.TANGLE_API_KEY`. > `optional` **baseUrl?**: `string` -Defined in: [intelligence/delivery.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L75) +Defined in: [intelligence/delivery.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L115) Plane base URL. Defaults to `process.env.TANGLE_INTELLIGENCE_URL` then `https://intelligence.tangle.tools`. @@ -641,7 +776,7 @@ Plane base URL. Defaults to `process.env.TANGLE_INTELLIGENCE_URL` then > `optional` **fetchImpl?**: (`input`, `init?`) => `Promise`\<`Response`\> -Defined in: [intelligence/delivery.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L77) +Defined in: [intelligence/delivery.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L117) fetch impl (tests / non-global-fetch runtimes). Defaults to global fetch. @@ -663,7 +798,7 @@ fetch impl (tests / non-global-fetch runtimes). Defaults to global fetch. > `optional` **timeoutMs?**: `number` -Defined in: [intelligence/delivery.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L81) +Defined in: [intelligence/delivery.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L121) Abort the pull after this many ms so a hung plane never blocks the caller. Default 10000. The timeout surfaces as a normal fail-closed `succeeded: @@ -673,12 +808,12 @@ Abort the pull after this many ms so a hung plane never blocks the caller. ### CertifiedPromptSource -Defined in: [intelligence/delivery.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L187) +Defined in: [intelligence/delivery.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L276) A cached, self-refreshing source of a target's certified prompt additions — the prompt-only delivery lane for callers that assemble their OWN system prompt (product chat routes) rather than wrapping an agent fn. Same - fail-closed semantics as [withCertifiedDelivery](#withcertifieddelivery): pulls at most every + fail-closed semantics as [pullCertified](#pullcertified): pulls at most every `refreshMs`, coalesces concurrent pulls, keeps the last-known profile on a failed/404 pull, never throws, never blocks past the pull timeout. @@ -688,7 +823,7 @@ A cached, self-refreshing source of a target's certified prompt additions — > **compose**(`base`): `Promise`\<`string`\> -Defined in: [intelligence/delivery.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L190) +Defined in: [intelligence/delivery.ts:279](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L279) Refresh (window-respecting) then fold the certified additions into a base system prompt. Returns `base` unchanged when nothing is promoted. @@ -707,7 +842,7 @@ Refresh (window-respecting) then fold the certified additions into a > **current**(): [`CertifiedProfile`](#certifiedprofile) \| `null` -Defined in: [intelligence/delivery.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L192) +Defined in: [intelligence/delivery.ts:281](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L281) The certified profile currently in effect (`null` = none pulled yet). @@ -719,7 +854,7 @@ The certified profile currently in effect (`null` = none pulled yet). > **refresh**(): `Promise`\<`void`\> -Defined in: [intelligence/delivery.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L194) +Defined in: [intelligence/delivery.ts:283](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L283) Pull now if the refresh window has elapsed; coalesced and fail-closed. @@ -731,7 +866,7 @@ Pull now if the refresh window has elapsed; coalesced and fail-closed. ### CertifiedPromptSourceOptions -Defined in: [intelligence/delivery.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L199) +Defined in: [intelligence/delivery.ts:288](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L288) Options for [createCertifiedPromptSource](#createcertifiedpromptsource) — the pull coordinates plus the refresh cadence. @@ -746,7 +881,7 @@ Options for [createCertifiedPromptSource](#createcertifiedpromptsource) — the > **target**: `string` -Defined in: [intelligence/delivery.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L70) +Defined in: [intelligence/delivery.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L110) The agent target certified artifacts are promoted under. @@ -758,7 +893,7 @@ The agent target certified artifacts are promoted under. > `optional` **apiKey?**: `string` -Defined in: [intelligence/delivery.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L72) +Defined in: [intelligence/delivery.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L112) Bearer key. Defaults to `process.env.TANGLE_API_KEY`. @@ -770,7 +905,7 @@ Bearer key. Defaults to `process.env.TANGLE_API_KEY`. > `optional` **baseUrl?**: `string` -Defined in: [intelligence/delivery.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L75) +Defined in: [intelligence/delivery.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L115) Plane base URL. Defaults to `process.env.TANGLE_INTELLIGENCE_URL` then `https://intelligence.tangle.tools`. @@ -783,7 +918,7 @@ Plane base URL. Defaults to `process.env.TANGLE_INTELLIGENCE_URL` then > `optional` **fetchImpl?**: (`input`, `init?`) => `Promise`\<`Response`\> -Defined in: [intelligence/delivery.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L77) +Defined in: [intelligence/delivery.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L117) fetch impl (tests / non-global-fetch runtimes). Defaults to global fetch. @@ -809,7 +944,7 @@ fetch impl (tests / non-global-fetch runtimes). Defaults to global fetch. > `optional` **timeoutMs?**: `number` -Defined in: [intelligence/delivery.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L81) +Defined in: [intelligence/delivery.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L121) Abort the pull after this many ms so a hung plane never blocks the caller. Default 10000. The timeout surfaces as a normal fail-closed `succeeded: @@ -823,1077 +958,2141 @@ Abort the pull after this many ms so a hung plane never blocks the caller. > `optional` **refreshMs?**: `number` -Defined in: [intelligence/delivery.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L201) +Defined in: [intelligence/delivery.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L290) Min interval between certified-profile pulls. Default 5m. *** -### AppliedIntelligence +### EffortSettings -Defined in: [intelligence/delivery.ts:246](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L246) +Defined in: [intelligence/effort.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L32) -What the delivery wrapper hands the agent each run. +The flat, resolved settings a tier compiles to. Every field is individually +overridable through `resolveEffort`. Pure data — read by the wrapper, never +self-executing. #### Properties -##### certified +##### analysts -> **certified**: [`CertifiedProfile`](#certifiedprofile) \| `null` +> **analysts**: `boolean` -Defined in: [intelligence/delivery.ts:249](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L249) +Defined in: [intelligence/effort.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L34) -The certified profile in effect (null when none promoted / pull failed — - fail-closed: the agent runs on its base surface). +Whether trace-derived analyst diagnosis may spawn. `false` ⇒ no analyst. -#### Methods +##### corpus -##### composePrompt() +> **corpus**: [`CorpusAccess`](#corpusaccess) -> **composePrompt**(`base`): `string` +Defined in: [intelligence/effort.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L36) -Defined in: [intelligence/delivery.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L251) +Cross-run corpus access this tier permits. -Fold the certified prompt surface into a base system prompt. +##### fanout -###### Parameters +> **fanout**: `number` -###### base +Defined in: [intelligence/effort.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L38) -`string` +Parallel candidate width. `1` ⇒ single-shot, no breadth. -###### Returns +##### loops -`string` +> **loops**: `boolean` -*** +Defined in: [intelligence/effort.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L40) -### DeliveryConfig +Whether multi-step improvement loops (refine / fanout-vote) may run. -Defined in: [intelligence/delivery.ts:259](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L259) +##### intelligenceBudgetUsd -Delivery config = the Observe config plus the pull target + refresh cadence. +> **intelligenceBudgetUsd**: `number` \| `null` -#### Extends +Defined in: [intelligence/effort.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L47) -- [`IntelligenceConfig`](#intelligenceconfig) +Ceiling, in USD, for INTELLIGENCE-class spawns only (analysts, corpus, +loops) — NOT base inference. `0` refuses every intelligence spawn; `null` +means uncapped (the spend lands on the Pareto receipt). Base-stream +inference is billed on its own channel and is never constrained here. -#### Properties +*** -##### target? +### EffortOverridesCompiled -> `optional` **target?**: `string` +Defined in: [intelligence/effort.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L157) -Defined in: [intelligence/delivery.ts:261](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L261) +The run-config overrides an `EffortSettings` compiles to — the bridge between the +pure effort policy and the orchestration entrypoints (`runPersonified` / the +improvement cycle). This is ONLY data: it never constructs an analyst or runs a +loop. The caller reads these flags to decide WHAT to pass: -Pull target. Defaults to `project`. + - `withAnalyst: false` ⇒ DO NOT construct/pass a `ScopeAnalyst` to `runPersonified` + (the dormant empty-findings path runs; the base agent still works). This is the + PRODUCT fail-closed at `off`/`eco` — "don't construct the analyst" — distinct from + the EXPERIMENT fail-closed inside `createScopeAnalyst` ("hard abort"), which stays + untouched. Degrade, never throw. + - `fanout` ⇒ the `ShapeBudget.fanout` width to pass (`1` at `off`, the tier's breadth + otherwise). Overrides the personify default fanout. + - `withLoops: false` ⇒ the improvement cycle is a no-op for this run (no refine / + fanout-vote multi-step loop spawns). + - `intelligenceBudgetUsd` ⇒ the intelligence-class spend ceiling carried through for + the billing clamp (passed verbatim; `0` refuses every intelligence spawn). -##### baseUrl? +#### Properties -> `optional` **baseUrl?**: `string` +##### withAnalyst -Defined in: [intelligence/delivery.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L264) +> **withAnalyst**: `boolean` -Plane base URL for the pull (NOT the OTLP `endpoint`). Defaults to - `TANGLE_INTELLIGENCE_URL` then `https://intelligence.tangle.tools`. +Defined in: [intelligence/effort.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L159) -##### refreshMs? +Construct + pass a `ScopeAnalyst`? `false` ⇒ omit it (degrade to the base agent). -> `optional` **refreshMs?**: `number` +##### fanout -Defined in: [intelligence/delivery.ts:266](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L266) +> **fanout**: `number` -Min interval between certified-profile pulls. Default 5m. +Defined in: [intelligence/effort.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L161) -##### timeoutMs? +`ShapeBudget.fanout` width to pass to `runPersonified`. -> `optional` **timeoutMs?**: `number` +##### withLoops -Defined in: [intelligence/delivery.ts:268](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L268) +> **withLoops**: `boolean` -Per-pull timeout in ms (fail-closed on a hung plane). Default 10000. +Defined in: [intelligence/effort.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L163) -##### fetchImpl? +Run the multi-step improvement cycle, or no-op it for this run? -> `optional` **fetchImpl?**: (`input`, `init?`) => `Promise`\<`Response`\> +##### intelligenceBudgetUsd -Defined in: [intelligence/delivery.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L270) +> **intelligenceBudgetUsd**: `number` \| `null` -fetch impl for the pull (tests). Defaults to global fetch. +Defined in: [intelligence/effort.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L165) -###### Parameters +Intelligence-class spend ceiling. `0` refuses every intelligence spawn; `null` uncapped. -###### input +*** -`string` \| `URL` \| `Request` +### AgentImprovementProposal -###### init? +Defined in: [intelligence/improvement-cycle.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L62) -`RequestInit` +#### Type Parameters -###### Returns +##### TScenario -`Promise`\<`Response`\> +`TScenario` *extends* `Scenario` = `Scenario` -##### project +##### TArtifact -> **project**: `string` +`TArtifact` = `unknown` -Defined in: [intelligence/index.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L132) +#### Properties -Stable project id — the tenant dimension every trace is tagged with. +##### schemaVersion -###### Inherited from +> **schemaVersion**: `1` -[`IntelligenceConfig`](#intelligenceconfig).[`project`](#project-1) +Defined in: [intelligence/improvement-cycle.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L66) -##### apiKey? +##### kind -> `optional` **apiKey?**: `string` +> **kind**: `"agent-improvement-proposal"` -Defined in: [intelligence/index.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L134) +Defined in: [intelligence/improvement-cycle.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L67) -Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. +##### runId -###### Inherited from +> **runId**: `string` -[`IntelligenceConfig`](#intelligenceconfig).[`apiKey`](#apikey-3) +Defined in: [intelligence/improvement-cycle.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L68) -##### effort? +##### surface -> `optional` **effort?**: [`EffortTier`](#efforttier) \| \{ `tier`: [`EffortTier`](#efforttier); `overrides?`: `Partial`\<[`EffortSettings`](#effortsettings)\>; \} +> **surface**: [`ImproveSurface`](index.md#improvesurface) -Defined in: [intelligence/index.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L136) +Defined in: [intelligence/improvement-cycle.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L69) -Effort tier (default `'standard'`) plus optional per-field overrides. +##### proposedAt -###### Inherited from +> **proposedAt**: `string` -[`IntelligenceConfig`](#intelligenceconfig).[`effort`](#effort-1) +Defined in: [intelligence/improvement-cycle.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L70) -##### endpoint? +##### baselineProfile -> `optional` **endpoint?**: `string` +> **baselineProfile**: `AgentProfile` -Defined in: [intelligence/index.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L143) +Defined in: [intelligence/improvement-cycle.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L71) -OTLP ingest base. The underlying exporter appends `/v1/traces`, so point -this at the OTLP route (e.g. `https://intelligence.tangle.tools/v1/otlp`). -Reads `INTELLIGENCE_OTLP_ENDPOINT` then `OTEL_EXPORTER_OTLP_ENDPOINT` when -omitted; absent all three, export is a no-op (best-effort by construction). +##### baselineProfileHash -###### Inherited from +> **baselineProfileHash**: `string` -[`IntelligenceConfig`](#intelligenceconfig).[`endpoint`](#endpoint-1) +Defined in: [intelligence/improvement-cycle.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L72) -##### redact? +##### candidateProfile -> `optional` **redact?**: `false` \| [`Redactor`](#redactor) +> **candidateProfile**: `AgentProfile` -Defined in: [intelligence/index.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L149) +Defined in: [intelligence/improvement-cycle.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L73) -Redaction hook run over every exported input/output. A function replaces -the default scrubber; `false` opts out entirely (raw fidelity, caller has -sanitized upstream); omitted ⇒ the built-in `defaultRedactor`. +##### candidateProfileHash -###### Inherited from +> **candidateProfileHash**: `string` -[`IntelligenceConfig`](#intelligenceconfig).[`redact`](#redact-1) +Defined in: [intelligence/improvement-cycle.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L74) -##### surfaces? +##### findings -> `optional` **surfaces?**: `string`[] +> **findings**: `AnalystFinding`[] -Defined in: [intelligence/index.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L151) +Defined in: [intelligence/improvement-cycle.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L75) -Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. +##### evaluation -###### Inherited from +> **evaluation**: [`AgentImprovementEvaluation`](#agentimprovementevaluation)\<`TScenario`, `TArtifact`\> -[`IntelligenceConfig`](#intelligenceconfig).[`surfaces`](#surfaces-1) +Defined in: [intelligence/improvement-cycle.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L76) -##### checks? +##### candidateBundle? -> `optional` **checks?**: `string`[] +> `optional` **candidateBundle?**: `AgentCandidateBundleV1` -Defined in: [intelligence/index.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L153) +Defined in: [intelligence/improvement-cycle.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L77) -Verification checks a later PR mode would gate on. Recorded for `doctor()` only. +##### digest -###### Inherited from +> **digest**: `` `sha256:${string}` `` -[`IntelligenceConfig`](#intelligenceconfig).[`checks`](#checks-1) +Defined in: [intelligence/improvement-cycle.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L78) -##### repo? +*** -> `optional` **repo?**: [`RepoConfig`](#repoconfig) +### AgentImprovementReview -Defined in: [intelligence/index.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L155) +Defined in: [intelligence/improvement-cycle.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L83) -Repo access a later PR mode would need. Recorded for `doctor()` only. +#### Properties -###### Inherited from +##### schemaVersion -[`IntelligenceConfig`](#intelligenceconfig).[`repo`](#repo-1) +> **schemaVersion**: `1` -*** +Defined in: [intelligence/improvement-cycle.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L84) -### EffortSettings +##### kind -Defined in: [intelligence/effort.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L32) +> **kind**: `"agent-improvement-review"` -The flat, resolved settings a tier compiles to. Every field is individually -overridable through `resolveEffort`. Pure data — read by the wrapper, never -self-executing. +Defined in: [intelligence/improvement-cycle.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L85) -#### Properties +##### proposalDigest -##### analysts +> **proposalDigest**: `` `sha256:${string}` `` -> **analysts**: `boolean` +Defined in: [intelligence/improvement-cycle.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L86) -Defined in: [intelligence/effort.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L34) +##### candidateBundleDigest? -Whether trace-derived analyst diagnosis may spawn. `false` ⇒ no analyst. +> `optional` **candidateBundleDigest?**: `` `sha256:${string}` `` -##### corpus +Defined in: [intelligence/improvement-cycle.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L87) -> **corpus**: [`CorpusAccess`](#corpusaccess) +##### decision -Defined in: [intelligence/effort.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L36) +> **decision**: [`AgentImprovementReviewDecision`](#agentimprovementreviewdecision) -Cross-run corpus access this tier permits. +Defined in: [intelligence/improvement-cycle.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L88) -##### fanout +##### reviewedBy -> **fanout**: `number` +> **reviewedBy**: `string` -Defined in: [intelligence/effort.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L38) +Defined in: [intelligence/improvement-cycle.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L89) -Parallel candidate width. `1` ⇒ single-shot, no breadth. +##### reviewedAt -##### loops +> **reviewedAt**: `string` -> **loops**: `boolean` +Defined in: [intelligence/improvement-cycle.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L90) -Defined in: [intelligence/effort.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L40) +##### reason -Whether multi-step improvement loops (refine / fanout-vote) may run. +> **reason**: `string` -##### intelligenceBudgetUsd +Defined in: [intelligence/improvement-cycle.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L91) -> **intelligenceBudgetUsd**: `number` \| `null` +##### feedback? -Defined in: [intelligence/effort.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L47) +> `optional` **feedback?**: `string` -Ceiling, in USD, for INTELLIGENCE-class spawns only (analysts, corpus, -loops) — NOT base inference. `0` refuses every intelligence spawn; `null` -means uncapped (the spend lands on the Pareto receipt). Base-stream -inference is billed on its own channel and is never constrained here. +Defined in: [intelligence/improvement-cycle.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L92) -*** +##### digest -### EffortOverridesCompiled +> **digest**: `` `sha256:${string}` `` -Defined in: [intelligence/effort.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L157) +Defined in: [intelligence/improvement-cycle.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L93) -The run-config overrides an `EffortSettings` compiles to — the bridge between the -pure effort policy and the orchestration entrypoints (`runPersonified` / the -improvement cycle). This is ONLY data: it never constructs an analyst or runs a -loop. The caller reads these flags to decide WHAT to pass: +*** - - `withAnalyst: false` ⇒ DO NOT construct/pass a `ScopeAnalyst` to `runPersonified` - (the dormant empty-findings path runs; the base agent still works). This is the - PRODUCT fail-closed at `off`/`eco` — "don't construct the analyst" — distinct from - the EXPERIMENT fail-closed inside `createScopeAnalyst` ("hard abort"), which stays - untouched. Degrade, never throw. - - `fanout` ⇒ the `ShapeBudget.fanout` width to pass (`1` at `off`, the tier's breadth - otherwise). Overrides the personify default fanout. - - `withLoops: false` ⇒ the improvement cycle is a no-op for this run (no refine / - fanout-vote multi-step loop spawns). - - `intelligenceBudgetUsd` ⇒ the intelligence-class spend ceiling carried through for - the billing clamp (passed verbatim; `0` refuses every intelligence spawn). +### CandidateExecutionEvidence -#### Properties +Defined in: [intelligence/improvement-cycle.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L96) -##### withAnalyst +#### Properties -> **withAnalyst**: `boolean` +##### proposalDigest -Defined in: [intelligence/effort.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L159) +> **proposalDigest**: `` `sha256:${string}` `` -Construct + pass a `ScopeAnalyst`? `false` ⇒ omit it (degrade to the base agent). +Defined in: [intelligence/improvement-cycle.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L97) -##### fanout +##### reviewDigest -> **fanout**: `number` +> **reviewDigest**: `` `sha256:${string}` `` -Defined in: [intelligence/effort.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L161) +Defined in: [intelligence/improvement-cycle.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L98) -`ShapeBudget.fanout` width to pass to `runPersonified`. +##### bundleDigest -##### withLoops +> **bundleDigest**: `` `sha256:${string}` `` -> **withLoops**: `boolean` +Defined in: [intelligence/improvement-cycle.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L99) -Defined in: [intelligence/effort.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L163) +##### executionId -Run the multi-step improvement cycle, or no-op it for this run? +> **executionId**: `string` -##### intelligenceBudgetUsd +Defined in: [intelligence/improvement-cycle.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L100) -> **intelligenceBudgetUsd**: `number` \| `null` +##### executionPlanDigest -Defined in: [intelligence/effort.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/effort.ts#L165) +> **executionPlanDigest**: `` `sha256:${string}` `` -Intelligence-class spend ceiling. `0` refuses every intelligence spawn; `null` uncapped. +Defined in: [intelligence/improvement-cycle.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L101) -*** +##### materializationReceiptDigest -### UsageSplit +> **materializationReceiptDigest**: `` `sha256:${string}` `` -Defined in: [intelligence/index.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L111) +Defined in: [intelligence/improvement-cycle.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L102) -The per-class cost split carried by every trace and outcome. `off` ⇒ -`intelligenceUsd: 0` by construction — there is no intelligence spawn to -bill. This is a classification on the trace, NOT a budget-pool split. +##### succeeded -#### Properties +> **succeeded**: `boolean` -##### inferenceUsd +Defined in: [intelligence/improvement-cycle.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L103) -> **inferenceUsd**: `number` +##### runReceiptDigest? -Defined in: [intelligence/index.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L113) +> `optional` **runReceiptDigest?**: `` `sha256:${string}` `` -Base-stream (model) spend in USD. +Defined in: [intelligence/improvement-cycle.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L104) -##### intelligenceUsd +*** -> **intelligenceUsd**: `number` +### ProposeAgentImprovementOptions -Defined in: [intelligence/index.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L115) +Defined in: [intelligence/improvement-cycle.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L107) -Intelligence-spawn spend in USD. Provably `0` at the OFF tier. +#### Type Parameters -*** +##### TScenario -### RepoConfig +`TScenario` *extends* `Scenario` -Defined in: [intelligence/index.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L121) +##### TArtifact -Repo coordinates a product may declare for the (later) Gated-PR mode. The - Observe slice only records their PRESENCE for `doctor()`; it never touches - the repo. +`TArtifact` #### Properties -##### owner - -> **owner**: `string` +##### runId -Defined in: [intelligence/index.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L122) +> **runId**: `string` -##### name +Defined in: [intelligence/improvement-cycle.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L108) -> **name**: `string` +##### profile -Defined in: [intelligence/index.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L123) +> **profile**: `AgentProfile` -##### baseBranch +Defined in: [intelligence/improvement-cycle.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L109) -> **baseBranch**: `string` +##### analysis -Defined in: [intelligence/index.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L124) +> **analysis**: `Omit`\<[`RunAnalystLoopOpts`](analyst-loop.md#runanalystloopopts), `"runId"` \| `"improvementAdapter"` \| `"autoApply"`\> -*** +Defined in: [intelligence/improvement-cycle.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L110) -### IntelligenceConfig +##### improvement -Defined in: [intelligence/index.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L130) +> **improvement**: [`ImproveOptions`](index.md#improveoptions)\<`TScenario`, `TArtifact`\> -Client configuration. `project` + `apiKey` are the Observe minimum; the - rest tune effort, endpoint, redaction, and (for `doctor()` readiness) - declare the surfaces/checks/repo a later PR mode would need. +Defined in: [intelligence/improvement-cycle.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L111) -#### Extended by +##### buildCandidate? -- [`DeliveryConfig`](#deliveryconfig) +> `optional` **buildCandidate?**: (`input`) => `AgentCandidateBundleV1` \| [`AgentCandidateBundleInput`](index.md#agentcandidatebundleinput) \| `Promise`\<`AgentCandidateBundleV1` \| [`AgentCandidateBundleInput`](index.md#agentcandidatebundleinput)\> -#### Properties +Defined in: [intelligence/improvement-cycle.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L117) -##### project +Optional environment adapter that freezes an executable bundle after the +measured comparison recommends the candidate. Return the sealed output of +`buildAgentCandidateBundle` directly, or a low-level digest-free input. -> **project**: `string` +###### Parameters -Defined in: [intelligence/index.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L132) +###### input -Stable project id — the tenant dimension every trace is tagged with. +###### analysis -##### apiKey? +[`RunAnalystLoopResult`](analyst-loop.md#runanalystloopresult) -> `optional` **apiKey?**: `string` +###### improvement -Defined in: [intelligence/index.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L134) +[`ImproveResult`](index.md#improveresult)\<`TScenario`, `TArtifact`\> -Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. +###### Returns -##### effort? +`AgentCandidateBundleV1` \| [`AgentCandidateBundleInput`](index.md#agentcandidatebundleinput) \| `Promise`\<`AgentCandidateBundleV1` \| [`AgentCandidateBundleInput`](index.md#agentcandidatebundleinput)\> -> `optional` **effort?**: [`EffortTier`](#efforttier) \| \{ `tier`: [`EffortTier`](#efforttier); `overrides?`: `Partial`\<[`EffortSettings`](#effortsettings)\>; \} +##### now? -Defined in: [intelligence/index.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L136) +> `optional` **now?**: () => `Date` -Effort tier (default `'standard'`) plus optional per-field overrides. +Defined in: [intelligence/improvement-cycle.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L124) -##### endpoint? +###### Returns -> `optional` **endpoint?**: `string` +`Date` -Defined in: [intelligence/index.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L143) +*** -OTLP ingest base. The underlying exporter appends `/v1/traces`, so point -this at the OTLP route (e.g. `https://intelligence.tangle.tools/v1/otlp`). -Reads `INTELLIGENCE_OTLP_ENDPOINT` then `OTEL_EXPORTER_OTLP_ENDPOINT` when -omitted; absent all three, export is a no-op (best-effort by construction). +### ProposeAgentImprovementResult -##### redact? +Defined in: [intelligence/improvement-cycle.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L127) -> `optional` **redact?**: `false` \| [`Redactor`](#redactor) +#### Type Parameters -Defined in: [intelligence/index.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L149) +##### TScenario -Redaction hook run over every exported input/output. A function replaces -the default scrubber; `false` opts out entirely (raw fidelity, caller has -sanitized upstream); omitted ⇒ the built-in `defaultRedactor`. +`TScenario` *extends* `Scenario` -##### surfaces? +##### TArtifact -> `optional` **surfaces?**: `string`[] +`TArtifact` -Defined in: [intelligence/index.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L151) +#### Properties -Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. +##### analysis -##### checks? +> **analysis**: [`RunAnalystLoopResult`](analyst-loop.md#runanalystloopresult) -> `optional` **checks?**: `string`[] +Defined in: [intelligence/improvement-cycle.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L128) -Defined in: [intelligence/index.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L153) +##### improvement -Verification checks a later PR mode would gate on. Recorded for `doctor()` only. +> **improvement**: [`ImproveResult`](index.md#improveresult)\<`TScenario`, `TArtifact`\> -##### repo? +Defined in: [intelligence/improvement-cycle.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L129) -> `optional` **repo?**: [`RepoConfig`](#repoconfig) +##### proposal -Defined in: [intelligence/index.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L155) +> **proposal**: [`AgentImprovementProposal`](#agentimprovementproposal)\<`TScenario`, `TArtifact`\> -Repo access a later PR mode would need. Recorded for `doctor()` only. +Defined in: [intelligence/improvement-cycle.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L130) *** -### TraceMeta - -Defined in: [intelligence/index.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L159) - -Metadata describing one traced run. `runId`/`traceId` default to fresh ids. - -#### Properties +### CreateAgentImprovementProposalOptions -##### input? +Defined in: [intelligence/improvement-cycle.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L133) -> `optional` **input?**: `unknown` +#### Type Parameters -Defined in: [intelligence/index.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L161) +##### TScenario -The run's input — exported through the redactor. +`TScenario` *extends* `Scenario` -##### runId? +##### TArtifact -> `optional` **runId?**: `string` +`TArtifact` -Defined in: [intelligence/index.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L163) +#### Properties -Stable run id. Defaults to a fresh id. +##### runId -##### traceId? +> **runId**: `string` -> `optional` **traceId?**: `string` +Defined in: [intelligence/improvement-cycle.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L134) -Defined in: [intelligence/index.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L165) +##### surface -32-hex trace id. Defaults to a fresh id. +> **surface**: [`ImproveSurface`](index.md#improvesurface) -##### model? +Defined in: [intelligence/improvement-cycle.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L135) -> `optional` **model?**: `string` +##### baselineProfile -Defined in: [intelligence/index.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L167) +> **baselineProfile**: `AgentProfile` -Model id, when known — stamped on the span. +Defined in: [intelligence/improvement-cycle.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L136) -##### provider? +##### candidateProfile -> `optional` **provider?**: `string` +> **candidateProfile**: `AgentProfile` -Defined in: [intelligence/index.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L169) +Defined in: [intelligence/improvement-cycle.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L137) -Provider name, when known — stamped on the span. +##### findings -##### labels? +> **findings**: readonly `AnalystFinding`[] -> `optional` **labels?**: `Record`\<`string`, `string` \| `number` \| `boolean`\> +Defined in: [intelligence/improvement-cycle.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L138) -Defined in: [intelligence/index.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L171) +##### evaluation -Arbitrary extra labels (string/number/boolean) stamped on the span. +> **evaluation**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> -*** +Defined in: [intelligence/improvement-cycle.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L139) -### TraceHandle +##### candidateBundle? -Defined in: [intelligence/index.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L180) +> `optional` **candidateBundle?**: `AgentCandidateBundleV1` \| [`AgentCandidateBundleInput`](index.md#agentcandidatebundleinput) -The trace handle a `traceRun` body records into. `recordOutput` captures the -agent's result (redacted on export); `recordOutcome` captures the scored -outcome + the `{ inferenceUsd, intelligenceUsd }` split. Both are optional — -an un-recorded run still exports a span with whatever was set. +Defined in: [intelligence/improvement-cycle.ts:140](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L140) -#### Methods +##### now? -##### recordOutput() +> `optional` **now?**: () => `Date` -> **recordOutput**(`output`): `void` +Defined in: [intelligence/improvement-cycle.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L141) -Defined in: [intelligence/index.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L182) +###### Returns -Capture the run's output. Exported through the redactor. +`Date` -###### Parameters +*** -###### output +### ReviewAgentImprovementInput -`unknown` +Defined in: [intelligence/improvement-cycle.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L144) -###### Returns +#### Properties -`void` +##### decision -##### recordOutcome() +> **decision**: [`AgentImprovementReviewDecision`](#agentimprovementreviewdecision) -> **recordOutcome**(`outcome`): `void` +Defined in: [intelligence/improvement-cycle.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L145) -Defined in: [intelligence/index.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L189) +##### reviewedBy -Capture the run's outcome. `usage` defaults to inference-only -(`intelligenceUsd: 0`) — the OFF baseline; an intelligence-enabled run -fills `intelligenceUsd` itself. `costUsd`, when given without a split, is -treated as pure inference. +> **reviewedBy**: `string` -###### Parameters +Defined in: [intelligence/improvement-cycle.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L146) -###### outcome +##### reason -###### success? +> **reason**: `string` -`boolean` +Defined in: [intelligence/improvement-cycle.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L147) -###### score? +##### feedback? -`number` +> `optional` **feedback?**: `string` -###### costUsd? +Defined in: [intelligence/improvement-cycle.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L148) -`number` +##### now? -###### usage? +> `optional` **now?**: () => `Date` -`Partial`\<[`UsageSplit`](#usagesplit)\> +Defined in: [intelligence/improvement-cycle.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L149) ###### Returns -`void` +`Date` *** -### RecordTraceMeta - -Defined in: [intelligence/index.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L198) +### ExecuteApprovedAgentCandidateOptions -Metadata for [IntelligenceClient.recordTrace](#recordtrace). +Defined in: [intelligence/improvement-cycle.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L152) #### Properties -##### traceId? +##### proposal -> `optional` **traceId?**: `string` +> **proposal**: [`AgentImprovementProposal`](#agentimprovementproposal) -Defined in: [intelligence/index.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L200) +Defined in: [intelligence/improvement-cycle.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L153) -32-hex trace id to anchor every span to. Defaults to a fresh id. +##### review -##### rootParentSpanId? +> **review**: [`AgentImprovementReview`](#agentimprovementreview) -> `optional` **rootParentSpanId?**: `string` +Defined in: [intelligence/improvement-cycle.ts:154](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L154) -Defined in: [intelligence/index.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L203) +##### authorizeReview -Span id of an enclosing span the loop root should parent under (e.g. a - `traceRun` span). Omitted ⇒ the loop root is the trace root. +> **authorizeReview**: (`review`, `proposal`) => `boolean` \| `Promise`\<`boolean`\> -*** +Defined in: [intelligence/improvement-cycle.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L156) -### TraceOutcome +Product-owned authentication check for the persisted approval record. -Defined in: [intelligence/index.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L208) +###### Parameters -The resolved outcome of one traced run, surfaced on the export span and - available to the caller for downstream billing assertions. +###### review -#### Properties +[`AgentImprovementReview`](#agentimprovementreview) -##### runId +###### proposal -> **runId**: `string` +[`AgentImprovementProposal`](#agentimprovementproposal) -Defined in: [intelligence/index.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L209) +###### Returns -##### traceId +`boolean` \| `Promise`\<`boolean`\> -> **traceId**: `string` +##### task -Defined in: [intelligence/index.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L210) +> **task**: [`AgentCandidateTaskExecution`](index.md#agentcandidatetaskexecution) -##### project +Defined in: [intelligence/improvement-cycle.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L160) -> **project**: `string` +##### ports -Defined in: [intelligence/index.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L211) +> **ports**: [`AgentCandidateExecutionPorts`](index.md#agentcandidateexecutionports) -##### effort +Defined in: [intelligence/improvement-cycle.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L161) -> **effort**: [`EffortSettings`](#effortsettings) +##### preparation? -Defined in: [intelligence/index.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L213) +> `optional` **preparation?**: [`PrepareAgentCandidateExecutionOptions`](index.md#prepareagentcandidateexecutionoptions) -The resolved effort settings this run executed under. +Defined in: [intelligence/improvement-cycle.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L162) -##### intelligenceOff +##### execution -> **intelligenceOff**: `boolean` +> **execution**: [`ExecutePreparedAgentCandidateOptions`](index.md#executepreparedagentcandidateoptions) -Defined in: [intelligence/index.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L215) +Defined in: [intelligence/improvement-cycle.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L163) -True when this run ran as pure passthrough (the OFF floor). +*** -##### success? +### ExecuteApprovedAgentCandidateResult -> `optional` **success?**: `boolean` +Defined in: [intelligence/improvement-cycle.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L166) -Defined in: [intelligence/index.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L216) +#### Properties -##### score? +##### finalization -> `optional` **score?**: `number` +> **finalization**: [`AgentCandidateRunFinalization`](index.md#agentcandidaterunfinalization) + +Defined in: [intelligence/improvement-cycle.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L167) + +##### evidence + +> **evidence**: [`CandidateExecutionEvidence`](#candidateexecutionevidence) + +Defined in: [intelligence/improvement-cycle.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L168) + +*** + +### UsageSplit + +Defined in: [intelligence/index.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L142) + +The per-class cost split carried by every trace and outcome. `off` ⇒ +`intelligenceUsd: 0` by construction — there is no intelligence spawn to +bill. This is a classification on the trace, NOT a budget-pool split. + +#### Properties + +##### inferenceUsd + +> **inferenceUsd**: `number` + +Defined in: [intelligence/index.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L144) + +Base-stream (model) spend in USD. + +##### intelligenceUsd + +> **intelligenceUsd**: `number` + +Defined in: [intelligence/index.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L146) + +Intelligence-spawn spend in USD. Provably `0` at the OFF tier. + +*** + +### RunRecord + +Defined in: [intelligence/index.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L156) + +The typed record `withIntelligence` sends per call — serialized through the +shipped OTLP builders to the plane's `/v1/otlp` ingest. `input`/`output` are +redacted on export; the per-class `usage` split carries the billing proof; +`loopEvents`, when present, export as the nested loop→round→iteration span +tree under the same `traceId`. + +#### Properties + +##### runId + +> **runId**: `string` + +Defined in: [intelligence/index.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L157) + +##### traceId + +> **traceId**: `string` + +Defined in: [intelligence/index.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L158) + +##### project + +> **project**: `string` + +Defined in: [intelligence/index.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L159) + +##### target + +> **target**: `string` + +Defined in: [intelligence/index.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L160) + +##### input + +> **input**: `unknown` + +Defined in: [intelligence/index.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L161) + +##### output + +> **output**: `unknown` + +Defined in: [intelligence/index.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L162) + +##### outcome + +> **outcome**: `object` + +Defined in: [intelligence/index.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L163) + +###### success? + +> `optional` **success?**: `boolean` + +###### score? + +> `optional` **score?**: `number` + +###### usage + +> **usage**: [`UsageSplit`](#usagesplit) + +##### model? + +> `optional` **model?**: `string` + +Defined in: [intelligence/index.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L168) + +##### provider? + +> `optional` **provider?**: `string` + +Defined in: [intelligence/index.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L169) + +##### loopEvents? + +> `optional` **loopEvents?**: [`LoopTraceEvent`](runtime.md#looptraceevent)[] + +Defined in: [intelligence/index.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L170) + +##### runtimeEvents? + +> `optional` **runtimeEvents?**: [`RuntimeStreamEvent`](index.md#runtimestreamevent)[] + +Defined in: [intelligence/index.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L171) + +##### profile? + +> `optional` **profile?**: `AgentProfile` + +Defined in: [intelligence/index.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L172) + +##### sessionId? + +> `optional` **sessionId?**: `string` + +Defined in: [intelligence/index.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L173) + +##### harness? + +> `optional` **harness?**: `string` + +Defined in: [intelligence/index.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L174) + +##### repository? + +> `optional` **repository?**: `string` + +Defined in: [intelligence/index.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L175) + +##### commitSha? + +> `optional` **commitSha?**: `string` + +Defined in: [intelligence/index.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L176) + +##### timing? + +> `optional` **timing?**: `object` + +Defined in: [intelligence/index.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L177) + +###### startedAt + +> **startedAt**: `number` + +###### completedAt + +> **completedAt**: `number` + +###### durationMs + +> **durationMs**: `number` + +##### tokens? + +> `optional` **tokens?**: `object` + +Defined in: [intelligence/index.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L178) + +###### input + +> **input**: `number` + +###### output + +> **output**: `number` + +###### cachedInput? + +> `optional` **cachedInput?**: `number` + +###### reasoning? + +> `optional` **reasoning?**: `number` + +##### error? + +> `optional` **error?**: `object` + +Defined in: [intelligence/index.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L184) + +###### name + +> **name**: `string` + +###### message + +> **message**: `string` + +###### code? + +> `optional` **code?**: `string` + +##### candidateExecution? + +> `optional` **candidateExecution?**: [`CandidateExecutionEvidence`](#candidateexecutionevidence) + +Defined in: [intelligence/index.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L186) + +Exact proposal → review → execution → receipt linkage for candidate runs. + +*** + +### RunReport + +Defined in: [intelligence/index.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L195) + +What an agent reports (via `applied.record`) to enrich the [RunRecord](#runrecord) +sent for its call. All optional — an un-recorded run still sends input/output +with an inference-only zero usage split. `costUsd` without a split is treated +as pure inference (the base stream). + +#### Properties + +##### success? + +> `optional` **success?**: `boolean` + +Defined in: [intelligence/index.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L196) + +##### score? + +> `optional` **score?**: `number` + +Defined in: [intelligence/index.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L197) + +##### usage? + +> `optional` **usage?**: `Partial`\<[`UsageSplit`](#usagesplit)\> + +Defined in: [intelligence/index.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L198) + +##### costUsd? + +> `optional` **costUsd?**: `number` + +Defined in: [intelligence/index.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L199) + +##### model? + +> `optional` **model?**: `string` + +Defined in: [intelligence/index.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L200) + +##### provider? + +> `optional` **provider?**: `string` + +Defined in: [intelligence/index.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L201) + +##### loopEvents? + +> `optional` **loopEvents?**: [`LoopTraceEvent`](runtime.md#looptraceevent)[] + +Defined in: [intelligence/index.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L202) + +##### runtimeEvents? + +> `optional` **runtimeEvents?**: [`RuntimeStreamEvent`](index.md#runtimestreamevent)[] + +Defined in: [intelligence/index.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L203) + +##### profile? + +> `optional` **profile?**: `AgentProfile` + +Defined in: [intelligence/index.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L204) + +##### sessionId? + +> `optional` **sessionId?**: `string` + +Defined in: [intelligence/index.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L205) + +##### harness? + +> `optional` **harness?**: `string` + +Defined in: [intelligence/index.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L206) + +##### commitSha? + +> `optional` **commitSha?**: `string` + +Defined in: [intelligence/index.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L207) + +##### tokens? + +> `optional` **tokens?**: `object` + +Defined in: [intelligence/index.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L208) + +###### input + +> **input**: `number` + +###### output + +> **output**: `number` + +###### cachedInput? + +> `optional` **cachedInput?**: `number` + +###### reasoning? + +> `optional` **reasoning?**: `number` + +##### error? + +> `optional` **error?**: `object` + +Defined in: [intelligence/index.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L209) + +###### name + +> **name**: `string` + +###### message + +> **message**: `string` + +###### code? + +> `optional` **code?**: `string` + +##### candidateExecution? + +> `optional` **candidateExecution?**: [`CandidateExecutionEvidence`](#candidateexecutionevidence) + +Defined in: [intelligence/index.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L210) + +*** + +### RepoConfig + +Defined in: [intelligence/index.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L216) + +Repo coordinates a product may declare for the (later) Gated-PR mode. The + Observe slice only records their PRESENCE for `doctor()`; it never touches + the repo. + +#### Properties + +##### owner + +> **owner**: `string` Defined in: [intelligence/index.ts:217](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L217) -##### usage +##### name + +> **name**: `string` + +Defined in: [intelligence/index.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L218) + +##### baseBranch + +> **baseBranch**: `string` + +Defined in: [intelligence/index.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L219) + +*** + +### IntelligenceConfig + +Defined in: [intelligence/index.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L225) + +Client configuration. `project` + `apiKey` are the Observe minimum; the + rest tune effort, endpoint, redaction, and (for `doctor()` readiness) + declare the surfaces/checks/repo a later PR mode would need. + +#### Extended by + +- [`IntelligenceHookConfig`](#intelligencehookconfig) + +#### Properties + +##### project + +> **project**: `string` + +Defined in: [intelligence/index.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L227) + +Stable project id — the tenant dimension every trace is tagged with. + +##### apiKey? + +> `optional` **apiKey?**: `string` + +Defined in: [intelligence/index.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L229) + +Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. + +##### effort? + +> `optional` **effort?**: [`EffortTier`](#efforttier) \| \{ `tier`: [`EffortTier`](#efforttier); `overrides?`: `Partial`\<[`EffortSettings`](#effortsettings)\>; \} + +Defined in: [intelligence/index.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L231) + +Effort tier (default `'standard'`) plus optional per-field overrides. + +##### baseUrl? + +> `optional` **baseUrl?**: `string` + +Defined in: [intelligence/index.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L239) + +The ONE Tangle Intelligence base URL — both the send (OTLP `/v1/otlp`) and +receive (`/v1/profiles/:target/composed`) paths derive from it. Reads +`TANGLE_INTELLIGENCE_URL` when omitted, else `https://intelligence.tangle.tools`. +Send is best-effort and only ships when an `apiKey` is present (the tenant +key the ingest requires); absent a key, export is a no-op. + +##### redact? + +> `optional` **redact?**: `false` \| [`Redactor`](#redactor) + +Defined in: [intelligence/index.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L245) + +Redaction hook run over every exported input/output. A function replaces +the default scrubber; `false` opts out entirely (raw fidelity, caller has +sanitized upstream); omitted ⇒ the built-in `defaultRedactor`. + +##### surfaces? + +> `optional` **surfaces?**: `string`[] + +Defined in: [intelligence/index.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L247) + +Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. + +##### checks? + +> `optional` **checks?**: `string`[] + +Defined in: [intelligence/index.ts:249](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L249) + +Verification checks a later PR mode would gate on. Recorded for `doctor()` only. + +##### repo? + +> `optional` **repo?**: [`RepoConfig`](#repoconfig) + +Defined in: [intelligence/index.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L251) + +Repo access a later PR mode would need. Recorded for `doctor()` only. + +##### profile? + +> `optional` **profile?**: `AgentProfile` + +Defined in: [intelligence/index.ts:253](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L253) + +Full canonical profile used for this agent. Exported redacted with a stable hash. + +##### commitSha? + +> `optional` **commitSha?**: `string` + +Defined in: [intelligence/index.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L255) + +Commit that produced the running agent, when known. + +##### runtimeTelemetry? + +> `optional` **runtimeTelemetry?**: [`RuntimeTelemetryOptions`](index.md#runtimetelemetryoptions) + +Defined in: [intelligence/index.ts:257](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L257) + +Runtime-event payload policy. Tool inputs/results remain off unless explicitly enabled. + +##### payloadAttributes? + +> `optional` **payloadAttributes?**: `"metadata"` \| `"full"` + +Defined in: [intelligence/index.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L264) + +Payloads are metadata-only by default: the run span carries a stable hash +and UTF-8 byte count, but not the redacted content. Set `full` only when +the configured OTLP destination is approved to receive complete redacted +inputs, outputs, and profiles. + +*** + +### TraceMeta + +Defined in: [intelligence/index.ts:268](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L268) + +Metadata describing one traced run. `runId`/`traceId` default to fresh ids. + +#### Properties + +##### input? + +> `optional` **input?**: `unknown` + +Defined in: [intelligence/index.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L270) + +The run's input — exported through the redactor. + +##### runId? + +> `optional` **runId?**: `string` + +Defined in: [intelligence/index.ts:272](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L272) + +Stable run id. Defaults to a fresh id. + +##### traceId? + +> `optional` **traceId?**: `string` + +Defined in: [intelligence/index.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L274) + +32-hex trace id. Defaults to a fresh id. + +##### model? + +> `optional` **model?**: `string` + +Defined in: [intelligence/index.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L276) + +Model id, when known — stamped on the span. + +##### provider? + +> `optional` **provider?**: `string` + +Defined in: [intelligence/index.ts:278](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L278) + +Provider name, when known — stamped on the span. + +##### labels? + +> `optional` **labels?**: `Record`\<`string`, `string` \| `number` \| `boolean`\> + +Defined in: [intelligence/index.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L280) + +Arbitrary extra labels (string/number/boolean) stamped on the span. + +*** + +### TraceHandle + +Defined in: [intelligence/index.ts:289](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L289) + +The trace handle a `traceRun` body records into. `recordOutput` captures the +agent's result (redacted on export); `recordOutcome` captures the scored +outcome + the `{ inferenceUsd, intelligenceUsd }` split. Both are optional — +an un-recorded run still exports a span with whatever was set. + +#### Methods + +##### recordOutput() + +> **recordOutput**(`output`): `void` + +Defined in: [intelligence/index.ts:291](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L291) + +Capture the run's output. Exported through the redactor. + +###### Parameters + +###### output + +`unknown` + +###### Returns + +`void` + +##### recordOutcome() + +> **recordOutcome**(`outcome`): `void` + +Defined in: [intelligence/index.ts:298](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L298) + +Capture the run's outcome. `usage` defaults to inference-only +(`intelligenceUsd: 0`) — the OFF baseline; an intelligence-enabled run +fills `intelligenceUsd` itself. `costUsd`, when given without a split, is +treated as pure inference. + +###### Parameters + +###### outcome + +###### success? + +`boolean` + +###### score? + +`number` + +###### costUsd? + +`number` + +###### usage? + +`Partial`\<[`UsageSplit`](#usagesplit)\> + +###### Returns + +`void` + +*** + +### RecordTraceMeta + +Defined in: [intelligence/index.ts:307](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L307) + +Metadata for [IntelligenceClient.recordTrace](#recordtrace). + +#### Properties + +##### traceId? + +> `optional` **traceId?**: `string` + +Defined in: [intelligence/index.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L309) + +32-hex trace id to anchor every span to. Defaults to a fresh id. + +##### rootParentSpanId? + +> `optional` **rootParentSpanId?**: `string` + +Defined in: [intelligence/index.ts:312](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L312) + +Span id of an enclosing span the loop root should parent under (e.g. a + `traceRun` span). Omitted ⇒ the loop root is the trace root. + +*** + +### TraceOutcome + +Defined in: [intelligence/index.ts:317](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L317) + +The resolved outcome of one traced run, surfaced on the export span and + available to the caller for downstream billing assertions. + +#### Properties + +##### runId + +> **runId**: `string` + +Defined in: [intelligence/index.ts:318](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L318) + +##### traceId + +> **traceId**: `string` + +Defined in: [intelligence/index.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L319) + +##### project + +> **project**: `string` + +Defined in: [intelligence/index.ts:320](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L320) + +##### effort + +> **effort**: [`EffortSettings`](#effortsettings) + +Defined in: [intelligence/index.ts:322](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L322) + +The resolved effort settings this run executed under. + +##### intelligenceOff + +> **intelligenceOff**: `boolean` + +Defined in: [intelligence/index.ts:324](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L324) + +True when this run ran as pure passthrough (the OFF floor). + +##### success? + +> `optional` **success?**: `boolean` + +Defined in: [intelligence/index.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L325) + +##### score? + +> `optional` **score?**: `number` + +Defined in: [intelligence/index.ts:326](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L326) + +##### usage + +> **usage**: [`UsageSplit`](#usagesplit) + +Defined in: [intelligence/index.ts:328](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L328) + +Per-class billing split. `intelligenceUsd` is `0` at the OFF tier. + +*** + +### IntelligenceClient + +Defined in: [intelligence/index.ts:332](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L332) + +The Observe-mode Intelligence client. + +#### Properties + +##### project + +> `readonly` **project**: `string` + +Defined in: [intelligence/index.ts:334](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L334) + +The resolved project id. + +##### effort + +> `readonly` **effort**: [`EffortSettings`](#effortsettings) + +Defined in: [intelligence/index.ts:336](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L336) + +The resolved effort settings. + +#### Methods + +##### traceRun() + +> **traceRun**\<`T`\>(`meta`, `fn`): `Promise`\<`T`\> + +Defined in: [intelligence/index.ts:342](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L342) + +Run `fn` under a trace, export one span best-effort, and return whatever +`fn` returns. Telemetry-export failures are swallowed; an error THROWN by +`fn` propagates to the caller (the agent's own failures are not masked). + +###### Type Parameters + +###### T + +`T` + +###### Parameters + +###### meta + +[`TraceMeta`](#tracemeta) + +###### fn + +(`trace`) => `Promise`\<`T`\> + +###### Returns + +`Promise`\<`T`\> + +##### recordTrace() + +> **recordTrace**(`events`, `meta?`): `string` + +Defined in: [intelligence/index.ts:352](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L352) + +Export a run's full loop topology — the ordered `LoopTraceEvent` stream a +`runLoop`/`Supervisor` run emits — as a nested OTLP span tree (loop → round → +iteration) into ONE trace. Reuses the shipped `buildLoopOtelSpans` builder +(NO second span builder), so the topology a viewer renders matches the +kernel's. `traceId` defaults to a fresh id; `rootParentSpanId` parents the +loop root under an enclosing span (e.g. a `traceRun` span) when given. +Best-effort: export failures are swallowed. Returns the resolved `traceId`. + +###### Parameters + +###### events + +readonly [`LoopTraceEvent`](runtime.md#looptraceevent)[] + +###### meta? + +[`RecordTraceMeta`](#recordtracemeta) + +###### Returns + +`string` + +##### exportRunRecord() + +> **exportRunRecord**(`record`): `string` + +Defined in: [intelligence/index.ts:360](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L360) + +Send one typed [RunRecord](#runrecord) — the run's flat span (input/output/outcome/ +usage/model/provider, redacted) plus, when `loopEvents` are present, the +nested loop topology under the same `traceId`. Reuses the shipped +`flatOtelSpan` + `buildLoopOtelSpans` builders (no second builder). +Best-effort: export failures are swallowed. Returns the record's `traceId`. + +###### Parameters + +###### record + +[`RunRecord`](#runrecord) + +###### Returns + +`string` + +##### freshRunId() + +> **freshRunId**(): `string` + +Defined in: [intelligence/index.ts:362](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L362) + +Mint a fresh run id (`run-`). + +###### Returns + +`string` + +##### freshTraceId() + +> **freshTraceId**(): `string` + +Defined in: [intelligence/index.ts:364](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L364) + +Mint a fresh 32-hex trace id. + +###### Returns + +`string` + +##### doctor() + +> **doctor**(): [`DoctorReport`](#doctorreport) + +Defined in: [intelligence/index.ts:370](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L370) + +Network-free readiness report: which adoption modes are reachable given +this config. Observe is always reachable; Recommend needs outcomes; PR +needs checks + surfaces + repo. + +###### Returns + +[`DoctorReport`](#doctorreport) + +##### flush() + +> **flush**(): `Promise`\<`void`\> + +Defined in: [intelligence/index.ts:372](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L372) + +Flush any pending export spans. Best-effort; resolves even if export fails. + +###### Returns + +`Promise`\<`void`\> + +*** + +### ModeReadiness + +Defined in: [intelligence/index.ts:376](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L376) + +One mode's readiness verdict. + +#### Properties + +##### ready + +> **ready**: `boolean` + +Defined in: [intelligence/index.ts:377](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L377) + +##### missing + +> **missing**: `string`[] + +Defined in: [intelligence/index.ts:379](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L379) + +Inputs this mode still needs, when not ready. Empty when ready. + +*** + +### DoctorReport + +Defined in: [intelligence/index.ts:383](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L383) + +The `doctor()` readiness report — Mode-readiness without any network call. + +#### Properties + +##### project + +> **project**: `string` + +Defined in: [intelligence/index.ts:384](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L384) + +##### effort + +> **effort**: [`EffortSettings`](#effortsettings) + +Defined in: [intelligence/index.ts:385](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L385) + +##### exportConfigured + +> **exportConfigured**: `boolean` + +Defined in: [intelligence/index.ts:387](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L387) -> **usage**: [`UsageSplit`](#usagesplit) +True when an OTLP endpoint is configured (export will actually ship). -Defined in: [intelligence/index.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L219) +##### modes -Per-class billing split. `intelligenceUsd` is `0` at the OFF tier. +> **modes**: `object` + +Defined in: [intelligence/index.ts:388](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L388) + +###### observe + +> **observe**: [`ModeReadiness`](#modereadiness) + +###### recommend + +> **recommend**: [`ModeReadiness`](#modereadiness) + +###### pr + +> **pr**: [`ModeReadiness`](#modereadiness) *** -### IntelligenceClient +### ProvisionedHost -Defined in: [intelligence/index.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L223) +Defined in: [intelligence/resolver.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L50) -The Observe-mode Intelligence client. +A live, provisioned host the resolver tore up for a `process-on-infra` arm. + `teardown()` runs at `dispose()` in reverse provisioning order. #### Properties -##### project +##### mcpConnection? -> `readonly` **project**: `string` +> `optional` **mcpConnection?**: `AgentProfileMcpServer` -Defined in: [intelligence/index.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L225) +Defined in: [intelligence/resolver.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L53) -The resolved project id. +Lower the inner binding's mcp connection now that the host is up; the URL/ + command points at the host. Absent when the host serves a non-mcp inner. -##### effort +#### Methods -> `readonly` **effort**: [`EffortSettings`](#effortsettings) +##### teardown() -Defined in: [intelligence/index.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L227) +> **teardown**(): `Promise`\<`void`\> -The resolved effort settings. +Defined in: [intelligence/resolver.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L54) -#### Methods +###### Returns -##### traceRun() +`Promise`\<`void`\> -> **traceRun**\<`T`\>(`meta`, `fn`): `Promise`\<`T`\> +*** -Defined in: [intelligence/index.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L233) +### ResolveCtx -Run `fn` under a trace, export one span best-effort, and return whatever -`fn` returns. Telemetry-export failures are swallowed; an error THROWN by -`fn` propagates to the caller (the agent's own failures are not masked). +Defined in: [intelligence/resolver.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L62) -###### Type Parameters +Per-call, per-tenant context the resolver reads. Everything that touches the +network, a secret, or an infra provisioner is INJECTED so the manifest carries +no live secret and the substrate-free caller wires only what it can host. -###### T +#### Properties -`T` +##### tenant? + +> `optional` **tenant?**: `string` + +Defined in: [intelligence/resolver.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L64) + +Stable tenant id — namespaces billing + teardown (`tenant#target`). + +##### fetchImpl? + +> `optional` **fetchImpl?**: (`input`, `init?`) => `Promise`\<`Response`\> + +Defined in: [intelligence/resolver.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L66) + +fetch impl for http tools. Defaults to global fetch; absent ⇒ http tools fail loud. ###### Parameters -###### meta +###### input -[`TraceMeta`](#tracemeta) +`string` \| `URL` \| `Request` -###### fn +###### init? -(`trace`) => `Promise`\<`T`\> +`RequestInit` ###### Returns -`Promise`\<`T`\> +`Promise`\<`Response`\> -##### recordTrace() +##### resolveSecret? -> **recordTrace**(`events`, `meta?`): `string` +> `optional` **resolveSecret?**: (`auth`, `tenant`) => `Promise`\<\{ `succeeded`: `true`; `value`: `string`; \} \| \{ `succeeded`: `false`; `error`: `string`; \}\> -Defined in: [intelligence/index.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L243) +Defined in: [intelligence/resolver.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L72) -Export a run's full loop topology — the ordered `LoopTraceEvent` stream a -`runLoop`/`Supervisor` run emits — as a nested OTLP span tree (loop → round → -iteration) into ONE trace. Reuses the shipped `buildLoopOtelSpans` builder -(NO second span builder), so the topology a viewer renders matches the -kernel's. `traceId` defaults to a fresh id; `rootParentSpanId` parents the -loop root under an enclosing span (e.g. a `traceRun` span) when given. -Best-effort: export failures are swallowed. Returns the resolved `traceId`. +Resolve a declared credential to a live secret for THIS tenant. Returns a +typed outcome — inspect `succeeded` before `value`. Absent ⇒ a binding that +declares non-`none` auth fails loud (never a request with no credential). ###### Parameters -###### events +###### auth -readonly [`LoopTraceEvent`](runtime.md#looptraceevent)[] +[`CapabilityAuth`](#capabilityauth) -###### meta? +###### tenant -[`RecordTraceMeta`](#recordtracemeta) +`string` \| `undefined` + +###### Returns + +`Promise`\<\{ `succeeded`: `true`; `value`: `string`; \} \| \{ `succeeded`: `false`; `error`: `string`; \}\> + +##### runSandboxCode? + +> `optional` **runSandboxCode?**: (`code`, `entry`, `args`, `task`) => `Promise`\<`string`\> + +Defined in: [intelligence/resolver.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L81) + +Run a `sandbox-code` body per call. Injected by the host that owns a sandbox +client (the spine does not import the sandbox executor). Absent ⇒ +`sandbox-code` bindings fail loud. + +###### Parameters + +###### code + +[`ContentRef`](#contentref) + +###### entry + +`string` + +###### args + +`Record`\<`string`, `unknown`\> + +###### task + +`unknown` ###### Returns +`Promise`\<`string`\> + +##### provisionHost? + +> `optional` **provisionHost?**: (`host`, `inner`, `costTag`) => `Promise`\<[`ProvisionedHost`](#provisionedhost)\> + +Defined in: [intelligence/resolver.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L93) + +Provision a host for a `process-on-infra` binding, then serve the inner +binding inside it. Injected by the host that owns `createExecutor`. Absent ⇒ +`process-on-infra` bindings fail loud. The provider resolves the inner +binding INSIDE the host and returns the connection + a teardown. + +###### Parameters + +###### host + +[`HostSpec`](#hostspec) + +###### inner + +[`DeliveryBinding`](#deliverybinding) + +###### costTag + `string` -##### doctor() +###### Returns -> **doctor**(): [`DoctorReport`](#doctorreport) +`Promise`\<[`ProvisionedHost`](#provisionedhost)\> -Defined in: [intelligence/index.ts:249](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L249) +##### probeLiveToolNames? -Network-free readiness report: which adoption modes are reachable given -this config. Observe is always reachable; Recommend needs outcomes; PR -needs checks + surfaces + repo. +> `optional` **probeLiveToolNames?**: (`capabilityId`) => `Promise`\<`string`[]\> + +Defined in: [intelligence/resolver.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L106) + +Drift probe: return the LIVE tool names a resolved surface exposes for a +given capability id (a `tools/list` over an mcp connection, the agent's +actual registered tool names for a host tool). When present, the post-resolve +drift check drops any tool/mcp whose live names diverge from the certified +interface — the only callable surfaces are gate-blessed ones. Absent ⇒ the +check enforces only the host-side executor↔spec parity (no live probe). + +###### Parameters + +###### capabilityId + +`string` ###### Returns -[`DoctorReport`](#doctorreport) +`Promise`\<`string`[]\> + +##### onDrop? + +> `optional` **onDrop?**: (`capabilityId`, `error`) => `void` + +Defined in: [intelligence/resolver.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L114) + +Observe a DROPPED capability — a per-capability resolve failure that is +fail-closed (the capability is omitted, never half-wired). The drop is the +contract; this surfaces the diagnostic so it is never silently erased. NOT +called for [CapabilityNotAdmittedError](#capabilitynotadmittederror) (that rethrows — a manifest +carrying an un-admitted binding kind is a hard error, not a soft drop). + +###### Parameters + +###### capabilityId + +`string` + +###### error + +`Error` + +###### Returns + +`void` + +*** + +### AppliedIntelligence + +Defined in: [intelligence/with-intelligence.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L53) + +What the hook hands the agent each run. Additive over the prompt-only + delivery: `composePrompt` folds the certified prompt surface (as before); + `proposals`/`applyProfile` surface the promoted profile DIFFS — never + auto-applied; `record` enriches the [RunRecord](#runrecord) that is sent. + +#### Properties + +##### runId + +> **runId**: `string` + +Defined in: [intelligence/with-intelligence.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L55) + +Stable ids shared by the run span and every nested runtime/loop span. + +##### traceId + +> **traceId**: `string` + +Defined in: [intelligence/with-intelligence.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L56) + +##### certified + +> **certified**: [`CertifiedProfile`](#certifiedprofile) \| `null` + +Defined in: [intelligence/with-intelligence.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L59) + +The certified profile in effect (null when none promoted / pull failed — + fail-closed: the agent runs on its base surface). + +##### proposals + +> **proposals**: [`ProposedProfileDiff`](#proposedprofilediff)[] + +Defined in: [intelligence/with-intelligence.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L65) + +The promoted, gate-certified profile diffs — surfaced for a human or the + gated `improve()` loop. NEVER auto-applied by this hook. Empty when none. + +#### Methods + +##### composePrompt() + +> **composePrompt**(`base`): `string` + +Defined in: [intelligence/with-intelligence.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L62) -##### flush() +Fold the certified prompt surface into a base system prompt (the promoted + prompt). The consumer opts in by calling it. -> **flush**(): `Promise`\<`void`\> +###### Parameters -Defined in: [intelligence/index.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L251) +###### base -Flush any pending export spans. Best-effort; resolves even if export fails. +`string` ###### Returns -`Promise`\<`void`\> - -*** +`string` -### ModeReadiness +##### applyProfile() -Defined in: [intelligence/index.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L255) +> **applyProfile**(`base`): `AgentProfile` -One mode's readiness verdict. +Defined in: [intelligence/with-intelligence.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L69) -#### Properties +Fold every proposal into `base` via `applyAgentProfileDiff`, in promotion + order, and return the result. The caller invokes this EXPLICITLY (it is the + human/gated apply step) — the hook never calls it on the run path. -##### ready +###### Parameters -> **ready**: `boolean` +###### base -Defined in: [intelligence/index.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L256) +`AgentProfile` -##### missing +###### Returns -> **missing**: `string`[] +`AgentProfile` -Defined in: [intelligence/index.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L258) +##### record() -Inputs this mode still needs, when not ready. Empty when ready. +> **record**(`report`): `void` -*** +Defined in: [intelligence/with-intelligence.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L73) -### DoctorReport +Enrich the [RunRecord](#runrecord) sent for this call — outcome, usage split, + model/provider, and the loop event stream. Optional; an un-recorded run + still sends input/output with an inference-only zero usage split. -Defined in: [intelligence/index.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L262) +###### Parameters -The `doctor()` readiness report — Mode-readiness without any network call. +###### report -#### Properties +[`RunReport`](#runreport) -##### project +###### Returns -> **project**: `string` +`void` -Defined in: [intelligence/index.ts:263](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L263) +*** -##### effort +### IntelligenceHookConfig -> **effort**: [`EffortSettings`](#effortsettings) +Defined in: [intelligence/with-intelligence.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L83) -Defined in: [intelligence/index.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L264) +`withIntelligence` config = the Observe config plus the pull target, refresh + cadence, and a proposals callback. One base URL (`baseUrl` / + `TANGLE_INTELLIGENCE_URL`) drives both the send and receive paths. -##### exportConfigured +#### Extends -> **exportConfigured**: `boolean` +- [`IntelligenceConfig`](#intelligenceconfig) -Defined in: [intelligence/index.ts:266](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L266) +#### Properties -True when an OTLP endpoint is configured (export will actually ship). +##### project -##### modes +> **project**: `string` -> **modes**: `object` +Defined in: [intelligence/index.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L227) -Defined in: [intelligence/index.ts:267](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L267) +Stable project id — the tenant dimension every trace is tagged with. -###### observe +###### Inherited from -> **observe**: [`ModeReadiness`](#modereadiness) +[`IntelligenceConfig`](#intelligenceconfig).[`project`](#project-1) -###### recommend +##### apiKey? -> **recommend**: [`ModeReadiness`](#modereadiness) +> `optional` **apiKey?**: `string` -###### pr +Defined in: [intelligence/index.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L229) -> **pr**: [`ModeReadiness`](#modereadiness) +Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. -*** +###### Inherited from -### ProvisionedHost +[`IntelligenceConfig`](#intelligenceconfig).[`apiKey`](#apikey-2) -Defined in: [intelligence/resolver.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L50) +##### effort? -A live, provisioned host the resolver tore up for a `process-on-infra` arm. - `teardown()` runs at `dispose()` in reverse provisioning order. +> `optional` **effort?**: [`EffortTier`](#efforttier) \| \{ `tier`: [`EffortTier`](#efforttier); `overrides?`: `Partial`\<[`EffortSettings`](#effortsettings)\>; \} -#### Properties +Defined in: [intelligence/index.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L231) -##### mcpConnection? +Effort tier (default `'standard'`) plus optional per-field overrides. -> `optional` **mcpConnection?**: `AgentProfileMcpServer` +###### Inherited from -Defined in: [intelligence/resolver.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L53) +[`IntelligenceConfig`](#intelligenceconfig).[`effort`](#effort) -Lower the inner binding's mcp connection now that the host is up; the URL/ - command points at the host. Absent when the host serves a non-mcp inner. +##### baseUrl? -#### Methods +> `optional` **baseUrl?**: `string` -##### teardown() +Defined in: [intelligence/index.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L239) -> **teardown**(): `Promise`\<`void`\> +The ONE Tangle Intelligence base URL — both the send (OTLP `/v1/otlp`) and +receive (`/v1/profiles/:target/composed`) paths derive from it. Reads +`TANGLE_INTELLIGENCE_URL` when omitted, else `https://intelligence.tangle.tools`. +Send is best-effort and only ships when an `apiKey` is present (the tenant +key the ingest requires); absent a key, export is a no-op. -Defined in: [intelligence/resolver.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L54) +###### Inherited from -###### Returns +[`IntelligenceConfig`](#intelligenceconfig).[`baseUrl`](#baseurl-2) -`Promise`\<`void`\> +##### redact? -*** +> `optional` **redact?**: `false` \| [`Redactor`](#redactor) -### ResolveCtx +Defined in: [intelligence/index.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L245) -Defined in: [intelligence/resolver.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L62) +Redaction hook run over every exported input/output. A function replaces +the default scrubber; `false` opts out entirely (raw fidelity, caller has +sanitized upstream); omitted ⇒ the built-in `defaultRedactor`. -Per-call, per-tenant context the resolver reads. Everything that touches the -network, a secret, or an infra provisioner is INJECTED so the manifest carries -no live secret and the substrate-free caller wires only what it can host. +###### Inherited from -#### Properties +[`IntelligenceConfig`](#intelligenceconfig).[`redact`](#redact) -##### tenant? +##### surfaces? -> `optional` **tenant?**: `string` +> `optional` **surfaces?**: `string`[] -Defined in: [intelligence/resolver.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L64) +Defined in: [intelligence/index.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L247) -Stable tenant id — namespaces billing + teardown (`tenant#target`). +Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. -##### fetchImpl? +###### Inherited from -> `optional` **fetchImpl?**: (`input`, `init?`) => `Promise`\<`Response`\> +[`IntelligenceConfig`](#intelligenceconfig).[`surfaces`](#surfaces) -Defined in: [intelligence/resolver.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L66) +##### checks? -fetch impl for http tools. Defaults to global fetch; absent ⇒ http tools fail loud. +> `optional` **checks?**: `string`[] -###### Parameters +Defined in: [intelligence/index.ts:249](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L249) -###### input +Verification checks a later PR mode would gate on. Recorded for `doctor()` only. -`string` \| `URL` \| `Request` +###### Inherited from -###### init? +[`IntelligenceConfig`](#intelligenceconfig).[`checks`](#checks) -`RequestInit` +##### repo? -###### Returns +> `optional` **repo?**: [`RepoConfig`](#repoconfig) -`Promise`\<`Response`\> +Defined in: [intelligence/index.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L251) -##### resolveSecret? +Repo access a later PR mode would need. Recorded for `doctor()` only. -> `optional` **resolveSecret?**: (`auth`, `tenant`) => `Promise`\<\{ `succeeded`: `true`; `value`: `string`; \} \| \{ `succeeded`: `false`; `error`: `string`; \}\> +###### Inherited from -Defined in: [intelligence/resolver.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L72) +[`IntelligenceConfig`](#intelligenceconfig).[`repo`](#repo) -Resolve a declared credential to a live secret for THIS tenant. Returns a -typed outcome — inspect `succeeded` before `value`. Absent ⇒ a binding that -declares non-`none` auth fails loud (never a request with no credential). +##### profile? -###### Parameters +> `optional` **profile?**: `AgentProfile` -###### auth +Defined in: [intelligence/index.ts:253](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L253) -[`CapabilityAuth`](#capabilityauth) +Full canonical profile used for this agent. Exported redacted with a stable hash. -###### tenant +###### Inherited from -`string` \| `undefined` +[`IntelligenceConfig`](#intelligenceconfig).[`profile`](#profile-3) -###### Returns +##### commitSha? -`Promise`\<\{ `succeeded`: `true`; `value`: `string`; \} \| \{ `succeeded`: `false`; `error`: `string`; \}\> +> `optional` **commitSha?**: `string` -##### runSandboxCode? +Defined in: [intelligence/index.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L255) -> `optional` **runSandboxCode?**: (`code`, `entry`, `args`, `task`) => `Promise`\<`string`\> +Commit that produced the running agent, when known. -Defined in: [intelligence/resolver.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L81) +###### Inherited from -Run a `sandbox-code` body per call. Injected by the host that owns a sandbox -client (the spine does not import the sandbox executor). Absent ⇒ -`sandbox-code` bindings fail loud. +[`IntelligenceConfig`](#intelligenceconfig).[`commitSha`](#commitsha-2) -###### Parameters +##### runtimeTelemetry? -###### code +> `optional` **runtimeTelemetry?**: [`RuntimeTelemetryOptions`](index.md#runtimetelemetryoptions) -[`ContentRef`](#contentref) +Defined in: [intelligence/index.ts:257](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L257) -###### entry +Runtime-event payload policy. Tool inputs/results remain off unless explicitly enabled. -`string` +###### Inherited from -###### args +[`IntelligenceConfig`](#intelligenceconfig).[`runtimeTelemetry`](#runtimetelemetry) -`Record`\<`string`, `unknown`\> +##### payloadAttributes? -###### task +> `optional` **payloadAttributes?**: `"metadata"` \| `"full"` -`unknown` +Defined in: [intelligence/index.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L264) -###### Returns +Payloads are metadata-only by default: the run span carries a stable hash +and UTF-8 byte count, but not the redacted content. Set `full` only when +the configured OTLP destination is approved to receive complete redacted +inputs, outputs, and profiles. -`Promise`\<`string`\> +###### Inherited from -##### provisionHost? +[`IntelligenceConfig`](#intelligenceconfig).[`payloadAttributes`](#payloadattributes) -> `optional` **provisionHost?**: (`host`, `inner`, `costTag`) => `Promise`\<[`ProvisionedHost`](#provisionedhost)\> +##### target? -Defined in: [intelligence/resolver.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L93) +> `optional` **target?**: `string` -Provision a host for a `process-on-infra` binding, then serve the inner -binding inside it. Injected by the host that owns `createExecutor`. Absent ⇒ -`process-on-infra` bindings fail loud. The provider resolves the inner -binding INSIDE the host and returns the connection + a teardown. +Defined in: [intelligence/with-intelligence.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L85) -###### Parameters +Pull target. Defaults to `project`. -###### host +##### refreshMs? -[`HostSpec`](#hostspec) +> `optional` **refreshMs?**: `number` -###### inner +Defined in: [intelligence/with-intelligence.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L87) -[`DeliveryBinding`](#deliverybinding) +Min interval between certified-profile pulls. Default 5m. -###### costTag +##### timeoutMs? -`string` +> `optional` **timeoutMs?**: `number` -###### Returns +Defined in: [intelligence/with-intelligence.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L89) -`Promise`\<[`ProvisionedHost`](#provisionedhost)\> +Per-pull timeout in ms (fail-closed on a hung plane). Default 10000. -##### probeLiveToolNames? +##### fetchImpl? -> `optional` **probeLiveToolNames?**: (`capabilityId`) => `Promise`\<`string`[]\> +> `optional` **fetchImpl?**: (`input`, `init?`) => `Promise`\<`Response`\> -Defined in: [intelligence/resolver.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L106) +Defined in: [intelligence/with-intelligence.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L91) -Drift probe: return the LIVE tool names a resolved surface exposes for a -given capability id (a `tools/list` over an mcp connection, the agent's -actual registered tool names for a host tool). When present, the post-resolve -drift check drops any tool/mcp whose live names diverge from the certified -interface — the only callable surfaces are gate-blessed ones. Absent ⇒ the -check enforces only the host-side executor↔spec parity (no live probe). +fetch impl for the pull (tests). Defaults to global fetch. ###### Parameters -###### capabilityId +###### input -`string` +`string` \| `URL` \| `Request` -###### Returns +###### init? -`Promise`\<`string`[]\> +`RequestInit` -##### onDrop? +###### Returns -> `optional` **onDrop?**: (`capabilityId`, `error`) => `void` +`Promise`\<`Response`\> -Defined in: [intelligence/resolver.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L114) +##### onProposals? -Observe a DROPPED capability — a per-capability resolve failure that is -fail-closed (the capability is omitted, never half-wired). The drop is the -contract; this surfaces the diagnostic so it is never silently erased. NOT -called for [CapabilityNotAdmittedError](#capabilitynotadmittederror) (that rethrows — a manifest -carrying an un-admitted binding kind is a hard error, not a soft drop). +> `optional` **onProposals?**: (`proposals`) => `void` -###### Parameters +Defined in: [intelligence/with-intelligence.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L94) -###### capabilityId +Notified when a refresh delivers a NEW set of promoted proposals (by + provenance content hash). Surfaces diffs without auto-applying them. -`string` +###### Parameters -###### error +###### proposals -`Error` +[`ProposedProfileDiff`](#proposedprofilediff)[] ###### Returns @@ -1982,45 +3181,10 @@ Every binding kind — the open set the resolver dispatches over. > **PullOutcome** = \{ `succeeded`: `true`; `value`: [`CertifiedProfile`](#certifiedprofile); \} \| \{ `succeeded`: `false`; `error`: `string`; `status?`: `number`; \} -Defined in: [intelligence/delivery.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L64) +Defined in: [intelligence/delivery.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L104) Typed outcome for the pull — inspect `succeeded` before `value`. A 404 - (nothing promoted yet) is a normal, non-error `succeeded: false`. - -*** - -### DeliveredAgent - -> **DeliveredAgent**\<`I`, `O`\> = (`input`, `applied`) => `Promise`\<`O`\> - -Defined in: [intelligence/delivery.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L256) - -An agent wrapped by [withCertifiedDelivery](#withcertifieddelivery): receives the input plus - the certified intelligence delivered for this run. - -#### Type Parameters - -##### I - -`I` - -##### O - -`O` - -#### Parameters - -##### input - -`I` - -##### applied - -[`AppliedIntelligence`](#appliedintelligence) - -#### Returns - -`Promise`\<`O`\> + (nothing promoted yet) is a normal, non-error `succeeded: false`. *** @@ -2058,11 +3222,37 @@ Per-field overrides applied on top of a tier preset. Any subset of the *** +### AgentImprovementEvaluation + +> **AgentImprovementEvaluation**\<`TScenario`, `TArtifact`\> = `Pick`\<`SelfImproveResult`\<`TScenario`, `TArtifact`\>, `"baseline"` \| `"winner"` \| `"lift"` \| `"diff"` \| `"provenance"` \| `"gateDecision"` \| `"generationsExplored"` \| `"durationMs"` \| `"totalCostUsd"` \| `"insight"` \| `"power"`\> + +Defined in: [intelligence/improvement-cycle.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L47) + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +*** + +### AgentImprovementReviewDecision + +> **AgentImprovementReviewDecision** = `"approve"` \| `"reject"` \| `"request-changes"` + +Defined in: [intelligence/improvement-cycle.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L81) + +*** + ### UsageClass > **UsageClass** = `"inference"` \| `"intelligence"` -Defined in: [intelligence/index.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L104) +Defined in: [intelligence/index.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L135) Usage class for billing. Base-stream tokens bill `'inference'`; every intelligence spawn (analyst, corpus, loop) bills `'intelligence'`. The @@ -2070,65 +3260,108 @@ Usage class for billing. Base-stream tokens bill `'inference'`; every *** -### Agent +### Redactor + +> **Redactor** = (`value`) => `unknown` + +Defined in: [intelligence/redact.ts:18](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/redact.ts#L18) + +A redactor maps an arbitrary trace value to a safe-to-export value. Pure; + must not throw on cyclic input (the default tolerates cycles). + +#### Parameters + +##### value + +`unknown` + +#### Returns + +`unknown` + +*** + +### IntelligenceAgent -> **Agent**\<`TInput`, `TOutput`\> = (`input`) => `Promise`\<`TOutput`\> +> **IntelligenceAgent**\<`I`, `O`\> = (`input`, `applied`) => `Promise`\<`O`\> -Defined in: [intelligence/index.ts:520](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L520) +Defined in: [intelligence/with-intelligence.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L78) -A generic agent: one async input → output. The shape `withTangleIntelligence` - preserves exactly. +An agent wrapped by [withIntelligence](#withintelligence): receives the input plus the + intelligence delivered for this run. #### Type Parameters -##### TInput +##### I -`TInput` +`I` -##### TOutput +##### O -`TOutput` +`O` #### Parameters ##### input -`TInput` +`I` + +##### applied + +[`AppliedIntelligence`](#appliedintelligence) #### Returns -`Promise`\<`TOutput`\> +`Promise`\<`O`\> *** -### ClientOrConfig +### IntelligenceWrapped -> **ClientOrConfig** = [`IntelligenceClient`](#intelligenceclient) \| [`IntelligenceConfig`](#intelligenceconfig) +> **IntelligenceWrapped**\<`I`, `O`\> = (`input`) => `Promise`\<`O`\> & `object` -Defined in: [intelligence/index.ts:523](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L523) +Defined in: [intelligence/with-intelligence.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L99) -Either a built client or the config to build one. +The wrapped agent — same `(input) => Promise` shape, plus a manual + `refresh()` and a `proposals()` accessor for the currently-promoted diffs. -*** +#### Type Declaration -### Redactor +##### refresh() -> **Redactor** = (`value`) => `unknown` +> **refresh**(): `Promise`\<`void`\> -Defined in: [intelligence/redact.ts:18](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/redact.ts#L18) +###### Returns -A redactor maps an arbitrary trace value to a safe-to-export value. Pure; - must not throw on cyclic input (the default tolerates cycles). +`Promise`\<`void`\> -#### Parameters +##### proposals() -##### value +> **proposals**(): [`ProposedProfileDiff`](#proposedprofilediff)[] -`unknown` +###### Returns -#### Returns +[`ProposedProfileDiff`](#proposedprofilediff)[] -`unknown` +##### flush() + +> **flush**(): `Promise`\<`void`\> + +Flush buffered trace spans before a short-lived process exits. + +###### Returns + +`Promise`\<`void`\> + +#### Type Parameters + +##### I + +`I` + +##### O + +`O` ## Variables @@ -2169,11 +3402,55 @@ This delivers the spine against today's wire before the plane changes. *** +### resolveIntelligenceBaseUrl() + +> **resolveIntelligenceBaseUrl**(`baseUrl`): `string` + +Defined in: [intelligence/delivery.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L128) + +Resolve the ONE Intelligence base URL — the single knob both the send and + receive paths derive from. Env fallback: `TANGLE_INTELLIGENCE_URL`. + +#### Parameters + +##### baseUrl + +`string` \| `undefined` + +#### Returns + +`string` + +*** + +### normalizeCertifiedProfile() + +> **normalizeCertifiedProfile**(`raw`): [`CertifiedProfile`](#certifiedprofile) + +Defined in: [intelligence/delivery.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L163) + +Deserialize the composed-endpoint response into a `CertifiedProfile`. The +previously-dropped `agentProfileDiffs`/`capabilities`/`agentProfile` are read +here so they round-trip to the consumer; a plane that has not yet promoted any +diffs simply yields empty arrays / a null profile (fail-closed, never a crash). + +#### Parameters + +##### raw + +`unknown` + +#### Returns + +[`CertifiedProfile`](#certifiedprofile) + +*** + ### pullCertified() > **pullCertified**(`opts`): `Promise`\<[`PullOutcome`](#pulloutcome)\> -Defined in: [intelligence/delivery.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L107) +Defined in: [intelligence/delivery.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L193) Pull the certified composed profile for a target. Fail-closed: a network error or a non-2xx returns a typed `succeeded: false` (never throws), so a @@ -2196,7 +3473,7 @@ the normal "nothing promoted yet" signal, carried as `status: 404`. > **composeCertifiedPrompt**(`base`, `certified`): `string` -Defined in: [intelligence/delivery.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L167) +Defined in: [intelligence/delivery.ts:253](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L253) Fold the certified prompt surface (and any certified prompt-folding artifacts: `prompt-surface` / `skill` / `instructions`) into a base system prompt under a @@ -2204,7 +3481,7 @@ marked section, so the deployed agent prompt == base + the gate-certified additions. Order is stable (prompt surface first, then artifact buckets in `promptFoldTypes` order, then by path within a bucket) so the same profile renders byte-identically each call. Returns `base` unchanged when there is no -usable certified content. +usable certified content. Reads only the prompt-folding slice of a profile. #### Parameters @@ -2214,7 +3491,7 @@ usable certified content. ##### certified -[`CertifiedProfile`](#certifiedprofile) \| `null` +`Pick`\<[`CertifiedProfile`](#certifiedprofile), `"promptSurface"` \| `"artifacts"`\> \| `null` #### Returns @@ -2226,12 +3503,12 @@ usable certified content. > **createCertifiedPromptSource**(`opts`): [`CertifiedPromptSource`](#certifiedpromptsource) -Defined in: [intelligence/delivery.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L210) +Defined in: [intelligence/delivery.ts:299](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L299) Create the cached certified-prompt source — the ONE module-scope-cache + coalesced-refresh + keep-last-known implementation. Product wiring uses this -rather than hand-rolling the same lines around `pullCertified`, and -[withCertifiedDelivery](#withcertifieddelivery) is built on it. +rather than hand-rolling the same lines around `pullCertified`. The +`withIntelligence` hook rides this same source for its prompt delivery. #### Parameters @@ -2245,45 +3522,6 @@ rather than hand-rolling the same lines around `pullCertified`, and *** -### withCertifiedDelivery() - -> **withCertifiedDelivery**\<`I`, `O`\>(`agent`, `config`): (`input`) => `Promise`\<`O`\> & `object` - -Defined in: [intelligence/delivery.ts:281](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/delivery.ts#L281) - -Wrap an agent so it (a) Observes each run via the shipped Observe client and -(b) RECEIVES the tenant's certified artifacts pulled from the deployed plane. -The certified profile is cached and refreshed at most every `refreshMs`; a -failed pull is fail-closed — the agent runs on its base surface and never -breaks because Intelligence is unreachable. When the plane promotes a new -gate-certified surface, the next refresh delivers it to the running agent. - -#### Type Parameters - -##### I - -`I` - -##### O - -`O` - -#### Parameters - -##### agent - -[`DeliveredAgent`](#deliveredagent)\<`I`, `O`\> - -##### config - -[`DeliveryConfig`](#deliveryconfig) - -#### Returns - -(`input`) => `Promise`\<`O`\> & `object` - -*** - ### resolveEffort() > **resolveEffort**(`tier`, `overrides?`): [`EffortSettings`](#effortsettings) @@ -2368,66 +3606,172 @@ compile to `withAnalyst: true`, the tier's `fanout`, and `withLoops: true`. *** -### createIntelligenceClient() +### proposeAgentImprovement() -> **createIntelligenceClient**(`config`): [`IntelligenceClient`](#intelligenceclient) +> **proposeAgentImprovement**\<`TScenario`, `TArtifact`\>(`options`): `Promise`\<[`ProposeAgentImprovementResult`](#proposeagentimprovementresult)\<`TScenario`, `TArtifact`\>\> + +Defined in: [intelligence/improvement-cycle.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L196) -Defined in: [intelligence/index.ts:331](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L331) +Analyze one run and produce one measured, review-only improvement proposal. -Create an Observe-mode Intelligence client. Resolves effort, endpoint, and -redactor up front; the exporter is built lazily and is `undefined` when no -endpoint is configured (export becomes a no-op — best-effort by -construction). +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` #### Parameters -##### config +##### options -[`IntelligenceConfig`](#intelligenceconfig) +[`ProposeAgentImprovementOptions`](#proposeagentimprovementoptions)\<`TScenario`, `TArtifact`\> #### Returns -[`IntelligenceClient`](#intelligenceclient) +`Promise`\<[`ProposeAgentImprovementResult`](#proposeagentimprovementresult)\<`TScenario`, `TArtifact`\>\> *** -### withTangleIntelligence() +### createAgentImprovementProposal() -> **withTangleIntelligence**\<`TInput`, `TOutput`\>(`agent`, `clientOrConfig`): [`Agent`](#agent)\<`TInput`, `TOutput`\> +> **createAgentImprovementProposal**\<`TScenario`, `TArtifact`\>(`options`): [`AgentImprovementProposal`](#agentimprovementproposal)\<`TScenario`, `TArtifact`\> -Defined in: [intelligence/index.ts:538](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L538) +Defined in: [intelligence/improvement-cycle.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L244) -Wrap a generic `agent` with best-effort Observe-mode tracing, returning the -SAME shape. Each call runs the agent under a trace and exports one span; an -export failure is swallowed (the live agent never fails because Intelligence -is down) but an error from the agent itself propagates unchanged. - -At `effort: 'off'` this is pure passthrough plus best-effort telemetry — -zero intelligence spawns, `intelligenceUsd: 0` on the trace. +Freeze an already-measured improvement into the one reviewable proposal +contract. Products that run analysis or evaluation in separate workers use +this constructor instead of rerunning either phase or rebuilding digests. #### Type Parameters -##### TInput +##### TScenario -`TInput` +`TScenario` *extends* `Scenario` -##### TOutput +##### TArtifact -`TOutput` +`TArtifact` #### Parameters -##### agent +##### options + +[`CreateAgentImprovementProposalOptions`](#createagentimprovementproposaloptions)\<`TScenario`, `TArtifact`\> + +#### Returns + +[`AgentImprovementProposal`](#agentimprovementproposal)\<`TScenario`, `TArtifact`\> + +*** + +### reviewAgentImprovementProposal() + +> **reviewAgentImprovementProposal**(`inputProposal`, `input`): [`AgentImprovementReview`](#agentimprovementreview) + +Defined in: [intelligence/improvement-cycle.ts:294](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L294) + +Persist an approve/reject/change-request decision bound to one exact proposal. + +#### Parameters + +##### inputProposal + +[`AgentImprovementProposal`](#agentimprovementproposal) + +##### input + +[`ReviewAgentImprovementInput`](#reviewagentimprovementinput) + +#### Returns + +[`AgentImprovementReview`](#agentimprovementreview) + +*** + +### executeApprovedAgentCandidate() + +> **executeApprovedAgentCandidate**(`options`): `Promise`\<[`ExecuteApprovedAgentCandidateResult`](#executeapprovedagentcandidateresult)\> + +Defined in: [intelligence/improvement-cycle.ts:324](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L324) + +Verify, materialize, run, grade, and receipt only the exact approved bundle. + +#### Parameters + +##### options + +[`ExecuteApprovedAgentCandidateOptions`](#executeapprovedagentcandidateoptions) + +#### Returns + +`Promise`\<[`ExecuteApprovedAgentCandidateResult`](#executeapprovedagentcandidateresult)\> + +*** + +### verifyAgentImprovementProposal() + +> **verifyAgentImprovementProposal**(`input`): [`AgentImprovementProposal`](#agentimprovementproposal) + +Defined in: [intelligence/improvement-cycle.ts:366](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L366) -[`Agent`](#agent)\<`TInput`, `TOutput`\> +Validate a proposal's schema, profile, sealed bundle, and canonical digest. -##### clientOrConfig +#### Parameters + +##### input + +`unknown` + +#### Returns + +[`AgentImprovementProposal`](#agentimprovementproposal) + +*** + +### verifyAgentImprovementReview() + +> **verifyAgentImprovementReview**(`input`): [`AgentImprovementReview`](#agentimprovementreview) + +Defined in: [intelligence/improvement-cycle.ts:483](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L483) + +Validate a review's decision fields and canonical digest. + +#### Parameters + +##### input + +`unknown` + +#### Returns + +[`AgentImprovementReview`](#agentimprovementreview) + +*** + +### createIntelligenceClient() + +> **createIntelligenceClient**(`config`): [`IntelligenceClient`](#intelligenceclient) + +Defined in: [intelligence/index.ts:454](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L454) + +Create an Observe-mode Intelligence client. Resolves effort, the base URL, and +the redactor up front; the exporter is built lazily and is `undefined` when no +`apiKey` is present (send becomes a no-op — the ingest requires a tenant key, +and best-effort export must never spam an unauthenticated plane). + +#### Parameters + +##### config -[`ClientOrConfig`](#clientorconfig) +[`IntelligenceConfig`](#intelligenceconfig) #### Returns -[`Agent`](#agent)\<`TInput`, `TOutput`\> +[`IntelligenceClient`](#intelligenceclient) *** @@ -2518,7 +3862,7 @@ Fail-closed: a `null` manifest returns the base surface only. > **composeCertifiedProfileFromWire**(`base`, `profile`, `ctx?`): `Promise`\<[`ResolvedSurface`](#resolvedsurface)\> -Defined in: [intelligence/resolver.ts:664](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L664) +Defined in: [intelligence/resolver.ts:660](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/resolver.ts#L660) Lower a plane `CertifiedProfile` straight into a `ResolvedSurface` via `manifestFromProfile` — the convenience the shipped pull lane calls when it @@ -2543,3 +3887,43 @@ Lower a plane `CertifiedProfile` straight into a `ResolvedSurface` via #### Returns `Promise`\<[`ResolvedSurface`](#resolvedsurface)\> + +*** + +### withIntelligence() + +> **withIntelligence**\<`I`, `O`\>(`agent`, `config`): [`IntelligenceWrapped`](#intelligencewrapped)\<`I`, `O`\> + +Defined in: [intelligence/with-intelligence.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L169) + +Wrap an agent so it (a) RECEIVES the tenant's certified profile — the prompt +surface to fold and the promoted profile diffs as proposals — and (b) SENDS a +typed [RunRecord](#runrecord) per call to the plane. The pull is cached and refreshed +at most every `refreshMs`; a failed pull is fail-closed (the agent runs on its +base surface, never breaks because Intelligence is unreachable). The send is +best-effort — an export failure never fails the agent's turn — while an error +thrown by the agent itself propagates unchanged. + +#### Type Parameters + +##### I + +`I` + +##### O + +`O` + +#### Parameters + +##### agent + +[`IntelligenceAgent`](#intelligenceagent)\<`I`, `O`\> + +##### config + +[`IntelligenceHookConfig`](#intelligencehookconfig) + +#### Returns + +[`IntelligenceWrapped`](#intelligencewrapped)\<`I`, `O`\> diff --git a/docs/api/mcp.md b/docs/api/mcp.md index ba9fc764..94756b9a 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -8,6 +8,68 @@ ## Classes +### CodexExecutionDiagnosticError + +Defined in: [mcp/codex-diagnostics.ts:19](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L19) + +Thrown when reproducible Codex exits without one valid terminal usage event. + +#### Extends + +- `Error` + +#### Constructors + +##### Constructor + +> **new CodexExecutionDiagnosticError**(`reason`, `diagnostic`, `cause?`): [`CodexExecutionDiagnosticError`](#codexexecutiondiagnosticerror) + +Defined in: [mcp/codex-diagnostics.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L22) + +###### Parameters + +###### reason + +`string` + +###### diagnostic + +[`CodexExecutionFailureDiagnostic`](#codexexecutionfailurediagnostic) + +###### cause? + +`unknown` + +###### Returns + +[`CodexExecutionDiagnosticError`](#codexexecutiondiagnosticerror) + +###### Overrides + +`Error.constructor` + +#### Properties + +##### code + +> `readonly` **code**: `"CODEX_EXECUTION_DIAGNOSTIC"` = `'CODEX_EXECUTION_DIAGNOSTIC'` + +Defined in: [mcp/codex-diagnostics.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L20) + +##### reason + +> `readonly` **reason**: `string` + +Defined in: [mcp/codex-diagnostics.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L23) + +##### diagnostic + +> `readonly` **diagnostic**: [`CodexExecutionFailureDiagnostic`](#codexexecutionfailurediagnostic) + +Defined in: [mcp/codex-diagnostics.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L24) + +*** + ### DelegationStateCorruptError Defined in: [mcp/delegation-store.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/delegation-store.ts#L55) @@ -727,6 +789,65 @@ shape against the structural `FleetHandle` contract. *** +### CodexExecutionFailureDiagnostic + +Defined in: [mcp/codex-diagnostics.ts:7](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L7) + +Bounded, credential-redacted process context attached when reproducible Codex output fails +validation. The process still fails closed; this only preserves enough evidence to diagnose it. + +#### Properties + +##### exitCode + +> **exitCode**: `number` \| `null` + +Defined in: [mcp/codex-diagnostics.ts:8](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L8) + +##### killedBySignal + +> **killedBySignal**: `Signals` \| `null` + +Defined in: [mcp/codex-diagnostics.ts:9](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L9) + +##### timedOut + +> **timedOut**: `boolean` + +Defined in: [mcp/codex-diagnostics.ts:10](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L10) + +##### durationMs + +> **durationMs**: `number` + +Defined in: [mcp/codex-diagnostics.ts:11](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L11) + +##### stdout + +> **stdout**: `string` + +Defined in: [mcp/codex-diagnostics.ts:12](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L12) + +##### stderr + +> **stderr**: `string` + +Defined in: [mcp/codex-diagnostics.ts:13](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L13) + +##### stdoutTruncated + +> **stdoutTruncated**: `boolean` + +Defined in: [mcp/codex-diagnostics.ts:14](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L14) + +##### stderrTruncated + +> **stderrTruncated**: `boolean` + +Defined in: [mcp/codex-diagnostics.ts:15](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/codex-diagnostics.ts#L15) + +*** + ### DelegateRunCtx Defined in: [mcp/delegates.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/delegates.ts#L57) @@ -833,7 +954,7 @@ Gate: only approved candidates are eligible to win. ##### recommendation -> **recommendation**: `"ship"` \| `"approve-with-nits"` \| `"changes-requested"` \| `"reject"` +> **recommendation**: `"ship"` \| `"reject"` \| `"approve-with-nits"` \| `"changes-requested"` Defined in: [mcp/delegates.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/delegates.ts#L102) @@ -2256,19 +2377,19 @@ Which harness handled this delegation. > **kind**: `"sibling"` \| `"fleet"` -Defined in: [runtime/types.ts:400](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L400) +Defined in: [runtime/types.ts:397](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L397) **`Experimental`** ###### Inherited from -[`LoopSandboxPlacement`](runtime.md#loopsandboxplacement).[`kind`](runtime.md#kind-3) +[`LoopSandboxPlacement`](runtime.md#loopsandboxplacement).[`kind`](runtime.md#kind-4) ##### sandboxId? > `optional` **sandboxId?**: `string` -Defined in: [runtime/types.ts:401](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L401) +Defined in: [runtime/types.ts:398](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L398) **`Experimental`** @@ -2280,7 +2401,7 @@ Defined in: [runtime/types.ts:401](https://github.com/tangle-network/agent-runti > `optional` **fleetId?**: `string` -Defined in: [runtime/types.ts:402](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L402) +Defined in: [runtime/types.ts:399](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L399) **`Experimental`** @@ -2292,7 +2413,7 @@ Defined in: [runtime/types.ts:402](https://github.com/tangle-network/agent-runti > `optional` **machineId?**: `string` -Defined in: [runtime/types.ts:403](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L403) +Defined in: [runtime/types.ts:400](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L400) **`Experimental`** @@ -2512,7 +2633,7 @@ Default `[]` (no circular check unless the consumer declares its kinds). ### RunLocalHarnessOptions -Defined in: [mcp/local-harness.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L108) +Defined in: [mcp/local-harness.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L243) **`Experimental`** @@ -2522,7 +2643,7 @@ Defined in: [mcp/local-harness.ts:108](https://github.com/tangle-network/agent-r > **harness**: [`LocalHarness`](#localharness) -Defined in: [mcp/local-harness.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L109) +Defined in: [mcp/local-harness.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L244) **`Experimental`** @@ -2530,7 +2651,7 @@ Defined in: [mcp/local-harness.ts:109](https://github.com/tangle-network/agent-r > **cwd**: `string` -Defined in: [mcp/local-harness.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L111) +Defined in: [mcp/local-harness.ts:246](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L246) **`Experimental`** @@ -2540,7 +2661,7 @@ Working directory for the subprocess (typically a worktree path). > **taskPrompt**: `string` -Defined in: [mcp/local-harness.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L113) +Defined in: [mcp/local-harness.ts:248](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L248) **`Experimental`** @@ -2550,7 +2671,7 @@ Prompt forwarded as the harness CLI's task argument. > `optional` **invocation?**: `object` -Defined in: [mcp/local-harness.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L121) +Defined in: [mcp/local-harness.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L256) **`Experimental`** @@ -2568,11 +2689,44 @@ is used unchanged. > **args**: readonly `string`[] +##### dangerouslySkipPermissions? + +> `optional` **dangerouslySkipPermissions?**: `boolean` + +Defined in: [mcp/local-harness.ts:259](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L259) + +**`Experimental`** + +Allow autonomous Claude edits without an interactive permission prompt. + Use only when `cwd` is an isolated candidate worktree. + +##### codexReproducible? + +> `optional` **codexReproducible?**: `boolean` + +Defined in: [mcp/local-harness.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L262) + +**`Experimental`** + +Isolate Codex from ambient configuration/instructions and require JSONL token usage. + The invocation should come from `harnessInvocation(..., { codexReproducible: true })`. + +##### codexReadDeniedPaths? + +> `optional` **codexReadDeniedPaths?**: readonly `string`[] + +Defined in: [mcp/local-harness.ts:265](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L265) + +**`Experimental`** + +Absolute host paths that reproducible Codex must not read. The normalized set is compiled + into the controlled permission profile and its digest is returned in execution evidence. + ##### timeoutMs? > `optional` **timeoutMs?**: `number` -Defined in: [mcp/local-harness.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L123) +Defined in: [mcp/local-harness.ts:267](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L267) **`Experimental`** @@ -2582,7 +2736,7 @@ Wall-clock kill deadline (ms). Default 5 min. Subprocess SIGTERMed on expiry. > `optional` **signal?**: `AbortSignal` -Defined in: [mcp/local-harness.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L125) +Defined in: [mcp/local-harness.ts:269](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L269) **`Experimental`** @@ -2592,7 +2746,7 @@ Caller cancellation. SIGTERM is sent on abort. > `optional` **env?**: `ProcessEnv` -Defined in: [mcp/local-harness.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L127) +Defined in: [mcp/local-harness.ts:271](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L271) **`Experimental`** @@ -2602,7 +2756,7 @@ Override env (defaults to inheriting from the parent). > `optional` **spawn?**: (`command`, `args`, `opts`) => `ChildProcess` -Defined in: [mcp/local-harness.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L132) +Defined in: [mcp/local-harness.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L276) **`Experimental`** @@ -2633,15 +2787,317 @@ readonly `string`[] `"pipe"` +###### detached + +`boolean` + ###### Returns `ChildProcess` +##### resolveCodexExecutable? + +> `optional` **resolveCodexExecutable?**: (`command`, `env`) => `Promise`\<`string`\> + +Defined in: [mcp/local-harness.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L287) + +**`Experimental`** + +Test seam for locating the native Codex executable before it is staged in the worktree. + +###### Parameters + +###### command + +`string` + +###### env + +`ProcessEnv` + +###### Returns + +`Promise`\<`string`\> + +*** + +### CodexTokenUsage + +Defined in: [mcp/local-harness.ts:291](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L291) + +Exact aggregate usage emitted by Codex's terminal `turn.completed` JSONL event. + +#### Properties + +##### inputTokens + +> **inputTokens**: `number` + +Defined in: [mcp/local-harness.ts:292](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L292) + +##### cachedInputTokens + +> **cachedInputTokens**: `number` + +Defined in: [mcp/local-harness.ts:293](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L293) + +##### outputTokens + +> **outputTokens**: `number` + +Defined in: [mcp/local-harness.ts:294](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L294) + +##### reasoningOutputTokens + +> **reasoningOutputTokens**: `number` + +Defined in: [mcp/local-harness.ts:295](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L295) + +*** + +### CodexExecutionPolicy + +Defined in: [mcp/local-harness.ts:299](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L299) + +Isolation settings asserted before a reproducible Codex run is allowed to start. + +#### Properties + +##### sessionPersistence + +> **sessionPersistence**: `"ephemeral"` + +Defined in: [mcp/local-harness.ts:300](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L300) + +##### userConfig + +> **userConfig**: `false` + +Defined in: [mcp/local-harness.ts:301](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L301) + +##### rules + +> **rules**: `false` + +Defined in: [mcp/local-harness.ts:302](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L302) + +##### projectInstructions + +> **projectInstructions**: `false` + +Defined in: [mcp/local-harness.ts:303](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L303) + +##### skillInstructions + +> **skillInstructions**: `false` + +Defined in: [mcp/local-harness.ts:304](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L304) + +##### appInstructions + +> **appInstructions**: `false` + +Defined in: [mcp/local-harness.ts:305](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L305) + +##### toolSuggestions + +> **toolSuggestions**: `false` + +Defined in: [mcp/local-harness.ts:306](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L306) + +##### multiAgentInstructions + +> **multiAgentInstructions**: `false` + +Defined in: [mcp/local-harness.ts:307](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L307) + +##### sandbox + +> **sandbox**: `"workspace-write"` + +Defined in: [mcp/local-harness.ts:308](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L308) + +##### permissionProfile + +> **permissionProfile**: `"agent_runtime_reproducible"` + +Defined in: [mcp/local-harness.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L309) + +##### approvalPolicy + +> **approvalPolicy**: `"never"` + +Defined in: [mcp/local-harness.ts:310](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L310) + +##### shellNetwork + +> **shellNetwork**: `false` + +Defined in: [mcp/local-harness.ts:311](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L311) + +##### webSearch + +> **webSearch**: `false` + +Defined in: [mcp/local-harness.ts:312](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L312) + +##### serviceTier + +> **serviceTier**: `"default"` + +Defined in: [mcp/local-harness.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L313) + +##### shellEnvironment + +> **shellEnvironment**: `"core-filtered"` + +Defined in: [mcp/local-harness.ts:314](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L314) + +##### loginShell + +> **loginShell**: `false` + +Defined in: [mcp/local-harness.ts:315](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L315) + +##### credentialsReadable + +> **credentialsReadable**: `false` + +Defined in: [mcp/local-harness.ts:316](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L316) + +##### hostHomeReadable + +> **hostHomeReadable**: `false` + +Defined in: [mcp/local-harness.ts:317](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L317) + +##### procEnvironment + +> **procEnvironment**: `"private-sanitized"` + +Defined in: [mcp/local-harness.ts:318](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L318) + +##### sensitiveEnvironmentNamesVisible + +> **sensitiveEnvironmentNamesVisible**: `false` + +Defined in: [mcp/local-harness.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L319) + +##### parentRepoRead + +> **parentRepoRead**: `false` + +Defined in: [mcp/local-harness.ts:320](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L320) + +##### gitMetadata + +> **gitMetadata**: `false` + +Defined in: [mcp/local-harness.ts:321](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L321) + +##### temporaryDirectory + +> **temporaryDirectory**: `"workspace-private"` + +Defined in: [mcp/local-harness.ts:322](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L322) + +##### stagedExecutable + +> **stagedExecutable**: `"static-elf-read-only"` + +Defined in: [mcp/local-harness.ts:323](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L323) + +##### callerReadDeniedPaths + +> **callerReadDeniedPaths**: `"enforced"` + +Defined in: [mcp/local-harness.ts:324](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L324) + +##### containerSockets + +> **containerSockets**: `false` + +Defined in: [mcp/local-harness.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L325) + +*** + +### CodexExecutionEvidence + +Defined in: [mcp/local-harness.ts:329](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L329) + +Zero-model-call evidence for the exact Codex process about to run. + +#### Properties + +##### cliVersion + +> **cliVersion**: `string` + +Defined in: [mcp/local-harness.ts:330](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L330) + +##### executableSha256 + +> **executableSha256**: `string` + +Defined in: [mcp/local-harness.ts:331](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L331) + +##### requestedPromptSha256 + +> **requestedPromptSha256**: `string` + +Defined in: [mcp/local-harness.ts:333](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L333) + +SHA-256 of the exact composed prompt argument proved present in the rendered prompt. + +##### effectivePromptSha256 + +> **effectivePromptSha256**: `string` + +Defined in: [mcp/local-harness.ts:334](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L334) + +##### nonPromptArgsSha256 + +> **nonPromptArgsSha256**: `string` + +Defined in: [mcp/local-harness.ts:335](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L335) + +##### controlledConfigSha256 + +> **controlledConfigSha256**: `string` + +Defined in: [mcp/local-harness.ts:336](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L336) + +##### readDeniedPaths + +> **readDeniedPaths**: `string`[] + +Defined in: [mcp/local-harness.ts:338](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L338) + +Sorted normalized paths compiled into the permission profile. + +##### readDeniedPathsSha256 + +> **readDeniedPathsSha256**: `string` + +Defined in: [mcp/local-harness.ts:339](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L339) + +##### readDeniedPathCount + +> **readDeniedPathCount**: `number` + +Defined in: [mcp/local-harness.ts:340](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L340) + +##### policy + +> **policy**: [`CodexExecutionPolicy`](#codexexecutionpolicy) + +Defined in: [mcp/local-harness.ts:341](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L341) + *** ### LocalHarnessResult -Defined in: [mcp/local-harness.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L144) +Defined in: [mcp/local-harness.ts:345](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L345) **`Experimental`** @@ -2651,7 +3107,7 @@ Defined in: [mcp/local-harness.ts:144](https://github.com/tangle-network/agent-r > **exitCode**: `number` \| `null` -Defined in: [mcp/local-harness.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L146) +Defined in: [mcp/local-harness.ts:347](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L347) **`Experimental`** @@ -2661,7 +3117,7 @@ OS exit code. `null` when killed before exit. > **stdout**: `string` -Defined in: [mcp/local-harness.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L148) +Defined in: [mcp/local-harness.ts:349](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L349) **`Experimental`** @@ -2671,7 +3127,7 @@ Concatenated stdout. > **stderr**: `string` -Defined in: [mcp/local-harness.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L150) +Defined in: [mcp/local-harness.ts:351](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L351) **`Experimental`** @@ -2681,7 +3137,7 @@ Concatenated stderr. > **killedBySignal**: `Signals` \| `null` -Defined in: [mcp/local-harness.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L152) +Defined in: [mcp/local-harness.ts:353](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L353) **`Experimental`** @@ -2691,7 +3147,7 @@ Set when the process exited via signal (timeout / abort). > **durationMs**: `number` -Defined in: [mcp/local-harness.ts:154](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L154) +Defined in: [mcp/local-harness.ts:355](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L355) **`Experimental`** @@ -2701,12 +3157,32 @@ Wall-clock duration ms (spawn → exit). > **timedOut**: `boolean` -Defined in: [mcp/local-harness.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L156) +Defined in: [mcp/local-harness.ts:357](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L357) **`Experimental`** Set when timeoutMs elapsed before exit. +##### usage? + +> `optional` **usage?**: [`CodexTokenUsage`](#codextokenusage) + +Defined in: [mcp/local-harness.ts:359](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L359) + +**`Experimental`** + +Present for a reproducible Codex run; parsed from the real terminal JSONL event. + +##### evidence? + +> `optional` **evidence?**: [`CodexExecutionEvidence`](#codexexecutionevidence) + +Defined in: [mcp/local-harness.ts:361](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L361) + +**`Experimental`** + +Present for reproducible Codex runs; generated and checked before model execution. + *** ### McpServerOptions @@ -3920,7 +4396,7 @@ Defined in: [mcp/tools/coordination.ts:44](https://github.com/tangle-network/age ###### Inherited from -[`Question`](#question).[`reason`](#reason-3) +[`Question`](#question).[`reason`](#reason-4) ##### urgency @@ -5430,7 +5906,7 @@ Defined in: [mcp/types.ts:308](https://github.com/tangle-network/agent-runtime/b ### WorktreeHandle -Defined in: [mcp/worktree.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L22) +Defined in: [mcp/worktree.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L23) **`Experimental`** @@ -5440,7 +5916,7 @@ Defined in: [mcp/worktree.ts:22](https://github.com/tangle-network/agent-runtime > **path**: `string` -Defined in: [mcp/worktree.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L24) +Defined in: [mcp/worktree.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L25) **`Experimental`** @@ -5450,7 +5926,7 @@ Absolute path to the worktree directory. > **baseSha**: `string` -Defined in: [mcp/worktree.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L26) +Defined in: [mcp/worktree.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L27) **`Experimental`** @@ -5460,7 +5936,7 @@ SHA the worktree was created at. > **branch**: `string` -Defined in: [mcp/worktree.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L28) +Defined in: [mcp/worktree.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L29) **`Experimental`** @@ -5470,7 +5946,7 @@ Branch name created for this worktree (typically `delegate/`). ### CreateWorktreeOptions -Defined in: [mcp/worktree.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L32) +Defined in: [mcp/worktree.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L33) **`Experimental`** @@ -5480,7 +5956,7 @@ Defined in: [mcp/worktree.ts:32](https://github.com/tangle-network/agent-runtime > **repoRoot**: `string` -Defined in: [mcp/worktree.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L34) +Defined in: [mcp/worktree.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L35) **`Experimental`** @@ -5490,7 +5966,7 @@ Absolute path to the main git checkout. > **runId**: `string` -Defined in: [mcp/worktree.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L36) +Defined in: [mcp/worktree.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L37) **`Experimental`** @@ -5500,7 +5976,7 @@ Unique id for the worktree path + branch. Use the delegation run id. > `optional` **variantsDir?**: `string` -Defined in: [mcp/worktree.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L38) +Defined in: [mcp/worktree.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L39) **`Experimental`** @@ -5510,7 +5986,7 @@ Parent directory the worktree lives under. Defaults to `.agent-worktrees`. > `optional` **baseRef?**: `string` -Defined in: [mcp/worktree.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L40) +Defined in: [mcp/worktree.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L41) **`Experimental`** @@ -5520,7 +5996,7 @@ Override the base ref (default `HEAD`). > `optional` **runGit?**: [`GitRunner`](#gitrunner) -Defined in: [mcp/worktree.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L42) +Defined in: [mcp/worktree.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L43) **`Experimental`** @@ -5530,7 +6006,7 @@ Test seam — inject a custom git runner. ### DiffOptions -Defined in: [mcp/worktree.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L46) +Defined in: [mcp/worktree.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L47) **`Experimental`** @@ -5540,7 +6016,7 @@ Defined in: [mcp/worktree.ts:46](https://github.com/tangle-network/agent-runtime > **worktree**: [`WorktreeHandle`](#worktreehandle) -Defined in: [mcp/worktree.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L48) +Defined in: [mcp/worktree.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L49) **`Experimental`** @@ -5550,17 +6026,29 @@ Worktree to diff. > `optional` **baseRef?**: `string` -Defined in: [mcp/worktree.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L50) +Defined in: [mcp/worktree.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L51) **`Experimental`** What to compare against. Default `worktree.baseSha`. +##### excludePaths? + +> `optional` **excludePaths?**: readonly `string`[] + +Defined in: [mcp/worktree.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L57) + +**`Experimental`** + +Repository-relative input paths to omit from the captured worker patch. +Paths are passed to Git with literal exclusion magic, so profile-provided +`*`, `?`, `[` and `:` characters can never expand into broader pathspecs. + ##### runGit? > `optional` **runGit?**: [`GitRunner`](#gitrunner) -Defined in: [mcp/worktree.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L52) +Defined in: [mcp/worktree.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L59) **`Experimental`** @@ -5570,7 +6058,7 @@ Test seam. ### DiffResult -Defined in: [mcp/worktree.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L56) +Defined in: [mcp/worktree.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L63) **`Experimental`** @@ -5580,7 +6068,7 @@ Defined in: [mcp/worktree.ts:56](https://github.com/tangle-network/agent-runtime > **patch**: `string` -Defined in: [mcp/worktree.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L57) +Defined in: [mcp/worktree.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L64) **`Experimental`** @@ -5588,7 +6076,7 @@ Defined in: [mcp/worktree.ts:57](https://github.com/tangle-network/agent-runtime > **stats**: `object` -Defined in: [mcp/worktree.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L58) +Defined in: [mcp/worktree.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L65) **`Experimental`** @@ -5608,7 +6096,7 @@ Defined in: [mcp/worktree.ts:58](https://github.com/tangle-network/agent-runtime ### RemoveWorktreeOptions -Defined in: [mcp/worktree.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L66) +Defined in: [mcp/worktree.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L73) **`Experimental`** @@ -5618,7 +6106,7 @@ Defined in: [mcp/worktree.ts:66](https://github.com/tangle-network/agent-runtime > **worktree**: [`WorktreeHandle`](#worktreehandle) -Defined in: [mcp/worktree.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L67) +Defined in: [mcp/worktree.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L74) **`Experimental`** @@ -5626,7 +6114,7 @@ Defined in: [mcp/worktree.ts:67](https://github.com/tangle-network/agent-runtime > **repoRoot**: `string` -Defined in: [mcp/worktree.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L68) +Defined in: [mcp/worktree.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L75) **`Experimental`** @@ -5634,7 +6122,7 @@ Defined in: [mcp/worktree.ts:68](https://github.com/tangle-network/agent-runtime > `optional` **force?**: `boolean` -Defined in: [mcp/worktree.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L70) +Defined in: [mcp/worktree.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L77) **`Experimental`** @@ -5644,7 +6132,7 @@ Force removal even if dirty (default true; the loser of a fanout has uncommitted > `optional` **runGit?**: [`GitRunner`](#gitrunner) -Defined in: [mcp/worktree.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L72) +Defined in: [mcp/worktree.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L79) **`Experimental`** @@ -5780,7 +6268,7 @@ SDK contract — re-invoking with the same ids returns the same outcome. > **LocalHarness** = `"claude"` \| `"codex"` \| `"opencode"` -Defined in: [mcp/local-harness.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L24) +Defined in: [mcp/local-harness.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L51) Local coding harness available inside the sandbox. @@ -5876,7 +6364,7 @@ The MCP wire carries it as JSON either way. > **GitRunner** = (`args`, `opts`) => `object` -Defined in: [mcp/worktree.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L76) +Defined in: [mcp/worktree.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L83) Pluggable git runner (sync) — replaceable in tests. @@ -7165,7 +7653,7 @@ then any consumer judges, returning on the first veto. > **runLocalHarness**(`options`): `Promise`\<[`LocalHarnessResult`](#localharnessresult)\> -Defined in: [mcp/local-harness.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L180) +Defined in: [mcp/local-harness.ts:386](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L386) **`Experimental`** @@ -7197,6 +7685,26 @@ Does NOT throw when: *** +### parseCodexTokenUsage() + +> **parseCodexTokenUsage**(`stdout`): [`CodexTokenUsage`](#codextokenusage) + +Defined in: [mcp/local-harness.ts:1350](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/local-harness.ts#L1350) + +Parse and validate the one terminal usage event emitted by `codex exec --json`. + +#### Parameters + +##### stdout + +`string` + +#### Returns + +[`CodexTokenUsage`](#codextokenusage) + +*** + ### createMcpServer() > **createMcpServer**(`options?`): [`McpServer`](#mcpserver) @@ -7719,7 +8227,7 @@ current trace context. > **createWorktree**(`options`): `Promise`\<[`WorktreeHandle`](#worktreehandle)\> -Defined in: [mcp/worktree.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L114) +Defined in: [mcp/worktree.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L128) **`Experimental`** @@ -7741,11 +8249,11 @@ Checkout a fresh git worktree for a delegation run on a new branch under `varian > **captureWorktreeDiff**(`options`): `Promise`\<[`DiffResult`](#diffresult)\> -Defined in: [mcp/worktree.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L134) +Defined in: [mcp/worktree.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L148) **`Experimental`** -Stage all changes in a worktree and return the diff patch + shortstat against the base ref. +Stage worker changes and return the diff + shortstat, excluding declared input paths. #### Parameters @@ -7763,11 +8271,12 @@ Stage all changes in a worktree and return the diff patch + shortstat against th > **removeWorktree**(`options`): `Promise`\<`void`\> -Defined in: [mcp/worktree.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L174) +Defined in: [mcp/worktree.ts:241](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree.ts#L241) **`Experimental`** -Remove a git worktree and delete its branch; tolerates already-removed paths. +Remove a git worktree and delete its branch. Already-removed paths are harmless; every other +Git failure rejects so callers cannot report a worktree as destroyed when cleanup failed. #### Parameters diff --git a/docs/api/primeintellect.md b/docs/api/primeintellect.md new file mode 100644 index 00000000..45dd6c11 --- /dev/null +++ b/docs/api/primeintellect.md @@ -0,0 +1,830 @@ +[**@tangle-network/agent-runtime**](README.md) + +*** + +[@tangle-network/agent-runtime](README.md) / primeintellect + +# primeintellect + +## Interfaces + +### WritePrimeIntellectPackageOptions + +Defined in: [primeintellect/package.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L23) + +#### Properties + +##### replace? + +> `optional` **replace?**: `boolean` + +Defined in: [primeintellect/package.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L25) + +Replace an existing generated package and restore it if the final swap fails. + +*** + +### RunPrimeIntellectProgramOptions + +Defined in: [primeintellect/runner.ts:17](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L17) + +#### Properties + +##### env? + +> `optional` **env?**: `ProcessEnv` + +Defined in: [primeintellect/runner.ts:18](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L18) + +*** + +### PrimeIntellectTrace + +Defined in: [primeintellect/traces.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L23) + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [primeintellect/traces.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L24) + +##### task + +> **task**: `object` + +Defined in: [primeintellect/traces.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L25) + +###### type + +> **type**: `string` + +###### data + +> **data**: `object` + +###### Index Signature + +\[`key`: `string`\]: `unknown` + +###### data.idx + +> **idx**: `number` + +###### data.name? + +> `optional` **name?**: `string` \| `null` + +###### data.split? + +> `optional` **split?**: `"train"` \| `"eval"` + +###### data.prompt? + +> `optional` **prompt?**: `unknown` + +###### data.system\_prompt? + +> `optional` **system\_prompt?**: `string` \| `null` + +###### data.metadata? + +> `optional` **metadata?**: `Record`\<`string`, `unknown`\> + +##### runtime? + +> `optional` **runtime?**: `unknown` + +Defined in: [primeintellect/traces.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L37) + +##### nodes + +> **nodes**: `PrimeTraceNode`[] + +Defined in: [primeintellect/traces.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L38) + +##### rewards + +> **rewards**: `Record`\<`string`, `number`\> + +Defined in: [primeintellect/traces.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L39) + +##### metrics + +> **metrics**: `Record`\<`string`, `number`\> + +Defined in: [primeintellect/traces.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L40) + +##### info? + +> `optional` **info?**: `Record`\<`string`, `unknown`\> + +Defined in: [primeintellect/traces.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L41) + +##### extra\_usage? + +> `optional` **extra\_usage?**: `PrimeUsage`[] + +Defined in: [primeintellect/traces.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L42) + +##### is\_completed? + +> `optional` **is\_completed?**: `boolean` + +Defined in: [primeintellect/traces.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L43) + +##### stop\_condition? + +> `optional` **stop\_condition?**: `string` \| `null` + +Defined in: [primeintellect/traces.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L44) + +##### errors? + +> `optional` **errors?**: `object`[] + +Defined in: [primeintellect/traces.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L45) + +###### type + +> **type**: `string` + +###### message + +> **message**: `string` + +###### traceback? + +> `optional` **traceback?**: `string` \| `null` + +##### timing? + +> `optional` **timing?**: `object` + +Defined in: [primeintellect/traces.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L46) + +###### start? + +> `optional` **start?**: `number` + +###### setup? + +> `optional` **setup?**: `PrimeTimeSpan` + +###### generation? + +> `optional` **generation?**: `PrimeTimeSpan` + +###### finalize? + +> `optional` **finalize?**: `PrimeTimeSpan` + +###### scoring? + +> `optional` **scoring?**: `PrimeTimeSpan` + +*** + +### PrimeIntellectTraceImportOptions + +Defined in: [primeintellect/traces.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L55) + +#### Properties + +##### experimentId + +> **experimentId**: `string` + +Defined in: [primeintellect/traces.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L56) + +##### candidateId + +> **candidateId**: `string` + +Defined in: [primeintellect/traces.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L57) + +##### seed + +> **seed**: `number` + +Defined in: [primeintellect/traces.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L58) + +##### model + +> **model**: `string` + +Defined in: [primeintellect/traces.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L60) + +Snapshot-pinned model id required by RunRecord validation. + +##### promptHash + +> **promptHash**: `string` + +Defined in: [primeintellect/traces.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L61) + +##### configHash + +> **configHash**: `string` + +Defined in: [primeintellect/traces.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L62) + +##### commitSha + +> **commitSha**: `string` + +Defined in: [primeintellect/traces.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L63) + +*** + +### PrimeIntellectTask + +Defined in: [primeintellect/types.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L32) + +One immutable problem. References stay inside Prime's task process. + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [primeintellect/types.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L33) + +##### split + +> **split**: [`PrimeIntellectSplit`](#primeintellectsplit) + +Defined in: [primeintellect/types.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L34) + +##### prompt + +> **prompt**: `string` \| [`PrimeIntellectMessage`](#primeintellectmessage)[] + +Defined in: [primeintellect/types.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L35) + +##### systemPrompt? + +> `optional` **systemPrompt?**: `string` + +Defined in: [primeintellect/types.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L36) + +##### answer? + +> `optional` **answer?**: `string` \| `string`[] + +Defined in: [primeintellect/types.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L37) + +##### metadata? + +> `optional` **metadata?**: `Record`\<`string`, [`PrimeIntellectJson`](#primeintellectjson)\> + +Defined in: [primeintellect/types.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L38) + +*** + +### PrimeIntellectRunner + +Defined in: [primeintellect/types.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L63) + +Files and commands that make the caller's real agent program runnable. + +#### Properties + +##### command + +> **command**: readonly \[`string`, `string`\] + +Defined in: [primeintellect/types.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L64) + +##### files? + +> `optional` **files?**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [primeintellect/types.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L65) + +##### setup? + +> `optional` **setup?**: readonly [`PrimeIntellectSetupCommand`](#primeintellectsetupcommand)[] + +Defined in: [primeintellect/types.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L66) + +##### forwardEnv? + +> `optional` **forwardEnv?**: readonly `string`[] + +Defined in: [primeintellect/types.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L67) + +##### image + +> **image**: `string` + +Defined in: [primeintellect/types.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L69) + +Container image used by the generated eval config. + +*** + +### PrimeIntellectPackageOptions + +Defined in: [primeintellect/types.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L72) + +#### Properties + +##### name + +> **name**: `string` + +Defined in: [primeintellect/types.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L73) + +##### version + +> **version**: `string` + +Defined in: [primeintellect/types.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L74) + +##### description? + +> `optional` **description?**: `string` + +Defined in: [primeintellect/types.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L75) + +##### tasks + +> **tasks**: readonly [`PrimeIntellectTask`](#primeintellecttask)[] + +Defined in: [primeintellect/types.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L76) + +##### scoring + +> **scoring**: [`PrimeIntellectScoring`](#primeintellectscoring) + +Defined in: [primeintellect/types.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L77) + +##### runner + +> **runner**: [`PrimeIntellectRunner`](#primeintellectrunner) + +Defined in: [primeintellect/types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L78) + +##### maxTurns? + +> `optional` **maxTurns?**: `number` + +Defined in: [primeintellect/types.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L80) + +Prime-enforced model turn cap. Default 16. + +##### maxInputTokens? + +> `optional` **maxInputTokens?**: `number` + +Defined in: [primeintellect/types.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L81) + +##### maxOutputTokens? + +> `optional` **maxOutputTokens?**: `number` + +Defined in: [primeintellect/types.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L82) + +##### maxTotalTokens? + +> `optional` **maxTotalTokens?**: `number` + +Defined in: [primeintellect/types.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L83) + +##### rolloutTimeoutSeconds? + +> `optional` **rolloutTimeoutSeconds?**: `number` + +Defined in: [primeintellect/types.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L84) + +##### scoringTimeoutSeconds? + +> `optional` **scoringTimeoutSeconds?**: `number` + +Defined in: [primeintellect/types.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L85) + +*** + +### PrimeIntellectPackageManifest + +Defined in: [primeintellect/types.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L88) + +#### Properties + +##### schema + +> **schema**: `"tangle.primeintellect.package/v1"` + +Defined in: [primeintellect/types.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L89) + +##### name + +> **name**: `string` + +Defined in: [primeintellect/types.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L90) + +##### moduleName + +> **moduleName**: `string` + +Defined in: [primeintellect/types.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L91) + +##### version + +> **version**: `string` + +Defined in: [primeintellect/types.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L92) + +##### verifiers + +> **verifiers**: `">=0.2.0,<0.3.0"` + +Defined in: [primeintellect/types.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L93) + +##### taskCount + +> **taskCount**: `number` + +Defined in: [primeintellect/types.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L94) + +##### splits + +> **splits**: `Record`\<[`PrimeIntellectSplit`](#primeintellectsplit), `number`\> + +Defined in: [primeintellect/types.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L95) + +##### taskIdsSha256 + +> **taskIdsSha256**: `string` + +Defined in: [primeintellect/types.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L96) + +##### filesSha256 + +> **filesSha256**: `Record`\<`string`, `string`\> + +Defined in: [primeintellect/types.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L97) + +*** + +### PrimeIntellectPackageBundle + +Defined in: [primeintellect/types.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L100) + +#### Properties + +##### manifest + +> **manifest**: [`PrimeIntellectPackageManifest`](#primeintellectpackagemanifest) + +Defined in: [primeintellect/types.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L101) + +##### files + +> **files**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [primeintellect/types.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L103) + +Relative package path to UTF-8 contents. + +*** + +### PrimeIntellectPublicTask + +Defined in: [primeintellect/types.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L107) + +The answer-free task exposed to the caller's runtime program. + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [primeintellect/types.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L108) + +##### split + +> **split**: [`PrimeIntellectSplit`](#primeintellectsplit) + +Defined in: [primeintellect/types.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L109) + +##### prompt + +> **prompt**: `string` \| [`PrimeIntellectMessage`](#primeintellectmessage)[] + +Defined in: [primeintellect/types.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L110) + +##### systemPrompt? + +> `optional` **systemPrompt?**: `string` + +Defined in: [primeintellect/types.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L111) + +##### metadata? + +> `optional` **metadata?**: `Record`\<`string`, [`PrimeIntellectJson`](#primeintellectjson)\> + +Defined in: [primeintellect/types.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L112) + +*** + +### PrimeIntellectEpisodeContext + +Defined in: [primeintellect/types.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L115) + +#### Properties + +##### protocol + +> **protocol**: `"tangle.primeintellect.episode/v1"` + +Defined in: [primeintellect/types.ts:116](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L116) + +##### task + +> **task**: [`PrimeIntellectPublicTask`](#primeintellectpublictask) + +Defined in: [primeintellect/types.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L117) + +##### model + +> **model**: `object` + +Defined in: [primeintellect/types.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L118) + +###### name + +> **name**: `string` + +###### baseUrl + +> **baseUrl**: `string` + +###### apiKey + +> **apiKey**: `string` + +##### mcpServers + +> **mcpServers**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [primeintellect/types.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L123) + +## Type Aliases + +### PrimeIntellectBackendOptions + +> **PrimeIntellectBackendOptions** = `Omit`\<`Parameters`\<*typeof* [`createOpenAICompatibleBackend`](index.md#createopenaicompatiblebackend)\>\[`0`\], `"apiKey"` \| `"baseUrl"` \| `"model"`\> + +Defined in: [primeintellect/runner.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L21) + +*** + +### PrimeIntellectImportDefaults + +> **PrimeIntellectImportDefaults** = [`PrimeIntellectTraceImportOptions`](#primeintellecttraceimportoptions) + +Defined in: [primeintellect/traces.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L66) + +*** + +### PrimeIntellectSplit + +> **PrimeIntellectSplit** = `"train"` \| `"eval"` + +Defined in: [primeintellect/types.ts:1](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L1) + +*** + +### PrimeIntellectJson + +> **PrimeIntellectJson** = `null` \| `boolean` \| `number` \| `string` \| [`PrimeIntellectJson`](#primeintellectjson)[] \| \{\[`key`: `string`\]: [`PrimeIntellectJson`](#primeintellectjson); \} + +Defined in: [primeintellect/types.ts:3](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L3) + +*** + +### PrimeIntellectContent + +> **PrimeIntellectContent** = `string` \| (\{ `type`: `"text"`; `text`: `string`; \} \| \{ `type`: `"image_url"`; `image_url`: \{ `url`: `string`; \}; \})[] + +Defined in: [primeintellect/types.ts:11](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L11) + +*** + +### PrimeIntellectMessage + +> **PrimeIntellectMessage** = \{ `role`: `"system"` \| `"user"`; `content`: [`PrimeIntellectContent`](#primeintellectcontent); \} \| \{ `role`: `"assistant"`; `content?`: `string` \| `null`; `reasoning_content?`: `string` \| `null`; `tool_calls?`: `object`[]; `provider_state?`: `Record`\<`string`, [`PrimeIntellectJson`](#primeintellectjson)\>[]; \} \| \{ `role`: `"tool"`; `tool_call_id`: `string`; `content`: [`PrimeIntellectContent`](#primeintellectcontent); `name?`: `string`; \} + +Defined in: [primeintellect/types.ts:15](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L15) + +*** + +### PrimeIntellectScoring + +> **PrimeIntellectScoring** = \{ `kind`: `"exact"`; `normalization?`: `"none"` \| `"trim"` \| `"trim-casefold"`; \} \| \{ `kind`: `"reference-judge"`; `model`: `string`; `prompt?`: `string`; `view?`: `"last_reply"` \| `"full_trace"`; \} \| \{ `kind`: `"command"`; `command`: readonly \[`string`, `...string[]`\]; `files?`: `Readonly`\<`Record`\<`string`, `string`\>\>; `forwardEnv?`: readonly `string`[]; `timeoutSeconds?`: `number`; \} + +Defined in: [primeintellect/types.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L41) + +*** + +### PrimeIntellectSetupCommand + +> **PrimeIntellectSetupCommand** = readonly \[`string`, `...string[]`\] + +Defined in: [primeintellect/types.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L60) + +## Functions + +### createPrimeIntellectPackage() + +> **createPrimeIntellectPackage**(`options`): [`PrimeIntellectPackageBundle`](#primeintellectpackagebundle) + +Defined in: [primeintellect/package.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L29) + +Build a complete PrimeIntellect Verifiers v1 package without writing to disk. + +#### Parameters + +##### options + +[`PrimeIntellectPackageOptions`](#primeintellectpackageoptions) + +#### Returns + +[`PrimeIntellectPackageBundle`](#primeintellectpackagebundle) + +*** + +### writePrimeIntellectPackage() + +> **writePrimeIntellectPackage**(`bundle`, `outputDirectory`, `options?`): `Promise`\<`string`\> + +Defined in: [primeintellect/package.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L90) + +Write a bundle through a sibling temporary directory, then rename it into place. + +#### Parameters + +##### bundle + +[`PrimeIntellectPackageBundle`](#primeintellectpackagebundle) + +##### outputDirectory + +`string` + +##### options? + +[`WritePrimeIntellectPackageOptions`](#writeprimeintellectpackageoptions) = `{}` + +#### Returns + +`Promise`\<`string`\> + +*** + +### readPrimeIntellectEpisodeContext() + +> **readPrimeIntellectEpisodeContext**(`env?`): [`PrimeIntellectEpisodeContext`](#primeintellectepisodecontext) + +Defined in: [primeintellect/runner.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L27) + +Read and validate the private process contract installed by the generated Prime harness. + +#### Parameters + +##### env? + +`ProcessEnv` = `process.env` + +#### Returns + +[`PrimeIntellectEpisodeContext`](#primeintellectepisodecontext) + +*** + +### createPrimeIntellectBackend() + +> **createPrimeIntellectBackend**(`context`, `options?`): [`AgentExecutionBackend`](index.md#agentexecutionbackend)\<[`AgentBackendInput`](index.md#agentbackendinput)\> + +Defined in: [primeintellect/runner.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L50) + +Build the existing runtime backend against Prime's intercepted model endpoint. + +#### Parameters + +##### context + +[`PrimeIntellectEpisodeContext`](#primeintellectepisodecontext) + +##### options? + +[`PrimeIntellectBackendOptions`](#primeintellectbackendoptions) = `{}` + +#### Returns + +[`AgentExecutionBackend`](index.md#agentexecutionbackend)\<[`AgentBackendInput`](index.md#agentbackendinput)\> + +*** + +### runPrimeIntellectProgram() + +> **runPrimeIntellectProgram**\<`Result`\>(`run`, `options?`): `Promise`\<`Result`\> + +Defined in: [primeintellect/runner.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L67) + +Execute the caller's canonical runtime program inside a Prime rollout. +The callback may call runPersonified, runAgentic, runLoop, or any product wrapper. + +#### Type Parameters + +##### Result + +`Result` + +#### Parameters + +##### run + +(`context`) => `Promise`\<`Result`\> + +##### options? + +[`RunPrimeIntellectProgramOptions`](#runprimeintellectprogramoptions) = `{}` + +#### Returns + +`Promise`\<`Result`\> + +*** + +### parsePrimeIntellectTraces() + +> **parsePrimeIntellectTraces**(`jsonl`): [`PrimeIntellectTrace`](#primeintellecttrace)[] + +Defined in: [primeintellect/traces.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L69) + +Parse Prime's durable `traces.jsonl` and reject malformed rows with a line number. + +#### Parameters + +##### jsonl + +`string` + +#### Returns + +[`PrimeIntellectTrace`](#primeintellecttrace)[] + +*** + +### importPrimeIntellectTraces() + +> **importPrimeIntellectTraces**(`jsonl`, `defaults`): `RunRecord`[] + +Defined in: [primeintellect/traces.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L90) + +Convert all Prime traces to agent-eval RunRecords while retaining one shared run config. + +#### Parameters + +##### jsonl + +`string` + +##### defaults + +[`PrimeIntellectTraceImportOptions`](#primeintellecttraceimportoptions) + +#### Returns + +`RunRecord`[] + +*** + +### primeIntellectTraceToRunRecord() + +> **primeIntellectTraceToRunRecord**(`trace`, `options`): `RunRecord` + +Defined in: [primeintellect/traces.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L100) + +Project one complete Prime trace into the common agent-eval analysis row. + +#### Parameters + +##### trace + +[`PrimeIntellectTrace`](#primeintellecttrace) + +##### options + +[`PrimeIntellectTraceImportOptions`](#primeintellecttraceimportoptions) + +#### Returns + +`RunRecord` diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index b553f220..b26bd8f2 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.89.0` and `@tangle-network/agent-eval@0.108.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.94.11` and `@tangle-network/agent-eval@0.117.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,25 +15,34 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 231 exports. +Import from `@tangle-network/agent-runtime` — 341 exports. | Symbol | Kind | Summary | |---|---|---| | `agenticGenerator` | function | Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. | +| `applyExactAgentProfileDiff` | function | Apply one exact diff and reject any value that cannot be preserved canonically. | +| `applyImprovementWinnerToProfile` | function | Apply a promoted winner surface back into the profile field for `surface`. | | `applyRunRecordDefaults` | function | Stamp cross-cutting defaults onto adapter-projected RunRecords without | | `auditLoopRunner` | function | `audit` mode — analyst loop over captured trace/run data. | +| `buildAgentCandidateBundle` | function | Compile one measured profile/code candidate into the immutable execution | | `buildForwardHeaders` | function | Build the headers to emit on an outbound participant call, given the | | `buildLoopOtelSpans` | function | Build a nested, real-duration OTLP span tree for ONE loop run from its full | | `buildLoopSpanNodes` | function | Sink-neutral core behind {@link buildLoopOtelSpans}: reconstruct the | +| `buildRuntimeEventOtelSpans` | function | Convert normalized runtime events into lossless, redacted child spans. | +| `candidateExecutionClaim` | function | Extract the complete durable claim from a prepared execution. | +| `captureAgentCandidateWorkspace` | function | Capture one exact regular-file workspace for immutable candidate execution. | +| `captureAgentCandidateWorkspaceFiles` | function | Capture detached files returned by a remote executor into the standard archive. | | `cleanModelId` | function | Trim a candidate model id; `undefined` for non-strings and blanks. | | `commandVerifier` | function | A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other | | `composeRuntimeHooks` | function | Merge several {@link RuntimeHooks} into one. Falsy entries are dropped (so you can | | `computeBackoff` | function | Compute the delay before the next attempt. Default: 250ms exponential with jitter. | +| `createAgentCandidateWorkspacePort` | function | Create the standard bounded materializer for candidate execution ports. | | `createAgentKnowledgeReadinessCheck` | function | Build the default readiness check backed by `@tangle-network/agent-knowledge` validation and scoring. | | `createConversationBackend` | function | Wrap a `Conversation` so it satisfies `AgentExecutionBackend`. The result is | | `createIterableBackend` | function | Wrap any custom async-iterable stream into a typed `AgentExecutionBackend`. | | `createOpenAICompatibleBackend` | function | OpenAI-compat streaming backend. Routes `runAgentTaskStream` through any | | `createOtelExporter` | function | Create an OTEL exporter. Returns undefined when no endpoint is configured. | +| `createProtectedAgentCandidateModelPort` | function | Bind a protected model-grant service to the immutable candidate runtime. | | `createRuntimeEventCollector` | function | Build an in-memory collector that sanitizes and accumulates `AgentRuntimeEvent`s for inspection. | | `createRuntimeStreamEventCollector` | function | Streaming-event counterpart of `createRuntimeEventCollector`. Pass each | | `createSandboxPromptBackend` | function | Build an `AgentExecutionBackend` backed by a sandbox/sidecar `streamPrompt` call. | @@ -43,6 +52,8 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `defineConversation` | function | Declarative constructor for a multi-agent `Conversation`. Validates inputs | | `defineRuntimeHooks` | function | Identity helper that types a {@link RuntimeHooks} literal so the fields are inferred. | | `deriveExecutionId` | function | Derive a stable executionId from the run identity. The same | +| `disposePreparedAgentCandidateExecution` | function | Revoke reservations held by a prepared candidate that will not be executed. | +| `executePreparedAgentCandidate` | function | Executes and finalizes one durably claimed candidate without exposing an unproven result. | | `exportEvalRuns` | function | Ship self-improvement eval-run events to Tangle Intelligence. Unlike the | | `formatSupervisedKnowledgeTask` | function | Format the supervisor task with the KB root, readiness requirements, current findings, and metadata. | | `getModels` | function | Fetch the model catalog from the router's `/v1/models`. Throws on a non-2xx | @@ -60,10 +71,16 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `mcpToolsForRuntimeMcpSubset` | function | Subset filter — return only the projected tools whose `function.name` | | `notifyRuntimeDecisionPoint` | function | Fire `hooks.onDecisionPoint`, swallowing sync throws and surfacing async failures to `onError`. | | `notifyRuntimeHookEvent` | function | Fire `hooks.onEvent`, swallowing sync throws and surfacing async failures to `onError`. | +| `parseExactAgentProfile` | function | Parse a complete profile without silently discarding unsupported fields. | +| `parseExactAgentProfileDiff` | function | Parse a profile diff without silently discarding unsupported fields. | | `parseLoopRunnerArgv` | function | Parse `--mode X --config Y` from an argv tail (`process.argv.slice(2)`). | +| `persistCandidateOutputArtifact` | function | Persist evaluator evidence, read it back, and bind the returned locator to the exact bytes. | +| `prepareAgentCandidateExecution` | function | Materializes a verified candidate into one immutable evaluator-owned execution plan. | +| `profileDiffProposer` | function | Turn exact AgentProfileDiffs from any source into full profile candidates for | | `rawTraceDistiller` | function | Build an `analyzeGeneration` producer that feeds the proposer RAW-TRACE | | `readDepth` | function | Read the depth counter off an inbound request. Missing → 0 (caller is the | | `readinessServerSentEvent` | function | Serialize a `KnowledgeReadinessReport` as a Server-Sent Event string. | +| `recoverExpiredAgentCandidateExecution` | function | Close an expired crashed attempt from persisted non-secret handles, then record failure. | | `reflectiveGenerator` | function | Cheap no-sandbox `CandidateGenerator` (the `shots=1` setting): draft surface edits via the improvement adapter and apply them as one coherent candidate. | | `researchLoopRunner` | function | `research` mode — research-in-a-loop with valid-only KB growth. | | `resolveAgentBackend` | function | Resolve the `AgentExecutionBackend` for the chosen `kind`. Reuse this instead | @@ -84,6 +101,7 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `sanitizeAgentRuntimeEvent` | function | Reduce an `AgentRuntimeEvent` to a PII-safe, serializable plain object for telemetry. | | `sanitizeKnowledgeReadinessReport` | function | Strip PII and large blobs from a `KnowledgeReadinessReport` for safe telemetry emission. | | `sanitizeRuntimeStreamEvent` | function | Reduce a `RuntimeStreamEvent` to a PII-safe, serializable plain object for telemetry. | +| `sealAgentCandidateBundle` | function | Validate and content-address a candidate bundle before it crosses an approval boundary. | | `selfImproveLoopRunner` | function | `self-improve` mode — agent-eval's one-call closed improvement loop (held-out gated). | | `sleep` | function | Resolve after `ms` milliseconds — used for retry backoff in conversation call policy. | | `slugifySpeaker` | function | Reduce a speaker name to ASCII alphanumerics + dashes. Preserves enough | @@ -92,7 +110,11 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `toolBuildPrompt` | function | Build the starting instruction for a coder agent tasked with implementing a new tool. | | `turnId` | function | Deterministic turn identifier. Stable across retries of the same logical | | `validateChatModelId` | function | Validate a caller-supplied chat-model id. Rejects non-strings, malformed | +| `verifyAgentCandidateBundle` | function | Verifies every digest, resource, workspace, and Git object in a candidate bundle. | | `worktreeLoopRunner` | function | `code` mode on the GENERIC recursive path: author one `AgentProfile` per harness, run them as a | +| `AGENTIC_PROFILE_RESOURCE_ROOT` | const | Dedicated ephemeral root for generic author-profile files. Every declared | +| `CANDIDATE_TRACE_ENV` | const | Environment keys used to propagate immutable candidate trace identity. | +| `CANDIDATE_TRACE_TAGS` | const | Protected trace tags that bind a run to one prepared candidate execution. | | `DEFAULT_MAX_DEPTH` | const | Hard cap on chained gateway hops; refused beyond this. Default keeps recursion bounded. | | `DEFAULT_ROUTER_BASE_URL` | const | Default Tangle Router base URL used when no env override is set. | | `defaultIsRetryable` | const | Default retryable classification — network/timeout class errors. Errors | @@ -106,7 +128,9 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `CircuitOpenError` | class | Thrown when the circuit breaker is open for a participant and no retry is allowed yet. | | `ConfigError` | class | Configuration missing or malformed (`HOME` unset, required image not supplied, env var absent). | | `DeadlineExceededError` | class | Thrown when a backend call exceeds its per-attempt deadline. | +| `FileAgentCandidateExecutionClaimStore` | class | Cross-process lifecycle implemented as fsynced, create-if-absent records. | | `FileConversationJournal` | class | JSONL on disk. One line per record; first line is the `begin`, subsequent | +| `InMemoryAgentCandidateExecutionClaimStore` | class | Single-process lifecycle implementation. | | `InMemoryConversationJournal` | class | In-memory `ConversationJournal` — suitable for testing and single-process runs. | | `InMemoryRuntimeSessionStore` | class | In-memory `RuntimeSessionStore` for single-process use and tests. | | `JudgeError` | class | A judge call failed in a way that's not retryable: schema parse failure, bad rubric, conflicting dimensions. | @@ -115,7 +139,29 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `RuntimeRunStateError` | class | A runtime-run lifecycle method was called in an order the state machine does | | `SqlConversationJournal` | class | SQL-backed ConversationJournal. Two tables — runs (one row per runId, holds | | `ValidationError` | class | Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). | +| `AgentCandidateArtifactPort` | interface | Reads one content-addressed object from the closed S3/IPFS locator set. | +| `AgentCandidateBenchmarkGraderPort` | interface | Evaluator-owned executable grader, pinned by immutable implementation bytes. | +| `AgentCandidateCodeSurfaceSource` | interface | The only accepted path from an agent-eval code candidate to executable bytes. | +| `AgentCandidateExecutionAttemptRecord` | interface | Persisted state available to a fresh trusted recovery worker after a crash. | +| `AgentCandidateExecutionClaim` | interface | Immutable signed identity stored for one execution attempt. | +| `AgentCandidateExecutionClaimStore` | interface | Atomic one-shot store for candidate execution attempts. | +| `AgentCandidateExecutionCleanupHandles` | interface | Non-secret identities a trusted recovery worker needs to close an abandoned attempt. | +| `AgentCandidateExecutionLease` | interface | Secret capability required to finish the acquired attempt. | +| `AgentCandidateExecutionRecoveryEvidence` | interface | Trusted, independently observed closure facts for one expired winning lease. | +| `AgentCandidateExecutionUsage` | interface | Exact fixed-point usage proven by the closed evaluator model ledger. | +| `AgentCandidateExecutorFinalCapture` | interface | Idempotent executor result after process death and trace drain. | +| `AgentCandidateExecutorMemoryCapture` | interface | Raw isolated-memory capture made only after access has been revoked. | +| `AgentCandidateExecutorPort` | interface | Executes one prepared request inside an evaluator-owned isolation boundary. | +| `AgentCandidateExecutorRequest` | interface | One detached request passed to the trusted environment-specific executor. | +| `AgentCandidateExecutorStopRequest` | interface | Opaque process identity used for termination without re-exposing launch credentials. | +| `AgentCandidateExecutorTaskOutcomeCapture` | interface | Raw evaluator capture made only after the candidate process is dead. | +| `AgentCandidateModelGrantClient` | interface | Narrow transport contract for a service that owns scoped model credentials | +| `AgentCandidateOutputArtifactPort` | interface | Durable content-addressed evidence store controlled only by the evaluator. | +| `AgentCandidateProtectedModelCall` | interface | One evaluator-gateway call in the final, revoked model-access ledger. | +| `AgentCandidateRepositoryPort` | interface | Resolves a declared GitHub repository to an already-present local Git object store. | +| `AgentCandidateWorkspacePort` | interface | Materializes an already-verified workspace archive. | | `BackendErrorDetail` | interface | Typed transport / backend failure detail. Carried on `backend_error` and | +| `BuildAgentCandidateBundleInput` | interface | Complete measured surfaces and execution policy compiled into one candidate bundle. | | `CandidateGenerator` | interface | The byte-producing seam — the ONE thing that differs between the cheap | | `ChatStreamEvent` | interface | The NDJSON line protocol every product chat client already speaks. | | `ChatTurnIdentity` | interface | Identity of a chat turn. `tenantId` is the workspace id for workspace- | @@ -136,10 +182,26 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `SqlAdapter` | interface | Minimal SQL driver shape. Implementations forward to whichever client the | | `ToolLoopAssistantToolCall` | interface | One OpenAI-shaped tool-call entry carried on an assistant message. | | `ToolLoopCall` | interface | Bounded turn-level tool-dispatch loop. | +| `VerifiedAgentCandidateTaskOutcome` | interface | Branded task outcome that has survived independent patch and tree verification. | | `VerifyResult` | interface | Outcome of verifying a candidate worktree. `feedback` (compiler errors, | | `AgentBackendKind` | type | The transport a chat backend runs on. | +| `AgentCandidateBundleInput` | type | Exact candidate wire shape before the runtime computes its canonical digest. | +| `AgentCandidateCodeSource` | type | Explicit control/no-op code or one finalized CodeSurface whose bytes must still verify. | +| `AgentCandidateExecutionClaimResult` | type | Result of atomically claiming one execution attempt. | +| `AgentCandidateExecutionFailureClass` | type | Only the first class is retryable, and only when the closed model ledger has zero calls. | +| `AgentCandidateExecutionFinishResult` | type | Result of atomically recording an attempt's terminal facts. | +| `AgentCandidateExecutionPhase` | type | Monotonic durable phase: the second value means candidate code could have started. | +| `AgentCandidateExecutionPhaseResult` | type | Result of crossing the irreversible candidate-may-run boundary. | +| `AgentCandidateExecutionStageResult` | type | Result of durably staging the one immutable terminal outbox entry. | +| `AgentCandidateExecutionTerminalRecord` | type | Durable terminal record for one acquired execution attempt. | +| `AgentCandidateExecutionTerminalResult` | type | Evaluator-owned terminal facts staged durably before the terminal CAS. | +| `AgentCandidateModelGrantReservation` | type | Secret-free response from the service's reservation endpoint. | +| `AgentCandidateModelLimits` | type | Limits mechanically enforced by the evaluator-owned model gateway. | +| `AgentCandidateProfileSource` | type | A complete profile that can be frozen without losing behavior. | | `AgentEvalErrorCode` | type | Error taxonomy for `@tangle-network/agent-eval`. | -| `ImproveSurface` | type | The agent-profile lever `improve` optimizes. Mirrors the AgentProfile-law | +| `AgenticGeneratorShotDisposition` | type | Worktree decision emitted before a completed shot is retried, accepted, or | +| `AgenticGeneratorShotExecution` | type | Frozen exact harness result for an author shot: full streams, process state, | +| `ImproveSurface` | type | The executable agent lever `improve` optimizes. Profile fields remain | | `OpenAIChatResponseFormat` | type | `response_format` parameter for OpenAI-compatible chat endpoints. Use | | `OpenAIChatToolChoice` | type | `tool_choice` parameter for OpenAI-compat chat. Same shape as the OpenAI | | `PersonaDriver` | type | A persona that drives the conversation: either a full driver `AgentProfile` | @@ -152,7 +214,7 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `ToolLoopStopReason` | type | Why the loop stopped. `completed` = model finished naturally; `stuck-loop` = | | `Verifier` | type | Verifies the edited worktree. Sync or async; throws only on a setup fault | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `HaltContext`, `HaltSignal`, `ImprovementDriverOptions`, `ImproveOptions`, `ImproveResult`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToolLoopResult`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateBenchmarkGraderIdentity`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorProfileFile`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateTaskExecution`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentProfileDiffProposal`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveMemoryOptions`, `ImprovementDriverOptions`, `ImproveResult`, `ImproveSkillsOptions`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `ManagedImprovementDriver`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `ProfileDiffProposerOptions`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveOptions`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `ProfileDiffProposerContext`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. ### Vertical agent — manifest + improvement adapter @@ -198,7 +260,7 @@ Import from `@tangle-network/agent-runtime/agent` — 48 exports. ### Intelligence SDK — Observe + provable-OFF billing -Import from `@tangle-network/agent-runtime/intelligence` — 63 exports. +Import from `@tangle-network/agent-runtime/intelligence` — 85 exports. | Symbol | Kind | Summary | |---|---|---| @@ -206,36 +268,46 @@ Import from `@tangle-network/agent-runtime/intelligence` — 63 exports. | `composeCertifiedProfile` | function | Compose a certified profile into a uniform `ResolvedSurface`. Additive over | | `composeCertifiedProfileFromWire` | function | Lower a plane `CertifiedProfile` straight into a `ResolvedSurface` via | | `composeCertifiedPrompt` | function | Fold the certified prompt surface (and any certified prompt-folding artifacts: | +| `createAgentImprovementProposal` | function | Freeze an already-measured improvement into the one reviewable proposal | | `createCertifiedPromptSource` | function | Create the cached certified-prompt source — the ONE module-scope-cache + | -| `createIntelligenceClient` | function | Create an Observe-mode Intelligence client. Resolves effort, endpoint, and | +| `createIntelligenceClient` | function | Create an Observe-mode Intelligence client. Resolves effort, the base URL, and | | `defaultRedactor` | function | The built-in redactor. Walks objects and arrays; replaces values under | +| `executeApprovedAgentCandidate` | function | Verify, materialize, run, grade, and receipt only the exact approved bundle. | | `isIntelligenceOff` | function | True when these settings admit NO intelligence spawn — the passthrough | | `manifestFromProfile` | function | Lower the EXISTING plane wire (`CertifiedProfile`) into a `CapabilityManifest`. | +| `normalizeCertifiedProfile` | function | Deserialize the composed-endpoint response into a `CertifiedProfile`. The | +| `proposeAgentImprovement` | function | Analyze one run and produce one measured, review-only improvement proposal. | | `pullCertified` | function | Pull the certified composed profile for a target. Fail-closed: a network | | `resolveEffort` | function | Compile a named tier (plus optional per-field overrides) into the flat | +| `resolveIntelligenceBaseUrl` | function | Resolve the ONE Intelligence base URL — the single knob both the send and | | `resolveRedactor` | function | Resolve the redactor a client uses. A caller-supplied hook replaces the | -| `withCertifiedDelivery` | function | Wrap an agent so it (a) Observes each run via the shipped Observe client and | -| `withTangleIntelligence` | function | Wrap a generic `agent` with best-effort Observe-mode tracing, returning the | +| `reviewAgentImprovementProposal` | function | Persist an approve/reject/change-request decision bound to one exact proposal. | +| `verifyAgentImprovementProposal` | function | Validate a proposal's schema, profile, sealed bundle, and canonical digest. | +| `verifyAgentImprovementReview` | function | Validate a review's decision fields and canonical digest. | +| `withIntelligence` | function | Wrap an agent so it (a) RECEIVES the tenant's certified profile — the prompt | | `defaultEffortTier` | const | The default tier when a client declares no effort. `'standard'` turns | | `CapabilityNotAdmittedError` | class | A binding kind whose resolver case is typed but not yet admitted (rag-index, | -| `AppliedIntelligence` | interface | What the delivery wrapper hands the agent each run. | +| `AppliedIntelligence` | interface | What the hook hands the agent each run. Additive over the prompt-only | | `CapabilityManifest` | interface | The strict generalization of `CertifiedProfile`. `promptSurface` is kept | | `CertifiedArtifact` | interface | A promoted, certified artifact (one entry in the composed profile). | | `CertifiedCapability` | interface | One certified unit of agent power. | +| `CertifiedCapabilitySummary` | interface | The composed endpoint's per-capability summary — the narrow shape on the | | `CertifiedProfile` | interface | The composed certified profile — exactly the shape the plane's | | `CertifiedPromptSource` | interface | A cached, self-refreshing source of a target's certified prompt additions — | | `CertifiedPromptSourceOptions` | interface | Options for {@link createCertifiedPromptSource} — the pull coordinates plus | | `CertifiedPromptSurface` | interface | The active promoted prompt surface for a target. | | `CertProvenance` | interface | The certify lane's held-out lift travelling WITH delivery. The shipped | | `CredentialRef` | interface | A named secret a binding requires — declared, never carried. | -| `DeliveryConfig` | interface | Delivery config = the Observe config plus the pull target + refresh cadence. | +| `DiffProvenance` | interface | The held-out provenance the plane's certify step stamps on a promoted diff. | | `DoctorReport` | interface | The `doctor()` readiness report — Mode-readiness without any network call. | | `EffortOverridesCompiled` | interface | The run-config overrides an `EffortSettings` compiles to — the bridge between the | | `EffortSettings` | interface | The flat, resolved settings a tier compiles to. Every field is individually | | `HostSpec` | interface | The host a `process-on-infra` binding provisions before its inner binding. | | `IntelligenceClient` | interface | The Observe-mode Intelligence client. | | `IntelligenceConfig` | interface | Client configuration. `project` + `apiKey` are the Observe minimum; the | +| `IntelligenceHookConfig` | interface | `withIntelligence` config = the Observe config plus the pull target, refresh | | `ModeReadiness` | interface | One mode's readiness verdict. | +| `ProposedProfileDiff` | interface | A gate-certified profile diff the plane has already promoted, plus the | | `ProvisionedHost` | interface | A live, provisioned host the resolver tore up for a `process-on-infra` arm. | | `RecordTraceMeta` | interface | Metadata for {@link IntelligenceClient.recordTrace}. | | `RepoConfig` | interface | Repo coordinates a product may declare for the (later) Gated-PR mode. The | @@ -244,32 +316,33 @@ Import from `@tangle-network/agent-runtime/intelligence` — 63 exports. | `ResolvedRetrieval` | interface | One retrieval handle. The agent never learns vector vs graph vs index. | | `ResolvedSubagent` | interface | One resolved subagent — folded into `AgentProfile.subagents`. | | `ResolvedSurface` | interface | What `composeCertifiedProfile` produces. Every binding fans into the same | +| `RunRecord` | interface | The typed record `withIntelligence` sends per call — serialized through the | +| `RunReport` | interface | What an agent reports (via `applied.record`) to enrich the {@link RunRecord} | | `TraceHandle` | interface | The trace handle a `traceRun` body records into. `recordOutput` captures the | | `TraceMeta` | interface | Metadata describing one traced run. `runId`/`traceId` default to fresh ids. | | `TraceOutcome` | interface | The resolved outcome of one traced run, surfaced on the export span and | | `UsageSplit` | interface | The per-class cost split carried by every trace and outcome. `off` ⇒ | -| `Agent` | type | A generic agent: one async input → output. The shape `withTangleIntelligence` | | `CapabilityAuth` | type | How a binding authenticates at resolve time. Declared as a REQUIREMENT in the | | `CapabilityInterface` | type | What the agent consumes. CLOSED — a new runtime kind NEVER extends this. Each | | `CapabilitySurface` | type | Every interface surface tag — the closed set the resolver fans into slots. | -| `ClientOrConfig` | type | Either a built client or the config to build one. | | `ContentRef` | type | Where a capability's bytes live. A leaked manifest carries no live secret and | | `CorpusAccess` | type | Corpus access an intelligence tier permits. `'off'` reads and writes | -| `DeliveredAgent` | type | An agent wrapped by {@link withCertifiedDelivery}: receives the input plus | | `DeliveryBinding` | type | How a capability is backed. OPEN tagged union — THE extension point. All arms | | `DeliveryBindingKind` | type | Every binding kind — the open set the resolver dispatches over. | | `EffortOverrides` | type | Per-field overrides applied on top of a tier preset. Any subset of the | | `EffortTier` | type | The named effort tiers, lowest to highest. `'off'` is the honest floor | +| `IntelligenceAgent` | type | An agent wrapped by {@link withIntelligence}: receives the input plus the | +| `IntelligenceWrapped` | type | The wrapped agent — same `(input) => Promise` shape, plus a manual | | `JsonSchema` | type | A JSON Schema object describing a tool's parameters. Kept structural — the | | `PullOutcome` | type | Typed outcome for the pull — inspect `succeeded` before `value`. A 404 | | `Redactor` | type | A redactor maps an arbitrary trace value to a safe-to-export value. Pure; | | `UsageClass` | type | Usage class for billing. Base-stream tokens bill `'inference'`; every | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `PullCertifiedOptions`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentImprovementProposal`, `AgentImprovementReview`, `CandidateExecutionEvidence`, `CreateAgentImprovementProposalOptions`, `ExecuteApprovedAgentCandidateOptions`, `ExecuteApprovedAgentCandidateResult`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `AgentImprovementEvaluation`, `AgentImprovementReviewDecision`. ### Recursive atom + loop kernel (alias of ./runtime) -Import from `@tangle-network/agent-runtime/loops` — 432 exports. +Import from `@tangle-network/agent-runtime/loops` — 457 exports. | Symbol | Kind | Summary | |---|---|---| @@ -285,8 +358,11 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `authorStrategy` | function | Author + load a strategy from losses. Throws when the author emits no loadable module; | | `breadthStrategy` | function | BREADTH: K independent rollouts (each own artifact), verifier picks the best. | | `buildSteerContext` | function | Build the `SteerContext` a combinator reads to steer (its `loopUntil.until`, `widen` gate, any | +| `canDisplace` | function | The repair keep-best guard: a challenger displaces the incumbent only when it is | | `collectAgentTurn` | function | Drain a `streamAgentTurn` stream (or any `RuntimeStreamEvent` stream that | +| `compareCheckOutcomes` | function | The selection order: crash < ran; then official pass-fraction; authored guesses only | | `completionAuthorizes` | function | Decide whether a `CompletionVerdict` may end the node under the policy: authority scales with the verdict's determinism, and probabilistic verdicts must clear `minConfidence`. | +| `composeCheckSources` | function | Concatenate check sources (official first by convention — ordering does not affect | | `computeFindingId` | function | Compute the stable finding_id from the identity-defining fields. | | `contentAddress` | function | Mint the content-addressed `outRef` for a result artifact: `sha256:` over a | | `createAgentEnvironmentProviderRegistry` | function | Create a registry that resolves provider names to concrete provider instances. | @@ -308,6 +384,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `createWaterfallCollector` | function | Build a `WaterfallCollector` that records agent spans and renders them as an ASCII timeline. | | `createWorktreeCliExecutor` | function | Build a worktree-CLI leaf `Executor`. Per-spawn (a fresh worktree + abort + teardown each), so a | | `decodeToolPart` | function | Decode a part with a specific harness's adapter when known, else try every registered adapter | +| `defaultExtractCandidate` | function | The candidate a shot produced, read from its conversation: the LAST `submit_answer` | | `defaultSelectWinner` | function | The kernel's winner argmax — best-valid-score, ties broken by earliest index, | | `defaultToolDetectors` | function | The default online panel for a tool-call pipe: a worker repeating the same call, or hammering | | `defineLeaderboard` | function | Assemble a declarative spec (`cases` + `prompt` + `score`) into a runnable | @@ -323,6 +400,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `extractLlmCallEvent` | function | Extract a `RuntimeStreamEvent`-shaped `llm_call` from a sandbox event when | | `failuresAnalyst` | function | The default self-improvement LENS — authored content, not a code path. On each settled worker it hands | | `fanout` | function | `fanout(items, opts)` — spawn one child per item in a single round (bounded by the conserved | +| `filterAuthoredAsserts` | function | The proven authored-assert filter (lifted from the rigs' generateTests): keep only | | `finalizeBestDelivered` | function | Keep-best finalize under the completion-oracle: return the highest-scoring DELIVERED child's | | `flatWidenGate` | function | The flat default `ScopeWidenGate` — never widens, keeping the R2 selector≠judge collision | | `gateOnDeliverable` | function | Wrap an `Executor` so its settlement `valid` reflects the deliverable check, not the | @@ -339,8 +417,10 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `makeFinding` | function | Convenience factory: produce a fully-formed AnalystFinding with the | | `mapSandboxEvent` | function | Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary, | | `mapSandboxToolEvent` | function | Project one `SandboxEvent` onto the `tool_call` / `tool_result` variants of | +| `modelAuthoredChecks` | function | Default authored-check source: one metered LLM call per task, before sampling, | | `naiveDriver` | function | `naiveDriver` — the no-signal steering control. | | `observe` | function | The third-person trace analyst: read a worker's trace and produce steer findings for the next attempt plus durable `learned` facts for the cross-run corpus. | +| `officialChecksFromMeta` | function | Official checks the surface stashed on the task (e.g. MBPP's shown assert). Reads | | `openSandboxRun` | function | Open a sandbox run. Harness-agnostic: the harness lives in | | `pairwiseSignificance` | function | Compare EVERY profile pair on the scenarios they both ran — paired-bootstrap effect + CI, a real | | `panel` | function | `panel(spec)` — spawn the M judge children over the SAME artifact, drain their settlements, | @@ -364,6 +444,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `renderReport` | function | Operator-facing report, split by who should act. The agent block is the | | `reportLoopUsage` | function | Forward a `LoopResult`'s aggregated cost + token usage into a campaign cost | | `resolveAgentEnvironmentProvider` | function | Resolve a provider instance or registry name, failing loudly when a name is unknown. | +| `resolveEntrySymbol` | function | The symbol authored checks are pinned to: `task.meta.entryPoint` when the surface | | `resolveSandboxClient` | function | Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core | | `routerBrain` | function | The router as a supervisor BRAIN: the canonical `ToolLoopChat` seam backed by the router's | | `routerChatWithTools` | function | A router completion WITH tool-calling — the operator driver's LLM seam. Passes OpenAI-shape | @@ -375,8 +456,10 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `runLoop` | function | The round-synchronous loop kernel: each round `driver.plan()` fans N tasks to sandboxes (bounded concurrency), parses + validates each output, and folds results through `driver.decide`. | | `runPersonified` | function | Compose the persona + chosen shape onto a fresh keystone `Supervisor`. Resolves the shape | | `runStrategyEvolution` | function | Multi-generation strategy search: author candidates from tournament losses, play them against the incumbent at equal budget, promote via `promotionGate` on an untouched holdout slice. | +| `sandboxCheckRunner` | function | Default CheckRunner backend: pipes the check program into `python3` over the sandbox | | `sandboxClientAsProvider` | function | Adapt a `SandboxClient` into the shared `AgentEnvironmentProvider` contract. | | `sandboxSessionTraceSource` | function | The SANDBOX / fleet trace source: read a box session's message parts and decode the harness's tool | +| `selectBestIndex` | function | Argmax by `compareCheckOutcomes`, FIRST index wins ties (deterministic; with zero | | `selectChampion` | function | Search-side champion selection over a tournament report. | | `selectValidWinner` | function | The single content-free valid-only winner selector. Among the gated-VALID children only | | `sentinelCompletion` | function | Completion for a sandbox-agent node: done iff the latest output carries the node's stop | @@ -385,6 +468,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `spendFromUsageEvents` | function | Fold a normalized `UsageEvent` array into a `Spend`. Tokens and usd are separate | | `stopSentinel` | function | A unique, attributable stop sentinel for a node (ralph-loop style). Deterministic from the | | `streamAgentTurn` | function | Run ONE agent turn on any backend kind and stream its events. Yields the | +| `structuralRollout` | function | Build the structuralRollout `Strategy`: k shots → score each by the frozen visible | | `sumSandboxUsage` | function | Sum the token usage + USD cost of a sandbox turn's events — the one honest way to meter an | | `supervise` | function | One-call supervisor: build + run a supervisor from its profile with sensible defaults; the raw `supervisorAgent` + `createSupervisor().run` seams stay available for power use. | | `superviseSurface` | function | Drive a team of agents (spawned + steered by `profile`) to solve a graded `AgenticSurface` task, and | @@ -392,6 +476,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `supervisorInstructions` | function | The supervisor SKILL — the how-to the supervisor reads (its system prompt). THE optimizable | | `trajectoryReport` | function | Reconstruct the whole spawn tree for `root` with per-node + rolled-up `Spend`. Reads the | | `verify` | function | `verify(spec)` — an IMPLEMENT child produces a candidate, then a SEPARATE VERIFIER child grades | +| `visibleCheckScore` | function | Display scalar for receipts/reports (the rigs' `visibleScore` shape): crash = -1, | | `watchTrace` | function | Subscribe to a `TraceSource` and run the streaming detectors over its live spans. Returns an | | `widen` | function | `widen(spec)` — the streaming spawn-on-completion driver. Spawns the seed lineages, then REACTS | | `workerFromBackend` | function | Build the worker seam from a backend (WHERE workers run) + an optional completion oracle (the | @@ -404,6 +489,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `defaultAuditorInstruction` | const | Default system instruction for intent-auditor agents: diagnose diverged/drifting trajectories. | | `defaultDelegateBudget` | const | The conserved pool a `delegate()` call applies when the caller does not pass its own `budget`. | | `defaultProfileRichnessThresholds` | const | Default thresholds for `ProfileRichnessThresholds` — 600 chars / 6 lines minimum system prompt. | +| `defaultStructuralRolloutPolicy` | const | The measured default recipe: 5 samples, 2 guarded repair rounds, 6 authored checks. | | `refine` | const | Built-in `Strategy`: attempt → `observe()` reads the trace → steer the next attempt → repeat (deepen one lineage). | | `sample` | const | Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N / resample). | | `sampleThenRefine` | const | The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept open), | @@ -431,7 +517,12 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `Budget` | interface | A budget envelope on a spawn or the root. All ceilings; the pool reserves against them. | | `BusEvent` | interface | Every bus event is a discriminated union member keyed by `type`. | | `BusRecord` | interface | A published event stamped for ordering and observability. `seq` is the monotonic publish index; | +| `CheckExecChannel` | interface | Minimal exec channel the default runner needs. `SandboxInstance` (and therefore | +| `CheckOutcome` | interface | How one candidate fared against the frozen visible checks, split by check kind. | | `CheckpointCapableBox` | interface | Loop-side widening of the box's optional checkpoint method. The | +| `CheckRunner` | interface | Executes the frozen checks against one candidate. Implementations MUST fail loud | +| `CheckSource` | interface | Produces the task's visible checks. MUST derive them from agent-visible information | +| `CheckSourceCtx` | interface | What a CheckSource composes with. `consult` is the strategy family's raw analyst | | `CollectedAgentTurn` | interface | A drained turn: the terminal summary plus every event the stream yielded. | | `CompletionAnalyst` | interface | Reads a node's trace → a completion verdict. Same input shape as the `analyze` hook, so | | `CompletionEvidence` | interface | Trace-derived evidence for a completion claim — an artifact (output) or a verifier metric, | @@ -483,7 +574,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `LoopLineageOptions` | interface | Opt-in box-lineage controls for `runLoop`. Default OFF — with both flags | | `LoopPlanPayload` | interface | Emitted once per `plan()` round, immediately after the driver plans. Carries | | `LoopTeardownFailedPayload` | interface | Emitted when a box's `delete()` throws or times out during teardown — the | -| `LoopTokenUsage` | interface | LLM token usage. Structurally matches agent-eval's `RunTokenUsage` / | +| `LoopTokenUsage` | interface | LLM token usage. Structurally maps into agent-eval's paid-call receipt so a | | `LoopUntilSpec` | interface | `loopUntil({ until, step })` — iterative deepening inside the conserved pool: spawn one `step` | | `LoopUntilState` | interface | The accumulated state `loopUntil` threads across rounds — the running candidate + the round | | `McpEndpoint` | interface | Where a handle's MCP server lives; headers carry per-artifact scoping. | @@ -536,6 +627,8 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `SteerContext` | interface | How a combinator's `act` consumes findings to steer — the SINGLE firewalled steer surface a | | `Strategy` | interface | A Strategy is HOW you spend the compute budget to beat the Environment's check — it | | `StrategyCtx` | interface | What a strategy body composes with: the artifact lifecycle, the budget, and the two steps. | +| `StructuralRolloutPolicy` | interface | The rollout's compute recipe — promoted from the proven rigs' env vars (K/REPAIRS/ | +| `StructuralRolloutResult` | interface | The body's deliverable — a `StrategyResult` plus selection provenance. The extra | | `SuperviseSurfaceResult` | interface | The deployable outcome of a supervised surface run. | | `Supervisor` | interface | Owns the conserved pool, the spawn log, the abort cascade, the OTP intensity breaker, | | `SupervisorProfile` | interface | The supervisor's profile — the subset of an `AgentProfile` that selects + shapes its brain. | @@ -551,12 +644,14 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `UsageSink` | interface | The slice of an agent-eval campaign `DispatchContext.cost` this needs. | | `VerifierEnvironmentOptions` | interface | createVerifierEnvironment — ANY checkable task as an `Environment`, no tool surface | | `VerifySpec` | interface | `verify({ implement, verifier })` — the 2-node sequential gate: an IMPLEMENT child produces a | +| `VisibleCheck` | interface | One task-visible executable check (e.g. a single-line Python assert). | | `WatchTraceOptions` | interface | The ONLINE analyst: watch a `TraceSource` and fold each tool span through agent-eval's published | | `WaterfallSpan` | interface | createWaterfallCollector — 100% trajectory observability from the lifecycle stream: | | `WidenGate` | interface | The progressive-widening gate (MCTS-PW). Decides whether a settled child is | | `WidenLineage` | interface | A lineage the gate may widen toward — the settled child that looked promising + the findings | | `WidenSpec` | interface | `widen({ gate })` (G5) — the STREAMING spawn-on-completion driver. Unlike the static-fanout | | `WorktreeCommandResult` | interface | Outcome of one verification command run in the worktree (test or typecheck). | +| `WorktreeProfileMaterializationReceipt` | interface | Proof of the profile inputs delivered before the worker process started. | | `AgentEnvironmentProviderRef` | type | Provider object or registry name accepted by runtime provider adapters. | | `AgentProfileRef` | type | Portable profile reference: inline profile or provider catalog id. | | `AgentTurnBackend` | type | The execution substrate one turn runs on — a closed discriminated union over | @@ -601,7 +696,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `WinnerStrategy` | type | Built-in valid-only winner strategies for `selectValidWinner` (selector≠judge): best gated-valid | | `WorktreePatchArtifact` | type | Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgenticOptions`, `AgenticRunResult`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `ArtifactHandle`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `DefinedLeaderboard`, `Driver`, `EventBus`, `EvolutionArchiveNode`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ForkRequest`, `GitWorkspaceOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `McpEnvironmentOptions`, `Observation`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ResourceRequest`, `RouterChatResult`, `RouterChatToolsResult`, `RouterToolLoopResult`, `RunAgenticOptions`, `SandboxRun`, `ShotSpec`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SurfaceScore`, `ToolSpec`, `TraceSource`, `ValidationCtx`, `Validator`, `WaterfallCollector`, `WaterfallReport`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `LoopTraceEvent`, `MakeWorkerAgent`, `WorkspaceCommit`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgenticOptions`, `AgenticRunResult`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `ArtifactHandle`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `DefinedLeaderboard`, `Driver`, `EventBus`, `EvolutionArchiveNode`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ForkRequest`, `GitWorkspaceOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `McpEnvironmentOptions`, `Observation`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ResourceRequest`, `RouterChatResult`, `RouterChatToolsResult`, `RouterToolLoopResult`, `RunAgenticOptions`, `SandboxRun`, `ShotSpec`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SurfaceScore`, `ToolSpec`, `TraceSource`, `ValidationCtx`, `Validator`, `WaterfallCollector`, `WaterfallReport`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `WorkspaceCommit`. ### Environment provider adapters — generic sandbox/compute bridge @@ -777,15 +872,100 @@ Import from `@tangle-network/agent-runtime/platform` — 20 exports. **Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AuthorizeUrlOptions`, `CatalogResult`, `ConnectionHealth`, `ConnectionHealthResult`, `ExchangeCodeResult`, `ExecInput`, `MintTokenInput`, `MintTokenResult`, `PlatformHubStatus`, `StartAuthInput`, `StartAuthResult`. +### PrimeIntellect: Verifiers v1 package and trace adapter + +Import from `@tangle-network/agent-runtime/primeintellect` — 27 exports. + +| Symbol | Kind | Summary | +|---|---|---| +| `createPrimeIntellectBackend` | function | Build the existing runtime backend against Prime's intercepted model endpoint. | +| `createPrimeIntellectPackage` | function | Build a complete PrimeIntellect Verifiers v1 package without writing to disk. | +| `importPrimeIntellectTraces` | function | Convert all Prime traces to agent-eval RunRecords while retaining one shared run config. | +| `parsePrimeIntellectTraces` | function | Parse Prime's durable `traces.jsonl` and reject malformed rows with a line number. | +| `primeIntellectTraceToRunRecord` | function | Project one complete Prime trace into the common agent-eval analysis row. | +| `readPrimeIntellectEpisodeContext` | function | Read and validate the private process contract installed by the generated Prime harness. | +| `runPrimeIntellectProgram` | function | Execute the caller's canonical runtime program inside a Prime rollout. | +| `writePrimeIntellectPackage` | function | Write a bundle through a sibling temporary directory, then rename it into place. | +| `PrimeIntellectPublicTask` | interface | The answer-free task exposed to the caller's runtime program. | +| `PrimeIntellectRunner` | interface | Files and commands that make the caller's real agent program runnable. | +| `PrimeIntellectTask` | interface | One immutable problem. References stay inside Prime's task process. | + +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `PrimeIntellectEpisodeContext`, `PrimeIntellectPackageBundle`, `PrimeIntellectPackageManifest`, `PrimeIntellectPackageOptions`, `PrimeIntellectTrace`, `PrimeIntellectTraceImportOptions`, `RunPrimeIntellectProgramOptions`, `WritePrimeIntellectPackageOptions`, `PrimeIntellectBackendOptions`, `PrimeIntellectContent`, `PrimeIntellectImportDefaults`, `PrimeIntellectJson`, `PrimeIntellectMessage`, `PrimeIntellectScoring`, `PrimeIntellectSetupCommand`, `PrimeIntellectSplit`. + +### Candidate execution — immutable prepare, run, grade, and receipt + +Import from `@tangle-network/agent-runtime/candidate-execution` — 95 exports. + +| Symbol | Kind | Summary | +|---|---|---| +| `applyExactAgentProfileDiff` | function | Apply one exact diff and reject any value that cannot be preserved canonically. | +| `buildAgentCandidateBundle` | function | Compile one measured profile/code candidate into the immutable execution | +| `candidateExecutionClaim` | function | Extract the complete durable claim from a prepared execution. | +| `captureAgentCandidateWorkspace` | function | Capture one exact regular-file workspace for immutable candidate execution. | +| `captureAgentCandidateWorkspaceFiles` | function | Capture detached files returned by a remote executor into the standard archive. | +| `createAgentCandidateWorkspacePort` | function | Create the standard bounded materializer for candidate execution ports. | +| `createProtectedAgentCandidateModelPort` | function | Bind a protected model-grant service to the immutable candidate runtime. | +| `disposePreparedAgentCandidateExecution` | function | Revoke reservations held by a prepared candidate that will not be executed. | +| `executePreparedAgentCandidate` | function | Executes and finalizes one durably claimed candidate without exposing an unproven result. | +| `parseExactAgentProfile` | function | Parse a complete profile without silently discarding unsupported fields. | +| `parseExactAgentProfileDiff` | function | Parse a profile diff without silently discarding unsupported fields. | +| `persistCandidateOutputArtifact` | function | Persist evaluator evidence, read it back, and bind the returned locator to the exact bytes. | +| `prepareAgentCandidateExecution` | function | Materializes a verified candidate into one immutable evaluator-owned execution plan. | +| `recoverExpiredAgentCandidateExecution` | function | Close an expired crashed attempt from persisted non-secret handles, then record failure. | +| `sealAgentCandidateBundle` | function | Validate and content-address a candidate bundle before it crosses an approval boundary. | +| `verifyAgentCandidateBundle` | function | Verifies every digest, resource, workspace, and Git object in a candidate bundle. | +| `CANDIDATE_TRACE_ENV` | const | Environment keys used to propagate immutable candidate trace identity. | +| `CANDIDATE_TRACE_TAGS` | const | Protected trace tags that bind a run to one prepared candidate execution. | +| `FileAgentCandidateExecutionClaimStore` | class | Cross-process lifecycle implemented as fsynced, create-if-absent records. | +| `InMemoryAgentCandidateExecutionClaimStore` | class | Single-process lifecycle implementation. | +| `AgentCandidateArtifactPort` | interface | Reads one content-addressed object from the closed S3/IPFS locator set. | +| `AgentCandidateBenchmarkGraderPort` | interface | Evaluator-owned executable grader, pinned by immutable implementation bytes. | +| `AgentCandidateCodeSurfaceSource` | interface | The only accepted path from an agent-eval code candidate to executable bytes. | +| `AgentCandidateExecutionAttemptRecord` | interface | Persisted state available to a fresh trusted recovery worker after a crash. | +| `AgentCandidateExecutionClaim` | interface | Immutable signed identity stored for one execution attempt. | +| `AgentCandidateExecutionClaimStore` | interface | Atomic one-shot store for candidate execution attempts. | +| `AgentCandidateExecutionCleanupHandles` | interface | Non-secret identities a trusted recovery worker needs to close an abandoned attempt. | +| `AgentCandidateExecutionLease` | interface | Secret capability required to finish the acquired attempt. | +| `AgentCandidateExecutionRecoveryEvidence` | interface | Trusted, independently observed closure facts for one expired winning lease. | +| `AgentCandidateExecutionUsage` | interface | Exact fixed-point usage proven by the closed evaluator model ledger. | +| `AgentCandidateExecutorFinalCapture` | interface | Idempotent executor result after process death and trace drain. | +| `AgentCandidateExecutorMemoryCapture` | interface | Raw isolated-memory capture made only after access has been revoked. | +| `AgentCandidateExecutorPort` | interface | Executes one prepared request inside an evaluator-owned isolation boundary. | +| `AgentCandidateExecutorRequest` | interface | One detached request passed to the trusted environment-specific executor. | +| `AgentCandidateExecutorStopRequest` | interface | Opaque process identity used for termination without re-exposing launch credentials. | +| `AgentCandidateExecutorTaskOutcomeCapture` | interface | Raw evaluator capture made only after the candidate process is dead. | +| `AgentCandidateModelGrantClient` | interface | Narrow transport contract for a service that owns scoped model credentials | +| `AgentCandidateOutputArtifactPort` | interface | Durable content-addressed evidence store controlled only by the evaluator. | +| `AgentCandidateProtectedModelCall` | interface | One evaluator-gateway call in the final, revoked model-access ledger. | +| `AgentCandidateRepositoryPort` | interface | Resolves a declared GitHub repository to an already-present local Git object store. | +| `AgentCandidateWorkspacePort` | interface | Materializes an already-verified workspace archive. | +| `BuildAgentCandidateBundleInput` | interface | Complete measured surfaces and execution policy compiled into one candidate bundle. | +| `VerifiedAgentCandidateTaskOutcome` | interface | Branded task outcome that has survived independent patch and tree verification. | +| `AgentCandidateBundleInput` | type | Exact candidate wire shape before the runtime computes its canonical digest. | +| `AgentCandidateCodeSource` | type | Explicit control/no-op code or one finalized CodeSurface whose bytes must still verify. | +| `AgentCandidateExecutionClaimResult` | type | Result of atomically claiming one execution attempt. | +| `AgentCandidateExecutionFailureClass` | type | Only the first class is retryable, and only when the closed model ledger has zero calls. | +| `AgentCandidateExecutionFinishResult` | type | Result of atomically recording an attempt's terminal facts. | +| `AgentCandidateExecutionPhase` | type | Monotonic durable phase: the second value means candidate code could have started. | +| `AgentCandidateExecutionPhaseResult` | type | Result of crossing the irreversible candidate-may-run boundary. | +| `AgentCandidateExecutionStageResult` | type | Result of durably staging the one immutable terminal outbox entry. | +| `AgentCandidateExecutionTerminalRecord` | type | Durable terminal record for one acquired execution attempt. | +| `AgentCandidateExecutionTerminalResult` | type | Evaluator-owned terminal facts staged durably before the terminal CAS. | +| `AgentCandidateModelGrantReservation` | type | Secret-free response from the service's reservation endpoint. | +| `AgentCandidateModelLimits` | type | Limits mechanically enforced by the evaluator-owned model gateway. | +| `AgentCandidateProfileSource` | type | A complete profile that can be frozen without losing behavior. | + +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateBenchmarkGraderIdentity`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorProfileFile`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateTaskExecution`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `CreateAgentCandidateWorkspacePortOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `DisposePreparedAgentCandidateOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResolvedAgentCandidateContainer`, `VerifiedAgentCandidate`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`. + ### MCP servers — delegate / coordination / detached-session -Import from `@tangle-network/agent-runtime/mcp` — 170 exports. +Import from `@tangle-network/agent-runtime/mcp` — 176 exports. | Symbol | Kind | Summary | |---|---|---| | `buildDelegationTraceSpans` | function | Derive the compact span tree for ONE loop run from its buffered | | `capDelegationTrace` | function | Enforce the trace caps over an ordered (oldest-first) span list. Drops the | -| `captureWorktreeDiff` | function | Stage all changes in a worktree and return the diff patch + shortstat against the base ref. | +| `captureWorktreeDiff` | function | Stage worker changes and return the diff + shortstat, excluding declared input paths. | | `coderTaskFromArgs` | function | Canonical `DelegateCodeArgs` → `CoderTask` mapping — the single source for | | `composeLoopTraceEmitters` | function | Fan one `LoopTraceEvent` stream into several emitters — e.g. the | | `createCoordinationTools` | function | Build the driver's MCP tools over a live scope. | @@ -814,9 +994,10 @@ Import from `@tangle-network/agent-runtime/mcp` — 170 exports. | `makeCheckRunner` | function | Build a `run_analyst` runner over a kind directory. | | `mcpToolsForRuntimeMcp` | function | Returns the queue-bound delegation tools projected into OpenAI Chat | | `mcpToolsForRuntimeMcpSubset` | function | Subset filter — return only the projected tools whose `function.name` | +| `parseCodexTokenUsage` | function | Parse and validate the one terminal usage event emitted by `codex exec --json`. | | `parseDetachedSessionRef` | function | Parse a `detachedSessionRef` string back to parts; throws `ValidationError` on malformed input. | | `readTraceContextFromEnv` | function | Read trace context from the process environment. | -| `removeWorktree` | function | Remove a git worktree and delete its branch; tolerates already-removed paths. | +| `removeWorktree` | function | Remove a git worktree and delete its branch. Already-removed paths are harmless; every other | | `renderTrace` | function | Render a worker's trace (tool calls + results) into the text an analyst lens reads. Generic over | | `runCheck` | function | Run ONE lens over a trace → findings. Generic over any kind: prompt = the lens + the agent-eval | | `runDetachedTurn` | function | Dispatch one detached turn and advance it to a terminal state with | @@ -846,6 +1027,7 @@ Import from `@tangle-network/agent-runtime/mcp` — 170 exports. | `DELEGATION_STATUS_TOOL_NAME` | const | MCP tool name for the `delegation_status` synchronous-poll tool. | | `DELEGATION_TRACE_MAX_BYTES` | const | Default cap on the serialized trace payload per record, in bytes. | | `DELEGATION_TRACE_MAX_SPANS` | const | Default cap on spans retained per delegation record. | +| `CodexExecutionDiagnosticError` | class | Thrown when reproducible Codex exits without one valid terminal usage event. | | `DelegationPersistenceError` | class | A delegation-store read or write failed (filesystem error, store | | `DelegationStateCorruptError` | class | The persisted delegation state exists but cannot be parsed into | | `DelegationTaskQueue` | class | In-process queue for async delegation tasks — submit, cancel, poll status, and read history. | @@ -853,6 +1035,10 @@ Import from `@tangle-network/agent-runtime/mcp` — 170 exports. | `InMemoryDelegationStore` | class | In-memory `DelegationStore` — suitable for single-process use and tests. | | `InMemoryFeedbackStore` | class | In-memory `FeedbackStore` — suitable for single-process use and tests. | | `Check` | interface | One lens — a composable analyst kind. Identity fields mirror `TraceAnalystKindSpec` so a kind is | +| `CodexExecutionEvidence` | interface | Zero-model-call evidence for the exact Codex process about to run. | +| `CodexExecutionFailureDiagnostic` | interface | Bounded, credential-redacted process context attached when reproducible Codex output fails | +| `CodexExecutionPolicy` | interface | Isolation settings asserted before a reproducible Codex run is allowed to start. | +| `CodexTokenUsage` | interface | Exact aggregate usage emitted by Codex's terminal `turn.completed` JSONL event. | | `CoordinationTools` | interface | The supervisor-side toolbox returned by {@link createCoordinationTools}: the MCP tool | | `DelegateArgs` | interface | Parsed `delegate` tool arguments. | | `DelegateCodeConfig` | interface | Minimal `CoderTask` overrides exposed over the MCP wire. The full | @@ -886,7 +1072,7 @@ The scoring/measurement/judge substrate. **Do NOT re-implement a judge, an authe ### JUDGE — LLM-as-judge, panels, calibration -Import from `@tangle-network/agent-eval` — 30 exports. +Import from `@tangle-network/agent-eval` — 32 exports. | Symbol | Kind | Summary | |---|---|---| @@ -899,6 +1085,7 @@ Import from `@tangle-network/agent-eval` — 30 exports. | `createCustomJudge` | function | Create a custom judge with a fully custom prompt. | | `createDomainExpertJudge` | function | Create a domain expert judge with a configurable domain. | | `createIntentMatchJudge` | function | Factory: pin LLM options once, return a closure. | +| `createReferenceEquivalenceJudge` | function | Build the campaign-native expected-answer judge. | | `createSemanticConceptJudge` | function | Factory: pin LLM options once, return a closure that accepts inputs. | | `ensembleJudge` | function | Build a campaign-shaped `JudgeConfig` whose `score()` runs every panel | | `judgeFamily` | function | Classify a model id into its provider family. Strips a `@snapshot` suffix | @@ -909,6 +1096,7 @@ Import from `@tangle-network/agent-eval` — 30 exports. | `replayTraceThroughJudge` | function | Apply a judge function to every LLM span in a run and record the | | `runIntentMatchJudge` | function | Run the intent-match judge. Soft-fails to available=false on error. | | `runKeywordCoverageJudge` | function | Score expected concepts against an already-fetched HTML payload + any | +| `runReferenceEquivalenceJudge` | function | Direct-call adapter over the campaign judge for product callers. | | `runSemanticConceptJudge` | function | Run the semantic concept judge. Soft-fails to available=false on | | `securityJudge` | function | Build a `SandboxJudgeSpec` that scores the harness output for security issues via a security scanner. | | `testJudge` | function | Build a `SandboxJudgeSpec` that scores the harness by its test-suite pass rate. | @@ -955,7 +1143,7 @@ Import from `@tangle-network/agent-eval` — 10 exports. ### STATISTICS — significance, intervals, effect size -Import from `@tangle-network/agent-eval` — 49 exports. +Import from `@tangle-network/agent-eval` — 51 exports. | Symbol | Kind | Summary | |---|---|---| @@ -978,6 +1166,7 @@ Import from `@tangle-network/agent-eval` — 49 exports. | `pairedEvalueSequence` | function | Run the paired e-value sequence over an in-order delta stream. | | `pairedMde` | function | Minimum detectable paired effect (standardised units) for a target paired | | `pairedRiskDifference` | function | Paired risk difference (the effect-size companion to {@link mcnemar}): the | +| `pairedSignTest` | function | Exact one-sided sign test over paired differences. | | `pairedTTest` | function | Paired t-test — before/after measurements on the SAME items. | | `partialCredit` | function | Partial credit: returns 0-1 ratio of current toward target | | `passAtK` | function | Unbiased pass@k for code generation (Chen et al. 2021, "Evaluating Large | @@ -994,25 +1183,34 @@ Import from `@tangle-network/agent-eval` — 49 exports. | `ProportionInterval` | interface | A binomial proportion estimate with a confidence interval. | | `RiskDifferenceResult` | interface | A paired binary effect size (treatment rate − control rate) with a CI. | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BootstrapOptions`, `BootstrapResult`, `CorpusAgreementOptions`, `CorpusAgreementPerDimension`, `CorpusAgreementReport`, `CorpusScoreRecord`, `EProcess`, `EProcessOptions`, `EProcessState`, `EProcessStep`, `PairedBootstrapOptions`, `PairedBootstrapResult`, `WeightedCompositeInput`, `WeightedCompositeResult`, `CliffsMagnitude`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BootstrapOptions`, `BootstrapResult`, `ClusterBootstrapInterval`, `CorpusAgreementOptions`, `CorpusAgreementPerDimension`, `CorpusAgreementReport`, `CorpusScoreRecord`, `EProcess`, `EProcessOptions`, `EProcessState`, `EProcessStep`, `PairedBootstrapOptions`, `PairedBootstrapResult`, `WeightedCompositeInput`, `WeightedCompositeResult`, `CliffsMagnitude`. ### CAMPAIGN — profile matrix, gates, improvement loop -Import from `@tangle-network/agent-eval/campaign` — 261 exports. +Import from `@tangle-network/agent-eval/campaign` — 379 exports. | Symbol | Kind | Summary | |---|---|---| | `aceProposer` | function | Append-only context engineering proposer: grows a skill playbook by appending generation-tagged lessons without merging or overwriting prior entries. | +| `acquireSingleRunLock` | function | Acquire the lock or throw naming the live holder. A stale lock (holder pid | +| `analyzeCrossSurfaceInteractions` | function | Build the complete cross-surface evidence matrix and derive all three frozen | | `applySkillPatch` | function | Apply a SkillOpt patch to a text surface. Ops apply in array order against | +| `assertCodeSurfaceIdentity` | function | Fail when a code surface does not carry a complete immutable identity. | +| `assertPolicyEditAuthorContextBudget` | function | Serialize once and fail before dispatch when author context exceeds its budget. | | `buildAnalystSurfaceDispatch` | function | Build the `dispatchWithSurface(surface, scenario, ctx)` the improvement loop | | `buildEvidenceVector` | function | The Evidence Bus. For each objective, pair candidate vs baseline by full | | `buildLoopProvenanceRecord` | function | Build the durable provenance record from a completed loop result. | | `callbackGovernor` | function | The LLM-supervisor slot: a governor whose `decide` defers to a caller-supplied | | `campaignBreakdown` | function | Per-candidate evidence a reflective/patch proposer grounds its next proposal | -| `campaignMeanComposite` | function | Mean composite across a campaign: per cell, the mean of its judges' | +| `campaignMeanComposite` | function | Mean composite across a campaign: per cell, the mean of its finite, | +| `classifyUngroundedLiterals` | function | Scan revised artifact text for single-quoted single-word literals (the | +| `codeSurfaceIdentityMaterial` | function | Canonical, location-independent identity of a finalized code candidate. | | `compareProposers` | function | Run a head-to-head lift benchmark across surface proposers on a shared holdout, returning per-proposer lift CIs and pairwise "who wins" verdicts. | | `composeGate` | function | Compose gates — all must `ship` for the composite to `ship`. First | +| `compositeProposer` | function | Fan the population budget across N proposers and merge their candidates into | | `countSentenceEdits` | function | Sentence-level edit distance — count distinct add/remove ops between | +| `createReferenceEquivalenceJudge` | function | Build the campaign-native expected-answer judge. | +| `createRunCostLedger` | function | Open the durable spend account stored beside a logical run. | | `defaultProductionGate` | function | Opinionated production gate composing held-out significance, red-team, reward-hacking, and canary checks into a single `Gate.decide` decision. | | `defaultRenderDiff` | function | Default surface diff renderer: produces a unified baseline/winner text diff for prompt surfaces or a worktree-ref summary for code surfaces. | | `detectScale` | function | Detect the native scale of a set of scores: 0-100 when any magnitude clears | @@ -1037,9 +1235,11 @@ Import from `@tangle-network/agent-eval/campaign` — 261 exports. | `heuristicGovernor` | function | The reference deterministic policy an agent {@link Governor} can replace. | | `inMemoryCampaignStorage` | function | In-memory storage for filesystem-less runtimes. Artifacts + trace spans | | `isProposedCandidate` | function | Type guard: a proposal carrying its rationale vs a bare | +| `isTransientTransportFailure` | function | True when the error text describes an infrastructure hiccup that should be | | `labelTrustRank` | function | Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). | | `lineageNodeId` | function | Deterministic node id: a hash of the node's lineage + content + proposer. | | `llmJudge` | function | Build a campaign-shaped `JudgeConfig` whose `score()` makes ONE LLM call | +| `llmPolicyEditProposer` | function | LLM-backed PolicyEdit author. It reads only the current JSON surface, | | `loadEvalFixture` | function | Load ONE fixture by name: reads `PROMPT.md` (plus `EVAL.ts`/`EVAL.tsx` and `package.json` under | | `loadEvalFixtureScenarios` | function | Load fixtures (all discovered, or just `names`) as campaign `Scenario`s tagged `eval-fixture`. | | `loopProvenanceSpans` | function | Build the loop's OTLP-ingestable spans from a provenance record. One root | @@ -1049,6 +1249,7 @@ Import from `@tangle-network/agent-eval/campaign` — 261 exports. | `neutralizationGate` | function | Composable placebo gate: ships only when the candidate's held-out lift is NOT | | `neutralizeText` | function | Blank every non-whitespace character to a 1-byte filler while preserving all | | `openAutoPr` | function | Open a GitHub PR for a gate-approved surface promotion, attaching the manifest hash, gate verdict, and diff as the PR body. | +| `openSearchLedger` | function | Open a durable filesystem search ledger. Construction performs no I/O; the | | `pairHoldout` | function | Pair candidate vs baseline holdout observations by FULL cellId. `select` | | `parameterSweepProposer` | function | Config/parameter-level proposer for FAPO's middle escalation level. | | `paretoSignificanceGate` | function | Wrap the bus + a policy as a `Gate`. Plugs into the existing | @@ -1058,47 +1259,67 @@ Import from `@tangle-network/agent-eval/campaign` — 261 exports. | `planEvalFixtureRun` | function | Dry-run planner for a fixture campaign: loads the scenarios, delegates to `planCampaignRun`, | | `policyEditProposer` | function | `SurfaceProposer` that admission-checks typed analyst `PolicyEdit`s and applies each | | `powerPreflight` | function | Estimate the minimum detectable lift a paired-holdout improvement run can | +| `projectPolicyEditHistory` | function | Projects scored history into the only fields a policy author may consume. | | `provenanceRecordPath` | function | Canonical durable paths under the run dir. | | `provenanceSpansPath` | function | Canonical path for the durable OTLP spans JSONL file under a loop run directory. | | `renderScoreboardMarkdown` | function | Render the scoreboard as a launch-readiness Markdown document — the literal | | `resolveRunDir` | function | Resolve a campaign `runDir`. An absolute path is honored as-is (the caller | -| `resolveWorktreePath` | function | Resolve a `CodeSurface`'s worktreeRef to a directory the measurement can | +| `resolveWorktreePath` | function | Resolve a code candidate for evaluation only after verifying its immutable | +| `rolloutArgumentDiff` | function | Deterministic per-field diff of call arguments between passing and failing | | `runCampaign` | function | Core campaign orchestrator: fan scenarios through dispatch, score with judges, aggregate bootstrap CIs, and persist reproducible `CampaignResult` records. | | `runEval` | function | Simplest evaluation preset: run scenarios through dispatch, score with judges, and return a `CampaignResult` — no optimizer, no gate, no PR. | | `runImprovementLoop` | function | Gated-promotion shell over `runOptimization`: scores the winner against the baseline on a holdout set, runs the release gate, and optionally opens a PR. | | `runLineage` | function | Drive a multi-track improvement DAG under an agent-managed governor. Seeds each | | `runLineageLoop` | function | Wire the {@link runLineage} DAG's `step`/`merge` seams to a real | -| `runOptimization` | function | Improvement loop body: N generations of propose → campaign → rank, maintaining a Pareto frontier and promoting the top-scoring candidates to the next generation. | +| `runOptimization` | function | Improvement loop body: N generations of propose → campaign → rank, maintaining a Pareto frontier and one global incumbent across generations. | | `runProfileMatrix` | function | Profile × scenario matrix runner: fan N agent profiles across M scenarios, project each cell to a validated `RunRecord` with real token usage, and enforce the backend-integrity guard before returning. | | `runSkillOpt` | function | SkillOpt sequential hill-climb: each epoch reflects on train-scenario weaknesses, proposes bounded patches, accepts the first patch that strictly improves the held-out composite, and anneals the edit | | `scoreboardSummary` | function | Roll the per-requirement rows up into the launch headline counts. | | `scoreDiscrimination` | function | Rank scenarios by how well they DISCRIMINATE candidates. | | `scoreUserStory` | function | Score one story's produced state against its requirements. Thin wrapper over | | `selectDiscriminative` | function | Select the top-`k` most discriminative scenario ids for a holdout, EXCLUDING | +| `selectPolicyEditAuthorRows` | function | Select a bounded, deterministic evidence slice for a PolicyEdit author. | | `sequentialDecide` | function | `SurfaceProposer.decide` adapter — stops the optimization loop the moment | | `sequentialPairedGate` | function | Anytime-valid sequential paired gate. Conforms to the existing `Gate` | | `skillOptEntry` | function | SkillOpt patch-mode hill-climb. Runs findings-BLIND: `runSkillOpt` owns its | | `skillOptProposer` | function | SkillOpt proposer: proposes bounded, anchored patch operations (add/delete/replace) on a skill document, conforming to both the patch-native `SkillOptProposer` and the generic `SurfaceProposer` interf | -| `surfaceContentHash` | function | Stable sha256 (full hex) of a surface's effective text. Code surfaces hash | -| `surfaceHash` | function | Short (16-char) sha256 fingerprint of a `MutableSurface`: hashes text content for prompt surfaces, or the worktree + base ref pair for code surfaces. | +| `surfaceContentHash` | function | Full SHA-256 content identity for a prompt or finalized code surface. | +| `surfaceHash` | function | Short loop key derived from the same content identity as provenance. | | `tangleTracesRoot` | function | The shared, out-of-repo root for campaign/benchmark run bundles. Keeping run | | `traceAnalystProposer` | function | Wrap agent-eval's trace-analyst registry as a SurfaceProposer (prompt-tier). | | `userStoryScoreboard` | function | Flatten story verdicts into the per-requirement scoreboard — the literal | +| `validatePolicyEditCandidateRecord` | function | _(no summary — add a TSDoc line at the declaration)_ | +| `validateSearchLedgerEvent` | function | Validate and return a canonical copy. Arrays whose order is not semantic are | +| `verifyCodeSurface` | function | Verify a finalized code surface against its current checkout. This rejects | +| `DEFAULT_POLICY_EDIT_HISTORY_LIMITS` | const | _(no summary — add a TSDoc line at the declaration)_ | | `paretoPolicy` | const | The default strategy: symmetric multi-objective Pareto significance. Ship iff | +| `POLICY_EDIT_CANDIDATE_RECORD_SCHEMA` | const | _(no summary — add a TSDoc line at the declaration)_ | +| `SEARCH_LEDGER_SCHEMA` | const | Durable append-only audit log for improvement searches. | +| `FileSearchLedger` | class | _(no summary — add a TSDoc line at the declaration)_ | | `FsLabeledScenarioStore` | class | Filesystem `LabeledScenarioStore`: appends one JSONL file per source with provenance and | | `LabeledScenarioStoreError` | class | Typed rejection from a labeled-scenario store (bad provenance, rate limit, invalid sample args) — carries a stable string `code`. | | `Lineage` | class | Deterministic improvement-candidate graph with mutation, merge, frontier, and persistence helpers. | | `ProfileMatrixError` | class | Thrown when the matrix is misconfigured (no profiles, a profile whose model | +| `SearchLedgerConflictError` | class | _(no summary — add a TSDoc line at the declaration)_ | +| `SearchLedgerError` | class | _(no summary — add a TSDoc line at the declaration)_ | +| `SearchLedgerIntegrityError` | class | _(no summary — add a TSDoc line at the declaration)_ | | `SkillPatchParseError` | class | Parse + validate the patch response. Throws `SkillPatchParseError` when the | | `WorktreeAdapterError` | class | Typed failure from a `WorktreeAdapter` operation (create/finalize/discard) — wraps the underlying git error as `cause`. | | `AceProposerOptions` | interface | `aceProposer` — Agentic Context Engineering: an APPEND-MOSTLY curator, the | | `AnalystArtifact` | interface | The analyst's output for one scenario — the artifact the judge scores. | | `AnalystScenario` | interface | A labeled trace scenario: a FIXED trace corpus plus the failure modes a | | `CampaignArtifactWriter` | interface | Scoped artifact writer — `write(path, content)` lands under | -| `CampaignCostMeter` | interface | Cell-scoped cost meter. NOTHING is captured automatically — | +| `CampaignCostMeter` | interface | Cell-scoped paid-call entry point. The dispatch places every paid operation | | `CampaignStorage` | interface | `CampaignStorage` — the filesystem seam `runCampaign` writes through | | `CampaignTraceWriter` | interface | Scoped trace writer handed to each dispatch — every span | -| `CodeSurface` | interface | A tier-4 code surface — a candidate change to the agent's | +| `CodeSurface` | interface | A tier-4 code surface — a finalized candidate change to the agent's | +| `CompositeProposerOptions` | interface | `compositeProposer` — run N proposers TOGETHER on the same surface. | +| `CrossSurfaceCandidate` | interface | Immutable identity for a single candidate or a materialized composition. | +| `CrossSurfaceComponent` | interface | One independently proposed change on one caller-defined surface. | +| `CrossSurfaceComponentEvidence` | interface | Per-component trace evidence captured during one task attempt. | +| `CrossSurfaceInteractionPath` | interface | One deterministic growth path starting from a compatible two-surface seed. | +| `CrossSurfaceSelectionPolicy` | interface | Predeclared candidate eligibility and composition policy. | +| `CrossSurfaceTaskRow` | interface | Canonical per-task input row. Consumers may extend this interface with | | `DefaultProductionGateOptions` | interface | `defaultProductionGate` — composes the substrate's existing safety | | `DispatchContext` | interface | Context handed to every dispatch invocation. Scoped — every | | `EvolutionaryProposerOptions` | interface | `evolutionaryProposer` — adapts a stateless `Mutator` (population mutation: | @@ -1112,6 +1333,7 @@ Import from `@tangle-network/agent-eval/campaign` — 261 exports. | `JudgeScore` | interface | The canonical judge verdict shape — one declaration, shared by campaign | | `LabeledScenarioWrite` | interface | Required-provenance write. The store rejects writes that | | `LineageNode` | interface | Lineage DAG — a git-graph of improvement candidates. | +| `LoopProvenanceCandidate` | interface | Loop provenance — the durable, queryable record of WHAT a self-improvement | | `LoopProvenanceRecord` | interface | The durable provenance record. Aligns to the hosted `EvalRunEvent` path but | | `MemoryCurationProposerOptions` | interface | `memoryCurationProposer` — a CURATOR `SurfaceProposer`, the complement to the | | `Mutator` | interface | Stateless surface mutation — given findings + current | @@ -1122,12 +1344,16 @@ Import from `@tangle-network/agent-eval/campaign` — 261 exports. | `PlaybackContext` | interface | Dispatch context plus the profile under test (which cheap model, etc.). | | `PlaybackDriver` | interface | Drives the real product through a story and returns the runtime event stream | | `PlaybackStep` | interface | One step of a user story — what the user does. The driver interprets | +| `PolicyEditAuthorScenarioRow` | interface | One measured scenario row eligible for PolicyEdit author context. | +| `PolicyEditCandidateRecord` | interface | JSON-safe attribution carried with a measured candidate and its scores. | +| `PolicyEditFindingInput` | interface | Trace-derived findings must name the measured profile that produced them. | | `PolicyEditProposerOptions` | interface | `policyEditProposer` turns typed analyst policy edits into measured candidate | | `PowerPreflightOptions` | interface | Power preflight — "can this budget detect the effect you are hunting?" | | `ProposeContext` | interface | Everything a proposer may read to plan the next | | `ProposedCandidate` | interface | A proposer output carrying the surface AND the WHY behind | | `ProposerEntry` | interface | What an optimizer produced: the surface it promoted + what it cost to get | | `RejectedEdit` | interface | A patch that was tried and not accepted — fed back to the model so it does | +| `RolloutCall` | interface | One tool/action call observed in a rollout: a name plus its arguments. | | `RunCampaignOptions` | interface | `runCampaign` — Pass A substrate primitive. ONE function that orchestrates | | `RunEvalOptions` | interface | `runEval` — the simplest preset over `runCampaign`. No optimizer, no | | `RunLineageLoopSeed` | interface | A seed track: the initial surface + track identity. Unlike | @@ -1136,17 +1362,24 @@ Import from `@tangle-network/agent-eval/campaign` — 261 exports. | `ScenarioSignal` | interface | Per-scenario observation: the composite scores each candidate earned on it. | | `ScoreboardRow` | interface | One row of the launch scoreboard — story × requirement → PASS/FAIL. | | `ScoreboardSummary` | interface | Launch-readiness headline counts rolled up from the per-requirement rows. | +| `ScoredRollout` | interface | A scored rollout: its calls plus the scalar outcome used to split pass/fail. | +| `ScoredSurfaceOutcome` | interface | Exact measured state for the surface an optimizer is learning from. | +| `SearchArtifactRef` | interface | Content-addressed artifact or receipt. Mutable paths are locators only; the | +| `SearchSourceRef` | interface | Repository, dataset, or package source pinned to an immutable commit or | +| `SearchSurfaceEvidence` | interface | Per-attempt proof that a declared candidate surface was or was not active, | | `SessionScript` | interface | One session within a multi-session journey. Dispatch is | +| `SingleRunLockOptions` | interface | Single-run lock for evaluations that share one mutable environment. | | `SkillOptEvidence` | interface | Evidence the optimizer reflects on: where the current surface is weakest. | | `SkillPatch` | interface | A named, attributable bundle of ops the optimizer proposes as one edit. | | `SurfaceProposer` | interface | A surface-improvement strategy. Given the current best | | `SurfaceScore` | interface | The measured fitness of one surface — the value recorded on a DAG node. | | `TraceAnalystProposerOptions` | interface | `traceAnalystProposer` — wraps agent-eval's OWN trace-analyst engine | +| `TransientFailureOptions` | interface | Transient-transport-failure classification for dispatch retry policies. | | `UserStory` | interface | A user story = a runnable product journey plus the requirements that define | | `UserStoryVerdict` | interface | A scored user story — the completion verdict plus its human title. | -| `Worktree` | interface | VCS-pluggable worktree adapter. One improvement = one worktree, PR-like | | `AxisVerdict` | type | Per-axis verdict from the good-direction paired bootstrap. | | `CampaignTokenUsage` | type | Token usage accumulated for a cell. Aliased to the canonical `RunTokenUsage` | +| `CrossSurfaceAttemptCompleteness` | type | Whether one candidate attempt produced a usable executable outcome. | | `DispatchFn` | type | One function: scenario + ctx → artifact. Dispatcher chooses | | `FapoOptimizationLevel` | type | FAPO (Fully Autonomous Prompt Optimization) is an orchestration policy, not | | `GateDecision` | type | Five-valued verdict taxonomy (MOSS-paper alignment). | @@ -1163,7 +1396,7 @@ Import from `@tangle-network/agent-eval/campaign` — 261 exports. | `SequentialDecision` | type | Anytime-valid sequential promotion gate — an e-process (betting | | `SkillPatchOp` | type | A single bounded edit against a skill surface. | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcceptedEdit`, `ApplySkillPatchResult`, `AxisEvidence`, `BuildAnalystSurfaceDispatchOptions`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CompareProposersOptions`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `FailureModeRecallJudgeOptions`, `FapoAttributionSignals`, `FapoFailureCluster`, `FapoProposerOptions`, `FapoReviewInput`, `FapoReviewIssue`, `FapoReviewResult`, `FapoScopeContract`, `GateContext`, `GateResult`, `GenerationRecord`, `GepaProposerOptions`, `GitWorktreeAdapterOptions`, `Governor`, `GovernorContext`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `HeuristicGovernorOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LineageEdge`, `LineageGraph`, `LineageStore`, `LlmJudgeOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceBackend`, `LoopProvenanceCandidate`, `NeutralizationGateOptions`, `OpenAutoPrResult`, `OptimizerConfig`, `ParameterCandidate`, `ParameterChange`, `ParameterSweepProposerOptions`, `ParetoSignificanceGateOptions`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PowerPreflight`, `ProfileSummary`, `PromotionObjective`, `ProposePatchesArgs`, `ProposerComparison`, `ProposerPairwise`, `ProposerScore`, `RunImprovementLoopResult`, `RunLineageLoopOptions`, `RunLineageLoopResult`, `RunLineageOptions`, `RunLineageResult`, `RunLineageSeed`, `RunLineageStepResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `RunSkillOptResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SkillOptEpochRecord`, `SkillOptProposer`, `SkillOptProposerOptions`, `SkillPatchRejection`, `TraceSpan`, `WorktreeAdapter`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `GovernorOp`, `JsonPrimitive`, `JsonValue`, `RedactionStatus`, `RunOptimizationOptions`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcceptedEdit`, `AnalyzeCrossSurfaceInteractionsInput`, `ApplySkillPatchResult`, `AxisEvidence`, `BuildAnalystSurfaceDispatchOptions`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CodeSurfaceVerification`, `CompareProposersOptions`, `CrossSurfaceAdditionDecision`, `CrossSurfaceBestSingleSelection`, `CrossSurfaceBootstrapPolicy`, `CrossSurfaceCandidateComparison`, `CrossSurfaceCandidateEvidence`, `CrossSurfaceCandidateOutcome`, `CrossSurfaceCandidateSummary`, `CrossSurfaceCompositionStep`, `CrossSurfaceDistribution`, `CrossSurfaceEligibility`, `CrossSurfaceEvidenceBreakdown`, `CrossSurfaceInteractionAwareSelection`, `CrossSurfaceInteractionEffect`, `CrossSurfaceInteractionReport`, `CrossSurfaceInteractionTask`, `CrossSurfaceNaiveStackSelection`, `CrossSurfacePairCompatibility`, `CrossSurfacePairEvidence`, `CrossSurfacePairwiseEntry`, `CrossSurfaceRankedSingle`, `CrossSurfaceRelativeCost`, `CrossSurfaceSelections`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `FailureModeRecallJudgeOptions`, `FapoAttributionSignals`, `FapoFailureCluster`, `FapoProposerOptions`, `FapoReviewInput`, `FapoReviewIssue`, `FapoReviewResult`, `FapoScopeContract`, `GateContext`, `GateResult`, `GenerationRecord`, `GepaProposerOptions`, `GitWorktreeAdapterOptions`, `Governor`, `GovernorContext`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `HeuristicGovernorOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LineageEdge`, `LineageGraph`, `LineageStore`, `LlmJudgeOptions`, `LlmPolicyEditProposerOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceBackend`, `NeutralizationGateOptions`, `OpenAutoPrResult`, `OpenSearchLedgerOptions`, `OptimizerConfig`, `ParameterCandidate`, `ParameterChange`, `ParameterSweepProposerOptions`, `ParetoSignificanceGateOptions`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PolicyEditCandidateSummary`, `PolicyEditHistoryCandidateContext`, `PolicyEditHistoryGenerationContext`, `PolicyEditHistoryProjectionOptions`, `PolicyEditObjective`, `PolicyEditOutcomeContext`, `PowerPreflight`, `ProfileSummary`, `PromotionObjective`, `ProposePatchesArgs`, `ProposerComparison`, `ProposerPairwise`, `ProposerScore`, `ReferenceEquivalenceJudgeOptions`, `ReferenceEquivalenceScenario`, `RolloutArgumentDiff`, `RolloutArgumentDiffOptions`, `RunImprovementLoopResult`, `RunLineageLoopOptions`, `RunLineageLoopResult`, `RunLineageOptions`, `RunLineageResult`, `RunLineageSeed`, `RunLineageStepResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `RunSkillOptResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SearchAttemptAccounting`, `SearchCandidateDecidedEvent`, `SearchCandidateLineage`, `SearchCandidateRegisteredEvent`, `SearchCandidateSlot`, `SearchCandidateSlotClosedEvent`, `SearchCandidateSurface`, `SearchCompletedEvent`, `SearchFailureReason`, `SearchLedger`, `SearchLedgerAppendResult`, `SearchLedgerEntry`, `SearchLedgerReplay`, `SearchModelIdentity`, `SearchOperationRecordedEvent`, `SearchPlan`, `SearchPlannedEvent`, `SearchPlannedOperation`, `SearchPlannedTask`, `SearchTaskAttemptedEvent`, `SelectPolicyEditAuthorRowsOptions`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SerializedJsonBudget`, `SingleRunLock`, `SkillOptEpochRecord`, `SkillOptProposer`, `SkillOptProposerOptions`, `SkillPatchRejection`, `TraceSpan`, `UngroundedLiteralReport`, `Worktree`, `WorktreeAdapter`, `CrossSurfaceAdditionRejectionReason`, `CrossSurfaceIneligibilityReason`, `CrossSurfacePairIncompatibilityReason`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `GovernorOp`, `JsonPolicyEditTargetSurface`, `JsonPrimitive`, `JsonValue`, `PolicyEditFindingSource`, `RedactionStatus`, `RunOptimizationOptions`, `SearchAccountingAudit`, `SearchCostAccounting`, `SearchLedgerEvent`, `SearchLedgerHash`, `SearchOperationKind`, `SearchSurfaceEffect`, `SearchSurfaceKind`, `SearchTaskOutcome`, `SearchTokenAccounting`. ### TOKEN / USAGE — usage extraction + run-record usage types diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 7f25ff74..6499dca5 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -448,7 +448,7 @@ Defined in: [mcp/tools/coordination.ts:65](https://github.com/tangle-network/age ### WorktreeCommandResult -Defined in: [mcp/worktree-harness.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L40) +Defined in: [mcp/worktree-harness.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L48) Outcome of one verification command run in the worktree (test or typecheck). @@ -458,7 +458,7 @@ Outcome of one verification command run in the worktree (test or typecheck). > **command**: `string` -Defined in: [mcp/worktree-harness.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L42) +Defined in: [mcp/worktree-harness.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L50) The shell command line that was run. @@ -466,7 +466,7 @@ The shell command line that was run. > **passed**: `boolean` -Defined in: [mcp/worktree-harness.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L44) +Defined in: [mcp/worktree-harness.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L52) Did the command exit 0? The PASS signal a deliverable gate / coder output reads. @@ -474,7 +474,7 @@ Did the command exit 0? The PASS signal a deliverable gate / coder output reads. > **exitCode**: `number` \| `null` -Defined in: [mcp/worktree-harness.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L46) +Defined in: [mcp/worktree-harness.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L54) OS exit code, or `null` when killed before exit. @@ -482,12 +482,82 @@ OS exit code, or `null` when killed before exit. > **output**: `string` -Defined in: [mcp/worktree-harness.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L48) +Defined in: [mcp/worktree-harness.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L56) Combined stdout+stderr (capped) — surfaced in traces for diagnosis. *** +### WorktreeProfileMaterializationReceipt + +Defined in: [mcp/worktree-harness.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L60) + +Proof of the profile inputs delivered before the worker process started. + +#### Properties + +##### workspacePlanDigest + +> **workspacePlanDigest**: `string` + +Defined in: [mcp/worktree-harness.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L62) + +Digest of the exact materializer plan: files, modes, environment, flags, and unsupported rows. + +##### writtenPaths + +> **writtenPaths**: `string`[] + +Defined in: [mcp/worktree-harness.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L64) + +Repository-relative profile input files written into the worker worktree. + +##### unsupported + +> **unsupported**: `Unsupported`[] + +Defined in: [mcp/worktree-harness.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L66) + +Must be empty on a successful run because this path fails closed. + +##### environmentNames + +> **environmentNames**: `string`[] + +Defined in: [mcp/worktree-harness.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L68) + +Environment variable names added to the worker process. Values remain out of telemetry. + +##### flags + +> **flags**: `string`[] + +Defined in: [mcp/worktree-harness.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L70) + +Exact additional CLI arguments emitted by the materializer. + +##### resourceInstructions + +> **resourceInstructions**: `object` + +Defined in: [mcp/worktree-harness.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/worktree-harness.ts#L72) + +`resources.instructions` bypasses native project files so reproducible Codex cannot drop it. + +###### delivery + +> **delivery**: `"none"` \| `"invocation-prompt"` + +###### sha256 + +> **sha256**: `string` \| `null` + +###### byteLength + +> **byteLength**: `number` + +*** + ### AnytimeTaskCurve Defined in: [runtime/anytime.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/anytime.ts#L25) @@ -1310,7 +1380,7 @@ Minimum confidence a PROBABILISTIC verdict must clear to end. Default 0.8. ### LeaderboardScore -Defined in: [runtime/define-leaderboard.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L61) +Defined in: [runtime/define-leaderboard.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L62) Structured per-case verdict a `score` function may return (a bare number is shorthand for `{ composite }`). `composite` is the [0,1] leaderboard score; @@ -1322,25 +1392,25 @@ Structured per-case verdict a `score` function may return (a bare number is > **composite**: `number` -Defined in: [runtime/define-leaderboard.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L62) +Defined in: [runtime/define-leaderboard.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L63) ##### dimensions? > `optional` **dimensions?**: `Record`\<`string`, `number`\> -Defined in: [runtime/define-leaderboard.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L63) +Defined in: [runtime/define-leaderboard.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L64) ##### notes? > `optional` **notes?**: `string` -Defined in: [runtime/define-leaderboard.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L64) +Defined in: [runtime/define-leaderboard.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L65) *** ### LeaderboardScenario -Defined in: [runtime/define-leaderboard.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L69) +Defined in: [runtime/define-leaderboard.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L70) The campaign scenario a case is wrapped into: the case rides along so judges and hooks can reach the full domain payload, not just its id. @@ -1361,13 +1431,13 @@ The campaign scenario a case is wrapped into: the case rides along so > **case**: `TCase` -Defined in: [runtime/define-leaderboard.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L70) +Defined in: [runtime/define-leaderboard.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L71) *** ### LeaderboardFlagSpec -Defined in: [runtime/define-leaderboard.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L75) +Defined in: [runtime/define-leaderboard.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L76) One extra CLI flag a spec declares. Parsed by `run()` as `-- ` and surfaced to every hook via `ctx.args`. @@ -1378,19 +1448,19 @@ One extra CLI flag a spec declares. Parsed by `run()` as `-- ` > `optional` **default?**: `string` -Defined in: [runtime/define-leaderboard.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L76) +Defined in: [runtime/define-leaderboard.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L77) ##### description > **description**: `string` -Defined in: [runtime/define-leaderboard.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L77) +Defined in: [runtime/define-leaderboard.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L78) *** ### LeaderboardRunContext -Defined in: [runtime/define-leaderboard.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L81) +Defined in: [runtime/define-leaderboard.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L82) Resolved run configuration handed to `setup` / `teardown` / `export`. @@ -1400,13 +1470,13 @@ Resolved run configuration handed to `setup` / `teardown` / `export`. > **name**: `string` -Defined in: [runtime/define-leaderboard.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L82) +Defined in: [runtime/define-leaderboard.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L83) ##### backend > **backend**: `string` -Defined in: [runtime/define-leaderboard.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L84) +Defined in: [runtime/define-leaderboard.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L85) Execution backend name (`--backend`), a key of `backends`. @@ -1414,19 +1484,19 @@ Execution backend name (`--backend`), a key of `backends`. > **runDir**: `string` -Defined in: [runtime/define-leaderboard.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L85) +Defined in: [runtime/define-leaderboard.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L86) ##### exportDir > **exportDir**: `string` -Defined in: [runtime/define-leaderboard.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L86) +Defined in: [runtime/define-leaderboard.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L87) ##### args > **args**: `Record`\<`string`, `string` \| `undefined`\> -Defined in: [runtime/define-leaderboard.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L88) +Defined in: [runtime/define-leaderboard.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L89) Every parsed flag (standard + `spec.flags`), by name without `--`. @@ -1434,13 +1504,13 @@ Every parsed flag (standard + `spec.flags`), by name without `--`. > **harnesses**: readonly `HarnessType`[] -Defined in: [runtime/define-leaderboard.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L89) +Defined in: [runtime/define-leaderboard.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L90) ##### models > **models**: readonly `string`[] -Defined in: [runtime/define-leaderboard.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L91) +Defined in: [runtime/define-leaderboard.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L92) Snapshot-stamped model ids (`name@snapshot`) — the eval identity models. @@ -1448,25 +1518,25 @@ Snapshot-stamped model ids (`name@snapshot`) — the eval identity models. > **caseIds**: readonly `string`[] -Defined in: [runtime/define-leaderboard.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L92) +Defined in: [runtime/define-leaderboard.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L93) ##### shots > **shots**: `number` -Defined in: [runtime/define-leaderboard.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L93) +Defined in: [runtime/define-leaderboard.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L94) ##### reps > **reps**: `number` -Defined in: [runtime/define-leaderboard.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L94) +Defined in: [runtime/define-leaderboard.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L95) *** ### LeaderboardBenchTask -Defined in: [runtime/define-leaderboard.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L99) +Defined in: [runtime/define-leaderboard.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L100) Structurally `BenchTask` (bench registry shape) — declared locally so this module adds no dependency on a benchmark package. @@ -1477,31 +1547,31 @@ Structurally `BenchTask` (bench registry shape) — declared locally so this > **id**: `string` -Defined in: [runtime/define-leaderboard.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L100) +Defined in: [runtime/define-leaderboard.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L101) ##### prompt > **prompt**: `string` -Defined in: [runtime/define-leaderboard.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L101) +Defined in: [runtime/define-leaderboard.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L102) ##### split? > `optional` **split?**: `string` -Defined in: [runtime/define-leaderboard.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L102) +Defined in: [runtime/define-leaderboard.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L103) ##### metadata? > `optional` **metadata?**: `Record`\<`string`, `unknown`\> -Defined in: [runtime/define-leaderboard.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L103) +Defined in: [runtime/define-leaderboard.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L104) *** ### LeaderboardBenchScore -Defined in: [runtime/define-leaderboard.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L107) +Defined in: [runtime/define-leaderboard.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L108) Structurally `BenchScore` (bench registry shape). @@ -1511,25 +1581,25 @@ Structurally `BenchScore` (bench registry shape). > **resolved**: `boolean` -Defined in: [runtime/define-leaderboard.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L108) +Defined in: [runtime/define-leaderboard.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L109) ##### score > **score**: `number` -Defined in: [runtime/define-leaderboard.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L109) +Defined in: [runtime/define-leaderboard.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L110) ##### detail? > `optional` **detail?**: `string` -Defined in: [runtime/define-leaderboard.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L110) +Defined in: [runtime/define-leaderboard.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L111) *** ### LeaderboardBenchmarkAdapter -Defined in: [runtime/define-leaderboard.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L117) +Defined in: [runtime/define-leaderboard.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L118) Structurally `BenchmarkAdapter` (bench registry shape): `name`, `preflight()`, `loadTasks()`, deterministic `judge()`, `goldArtifact()`. @@ -1548,7 +1618,7 @@ Structurally `BenchmarkAdapter` (bench registry shape): `name`, > `readonly` **name**: `string` -Defined in: [runtime/define-leaderboard.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L118) +Defined in: [runtime/define-leaderboard.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L119) #### Methods @@ -1556,7 +1626,7 @@ Defined in: [runtime/define-leaderboard.ts:118](https://github.com/tangle-networ > **preflight**(): `Promise`\<`void`\> -Defined in: [runtime/define-leaderboard.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L119) +Defined in: [runtime/define-leaderboard.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L120) ###### Returns @@ -1566,7 +1636,7 @@ Defined in: [runtime/define-leaderboard.ts:119](https://github.com/tangle-networ > **loadTasks**(`opts?`): `Promise`\<[`LeaderboardBenchTask`](#leaderboardbenchtask)[]\> -Defined in: [runtime/define-leaderboard.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L120) +Defined in: [runtime/define-leaderboard.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L121) ###### Parameters @@ -1592,7 +1662,7 @@ Defined in: [runtime/define-leaderboard.ts:120](https://github.com/tangle-networ > **judge**(`task`, `artifact`): `Promise`\<[`LeaderboardBenchScore`](#leaderboardbenchscore)\> -Defined in: [runtime/define-leaderboard.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L125) +Defined in: [runtime/define-leaderboard.ts:126](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L126) ###### Parameters @@ -1612,7 +1682,7 @@ Defined in: [runtime/define-leaderboard.ts:125](https://github.com/tangle-networ > **goldArtifact**(`task`): `Promise`\<`string` \| `undefined`\> -Defined in: [runtime/define-leaderboard.ts:126](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L126) +Defined in: [runtime/define-leaderboard.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L127) ###### Parameters @@ -1628,7 +1698,7 @@ Defined in: [runtime/define-leaderboard.ts:126](https://github.com/tangle-networ ### LeaderboardIterationInfo -Defined in: [runtime/define-leaderboard.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L132) +Defined in: [runtime/define-leaderboard.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L133) Per-shot outcome context passed as `onCellEvents`'s third argument — how a thrown shot (which never reaches `parseOutput`) stays visible through the @@ -1640,7 +1710,7 @@ Per-shot outcome context passed as `onCellEvents`'s third argument — how a > **index**: `number` -Defined in: [runtime/define-leaderboard.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L134) +Defined in: [runtime/define-leaderboard.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L135) 0-based shot index within the cell. @@ -1648,7 +1718,7 @@ Defined in: [runtime/define-leaderboard.ts:134](https://github.com/tangle-networ > `optional` **error?**: `string` -Defined in: [runtime/define-leaderboard.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L136) +Defined in: [runtime/define-leaderboard.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L137) The shot's thrown error message, when the shot failed before scoring. @@ -1656,7 +1726,7 @@ The shot's thrown error message, when the shot failed before scoring. > `optional` **verdict?**: `object` -Defined in: [runtime/define-leaderboard.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L138) +Defined in: [runtime/define-leaderboard.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L139) The shot's validator verdict, when the shot reached scoring. @@ -1668,7 +1738,7 @@ The shot's validator verdict, when the shot reached scoring. ### LeaderboardSpec -Defined in: [runtime/define-leaderboard.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L147) +Defined in: [runtime/define-leaderboard.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L148) The declarative leaderboard spec. `TArtifact` is the artifact channel the dispatch produces and the judges score — `string` (the default) is the plain @@ -1691,7 +1761,7 @@ spec supplies `parseOutput` (or a LEVEL-2 `dispatch`) producing it. > **name**: `string` -Defined in: [runtime/define-leaderboard.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L149) +Defined in: [runtime/define-leaderboard.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L150) Leaderboard name — the scenario `kind`, default profile name, and report title. @@ -1699,7 +1769,7 @@ Leaderboard name — the scenario `kind`, default profile name, and report title > **cases**: `TCase`[] -Defined in: [runtime/define-leaderboard.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L151) +Defined in: [runtime/define-leaderboard.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L152) The case corpus. Every case needs a stable string id (see `caseId`). @@ -1707,7 +1777,7 @@ The case corpus. Every case needs a stable string id (see `caseId`). > `optional` **caseId?**: (`c`) => `string` -Defined in: [runtime/define-leaderboard.ts:154](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L154) +Defined in: [runtime/define-leaderboard.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L155) Stable id extractor. Default: the case's own `id` property (fail-loud when absent or not a string). @@ -1726,7 +1796,7 @@ Stable id extractor. Default: the case's own `id` property (fail-loud > **prompt**: (`c`) => `string` \| `Promise`\<`string`\> -Defined in: [runtime/define-leaderboard.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L157) +Defined in: [runtime/define-leaderboard.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L158) The per-case task prompt. May be async (e.g. built by shelling out to a reference implementation); resolved ONCE per case before dispatch. @@ -1745,7 +1815,7 @@ The per-case task prompt. May be async (e.g. built by shelling out to a > **score**: (`output`, `c`) => `number` \| [`LeaderboardScore`](#leaderboardscore) -Defined in: [runtime/define-leaderboard.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L161) +Defined in: [runtime/define-leaderboard.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L162) The domain grader: agent output artifact → score. Used BOTH as the per-shot validator (a shot with `composite > 0` stops the naive retry @@ -1769,7 +1839,7 @@ The domain grader: agent output artifact → score. Used BOTH as the > `optional` **axis?**: `object` -Defined in: [runtime/define-leaderboard.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L165) +Defined in: [runtime/define-leaderboard.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L166) Harness × model axes for `expandProfileAxes`. Defaults: the canonical `CODING_HARNESSES` × the base profile's `model.default`. `--harnesses` / @@ -1787,7 +1857,7 @@ Harness × model axes for `expandProfileAxes`. Defaults: the canonical > `optional` **baseProfile?**: `AgentProfile` -Defined in: [runtime/define-leaderboard.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L168) +Defined in: [runtime/define-leaderboard.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L169) Base profile the axes expand over (prompt/tools/skills held fixed). Default: a minimal `{ name, model: { default: } }`. @@ -1796,7 +1866,7 @@ Base profile the axes expand over (prompt/tools/skills held fixed). > `optional` **backends?**: `Record`\<`string`, (() => [`SandboxClient`](#sandboxclient-3)) \| `undefined`\> -Defined in: [runtime/define-leaderboard.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L178) +Defined in: [runtime/define-leaderboard.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L179) Execution-backend registry: `--backend ` picks the factory that yields the `SandboxClient` every cell runs on. Merged over the defaults: @@ -1810,7 +1880,7 @@ yields the `SandboxClient` every cell runs on. Merged over the defaults: > `optional` **flags?**: `Record`\<`string`, [`LeaderboardFlagSpec`](#leaderboardflagspec)\> -Defined in: [runtime/define-leaderboard.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L180) +Defined in: [runtime/define-leaderboard.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L181) Extra `--flag value` CLI args `run()` parses and surfaces via `ctx.args`. @@ -1818,7 +1888,7 @@ Extra `--flag value` CLI args `run()` parses and surfaces via `ctx.args`. > `optional` **modelBackend?**: `Record`\<`string`, `unknown`\> -Defined in: [runtime/define-leaderboard.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L184) +Defined in: [runtime/define-leaderboard.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L185) Extra fields merged into each cell's `backend.model` create override — e.g. `{ provider: 'openai-compat', apiKey, baseUrl }` for a router-backed @@ -1828,7 +1898,7 @@ Extra fields merged into each cell's `backend.model` create override — > `optional` **setup?**: (`ctx`) => `void` \| `Promise`\<`void`\> -Defined in: [runtime/define-leaderboard.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L186) +Defined in: [runtime/define-leaderboard.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L187) Runs once before the matrix (fetch fixtures, warm caches). @@ -1846,7 +1916,7 @@ Runs once before the matrix (fetch fixtures, warm caches). > `optional` **teardown?**: (`ctx`) => `void` \| `Promise`\<`void`\> -Defined in: [runtime/define-leaderboard.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L188) +Defined in: [runtime/define-leaderboard.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L189) Runs once after the matrix, even on failure (reap boxes, close handles). @@ -1864,7 +1934,7 @@ Runs once after the matrix, even on failure (reap boxes, close handles). > `optional` **onCellEvents?**: (`events`, `c`, `iteration?`) => `void` -Defined in: [runtime/define-leaderboard.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L194) +Defined in: [runtime/define-leaderboard.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L195) Per-cell event tap: the raw sandbox events of EVERY shot, with the case — the seam for domain metric capture (search counts, citations) without a @@ -1894,7 +1964,7 @@ readonly `SandboxEvent`[] > `optional` **parseOutput?**: (`events`, `c`) => `TArtifact` -Defined in: [runtime/define-leaderboard.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L204) +Defined in: [runtime/define-leaderboard.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L205) Output decode override: raw events → the scored artifact. Default: the sandbox SDK's `collectAgentResponseText` (final answer text; empty string @@ -1920,14 +1990,14 @@ readonly `SandboxEvent`[] > `optional` **resolveModel?**: (`events`) => `string` \| `undefined` -Defined in: [runtime/define-leaderboard.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L214) +Defined in: [runtime/define-leaderboard.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L215) Resolve the model the backend ACTUALLY served off a shot's raw events. Required for HARNESS_NATIVE_MODEL-snapped cells (a vendor-locked harness × an out-of-family model expands to the `default` sentinel): the RunRecord must pin a real snapshot-bearing model id, which only the dispatch — reading the backend's usage/terminal events — can know. When this returns -a value the default dispatch reports it via `ctx.cost.observeModel`; +a value the default dispatch records it on the paid-call receipt; in-family cells (concrete declared model) never need it. ###### Parameters @@ -1944,7 +2014,7 @@ readonly `SandboxEvent`[] > `optional` **export?**: (`result`, `ctx`) => `void` \| `Promise`\<`void`\> -Defined in: [runtime/define-leaderboard.ts:217](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L217) +Defined in: [runtime/define-leaderboard.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L218) Result export. Default: write `matrix-result.json` under the run dir and print (+ write) the ranked leaderboard markdown under the export dir. @@ -1967,7 +2037,7 @@ Result export. Default: write `matrix-result.json` under the run dir and > `optional` **dispatch?**: `ProfileDispatchFn`\<[`LeaderboardScenario`](#leaderboardscenario)\<`TCase`\>, `TArtifact`\> -Defined in: [runtime/define-leaderboard.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L223) +Defined in: [runtime/define-leaderboard.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L224) LEVEL 2 — full dispatch replacement (in-process products bring their own). The default is `loopDispatch` + `naiveDriver` over the resolved backend. @@ -1976,7 +2046,7 @@ LEVEL 2 — full dispatch replacement (in-process products bring their own). > `optional` **judges?**: `JudgeConfig`\<`TArtifact`, [`LeaderboardScenario`](#leaderboardscenario)\<`TCase`\>\>[] -Defined in: [runtime/define-leaderboard.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L225) +Defined in: [runtime/define-leaderboard.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L226) LEVEL 2 — full judge replacement. Default: `score` wrapped as one judge. @@ -1984,7 +2054,7 @@ LEVEL 2 — full judge replacement. Default: `score` wrapped as one judge. > `optional` **shots?**: `number` -Defined in: [runtime/define-leaderboard.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L227) +Defined in: [runtime/define-leaderboard.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L228) Naive-retry shot cap per cell (`--shots`). Default 1. @@ -1992,15 +2062,24 @@ Naive-retry shot cap per cell (`--shots`). Default 1. > `optional` **reps?**: `number` -Defined in: [runtime/define-leaderboard.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L229) +Defined in: [runtime/define-leaderboard.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L230) Replicates per cell (`--reps`). Default 1. +##### maximumCharge? + +> `optional` **maximumCharge?**: `MaximumCharge` \| ((`profile`, `scenario`) => MaximumCharge \| undefined) + +Defined in: [runtime/define-leaderboard.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L233) + +Provider- or executor-enforced maximum for one cell dispatch. Required +before execution when `matrix.costCeiling` is configured. + ##### matrix? > `optional` **matrix?**: `Partial`\<`RunProfileMatrixOptions`\<[`LeaderboardScenario`](#leaderboardscenario)\<`TCase`\>, `TArtifact`\>\> -Defined in: [runtime/define-leaderboard.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L233) +Defined in: [runtime/define-leaderboard.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L239) Passthrough overrides spread onto the final `runProfileMatrix` call (e.g. `maxConcurrency`, `costCeiling`, `integrity`, `storage`) — spread @@ -2010,7 +2089,7 @@ Passthrough overrides spread onto the final `runProfileMatrix` call ### DefinedLeaderboard -Defined in: [runtime/define-leaderboard.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L236) +Defined in: [runtime/define-leaderboard.ts:242](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L242) #### Type Parameters @@ -2028,7 +2107,7 @@ Defined in: [runtime/define-leaderboard.ts:236](https://github.com/tangle-networ > **run**(`argv?`): `Promise`\<`RunProfileMatrixResult`\<`TArtifact`, [`LeaderboardScenario`](#leaderboardscenario)\<`TCase`\>\>\> -Defined in: [runtime/define-leaderboard.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L250) +Defined in: [runtime/define-leaderboard.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L256) Parse flags, run the matrix, export, and return the raw result. @@ -2056,7 +2135,7 @@ only an explicit `--run-dir` opts into that resume behavior. > **toBenchmarkAdapter**(): [`LeaderboardBenchmarkAdapter`](#leaderboardbenchmarkadapter)\<`TArtifact`\> -Defined in: [runtime/define-leaderboard.ts:252](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L252) +Defined in: [runtime/define-leaderboard.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L258) The same domain surface in the structural `BenchmarkAdapter` shape. @@ -2320,7 +2399,7 @@ tags, so set it when a demo's output reads on a meaningful sandbox id. ### LoopDispatchOptions -Defined in: [runtime/loop-dispatch.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L50) +Defined in: [runtime/loop-dispatch.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L49) #### Type Parameters @@ -2350,7 +2429,7 @@ Defined in: [runtime/loop-dispatch.ts:50](https://github.com/tangle-network/agen > **sandboxClient**: [`SandboxClient`](#sandboxclient-3) -Defined in: [runtime/loop-dispatch.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L58) +Defined in: [runtime/loop-dispatch.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L57) Sandbox client used for every cell's `runLoop`. Supplied once. @@ -2358,7 +2437,7 @@ Sandbox client used for every cell's `runLoop`. Supplied once. > **toLoopOptions**: (`scenario`, `profile`) => [`LoopOptionsForDispatch`](#loopoptionsfordispatch)\<`Task`, `Output`, `Decision`\> -Defined in: [runtime/loop-dispatch.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L61) +Defined in: [runtime/loop-dispatch.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L60) Build the per-cell runLoop options from the scenario (+ profile, when used with `runProfileMatrix`). @@ -2381,7 +2460,7 @@ Build the per-cell runLoop options from the scenario (+ profile, when > `optional` **toArtifact?**: (`result`) => `TArtifact` -Defined in: [runtime/loop-dispatch.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L69) +Defined in: [runtime/loop-dispatch.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L68) Map the finished loop to the artifact the judges score. Default: `result.winner?.output`. A loop with no winner yields `undefined` (judges @@ -2402,7 +2481,7 @@ Map the finished loop to the artifact the judges score. Default: > `optional` **forwardTrace?**: `boolean` -Defined in: [runtime/loop-dispatch.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L72) +Defined in: [runtime/loop-dispatch.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L71) Forward `loop.*` trace events into the campaign's scoped trace so loop spans correlate with the cell. Default true. @@ -2411,15 +2490,50 @@ Forward `loop.*` trace events into the campaign's scoped trace so loop > `optional` **costSource?**: `string` -Defined in: [runtime/loop-dispatch.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L74) +Defined in: [runtime/loop-dispatch.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L73) Cost-meter source label for the loop's spend. Default `'loop'`. +##### maximumCharge? + +> `optional` **maximumCharge?**: `MaximumCharge` \| ((`scenario`, `profile`) => MaximumCharge \| undefined) + +Defined in: [runtime/loop-dispatch.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L76) + +Provider- or executor-enforced maximum for this whole cell dispatch. +Required by agent-eval before execution when the campaign is cost-capped. + +##### resolveCostModel? + +> `optional` **resolveCostModel?**: (`result`, `scenario`, `profile`) => `string` \| `undefined` + +Defined in: [runtime/loop-dispatch.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L80) + +Resolve the model actually served from the completed loop. + +###### Parameters + +###### result + +[`LoopResult`](#loopresult)\<`Task`, `Output`, `Decision`\> + +###### scenario + +`TScenario` + +###### profile + +`AgentProfile` + +###### Returns + +`string` \| `undefined` + *** ### LoopCampaignDispatchOptions -Defined in: [runtime/loop-dispatch.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L124) +Defined in: [runtime/loop-dispatch.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L186) Options for adapting plain agent-eval campaign scenarios into runtime `runLoop` cells. @@ -2451,7 +2565,7 @@ Options for adapting plain agent-eval campaign scenarios into runtime `runLoop` > **sandboxClient**: [`SandboxClient`](#sandboxclient-3) -Defined in: [runtime/loop-dispatch.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L132) +Defined in: [runtime/loop-dispatch.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L194) Sandbox client used for every campaign cell's `runLoop`. @@ -2459,7 +2573,7 @@ Sandbox client used for every campaign cell's `runLoop`. > **toLoopOptions**: (`scenario`) => [`LoopOptionsForDispatch`](#loopoptionsfordispatch)\<`Task`, `Output`, `Decision`\> -Defined in: [runtime/loop-dispatch.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L134) +Defined in: [runtime/loop-dispatch.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L196) Build the per-cell runLoop options from the campaign scenario. @@ -2477,7 +2591,7 @@ Build the per-cell runLoop options from the campaign scenario. > `optional` **toArtifact?**: (`result`) => `TArtifact` -Defined in: [runtime/loop-dispatch.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L136) +Defined in: [runtime/loop-dispatch.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L198) Map the finished loop to the artifact the campaign judges score. @@ -2495,7 +2609,7 @@ Map the finished loop to the artifact the campaign judges score. > `optional` **forwardTrace?**: `boolean` -Defined in: [runtime/loop-dispatch.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L138) +Defined in: [runtime/loop-dispatch.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L200) Forward `loop.*` trace events into the campaign's scoped trace. Default true. @@ -2503,10 +2617,40 @@ Forward `loop.*` trace events into the campaign's scoped trace. Default true. > `optional` **costSource?**: `string` -Defined in: [runtime/loop-dispatch.ts:140](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L140) +Defined in: [runtime/loop-dispatch.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L202) Cost-meter source label for the loop's spend. Default `'loop'`. +##### maximumCharge? + +> `optional` **maximumCharge?**: `MaximumCharge` \| ((`scenario`) => MaximumCharge \| undefined) + +Defined in: [runtime/loop-dispatch.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L204) + +Provider- or executor-enforced maximum for this whole cell dispatch. + +##### resolveCostModel? + +> `optional` **resolveCostModel?**: (`result`, `scenario`) => `string` \| `undefined` + +Defined in: [runtime/loop-dispatch.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L206) + +Resolve the model actually served from the completed loop. + +###### Parameters + +###### result + +[`LoopResult`](#loopresult)\<`Task`, `Output`, `Decision`\> + +###### scenario + +`TScenario` + +###### Returns + +`string` \| `undefined` + *** ### McpEndpoint @@ -2667,7 +2811,7 @@ The worker's trace — any event array (sandbox events, tool-call records). ##### outcome? -> `optional` **outcome?**: `"failed"` \| `"passed"` \| `"unknown"` +> `optional` **outcome?**: `"failed"` \| `"unknown"` \| `"passed"` Defined in: [runtime/observe.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/observe.ts#L32) @@ -5966,7 +6110,7 @@ Defined in: [runtime/sandbox-capabilities.ts:75](https://github.com/tangle-netwo ### SandboxToolPartState -Defined in: [runtime/sandbox-events.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L146) +Defined in: [runtime/sandbox-events.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L147) **`Experimental`** @@ -5983,7 +6127,7 @@ state per turn via [createSandboxToolPartState](#createsandboxtoolpartstate). > **statusByCall**: `Map`\<`string`, `string`\> -Defined in: [runtime/sandbox-events.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L149) +Defined in: [runtime/sandbox-events.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L150) **`Experimental`** @@ -5994,7 +6138,7 @@ Last seen status per tool call id. A terminal status is sticky — later > **seq**: `number` -Defined in: [runtime/sandbox-events.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L151) +Defined in: [runtime/sandbox-events.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L152) **`Experimental`** @@ -7860,7 +8004,7 @@ Default false: only facts tagged `audience:agent` are injected into the worker. ### AgenticRunResult -Defined in: [runtime/strategy.ts:594](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L594) +Defined in: [runtime/strategy.ts:608](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L608) #### Properties @@ -7868,7 +8012,7 @@ Defined in: [runtime/strategy.ts:594](https://github.com/tangle-network/agent-ru > **mode**: `string` -Defined in: [runtime/strategy.ts:596](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L596) +Defined in: [runtime/strategy.ts:610](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L610) The strategy name (built-in 'depth'/'breadth' or a custom strategy's name). @@ -7876,25 +8020,25 @@ The strategy name (built-in 'depth'/'breadth' or a custom strategy's name). > **score**: `number` -Defined in: [runtime/strategy.ts:597](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L597) +Defined in: [runtime/strategy.ts:611](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L611) ##### resolved > **resolved**: `boolean` -Defined in: [runtime/strategy.ts:598](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L598) +Defined in: [runtime/strategy.ts:612](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L612) ##### completions > **completions**: `number` -Defined in: [runtime/strategy.ts:599](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L599) +Defined in: [runtime/strategy.ts:613](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L613) ##### progression > **progression**: `number`[] -Defined in: [runtime/strategy.ts:601](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L601) +Defined in: [runtime/strategy.ts:615](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L615) DEPTH: score after each shot — the progress-over-rounds curve. BREADTH: best-so-far per rollout. @@ -7902,13 +8046,13 @@ DEPTH: score after each shot — the progress-over-rounds curve. BREADTH: best-s > **shots**: `number` -Defined in: [runtime/strategy.ts:602](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L602) +Defined in: [runtime/strategy.ts:616](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L616) ##### usd > **usd**: `number` -Defined in: [runtime/strategy.ts:605](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L605) +Defined in: [runtime/strategy.ts:619](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L619) The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: real router tokens, priced usd (0 when the model is unpriced — never fabricated), wall ms. @@ -7917,13 +8061,13 @@ The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: r > **ms**: `number` -Defined in: [runtime/strategy.ts:606](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L606) +Defined in: [runtime/strategy.ts:620](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L620) ##### tokens > **tokens**: `object` -Defined in: [runtime/strategy.ts:607](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L607) +Defined in: [runtime/strategy.ts:621](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L621) ###### input @@ -7937,7 +8081,7 @@ Defined in: [runtime/strategy.ts:607](https://github.com/tangle-network/agent-ru ### Strategy -Defined in: [runtime/strategy.ts:744](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L744) +Defined in: [runtime/strategy.ts:758](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L758) A Strategy is HOW you spend the compute budget to beat the Environment's check — it builds the driver `Agent` the Supervisor runs. This is the OPEN extension point: a dev @@ -7954,7 +8098,7 @@ the reference implementations to copy: > `readonly` **name**: `string` -Defined in: [runtime/strategy.ts:745](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L745) +Defined in: [runtime/strategy.ts:759](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L759) #### Methods @@ -7962,7 +8106,7 @@ Defined in: [runtime/strategy.ts:745](https://github.com/tangle-network/agent-ru > **driver**(`surface`, `task`, `opts`, `budget`): [`Agent`](#agent)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:746](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L746) +Defined in: [runtime/strategy.ts:760](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L760) ###### Parameters @@ -7990,7 +8134,7 @@ Defined in: [runtime/strategy.ts:746](https://github.com/tangle-network/agent-ru ### ShotPersona -Defined in: [runtime/strategy.ts:776](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L776) +Defined in: [runtime/strategy.ts:790](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L790) A role for one shot — multi-agent loops (researcher + engineer, a panel of k researchers) give each shot its own system prompt and optionally its own model. @@ -8001,7 +8145,7 @@ A role for one shot — multi-agent loops (researcher + engineer, a panel of k > `optional` **systemPrompt?**: `string` -Defined in: [runtime/strategy.ts:779](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L779) +Defined in: [runtime/strategy.ts:793](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L793) Replaces the task's systemPrompt for a FRESH shot; on a carried conversation it is injected as a hand-off message (the transcript's earlier roles stay intact). @@ -8010,7 +8154,7 @@ Replaces the task's systemPrompt for a FRESH shot; on a carried conversation it > `optional` **model?**: `string` -Defined in: [runtime/strategy.ts:781](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L781) +Defined in: [runtime/strategy.ts:795](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L795) Per-shot model override (e.g. a stronger model for the engineer shot). @@ -8018,7 +8162,7 @@ Per-shot model override (e.g. a stronger model for the engineer shot). ### ShotSpec -Defined in: [runtime/strategy.ts:784](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L784) +Defined in: [runtime/strategy.ts:798](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L798) #### Properties @@ -8026,7 +8170,7 @@ Defined in: [runtime/strategy.ts:784](https://github.com/tangle-network/agent-ru > `optional` **handle?**: [`ArtifactHandle`](#artifacthandle) -Defined in: [runtime/strategy.ts:786](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L786) +Defined in: [runtime/strategy.ts:800](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L800) present ⇒ continue this artifact (depth); absent ⇒ the shot opens a fresh one (sample/restart). @@ -8034,25 +8178,25 @@ present ⇒ continue this artifact (depth); absent ⇒ the shot opens a fresh on > `optional` **messages?**: `Msg`[] -Defined in: [runtime/strategy.ts:787](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L787) +Defined in: [runtime/strategy.ts:801](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L801) ##### steer? > `optional` **steer?**: `string` -Defined in: [runtime/strategy.ts:788](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L788) +Defined in: [runtime/strategy.ts:802](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L802) ##### persona? > `optional` **persona?**: [`ShotPersona`](#shotpersona) -Defined in: [runtime/strategy.ts:789](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L789) +Defined in: [runtime/strategy.ts:803](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L803) ##### tools? > `optional` **tools?**: `string`[] -Defined in: [runtime/strategy.ts:792](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L792) +Defined in: [runtime/strategy.ts:806](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L806) Restrict THIS shot to a subset of the domain's tools (by name) — focus a shot on the relevant capabilities. Restriction-only; unknown names throw. Omitted ⇒ all. @@ -8061,7 +8205,11 @@ Restrict THIS shot to a subset of the domain's tools (by name) — focus a shot ### StrategyResult -Defined in: [runtime/strategy.ts:794](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L794) +Defined in: [runtime/strategy.ts:808](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L808) + +#### Extended by + +- [`StructuralRolloutResult`](#structuralrolloutresult) #### Properties @@ -8069,37 +8217,37 @@ Defined in: [runtime/strategy.ts:794](https://github.com/tangle-network/agent-ru > **score**: `number` -Defined in: [runtime/strategy.ts:795](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L795) +Defined in: [runtime/strategy.ts:809](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L809) ##### resolved > **resolved**: `boolean` -Defined in: [runtime/strategy.ts:796](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L796) +Defined in: [runtime/strategy.ts:810](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L810) ##### completions > **completions**: `number` -Defined in: [runtime/strategy.ts:797](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L797) +Defined in: [runtime/strategy.ts:811](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L811) ##### progression > **progression**: `number`[] -Defined in: [runtime/strategy.ts:798](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L798) +Defined in: [runtime/strategy.ts:812](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L812) ##### shots > **shots**: `number` -Defined in: [runtime/strategy.ts:799](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L799) +Defined in: [runtime/strategy.ts:813](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L813) *** ### StrategyCtx -Defined in: [runtime/strategy.ts:811](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L811) +Defined in: [runtime/strategy.ts:825](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L825) What a strategy body composes with: the artifact lifecycle, the budget, and the two steps. @@ -8109,7 +8257,7 @@ What a strategy body composes with: the artifact lifecycle, the budget, and the > `readonly` **surface**: `StrategyArtifacts` -Defined in: [runtime/strategy.ts:813](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L813) +Defined in: [runtime/strategy.ts:827](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L827) Open/close artifacts the body manages itself (e.g. one persistent handle for depth). @@ -8117,25 +8265,25 @@ Open/close artifacts the body manages itself (e.g. one persistent handle for dep > `readonly` **task**: [`AgenticTask`](#agentictask) -Defined in: [runtime/strategy.ts:814](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L814) +Defined in: [runtime/strategy.ts:828](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L828) ##### opts > `readonly` **opts**: [`AgenticOptions`](#agenticoptions) -Defined in: [runtime/strategy.ts:815](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L815) +Defined in: [runtime/strategy.ts:829](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L829) ##### budget > `readonly` **budget**: `number` -Defined in: [runtime/strategy.ts:816](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L816) +Defined in: [runtime/strategy.ts:830](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L830) ##### scope > `readonly` **scope**: [`Scope`](#scope-1)\<[`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:817](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L817) +Defined in: [runtime/strategy.ts:831](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L831) #### Methods @@ -8143,7 +8291,7 @@ Defined in: [runtime/strategy.ts:817](https://github.com/tangle-network/agent-ru > **shot**(`spec?`): `Promise`\<`ShotResult` \| `null`\> -Defined in: [runtime/strategy.ts:819](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L819) +Defined in: [runtime/strategy.ts:833](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L833) Run ONE worker shot; its harness-scored result, or null if it went down. @@ -8161,7 +8309,7 @@ Run ONE worker shot; its harness-scored result, or null if it went down. > **critique**(`messages`): `Promise`\<`string` \| `null`\> -Defined in: [runtime/strategy.ts:821](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L821) +Defined in: [runtime/strategy.ts:835](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L835) The firewalled critic reads the trajectory → a steer string, or null on COMPLETE/down. @@ -8179,7 +8327,7 @@ The firewalled critic reads the trajectory → a steer string, or null on COMPLE > **consult**(`messages`, `instruction`): `Promise`\<`string` \| `null`\> -Defined in: [runtime/strategy.ts:826](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L826) +Defined in: [runtime/strategy.ts:840](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L840) The RAW analyst channel: the firewalled critic answers `instruction` over the trajectory verbatim — no findings extraction, so verdict-shaped formats @@ -8204,7 +8352,7 @@ The RAW analyst channel: the firewalled critic answers `instruction` over the > **listTools**(`handle`): `Promise`\<`object`[]\> -Defined in: [runtime/strategy.ts:830](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L830) +Defined in: [runtime/strategy.ts:844](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L844) The tools THIS artifact's task actually offers (names + descriptions only — never the implementations). Tool sets vary per task on heterogeneous domains; a strategy @@ -8224,7 +8372,7 @@ The tools THIS artifact's task actually offers (names + descriptions only — ne ### RunAgenticOptions -Defined in: [runtime/strategy.ts:1059](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1059) +Defined in: [runtime/strategy.ts:1073](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1073) #### Extends @@ -8392,19 +8540,19 @@ In-context learning: when set, query `corpus` before each depth shot and inject > **surface**: [`AgenticSurface`](#agenticsurface) -Defined in: [runtime/strategy.ts:1060](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1060) +Defined in: [runtime/strategy.ts:1074](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1074) ##### task > **task**: [`AgenticTask`](#agentictask) -Defined in: [runtime/strategy.ts:1061](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1061) +Defined in: [runtime/strategy.ts:1075](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1075) ##### hooks? > `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) -Defined in: [runtime/strategy.ts:1064](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1064) +Defined in: [runtime/strategy.ts:1078](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1078) Lifecycle observability — every spawn/settle (shots, analysts) streams here live. The seam online watchdogs/route-auditors subscribe to. @@ -8413,7 +8561,7 @@ Lifecycle observability — every spawn/settle (shots, analysts) streams here li > `optional` **strategy?**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:1066](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1066) +Defined in: [runtime/strategy.ts:1080](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1080) A Strategy (the open way) — author/pass your own. Overrides `mode` when present. @@ -8421,7 +8569,7 @@ A Strategy (the open way) — author/pass your own. Overrides `mode` when presen > `optional` **mode?**: `"depth"` \| `"breadth"` -Defined in: [runtime/strategy.ts:1068](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1068) +Defined in: [runtime/strategy.ts:1082](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1082) Built-in shorthand: 'depth'→refine, 'breadth'→sample. Default 'depth'. @@ -8429,7 +8577,7 @@ Built-in shorthand: 'depth'→refine, 'breadth'→sample. Default 'depth'. > **budget**: `number` -Defined in: [runtime/strategy.ts:1070](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1070) +Defined in: [runtime/strategy.ts:1084](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1084) budget: refine→max shots; sample→rollout width. @@ -8437,7 +8585,7 @@ budget: refine→max shots; sample→rollout width. > `optional` **rootBudget?**: [`Budget`](#budget-12) -Defined in: [runtime/strategy.ts:1071](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1071) +Defined in: [runtime/strategy.ts:1085](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1085) *** @@ -8615,155 +8763,618 @@ Defined in: [runtime/stream-agent-turn.ts:198](https://github.com/tangle-network *** -### SurfaceWorkerOut +### StructuralRolloutPolicy -Defined in: [runtime/supervise-surface.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L34) +Defined in: [runtime/structural-rollout.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L45) -What a surface worker settles with — the surface verdict the driver + deliverable read. `resolved` is - the surface check's pass/fail (settled ⟺ resolved); `score` is the partial-credit fraction; `failing` - carries the tests this worker left red (so the analyst can target them). +The rollout's compute recipe — promoted from the proven rigs' env vars (K/REPAIRS/ + TESTGEN/DIVERSE/TEMPERATURE). Defaults are the measured sweet spot: repair value + concentrates at low k (~+12pp at k=1, +1–3pp at k=5), so `k=5, repairRounds=2` is the + full recipe and `k=1, repairRounds=2` the low-compute preset. #### Properties -##### resolved +##### k -> `readonly` **resolved**: `boolean` +> **k**: `number` -Defined in: [runtime/supervise-surface.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L35) +Defined in: [runtime/structural-rollout.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L47) -##### score +Independent samples per task (selection breadth). -> `readonly` **score**: `number` +##### repairRounds -Defined in: [runtime/supervise-surface.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L36) +> **repairRounds**: `number` -##### shots +Defined in: [runtime/structural-rollout.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L49) -> `readonly` **shots**: `number` +Repair shots after selection, each steered by the checks' failure output. -Defined in: [runtime/supervise-surface.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L37) +##### testgen -##### summary +> **testgen**: `number` -> `readonly` **summary**: `string` +Defined in: [runtime/structural-rollout.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L51) -Defined in: [runtime/supervise-surface.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L38) +Model-authored visible checks requested per task; 0 disables authoring. -##### failing? +##### diverse? -> `readonly` `optional` **failing?**: readonly `string`[] +> `optional` **diverse?**: `boolean` -Defined in: [runtime/supervise-surface.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L39) +Defined in: [runtime/structural-rollout.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L54) -*** +Per-slot strategy-lens prefixes on the k samples (attacks the all-k-fail bucket). + Measured as a paired null (+0.6pp) — kept as an optional knob, off by default. -### SurfaceWorkerConfig +##### temperature? -Defined in: [runtime/supervise-surface.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L102) +> `optional` **temperature?**: `number` -How a worker runs the surface task (its router substrate + per-attempt bounds). +Defined in: [runtime/structural-rollout.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L56) -#### Properties +Sampling temperature for every shot of this strategy; omitted ⇒ the worker default. -##### routerBaseUrl +*** -> `readonly` **routerBaseUrl**: `string` +### VisibleCheck -Defined in: [runtime/supervise-surface.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L103) +Defined in: [runtime/structural-rollout.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L87) -##### routerKey +One task-visible executable check (e.g. a single-line Python assert). -> `readonly` **routerKey**: `string` +#### Properties -Defined in: [runtime/supervise-surface.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L104) +##### code -##### model +> **code**: `string` -> `readonly` **model**: `string` +Defined in: [runtime/structural-rollout.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L88) -Defined in: [runtime/supervise-surface.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L105) +##### kind -##### maxTokens? +> **kind**: `"authored"` \| `"official"` -> `readonly` `optional` **maxTokens?**: `number` +Defined in: [runtime/structural-rollout.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L91) -Defined in: [runtime/supervise-surface.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L106) +'official' = shown in the task itself (docstring example, shown assert); + 'authored' = the model's own guess. Official outranks authored in selection. -##### innerTurns? +*** -> `readonly` `optional` **innerTurns?**: `number` +### CheckSourceCtx -Defined in: [runtime/supervise-surface.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L107) +Defined in: [runtime/structural-rollout.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L97) -##### budget? +What a CheckSource composes with. `consult` is the strategy family's raw analyst + channel (metered by the conserved pool, offline-injectable via `opts.complete`) — + check authoring goes through it rather than a bespoke model client. -> `readonly` `optional` **budget?**: `number` +#### Properties -Defined in: [runtime/supervise-surface.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L109) +##### count -Refine-shot budget for ONE worker attempt (max steered shots). Default 1. +> **count**: `number` -*** +Defined in: [runtime/structural-rollout.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L99) -### SuperviseSurfaceOptions +Authored-check budget for this task (`policy.testgen`). -Defined in: [runtime/supervise-surface.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L168) +##### entrySymbol? -#### Properties +> `optional` **entrySymbol?**: `string` -##### surface +Defined in: [runtime/structural-rollout.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L102) -> `readonly` **surface**: [`AgenticSurface`](#agenticsurface) +The symbol authored checks must reference; undefined ⇒ authoring is skipped + (no guesses beats guesses pinned to nothing). -Defined in: [runtime/supervise-surface.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L170) +#### Methods -The graded surface workers solve (open/tools/call/score/close). +##### consult() -##### worker +> **consult**(`instruction`): `Promise`\<`string` \| `null`\> -> `readonly` **worker**: [`SurfaceWorkerConfig`](#surfaceworkerconfig) +Defined in: [runtime/structural-rollout.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L105) -Defined in: [runtime/supervise-surface.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L172) +One metered LLM call: instruction in, reply text out, null when the channel went + down. The task's visible prompt is included by the channel itself. -Where/how each worker runs the surface task. +###### Parameters -##### budget? +###### instruction -> `readonly` `optional` **budget?**: [`Budget`](#budget-12) +`string` -Defined in: [runtime/supervise-surface.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L175) +###### Returns -The conserved compute pool for the whole supervised run. Default: sized off the worker's inner-loop - bounds for a handful of worker spawns — raise it to let the driver try more. +`Promise`\<`string` \| `null`\> -##### router? +*** -> `readonly` `optional` **router?**: [`RouterConfig`](#routerconfig) +### CheckSource -Defined in: [runtime/supervise-surface.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L178) +Defined in: [runtime/structural-rollout.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L111) -The driver brain's router substrate (its own inference). Default: the worker's router + model — the - driver and workers share one router unless you separate them (e.g. a stronger driver model). +Produces the task's visible checks. MUST derive them from agent-visible information + only, before any candidate exists — the strategy freezes the returned set for every + sample and repair round of the task. -##### analysts? +#### Methods -> `readonly` `optional` **analysts?**: [`AnalystRegistry`](#analystregistry) \| `null` +##### generate() -Defined in: [runtime/supervise-surface.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L182) +> **generate**(`task`, `ctx`): `Promise`\<[`VisibleCheck`](#visiblecheck)[]\> -The self-improvement lens fed to the driver on each settled worker. Default `failuresAnalyst()` - (target the still-failing tests). Pass a custom registry to change it, or `null` to turn the - within-run self-improvement OFF (the driver sees raw settled outputs). +Defined in: [runtime/structural-rollout.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L112) -##### strategy? +###### Parameters -> `readonly` `optional` **strategy?**: [`Strategy`](#strategy-3) +###### task -Defined in: [runtime/supervise-surface.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L184) +[`AgenticTask`](#agentictask) -The strategy each worker runs over the surface. Default `refine` (iterate-with-feedback). +###### ctx -##### maxLiveWorkers? +[`CheckSourceCtx`](#checksourcectx) + +###### Returns + +`Promise`\<[`VisibleCheck`](#visiblecheck)[]\> + +*** + +### CheckOutcome + +Defined in: [runtime/structural-rollout.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L205) + +How one candidate fared against the frozen visible checks, split by check kind. + +#### Properties + +##### passedOfficial + +> **passedOfficial**: `number` + +Defined in: [runtime/structural-rollout.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L206) + +##### totalOfficial + +> **totalOfficial**: `number` + +Defined in: [runtime/structural-rollout.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L207) + +##### passedAuthored + +> **passedAuthored**: `number` + +Defined in: [runtime/structural-rollout.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L208) + +##### totalAuthored + +> **totalAuthored**: `number` + +Defined in: [runtime/structural-rollout.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L209) + +##### failureOutput + +> **failureOutput**: `string` + +Defined in: [runtime/structural-rollout.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L211) + +The checks' failure report — the ONLY feedback the repair loop may see. + +##### crashed? + +> `optional` **crashed?**: `boolean` + +Defined in: [runtime/structural-rollout.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L214) + +True when the candidate crashed before any check could run — ranks below a + candidate that ran and failed everything. + +*** + +### CheckExecChannel + +Defined in: [runtime/structural-rollout.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L219) + +Minimal exec channel the default runner needs. `SandboxInstance` (and therefore + `ValidationCtx.box`) satisfies it structurally. + +#### Methods + +##### exec() + +> **exec**(`command`, `options?`): `Promise`\<\{ `exitCode`: `number`; `stdout`: `string`; `stderr`: `string`; \}\> + +Defined in: [runtime/structural-rollout.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L220) + +###### Parameters + +###### command + +`string` + +###### options? + +###### timeoutMs? + +`number` + +###### Returns + +`Promise`\<\{ `exitCode`: `number`; `stdout`: `string`; `stderr`: `string`; \}\> + +*** + +### CheckRunContext + +Defined in: [runtime/structural-rollout.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L226) + +#### Properties + +##### task + +> **task**: [`AgenticTask`](#agentictask) + +Defined in: [runtime/structural-rollout.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L227) + +##### box? + +> `optional` **box?**: [`CheckExecChannel`](#checkexecchannel) + +Defined in: [runtime/structural-rollout.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L229) + +Live exec channel for this run (`ValidationCtx.box` / a sandbox instance). + +##### signal? + +> `optional` **signal?**: `AbortSignal` + +Defined in: [runtime/structural-rollout.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L230) + +*** + +### CheckRunner + +Defined in: [runtime/structural-rollout.ts:235](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L235) + +Executes the frozen checks against one candidate. Implementations MUST fail loud + (throw) when they cannot execute — a silent zero poisons selection. + +#### Methods + +##### run() + +> **run**(`candidate`, `checks`, `ctx`): `Promise`\<[`CheckOutcome`](#checkoutcome)\> + +Defined in: [runtime/structural-rollout.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L236) + +###### Parameters + +###### candidate + +`string` + +###### checks + +[`VisibleCheck`](#visiblecheck)[] + +###### ctx + +[`CheckRunContext`](#checkruncontext) + +###### Returns + +`Promise`\<[`CheckOutcome`](#checkoutcome)\> + +*** + +### StructuralRolloutResult + +Defined in: [runtime/structural-rollout.ts:489](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L489) + +The body's deliverable — a `StrategyResult` plus selection provenance. The extra + fields ride through `defineStrategy`'s deliverable spread onto `AgenticRunResult` + (score/resolved stay harness-verified, exactly as for every authored strategy). + +#### Extends + +- [`StrategyResult`](#strategyresult) + +#### Properties + +##### score + +> **score**: `number` + +Defined in: [runtime/strategy.ts:809](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L809) + +###### Inherited from + +[`StrategyResult`](#strategyresult).[`score`](#score-9) + +##### resolved + +> **resolved**: `boolean` + +Defined in: [runtime/strategy.ts:810](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L810) + +###### Inherited from + +[`StrategyResult`](#strategyresult).[`resolved`](#resolved-4) + +##### completions + +> **completions**: `number` + +Defined in: [runtime/strategy.ts:811](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L811) + +###### Inherited from + +[`StrategyResult`](#strategyresult).[`completions`](#completions-1) + +##### progression + +> **progression**: `number`[] + +Defined in: [runtime/strategy.ts:812](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L812) + +###### Inherited from + +[`StrategyResult`](#strategyresult).[`progression`](#progression-2) + +##### shots + +> **shots**: `number` + +Defined in: [runtime/strategy.ts:813](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L813) + +###### Inherited from + +[`StrategyResult`](#strategyresult).[`shots`](#shots-3) + +##### selection + +> **selection**: [`SelectionReceipt`](#selectionreceipt)[] + +Defined in: [runtime/structural-rollout.ts:492](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L492) + +One receipt per scored candidate (k samples, then repairs), `SelectionReceipt` + shaped like the kernel's (`types.ts`), selector 'driver'. + +##### repairStop + +> **repairStop**: [`RepairStop`](#repairstop) + +Defined in: [runtime/structural-rollout.ts:493](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L493) + +##### officialChecks + +> **officialChecks**: `number` + +Defined in: [runtime/structural-rollout.ts:494](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L494) + +##### authoredChecks + +> **authoredChecks**: `number` + +Defined in: [runtime/structural-rollout.ts:495](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L495) + +*** + +### StructuralRolloutConfig + +Defined in: [runtime/structural-rollout.ts:498](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L498) + +#### Properties + +##### policy? + +> `optional` **policy?**: `Partial`\<[`StructuralRolloutPolicy`](#structuralrolloutpolicy)\> + +Defined in: [runtime/structural-rollout.ts:500](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L500) + +Knobs; missing fields take the measured defaults (k=5, repairRounds=2, testgen=6). + +##### checkSource? + +> `optional` **checkSource?**: [`CheckSource`](#checksource) + +Defined in: [runtime/structural-rollout.ts:503](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L503) + +Where the visible checks come from. Default: official checks from + `task.meta.visibleChecks` composed with `modelAuthoredChecks()`. + +##### checkRunner? + +> `optional` **checkRunner?**: [`CheckRunner`](#checkrunner) + +Defined in: [runtime/structural-rollout.ts:506](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L506) + +How candidates are measured. Default `sandboxCheckRunner()` — it needs an exec + channel (bind one to the runner, or pass `box` here) and fails loud without one. + +##### box? + +> `optional` **box?**: [`CheckExecChannel`](#checkexecchannel) + +Defined in: [runtime/structural-rollout.ts:510](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L510) + +Exec channel threaded into every check run of this strategy (a sandbox instance / + `ValidationCtx.box`). The strategy seam itself carries no sandbox, so the caller + who owns one supplies it here or binds it into the runner. + +##### extractCandidate? + +> `optional` **extractCandidate?**: (`messages`) => `string` + +Defined in: [runtime/structural-rollout.ts:512](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L512) + +Candidate extraction from a shot's conversation. Default `defaultExtractCandidate`. + +###### Parameters + +###### messages + +readonly `Msg`[] + +###### Returns + +`string` + +*** + +### SurfaceWorkerOut + +Defined in: [runtime/supervise-surface.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L34) + +What a surface worker settles with — the surface verdict the driver + deliverable read. `resolved` is + the surface check's pass/fail (settled ⟺ resolved); `score` is the partial-credit fraction; `failing` + carries the tests this worker left red (so the analyst can target them). + +#### Properties + +##### resolved + +> `readonly` **resolved**: `boolean` + +Defined in: [runtime/supervise-surface.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L35) + +##### score + +> `readonly` **score**: `number` + +Defined in: [runtime/supervise-surface.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L36) + +##### shots + +> `readonly` **shots**: `number` + +Defined in: [runtime/supervise-surface.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L37) + +##### summary + +> `readonly` **summary**: `string` + +Defined in: [runtime/supervise-surface.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L38) + +##### failing? + +> `readonly` `optional` **failing?**: readonly `string`[] + +Defined in: [runtime/supervise-surface.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L39) + +*** + +### SurfaceWorkerConfig + +Defined in: [runtime/supervise-surface.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L102) + +How a worker runs the surface task (its router substrate + per-attempt bounds). + +#### Properties + +##### routerBaseUrl + +> `readonly` **routerBaseUrl**: `string` + +Defined in: [runtime/supervise-surface.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L103) + +##### routerKey + +> `readonly` **routerKey**: `string` + +Defined in: [runtime/supervise-surface.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L104) + +##### model + +> `readonly` **model**: `string` + +Defined in: [runtime/supervise-surface.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L105) + +##### maxTokens? + +> `readonly` `optional` **maxTokens?**: `number` + +Defined in: [runtime/supervise-surface.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L106) + +##### innerTurns? + +> `readonly` `optional` **innerTurns?**: `number` + +Defined in: [runtime/supervise-surface.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L107) + +##### budget? + +> `readonly` `optional` **budget?**: `number` + +Defined in: [runtime/supervise-surface.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L109) + +Refine-shot budget for ONE worker attempt (max steered shots). Default 1. + +*** + +### SuperviseSurfaceOptions + +Defined in: [runtime/supervise-surface.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L168) + +#### Properties + +##### surface + +> `readonly` **surface**: [`AgenticSurface`](#agenticsurface) + +Defined in: [runtime/supervise-surface.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L170) + +The graded surface workers solve (open/tools/call/score/close). + +##### worker + +> `readonly` **worker**: [`SurfaceWorkerConfig`](#surfaceworkerconfig) + +Defined in: [runtime/supervise-surface.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L172) + +Where/how each worker runs the surface task. + +##### budget? + +> `readonly` `optional` **budget?**: [`Budget`](#budget-12) + +Defined in: [runtime/supervise-surface.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L175) + +The conserved compute pool for the whole supervised run. Default: sized off the worker's inner-loop + bounds for a handful of worker spawns — raise it to let the driver try more. + +##### router? + +> `readonly` `optional` **router?**: [`RouterConfig`](#routerconfig) + +Defined in: [runtime/supervise-surface.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L178) + +The driver brain's router substrate (its own inference). Default: the worker's router + model — the + driver and workers share one router unless you separate them (e.g. a stronger driver model). + +##### analysts? + +> `readonly` `optional` **analysts?**: [`AnalystRegistry`](#analystregistry) \| `null` + +Defined in: [runtime/supervise-surface.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L182) + +The self-improvement lens fed to the driver on each settled worker. Default `failuresAnalyst()` + (target the still-failing tests). Pass a custom registry to change it, or `null` to turn the + within-run self-improvement OFF (the driver sees raw settled outputs). + +##### strategy? + +> `readonly` `optional` **strategy?**: [`Strategy`](#strategy-3) + +Defined in: [runtime/supervise-surface.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise-surface.ts#L184) + +The strategy each worker runs over the surface. Default `refine` (iterate-with-feedback). + +##### maxLiveWorkers? > `readonly` `optional` **maxLiveWorkers?**: `number` @@ -10083,7 +10694,7 @@ Defined in: [runtime/supervise/run-context.ts:50](https://github.com/tangle-netw ### ProviderSeam -Defined in: [runtime/supervise/runtime.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L197) +Defined in: [runtime/supervise/runtime.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L201) Generic environment provider executor config. External packages implement `AgentEnvironmentProvider`; this built-in wrapper lets `createExecutor` @@ -10173,13 +10784,13 @@ Defined in: [runtime/environment-provider.ts:273](https://github.com/tangle-netw > **provider**: `string` \| `AgentEnvironmentProvider` -Defined in: [runtime/supervise/runtime.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L198) +Defined in: [runtime/supervise/runtime.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L202) ##### registry? > `optional` **registry?**: [`AgentEnvironmentProviderRegistry`](runtime/environment-provider.md#agentenvironmentproviderregistry) -Defined in: [runtime/supervise/runtime.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L199) +Defined in: [runtime/supervise/runtime.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L203) *** @@ -10634,8 +11245,8 @@ Structured run summary (tool-call count, step order). Steps carry a single times Defined in: [runtime/supervise/trajectory-recorder.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/trajectory-recorder.ts#L22) -Full-run repeated-call view (total occurrences + window) — catches a loop the online consecutive - detector interleaves past. +Full-run repeated-call view (total occurrences + window) — allows one intervening call so it +catches a loop the online consecutive detector interleaves past. ##### toolWaste @@ -11089,23 +11700,32 @@ Defined in: [runtime/supervise/types.ts:210](https://github.com/tangle-network/a Defined in: [runtime/supervise/types.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L211) +##### usdKnown? + +> `optional` **usdKnown?**: `boolean` + +Defined in: [runtime/supervise/types.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L214) + +Dollar accounting is known unless explicitly false. A false value must not be treated as $0 + when enforcing a dollar-denominated comparison or limit. + ##### usd > **usd**: `number` -Defined in: [runtime/supervise/types.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L212) +Defined in: [runtime/supervise/types.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L215) ##### ms > **ms**: `number` -Defined in: [runtime/supervise/types.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L213) +Defined in: [runtime/supervise/types.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L216) *** ### Scope -Defined in: [runtime/supervise/types.ts:284](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L284) +Defined in: [runtime/supervise/types.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L287) The budget-conserving reactive scope an `Agent.act` runs inside. `spawn` reserves budget atomically from the shared pool and fails closed when the pool cannot cover it. @@ -11124,7 +11744,7 @@ not the replay log. > `readonly` **signal**: `AbortSignal` -Defined in: [runtime/supervise/types.ts:318](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L318) +Defined in: [runtime/supervise/types.ts:321](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L321) This scope's abort signal — aborted when the run is cancelled, a breaker trips, the pool is exhausted, or a parent scope cascades. A long-running driver `act` over this scope reads @@ -11135,7 +11755,7 @@ This scope's abort signal — aborted when the run is cancelled, a breaker trips > `readonly` **view**: [`TreeView`](#treeview) -Defined in: [runtime/supervise/types.ts:332](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L332) +Defined in: [runtime/supervise/types.ts:335](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L335) The live tree — reads the in-memory nursery, not the journal. @@ -11143,7 +11763,7 @@ The live tree — reads the in-memory nursery, not the journal. > `readonly` **budget**: `Readonly`\<\{ `tokensLeft`: `number`; `usdLeft`: `number`; `usdCapped`: `boolean`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> -Defined in: [runtime/supervise/types.ts:334](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L334) +Defined in: [runtime/supervise/types.ts:337](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L337) Conserved-pool readouts (post-reservation). @@ -11153,7 +11773,7 @@ Conserved-pool readouts (post-reservation). > **spawn**\<`C`\>(`agent`, `task`, `opts`): \{ `ok`: `true`; `handle`: `Handle`\<`C`\>; \} \| \{ `ok`: `false`; `reason`: `"budget-exhausted"` \| `"depth-exceeded"`; \} -Defined in: [runtime/supervise/types.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L290) +Defined in: [runtime/supervise/types.ts:293](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L293) Spawn a child. Reserves `opts.budget` from the conserved pool atomically; refunds the unspent remainder on settle. Returns a typed outcome — fail-closed on an exhausted @@ -11187,7 +11807,7 @@ pool or an exceeded depth ceiling (the caller inspects `ok` before `handle`). > **next**(): `Promise`\<[`Settled`](#settled-2)\<`Out`\> \| `null`\> -Defined in: [runtime/supervise/types.ts:297](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L297) +Defined in: [runtime/supervise/types.ts:300](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L300) ray.wait n=1 over this scope's in-memory live set; resolves as each child settles; `null` when the live set is empty. @@ -11200,7 +11820,7 @@ ray.wait n=1 over this scope's in-memory live set; resolves as each child settle > **nextResolved**(): `Promise`\<[`Settled`](#settled-2)\<`Out`\> \| `null`\> -Defined in: [runtime/supervise/types.ts:304](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L304) +Defined in: [runtime/supervise/types.ts:307](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L307) Non-blocking twin of `next()`: deliver an ALREADY-settled, undelivered child, or `null` when none is ready — never awaits a live child. The driver's post-loop drain reads this so @@ -11215,7 +11835,7 @@ the finalize ledger instead of being silently lost. > **send**(`nodeId`, `msg`): `boolean` -Defined in: [runtime/supervise/types.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L313) +Defined in: [runtime/supervise/types.ts:316](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L316) Steer a RUNNING child out-of-band — deliver a message to its executor's inbox (the driver's `send` verb: next-instruction, interrupt, or resume). Returns `true` if the message was @@ -11242,7 +11862,7 @@ is a direct call; the sandbox/Agent-Bus transports surface the SAME verb as an M > **meter**(`spend`, `detail?`): `Promise`\<`void`\> -Defined in: [runtime/supervise/types.ts:330](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L330) +Defined in: [runtime/supervise/types.ts:333](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L333) Meter the driver's OWN compute against the conserved pool — its inference turns, which are real tokens/usd but not a spawned child (no reserve/reconcile). A direct `free → committed` @@ -11272,7 +11892,7 @@ metered event is cost-critical, so it lands before the join-barrier roll-up). ### TreeView -Defined in: [runtime/supervise/types.ts:359](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L359) +Defined in: [runtime/supervise/types.ts:362](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L362) The live tree — what `scope.view` / `RootHandle.view()` materialize for a viewer. @@ -11282,19 +11902,19 @@ The live tree — what `scope.view` / `RootHandle.view()` materialize for a view > `readonly` **root**: `string` -Defined in: [runtime/supervise/types.ts:360](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L360) +Defined in: [runtime/supervise/types.ts:363](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L363) ##### nodes > `readonly` **nodes**: readonly `NodeSnapshot`[] -Defined in: [runtime/supervise/types.ts:361](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L361) +Defined in: [runtime/supervise/types.ts:364](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L364) ##### inFlight > `readonly` **inFlight**: `number` -Defined in: [runtime/supervise/types.ts:363](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L363) +Defined in: [runtime/supervise/types.ts:366](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L366) Count of nodes in `running` or `acquiring` — the "what's in flow?" answer. @@ -11302,7 +11922,7 @@ Count of nodes in `running` or `acquiring` — the "what's in flow?" answer. ### ResultBlobStore -Defined in: [runtime/supervise/types.ts:423](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L423) +Defined in: [runtime/supervise/types.ts:426](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L426) Content-addressed result blobs (the `outRef` → artifact map) backing the replay invariant. Split from the journal so the journal stays small (decisions) and the @@ -11314,7 +11934,7 @@ Content-addressed result blobs (the `outRef` → artifact map) backing the repla > **put**(`outRef`, `artifact`): `Promise`\<`void`\> -Defined in: [runtime/supervise/types.ts:424](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L424) +Defined in: [runtime/supervise/types.ts:427](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L427) ###### Parameters @@ -11334,7 +11954,7 @@ Defined in: [runtime/supervise/types.ts:424](https://github.com/tangle-network/a > **get**(`outRef`): `Promise`\<`unknown`\> -Defined in: [runtime/supervise/types.ts:425](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L425) +Defined in: [runtime/supervise/types.ts:428](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L428) ###### Parameters @@ -11350,7 +11970,7 @@ Defined in: [runtime/supervise/types.ts:425](https://github.com/tangle-network/a ### Supervisor -Defined in: [runtime/supervise/types.ts:435](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L435) +Defined in: [runtime/supervise/types.ts:438](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L438) Owns the conserved pool, the spawn log, the abort cascade, the OTP intensity breaker, and the root handle. `run` executes the root `Agent` to completion; `attach` wires a @@ -11372,7 +11992,7 @@ live `RootHandle` (the Q2 substrate the chat/pi-viz client later consumes). > **run**(`root`, `task`, `opts`): `Promise`\<[`SupervisedResult`](#supervisedresult)\<`Out`\>\> -Defined in: [runtime/supervise/types.ts:436](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L436) +Defined in: [runtime/supervise/types.ts:439](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L439) ###### Parameters @@ -11396,7 +12016,7 @@ Defined in: [runtime/supervise/types.ts:436](https://github.com/tangle-network/a > **attach**(`h`): `void` -Defined in: [runtime/supervise/types.ts:437](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L437) +Defined in: [runtime/supervise/types.ts:440](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L440) ###### Parameters @@ -11412,7 +12032,7 @@ Defined in: [runtime/supervise/types.ts:437](https://github.com/tangle-network/a ### SupervisorOpts -Defined in: [runtime/supervise/types.ts:440](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L440) +Defined in: [runtime/supervise/types.ts:443](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L443) #### Properties @@ -11420,7 +12040,7 @@ Defined in: [runtime/supervise/types.ts:440](https://github.com/tangle-network/a > `readonly` **budget**: [`Budget`](#budget-12) -Defined in: [runtime/supervise/types.ts:442](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L442) +Defined in: [runtime/supervise/types.ts:445](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L445) The root conserved-pool ceiling (tokens + usd + iterations + deadline). @@ -11428,7 +12048,7 @@ The root conserved-pool ceiling (tokens + usd + iterations + deadline). > `readonly` **runId**: `string` -Defined in: [runtime/supervise/types.ts:444](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L444) +Defined in: [runtime/supervise/types.ts:447](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L447) Trace-correlation root + the journal/blob root key. @@ -11436,7 +12056,7 @@ Trace-correlation root + the journal/blob root key. > `readonly` **journal**: `SpawnJournal` -Defined in: [runtime/supervise/types.ts:446](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L446) +Defined in: [runtime/supervise/types.ts:449](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L449) Event source — defaults to the in-memory journal in the impl; pass JSONL/FS for durability. @@ -11444,7 +12064,7 @@ Event source — defaults to the in-memory journal in the impl; pass JSONL/FS fo > `readonly` **blobs**: [`ResultBlobStore`](#resultblobstore) -Defined in: [runtime/supervise/types.ts:448](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L448) +Defined in: [runtime/supervise/types.ts:451](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L451) Result payload store backing `outRef` rehydration. @@ -11452,7 +12072,7 @@ Result payload store backing `outRef` rehydration. > `readonly` **executors**: [`ExecutorRegistry`](#executorregistry) -Defined in: [runtime/supervise/types.ts:450](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L450) +Defined in: [runtime/supervise/types.ts:453](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L453) Executor resolution — the open registry mapping `AgentSpec` → `Executor`. @@ -11460,7 +12080,7 @@ Executor resolution — the open registry mapping `AgentSpec` → `Executor`. > `readonly` `optional` **maxDepth?**: `number` -Defined in: [runtime/supervise/types.ts:452](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L452) +Defined in: [runtime/supervise/types.ts:455](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L455) Runtime recursion-depth ceiling (paired with the conserved pool per R3). @@ -11468,7 +12088,7 @@ Runtime recursion-depth ceiling (paired with the conserved pool per R3). > `readonly` `optional` **maxRestarts?**: `number` -Defined in: [runtime/supervise/types.ts:457](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L457) +Defined in: [runtime/supervise/types.ts:460](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L460) OTP intensity breaker: more than `maxRestarts` child restarts within `withinMs` trips the supervisor to `no-winner` rather than restarting forever. @@ -11477,13 +12097,13 @@ trips the supervisor to `no-winner` rather than restarting forever. > `readonly` `optional` **withinMs?**: `number` -Defined in: [runtime/supervise/types.ts:458](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L458) +Defined in: [runtime/supervise/types.ts:461](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L461) ##### now? > `readonly` `optional` **now?**: () => `number` -Defined in: [runtime/supervise/types.ts:459](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L459) +Defined in: [runtime/supervise/types.ts:462](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L462) ###### Returns @@ -11493,13 +12113,13 @@ Defined in: [runtime/supervise/types.ts:459](https://github.com/tangle-network/a > `readonly` `optional` **signal?**: `AbortSignal` -Defined in: [runtime/supervise/types.ts:460](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L460) +Defined in: [runtime/supervise/types.ts:463](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L463) ##### hooks? > `readonly` `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) -Defined in: [runtime/supervise/types.ts:463](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L463) +Defined in: [runtime/supervise/types.ts:466](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L466) Lifecycle stream sink, threaded into the root `Scope` so every `spawn`/settle emits on the same `agent.spawn`/`agent.child` stream `runLoop` feeds — one observable recursive tree. @@ -11508,7 +12128,7 @@ Lifecycle stream sink, threaded into the root `Scope` so every `spawn`/settle em ### WidenGate -Defined in: [runtime/supervise/types.ts:519](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L519) +Defined in: [runtime/supervise/types.ts:522](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L522) The progressive-widening gate (MCTS-PW). Decides whether a settled child is `promising` enough to spawn another under the remaining pool. DEFAULTS TO FLAT @@ -11529,7 +12149,7 @@ an explicit, argued `judgeExempt: true` (the documented escape hatch, off by def > `readonly` `optional` **judgeExempt?**: `boolean` -Defined in: [runtime/supervise/types.ts:524](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L524) +Defined in: [runtime/supervise/types.ts:527](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L527) When true, widening may read `verdict` directly (collides with the steer firewall — must be explicitly argued per cell, never defaulted on). @@ -11540,7 +12160,7 @@ When true, widening may read `verdict` directly (collides with the steer firewal > **shouldWiden**(`settled`, `budget`): `boolean` -Defined in: [runtime/supervise/types.ts:521](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L521) +Defined in: [runtime/supervise/types.ts:524](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L524) Default impl returns false for every settlement (flat — never widens). @@ -11562,7 +12182,7 @@ Default impl returns false for every settlement (flat — never widens). ### WorktreeCliExecutorOptions -Defined in: [runtime/supervise/worktree-cli-executor.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L47) +Defined in: [runtime/supervise/worktree-cli-executor.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L45) **`Experimental`** @@ -11572,7 +12192,7 @@ Defined in: [runtime/supervise/worktree-cli-executor.ts:47](https://github.com/t > **repoRoot**: `string` -Defined in: [runtime/supervise/worktree-cli-executor.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L49) +Defined in: [runtime/supervise/worktree-cli-executor.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L47) **`Experimental`** @@ -11582,27 +12202,31 @@ Absolute path to the git checkout the worktree is cut from. > **profile**: `AgentProfile` -Defined in: [runtime/supervise/worktree-cli-executor.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L51) +Defined in: [runtime/supervise/worktree-cli-executor.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L55) **`Experimental`** -The SUPERVISOR-AUTHORED profile (the §1.5 payload: systemPrompt + model). +The supervisor-authored prompt/model plus materializable structural resources. +`model.default` selects the one-shot model; `small`, `provider`, and `metadata` remain hints. +Resource failures are fatal regardless of `resources.failOnError`. +Tools, permissions, connections, confidential execution, modes, and extensions fail closed. +Harness-specific nested controls that the pinned materializer cannot preserve also fail closed. ##### harness > **harness**: [`LocalHarness`](mcp.md#localharness) -Defined in: [runtime/supervise/worktree-cli-executor.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L53) +Defined in: [runtime/supervise/worktree-cli-executor.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L57) **`Experimental`** -Which local harness CLI drives this leaf (`claude` | `codex` | `opencode`). +Local CLI for this leaf. This explicit choice overrides `profile.harness`. ##### taskPrompt > **taskPrompt**: `string` -Defined in: [runtime/supervise/worktree-cli-executor.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L55) +Defined in: [runtime/supervise/worktree-cli-executor.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L59) **`Experimental`** @@ -11612,7 +12236,7 @@ The per-task instruction handed to the harness (composed under the system prompt > `optional` **runId?**: `string` -Defined in: [runtime/supervise/worktree-cli-executor.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L57) +Defined in: [runtime/supervise/worktree-cli-executor.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L61) **`Experimental`** @@ -11622,7 +12246,7 @@ Unique id for the worktree path + branch. Defaults to a fresh UUID. > `optional` **baseRef?**: `string` -Defined in: [runtime/supervise/worktree-cli-executor.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L59) +Defined in: [runtime/supervise/worktree-cli-executor.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L63) **`Experimental`** @@ -11632,17 +12256,39 @@ Override the base ref the worktree is cut from (default `HEAD`). > `optional` **harnessTimeoutMs?**: `number` -Defined in: [runtime/supervise/worktree-cli-executor.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L61) +Defined in: [runtime/supervise/worktree-cli-executor.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L65) **`Experimental`** Wall-clock cap per harness subprocess (ms). Default 5 min (the `runLocalHarness` default). +##### codexReproducible? + +> `optional` **codexReproducible?**: `boolean` + +Defined in: [runtime/supervise/worktree-cli-executor.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L68) + +**`Experimental`** + +Run Codex with an ephemeral session, isolated config/instructions, network disabled, and + JSONL usage capture. Requires `harness: 'codex'`; metered by default. + +##### codexReadDeniedPaths? + +> `optional` **codexReadDeniedPaths?**: readonly `string`[] + +Defined in: [runtime/supervise/worktree-cli-executor.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L71) + +**`Experimental`** + +Absolute host paths denied to reproducible Codex (for benchmark answer copies, credentials, + or other task-specific ambient state). + ##### testCmd? > `optional` **testCmd?**: `string` -Defined in: [runtime/supervise/worktree-cli-executor.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L66) +Defined in: [runtime/supervise/worktree-cli-executor.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L76) **`Experimental`** @@ -11653,7 +12299,7 @@ Its exit code becomes `artifact.checks.tests.passed`. Omit to skip (no signal de > `optional` **typecheckCmd?**: `string` -Defined in: [runtime/supervise/worktree-cli-executor.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L68) +Defined in: [runtime/supervise/worktree-cli-executor.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L78) **`Experimental`** @@ -11663,7 +12309,7 @@ Shell command run in the live worktree to derive the typecheck-PASS signal (e.g. > `optional` **checkTimeoutMs?**: `number` -Defined in: [runtime/supervise/worktree-cli-executor.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L70) +Defined in: [runtime/supervise/worktree-cli-executor.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L80) **`Experimental`** @@ -11673,7 +12319,7 @@ Wall-clock cap per verification command (ms). Default = `harnessTimeoutMs` or 5 > `optional` **checkOutputCap?**: `number` -Defined in: [runtime/supervise/worktree-cli-executor.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L72) +Defined in: [runtime/supervise/worktree-cli-executor.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L82) **`Experimental`** @@ -11683,7 +12329,7 @@ Cap on each check's captured output. Default 16k. > `optional` **runGit?**: [`GitRunner`](mcp.md#gitrunner) -Defined in: [runtime/supervise/worktree-cli-executor.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L74) +Defined in: [runtime/supervise/worktree-cli-executor.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L84) **`Experimental`** @@ -11693,7 +12339,7 @@ Test seam — inject a git runner so unit tests drive the worktree helpers witho > `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> -Defined in: [runtime/supervise/worktree-cli-executor.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L76) +Defined in: [runtime/supervise/worktree-cli-executor.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L86) **`Experimental`** @@ -11731,7 +12377,7 @@ Does NOT throw when: > `optional` **runCommand?**: `WorktreeCheckRunner` -Defined in: [runtime/supervise/worktree-cli-executor.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L79) +Defined in: [runtime/supervise/worktree-cli-executor.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L89) **`Experimental`** @@ -11742,14 +12388,13 @@ Test seam — inject the verification-command runner so unit tests script test/t > `optional` **budgetExempt?**: `boolean` -Defined in: [runtime/supervise/worktree-cli-executor.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L86) +Defined in: [runtime/supervise/worktree-cli-executor.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L95) **`Experimental`** -Exclude this leaf's spend from the conserved pool + equal-k arms. Defaults to `true` because a -coding-harness CLI does not surface token usage, so metering it would record a fabricated zero -(the no-silent-zeros rule forbids that). Set `false` ONLY for a harness that surfaces real -token/usd usage worth metering — the executor would then debit the (real) spend it captures. +Exclude this leaf's spend from accounting. Defaults to `true` for ordinary CLI runs and +`false` for `codexReproducible`, which captures real token usage. A metered custom runner must +likewise return `LocalHarnessResult.usage`. *** @@ -12371,12 +13016,10 @@ Defined in: [runtime/types.ts:115](https://github.com/tangle-network/agent-runti ### LoopTokenUsage -Defined in: [runtime/types.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L122) +Defined in: [runtime/types.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L120) -LLM token usage. Structurally matches agent-eval's `RunTokenUsage` / - `CampaignTokenUsage` ({ input, output }) so a loop result maps straight - onto `ctx.cost.observeTokens` in a `runProfileMatrix` dispatch — without - which the backend-integrity guard reads the run as a stub. +LLM token usage. Structurally maps into agent-eval's paid-call receipt so a +campaign dispatch settles real usage instead of appearing as a stub. #### Properties @@ -12384,19 +13027,19 @@ LLM token usage. Structurally matches agent-eval's `RunTokenUsage` / > **input**: `number` -Defined in: [runtime/types.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L123) +Defined in: [runtime/types.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L121) ##### output > **output**: `number` -Defined in: [runtime/types.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L124) +Defined in: [runtime/types.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L122) *** ### MountManifestEntry -Defined in: [runtime/types.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L138) +Defined in: [runtime/types.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L136) **`Experimental`** @@ -12414,7 +13057,7 @@ auditable after the fact ("what exactly was this agent given?"). > **path**: `string` -Defined in: [runtime/types.ts:140](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L140) +Defined in: [runtime/types.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L138) **`Experimental`** @@ -12424,7 +13067,7 @@ Destination path inside the box where the resource was placed. > **sha256**: `string` -Defined in: [runtime/types.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L143) +Defined in: [runtime/types.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L141) **`Experimental`** @@ -12435,7 +13078,7 @@ Hex SHA-256 of the mounted bytes. The caller computes it from the bytes > **bytes**: `number` -Defined in: [runtime/types.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L145) +Defined in: [runtime/types.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L143) **`Experimental`** @@ -12445,7 +13088,7 @@ Size of the mounted resource in bytes. > **source**: `string` -Defined in: [runtime/types.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L148) +Defined in: [runtime/types.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L146) **`Experimental`** @@ -12456,7 +13099,7 @@ Free-form origin of the resource (e.g. a repo ref, a corpus id, a local ### SelectionReceipt -Defined in: [runtime/types.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L160) +Defined in: [runtime/types.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L158) **`Experimental`** @@ -12472,7 +13115,7 @@ per scored candidate at finalize so a run answers "why did THIS one win?". > **candidateIndex**: `number` -Defined in: [runtime/types.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L162) +Defined in: [runtime/types.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L160) **`Experimental`** @@ -12482,7 +13125,7 @@ Iteration index this receipt is about. > **selected**: `boolean` -Defined in: [runtime/types.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L164) +Defined in: [runtime/types.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L162) **`Experimental`** @@ -12492,7 +13135,7 @@ True for the iteration the selector chose as winner; false otherwise. > `optional` **score?**: `number` -Defined in: [runtime/types.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L166) +Defined in: [runtime/types.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L164) **`Experimental`** @@ -12502,7 +13145,7 @@ The candidate's verdict score, when it has one. > `optional` **reason?**: `string` -Defined in: [runtime/types.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L168) +Defined in: [runtime/types.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L166) **`Experimental`** @@ -12510,9 +13153,9 @@ Why this candidate was (or was not) selected, when the selector states it. ##### selector -> **selector**: `"driver"` \| `"caller"` \| `"default"` +> **selector**: `"default"` \| `"driver"` \| `"caller"` -Defined in: [runtime/types.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L172) +Defined in: [runtime/types.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L170) **`Experimental`** @@ -12524,7 +13167,7 @@ Identity of the selector that produced this receipt — `'caller'` (an ### RunProvenance -Defined in: [runtime/types.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L184) +Defined in: [runtime/types.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L182) **`Experimental`** @@ -12540,7 +13183,7 @@ candidate to select. > **mounts**: [`MountManifestEntry`](#mountmanifestentry)[] -Defined in: [runtime/types.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L186) +Defined in: [runtime/types.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L184) **`Experimental`** @@ -12550,7 +13193,7 @@ Every resource recorded via `prepareBox`'s `recordMount`, in record order. > **selectionReceipts**: [`SelectionReceipt`](#selectionreceipt)[] -Defined in: [runtime/types.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L188) +Defined in: [runtime/types.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L186) **`Experimental`** @@ -12560,7 +13203,7 @@ One receipt per scored candidate at finalize, in iteration order. ### Iteration -Defined in: [runtime/types.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L201) +Defined in: [runtime/types.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L199) **`Experimental`** @@ -12580,7 +13223,7 @@ Defined in: [runtime/types.ts:201](https://github.com/tangle-network/agent-runti > **index**: `number` -Defined in: [runtime/types.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L203) +Defined in: [runtime/types.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L201) **`Experimental`** @@ -12590,7 +13233,7 @@ Defined in: [runtime/types.ts:203](https://github.com/tangle-network/agent-runti > **task**: `Task` -Defined in: [runtime/types.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L204) +Defined in: [runtime/types.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L202) **`Experimental`** @@ -12598,7 +13241,7 @@ Defined in: [runtime/types.ts:204](https://github.com/tangle-network/agent-runti > **agentRunName**: `string` -Defined in: [runtime/types.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L206) +Defined in: [runtime/types.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L204) **`Experimental`** @@ -12608,7 +13251,7 @@ Stable name of the `AgentRunSpec` that produced this iteration. > `optional` **output?**: `Output` -Defined in: [runtime/types.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L207) +Defined in: [runtime/types.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L205) **`Experimental`** @@ -12616,7 +13259,7 @@ Defined in: [runtime/types.ts:207](https://github.com/tangle-network/agent-runti > `optional` **verdict?**: `DefaultVerdict` -Defined in: [runtime/types.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L208) +Defined in: [runtime/types.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L206) **`Experimental`** @@ -12624,7 +13267,7 @@ Defined in: [runtime/types.ts:208](https://github.com/tangle-network/agent-runti > `optional` **error?**: `Error` -Defined in: [runtime/types.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L209) +Defined in: [runtime/types.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L207) **`Experimental`** @@ -12632,7 +13275,7 @@ Defined in: [runtime/types.ts:209](https://github.com/tangle-network/agent-runti > **events**: `SandboxEvent`[] -Defined in: [runtime/types.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L211) +Defined in: [runtime/types.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L209) **`Experimental`** @@ -12642,7 +13285,7 @@ Raw sandbox event stream collected for this iteration. > **startedAt**: `number` -Defined in: [runtime/types.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L212) +Defined in: [runtime/types.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L210) **`Experimental`** @@ -12650,7 +13293,7 @@ Defined in: [runtime/types.ts:212](https://github.com/tangle-network/agent-runti > **endedAt**: `number` -Defined in: [runtime/types.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L213) +Defined in: [runtime/types.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L211) **`Experimental`** @@ -12658,7 +13301,7 @@ Defined in: [runtime/types.ts:213](https://github.com/tangle-network/agent-runti > **costUsd**: `number` -Defined in: [runtime/types.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L214) +Defined in: [runtime/types.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L212) **`Experimental`** @@ -12666,7 +13309,7 @@ Defined in: [runtime/types.ts:214](https://github.com/tangle-network/agent-runti > **tokenUsage**: [`LoopTokenUsage`](#looptokenusage) -Defined in: [runtime/types.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L216) +Defined in: [runtime/types.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L214) **`Experimental`** @@ -12676,7 +13319,7 @@ Summed LLM token usage across every `llm_call` event in this iteration. ### Driver -Defined in: [runtime/types.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L220) +Defined in: [runtime/types.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L218) **`Experimental`** @@ -12700,7 +13343,7 @@ Defined in: [runtime/types.ts:220](https://github.com/tangle-network/agent-runti > `readonly` `optional` **name?**: `string` -Defined in: [runtime/types.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L224) +Defined in: [runtime/types.ts:222](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L222) **`Experimental`** @@ -12712,7 +13355,7 @@ Stable identifier surfaced in trace events. Default `'driver'`. > **plan**(`task`, `history`): `Promise`\<`Task`[]\> -Defined in: [runtime/types.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L229) +Defined in: [runtime/types.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L227) **`Experimental`** @@ -12737,7 +13380,7 @@ readonly [`Iteration`](#iteration-1)\<`Task`, `Output`\>[] > **decide**(`history`): `Decision` \| `Promise`\<`Decision`\> -Defined in: [runtime/types.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L236) +Defined in: [runtime/types.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L234) **`Experimental`** @@ -12760,7 +13403,7 @@ readonly [`Iteration`](#iteration-1)\<`Task`, `Output`\>[] > `optional` **describePlan**(): [`LoopPlanDescription`](#loopplandescription) \| `undefined` -Defined in: [runtime/types.ts:246](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L246) +Defined in: [runtime/types.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L244) **`Experimental`** @@ -12780,7 +13423,7 @@ own topology returns its chosen move's kind + rationale here. > `optional` **selectWinner**(`history`): [`LoopWinner`](#loopwinner)\<`Task`, `Output`\> \| `undefined` -Defined in: [runtime/types.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L256) +Defined in: [runtime/types.ts:254](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L254) **`Experimental`** @@ -12805,7 +13448,7 @@ readonly [`Iteration`](#iteration-1)\<`Task`, `Output`\>[] ### LoopPlanDescription -Defined in: [runtime/types.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L262) +Defined in: [runtime/types.ts:260](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L260) **`Experimental`** @@ -12817,7 +13460,7 @@ Driver-supplied description of the just-planned move. > **kind**: `string` -Defined in: [runtime/types.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L264) +Defined in: [runtime/types.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L262) **`Experimental`** @@ -12827,7 +13470,7 @@ Topology move this round — e.g. `'refine' | 'fanout' | 'verify' | 'stop'`. > `optional` **rationale?**: `string` -Defined in: [runtime/types.ts:266](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L266) +Defined in: [runtime/types.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L264) **`Experimental`** @@ -12837,7 +13480,7 @@ Why the driver chose this move (the agent's rationale), when available. > `optional` **parentIndex?**: `number` -Defined in: [runtime/types.ts:273](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L273) +Defined in: [runtime/types.ts:271](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L271) **`Experimental`** @@ -12850,7 +13493,7 @@ Omit to keep the inferred (best-valid / latest) branch point. ### LoopWinner -Defined in: [runtime/types.ts:277](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L277) +Defined in: [runtime/types.ts:275](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L275) **`Experimental`** @@ -12870,7 +13513,7 @@ Defined in: [runtime/types.ts:277](https://github.com/tangle-network/agent-runti > **task**: `Task` -Defined in: [runtime/types.ts:278](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L278) +Defined in: [runtime/types.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L276) **`Experimental`** @@ -12878,7 +13521,7 @@ Defined in: [runtime/types.ts:278](https://github.com/tangle-network/agent-runti > **output**: `Output` -Defined in: [runtime/types.ts:279](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L279) +Defined in: [runtime/types.ts:277](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L277) **`Experimental`** @@ -12886,7 +13529,7 @@ Defined in: [runtime/types.ts:279](https://github.com/tangle-network/agent-runti > `optional` **verdict?**: `DefaultVerdict` -Defined in: [runtime/types.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L280) +Defined in: [runtime/types.ts:278](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L278) **`Experimental`** @@ -12894,7 +13537,7 @@ Defined in: [runtime/types.ts:280](https://github.com/tangle-network/agent-runti > **iterationIndex**: `number` -Defined in: [runtime/types.ts:281](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L281) +Defined in: [runtime/types.ts:279](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L279) **`Experimental`** @@ -12902,7 +13545,7 @@ Defined in: [runtime/types.ts:281](https://github.com/tangle-network/agent-runti > **agentRunName**: `string` -Defined in: [runtime/types.ts:282](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L282) +Defined in: [runtime/types.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L280) **`Experimental`** @@ -12910,7 +13553,7 @@ Defined in: [runtime/types.ts:282](https://github.com/tangle-network/agent-runti ### LoopResult -Defined in: [runtime/types.ts:286](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L286) +Defined in: [runtime/types.ts:284](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L284) **`Experimental`** @@ -12934,7 +13577,7 @@ Defined in: [runtime/types.ts:286](https://github.com/tangle-network/agent-runti > **decision**: `Decision` -Defined in: [runtime/types.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L287) +Defined in: [runtime/types.ts:285](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L285) **`Experimental`** @@ -12942,7 +13585,7 @@ Defined in: [runtime/types.ts:287](https://github.com/tangle-network/agent-runti > **iterations**: [`Iteration`](#iteration-1)\<`Task`, `Output`\>[] -Defined in: [runtime/types.ts:288](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L288) +Defined in: [runtime/types.ts:286](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L286) **`Experimental`** @@ -12950,7 +13593,7 @@ Defined in: [runtime/types.ts:288](https://github.com/tangle-network/agent-runti > `optional` **winner?**: [`LoopWinner`](#loopwinner)\<`Task`, `Output`\> -Defined in: [runtime/types.ts:289](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L289) +Defined in: [runtime/types.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L287) **`Experimental`** @@ -12958,7 +13601,7 @@ Defined in: [runtime/types.ts:289](https://github.com/tangle-network/agent-runti > **durationMs**: `number` -Defined in: [runtime/types.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L290) +Defined in: [runtime/types.ts:288](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L288) **`Experimental`** @@ -12966,7 +13609,7 @@ Defined in: [runtime/types.ts:290](https://github.com/tangle-network/agent-runti > **costUsd**: `number` -Defined in: [runtime/types.ts:292](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L292) +Defined in: [runtime/types.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L290) **`Experimental`** @@ -12976,19 +13619,18 @@ Sum of every iteration's `costUsd`. > **tokenUsage**: [`LoopTokenUsage`](#looptokenusage) -Defined in: [runtime/types.ts:296](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L296) +Defined in: [runtime/types.ts:293](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L293) **`Experimental`** -Sum of every iteration's token usage. Forward to - `ctx.cost.observeTokens` in a `runProfileMatrix` dispatch so the - integrity guard sees real LLM activity. +Sum of every iteration's token usage. `loopDispatch` commits it through + the campaign's paid-call receipt. ##### provenance > **provenance**: [`RunProvenance`](#runprovenance) -Defined in: [runtime/types.ts:300](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L300) +Defined in: [runtime/types.ts:297](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L297) **`Experimental`** @@ -13000,7 +13642,7 @@ Domain-free run provenance for auditability: the mount manifest recorded ### SandboxClient -Defined in: [runtime/types.ts:316](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L316) +Defined in: [runtime/types.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L313) **`Experimental`** @@ -13020,7 +13662,7 @@ the kernel falls back to `{ placement: 'sibling', sandboxId: box.id }`. > **create**(`options?`): `Promise`\<`SandboxInstance`\> -Defined in: [runtime/types.ts:317](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L317) +Defined in: [runtime/types.ts:314](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L314) **`Experimental`** @@ -13038,7 +13680,7 @@ Defined in: [runtime/types.ts:317](https://github.com/tangle-network/agent-runti > `optional` **describePlacement**(`box`): [`LoopSandboxPlacement`](#loopsandboxplacement) -Defined in: [runtime/types.ts:318](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L318) +Defined in: [runtime/types.ts:315](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L315) **`Experimental`** @@ -13056,7 +13698,7 @@ Defined in: [runtime/types.ts:318](https://github.com/tangle-network/agent-runti > `optional` **criuStatus**(): `Promise`\<\{ `available`: `boolean`; `criuVersion?`: `string`; `reason?`: `string`; \}\> -Defined in: [runtime/types.ts:329](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L329) +Defined in: [runtime/types.ts:326](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L326) **`Experimental`** @@ -13076,7 +13718,7 @@ The raw `Sandbox` SDK class satisfies it; the loop's test fakes omit it ### LoopLineageOptions -Defined in: [runtime/types.ts:353](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L353) +Defined in: [runtime/types.ts:350](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L350) **`Experimental`** @@ -13104,7 +13746,7 @@ are copy-on-write, but each is still a live box until loop end). > `optional` **sessionContinuity?**: `boolean` -Defined in: [runtime/types.ts:368](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L368) +Defined in: [runtime/types.ts:365](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L365) **`Experimental`** @@ -13125,7 +13767,7 @@ proves the session EXISTS server-side, not that prior turns replay into it. > `optional` **forkFanout?**: `boolean` -Defined in: [runtime/types.ts:383](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L383) +Defined in: [runtime/types.ts:380](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L380) **`Experimental`** @@ -13146,7 +13788,7 @@ different-per-branch profiles use the unforked fanout path. > `optional` **streaming?**: `"sse"` \| `"poll"` -Defined in: [runtime/types.ts:395](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L395) +Defined in: [runtime/types.ts:392](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L392) **`Experimental`** @@ -13164,7 +13806,7 @@ idle-drop. Applies to the default fresh-box path too, not only when ### LoopSandboxPlacement -Defined in: [runtime/types.ts:399](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L399) +Defined in: [runtime/types.ts:396](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L396) **`Experimental`** @@ -13178,7 +13820,7 @@ Defined in: [runtime/types.ts:399](https://github.com/tangle-network/agent-runti > **kind**: `"sibling"` \| `"fleet"` -Defined in: [runtime/types.ts:400](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L400) +Defined in: [runtime/types.ts:397](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L397) **`Experimental`** @@ -13186,7 +13828,7 @@ Defined in: [runtime/types.ts:400](https://github.com/tangle-network/agent-runti > `optional` **sandboxId?**: `string` -Defined in: [runtime/types.ts:401](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L401) +Defined in: [runtime/types.ts:398](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L398) **`Experimental`** @@ -13194,7 +13836,7 @@ Defined in: [runtime/types.ts:401](https://github.com/tangle-network/agent-runti > `optional` **fleetId?**: `string` -Defined in: [runtime/types.ts:402](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L402) +Defined in: [runtime/types.ts:399](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L399) **`Experimental`** @@ -13202,7 +13844,7 @@ Defined in: [runtime/types.ts:402](https://github.com/tangle-network/agent-runti > `optional` **machineId?**: `string` -Defined in: [runtime/types.ts:403](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L403) +Defined in: [runtime/types.ts:400](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L400) **`Experimental`** @@ -13210,7 +13852,7 @@ Defined in: [runtime/types.ts:403](https://github.com/tangle-network/agent-runti ### LoopTraceEmitter -Defined in: [runtime/types.ts:407](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L407) +Defined in: [runtime/types.ts:404](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L404) **`Experimental`** @@ -13220,7 +13862,7 @@ Defined in: [runtime/types.ts:407](https://github.com/tangle-network/agent-runti > **emit**(`event`): `void` \| `Promise`\<`void`\> -Defined in: [runtime/types.ts:408](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L408) +Defined in: [runtime/types.ts:405](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L405) **`Experimental`** @@ -13238,7 +13880,7 @@ Defined in: [runtime/types.ts:408](https://github.com/tangle-network/agent-runti ### LoopStartedPayload -Defined in: [runtime/types.ts:443](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L443) +Defined in: [runtime/types.ts:440](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L440) **`Experimental`** @@ -13248,7 +13890,7 @@ Defined in: [runtime/types.ts:443](https://github.com/tangle-network/agent-runti > **driver**: `string` -Defined in: [runtime/types.ts:444](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L444) +Defined in: [runtime/types.ts:441](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L441) **`Experimental`** @@ -13256,7 +13898,7 @@ Defined in: [runtime/types.ts:444](https://github.com/tangle-network/agent-runti > **agentRunNames**: `string`[] -Defined in: [runtime/types.ts:445](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L445) +Defined in: [runtime/types.ts:442](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L442) **`Experimental`** @@ -13264,7 +13906,7 @@ Defined in: [runtime/types.ts:445](https://github.com/tangle-network/agent-runti > **maxIterations**: `number` -Defined in: [runtime/types.ts:446](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L446) +Defined in: [runtime/types.ts:443](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L443) **`Experimental`** @@ -13272,7 +13914,7 @@ Defined in: [runtime/types.ts:446](https://github.com/tangle-network/agent-runti > **maxConcurrency**: `number` -Defined in: [runtime/types.ts:447](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L447) +Defined in: [runtime/types.ts:444](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L444) **`Experimental`** @@ -13280,7 +13922,7 @@ Defined in: [runtime/types.ts:447](https://github.com/tangle-network/agent-runti ### LoopPlanPayload -Defined in: [runtime/types.ts:458](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L458) +Defined in: [runtime/types.ts:455](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L455) **`Experimental`** @@ -13295,7 +13937,7 @@ provided, else inferred from `plannedCount` (0→stop, 1→refine, N→fanout). > **roundIndex**: `number` -Defined in: [runtime/types.ts:460](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L460) +Defined in: [runtime/types.ts:457](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L457) **`Experimental`** @@ -13305,7 +13947,7 @@ Defined in: [runtime/types.ts:460](https://github.com/tangle-network/agent-runti > **plannedCount**: `number` -Defined in: [runtime/types.ts:462](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L462) +Defined in: [runtime/types.ts:459](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L459) **`Experimental`** @@ -13315,7 +13957,7 @@ Tasks the driver issued this round. > **moveKind**: `string` -Defined in: [runtime/types.ts:464](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L464) +Defined in: [runtime/types.ts:461](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L461) **`Experimental`** @@ -13325,7 +13967,7 @@ Topology move — `'refine' | 'fanout' | 'verify' | 'stop'` etc. > `optional` **rationale?**: `string` -Defined in: [runtime/types.ts:466](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L466) +Defined in: [runtime/types.ts:463](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L463) **`Experimental`** @@ -13335,7 +13977,7 @@ Driver rationale for the move, when available. > `optional` **parentIndex?**: `number` -Defined in: [runtime/types.ts:472](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L472) +Defined in: [runtime/types.ts:469](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L469) **`Experimental`** @@ -13347,7 +13989,7 @@ latest) iteration so far — unless a driver later declares it explicitly. > **childIndices**: `number`[] -Defined in: [runtime/types.ts:474](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L474) +Defined in: [runtime/types.ts:471](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L471) **`Experimental`** @@ -13357,7 +13999,7 @@ Iteration indices this round dispatched (the edge targets). ### LoopIterationStartedPayload -Defined in: [runtime/types.ts:478](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L478) +Defined in: [runtime/types.ts:475](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L475) **`Experimental`** @@ -13367,7 +14009,7 @@ Defined in: [runtime/types.ts:478](https://github.com/tangle-network/agent-runti > **iterationIndex**: `number` -Defined in: [runtime/types.ts:479](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L479) +Defined in: [runtime/types.ts:476](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L476) **`Experimental`** @@ -13375,7 +14017,7 @@ Defined in: [runtime/types.ts:479](https://github.com/tangle-network/agent-runti > **agentRunName**: `string` -Defined in: [runtime/types.ts:480](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L480) +Defined in: [runtime/types.ts:477](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L477) **`Experimental`** @@ -13383,7 +14025,7 @@ Defined in: [runtime/types.ts:480](https://github.com/tangle-network/agent-runti > **taskHash**: `string` -Defined in: [runtime/types.ts:481](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L481) +Defined in: [runtime/types.ts:478](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L478) **`Experimental`** @@ -13391,7 +14033,7 @@ Defined in: [runtime/types.ts:481](https://github.com/tangle-network/agent-runti > `optional` **groupId?**: `number` -Defined in: [runtime/types.ts:483](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L483) +Defined in: [runtime/types.ts:480](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L480) **`Experimental`** @@ -13401,7 +14043,7 @@ Plan round (== `LoopPlanPayload.roundIndex`) this iteration belongs to. > `optional` **parentIndex?**: `number` -Defined in: [runtime/types.ts:485](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L485) +Defined in: [runtime/types.ts:482](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L482) **`Experimental`** @@ -13411,7 +14053,7 @@ Iteration this one was planned from; `undefined` ⇒ root. ### LoopIterationDispatchPayload -Defined in: [runtime/types.ts:496](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L496) +Defined in: [runtime/types.ts:493](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L493) **`Experimental`** @@ -13426,7 +14068,7 @@ they write lands on it directly. > **iterationIndex**: `number` -Defined in: [runtime/types.ts:497](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L497) +Defined in: [runtime/types.ts:494](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L494) **`Experimental`** @@ -13434,7 +14076,7 @@ Defined in: [runtime/types.ts:497](https://github.com/tangle-network/agent-runti > **agentRunName**: `string` -Defined in: [runtime/types.ts:498](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L498) +Defined in: [runtime/types.ts:495](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L495) **`Experimental`** @@ -13442,7 +14084,7 @@ Defined in: [runtime/types.ts:498](https://github.com/tangle-network/agent-runti > **placement**: `"sibling"` \| `"fleet"` -Defined in: [runtime/types.ts:499](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L499) +Defined in: [runtime/types.ts:496](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L496) **`Experimental`** @@ -13450,7 +14092,7 @@ Defined in: [runtime/types.ts:499](https://github.com/tangle-network/agent-runti > `optional` **sandboxId?**: `string` -Defined in: [runtime/types.ts:501](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L501) +Defined in: [runtime/types.ts:498](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L498) **`Experimental`** @@ -13460,7 +14102,7 @@ Set on every placement. Lets analyst loops correlate per-iteration logs. > `optional` **fleetId?**: `string` -Defined in: [runtime/types.ts:503](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L503) +Defined in: [runtime/types.ts:500](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L500) **`Experimental`** @@ -13470,7 +14112,7 @@ Set only when `placement === 'fleet'`. > `optional` **machineId?**: `string` -Defined in: [runtime/types.ts:505](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L505) +Defined in: [runtime/types.ts:502](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L502) **`Experimental`** @@ -13480,7 +14122,7 @@ Set only when `placement === 'fleet'`. > `optional` **groupId?**: `number` -Defined in: [runtime/types.ts:507](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L507) +Defined in: [runtime/types.ts:504](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L504) **`Experimental`** @@ -13490,7 +14132,7 @@ Plan round this iteration belongs to. > `optional` **parentIndex?**: `number` -Defined in: [runtime/types.ts:509](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L509) +Defined in: [runtime/types.ts:506](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L506) **`Experimental`** @@ -13500,7 +14142,7 @@ Iteration this one was planned from; `undefined` ⇒ root. ### LoopIterationEndedPayload -Defined in: [runtime/types.ts:513](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L513) +Defined in: [runtime/types.ts:510](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L510) **`Experimental`** @@ -13510,7 +14152,7 @@ Defined in: [runtime/types.ts:513](https://github.com/tangle-network/agent-runti > **iterationIndex**: `number` -Defined in: [runtime/types.ts:514](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L514) +Defined in: [runtime/types.ts:511](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L511) **`Experimental`** @@ -13518,7 +14160,7 @@ Defined in: [runtime/types.ts:514](https://github.com/tangle-network/agent-runti > **agentRunName**: `string` -Defined in: [runtime/types.ts:515](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L515) +Defined in: [runtime/types.ts:512](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L512) **`Experimental`** @@ -13526,7 +14168,7 @@ Defined in: [runtime/types.ts:515](https://github.com/tangle-network/agent-runti > `optional` **outputHash?**: `string` -Defined in: [runtime/types.ts:516](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L516) +Defined in: [runtime/types.ts:513](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L513) **`Experimental`** @@ -13534,7 +14176,7 @@ Defined in: [runtime/types.ts:516](https://github.com/tangle-network/agent-runti > `optional` **verdict?**: `DefaultVerdict` -Defined in: [runtime/types.ts:517](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L517) +Defined in: [runtime/types.ts:514](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L514) **`Experimental`** @@ -13542,7 +14184,7 @@ Defined in: [runtime/types.ts:517](https://github.com/tangle-network/agent-runti > `optional` **error?**: `string` -Defined in: [runtime/types.ts:518](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L518) +Defined in: [runtime/types.ts:515](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L515) **`Experimental`** @@ -13550,7 +14192,7 @@ Defined in: [runtime/types.ts:518](https://github.com/tangle-network/agent-runti > **costUsd**: `number` -Defined in: [runtime/types.ts:519](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L519) +Defined in: [runtime/types.ts:516](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L516) **`Experimental`** @@ -13558,7 +14200,7 @@ Defined in: [runtime/types.ts:519](https://github.com/tangle-network/agent-runti > **durationMs**: `number` -Defined in: [runtime/types.ts:520](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L520) +Defined in: [runtime/types.ts:517](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L517) **`Experimental`** @@ -13566,7 +14208,7 @@ Defined in: [runtime/types.ts:520](https://github.com/tangle-network/agent-runti > `optional` **tokenUsage?**: [`LoopTokenUsage`](#looptokenusage) -Defined in: [runtime/types.ts:523](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L523) +Defined in: [runtime/types.ts:520](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L520) **`Experimental`** @@ -13577,7 +14219,7 @@ Summed LLM token usage for this iteration — maps to gen_ai.usage.* on the > `optional` **groupId?**: `number` -Defined in: [runtime/types.ts:525](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L525) +Defined in: [runtime/types.ts:522](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L522) **`Experimental`** @@ -13587,7 +14229,7 @@ Plan round this iteration belongs to. > `optional` **parentIndex?**: `number` -Defined in: [runtime/types.ts:527](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L527) +Defined in: [runtime/types.ts:524](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L524) **`Experimental`** @@ -13597,7 +14239,7 @@ Iteration this one was planned from; `undefined` ⇒ root. > `optional` **outputPreview?**: `string` -Defined in: [runtime/types.ts:530](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L530) +Defined in: [runtime/types.ts:527](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L527) **`Experimental`** @@ -13608,7 +14250,7 @@ Truncated string preview of the parsed output — for a viewer's drawer. ### LoopDecisionPayload -Defined in: [runtime/types.ts:534](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L534) +Defined in: [runtime/types.ts:531](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L531) **`Experimental`** @@ -13618,7 +14260,7 @@ Defined in: [runtime/types.ts:534](https://github.com/tangle-network/agent-runti > **decision**: `string` -Defined in: [runtime/types.ts:535](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L535) +Defined in: [runtime/types.ts:532](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L532) **`Experimental`** @@ -13626,7 +14268,7 @@ Defined in: [runtime/types.ts:535](https://github.com/tangle-network/agent-runti > **historyLength**: `number` -Defined in: [runtime/types.ts:536](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L536) +Defined in: [runtime/types.ts:533](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L533) **`Experimental`** @@ -13634,7 +14276,7 @@ Defined in: [runtime/types.ts:536](https://github.com/tangle-network/agent-runti ### LoopEndedPayload -Defined in: [runtime/types.ts:540](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L540) +Defined in: [runtime/types.ts:537](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L537) **`Experimental`** @@ -13644,7 +14286,7 @@ Defined in: [runtime/types.ts:540](https://github.com/tangle-network/agent-runti > `optional` **winnerIterationIndex?**: `number` -Defined in: [runtime/types.ts:541](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L541) +Defined in: [runtime/types.ts:538](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L538) **`Experimental`** @@ -13652,7 +14294,7 @@ Defined in: [runtime/types.ts:541](https://github.com/tangle-network/agent-runti > **totalCostUsd**: `number` -Defined in: [runtime/types.ts:542](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L542) +Defined in: [runtime/types.ts:539](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L539) **`Experimental`** @@ -13660,7 +14302,7 @@ Defined in: [runtime/types.ts:542](https://github.com/tangle-network/agent-runti > **durationMs**: `number` -Defined in: [runtime/types.ts:543](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L543) +Defined in: [runtime/types.ts:540](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L540) **`Experimental`** @@ -13668,7 +14310,7 @@ Defined in: [runtime/types.ts:543](https://github.com/tangle-network/agent-runti > **iterations**: `number` -Defined in: [runtime/types.ts:544](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L544) +Defined in: [runtime/types.ts:541](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L541) **`Experimental`** @@ -13676,7 +14318,7 @@ Defined in: [runtime/types.ts:544](https://github.com/tangle-network/agent-runti ### LoopTeardownFailedPayload -Defined in: [runtime/types.ts:550](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L550) +Defined in: [runtime/types.ts:547](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L547) **`Experimental`** @@ -13690,7 +14332,7 @@ Emitted when a box's `delete()` throws or times out during teardown — the > `optional` **sandboxId?**: `string` -Defined in: [runtime/types.ts:551](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L551) +Defined in: [runtime/types.ts:548](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L548) **`Experimental`** @@ -13698,7 +14340,7 @@ Defined in: [runtime/types.ts:551](https://github.com/tangle-network/agent-runti > **reason**: `string` -Defined in: [runtime/types.ts:553](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L553) +Defined in: [runtime/types.ts:550](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L550) **`Experimental`** @@ -13708,7 +14350,7 @@ Defined in: [runtime/types.ts:553](https://github.com/tangle-network/agent-runti ### ExecCtx -Defined in: [runtime/types.ts:561](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L561) +Defined in: [runtime/types.ts:558](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L558) **`Experimental`** @@ -13720,7 +14362,7 @@ Execution context for `runLoop`: the sandbox client the kernel creates boxes thr > **sandboxClient**: [`SandboxClient`](#sandboxclient-3) -Defined in: [runtime/types.ts:563](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L563) +Defined in: [runtime/types.ts:560](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L560) **`Experimental`** @@ -13730,7 +14372,7 @@ Sandbox SDK client — the kernel calls `.create()` per iteration. > `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) -Defined in: [runtime/types.ts:565](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L565) +Defined in: [runtime/types.ts:562](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L562) **`Experimental`** @@ -13740,7 +14382,7 @@ Optional runtime hooks. Execution-scoped; never part of `AgentProfile`. > `optional` **traceEmitter?**: [`LoopTraceEmitter`](#looptraceemitter) -Defined in: [runtime/types.ts:567](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L567) +Defined in: [runtime/types.ts:564](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L564) **`Experimental`** @@ -13750,7 +14392,7 @@ Optional trace emitter. When set, the kernel emits `loop.*` events. > `optional` **onSandboxEvent?**: (`event`, `meta`) => `void` \| `PromiseLike`\<`void`\> -Defined in: [runtime/types.ts:585](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L585) +Defined in: [runtime/types.ts:582](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L582) **`Experimental`** @@ -13792,7 +14434,7 @@ on that. > `optional` **runHandle?**: [`RuntimeRunHandle`](index.md#runtimerunhandle) -Defined in: [runtime/types.ts:594](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L594) +Defined in: [runtime/types.ts:591](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L591) **`Experimental`** @@ -13804,7 +14446,7 @@ the kernel infers from a sandbox event stream is forwarded via > `optional` **signal?**: `AbortSignal` -Defined in: [runtime/types.ts:596](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L596) +Defined in: [runtime/types.ts:593](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L593) **`Experimental`** @@ -13814,7 +14456,7 @@ Cooperative cancellation signal. > `optional` **traceId?**: `string` -Defined in: [runtime/types.ts:602](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L602) +Defined in: [runtime/types.ts:599](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L599) **`Experimental`** @@ -13826,7 +14468,7 @@ inherited from TRACE_ID env var in MCP subprocess mode. > `optional` **parentSpanId?**: `string` -Defined in: [runtime/types.ts:607](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L607) +Defined in: [runtime/types.ts:604](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L604) **`Experimental`** @@ -14286,7 +14928,7 @@ async iterable for streaming. The callback may also write files into > **LoopOptionsForDispatch**\<`Task`, `Output`, `Decision`\> = `Omit`\<`RunLoopOptions`\<`Task`, `Output`, `Decision`\>, `"ctx"`\> -Defined in: [runtime/loop-dispatch.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L45) +Defined in: [runtime/loop-dispatch.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L44) runLoop options minus the `ctx` (loopDispatch builds the ctx). @@ -15058,6 +15700,14 @@ any custom backend): the turn is one `backend.stream()` call. *** +### RepairStop + +> **RepairStop** = `"already-passing"` \| `"no-signal"` \| `"repaired-pass"` \| `"rounds-exhausted"` \| `"no-candidates"` + +Defined in: [runtime/structural-rollout.ts:479](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L479) + +*** + ### BudgetReadout > **BudgetReadout** = `Readonly`\<\{ `tokensLeft`: `number`; `usdLeft`: `number`; `usdCapped`: `boolean`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> @@ -15076,7 +15726,7 @@ Post-reservation pool readout — the shape `Scope.budget` exposes. `tokensLeft` > **ExecutorConfig** = `object` & `RouterSeam` \| `object` & `RouterToolsSeam` \| `object` & `BridgeSeam` \| `object` & `CliSeam` \| `object` & `CliWorktreeSeam` \| `object` & [`ProviderSeam`](#providerseam) \| `object` & `SandboxSeam` -Defined in: [runtime/supervise/runtime.ts:1534](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L1534) +Defined in: [runtime/supervise/runtime.ts:1540](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L1540) Config for [createExecutor](#createexecutor): the backend is DATA — the cost dial a profile, an experiment config, or a replay journal can name — not an import choice. Each @@ -15180,7 +15830,7 @@ user supply construction args without pre-instantiating. > **Settled**\<`Out`\> = \{ `kind`: `"done"`; `handle`: `Handle`\<`Out`\>; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `seq`: `number`; \} \| \{ `kind`: `"down"`; `handle`: `Handle`\<`Out`\>; `reason`: `string`; `infra`: `boolean`; `restartCount`: `number`; `seq`: `number`; \} -Defined in: [runtime/supervise/types.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L256) +Defined in: [runtime/supervise/types.ts:259](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L259) A settled child, delivered by `scope.next()`. `seq` is the monotonic cursor order `next()` yielded this settlement (B2) — NOT wall-clock — and replay delivers strictly @@ -15236,7 +15886,7 @@ True = infrastructure failure (excluded from merge `n` / equal-k), not a bad res > **SupervisedResult**\<`Out`\> = \{ `kind`: `"winner"`; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](#treeview); `spentTotal`: [`Spend`](#spend); `spentBreakdown?`: \{ `driverInference`: [`Spend`](#spend); `childWork`: [`Spend`](#spend); \}; \} \| \{ `kind`: `"no-winner"`; `reason`: `"all-children-down"` \| `"budget-exhausted"` \| `"aborted"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); \} -Defined in: [runtime/supervise/types.ts:467](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L467) +Defined in: [runtime/supervise/types.ts:470](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/types.ts#L470) Typed terminal result (M2) — a no-winner is NEVER coerced to a best-effort output. @@ -15328,7 +15978,7 @@ The conserved spend incurred before the run failed — real cost is paid even wh > **WorktreePatchArtifact** = `WorktreeHarnessResult` -Defined in: [runtime/supervise/worktree-cli-executor.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L44) +Defined in: [runtime/supervise/worktree-cli-executor.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L42) Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured diff + the harness's run record + the derived checks). @@ -15381,7 +16031,7 @@ Public supervisor-facing compaction config: same knobs as the primitive, but `di > **MountRecorder** = (`entry`) => `void` -Defined in: [runtime/types.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L198) +Defined in: [runtime/types.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L196) **`Experimental`** @@ -15405,7 +16055,7 @@ declares what it mounted without the kernel having to inspect box contents. > **LoopTraceEvent** = \{ `kind`: `"loop.started"`; `runId`: `string`; `timestamp`: `number`; `payload`: [`LoopStartedPayload`](#loopstartedpayload); \} \| \{ `kind`: `"loop.plan"`; `runId`: `string`; `timestamp`: `number`; `payload`: [`LoopPlanPayload`](#loopplanpayload); \} \| \{ `kind`: `"loop.iteration.started"`; `runId`: `string`; `timestamp`: `number`; `payload`: [`LoopIterationStartedPayload`](#loopiterationstartedpayload); \} \| \{ `kind`: `"loop.iteration.dispatch"`; `runId`: `string`; `timestamp`: `number`; `payload`: [`LoopIterationDispatchPayload`](#loopiterationdispatchpayload); \} \| \{ `kind`: `"loop.iteration.ended"`; `runId`: `string`; `timestamp`: `number`; `payload`: [`LoopIterationEndedPayload`](#loopiterationendedpayload); \} \| \{ `kind`: `"loop.decision"`; `runId`: `string`; `timestamp`: `number`; `payload`: [`LoopDecisionPayload`](#loopdecisionpayload); \} \| \{ `kind`: `"loop.ended"`; `runId`: `string`; `timestamp`: `number`; `payload`: [`LoopEndedPayload`](#loopendedpayload); \} \| \{ `kind`: `"loop.teardown.failed"`; `runId`: `string`; `timestamp`: `number`; `payload`: [`LoopTeardownFailedPayload`](#loopteardownfailedpayload); \} -Defined in: [runtime/types.ts:412](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L412) +Defined in: [runtime/types.ts:409](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L409) **`Experimental`** @@ -15496,7 +16146,7 @@ The compressed consumable a skill carries: everything an author needs to emit a > `const` **sample**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:755](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L755) +Defined in: [runtime/strategy.ts:769](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L769) Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N / resample). @@ -15506,7 +16156,7 @@ Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N > `const` **refine**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:760](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L760) +Defined in: [runtime/strategy.ts:774](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L774) Built-in `Strategy`: attempt → `observe()` reads the trace → steer the next attempt → repeat (deepen one lineage). @@ -15516,7 +16166,7 @@ Built-in `Strategy`: attempt → `observe()` reads the trace → steer the next > `const` **adaptiveRefine**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:960](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L960) +Defined in: [runtime/strategy.ts:974](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L974) A NEW strategy, authored from the steps (~20 lines): refine, but when a steered shot fails to improve the score it ABANDONS that line and restarts fresh (branch-when-stuck) @@ -15530,7 +16180,7 @@ A NEW strategy, authored from the steps (~20 lines): refine, but when a steered > `const` **sampleThenRefine**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:1003](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1003) +Defined in: [runtime/strategy.ts:1017](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1017) The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept open), then refine the best-verifying line with the remaining budget. Sample's basin escape + @@ -15538,6 +16188,16 @@ The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept *** +### defaultStructuralRolloutPolicy + +> `const` **defaultStructuralRolloutPolicy**: [`StructuralRolloutPolicy`](#structuralrolloutpolicy) + +Defined in: [runtime/structural-rollout.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L60) + +The measured default recipe: 5 samples, 2 guarded repair rounds, 6 authored checks. + +*** + ### defaultProfileRichnessThresholds > `const` **defaultProfileRichnessThresholds**: [`ProfileRichnessThresholds`](#profilerichnessthresholds) @@ -15564,7 +16224,7 @@ The conserved pool a `delegate()` call applies when the caller does not pass its > `const` **cliWorktreeExecutor**: [`ExecutorFactory`](#executorfactory)\<`unknown`\> -Defined in: [runtime/supervise/runtime.ts:1498](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L1498) +Defined in: [runtime/supervise/runtime.ts:1502](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L1502) The leaf `createWorktreeCliExecutor` as a backend-as-data factory: a supervisor-authored `AgentProfile` driving claude / codex / opencode on its own worktree. `budgetExempt` like @@ -15929,7 +16589,7 @@ passes. Ground truth — the driver ends directly, no validation. The check read > **defineLeaderboard**\<`TCase`, `TArtifact`\>(`spec`): [`DefinedLeaderboard`](#definedleaderboard)\<`TCase`, `TArtifact`\> -Defined in: [runtime/define-leaderboard.ts:299](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L299) +Defined in: [runtime/define-leaderboard.ts:305](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/define-leaderboard.ts#L305) Assemble a declarative spec (`cases` + `prompt` + `score`) into a runnable harness×model leaderboard — `run()` executes the matrix, `toBenchmarkAdapter()` @@ -16028,7 +16688,7 @@ run once on the prompt, emit the terminal result event, tear down. > **loopCampaignDispatch**\<`Task`, `Output`, `Decision`, `TScenario`, `TArtifact`\>(`opts`): `DispatchFn`\<`TScenario`, `TArtifact`\> -Defined in: [runtime/loop-dispatch.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L148) +Defined in: [runtime/loop-dispatch.ts:217](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L217) Adapter for plain `runCampaign` scenarios. This is the runtime-side pair for agent-eval fixture scenarios: load fixtures in `agent-eval/campaign`, build @@ -16072,7 +16732,7 @@ the runtime loop here, and keep cost + token + trace reporting automatic. > **loopDispatch**\<`Task`, `Output`, `Decision`, `TScenario`, `TArtifact`\>(`opts`): `ProfileDispatchFn`\<`TScenario`, `TArtifact`\> -Defined in: [runtime/loop-dispatch.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L159) +Defined in: [runtime/loop-dispatch.ts:240](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/loop-dispatch.ts#L240) Adapter for `runProfileMatrix` (profile is an axis). Returns a `ProfileDispatchFn` that runs `runLoop` per (profile, scenario) cell and @@ -16923,7 +17583,7 @@ One OpenAI-compatible chat completion through the Tangle router, returning text ###### reasoningEffort? -`"none"` \| `"high"` \| `"medium"` \| `"low"` +`"none"` \| `"low"` \| `"medium"` \| `"high"` Reasoning control for thinking models, forwarded as `reasoning_effort`. 'none' is the load-bearing value: binary/single-token decisions (routing, @@ -17291,16 +17951,17 @@ RuntimeStreamEvent & \{ type: "llm\_call"; \} \| `undefined` > **sumSandboxUsage**(`events`, `agentRunName?`): `object` -Defined in: [runtime/sandbox-events.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L90) +Defined in: [runtime/sandbox-events.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L91) Sum the token usage + USD cost of a sandbox turn's events — the one honest way to meter an `openSandboxRun` cell. Folds `extractLlmCallEvent` over the stream (which reads usage off EVERY backend event shape), so a `runProfileMatrix` dispatch can report it to `ctx.cost`: - const turn = await run.start(prompt) - const u = sumSandboxUsage(turn.events) - if (u.input || u.output) ctx.cost.observeTokens({ input: u.input, output: u.output }) - if (u.costUsd) ctx.cost.observe(u.costUsd, 'sandbox-cell') + receipt: (turn) => { + const u = sumSandboxUsage(turn.events) + return { model, inputTokens: u.input, outputTokens: u.output, + ...(u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}) } + } Without this a cell reads `{tokens:0, cost:0}` and the backend-integrity guard correctly aborts the matrix as a stub. `agentRunName` is the fallback model label for cost-only events (default `'agent'`). @@ -17337,7 +17998,7 @@ readonly `SandboxEvent`[] > **createSandboxToolPartState**(): [`SandboxToolPartState`](#sandboxtoolpartstate) -Defined in: [runtime/sandbox-events.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L160) +Defined in: [runtime/sandbox-events.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L161) **`Experimental`** @@ -17354,7 +18015,7 @@ empty call-status map so each turn projects tool frames independently. > **mapSandboxToolEvent**(`event`, `state`): [`RuntimeStreamEvent`](index.md#runtimestreamevent) & `object`[] -Defined in: [runtime/sandbox-events.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L191) +Defined in: [runtime/sandbox-events.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L192) **`Experimental`** @@ -17398,7 +18059,7 @@ Returns `[]` for every non-tool event. > **mapSandboxEvent**(`event`, `opts?`): [`RuntimeStreamEvent`](index.md#runtimestreamevent) \| `undefined` -Defined in: [runtime/sandbox-events.ts:318](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L318) +Defined in: [runtime/sandbox-events.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-events.ts#L319) Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary, for runtimes that bridge a sandbox `streamPrompt` into the @@ -17767,7 +18428,7 @@ Multi-generation strategy search: author candidates from tournament losses, play > **depthStrategy**(`surface`, `task`, `opts`, `cfg`): [`Agent`](#agent)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:616](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L616) +Defined in: [runtime/strategy.ts:630](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L630) DEPTH: one persistent artifact, carried across analyst-steered shots. @@ -17801,7 +18462,7 @@ DEPTH: one persistent artifact, carried across analyst-steered shots. > **breadthStrategy**(`_surface`, `task`, `opts`, `cfg`): [`Agent`](#agent)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:687](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L687) +Defined in: [runtime/strategy.ts:701](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L701) BREADTH: K independent rollouts (each own artifact), verifier picks the best. @@ -17835,7 +18496,7 @@ BREADTH: K independent rollouts (each own artifact), verifier picks the best. > **defineStrategy**(`name`, `run`): [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:834](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L834) +Defined in: [runtime/strategy.ts:848](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L848) Author a Strategy from the composable steps — the open, compact way. @@ -17859,7 +18520,7 @@ Author a Strategy from the composable steps — the open, compact way. > **runAgentic**(`opts`): `Promise`\<[`AgenticRunResult`](#agenticrunresult)\> -Defined in: [runtime/strategy.ts:1075](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1075) +Defined in: [runtime/strategy.ts:1089](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1089) Run a Strategy through the keystone Supervisor — `Agent.act` over a conserved-budget Scope. @@ -17935,6 +18596,306 @@ event — a stream that violates the contract must not read as an empty turn. *** +### filterAuthoredAsserts() + +> **filterAuthoredAsserts**(`reply`, `entrySymbol`, `count`): `string`[] + +Defined in: [runtime/structural-rollout.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L125) + +The proven authored-assert filter (lifted from the rigs' generateTests): keep only + single-line, paren-balanced asserts that reference the entry symbol — malformed lines + are dropped here rather than poisoning every candidate's score identically. + +#### Parameters + +##### reply + +`string` + +##### entrySymbol + +`string` + +##### count + +`number` + +#### Returns + +`string`[] + +*** + +### modelAuthoredChecks() + +> **modelAuthoredChecks**(`overrides?`): [`CheckSource`](#checksource) + +Defined in: [runtime/structural-rollout.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L149) + +Default authored-check source: one metered LLM call per task, before sampling, + filtered through `filterAuthoredAsserts`. Returns [] (no signal, never a fabricated + check) when the budget is 0, no entry symbol resolves, or the channel went down. + +#### Parameters + +##### overrides? + +###### count? + +`number` + +#### Returns + +[`CheckSource`](#checksource) + +*** + +### officialChecksFromMeta() + +> **officialChecksFromMeta**(`key?`): [`CheckSource`](#checksource) + +Defined in: [runtime/structural-rollout.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L167) + +Official checks the surface stashed on the task (e.g. MBPP's shown assert). Reads + `task.meta[key]` as a string array; anything else means no official checks. + +#### Parameters + +##### key? + +`string` = `'visibleChecks'` + +#### Returns + +[`CheckSource`](#checksource) + +*** + +### composeCheckSources() + +> **composeCheckSources**(...`sources`): [`CheckSource`](#checksource) + +Defined in: [runtime/structural-rollout.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L181) + +Concatenate check sources (official first by convention — ordering does not affect + scoring, which reads each check's `kind`). + +#### Parameters + +##### sources + +...[`CheckSource`](#checksource)[] + +#### Returns + +[`CheckSource`](#checksource) + +*** + +### resolveEntrySymbol() + +> **resolveEntrySymbol**(`task`): `string` \| `undefined` + +Defined in: [runtime/structural-rollout.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L194) + +The symbol authored checks are pinned to: `task.meta.entryPoint` when the surface + provides it, else the LAST `def name(` in the visible prompt (a code-completion stub + lists helpers first, the entry stub last). Undefined ⇒ authoring is skipped. + +#### Parameters + +##### task + +[`AgenticTask`](#agentictask) + +#### Returns + +`string` \| `undefined` + +*** + +### sandboxCheckRunner() + +> **sandboxCheckRunner**(`options?`): [`CheckRunner`](#checkrunner) + +Defined in: [runtime/structural-rollout.ts:279](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L279) + +Default CheckRunner backend: pipes the check program into `python3` over the sandbox + exec channel (`ctx.box`, or one bound at construction). Never shells out to docker + itself — the jail is the sandbox's concern. No channel ⇒ throws; it must never + silently score 0. Empty check sets short-circuit to a no-signal outcome (nothing to + execute, so no channel is required). + +#### Parameters + +##### options? + +###### box? + +[`CheckExecChannel`](#checkexecchannel) + +###### python? + +`string` + +###### timeoutMs? + +`number` + +#### Returns + +[`CheckRunner`](#checkrunner) + +*** + +### compareCheckOutcomes() + +> **compareCheckOutcomes**(`a`, `b`): `number` + +Defined in: [runtime/structural-rollout.ts:347](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L347) + +The selection order: crash < ran; then official pass-fraction; authored guesses only + break ties. Returns > 0 when `a` outranks `b`. Strictly lexicographic — on MBPP, + letting 6 noisy guesses outvote the one official check flipped selection negative. + +#### Parameters + +##### a + +[`CheckOutcome`](#checkoutcome) + +##### b + +[`CheckOutcome`](#checkoutcome) + +#### Returns + +`number` + +*** + +### visibleCheckScore() + +> **visibleCheckScore**(`o`): `number` + +Defined in: [runtime/structural-rollout.ts:360](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L360) + +Display scalar for receipts/reports (the rigs' `visibleScore` shape): crash = -1, + else official fraction + 0.001 × authored fraction. Selection itself uses the exact + lexicographic comparator, never this scalar. + +#### Parameters + +##### o + +[`CheckOutcome`](#checkoutcome) + +#### Returns + +`number` + +*** + +### selectBestIndex() + +> **selectBestIndex**(`outcomes`): `number` + +Defined in: [runtime/structural-rollout.ts:367](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L367) + +Argmax by `compareCheckOutcomes`, FIRST index wins ties (deterministic; with zero + visible coverage every candidate ties at no-signal and index 0 is the blind pick). + +#### Parameters + +##### outcomes + +readonly [`CheckOutcome`](#checkoutcome)[] + +#### Returns + +`number` + +*** + +### canDisplace() + +> **canDisplace**(`challenger`, `incumbent`): `boolean` + +Defined in: [runtime/structural-rollout.ts:382](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L382) + +The repair keep-best guard: a challenger displaces the incumbent only when it is + strictly better in the selection order AND passes at least as many official checks. + The raw-count clause is deliberate belt-and-braces over the comparator (a custom + runner can report shifted totals): repair must NEVER replace a candidate that passes + more official checks with one that passes fewer. + +#### Parameters + +##### challenger + +[`CheckOutcome`](#checkoutcome) + +##### incumbent + +[`CheckOutcome`](#checkoutcome) + +#### Returns + +`boolean` + +*** + +### defaultExtractCandidate() + +> **defaultExtractCandidate**(`messages`): `string` + +Defined in: [runtime/structural-rollout.ts:403](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L403) + +The candidate a shot produced, read from its conversation: the LAST `submit_answer` + tool-call argument (verifier environments submit the artifact explicitly), else the + latest assistant reply's fenced code block — preferring a block containing a `def`, + because repair replies echo the failure report in a bare fence BEFORE the fixed code + (the rigs' extractRepairCode lesson) — else the latest non-empty assistant text. + +#### Parameters + +##### messages + +readonly `Msg`[] + +#### Returns + +`string` + +*** + +### structuralRollout() + +> **structuralRollout**(`config?`): [`Strategy`](#strategy-3) + +Defined in: [runtime/structural-rollout.ts:525](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L525) + +Build the structuralRollout `Strategy`: k shots → score each by the frozen visible +checks (official above authored, crash lowest) → argmax with first-index tie-break → +up to `repairRounds` repair shots steered by the failure output, keep-best under the +official-check guard. Authored via `defineStrategy`, so the deliverable score stays +harness-verified and every shot is metered by the conserved pool. + +Budget note: `runAgentic`'s `budget` sizes the pool — pass at least +`k + repairRounds + 1` so the samples, repairs, and the check-author consult all admit. + +#### Parameters + +##### config? + +[`StructuralRolloutConfig`](#structuralrolloutconfig) = `{}` + +#### Returns + +[`Strategy`](#strategy-3) + +*** + ### failuresAnalyst() > **failuresAnalyst**(): [`AnalystRegistry`](#analystregistry) @@ -18534,7 +19495,7 @@ state between runs), so two runs never cross-contaminate their journals/blobs. > **createExecutor**(`config`): [`ExecutorFactory`](#executorfactory)\<`unknown`\> -Defined in: [runtime/supervise/runtime.ts:1551](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L1551) +Defined in: [runtime/supervise/runtime.ts:1557](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L1557) The single built-in executor factory. Picks a leaf backend by data (`config.backend`), injects the matching seam, and delegates to that backend's built-in implementation. @@ -18559,7 +19520,7 @@ per-vendor adapter or a closed `inline|sandbox|cli` switch — those bypass the > **createExecutorRegistry**(): [`ExecutorRegistry`](#executorregistry) -Defined in: [runtime/supervise/runtime.ts:1597](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L1597) +Defined in: [runtime/supervise/runtime.ts:1603](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/runtime.ts#L1603) The open resolver/registry. Pre-registers the three built-ins under their runtime tags (`'router'`, `'sandbox'`, `'cli'`) and accepts `register(name, @@ -18883,7 +19844,7 @@ Collect the source's spans and run the agent-eval batch analyzers over them unde > **createWorktreeCliExecutor**(`options`): [`Executor`](#executor)\<`WorktreeHarnessResult`\> -Defined in: [runtime/supervise/worktree-cli-executor.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L98) +Defined in: [runtime/supervise/worktree-cli-executor.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/supervise/worktree-cli-executor.ts#L107) **`Experimental`** diff --git a/docs/archive/go-live-plan.md b/docs/archive/go-live-plan.md index 4063b3da..9df2a765 100644 --- a/docs/archive/go-live-plan.md +++ b/docs/archive/go-live-plan.md @@ -31,7 +31,7 @@ The two replicated survivors, with provenance: | Certification rigor predicts transfer | Only a full-statistical-gate-certified artifact transfers: holdout cert-vs-prose **+31.7pp CI[+13.3,+48.3]** and **+36.7pp CI[+11.7,+66.7]**, twice | E3d (5-run, agent-lab) | | The cost flywheel | Certified memory + compression **HOLD quality at −12% to −31% cost**; steer+compress promoted ×2 on AIME; author lands near-parity at 30–55% cost, replicated ×3 | E3/E3b/E3d, cost-arc, powered run | -**Mandatory provenance rider on every public claim** (both survivors are measured on EOPS/AIME via the agent-lab harness, **not yet through `withTangleIntelligence` in a product**): +**Mandatory provenance rider on every public claim** (both survivors are measured on EOPS/AIME via the agent-lab harness, **not yet through `withIntelligence` in a product**): > *"Demonstrated on internal benchmarks; in-product replication in progress."* --- @@ -89,7 +89,7 @@ This is where "take risks" stops. Boldness is allowed in *speed, scope, and dogf **First customer: gtm.tangle.tools** (NOT Harvey). Already deployed on agent-runtime, already on the platform billing rail, richest live tool surface, friendliest checks, named first in the playbook. Harvey is **external** — repo access + contract + trust risk *before the wrapper has ever run in production.* Harvey/legal is case study #2. -**First PR / first turn (the go-live proof):** wrap gtm's production `handleChatTurn` with `withTangleIntelligence` in Observe mode, ship **one real user turn**, confirm the trace lands in `/v1/traces` AND live chat is unaffected when that endpoint is down. That single live trace + survived-outage = go-live. +**First PR / first turn (the go-live proof):** wrap gtm's production `handleChatTurn` with `withIntelligence` in Observe mode, ship **one real user turn**, confirm the trace lands in `/v1/traces` AND live chat is unaffected when that endpoint is down. That single live trace + survived-outage = go-live. **Harvest, don't rebuild:** cherry-pick `origin/examples/tangle-intelligence-export` (`8fa16af`, 122 LOC) into `examples/intelligence-drop-in/`; reconcile `origin/feat/observe-closed-loop` (`37e373f`) into the Phase-1 PR. Both branches exist (verified) — pay the re-discovery tax once. @@ -113,7 +113,7 @@ Ship Observe to gtm **only when all green:** ## 6. THE #1 BUILD AND THE #1 EXPERIMENT **#1 BUILD — `src/intelligence/index.ts` + the conserved-pool usd split.** -`createIntelligenceClient` + `withTangleIntelligence(agent, config)` + `traceRun(meta, fn)` — a thin best-effort layer over the shipped `createOtelExporter` + the Mode-0 sandbox-stream base. Add the `./intelligence` export. **Cap at Observe + intelligence-off; do NOT bundle loops.** Bundle the `{inferenceUsd, intelligenceUsd}` split with it — without it there is no honest OFF tier, no billing boundary, no metered baseline for the cost claim. Everything in §4 depends on this one PR. +`createIntelligenceClient` + `withIntelligence(agent, config)` + `traceRun(meta, fn)` — a thin best-effort layer over the shipped `createOtelExporter` + the Mode-0 sandbox-stream base. Add the `./intelligence` export. **Cap at Observe + intelligence-off; do NOT bundle loops.** Bundle the `{inferenceUsd, intelligenceUsd}` split with it — without it there is no honest OFF tier, no billing boundary, no metered baseline for the cost claim. Everything in §4 depends on this one PR. **#1 EXPERIMENT — E4: does the cost flywheel COMPOUND, or is it one-shot selection?** The surviving thesis hinges on this. Every cost win so far is single-step and partly *selection, not accumulation* (E3c/E3d killed in-stream admission). "Cost keeps falling as the certified store grows" is **unmeasured** (no E4 doc exists yet — it's the designed-not-launched NEXT in `current.json`). diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 1149d8da..9e055b63 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -2,7 +2,7 @@ -> **Version 0.89.0.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.101.0 <1.0.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.14.0 <1.0.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. +> **Version 0.94.11.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.117.1 <0.118.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.25.0 <0.26.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. > > **`./loops` is the runtime barrel** — `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. > @@ -86,7 +86,7 @@ A general "loop" primitive is the single most common modelling error in this rep | Pick the **chat backend an in-process turn runs on** (`router`/`tcloud`/`cli-bridge`/`sandbox`) from a product flag | `resolveAgentBackend({ backend })` — root `.` | the copy-pasted `backend-name → createOpenAICompatibleBackend` branch every eval product hand-rolled (the copies drift) | | Pick / register a leaf backend, or bring your own agent | `createExecutor({ backend })` / `createExecutorRegistry()` / implement `Executor` — `/loops` | a per-vendor adapter or closed `inline\|sandbox\|cli` switch (won't report through the `UsageEvent` channel) | | Evolve a **prompt/string** surface | `gepaProposer({ llm, model, target })` (default inside `selfImprove`; the skill-surface twin is `skillOptProposer`, same source) — `agent-eval/campaign` | a hand-rolled prompt-mutation reflection loop with its own Pareto bookkeeping | -| Self-improve a profile (one pluggable verb) — START HERE (self-improvement) | `improve(profile, findings, { surface, gate })` — root `.` (the RSI verb; defaults the generator from `surface`, wraps `selfImprove`) | a bespoke optimize loop, or calling `selfImprove`/a skill-optimizer directly for the common case | +| Self-improve an agent (one pluggable verb) — START HERE (self-improvement) | `improve(profile, findings, { surface, gate })` — root `.`; prompt, skill-document, and curated-memory surfaces have built-in generators, tools/MCP/hooks/subagents/whole-profile take an explicit generator, and code gets isolated incumbent/candidate worktrees. Workflow and rollout-policy files are code/config changes; use the code surface plus agent-eval's `parameterSweepProposer` for JSON sweeps. | a bespoke optimize loop, calling `selfImprove`/a skill or memory optimizer directly for the common case, storing orchestration in opaque profile extensions, or comparing code against an empty/string stand-in | | Measure **one profile artifact's marginal lift** (with-vs-without, score+cost) / catalog artifacts | `measureMarginalLift(...)` / `ArtifactRegistry` (`applyArtifact` is the one `ArtifactKind`→`AgentProfile`-field bridge) — `/lifecycle` | a hand-rolled with/without ablation loop, or a per-kind `if kind==='skill'…` profile-field switch | | Run the **whole artifact lifecycle** — generate→measure→promote→store→compose, then drift-watch/dedupe the live set — over ANY profile surface (skill/prompt/tool/MCP) | `runLifecycle({ baseline, generators, evalRunner, gate })` then `composeProfile(registry, base, query)`; maintain with `driftWatch(...)` / `dedupeArtifacts(...)` — `/lifecycle` | a per-surface improve loop, a hand-rolled promote→compose step, or re-running `measureMarginalLift` without the registry/gate spine. The ONLY per-surface code is a thin `CandidateGenerator` (`skillGenerator` distills, `promptGenerator`/`buildableGenerator` for the rest) | | Run the self-improvement loop with full substrate control | `selfImprove({ agent, scenarios, judge, baselineSurface })` — `agent-eval/contract` | a bespoke optimize loop or a parallel skill-optimizer | @@ -109,8 +109,13 @@ A general "loop" primitive is the single most common modelling error in this rep | Run a coding task INSIDE the agent's OWN sandbox session (a sibling box, fresh branch, validated patch) | `detachedSessionDelegate({ sandboxClient \| executor, workerProfile? })` — `/mcp` (pass the worker `AgentProfile`; omit for a minimal model-only default) | a hardcoded coder profile baked into the delegate; `delegate()` (that spawns workers in a *chosen* backend, not the agent's own session) | | Have a **supervisor spawn + live-drive workers in a backend you choose** and observe/steer/resume them | the **coordination MCP** — `createCoordinationTools` / `serveCoordinationMcp` over a live `Scope`; each worker's leaf is `createExecutor({ backend })` — `/mcp`,`/loops` | `detachedSessionDelegate` — own-sandbox-session only, one-shot, no live steer/recursion/conserved-budget | | Stand up a vertical agent in the eval loop | `defineAgent(manifest)` + `createSurfaceImprovementAdapter` — `/agent` | a per-vertical manifest parser, surface-validator, or bespoke `ImprovementAdapter` | -| Turn intelligence/observation OFF (prove inference-only billing) | `withTangleIntelligence(agent, { effort: 'off' })` — `/intelligence` | a custom trace-wrapper or hand-rolled effort/tier config | -| Fold **certified prompt additions into a system prompt you assemble yourself** (product chat routes) | `createCertifiedPromptSource({ target })` → `source.compose(base)` — `/intelligence` (cached, coalesced, fail-closed; `withCertifiedDelivery` rides the same source) | a module-scope cache + refresh-window + keep-last-known loop around `pullCertified` in product wiring | +| Observe + deliver Intelligence on a live agent (send RunRecords + receive certified profile/diffs) | `withIntelligence(agent, { project, target })` — `/intelligence` (proposals surfaced, never auto-applied; `effort: 'off'` proves inference-only billing) | a custom trace-wrapper, a second receive path, or hand-rolled effort/tier config | +| Turn trace evidence into one measured, review-only agent proposal | `proposeAgentImprovement({ analysis, profile, improvement })` — `/intelligence` | manually joining analysts, optimizers, evidence, uncertainty, and candidate identity | +| Freeze a measured profile/diff plus a content-addressed code surface into one executable candidate | `buildAgentCandidateBundle(...)`, then `verifyAgentCandidateBundle(...)` at execution — `/candidate-execution` | a product callback that converts profile fields, reproduces Git diff flags, hashes bytes, or assembles `AgentCandidateBundle` by hand | +| Record approve/reject/change-request feedback against one exact proposal | `reviewAgentImprovementProposal(proposal, review)` — `/intelligence` | a mutable status row that is not bound to candidate bytes | +| Execute and grade only an authenticated approved candidate | `executeApprovedAgentCandidate(options)` — `/intelligence`; low-level ports live at `/candidate-execution` | product-local claim/retry/isolation/receipt orchestration | +| Capture and restore exact task, candidate, or memory workspace bytes | `captureAgentCandidateWorkspace(...)` + `createAgentCandidateWorkspacePort(...)` — `/candidate-execution` | a product-specific archive format, ambient `git checkout`, or a materializer that skips byte/path/mode verification | +| Fold **certified prompt additions into a system prompt you assemble yourself** (product chat routes) | `createCertifiedPromptSource({ target })` → `source.compose(base)` — `/intelligence` (cached, coalesced, fail-closed; `withIntelligence` rides the same source) | a module-scope cache + refresh-window + keep-last-known loop around `pullCertified` in product wiring | | Improve a KB with runtime agents, candidate workspaces, readiness checks, and measured supervised spend | `runKnowledgeImprovementJob(options)` from `/knowledge` | hand-wiring `improveKnowledgeBase` + a supervised updater + a readiness callback in every product | For the full export inventory (every primitive, its import path, its summary — generated, never stale), see `docs/api/primitive-catalog.md`; for per-symbol signatures, the per-module `docs/api/` pages. For the recursive atom (recursion · isolated-or-collaborative artifact · conserved budget · analysts) and the two-timescale architecture, see `docs/architecture.md`. For the genome→run→optimize→ship spine in depth, `docs/concepts.md` + `docs/learning-flywheel.md`. For the Intelligence SDK (Observe + the provable-OFF billing boundary), `docs/intelligence-sdk.md`. diff --git a/docs/concepts.md b/docs/concepts.md index 3130b9c5..fa3926f5 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -97,8 +97,8 @@ execution state. ## The agent manifest -`defineAgent(...)` is how a vertical declares the **surfaces** (prompt, -skills, tools — the levers `agent-eval`'s analyst loop can edit), the +`defineAgent(...)` is how a vertical declares the **surfaces** (the full +`AgentProfile`: prompt, skills, tools, MCP, hooks, subagents, and extensions), the **knowledge** requirements, the **rubric**, and the **run** function that ties it all together. The manifest is what the eval harness benchmarks, what the analyst loop improves, and (in time) what the diff --git a/docs/design/structural-rollout-integration.md b/docs/design/structural-rollout-integration.md new file mode 100644 index 00000000..f2533ac0 --- /dev/null +++ b/docs/design/structural-rollout-integration.md @@ -0,0 +1,45 @@ +# Structural rollout policy — integration design + +Status: design accepted 2026-07-09; measured basis in `supervisor-lab/docs/results/structural-lever-humaneval.md`. + +## Measured basis + +Best-of-k selection + self-repair, grounded ONLY on task-visible checks (shown examples + model-authored asserts), graded on hidden tests: + +| model × dataset | baseline → full loop | lift | p (exact sign test) | +|---|---|---|---| +| Llama-3-8B × MBPP (n=427) | 51.8% → 73.1% | **+21.3pp** | 2.3e-51 (+226/−13) | +| Llama-3-8B × HumanEval (n=164) | 43.9% → 62.2% | +18.3pp | 9.2e-11 | +| Qwen2.5-7B × HumanEval (n=164) | 82.4% → 91.5% | +9.0pp | 1.0e-8 | +| Qwen2.5-7B × MBPP (n=427) | 76.7% → 85.2% | +8.5pp | 4.6e-16 | +| glm-4.5-air / glm-5.2 × HumanEval (saturated 99.4/99.7%) | null | −0.6 / −0.4pp | the only regressions are the two calibration-flagged wrong-example tasks (/47, /116) | + +Every positive cell captures ≥93% of the pass@k bound. Selection is 85–92% of the effect; repair is a small always-positive increment. Prompt-diversity per slot is a paired null (+0.6pp). Independently verified: 1,968/1,968 regrade cells, 328/328 selection replays, 0 hidden-test leaks. + +## The finding that shapes the design + +The runtime already owns five of the six pieces: +1. best-of-k + repair strategy family — `src/runtime/strategy.ts` `sample`/`refine`/`sampleThenRefine` (:755/:760/:1003), authored via `defineStrategy` (:834) +2. jailed check execution — `createVerifierEnvironment` (`src/runtime/verifier-environment.ts:68`), agent-eval `testJudge`/`runJudgeFleet`, `@tangle-network/sandbox` — retire the bench rigs' bespoke `docker run` jails +3. selection + audit — `defaultSelectWinner` + `SelectionReceipt` (`src/runtime/run-loop.ts:1131`, `types.ts:160`) +4. visible/hidden firewall as typed field routing — agent-eval `FieldDestination`: `develop-against` (the visible-check source) vs `grading-only`, with `assertNoHiddenLeak` + `gradeOnHidden` +5. config/knob plumbing — `budget` on runBenchmark/runAgentic/superviseSurface; `directives.ts` for slot prefixes + +The ONLY net-new seam: **the model authors its own visible checks** (`CheckSource`). + +## Design (no caller changes) + +New module `src/runtime/structural-rollout.ts`: +- `CheckSource` — `generate(task, ctx) → VisibleCheck[]` from agent-visible/develop-against fields only. Default impl lifted from the proven `bench/src/hev-structural.mts generateTests` (:282) with the MBPP lesson baked in: **official shown examples outrank model-authored guesses in scoring** (guesses are 17–70% wrong depending on model × spec richness; unweighted they can flip selection negative). +- `CheckRunner` — `run(candidate, checks, ctx) → { passed, total, failureOutput }`, backend = sandbox exec / agent-eval `testJudge`, result shaped to `SurfaceScore`. +- `structuralRollout({ policy, checkSource, checkRunner }) → Strategy` — a fourth member of the sample/refine family via `defineStrategy`; argmax by weighted visible score, ≤`repairRounds` repair shots steered by `failureOutput`, keep-best-by-score. Emits `SelectionReceipt`s. +- `StructuralRolloutPolicy { k, repairRounds, testgen, diverse?, temperature? }` — promoted from the rig env vars; optimize its owning config/code file with the code surface and agent-eval's `parameterSweepProposer` so the winner remains executable. + +Placement rule: this is an INFERENCE-TIME capability (wraps the model call). It does not go into `improve()`/`selfImprove` (training-time); `improve()` may later tune the policy knobs. + +Extend-don't-fork list: strategy family, verifier-environment, selectWinner/receipts, agent-eval field routing + judges, the rigs' `generateTests`/`extractRepairCode` as default impls. + +## Known behavior to preserve/handle +- Wrong visible examples poison repair at saturation (glm regressions on /47,/116): repair must never replace a candidate that passes MORE official checks with one that passes fewer; consider a no-repair-when-only-defect-signal guard. +- Repair value concentrates at low k (~+12pp at k=1, +1–3pp at k=5): default policy `k=5, repairRounds=2, testgen=6`; low-compute preset `k=1, repairRounds=2`. +- Exact-equality float asserts are a known wrong-test class (HumanEval/2 case). diff --git a/docs/intelligence-sdk.md b/docs/intelligence-sdk.md index 01e31e55..fcc43b3d 100644 --- a/docs/intelligence-sdk.md +++ b/docs/intelligence-sdk.md @@ -8,19 +8,20 @@ Per-symbol signatures live in the generated [api/intelligence.md](./api/intellig ## Quickstart ```ts -import { withTangleIntelligence } from '@tangle-network/agent-runtime/intelligence' - -export const agent = withTangleIntelligence(myAgent, { - project: 'legal-agent', - apiKey: process.env.TANGLE_API_KEY, - repo: { owner: 'acme', name: 'legal-agent', baseBranch: 'main' }, - checks: ['pnpm test', 'pnpm typecheck'], - surfaces: ['src/agent/**', 'src/tools/**', 'skills/**'], -}) +import { withIntelligence } from '@tangle-network/agent-runtime/intelligence' + +export const agent = withIntelligence( + async (input, applied) => myAgent(input, { systemPrompt: applied.composePrompt(BASE) }), + { + project: 'legal-agent', + target: 'legal-agent', + apiKey: process.env.TANGLE_API_KEY, + }, +) ``` -`withTangleIntelligence` wraps any `(input) => Promise` agent and returns the same shape: each call runs under a trace and exports one span, best-effort. -`repo`, `checks`, and `surfaces` are recorded for `IntelligenceClient.doctor()` readiness only — the Observe slice never touches the repo. +`withIntelligence` is the ONE hook: it SENDS a typed `RunRecord` per call and RECEIVES the tenant's certified profile — folding the certified prompt surface (`applied.composePrompt`) and surfacing the plane's promoted profile diffs as PROPOSALS (`agent.proposals()` / `onProposals`). +It is **observe + deliver only** — it never auto-applies a received diff at runtime; a human, or the gated `improve()` loop, turns a proposal into a shipped profile (`applied.applyProfile(base)` is caller-invoked). That preserves the held-out invariant the plane enforces. Prove it offline: `pnpm tsx examples/intelligence-drop-in/intelligence-drop-in.ts` ($0, no credentials — it reads the exported span back and asserts the OFF tier's `intelligence_usd` is 0). ## Normative claims policy @@ -44,8 +45,10 @@ The delivery code enforces the same framing: a `CertifiedArtifact` carries its h - `project` (required) — the tenant dimension every trace is tagged with. - `apiKey` — bearer key for the ingest; reads `TANGLE_API_KEY` when omitted. - `effort` — a tier name or `{ tier, overrides }`; default `standard`. See [Effort tiers](#effort-tiers-and-the-off-billing-floor). -- `endpoint` — OTLP ingest base (the exporter appends `/v1/traces`). Reads `INTELLIGENCE_OTLP_ENDPOINT` then `OTEL_EXPORTER_OTLP_ENDPOINT` when omitted; absent all three, export is a no-op — best-effort by construction. +- `baseUrl` — the ONE Tangle Intelligence base URL both send (`/v1/otlp`) and receive (`/v1/profiles/:target/composed`) derive from. Reads `TANGLE_INTELLIGENCE_URL` when omitted, else `https://intelligence.tangle.tools`. Send ships only when an `apiKey` is present; absent a tenant key, export is a no-op — best-effort by construction. - `redact` — a `Redactor` replaces the default scrubber; `false` opts out loudly; omitted ⇒ `defaultRedactor`. See [Redaction](#redaction). +- `profile` / `commitSha` — the canonical agent configuration and code revision to attach to every run. +- `runtimeTelemetry` — controls whether raw tool inputs/results and other sensitive runtime payloads are included; identities, status, token use, cost, and failures are always retained. - `surfaces` / `checks` / `repo` — declared for `IntelligenceClient.doctor()` readiness only. The client surface: @@ -56,7 +59,7 @@ The client surface: - `client.flush()` — flush pending spans; resolves even if export fails. The best-effort law: telemetry-export failures are swallowed — a live agent never fails because Intelligence is down — but an error thrown by the agent itself propagates unchanged. -Every span carries the billing split as attributes (`tangle.usage.inference_usd`, `tangle.usage.intelligence_usd`, `tangle.effort.intelligence_off`); inputs/outputs pass through the redactor and are bounded to a 4 KB preview on the span. +Every span carries the billing split as attributes (`tangle.usage.inference_usd`, `tangle.usage.intelligence_usd`, `tangle.effort.intelligence_off`); inputs, outputs, and opted-in runtime payloads pass through the redactor and are exported without content truncation. ## Two lanes: traces UP, certified artifacts DOWN @@ -67,23 +70,29 @@ GET {base}/v1/profiles/:target/composed Authorization: Bearer ``` -The base URL reads `TANGLE_INTELLIGENCE_URL`, defaulting to the deployed plane at intelligence.tangle.tools. Four entrypoints, all fail-closed: +The base URL reads `TANGLE_INTELLIGENCE_URL`, defaulting to the deployed plane at intelligence.tangle.tools. The composed response carries the certified prompt surface + artifacts AND the typed `agentProfileDiffs` (each a promoted `AgentProfileDiff` + its held-out provenance), the composed `agentProfile`, and `capabilities`. Four entrypoints, all fail-closed: -- `pullCertified(opts)` — one pull, returning a typed `PullOutcome` (inspect `succeeded` before `value`); it never throws. A 404 is the normal "nothing promoted yet" signal, carried as `status: 404`; a hung plane is cut by the 10-second default timeout and surfaces as an ordinary failed pull. +- `pullCertified(opts)` — one pull, returning a typed `PullOutcome` (inspect `succeeded` before `value`); it never throws. It deserializes the full composed response — including the typed `agentProfileDiffs` earlier receive paths dropped. A 404 is the normal "nothing promoted yet" signal, carried as `status: 404`; a hung plane is cut by the 10-second default timeout and surfaces as an ordinary failed pull. - `createCertifiedPromptSource(opts)` — the cached, self-refreshing source: pulls at most every 5 minutes (`refreshMs`), coalesces concurrent pulls, and keeps the last-known profile on a failed or 404 pull — a good surface is never wiped by a bad refresh. - `composeCertifiedPrompt(base, certified)` — folds the certified prompt surface plus the prompt-folding artifact buckets (`promptFoldTypes`: prompt-surface, skill, instructions) into the base prompt under a marked section. The fold is byte-stable — prompt surface first, then bucket order, then path order — so the same profile renders identically on every call; it returns `base` unchanged when nothing usable is promoted. -- `withCertifiedDelivery(agent, config)` — the wrapper that rides both lanes at once: +- `withIntelligence(agent, config)` — the ONE hook that rides both lanes at once: ```ts -import { withCertifiedDelivery } from '@tangle-network/agent-runtime/intelligence' - -export const agent = withCertifiedDelivery( - async (input, applied) => myAgent(input, { systemPrompt: applied.composePrompt(BASE) }), - { project: 'support-agent', target: 'support-agent' }, +import { withIntelligence } from '@tangle-network/agent-runtime/intelligence' + +export const agent = withIntelligence( + async (input, applied) => { + const out = await myAgent(input, { systemPrompt: applied.composePrompt(BASE) }) + applied.record({ success: true, usage: { inferenceUsd: 0.002, intelligenceUsd: 0 } }) + return out + }, + { project: 'support-agent', target: 'support-agent', onProposals: (diffs) => review(diffs) }, ) ``` -Each call refreshes the certified profile (window-respecting), hands the agent an `AppliedIntelligence` handle (`certified` + `composePrompt`), and Observes the run with the certified version stamped on the span. +Each call refreshes the certified profile (window-respecting), hands the agent an `AppliedIntelligence` handle (`runId`, `traceId`, `certified`, `composePrompt`, `proposals`, `applyProfile`, `record`), and SENDs a typed `RunRecord` for the run. +The run record includes timing, failures, profile/config hash, repository revision, model, tokens, costs, runtime events, and loop events when supplied; thrown agent errors are exported before being rethrown. +The promoted profile diffs surface as `agent.proposals()` / the `onProposals` callback — the hook NEVER auto-applies them; `applied.applyProfile(base)` folds them only when the caller explicitly asks. When the plane promotes a new gate-certified surface, the next refresh delivers it to the running agent; when the plane is unreachable, the agent runs on its base surface. ### The capability resolver @@ -139,7 +148,7 @@ A caller-supplied `redact` hook replaces the default entirely (the customer owns | Verified PRs | **designed** — the readiness check ships | 100-300 LOC | branch/PR with passing checks | surfaces, checks, repo access | fail closed | | Advanced Loops | **designed** — the loop substrate ships | 300+ LOC | scheduled optimization, holdout gates, matrix runs | eval adapter, budgets, gates | explicit opt-in | -`IntelligenceClient.doctor()` reports exactly this ladder without a network call: Observe is always ready; Recommend needs effort above off; PR needs checks + surfaces + repo; `exportConfigured` says whether an OTLP endpoint is set. +`IntelligenceClient.doctor()` reports exactly this ladder without a network call: Observe is always ready; Recommend needs effort above off; PR needs checks + surfaces + repo; `exportConfigured` says whether a tenant `apiKey` is set (the key the ingest requires for a send to land). If a product has no checks, it is not eligible for PR mode. If it has no traces, it is not eligible for credible recommendations. If it has neither, start at Observe. Recommend today, without the hosted verb: record a run's trace with `client.recordTrace(events)`, derive analyst findings from it, and feed them to `improve()` for a gated candidate. @@ -192,8 +201,7 @@ import { createCertifiedPromptSource, createIntelligenceClient, pullCertified, - withCertifiedDelivery, - withTangleIntelligence, + withIntelligence, } from '@tangle-network/agent-runtime/intelligence' ``` diff --git a/examples/README.md b/examples/README.md index 15de1db2..2cf8c12c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -84,7 +84,7 @@ repeat. A failing validator prunes a bad candidate so the loop can't keep it. | 17b | [`self-improving-coder/`](./self-improving-coder/) | The flywheel on a contamination-proof coding task: an agent writes strategies from its training losses, graded by real pytest, promoted only if a fresh holdout confirms the gain. `CALIBRATE=1` is a $0 no-key check. | | 18 | [`self-improving-loop/`](./self-improving-loop/) | #17 unrolled step by step: v0 → judge → analyst → mutation → v1 → gate, showing which part owns each phase. Offline. | | 19 | [`intelligence-recommend/`](./intelligence-recommend/) | The improvement loop offline end to end: read a run's trace → derive findings → `improve()` → a gated candidate. | -| 20 | [`intelligence-drop-in/`](./intelligence-drop-in/) | Wrap any agent with `withTangleIntelligence` to emit one trace per call — best-effort, and a proof that "off" is a zero-cost passthrough. | +| 20 | [`intelligence-drop-in/`](./intelligence-drop-in/) | Wrap any agent with `withIntelligence` to send one RunRecord per call — best-effort, and a proof that "off" is a zero-cost passthrough. | | 20b | [`intelligence-webcode/`](./intelligence-webcode/) | The full observability SDK (billing boundary, effort tiers, per-tool cost breakdown, OTLP export) instrumented over every cell of the WebCode benchmark. Needs a sandbox key. | | 21 | [`agents-of-all-shapes/`](./agents-of-all-shapes/) | Proof that any framework's traces converge on one open telemetry contract and produce one insight report. CI-tested. Offline. | | 22 | [`product-eval/`](./product-eval/) | Test an agent against a simulated user: a persona holds a multi-round conversation, then the transcript is scored. Needs `TANGLE_API_KEY`. | diff --git a/examples/ablation-suite/aisdk-env.ts b/examples/ablation-suite/aisdk-env.ts new file mode 100644 index 00000000..3c98a5e9 --- /dev/null +++ b/examples/ablation-suite/aisdk-env.ts @@ -0,0 +1,279 @@ +/** + * aisdk-env — a long-horizon coding task on a genuinely POST-CUTOFF contract: build a working tool + * for the CURRENT Vercel `ai` SDK (v7). The coder's training knows v4, where the tool schema field + * was `parameters:`; v5+ renamed it to `inputSchema:`, so v4-shaped code no longer type-checks. The + * grader is the TypeScript compiler itself against the installed ai@7 — no rubric, the compiler is + * the judge. This is the first web-grounded task where the knowledge is genuinely absent from the + * coder AND search returns it AND a real compiler catches the difference. + * + * ABLATION DELTA: `search` toggles a `web_search` tool (you.com via the Tangle router). read/write/ + * run_tests (= tsc, returns the compiler errors as feedback) stay ON in both arms — additive + * isolation. The NO-SEARCH arm is the control: with compiler feedback but no web, does the coder + * recover the current contract on its own, or stay stuck on its v4 prior? + */ + +import { execFileSync } from 'node:child_process' +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import type { + AgenticSurface, + AgenticTask, + AgenticTool, + ArtifactHandle, + SurfaceScore, +} from '../../src/runtime/strategy.js' + +// Prebuilt template with node_modules { ai@7, zod, typescript } + the proven tsc invocation. Each +// workspace symlinks its node_modules and .bin from here so there is no per-task npm install. +const TEMPLATE = + process.env.AISDK_TEMPLATE ?? + '/tmp/claude-1000/-home-drew-code-blueprint-agent/8b62a3c4-97a5-46cd-92bf-5a4f3bf95f75/scratchpad/aisdk-probe' +const TSC = join(TEMPLATE, 'node_modules', '.bin', 'tsc') +const TSC_ARGS = [ + '--ignoreConfig', + '--pretty', + 'false', + '--noEmit', + '--strict', + '--skipLibCheck', + '--moduleResolution', + 'bundler', + '--module', + 'esnext', + '--target', + 'es2022', +] + +// Each task is a structurally-required rename in the current ai SDK: the coder's v4 memory writes a +// name the compiler now rejects, and the task cannot be completed without the current name — so a +// pass means the coder produced the current API, not a lucky stub. `starter` fails to compile (an +// unedited attempt is a fail); `check` compiles solution.ts with a real use of the export. +interface SdkTask { + key: string + systemPrompt: string + userPrompt: string + starter: string + check: string +} + +const SDK_TASKS: SdkTask[] = [ + { + key: 'tool', + systemPrompt: + 'You are a senior TypeScript engineer. Build code for the CURRENT version of the library, not the version you remember.', + userPrompt: + 'Edit `solution.ts` so it exports a tool named `weatherTool`, built with the CURRENT `ai` package via its `tool()` helper: a description, a zod schema for a `location: string` input, and an `execute` returning a string. The tool() schema field may have been renamed since your training — if run_tests fails, the current field name differs from what you wrote.', + starter: `import { tool } from 'ai'\nimport { z } from 'zod'\n// Build weatherTool with the CURRENT ai SDK tool() helper.\nexport const weatherTool = tool({})\n`, + check: `import { generateText } from 'ai'\nimport { weatherTool } from './solution'\nexport async function _check() {\n return generateText({ model: 'gpt' as unknown as never, tools: { weather: weatherTool }, prompt: 'x' })\n}\n`, + }, + { + key: 'stream', + systemPrompt: + 'You are a senior TypeScript engineer. Build code for the CURRENT version of the library, not the version you remember.', + userPrompt: + 'Edit `solution.ts` so it exports a function `handler(): Response` that calls `streamText` (model can be `"x" as any`, prompt `"hi"`) and returns its HTTP streaming Response for a chat UI. The method on the streamText result that returns this Response may have been renamed since your training — if run_tests fails, the current method name differs from what you wrote.', + starter: `import { streamText } from 'ai'\n// Return the streamText result's HTTP streaming Response.\nexport function handler(): Response {\n}\n`, + check: `import { handler } from './solution'\nexport const _r: Response = handler()\n`, + }, +] + +function taskSpec(id: string): SdkTask { + const key = id.split('#')[0] + return SDK_TASKS.find((t) => t.key === key) ?? SDK_TASKS[0] +} + +let searchCallTotal = 0 +export function resetSearchCalls(): void { + searchCallTotal = 0 +} +export function searchCalls(): number { + return searchCallTotal +} + +function ensureTemplate(): void { + if (!existsSync(join(TEMPLATE, 'node_modules', 'ai'))) { + throw new Error( + `AISDK_TEMPLATE has no node_modules/ai — install ai@7 zod typescript in ${TEMPLATE}`, + ) + } +} + +/** tsc the workspace (solution.ts + check.ts). Returns { pass, output } — output is the feedback. */ +function compile(dir: string): { pass: boolean; output: string } { + try { + execFileSync(TSC, [...TSC_ARGS, 'solution.ts', 'check.ts'], { + cwd: dir, + encoding: 'utf8', + timeout: 120_000, + stdio: ['ignore', 'pipe', 'pipe'], + }) + return { pass: true, output: 'tsc: 0 errors — compiles against the current ai SDK.' } + } catch (e) { + const out = + ((e as { stdout?: string }).stdout ?? '') + ((e as { stderr?: string }).stderr ?? '') + return { pass: false, output: out.trim().slice(0, 2_000) || 'tsc failed (no output)' } + } +} + +async function youSearch(query: string): Promise { + const key = process.env.TANGLE_API_KEY + if (!key) return 'web_search error: TANGLE_API_KEY unset' + try { + const res = await fetch('https://router.tangle.tools/v1/search', { + method: 'POST', + headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' }, + body: JSON.stringify({ query, provider: 'you' }), + }) + const j = (await res.json()) as { + data?: Array<{ title?: string; url?: string; snippet?: string }> + } + const rows = (j.data ?? []) + .slice(0, 6) + .map((r, i) => `${i + 1}. ${r.title ?? ''}\n ${r.url ?? ''}\n ${r.snippet ?? ''}`) + return rows.length ? rows.join('\n') : `web_search: no results for ${JSON.stringify(query)}` + } catch (err) { + return `web_search error: ${err instanceof Error ? err.message : String(err)}` + } +} + +export function makeAiSdkSurface(opts: { search: boolean }): AgenticSurface { + const name = `aisdk-v7${opts.search ? '+search' : ''}` + return { + name, + async open(task: AgenticTask): Promise { + ensureTemplate() + const spec = taskSpec(task.id) + const dir = mkdtempSync(join(tmpdir(), 'aisdk-')) + symlinkSync(join(TEMPLATE, 'node_modules'), join(dir, 'node_modules')) + writeFileSync(join(dir, 'solution.ts'), spec.starter) + writeFileSync(join(dir, 'check.ts'), spec.check) + return { id: `${name}:${task.id}`, surface: name, ctx: { dir } } + }, + async tools(): Promise { + const base: AgenticTool[] = [ + { + type: 'function', + function: { + name: 'write_file', + description: 'Write a file (path relative to project root, e.g. solution.ts).', + parameters: { + type: 'object', + properties: { path: { type: 'string' }, content: { type: 'string' } }, + required: ['path', 'content'], + }, + }, + }, + { + type: 'function', + function: { + name: 'read_file', + description: 'Read a file in the project.', + parameters: { + type: 'object', + properties: { path: { type: 'string' } }, + required: ['path'], + }, + }, + }, + ] + // NO_RUN_TESTS models the no-oracle regime (one-shot / non-executing agent): remove the + // compiler-feedback tool so the coder cannot trial-and-error to the current API. This is + // where web search should stop being redundant. + if (!process.env.NO_RUN_TESTS) + base.push({ + type: 'function', + function: { + name: 'run_tests', + description: + 'Type-check solution.ts against the installed ai SDK and get the compiler errors.', + parameters: { type: 'object', properties: {}, required: [] }, + }, + }) + if (opts.search) + base.push({ + type: 'function', + function: { + name: 'web_search', + description: + 'Search the live web (you.com) for the CURRENT ai SDK API you cannot recall (e.g. the tool() schema field name).', + parameters: { + type: 'object', + properties: { query: { type: 'string' } }, + required: ['query'], + }, + }, + }) + return base + }, + async call( + handle: ArtifactHandle, + toolName: string, + args: Record, + ): Promise { + const dir = (handle.ctx as { dir: string }).dir + if (toolName === 'web_search') { + searchCallTotal += 1 + return youSearch(String(args.query ?? '')) + } + if (toolName === 'write_file') { + const p = join(dir, String(args.path)) + mkdirSync(dirname(p), { recursive: true }) + writeFileSync(p, String(args.content ?? '')) + return `wrote ${args.path}` + } + if (toolName === 'read_file') { + const p = join(dir, String(args.path)) + return existsSync(p) + ? readFileSync(p, 'utf8').slice(0, 20_000) + : `no such file: ${args.path}` + } + if (toolName === 'run_tests') { + const { pass, output } = compile(dir) + return pass + ? output + : `tsc FAILED — fix the tool definition for the current SDK:\n${output}` + } + return `unknown tool: ${toolName}` + }, + async score(_task: AgenticTask, handle: ArtifactHandle): Promise { + const dir = (handle.ctx as { dir: string }).dir + return { passes: compile(dir).pass ? 1 : 0, total: 1, errored: 0 } + }, + async close(handle: ArtifactHandle): Promise { + const dir = (handle.ctx as { dir: string }).dir + if (process.env.AISDK_KEEP) { + console.error(` kept workspace: ${dir}`) + return + } + try { + rmSync(dir, { recursive: true, force: true }) + } catch { + /* best effort */ + } + }, + } +} + +// Cycle the verified tasks so n tasks spread evenly across the distinct renames (n=12 → 6 each), +// proving the search effect is not specific to one API change. +export function aiSdkTasks(n: number): Promise { + return Promise.resolve( + Array.from({ length: n }, (_, i) => { + const spec = SDK_TASKS[i % SDK_TASKS.length] + return { + id: `${spec.key}#${i}`, + systemPrompt: spec.systemPrompt, + userPrompt: spec.userPrompt, + } + }), + ) +} diff --git a/examples/ablation-suite/extract-contracts.mjs b/examples/ablation-suite/extract-contracts.mjs new file mode 100644 index 00000000..730eb644 --- /dev/null +++ b/examples/ablation-suite/extract-contracts.mjs @@ -0,0 +1,50 @@ +/** + * Auto-discover every reusable (vendor, fn, testFile, graderB64) sub-contract in blueprint-agent's + * VB web-grounded API-integration leaves. Each grader is a self-contained mock+pytest that boots its + * own mock and tests ONE exported function of the worker's solution.py — compose N of them into one + * long-horizon integration (multi-contract-env). The fn is read from the decoded grader itself + * (hasattr(module, "")), the ground truth, not guessed. + */ +import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const here = dirname(fileURLToPath(import.meta.url)) +const VB = '/home/drew/code/blueprint-agent/scripts/experiments/scenarios/verticals/web-grounded' + +// The HTTP-client-shaped API-integration files (composable: solution.py exports base_url/api_key fns). +const FILES = readdirSync(VB).filter((f) => (f.startsWith('api-') || f.startsWith('pipedrive')) && f.endsWith('.ts')) + +function constVal(src, name) { + const m = src.match(new RegExp(name + "\\s*=\\s*((?:'[^']*'\\s*\\+?\\s*)+)")) + return m ? [...m[1].matchAll(/'([^']*)'/g)].map((x) => x[1]).join('') : null +} + +const all = [] +for (const file of FILES) { + const src = readFileSync(join(VB, file), 'utf8') + const vendor = (src.match(/partner:\s*'([^']+)'/) || src.match(/company:\s*'([^']+)'/) || [, file.replace(/\.ts$/, '')])[1] + for (const call of src.matchAll(/oracleCommand\('([^']+)'\s*,\s*([A-Z0-9_]+)\)/g)) { + const [, testFile, constName] = call + const b64 = constVal(src, constName) + if (!b64) { console.error('MISS b64', constName, file); continue } + let grader + try { grader = Buffer.from(b64, 'base64').toString('utf8') } catch { continue } + const fn = grader.match(/hasattr\(module,\s*"([a-z_][a-z0-9_]*)"\)/)?.[1] + ?? grader.match(/module\.([a-z_][a-z0-9_]*)\(/)?.[1] + if (!fn) { console.error('MISS fn in grader', testFile, file); continue } + // pull the signature from the grader's call or the seed + const sig = (src.match(new RegExp('def ' + fn + '\\([^\\n)]*\\)[^\\n]*')) || [, `def ${fn}(base_url, api_key, ...)`])[0] + all.push({ vendor: `${vendor}:${fn}`, fn, testFile, signature: (typeof sig === 'string' ? sig : `def ${fn}(base_url, api_key)`).trim(), graderB64: b64 }) + console.error('OK', vendor, fn, `(${testFile}, ${b64.length}b64)`) + } +} + +// dedup by fn (a solution.py exports each fn once) + keep GET-single-resource style first +const seen = new Set() +const contracts = all.filter((c) => (seen.has(c.fn) ? false : (seen.add(c.fn), true))) +const outDir = join(here, 'fixtures', 'multi-contract') +mkdirSync(outDir, { recursive: true }) +writeFileSync(join(outDir, 'contracts.all.json'), JSON.stringify(contracts, null, 2)) +console.error(`\nDISCOVERED ${contracts.length} unique-fn contracts -> contracts.all.json`) +console.error('vendors:', contracts.map((c) => c.fn).join(', ')) diff --git a/examples/ablation-suite/fixtures/multi-contract/contracts.all.json b/examples/ablation-suite/fixtures/multi-contract/contracts.all.json new file mode 100644 index 00000000..7ec1d8a7 --- /dev/null +++ b/examples/ablation-suite/fixtures/multi-contract/contracts.all.json @@ -0,0 +1,135 @@ +[ + { + "vendor": "posthog:get_flag_variant_payload", + "fn": "get_flag_variant_payload", + "testFile": "test_posthog_flags_variant_payload.py", + "signature": "def get_flag_variant_payload(base_url: str, project_api_key: str, distinct_id: str, flag_key: str) -> tuple\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcG9zdGhvZy1mbGFncy12Mi12YXJpYW50LXBheWxvYWQgKGhhcmQpIGxlYWYuCgpTYW1lIFBvc3RIb2cgQ1VSUkVOVCBmZWF0dXJlLWZsYWcgY29udHJhY3QgYXMgdGhlIG1lZGl1bSBsZWFmLCBidXQgZXhlcmNpc2VzIHRoZSByZXNwb25zZS1TSEFQRQpheGlzIGluIGZ1bGw6IHRoZSBhZ2VudCBtdXN0IHJlYWQgYSBtdWx0aXZhcmlhdGUgZmxhZydzIGFzc2lnbmVkIGB2YXJpYW50YCBBTkQgZGVjb2RlIGl0cwpKU09OLWVuY29kZWQgYHBheWxvYWRgLCBib3RoIG9mIHdoaWNoIG1vdmVkIGxvY2F0aW9uIGluIHRoZSB2MiBlbnZlbG9wZS4KCiAgLSBlbmRwb2ludCAgICAgUE9TVCAvZmxhZ3M/dj0yCiAgLSBhdXRoICAgICAgICAgcHJvamVjdCB0b2tlbiBhcyBhcGlfa2V5IGluIHRoZSBKU09OIGJvZHkKICAtIHYyIHNoYXBlICAgICBmbGFnc1trZXldWyJ2YXJpYW50Il0gICAgICAgICAgICAgICAgICAodjEvZGVjaWRlOiBmZWF0dXJlRmxhZ3Nba2V5XSBJUyB0aGUgdmFyaWFudCBzdHJpbmcpCiAgICAgICAgICAgICAgICAgZmxhZ3Nba2V5XVsibWV0YWRhdGEiXVsicGF5bG9hZCJdICAgICAgKHYxL2RlY2lkZTogYSBzZXBhcmF0ZSBmZWF0dXJlRmxhZ1BheWxvYWRzW2tleV0gbWFwKQogICAgICAgICAgICAgICAgIHBheWxvYWQgaXMgYSBKU09OLUVOQ09ERUQgU1RSSU5HIHRoYXQgdGhlIGNsaWVudCBtdXN0IGpzb24ubG9hZHMgaW50byBhIGRpY3QuCgpBIGNsaWVudCB3cml0dGVuIGZyb20gQ1VSUkVOVCBQb3N0SG9nIGRvY3MgcmVhZHMgZmxhZ3NbJ2NoZWNrb3V0LWV4cGVyaW1lbnQnXVsndmFyaWFudCddID09Cid2YXJpYW50LWInIGFuZCBkZWNvZGVzIG1ldGFkYXRhLnBheWxvYWQgLT4geyJkaXNjb3VudF9wY3QiOiAyMH0gYW5kIHBhc3Nlcy4gQSBzdGFsZS9mcm9tLW1lbW9yeQpjbGllbnQgUE9TVHMgL2RlY2lkZSBhbmQgcmVhZHMgZmVhdHVyZUZsYWdzIC8gZmVhdHVyZUZsYWdQYXlsb2FkcyAoYWJzZW50IGluIHRoZSB2MiBlbnZlbG9wZSkgYW5kCkZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL3Bvc3Rob2cuY29tL2RvY3MvYXBpL2ZsYWdzCiAgaHR0cHM6Ly9wb3N0aG9nLmNvbS9kb2NzL2ludGVncmF0ZS9mZWF0dXJlLWZsYWdzLWNvZGUKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9UT0tFTiA9ICJwaGNfbGl2ZV9wcm9qZWN0X3Rva2VuX2FiYzEyMyIKCkZMQUdTID0gewogICAgIm5ldy1kYXNoYm9hcmQiOiB7CiAgICAgICAgImtleSI6ICJuZXctZGFzaGJvYXJkIiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJjb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogMCwgImRlc2NyaXB0aW9uIjogIkNvbmRpdGlvbiBzZXQgMSBtYXRjaGVkIn0sCiAgICAgICAgIm1ldGFkYXRhIjogeyJpZCI6IDEsICJ2ZXJzaW9uIjogMywgInBheWxvYWQiOiBOb25lfSwKICAgIH0sCiAgICAibGVnYWN5LWJhbm5lciI6IHsKICAgICAgICAia2V5IjogImxlZ2FjeS1iYW5uZXIiLAogICAgICAgICJlbmFibGVkIjogRmFsc2UsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJub19jb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogTm9uZSwgImRlc2NyaXB0aW9uIjogIk5vIGNvbmRpdGlvbiBzZXQgbWF0Y2hlZCJ9LAogICAgICAgICJtZXRhZGF0YSI6IHsiaWQiOiAyLCAidmVyc2lvbiI6IDEsICJwYXlsb2FkIjogTm9uZX0sCiAgICB9LAogICAgImNoZWNrb3V0LWV4cGVyaW1lbnQiOiB7CiAgICAgICAgImtleSI6ICJjaGVja291dC1leHBlcmltZW50IiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInZhcmlhbnQiOiAidmFyaWFudC1iIiwKICAgICAgICAicmVhc29uIjogeyJjb2RlIjogImNvbmRpdGlvbl9tYXRjaCIsICJjb25kaXRpb25faW5kZXgiOiAxLCAiZGVzY3JpcHRpb24iOiAiQ29uZGl0aW9uIHNldCAyIG1hdGNoZWQifSwKICAgICAgICAibWV0YWRhdGEiOiB7ImlkIjogMywgInZlcnNpb24iOiA3LCAicGF5bG9hZCI6ICJ7XCJkaXNjb3VudF9wY3RcIjogMjB9In0sCiAgICB9LAp9CgoKY2xhc3MgX01vY2tQb3N0SG9nKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOgogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fUE9TVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIGxlbmd0aCA9IGludChzZWxmLmhlYWRlcnMuZ2V0KCJDb250ZW50LUxlbmd0aCIsIDApIG9yIDApCiAgICAgICAgcmF3ID0gc2VsZi5yZmlsZS5yZWFkKGxlbmd0aCkgaWYgbGVuZ3RoIGVsc2UgYiIiCiAgICAgICAgdHJ5OgogICAgICAgICAgICBib2R5ID0ganNvbi5sb2FkcyhyYXcuZGVjb2RlKCkpIGlmIHJhdyBlbHNlIHt9CiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAgICAgYm9keSA9IE5vbmUKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJ2IjogcXMuZ2V0KCJ2IiwgW05vbmVdKVswXSwKICAgICAgICAgICAgImJvZHlfa2V5cyI6IHNvcnRlZChib2R5LmtleXMoKSkgaWYgaXNpbnN0YW5jZShib2R5LCBkaWN0KSBlbHNlIE5vbmUsCiAgICAgICAgfSkKCiAgICAgICAgaWYgcGF0aCAhPSAiL2ZsYWdzIjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsidHlwZSI6ICJpbnZhbGlkX3JlcXVlc3QiLCAiY29kZSI6ICJub3RfZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXRhaWwiOiAidW5rbm93biBlbmRwb2ludCAoUG9zdEhvZyBldmFsdWF0ZXMgZmVhdHVyZSBmbGFncyBhdCBQT1NUIC9mbGFncz92PTIpIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBpZiBxcy5nZXQoInYiLCBbTm9uZV0pWzBdICE9ICIyIjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsidHlwZSI6ICJ2YWxpZGF0aW9uX2Vycm9yIiwgImNvZGUiOiAibWlzc2luZ192ZXJzaW9uIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGV0YWlsIjogInRoZSAvZmxhZ3MgZW5kcG9pbnQgcmVxdWlyZXMgP3Y9MiBmb3IgdGhlIGN1cnJlbnQgZmxhZ3MgcmVzcG9uc2UgZm9ybWF0In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBpZiBub3QgaXNpbnN0YW5jZShib2R5LCBkaWN0KToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsidHlwZSI6ICJ2YWxpZGF0aW9uX2Vycm9yIiwgImNvZGUiOiAibWFsZm9ybWVkX2JvZHkiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXRhaWwiOiAiUE9TVCBib2R5IG11c3QgYmUgSlNPTiBjb250YWluaW5nIGFwaV9rZXkgYW5kIGRpc3RpbmN0X2lkIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBpZiBib2R5LmdldCgiYXBpX2tleSIpICE9IEVYUEVDVEVEX1RPS0VOOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJ0eXBlIjogImF1dGhlbnRpY2F0aW9uX2Vycm9yIiwgImNvZGUiOiAiaW52YWxpZF9hcGlfa2V5IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGV0YWlsIjogInByb3ZpZGUgdGhlIHByb2plY3QgQVBJIGtleSBhcyBhcGlfa2V5IGluIHRoZSBKU09OIGJvZHkifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIG5vdCBib2R5LmdldCgiZGlzdGluY3RfaWQiKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsidHlwZSI6ICJ2YWxpZGF0aW9uX2Vycm9yIiwgImNvZGUiOiAibWlzc2luZ19kaXN0aW5jdF9pZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJkaXN0aW5jdF9pZCBpcyByZXF1aXJlZCB0byBldmFsdWF0ZSBmbGFncyBmb3IgYSB1c2VyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAiZmxhZ3MiOiBGTEFHUywKICAgICAgICAgICAgImVycm9yc1doaWxlQ29tcHV0aW5nRmxhZ3MiOiBGYWxzZSwKICAgICAgICAgICAgInJlcXVlc3RJZCI6ICI1NTBlODQwMC1lMjliLTQxZDQtYTcxNi00NDY2NTU0NDAwMDAiLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1Bvc3RIb2cpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpLnN0YXJ0KCkKICAgIHRyeToKICAgICAgICB5aWVsZCBmImh0dHA6Ly8xMjcuMC4wLjE6e3BvcnR9IgogICAgZmluYWxseToKICAgICAgICBzZXJ2ZXIuc2h1dGRvd24oKQogICAgICAgIHNlcnZlci5zZXJ2ZXJfY2xvc2UoKQoKCmRlZiBfbG9hZF9hZ2VudF9zb2x1dGlvbigpOgogICAgcm9vdCA9IG9zLmVudmlyb24uZ2V0KCJWQl9TT0xVVElPTl9ST09UIiwgb3MuZ2V0Y3dkKCkpCiAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4ocm9vdCwgInNvbHV0aW9uIiwgInNvbHV0aW9uLnB5IikKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIGZvciBkaXJwYXRoLCBfZGlycywgZmlsZXMgaW4gb3Mud2Fsayhyb290KToKICAgICAgICAgICAgaWYgInNvbHV0aW9uLnB5IiBpbiBmaWxlczoKICAgICAgICAgICAgICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihkaXJwYXRoLCAic29sdXRpb24ucHkiKQogICAgICAgICAgICAgICAgYnJlYWsKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIHB5dGVzdC5mYWlsKGYiYWdlbnQgc29sdXRpb24gbm90IGZvdW5kOiBleHBlY3RlZCBzb2x1dGlvbi9zb2x1dGlvbi5weSB1bmRlciB7cm9vdH0iKQogICAgc3BlYyA9IGltcG9ydGxpYi51dGlsLnNwZWNfZnJvbV9maWxlX2xvY2F0aW9uKCJhZ2VudF9zb2x1dGlvbiIsIGNhbmRpZGF0ZSkKICAgIG1vZHVsZSA9IGltcG9ydGxpYi51dGlsLm1vZHVsZV9mcm9tX3NwZWMoc3BlYykKICAgIHNwZWMubG9hZGVyLmV4ZWNfbW9kdWxlKG1vZHVsZSkKICAgIHJldHVybiBtb2R1bGUKCgpkZWYgdGVzdF9nZXRfZmxhZ192YXJpYW50X2FuZF9wYXlsb2FkX2Zyb21fdjJfbWV0YWRhdGEobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAiZ2V0X2ZsYWdfdmFyaWFudF9wYXlsb2FkIiksIFwKICAgICAgICAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgZ2V0X2ZsYWdfdmFyaWFudF9wYXlsb2FkKGJhc2VfdXJsLCBwcm9qZWN0X2FwaV9rZXksIGRpc3RpbmN0X2lkLCBmbGFnX2tleSkiCgogICAgcmVzdWx0ID0gbW9kdWxlLmdldF9mbGFnX3ZhcmlhbnRfcGF5bG9hZChtb2NrX3NlcnZlciwgRVhQRUNURURfVE9LRU4sICJ1c2VyLTEyMyIsICJjaGVja291dC1leHBlcmltZW50IikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsICh0dXBsZSwgbGlzdCkpIGFuZCBsZW4ocmVzdWx0KSA9PSAyLCAoCiAgICAgICAgZiJleHBlY3RlZCBhICh2YXJpYW50LCBwYXlsb2FkKSBwYWlyLCBnb3Qge3Jlc3VsdCFyfS4gU2VydmVyIHNhdzoge19Nb2NrUG9zdEhvZy5yZXF1ZXN0X2xvZ30iCiAgICApCiAgICB2YXJpYW50LCBwYXlsb2FkID0gcmVzdWx0WzBdLCByZXN1bHRbMV0KCiAgICBhc3NlcnQgdmFyaWFudCA9PSAidmFyaWFudC1iIiwgKAogICAgICAgIGYiZXhwZWN0ZWQgdmFyaWFudCAndmFyaWFudC1iJyBmcm9tIGZsYWdzWydjaGVja291dC1leHBlcmltZW50J11bJ3ZhcmlhbnQnXSwgZ290IHt2YXJpYW50IXJ9LiAiCiAgICAgICAgZiJBIHN0YWxlIGNsaWVudCByZWFkaW5nIHRoZSAvZGVjaWRlIGZlYXR1cmVGbGFncyBtYXAgKHdoZXJlIHRoZSB2YWx1ZSBpdHNlbGYgaXMgdGhlIHZhcmlhbnQgc3RyaW5nKSBvciAiCiAgICAgICAgZiJ0aGUgd3JvbmcgZW5kcG9pbnQgZmFpbHMgaGVyZS4gU2VydmVyIHNhdzoge19Nb2NrUG9zdEhvZy5yZXF1ZXN0X2xvZ30iCiAgICApCgogICAgcGFyc2VkID0ganNvbi5sb2FkcyhwYXlsb2FkKSBpZiBpc2luc3RhbmNlKHBheWxvYWQsIHN0cikgZWxzZSBwYXlsb2FkCiAgICBhc3NlcnQgcGFyc2VkID09IHsiZGlzY291bnRfcGN0IjogMjB9LCAoCiAgICAgICAgZiJleHBlY3RlZCBwYXlsb2FkIHt7J2Rpc2NvdW50X3BjdCc6IDIwfX0gZGVjb2RlZCBmcm9tIGZsYWdzWydjaGVja291dC1leHBlcmltZW50J11bJ21ldGFkYXRhJ11bJ3BheWxvYWQnXSAiCiAgICAgICAgZiIoYSBKU09OLWVuY29kZWQgc3RyaW5nKSwgZ290IHtwYXlsb2FkIXJ9LiBBIHN0YWxlIGNsaWVudCByZWFkcyBhIHRvcC1sZXZlbCBmZWF0dXJlRmxhZ1BheWxvYWRzIG1hcCAoYWJzZW50ICIKICAgICAgICBmImluIHRoZSB2MiBlbnZlbG9wZSkgYW5kIGZhaWxzLiBTZXJ2ZXIgc2F3OiB7X01vY2tQb3N0SG9nLnJlcXVlc3RfbG9nfSIKICAgICkK" + }, + { + "vendor": "posthog:is_feature_enabled", + "fn": "is_feature_enabled", + "testFile": "test_posthog_flags_is_enabled.py", + "signature": "def is_feature_enabled(base_url: str, project_api_key: str, distinct_id: str, flag_key: str) -> bool\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcG9zdGhvZy1mbGFncy12Mi1pcy1lbmFibGVkIChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBQb3N0SG9nJ3MgQ1VSUkVOVCBmZWF0dXJlLWZsYWcKZXZhbHVhdGlvbiBjb250cmFjdDoKICAtIGVuZHBvaW50ICAgICBQT1NUIC9mbGFncz92PTIgICAgICAgICAgICAodGhlIG9sZGVyIC9kZWNpZGUgZW5kcG9pbnQgaXMgbm90IHNlcnZlZCkKICAtIGF1dGggICAgICAgICBwcm9qZWN0IHRva2VuIGFzIGFwaV9rZXkgaW4gdGhlIEpTT04gYm9keQogIC0gcmVzcG9uc2UgICAgIHsiZmxhZ3MiOiB7PGtleT46IHsia2V5IiwiZW5hYmxlZCIsInZhcmlhbnQiPywicmVhc29uIiwibWV0YWRhdGEifX0sIC4uLn0KICAgICAgICAgICAgICAgICAodGhlIG9sZGVyIC9kZWNpZGUgcmV0dXJuZWQgYSBmbGF0IHsiZmVhdHVyZUZsYWdzIjogezxrZXk+OiB0cnVlfCJ2YXJpYW50In19IG1hcCkKCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIFBvc3RIb2cgZG9jcyBQT1NUcyAvZmxhZ3M/dj0yIGFuZCByZWFkcwpmbGFnc1trZXldWyJlbmFibGVkIl0sIHNvIGl0IHJlcG9ydHMgbmV3LWRhc2hib2FyZCBUcnVlIGFuZCBsZWdhY3ktYmFubmVyIEZhbHNlIGFuZCBwYXNzZXMuCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtY3V0b2ZmIG1lbW9yeSBQT1NUcyAvZGVjaWRlLz92PTMgYW5kIHJlYWRzIGZlYXR1cmVGbGFnc1trZXldOwppdCA0MDRzIG9uIHRoZSBlbmRwb2ludCAob3IgS2V5RXJyb3JzIG9uIHRoZSBtaXNzaW5nIGZlYXR1cmVGbGFncyBtYXApIGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9wb3N0aG9nLmNvbS9kb2NzL2FwaS9mbGFncwogIGh0dHBzOi8vcG9zdGhvZy5jb20vZG9jcy9pbnRlZ3JhdGUvZmVhdHVyZS1mbGFncy1jb2RlCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgcHJvamVjdCB0b2tlbiB0aGUgY2xpZW50IG11c3Qgc2VuZCBhcyBgYXBpX2tleWAgaW4gdGhlIEpTT04gYm9keSAodjIgY29udHJhY3QpLgpFWFBFQ1RFRF9UT0tFTiA9ICJwaGNfbGl2ZV9wcm9qZWN0X3Rva2VuX2FiYzEyMyIKCiMgVGhlIGhpZGRlbiBmbGFnIHN0YXRlLiBUaGUgYWdlbnQgY2Fubm90IGhhcmRjb2RlIHRoaXMg4oCUIGl0IGlzIG9ubHkgb2JzZXJ2YWJsZSBieSBhY3R1YWxseQojIFBPU1RpbmcgL2ZsYWdzP3Y9MiBhbmQgcGFyc2luZyB0aGUgdjIgZW52ZWxvcGUuCkZMQUdTID0gewogICAgIm5ldy1kYXNoYm9hcmQiOiB7CiAgICAgICAgImtleSI6ICJuZXctZGFzaGJvYXJkIiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJjb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogMCwgImRlc2NyaXB0aW9uIjogIkNvbmRpdGlvbiBzZXQgMSBtYXRjaGVkIn0sCiAgICAgICAgIm1ldGFkYXRhIjogeyJpZCI6IDEsICJ2ZXJzaW9uIjogMywgInBheWxvYWQiOiBOb25lfSwKICAgIH0sCiAgICAibGVnYWN5LWJhbm5lciI6IHsKICAgICAgICAia2V5IjogImxlZ2FjeS1iYW5uZXIiLAogICAgICAgICJlbmFibGVkIjogRmFsc2UsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJub19jb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogTm9uZSwgImRlc2NyaXB0aW9uIjogIk5vIGNvbmRpdGlvbiBzZXQgbWF0Y2hlZCJ9LAogICAgICAgICJtZXRhZGF0YSI6IHsiaWQiOiAyLCAidmVyc2lvbiI6IDEsICJwYXlsb2FkIjogTm9uZX0sCiAgICB9LAogICAgImNoZWNrb3V0LWV4cGVyaW1lbnQiOiB7CiAgICAgICAgImtleSI6ICJjaGVja291dC1leHBlcmltZW50IiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInZhcmlhbnQiOiAidmFyaWFudC1iIiwKICAgICAgICAicmVhc29uIjogeyJjb2RlIjogImNvbmRpdGlvbl9tYXRjaCIsICJjb25kaXRpb25faW5kZXgiOiAxLCAiZGVzY3JpcHRpb24iOiAiQ29uZGl0aW9uIHNldCAyIG1hdGNoZWQifSwKICAgICAgICAibWV0YWRhdGEiOiB7ImlkIjogMywgInZlcnNpb24iOiA3LCAicGF5bG9hZCI6ICJ7XCJkaXNjb3VudF9wY3RcIjogMjB9In0sCiAgICB9LAp9CgoKY2xhc3MgX01vY2tQb3N0SG9nKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgIyBDbGFzcy1sZXZlbCBsb2cgc28gdGhlIHRlc3QgY2FuIHJlcG9ydCB0aGUgcGF0aC92ZXJzaW9uL2JvZHkgdGhlIGNsaWVudCBhY3R1YWxseSB1c2VkLgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOiAgIyBzaWxlbmNlIHN0ZGVyciBhY2Nlc3MgbG9nCiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19QT1NUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgbGVuZ3RoID0gaW50KHNlbGYuaGVhZGVycy5nZXQoIkNvbnRlbnQtTGVuZ3RoIiwgMCkgb3IgMCkKICAgICAgICByYXcgPSBzZWxmLnJmaWxlLnJlYWQobGVuZ3RoKSBpZiBsZW5ndGggZWxzZSBiIiIKICAgICAgICB0cnk6CiAgICAgICAgICAgIGJvZHkgPSBqc29uLmxvYWRzKHJhdy5kZWNvZGUoKSkgaWYgcmF3IGVsc2Uge30KICAgICAgICBleGNlcHQgRXhjZXB0aW9uOgogICAgICAgICAgICBib2R5ID0gTm9uZQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgInYiOiBxcy5nZXQoInYiLCBbTm9uZV0pWzBdLAogICAgICAgICAgICAiYm9keV9rZXlzIjogc29ydGVkKGJvZHkua2V5cygpKSBpZiBpc2luc3RhbmNlKGJvZHksIGRpY3QpIGVsc2UgTm9uZSwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogY3VycmVudCBmbGFnIGV2YWx1YXRpb24gbGl2ZXMgYXQgL2ZsYWdzICh0aGUgb2xkIC9kZWNpZGUgaXMgZ29uZSBoZXJlKS4KICAgICAgICBpZiBwYXRoICE9ICIvZmxhZ3MiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdCIsICJjb2RlIjogIm5vdF9mb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJ1bmtub3duIGVuZHBvaW50IChQb3N0SG9nIGV2YWx1YXRlcyBmZWF0dXJlIGZsYWdzIGF0IFBPU1QgL2ZsYWdzP3Y9MikifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgdmVyc2lvbiBheGlzOiB0aGUgY3VycmVudCBmbGFncyByZXNwb25zZSBzaGFwZSByZXF1aXJlcyA/dj0yLgogICAgICAgIGlmIHFzLmdldCgidiIsIFtOb25lXSlbMF0gIT0gIjIiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJ0eXBlIjogInZhbGlkYXRpb25fZXJyb3IiLCAiY29kZSI6ICJtaXNzaW5nX3ZlcnNpb24iLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXRhaWwiOiAidGhlIC9mbGFncyBlbmRwb2ludCByZXF1aXJlcyA/dj0yIGZvciB0aGUgY3VycmVudCBmbGFncyByZXNwb25zZSBmb3JtYXQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIG5vdCBpc2luc3RhbmNlKGJvZHksIGRpY3QpOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJ0eXBlIjogInZhbGlkYXRpb25fZXJyb3IiLCAiY29kZSI6ICJtYWxmb3JtZWRfYm9keSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJQT1NUIGJvZHkgbXVzdCBiZSBKU09OIGNvbnRhaW5pbmcgYXBpX2tleSBhbmQgZGlzdGluY3RfaWQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiB0aGUgcHJvamVjdCB0b2tlbiB0cmF2ZWxzIGFzIGFwaV9rZXkgaW4gdGhlIEpTT04gYm9keS4KICAgICAgICBpZiBib2R5LmdldCgiYXBpX2tleSIpICE9IEVYUEVDVEVEX1RPS0VOOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJ0eXBlIjogImF1dGhlbnRpY2F0aW9uX2Vycm9yIiwgImNvZGUiOiAiaW52YWxpZF9hcGlfa2V5IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGV0YWlsIjogInByb3ZpZGUgdGhlIHByb2plY3QgQVBJIGtleSBhcyBhcGlfa2V5IGluIHRoZSBKU09OIGJvZHkifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIG5vdCBib2R5LmdldCgiZGlzdGluY3RfaWQiKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsidHlwZSI6ICJ2YWxpZGF0aW9uX2Vycm9yIiwgImNvZGUiOiAibWlzc2luZ19kaXN0aW5jdF9pZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJkaXN0aW5jdF9pZCBpcyByZXF1aXJlZCB0byBldmFsdWF0ZSBmbGFncyBmb3IgYSB1c2VyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAiZmxhZ3MiOiBGTEFHUywKICAgICAgICAgICAgImVycm9yc1doaWxlQ29tcHV0aW5nRmxhZ3MiOiBGYWxzZSwKICAgICAgICAgICAgInJlcXVlc3RJZCI6ICI1NTBlODQwMC1lMjliLTQxZDQtYTcxNi00NDY2NTU0NDAwMDAiLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1Bvc3RIb2cpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpLnN0YXJ0KCkKICAgIHRyeToKICAgICAgICB5aWVsZCBmImh0dHA6Ly8xMjcuMC4wLjE6e3BvcnR9IgogICAgZmluYWxseToKICAgICAgICBzZXJ2ZXIuc2h1dGRvd24oKQogICAgICAgIHNlcnZlci5zZXJ2ZXJfY2xvc2UoKQoKCmRlZiBfbG9hZF9hZ2VudF9zb2x1dGlvbigpOgogICAgcm9vdCA9IG9zLmVudmlyb24uZ2V0KCJWQl9TT0xVVElPTl9ST09UIiwgb3MuZ2V0Y3dkKCkpCiAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4ocm9vdCwgInNvbHV0aW9uIiwgInNvbHV0aW9uLnB5IikKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIGZvciBkaXJwYXRoLCBfZGlycywgZmlsZXMgaW4gb3Mud2Fsayhyb290KToKICAgICAgICAgICAgaWYgInNvbHV0aW9uLnB5IiBpbiBmaWxlczoKICAgICAgICAgICAgICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihkaXJwYXRoLCAic29sdXRpb24ucHkiKQogICAgICAgICAgICAgICAgYnJlYWsKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIHB5dGVzdC5mYWlsKGYiYWdlbnQgc29sdXRpb24gbm90IGZvdW5kOiBleHBlY3RlZCBzb2x1dGlvbi9zb2x1dGlvbi5weSB1bmRlciB7cm9vdH0iKQogICAgc3BlYyA9IGltcG9ydGxpYi51dGlsLnNwZWNfZnJvbV9maWxlX2xvY2F0aW9uKCJhZ2VudF9zb2x1dGlvbiIsIGNhbmRpZGF0ZSkKICAgIG1vZHVsZSA9IGltcG9ydGxpYi51dGlsLm1vZHVsZV9mcm9tX3NwZWMoc3BlYykKICAgIHNwZWMubG9hZGVyLmV4ZWNfbW9kdWxlKG1vZHVsZSkKICAgIHJldHVybiBtb2R1bGUKCgpkZWYgdGVzdF9pc19mZWF0dXJlX2VuYWJsZWRfcmVhZHNfdjJfZmxhZ3NfZW52ZWxvcGUobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAiaXNfZmVhdHVyZV9lbmFibGVkIiksIFwKICAgICAgICAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgaXNfZmVhdHVyZV9lbmFibGVkKGJhc2VfdXJsLCBwcm9qZWN0X2FwaV9rZXksIGRpc3RpbmN0X2lkLCBmbGFnX2tleSkiCgogICAgZW5hYmxlZCA9IG1vZHVsZS5pc19mZWF0dXJlX2VuYWJsZWQobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX1RPS0VOLCAidXNlci0xMjMiLCAibmV3LWRhc2hib2FyZCIpCiAgICBkaXNhYmxlZCA9IG1vZHVsZS5pc19mZWF0dXJlX2VuYWJsZWQobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX1RPS0VOLCAidXNlci0xMjMiLCAibGVnYWN5LWJhbm5lciIpCgogICAgYXNzZXJ0IGVuYWJsZWQgPT0gVHJ1ZSwgKCAgIyBub3FhOiBFNzEyIC0gcmVqZWN0IGRpY3QvTm9uZS90cnV0aHktb2JqZWN0LCBub3QganVzdCBmYWxzeQogICAgICAgIGYiZXhwZWN0ZWQgbmV3LWRhc2hib2FyZCBlbmFibGVkPVRydWUsIGdvdCB7ZW5hYmxlZCFyfS4gQSBjb3JyZWN0IGNsaWVudCBQT1NUcyAvZmxhZ3M/dj0yIGFuZCByZWFkcyAiCiAgICAgICAgZiJmbGFnc1snbmV3LWRhc2hib2FyZCddWydlbmFibGVkJ10uIFNlcnZlciBzYXc6IHtfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2d9IgogICAgKQogICAgYXNzZXJ0IGRpc2FibGVkID09IEZhbHNlLCAoICAjIG5vcWE6IEU3MTIgLSBhIHRydXRoeSBmbGFnIG9iamVjdCBvciBhIC9kZWNpZGUgZmVhdHVyZUZsYWdzIHJlYWQgZmFpbHMgaGVyZQogICAgICAgIGYiZXhwZWN0ZWQgbGVnYWN5LWJhbm5lciBlbmFibGVkPUZhbHNlLCBnb3Qge2Rpc2FibGVkIXJ9LiBBIHN0YWxlIGNsaWVudCAocmVhZGluZyB0aGUgL2RlY2lkZSBmZWF0dXJlRmxhZ3MgIgogICAgICAgIGYibWFwLCBvciB0cmVhdGluZyB0aGUgZmxhZyBPQkpFQ1QgYXMgdHJ1dGh5KSBnZXRzIHRoaXMgd3JvbmcuIFNlcnZlciBzYXc6IHtfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2d9IgogICAgKQo=" + }, + { + "vendor": "stripe:list_all_events", + "fn": "list_all_events", + "testFile": "test_stripe_list_events.py", + "signature": "def list_all_events(base_url: str, api_key: str, object_id: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgc3RyaXBlLXYyLWxpc3QtZXZlbnRzIChoYXJkKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUKdGltZSAoYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdAp0aGUgYWdlbnQncyBzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgU3RyaXBlJ3MgQ1VSUkVOVCAoQVBJIHYyKQpldmVudHMtbGlzdCBjb250cmFjdCBmb3IgdXNhZ2UtYmlsbGluZyBtZXRlciBldmVudHM6CiAgLSBlbmRwb2ludCBwYXRoICAgICAvdjIvY29yZS9ldmVudHMgICAgICAgICAgICAgICAgICAgICAgICh2MSB3YXMgL3YxL2V2ZW50cykKICAtIHZlcnNpb24gaGVhZGVyICAgIFN0cmlwZS1WZXJzaW9uOiA8WVlZWS1NTS1ERC5jb2RlbmFtZT4gICh2MSByZXF1aXJlZCBOTyB2ZXJzaW9uIGhlYWRlcikKICAtIGF1dGggICAgICAgICAgICAgIEF1dGhvcml6YXRpb246IEJlYXJlciA8a2V5PiAgICAgICAgICAgIChzYW1lIGFjcm9zcyB2MS92MikKICAtIHBhZ2luYXRpb24gICAgICAgIGZvbGxvdyB0aGUgZnVsbC1VUkwgYG5leHRfcGFnZV91cmxgIHVudGlsIGl0IGlzIG51bGwKICAgICAgICAgICAgICAgICAgICAgICh2MSB1c2VkID9zdGFydGluZ19hZnRlcj08bGFzdF9pZD4gKyByZXNwb25zZSBgaGFzX21vcmVgKQoKQSBjbGllbnQgd3JpdHRlbiBmcm9tIENVUlJFTlQgU3RyaXBlIHYyIGRvY3MgY29sbGVjdHMgYWxsIDcgZXZlbnRzIGFjcm9zcyAzIHBhZ2VzCmJ5IGZvbGxvd2luZyBuZXh0X3BhZ2VfdXJsIGFuZCBwYXNzZXMuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtY3V0b2ZmCm1lbW9yeSAodjEgL3YxL2V2ZW50cyBwYXRoLCBubyB2ZXJzaW9uIGhlYWRlciwgb3Igc3RhcnRpbmdfYWZ0ZXIgKyBoYXNfbW9yZQpwYWdpbmF0aW9uKSBoaXRzIDQwNCAvIDQwMCAvIHN0b3BzIGFmdGVyIHBhZ2UgMSBhbmQgRkFJTFMg4oCUIHRoZSB2MiBlbnZlbG9wZSBoYXMKbm8gYGhhc19tb3JlYCBhbmQgaWdub3JlcyBgc3RhcnRpbmdfYWZ0ZXJgLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGktdjItb3ZlcnZpZXcKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGkvdjIvY29yZS9ldmVudHMvbGlzdC5tZAogIGh0dHBzOi8vZG9jcy5zdHJpcGUuY29tL2FwaS9wYWdpbmF0aW9uICAgKHRoZSB2MSBzaGFwZSB0aGUgZnJvbS1tZW1vcnkgY2xpZW50IGVtaXRzKQoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9LRVkgPSAic2tfdGVzdF92Ml9rZXlfYWJjMTIzIgpPQkpFQ1RfSUQgPSAibXRyX3Rlc3RfNjFSQ2ppcWRUREM5MXpnaXA0MUlxUEN6UG54cSIKX1ZFUlNJT05fUkUgPSByZS5jb21waWxlKHIiXlxkezR9LVxkezJ9LVxkezJ9KFwuW2EtejAtOV0rKT8kIikKCiMgNyBiaWxsaW5nLW1ldGVyIGV2ZW50cywgbW9jayBwYWdlIHNpemUgMyAtPiAzIHBhZ2VzLiBPbmx5IG9ic2VydmFibGUgYnkKIyBwYWdpbmF0aW5nIHRoZSBtb2NrIHRvIHRoZSBlbmQ7IHRoZSBhZ2VudCBjYW5ub3QgaGFyZGNvZGUgdGhpcy4KRVZFTlRTID0gWwogICAgewogICAgICAgICJpZCI6IGYiZXZ0X3Rlc3RfbWV0ZXJfe2l9IiwKICAgICAgICAib2JqZWN0IjogInYyLmNvcmUuZXZlbnQiLAogICAgICAgICJ0eXBlIjogInYxLmJpbGxpbmcubWV0ZXIuZXJyb3JfcmVwb3J0X3RyaWdnZXJlZCIsCiAgICAgICAgImNyZWF0ZWQiOiBmIjIwMjYtMDYtezEwICsgaX1UMTc6NDY6MjIuMTM0WiIsCiAgICAgICAgImxpdmVtb2RlIjogRmFsc2UsCiAgICAgICAgInJlbGF0ZWRfb2JqZWN0IjogeyJpZCI6IE9CSkVDVF9JRCwgInR5cGUiOiAiYmlsbGluZy5tZXRlciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICJ1cmwiOiBmIi92MS9iaWxsaW5nL21ldGVycy97T0JKRUNUX0lEfSJ9LAogICAgfQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgOCkKXQpQQUdFX1NJWkUgPSAzCgoKY2xhc3MgX01vY2tTdHJpcGVWMihCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KICAgICMgQ2FwdHVyZWQgb24gdGhlIGZpeHR1cmUgc28gbmV4dF9wYWdlX3VybCBjYW4gYmUgYnVpbHQgYXMgYSBmdWxsIGFic29sdXRlIFVSTCwKICAgICMgbWF0Y2hpbmcgU3RyaXBlIHYyIChuZXh0X3BhZ2VfdXJsIGlzIGEgY29tcGxldGUgVVJMLCBub3QgYSBiYXJlIGN1cnNvcikuCiAgICBvcmlnaW4gPSAiIgoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOgogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgdmVyc2lvbiA9IHNlbGYuaGVhZGVycy5nZXQoIlN0cmlwZS1WZXJzaW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJzdHJpcGVfdmVyc2lvbiI6IHZlcnNpb24sCiAgICAgICAgICAgICJoYXNfYmVhcmVyIjogKHNlbGYuaGVhZGVycy5nZXQoIkF1dGhvcml6YXRpb24iKSBvciAiIikuc3RhcnRzd2l0aCgiQmVhcmVyICIpLAogICAgICAgICAgICAicGFnZSI6IHFzLmdldCgicGFnZSIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJzdGFydGluZ19hZnRlciI6IHFzLmdldCgic3RhcnRpbmdfYWZ0ZXIiLCBbTm9uZV0pWzBdLAogICAgICAgIH0pCgogICAgICAgICMgKDEpIGVuZHBvaW50LXZlcnNpb24gYXhpczogb25seSB0aGUgdjIgZXZlbnRzIHBhdGggZXhpc3RzLgogICAgICAgIGlmIHBhdGggIT0gIi92Mi9jb3JlL2V2ZW50cyI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiVW5yZWNvZ25pemVkIHJlcXVlc3QgVVJMLiBTdHJpcGUgQVBJIHYyIGxpc3RzIGV2ZW50cyBhdCAvdjIvY29yZS9ldmVudHMuIn19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMikgdmVyc2lvbiBheGlzOiB2MiByZXF1aXJlcyBhIHdlbGwtZm9ybWVkIFN0cmlwZS1WZXJzaW9uIGhlYWRlciBvbiBFVkVSWSByZXF1ZXN0LgogICAgICAgIGlmIG5vdCB2ZXJzaW9uIG9yIG5vdCBfVkVSU0lPTl9SRS5tYXRjaCh2ZXJzaW9uKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsiZXJyb3IiOiB7InR5cGUiOiAiaW52YWxpZF9yZXF1ZXN0X2Vycm9yIiwKICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJNaXNzaW5nIG9yIG1hbGZvcm1lZCBTdHJpcGUtVmVyc2lvbiBoZWFkZXIuIFRoZSB2MiBBUEkgcmVxdWlyZXMgU3RyaXBlLVZlcnNpb246IDxZWVlZLU1NLURELmNvZGVuYW1lPi4ifX0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgzKSBhdXRoIGF4aXM6IGJlYXJlciBzZWNyZXQga2V5IG9uIEVWRVJZIHJlcXVlc3QgKHNoYXJlZCBzY2hlbWUgd2l0aCB2MSkuCiAgICAgICAgaWYgc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIsICIiKSAhPSBmIkJlYXJlciB7RVhQRUNURURfS0VZfSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiSW52YWxpZCBBUEkga2V5LiBBdXRoZW50aWNhdGUgd2l0aCBBdXRob3JpemF0aW9uOiBCZWFyZXIgPHNlY3JldCBrZXk+LiJ9fSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDQpIHBhZ2luYXRpb24gYXhpczogY3Vyc29yIHJpZGVzIGFuIG9wYXF1ZSBgcGFnZWAgcGFyYW0sIHN1cmZhY2VkIE9OTFkgdmlhIHRoZQogICAgICAgICMgZnVsbC1VUkwgbmV4dF9wYWdlX3VybC4gYHN0YXJ0aW5nX2FmdGVyYCAodGhlIHYxIGN1cnNvcikgaXMgaWdub3JlZCDigJQgYSB2MSBjbGllbnQKICAgICAgICAjIHRoZXJlZm9yZSByZS1yZWFkcyBwYWdlIDEgZm9yZXZlciBvciwgcmVhZGluZyB0aGUgYWJzZW50IGBoYXNfbW9yZWAsIHN0b3BzIGFmdGVyCiAgICAgICAgIyBvbmUgcGFnZS4KICAgICAgICBwYWdlX3BhcmFtID0gcXMuZ2V0KCJwYWdlIiwgW05vbmVdKVswXQogICAgICAgIG9mZnNldCA9IGludChwYWdlX3BhcmFtKSBpZiBwYWdlX3BhcmFtIGlzIG5vdCBOb25lIGFuZCBwYWdlX3BhcmFtLmlzZGlnaXQoKSBlbHNlIDAKCiAgICAgICAgd2luZG93ID0gRVZFTlRTW29mZnNldDpvZmZzZXQgKyBQQUdFX1NJWkVdCiAgICAgICAgbmV4dF9vZmZzZXQgPSBvZmZzZXQgKyBQQUdFX1NJWkUKICAgICAgICBpZiBuZXh0X29mZnNldCA8IGxlbihFVkVOVFMpOgogICAgICAgICAgICBuZXh0X3VybCA9IGYie3R5cGUoc2VsZikub3JpZ2lufS92Mi9jb3JlL2V2ZW50cz9vYmplY3RfaWQ9e09CSkVDVF9JRH0mcGFnZT17bmV4dF9vZmZzZXR9IgogICAgICAgIGVsc2U6CiAgICAgICAgICAgIG5leHRfdXJsID0gTm9uZQogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7CiAgICAgICAgICAgICJkYXRhIjogd2luZG93LAogICAgICAgICAgICAibmV4dF9wYWdlX3VybCI6IG5leHRfdXJsLAogICAgICAgICAgICAicHJldmlvdXNfcGFnZV91cmwiOiBOb25lLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja1N0cmlwZVYyLnJlcXVlc3RfbG9nID0gW10KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tTdHJpcGVWMikKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIF9Nb2NrU3RyaXBlVjIub3JpZ2luID0gZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIF9Nb2NrU3RyaXBlVjIub3JpZ2luCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9ldmVudHNfYWNyb3NzX3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2V2ZW50cyIpLCBcCiAgICAgICAgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IGxpc3RfYWxsX2V2ZW50cyhiYXNlX3VybCwgYXBpX2tleSwgb2JqZWN0X2lkKSIKCiAgICByZXN1bHQgPSBtb2R1bGUubGlzdF9hbGxfZXZlbnRzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9LRVksIE9CSkVDVF9JRCkKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBldmVudHMsIGdvdCB7dHlwZShyZXN1bHQpLl9fbmFtZV9ffSIKICAgIGlkcyA9IHNvcnRlZChlWyJpZCJdIGZvciBlIGluIHJlc3VsdCBpZiBpc2luc3RhbmNlKGUsIGRpY3QpIGFuZCAiaWQiIGluIGUpCiAgICBleHBlY3RlZF9pZHMgPSBzb3J0ZWQoZVsiaWQiXSBmb3IgZSBpbiBFVkVOVFMpCiAgICBhc3NlcnQgaWRzID09IGV4cGVjdGVkX2lkcywgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIGV2ZW50IGlkcyB7aWRzfSwgZXhwZWN0ZWQge2V4cGVjdGVkX2lkc30uIEEgY29ycmVjdCB2MiBjbGllbnQgZm9sbG93cyAiCiAgICAgICAgZiJuZXh0X3BhZ2VfdXJsIChhIGZ1bGwgVVJMKSB0byB0aGUgZW5kOyBpdCBkb2VzIE5PVCB1c2UgdjEgc3RhcnRpbmdfYWZ0ZXIvaGFzX21vcmUuICIKICAgICAgICBmIlNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja1N0cmlwZVYyLnJlcXVlc3RfbG9nfSIKICAgICkK" + }, + { + "vendor": "stripe:get_recent_events", + "fn": "get_recent_events", + "testFile": "test_stripe_get_events.py", + "signature": "def get_recent_events(base_url: str, api_key: str, object_id: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgc3RyaXBlLXYyLWdldC1ldmVudHMgKG1lZGl1bSkgbGVhZi4KClNpbXBsZXIgc2xpY2Ugb2YgdGhlIFNBTUUgU3RyaXBlIEFQSSB2MiBjb250cmFjdCDigJQgYSBzaW5nbGUtcGFnZSBHRVQgb2YgdGhlCnVzYWdlLWJpbGxpbmcgbWV0ZXIgZXZlbnQgc3RyZWFtLCBubyBwYWdpbmF0aW9uLiBTdGlsbCBzZWFyY2gtbmVjZXNzYXJ5IG9uIHR3bwpheGVzOiB0aGUgdjIgZW5kcG9pbnQgbmFtZXNwYWNlIGFuZCB0aGUgTUFOREFUT1JZIFN0cmlwZS1WZXJzaW9uIGhlYWRlciAoYQpmcm9tLW1lbW9yeSB2MSBjbGllbnQgaGl0cyAvdjEvZXZlbnRzIHdpdGggbm8gdmVyc2lvbiBoZWFkZXIgYW5kIGZhaWxzKS4KCiAgLSBlbmRwb2ludCBwYXRoICAgICAvdjIvY29yZS9ldmVudHMgICAgICAgICAgICAgICh2MSB3YXMgL3YxL2V2ZW50cykKICAtIHZlcnNpb24gaGVhZGVyICAgIFN0cmlwZS1WZXJzaW9uOiA8WVlZWS1NTS1ERC5jb2RlbmFtZT4gICAodjEgcmVxdWlyZWQgTk8gdmVyc2lvbiBoZWFkZXIpCiAgLSBhdXRoICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGtleT4gIChzYW1lIGFjcm9zcyB2MS92MikKICAtIHJlc3BvbnNlIGFycmF5ICAgIGRhdGEgICAgICAgICAgICAgICAgICAgICAgICAgIChwYXJzZWQgZnJvbSB0aGUgSlNPTiBib2R5KQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGktdjItb3ZlcnZpZXcKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGkvdjIvY29yZS9ldmVudHMKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGkvdjIvY29yZS9ldmVudHMvbGlzdC5tZAoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgojIFRoZSBiZWFyZXIgc2VjcmV0IGtleSB0aGUgY2xpZW50IG11c3Qgc2VuZCAoc2FtZSBhdXRoIHNjaGVtZSBhcyB2MSkuCkVYUEVDVEVEX0tFWSA9ICJza190ZXN0X3YyX2tleV9hYmMxMjMiCiMgVGhlIG1ldGVyIHdob3NlIGV2ZW50cyB0aGUgY2xpZW50IGZldGNoZXMgKGEgdXNhZ2UtYmlsbGluZyBtZXRlciBpZCkuCk9CSkVDVF9JRCA9ICJtdHJfdGVzdF82MVJDamlxZFREQzkxemdpcDQxSXFQQ3pQbnhxIgojIHYyIG1hbmRhdGVzIGEgU3RyaXBlLVZlcnNpb24gaGVhZGVyIHNoYXBlZCBZWVlZLU1NLUREIHdpdGggYW4gb3B0aW9uYWwgY29kZW5hbWUKIyAoZS5nLiAyMDI2LTA2LTI0LmRhaGxpYSkuIFdlIGFjY2VwdCBhbnkgd2VsbC1mb3JtZWQgZGF0ZStvcHRpb25hbC1jb2RlbmFtZSBzbyBhCiMgY2xpZW50IHRoYXQgcmVhZHMgdGhlIGxpdmUgZG9jcyBhbmQgcGlja3MgYW55IGRvY3VtZW50ZWQgdmVyc2lvbiBwYXNzZXM7IHRoZQojIGRpc2NvdmVyYWJsZSBmYWN0IHVuZGVyIHRlc3QgaXMgdGhhdCB0aGUgaGVhZGVyIG11c3QgYmUgUFJFU0VOVCBhdCBhbGwuCl9WRVJTSU9OX1JFID0gcmUuY29tcGlsZShyIl5cZHs0fS1cZHsyfS1cZHsyfShcLlthLXowLTldKyk/JCIpCgojIFNpbmdsZSBwYWdlIG9mIGJpbGxpbmctbWV0ZXIgZXZlbnRzIGZvciBPQkpFQ1RfSUQuIE9ubHkgb2JzZXJ2YWJsZSBieSBjYWxsaW5nCiMgdGhlIG1vY2s7IHRoZSBhZ2VudCBjYW5ub3QgaGFyZGNvZGUgaXQuCkVWRU5UUyA9IFsKICAgIHsKICAgICAgICAiaWQiOiBmImV2dF90ZXN0X21ldGVyX3tpfSIsCiAgICAgICAgIm9iamVjdCI6ICJ2Mi5jb3JlLmV2ZW50IiwKICAgICAgICAidHlwZSI6ICJ2MS5iaWxsaW5nLm1ldGVyLmVycm9yX3JlcG9ydF90cmlnZ2VyZWQiLAogICAgICAgICJjcmVhdGVkIjogZiIyMDI2LTA2LTJ7aX1UMTc6NDY6MjIuMTM0WiIsCiAgICAgICAgImxpdmVtb2RlIjogRmFsc2UsCiAgICAgICAgInJlbGF0ZWRfb2JqZWN0IjogeyJpZCI6IE9CSkVDVF9JRCwgInR5cGUiOiAiYmlsbGluZy5tZXRlciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICJ1cmwiOiBmIi92MS9iaWxsaW5nL21ldGVycy97T0JKRUNUX0lEfSJ9LAogICAgfQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgNCkKXQoKCmNsYXNzIF9Nb2NrU3RyaXBlVjIoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICB2ZXJzaW9uID0gc2VsZi5oZWFkZXJzLmdldCgiU3RyaXBlLVZlcnNpb24iKQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgInN0cmlwZV92ZXJzaW9uIjogdmVyc2lvbiwKICAgICAgICAgICAgImhhc19iZWFyZXIiOiAoc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpIG9yICIiKS5zdGFydHN3aXRoKCJCZWFyZXIgIiksCiAgICAgICAgICAgICJzdGFydGluZ19hZnRlciI6IHFzLmdldCgic3RhcnRpbmdfYWZ0ZXIiLCBbTm9uZV0pWzBdLAogICAgICAgIH0pCgogICAgICAgICMgKDEpIGVuZHBvaW50LXZlcnNpb24gYXhpczogb25seSB0aGUgdjIgZXZlbnRzIHBhdGggZXhpc3RzLgogICAgICAgIGlmIHBhdGggIT0gIi92Mi9jb3JlL2V2ZW50cyI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiVW5yZWNvZ25pemVkIHJlcXVlc3QgVVJMLiBTdHJpcGUgQVBJIHYyIGxpc3RzIGV2ZW50cyBhdCAvdjIvY29yZS9ldmVudHMuIn19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMikgdmVyc2lvbiBheGlzOiB2MiByZXF1aXJlcyBhIHdlbGwtZm9ybWVkIFN0cmlwZS1WZXJzaW9uIGhlYWRlci4KICAgICAgICBpZiBub3QgdmVyc2lvbiBvciBub3QgX1ZFUlNJT05fUkUubWF0Y2godmVyc2lvbik6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAwLCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiTWlzc2luZyBvciBtYWxmb3JtZWQgU3RyaXBlLVZlcnNpb24gaGVhZGVyLiBUaGUgdjIgQVBJIHJlcXVpcmVzIFN0cmlwZS1WZXJzaW9uOiA8WVlZWS1NTS1ERC5jb2RlbmFtZT4uIn19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMykgYXV0aCBheGlzOiBiZWFyZXIgc2VjcmV0IGtleSAoc2hhcmVkIHdpdGggdjEpLgogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIiwgIiIpCiAgICAgICAgaWYgYXV0aCAhPSBmIkJlYXJlciB7RVhQRUNURURfS0VZfSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiSW52YWxpZCBBUEkga2V5LiBBdXRoZW50aWNhdGUgd2l0aCBBdXRob3JpemF0aW9uOiBCZWFyZXIgPHNlY3JldCBrZXk+LiJ9fSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7CiAgICAgICAgICAgICJkYXRhIjogRVZFTlRTLAogICAgICAgICAgICAibmV4dF9wYWdlX3VybCI6IE5vbmUsCiAgICAgICAgICAgICJwcmV2aW91c19wYWdlX3VybCI6IE5vbmUsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrU3RyaXBlVjIucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1N0cmlwZVYyKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X3JlY2VudF9ldmVudHMobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAiZ2V0X3JlY2VudF9ldmVudHMiKSwgXAogICAgICAgICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfcmVjZW50X2V2ZW50cyhiYXNlX3VybCwgYXBpX2tleSwgb2JqZWN0X2lkKSIKCiAgICBldmVudHMgPSBtb2R1bGUuZ2V0X3JlY2VudF9ldmVudHMobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX0tFWSwgT0JKRUNUX0lEKQoKICAgIGFzc2VydCBpc2luc3RhbmNlKGV2ZW50cywgbGlzdCksIGYiZXhwZWN0ZWQgYSBsaXN0IG9mIGV2ZW50cywgZ290IHt0eXBlKGV2ZW50cykuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGVbImlkIl0gZm9yIGUgaW4gZXZlbnRzIGlmIGlzaW5zdGFuY2UoZSwgZGljdCkgYW5kICJpZCIgaW4gZSkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChlWyJpZCJdIGZvciBlIGluIEVWRU5UUykKICAgIGFzc2VydCBpZHMgPT0gZXhwZWN0ZWRfaWRzLCAoCiAgICAgICAgZiJjbGllbnQgcmV0dXJuZWQgZXZlbnQgaWRzIHtpZHN9LCBleHBlY3RlZCB7ZXhwZWN0ZWRfaWRzfS4gQSBjb3JyZWN0IHYyIGNsaWVudCBHRVRzICIKICAgICAgICBmIi92Mi9jb3JlL2V2ZW50cyB3aXRoIGEgU3RyaXBlLVZlcnNpb24gaGVhZGVyIGFuZCByZWFkcyB0aGUgYGRhdGFgIGFycmF5LiAiCiAgICAgICAgZiJTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tTdHJpcGVWMi5yZXF1ZXN0X2xvZ30iCiAgICApCiAgICBhc3NlcnQgYWxsKGUuZ2V0KCJvYmplY3QiKSA9PSAidjIuY29yZS5ldmVudCIgZm9yIGUgaW4gZXZlbnRzKSwgKAogICAgICAgIGYiZXhwZWN0ZWQgdjIuY29yZS5ldmVudCBvYmplY3RzLiBTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tTdHJpcGVWMi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" + }, + { + "vendor": "novu:list_all_subscribers", + "fn": "list_all_subscribers", + "testFile": "test_novu_list_subscribers.py", + "signature": "def list_all_subscribers(base_url: str, api_key: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgbm92dS12Mi1saXN0LWFsbC1zdWJzY3JpYmVycyAoaGFyZCkgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpJdCBzdGFuZHMgdXAgYSBMT0NBTCBtb2NrIHRoYXQgZmFpdGhmdWxseSBpbXBsZW1lbnRzIE5vdnUncyBDVVJSRU5UIHN1YnNjcmliZXJzLWxpc3QKY29udHJhY3Qgb24gdGhyZWUgYXhlczoKICAtIGVuZHBvaW50IHBhdGggICAvdjIvc3Vic2NyaWJlcnMgICAgICAgICAgICAgICAgKGEgZnJvbS1tZW1vcnkgY2xpZW50IHVzZXMgL3YxL3N1YnNjcmliZXJzKQogIC0gYXV0aCAgICAgICAgICAgIEF1dGhvcml6YXRpb246IEFwaUtleSA8dG9rZW4+ICAgKE5vdnUgZG9lcyBOT1QgdXNlIEJlYXJlcjsgZG9jcyB3YXJuIGFnYWluc3QgaXQpCiAgLSBwYWdpbmF0aW9uICAgICAgY3Vyc29yOiBxdWVyeSBgYWZ0ZXJgICsgcmVzcG9uc2UgYG5leHRgIChvcGFxdWUgY3Vyc29yLCBudWxsID0gZW5kKSwKICAgICAgICAgICAgICAgICAgICBpbnNpZGUgZW52ZWxvcGUge2RhdGEsIG5leHQsIHByZXZpb3VzLCB0b3RhbENvdW50LCB0b3RhbENvdW50Q2FwcGVkfQogICAgICAgICAgICAgICAgICAgIChhIGZyb20tbWVtb3J5IGNsaWVudCB1c2VzIHBhZ2UvbGltaXQgb2Zmc2V0IHBhZ2luYXRpb24gYW5kIGEgYGhhc01vcmVgCiAgICAgICAgICAgICAgICAgICAgIGZsYWcgdGhlIHYyIGVudmVsb3BlIG5ldmVyIHNlbmRzIC0+IHN0b3BzIGFmdGVyIHRoZSBmaXJzdCBwYWdlKQoKQSBjbGllbnQgd3JpdHRlbiBmcm9tIENVUlJFTlQgTm92dSBkb2NzIGNvbGxlY3RzIGFsbCA3IHN1YnNjcmliZXJzIGFjcm9zcyAzIGN1cnNvciBwYWdlcyBhbmQKcGFzc2VzLiBBIGNsaWVudCB3cml0dGVuIGZyb20gc3RhbGUvcHJlLWN1dG9mZiBtZW1vcnkgKHYxIHBhdGgsIEJlYXJlciBhdXRoLCBvciBwYWdlLW51bWJlcgpvZmZzZXQgcGFnaW5hdGlvbikgaGl0cyA0MDQgLyA0MDEgLyBzdG9wcyBhZnRlciBwYWdlIDEgYW5kIEZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Mubm92dS5jby9hcGktcmVmZXJlbmNlL2F1dGhlbnRpY2F0aW9uCiAgaHR0cHM6Ly9kb2NzLm5vdnUuY28vYXBpLXJlZmVyZW5jZS9zdWJzY3JpYmVycy9zZWFyY2gtc3Vic2NyaWJlcnMKIiIiCgppbXBvcnQgYmFzZTY0CmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX1RPS0VOID0gIm52X3NlY3JldF9rZXlfYWJjMTIzIgoKIyBUaGUgaGlkZGVuIGRhdGFzZXQuIFRoZSBhZ2VudCBjYW5ub3QgaGFyZGNvZGUgdGhpcyDigJQgaXQgaXMgb25seSBvYnNlcnZhYmxlIGJ5IGFjdHVhbGx5CiMgY2FsbGluZyB0aGUgbW9jayBhbmQgZm9sbG93aW5nIHRoZSBjdXJzb3IgdG8gdGhlIGVuZC4gNyBzdWJzY3JpYmVycywgc2VydmVyIHBhZ2Ugc2l6ZSAzLgpTVUJTQ1JJQkVSUyA9IFsKICAgIHsiX2lkIjogZiJfaWRfe2l9IiwgInN1YnNjcmliZXJJZCI6IGYidXNlcl97aX0iLCAiZW1haWwiOiBmInVzZXJ7aX1AZXhhbXBsZS5jb20ifQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgOCkKXQpQQUdFX1NJWkUgPSAzCgoKZGVmIF9lbmNvZGVfY3Vyc29yKG9mZnNldDogaW50KSAtPiBzdHI6CiAgICAjIE9wYXF1ZSBjdXJzb3Igc3RyaW5nLCBtYXRjaGluZyBOb3Z1J3Mgb3BhcXVlLWN1cnNvciBjb250cmFjdC4KICAgIHJldHVybiBiYXNlNjQudXJsc2FmZV9iNjRlbmNvZGUoc3RyKG9mZnNldCkuZW5jb2RlKCkpLmRlY29kZSgpCgoKZGVmIF9kZWNvZGVfY3Vyc29yKGN1cnNvcjogc3RyKSAtPiBpbnQ6CiAgICByZXR1cm4gaW50KGJhc2U2NC51cmxzYWZlX2I2NGRlY29kZShjdXJzb3IuZW5jb2RlKCkpLmRlY29kZSgpKQoKCmNsYXNzIF9Nb2NrTm92dShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImFwaWtleV9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkFwaUtleSAiKSwKICAgICAgICAgICAgImJlYXJlcl9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkJlYXJlciAiKSwKICAgICAgICAgICAgImFmdGVyIjogcXMuZ2V0KCJhZnRlciIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJwYWdlIjogcXMuZ2V0KCJwYWdlIiwgW05vbmVdKVswXSwKICAgICAgICB9KQoKICAgICAgICAjICgxKSBlbmRwb2ludC12ZXJzaW9uIGF4aXM6IG9ubHkgdGhlIHYyIHBhdGggZXhpc3RzLgogICAgICAgIGlmIHBhdGggIT0gIi92Mi9zdWJzY3JpYmVycyI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN0YXR1c0NvZGUiOiA0MDQsICJtZXNzYWdlIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yIjogIk5vdnUgQVBJIHYyIGxpc3RzIHN1YnNjcmliZXJzIGF0IC92Mi9zdWJzY3JpYmVycyJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMikgYXV0aCBheGlzOiB2MiByZXF1aXJlcyBgQXV0aG9yaXphdGlvbjogQXBpS2V5IDx0b2tlbj5gOyBCZWFyZXIgaXMgbm90IGhvbm9yZWQuCiAgICAgICAgaWYgYXV0aCAhPSBmIkFwaUtleSB7RVhQRUNURURfVE9LRU59IjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsic3RhdHVzQ29kZSI6IDQwMSwgIm1lc3NhZ2UiOiAidW5hdXRob3JpemVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3IiOiAiTm92dSBhdXRoZW50aWNhdGVzIHZpYSBgQXV0aG9yaXphdGlvbjogQXBpS2V5IDxzZWNyZXQ+YCAobm90IEJlYXJlcikifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDMpIHBhZ2luYXRpb24gYXhpczogY3Vyc29yLWJhc2VkIHZpYSBgYWZ0ZXJgLiBwYWdlLW51bWJlciBvZmZzZXQgaXMgaWdub3JlZCAodjIgaGFzIG5vIGBwYWdlYCkuCiAgICAgICAgYWZ0ZXIgPSBxcy5nZXQoImFmdGVyIiwgW05vbmVdKVswXQogICAgICAgIGlmIGFmdGVyIGlzIE5vbmU6CiAgICAgICAgICAgIG9mZnNldCA9IDAKICAgICAgICBlbHNlOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBvZmZzZXQgPSBfZGVjb2RlX2N1cnNvcihhZnRlcikKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAgICAgICAgIHNlbGYuX2pzb24oNDAwLCB7InN0YXR1c0NvZGUiOiA0MDAsICJtZXNzYWdlIjogImludmFsaWQgY3Vyc29yIn0pCiAgICAgICAgICAgICAgICByZXR1cm4KCiAgICAgICAgcGFnZSA9IFNVQlNDUklCRVJTW29mZnNldDpvZmZzZXQgKyBQQUdFX1NJWkVdCiAgICAgICAgbmV4dF9vZmZzZXQgPSBvZmZzZXQgKyBQQUdFX1NJWkUKICAgICAgICBuZXh0X2N1cnNvciA9IF9lbmNvZGVfY3Vyc29yKG5leHRfb2Zmc2V0KSBpZiBuZXh0X29mZnNldCA8IGxlbihTVUJTQ1JJQkVSUykgZWxzZSBOb25lCiAgICAgICAgcHJldl9jdXJzb3IgPSBfZW5jb2RlX2N1cnNvcihtYXgob2Zmc2V0IC0gUEFHRV9TSVpFLCAwKSkgaWYgb2Zmc2V0ID4gMCBlbHNlIE5vbmUKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAiZGF0YSI6IHBhZ2UsCiAgICAgICAgICAgICJuZXh0IjogbmV4dF9jdXJzb3IsCiAgICAgICAgICAgICJwcmV2aW91cyI6IHByZXZfY3Vyc29yLAogICAgICAgICAgICAidG90YWxDb3VudCI6IGxlbihTVUJTQ1JJQkVSUyksCiAgICAgICAgICAgICJ0b3RhbENvdW50Q2FwcGVkIjogRmFsc2UsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrTm92dS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrTm92dSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9zdWJzY3JpYmVyc19hY3Jvc3NfY3Vyc29yX3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX3N1YnNjcmliZXJzIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBsaXN0X2FsbF9zdWJzY3JpYmVycyhiYXNlX3VybCwgYXBpX2tleSkiCgogICAgcmVzdWx0ID0gbW9kdWxlLmxpc3RfYWxsX3N1YnNjcmliZXJzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBzdWJzY3JpYmVycywgZ290IHt0eXBlKHJlc3VsdCkuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKHNbInN1YnNjcmliZXJJZCJdIGZvciBzIGluIHJlc3VsdCBpZiBpc2luc3RhbmNlKHMsIGRpY3QpIGFuZCAic3Vic2NyaWJlcklkIiBpbiBzKQogICAgZXhwZWN0ZWRfaWRzID0gc29ydGVkKHNbInN1YnNjcmliZXJJZCJdIGZvciBzIGluIFNVQlNDUklCRVJTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBzdWJzY3JpYmVyIGlkcyB7aWRzfSwgZXhwZWN0ZWQge2V4cGVjdGVkX2lkc30uICIKICAgICAgICBmIkEgY29ycmVjdCB2MiBjbGllbnQgZm9sbG93cyB0aGUgYG5leHRgIGN1cnNvciB0byB0aGUgZW5kLiAiCiAgICAgICAgZiJTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tOb3Z1LnJlcXVlc3RfbG9nfSIKICAgICkK" + }, + { + "vendor": "novu:get_subscriber", + "fn": "get_subscriber", + "testFile": "test_novu_get_subscriber.py", + "signature": "def get_subscriber(base_url: str, api_key: str, subscriber_id: str) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgbm92dS12Mi1nZXQtc3Vic2NyaWJlciAobWVkaXVtKSBsZWFmLgoKU2ltcGxlciBzbGljZSBvZiB0aGUgU0FNRSBOb3Z1IEFQSSBjb250cmFjdCDigJQgYSBzaW5nbGUtcmVzb3VyY2UgR0VULCBubyBwYWdpbmF0aW9uLgpTdGlsbCBzZWFyY2gtbmVjZXNzYXJ5IG9uIHR3byBheGVzOiB0aGUgdjIgZW5kcG9pbnQgcGF0aCBhbmQgdGhlIE5PTi1TVEFOREFSRCBBcGlLZXkKYXV0aCBzY2hlbWUgKGEgZnJvbS1tZW1vcnkgY2xpZW50IHVzZXMgQmVhcmVyIGF1dGggYW5kL29yIHRoZSBvbGRlciAvdjEgcGF0aCBhbmQgZmFpbHMpLgoKICAtIGVuZHBvaW50IHBhdGggICAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0gICAoYSBmcm9tLW1lbW9yeSBjbGllbnQgdXNlcyAvdjEvc3Vic2NyaWJlcnMve2lkfSkKICAtIGF1dGggICAgICAgICAgICBBdXRob3JpemF0aW9uOiBBcGlLZXkgPHRva2VuPiAgICAgKE5vdnUgZG9lcyBOT1QgdXNlIEJlYXJlcjsgdGhlIGRvY3MKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBleHBsaWNpdGx5IHdhcm4gYWdhaW5zdCB0aGUgQmVhcmVyIHByZWZpeCkKICAtIHJlc3BvbnNlIHNoYXBlICB0aGUgc3Vic2NyaWJlciBvYmplY3QgaXMgcmV0dXJuZWQgRElSRUNUTFkgKG5vIGBkYXRhYCB3cmFwcGVyKQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Mubm92dS5jby9hcGktcmVmZXJlbmNlL2F1dGhlbnRpY2F0aW9uCiAgaHR0cHM6Ly9kb2NzLm5vdnUuY28vYXBpLXJlZmVyZW5jZS9zdWJzY3JpYmVycy9yZXRyaWV2ZS1hLXN1YnNjcmliZXIKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgc2VjcmV0IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBBdXRob3JpemF0aW9uIGhlYWRlciwgcHJlZml4ZWQgd2l0aCBgQXBpS2V5YC4KRVhQRUNURURfVE9LRU4gPSAibnZfc2VjcmV0X2tleV9hYmMxMjMiClNVQlNDUklCRVJfSUQgPSAidXNlcl80MiIgICMgdGhlIGlkIHRoZSB0ZXN0IGZldGNoZXM7IHRoZSBtb2NrIHN5bnRoZXNpemVzIGFueSB2YWxpZCBpZAoKCmNsYXNzIF9Nb2NrTm92dShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImFwaWtleV9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkFwaUtleSAiKSwKICAgICAgICAgICAgImJlYXJlcl9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkJlYXJlciAiKSwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogdjIgZmV0Y2hlcyBvbmUgc3Vic2NyaWJlciBhdCAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi92Mi9zdWJzY3JpYmVycy8oW14vXSspIiwgcGF0aCkKICAgICAgICBpZiBub3QgbToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsic3RhdHVzQ29kZSI6IDQwNCwgIm1lc3NhZ2UiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3IiOiAiTm92dSBBUEkgdjIgcmV0cmlldmVzIGEgc3Vic2NyaWJlciBhdCAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiBOb3Z1IGF1dGhlbnRpY2F0ZXMgd2l0aCBgQXV0aG9yaXphdGlvbjogQXBpS2V5IDx0b2tlbj5gIChOT1QgQmVhcmVyKS4KICAgICAgICBpZiBhdXRoICE9IGYiQXBpS2V5IHtFWFBFQ1RFRF9UT0tFTn0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJzdGF0dXNDb2RlIjogNDAxLCAibWVzc2FnZSI6ICJ1bmF1dGhvcml6ZWQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6ICJOb3Z1IGF1dGhlbnRpY2F0ZXMgdmlhIGBBdXRob3JpemF0aW9uOiBBcGlLZXkgPHNlY3JldD5gIChub3QgQmVhcmVyKSJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc3Vic2NyaWJlcl9pZCA9IG0uZ3JvdXAoMSkKICAgICAgICAjIHJlc3BvbnNlLXNoYXBlIGF4aXM6IHRoZSBzdWJzY3JpYmVyIG9iamVjdCBpcyByZXR1cm5lZCBESVJFQ1RMWSAobm8gYGRhdGFgIHdyYXBwZXIpLgogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7CiAgICAgICAgICAgICJfaWQiOiBmIl9pZF97c3Vic2NyaWJlcl9pZH0iLAogICAgICAgICAgICAic3Vic2NyaWJlcklkIjogc3Vic2NyaWJlcl9pZCwKICAgICAgICAgICAgImZpcnN0TmFtZSI6ICJBZGEiLAogICAgICAgICAgICAibGFzdE5hbWUiOiAiTG92ZWxhY2UiLAogICAgICAgICAgICAiZW1haWwiOiBmIntzdWJzY3JpYmVyX2lkfUBleGFtcGxlLmNvbSIsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrTm92dS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrTm92dSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9zdWJzY3JpYmVyX2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9zdWJzY3JpYmVyIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfc3Vic2NyaWJlcihiYXNlX3VybCwgYXBpX2tleSwgc3Vic2NyaWJlcl9pZCkiCgogICAgc3Vic2NyaWJlciA9IG1vZHVsZS5nZXRfc3Vic2NyaWJlcihtb2NrX3NlcnZlciwgRVhQRUNURURfVE9LRU4sIFNVQlNDUklCRVJfSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2Uoc3Vic2NyaWJlciwgZGljdCksIGYiZXhwZWN0ZWQgYSBzdWJzY3JpYmVyIGRpY3QsIGdvdCB7dHlwZShzdWJzY3JpYmVyKS5fX25hbWVfX30iCiAgICBhc3NlcnQgc3Vic2NyaWJlci5nZXQoInN1YnNjcmliZXJJZCIpID09IFNVQlNDUklCRVJfSUQgYW5kIHN1YnNjcmliZXIuZ2V0KCJfaWQiKSA9PSBmIl9pZF97U1VCU0NSSUJFUl9JRH0iLCAoCiAgICAgICAgZiJjbGllbnQgcmV0dXJuZWQge3N1YnNjcmliZXIhcn07IGEgY29ycmVjdCB2MiBjbGllbnQgR0VUcyAvdjIvc3Vic2NyaWJlcnMve1NVQlNDUklCRVJfSUR9IHdpdGggdGhlICIKICAgICAgICBmImBBdXRob3JpemF0aW9uOiBBcGlLZXlgIGhlYWRlciBhbmQgcmV0dXJucyB0aGUgc3Vic2NyaWJlciBvYmplY3QuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja05vdnUucmVxdWVzdF9sb2d9IgogICAgKQo=" + }, + { + "vendor": "honeycomb:run_query", + "fn": "run_query", + "testFile": "test_honeycomb_run_query.py", + "signature": "def run_query(base_url: str, api_key: str, dataset_slug: str, query: dict) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgaG9uZXljb21iLXJ1bi1xdWVyeSAoaGFyZCkgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpJdCBzdGFuZHMgdXAgYSBMT0NBTCBtb2NrIHRoYXQgZmFpdGhmdWxseSBpbXBsZW1lbnRzIEhvbmV5Y29tYidzIENVUlJFTlQgKEFQSSB2MSkgUXVlcnkgRGF0YQpjb250cmFjdCBvbiBUSFJFRSBzZWFyY2gtbmVjZXNzYXJ5IGF4ZXM6CiAgLSBhdXRoICAgICAgICBYLUhvbmV5Y29tYi1UZWFtOiA8a2V5PiAgICAgICAgICAoY3VzdG9tIGhlYWRlciwgTk8gQXV0aG9yaXphdGlvbjogQmVhcmVyKQogIC0gZW5kcG9pbnQgICAgLzEvcXVlcmllcyArIC8xL3F1ZXJ5X3Jlc3VsdHMgICAgKHZlcnNpb24gcHJlZml4IGlzIHRoZSBsaXRlcmFsICIxIikKICAtIGZsb3cgICAgICAgIGFzeW5jIFRIUkVFLVNURVAgKyBwb2xsICAgICAgICAgIChjcmVhdGUgYSBRdWVyeSwgY3JlYXRlIGEgUXVlcnkgUmVzdWx0IHRoYXQKICAgICAgICAgICAgICAgIHJlZmVyZW5jZXMgdGhlIHF1ZXJ5IGlkLCB0aGVuIEdFVC1wb2xsIHRoZSBxdWVyeS1yZXN1bHQgZW5kcG9pbnQgdW50aWwKICAgICAgICAgICAgICAgIGNvbXBsZXRlID09IHRydWUgYW5kIHJlYWQgZGF0YS5yZXN1bHRzKQoKR2V0dGluZyByZXN1bHRzIGlzIE5PVCBhIHNpbmdsZSBzeW5jaHJvbm91cyBjYWxsLiBBIGZyb20tbWVtb3J5IGNsaWVudCB0aGF0IFBPU1RzIGEgcXVlcnkgYW5kCmV4cGVjdHMgcm93cyBiYWNrIGluIHRoZSByZXNwb25zZSwgb3IgcmVhZHMgcmVzdWx0cyBiZWZvcmUgY29tcGxldGUgPT0gdHJ1ZSwgb3IgdXNlcyBCZWFyZXIKYXV0aCwgRkFJTFMuIEFuIGVtcHR5IHNvbHV0aW9uIEZBSUxTLiBPbmx5IGEgY2xpZW50IGJ1aWx0IGZyb20gdGhlIGN1cnJlbnQgSG9uZXljb21iIFF1ZXJ5IERhdGEKZG9jcyBwYXNzZXMuIFRoZSByZXN1bHQgcm93cyBhcmUgb25seSBvYnNlcnZhYmxlIGJ5IHJ1bm5pbmcgdGhlIGZ1bGwgZmxvdywgc28gdGhlIGFuc3dlciBjYW5ub3QKYmUgaGFyZGNvZGVkLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3MuaG9uZXljb21iLmlvL2FwaS9hdXRoCiAgaHR0cHM6Ly9hcGktZG9jcy5ob25leWNvbWIuaW8vYXBpL3F1ZXJ5LWRhdGEKICBodHRwczovL2FwaS1kb2NzLmhvbmV5Y29tYi5pby9hcGkvcXVlcmllcy9jcmVhdGVxdWVyeQoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX0tFWSA9ICJoY2FpY19ob25leWNvbWJfY29uZmlnX2tleV9hYmMxMjMiCkRBVEFTRVRfU0xVRyA9ICJjaGVja291dC1zZXJ2aWNlIgoKIyBUaGUgaGlkZGVuIHJlc3VsdCByb3dzLiBIb25leWNvbWIgcmV0dXJucyBxdWVyeSByZXN1bHRzIGFzIGEgbGlzdCBvZiB7ImRhdGEiOiB7Li4ufX0gb2JqZWN0cy4KIyBPbmx5IG9ic2VydmFibGUgYnkgcnVubmluZyB0aGUgcXVlcnkgYW5kIHBvbGxpbmcgdG8gY29tcGxldGlvbjsgY2Fubm90IGJlIGhhcmRjb2RlZC4KRVhQRUNURURfUkVTVUxUUyA9IFsKICAgIHsiZGF0YSI6IHsic2VydmljZSI6ICJhcGkiLCAiQ09VTlQiOiA0MjEwfX0sCiAgICB7ImRhdGEiOiB7InNlcnZpY2UiOiAid2ViIiwgIkNPVU5UIjogMTg3NX19LAogICAgeyJkYXRhIjogeyJzZXJ2aWNlIjogIndvcmtlciIsICJDT1VOVCI6IDkzMn19LApdCiMgVGhlIHF1ZXJ5IHJlc3VsdCBiZWNvbWVzIGNvbXBsZXRlIG9ubHkgb24gdGhlIDJuZCBwb2xsLCBzbyBhIGNvcnJlY3QgY2xpZW50IE1VU1QgbG9vcC4KUE9MTFNfVU5USUxfQ09NUExFVEUgPSAyCgoKY2xhc3MgX01vY2tIb25leWNvbWIoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCiAgICBfcXVlcnlfaWRzID0gc2V0KCkKICAgIF9yZXN1bHRfcG9sbHMgPSB7fQogICAgX2NvdW50ZXIgPSB7InEiOiAwLCAiciI6IDB9CgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBfcmVhZF9qc29uX2JvZHkoc2VsZik6CiAgICAgICAgbGVuZ3RoID0gaW50KHNlbGYuaGVhZGVycy5nZXQoIkNvbnRlbnQtTGVuZ3RoIikgb3IgMCkKICAgICAgICBpZiBsZW5ndGggPD0gMDoKICAgICAgICAgICAgcmV0dXJuIE5vbmUKICAgICAgICB0cnk6CiAgICAgICAgICAgIHJldHVybiBqc29uLmxvYWRzKHNlbGYucmZpbGUucmVhZChsZW5ndGgpLmRlY29kZSgpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb246CiAgICAgICAgICAgIHJldHVybiBOb25lCgogICAgZGVmIF9hdXRoZWQoc2VsZik6CiAgICAgICAgcmV0dXJuIHNlbGYuaGVhZGVycy5nZXQoIlgtSG9uZXljb21iLVRlYW0iKSA9PSBFWFBFQ1RFRF9LRVkKCiAgICBkZWYgX2xvZyhzZWxmLCBtZXRob2QsIHBhdGgpOgogICAgICAgIGhlYWRlcnNfbG93ZXIgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAibWV0aG9kIjogbWV0aG9kLAogICAgICAgICAgICAicGF0aCI6IHBhdGgsCiAgICAgICAgICAgICJoYXNfdGVhbV9oZWFkZXIiOiAieC1ob25leWNvbWItdGVhbSIgaW4gaGVhZGVyc19sb3dlciwKICAgICAgICAgICAgImhhc19iZWFyZXIiOiAiYXV0aG9yaXphdGlvbiIgaW4gaGVhZGVyc19sb3dlciwKICAgICAgICB9KQoKICAgIGRlZiBkb19QT1NUKHNlbGYpOgogICAgICAgIHBhdGggPSB1cmxwYXJzZShzZWxmLnBhdGgpLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgYm9keSA9IHNlbGYuX3JlYWRfanNvbl9ib2R5KCkKICAgICAgICBzZWxmLl9sb2coIlBPU1QiLCBwYXRoKQoKICAgICAgICBpZiBub3Qgc2VsZi5fYXV0aGVkKCk6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiSG9uZXljb21iIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUhvbmV5Y29tYi1UZWFtIGhlYWRlciJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyBTdGVwIDEg4oCUIGNyZWF0ZSBhIFF1ZXJ5ICh2YWxpZGF0ZXMgdGhlIHNwZWMsIHJldHVybnMgYW4gaWQ7IGRvZXMgTk9UIHJ1biBpdCkuCiAgICAgICAgbV9xID0gcmUuZnVsbG1hdGNoKHIiLzEvcXVlcmllcy8oW14vXSspIiwgcGF0aCkKICAgICAgICBpZiBtX3E6CiAgICAgICAgICAgIGlmIG5vdCBpc2luc3RhbmNlKGJvZHksIGRpY3QpIG9yICJjYWxjdWxhdGlvbnMiIG5vdCBpbiBib2R5OgogICAgICAgICAgICAgICAgc2VsZi5fanNvbig0MjIsIHsiZXJyb3IiOiAidW5wcm9jZXNzYWJsZSIsICJlcnJvcl9pbmZvIjogInF1ZXJ5IHNwZWMgcmVxdWlyZXMgY2FsY3VsYXRpb25zIn0pCiAgICAgICAgICAgICAgICByZXR1cm4KICAgICAgICAgICAgdHlwZShzZWxmKS5fY291bnRlclsicSJdICs9IDEKICAgICAgICAgICAgcWlkID0gZiJxLXt0eXBlKHNlbGYpLl9jb3VudGVyWydxJ119IgogICAgICAgICAgICB0eXBlKHNlbGYpLl9xdWVyeV9pZHMuYWRkKHFpZCkKICAgICAgICAgICAgc2VsZi5fanNvbigyMDEsIHsiaWQiOiBxaWQsICJkYXRhc2V0IjogbV9xLmdyb3VwKDEpfSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgU3RlcCAyIOKAlCBjcmVhdGUgYSBRdWVyeSBSZXN1bHQgdGhhdCByZWZlcmVuY2VzIHRoZSBxdWVyeSBpZCAocnVucyBpdCBhc3luY2hyb25vdXNseSkuCiAgICAgICAgbV9yID0gcmUuZnVsbG1hdGNoKHIiLzEvcXVlcnlfcmVzdWx0cy8oW14vXSspIiwgcGF0aCkKICAgICAgICBpZiBtX3I6CiAgICAgICAgICAgIHFpZCA9IGJvZHkuZ2V0KCJxdWVyeV9pZCIpIGlmIGlzaW5zdGFuY2UoYm9keSwgZGljdCkgZWxzZSBOb25lCiAgICAgICAgICAgIGlmIHFpZCBub3QgaW4gdHlwZShzZWxmKS5fcXVlcnlfaWRzOgogICAgICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiZXJyb3IiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiY3JlYXRlIGEgUXVlcnkgZmlyc3QsIHRoZW4gcmVmZXJlbmNlIGl0cyBpZCBhcyBxdWVyeV9pZCJ9KQogICAgICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgICAgIHR5cGUoc2VsZikuX2NvdW50ZXJbInIiXSArPSAxCiAgICAgICAgICAgIHJpZCA9IGYici17dHlwZShzZWxmKS5fY291bnRlclsnciddfSIKICAgICAgICAgICAgdHlwZShzZWxmKS5fcmVzdWx0X3BvbGxzW3JpZF0gPSAwCiAgICAgICAgICAgIHNlbGYuX2pzb24oMjAxLCB7ImlkIjogcmlkLCAiY29tcGxldGUiOiBGYWxzZX0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJlcnJvciI6ICJub3QgZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiSG9uZXljb21iIHYxIHF1ZXJ5IGRhdGEgbGl2ZXMgYXQgLzEvcXVlcmllcyBhbmQgLzEvcXVlcnlfcmVzdWx0cyJ9KQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGF0aCA9IHVybHBhcnNlKHNlbGYucGF0aCkucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBzZWxmLl9sb2coIkdFVCIsIHBhdGgpCgogICAgICAgIGlmIG5vdCBzZWxmLl9hdXRoZWQoKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsiZXJyb3IiOiAidW5hdXRob3JpemVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3JfaW5mbyI6ICJIb25leWNvbWIgYXV0aGVudGljYXRlcyB2aWEgdGhlIFgtSG9uZXljb21iLVRlYW0gaGVhZGVyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjIFN0ZXAgMyDigJQgcG9sbCB0aGUgcXVlcnkgcmVzdWx0IHVudGlsIGNvbXBsZXRlID09IHRydWUsIHRoZW4gcmVhZCBkYXRhLnJlc3VsdHMuCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi8xL3F1ZXJ5X3Jlc3VsdHMvKFteL10rKS8oW14vXSspIiwgcGF0aCkKICAgICAgICBpZiBub3QgbToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiZXJyb3IiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3JfaW5mbyI6ICJwb2xsIGEgcXVlcnkgcmVzdWx0IGF0IC8xL3F1ZXJ5X3Jlc3VsdHMve2RhdGFzZXR9L3tyZXN1bHRfaWR9In0pCiAgICAgICAgICAgIHJldHVybgogICAgICAgIHJpZCA9IG0uZ3JvdXAoMikKICAgICAgICBpZiByaWQgbm90IGluIHR5cGUoc2VsZikuX3Jlc3VsdF9wb2xsczoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiZXJyb3IiOiAibm90IGZvdW5kIiwgImVycm9yX2luZm8iOiAidW5rbm93biBxdWVyeSByZXN1bHQgaWQifSkKICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgdHlwZShzZWxmKS5fcmVzdWx0X3BvbGxzW3JpZF0gKz0gMQogICAgICAgIGlmIHR5cGUoc2VsZikuX3Jlc3VsdF9wb2xsc1tyaWRdIDwgUE9MTFNfVU5USUxfQ09NUExFVEU6CiAgICAgICAgICAgICMgTm90IGRvbmUgeWV0OiBubyBkYXRhIGZpZWxkIHVudGlsIGNvbXBsZXRlIChtYXRjaGVzIHRoZSByZWFsIGFzeW5jIGNvbnRyYWN0KS4KICAgICAgICAgICAgc2VsZi5fanNvbigyMDAsIHsiaWQiOiByaWQsICJjb21wbGV0ZSI6IEZhbHNlfSkKICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgImlkIjogcmlkLAogICAgICAgICAgICAiY29tcGxldGUiOiBUcnVlLAogICAgICAgICAgICAiZGF0YSI6IHsic2VyaWVzIjogW10sICJyZXN1bHRzIjogRVhQRUNURURfUkVTVUxUU30sCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrSG9uZXljb21iLnJlcXVlc3RfbG9nID0gW10KICAgIF9Nb2NrSG9uZXljb21iLl9xdWVyeV9pZHMgPSBzZXQoKQogICAgX01vY2tIb25leWNvbWIuX3Jlc3VsdF9wb2xscyA9IHt9CiAgICBfTW9ja0hvbmV5Y29tYi5fY291bnRlciA9IHsicSI6IDAsICJyIjogMH0KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tIb25leWNvbWIpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpLnN0YXJ0KCkKICAgIHRyeToKICAgICAgICB5aWVsZCBmImh0dHA6Ly8xMjcuMC4wLjE6e3BvcnR9IgogICAgZmluYWxseToKICAgICAgICBzZXJ2ZXIuc2h1dGRvd24oKQogICAgICAgIHNlcnZlci5zZXJ2ZXJfY2xvc2UoKQoKCmRlZiBfbG9hZF9hZ2VudF9zb2x1dGlvbigpOgogICAgcm9vdCA9IG9zLmVudmlyb24uZ2V0KCJWQl9TT0xVVElPTl9ST09UIiwgb3MuZ2V0Y3dkKCkpCiAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4ocm9vdCwgInNvbHV0aW9uIiwgInNvbHV0aW9uLnB5IikKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIGZvciBkaXJwYXRoLCBfZGlycywgZmlsZXMgaW4gb3Mud2Fsayhyb290KToKICAgICAgICAgICAgaWYgInNvbHV0aW9uLnB5IiBpbiBmaWxlczoKICAgICAgICAgICAgICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihkaXJwYXRoLCAic29sdXRpb24ucHkiKQogICAgICAgICAgICAgICAgYnJlYWsKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIHB5dGVzdC5mYWlsKGYiYWdlbnQgc29sdXRpb24gbm90IGZvdW5kOiBleHBlY3RlZCBzb2x1dGlvbi9zb2x1dGlvbi5weSB1bmRlciB7cm9vdH0iKQogICAgc3BlYyA9IGltcG9ydGxpYi51dGlsLnNwZWNfZnJvbV9maWxlX2xvY2F0aW9uKCJhZ2VudF9zb2x1dGlvbiIsIGNhbmRpZGF0ZSkKICAgIG1vZHVsZSA9IGltcG9ydGxpYi51dGlsLm1vZHVsZV9mcm9tX3NwZWMoc3BlYykKICAgIHNwZWMubG9hZGVyLmV4ZWNfbW9kdWxlKG1vZHVsZSkKICAgIHJldHVybiBtb2R1bGUKCgpkZWYgdGVzdF9ydW5fcXVlcnlfYXN5bmNfZmxvdyhtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJydW5fcXVlcnkiKSwgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IHJ1bl9xdWVyeShiYXNlX3VybCwgYXBpX2tleSwgZGF0YXNldF9zbHVnLCBxdWVyeSkiCgogICAgcXVlcnkgPSB7ImNhbGN1bGF0aW9ucyI6IFt7Im9wIjogIkNPVU5UIn1dLCAiYnJlYWtkb3ducyI6IFsic2VydmljZSJdLCAidGltZV9yYW5nZSI6IDcyMDB9CiAgICByZXN1bHRzID0gbW9kdWxlLnJ1bl9xdWVyeShtb2NrX3NlcnZlciwgRVhQRUNURURfS0VZLCBEQVRBU0VUX1NMVUcsIHF1ZXJ5KQoKICAgIGFzc2VydCBpc2luc3RhbmNlKHJlc3VsdHMsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiByZXN1bHQgcm93cywgZ290IHt0eXBlKHJlc3VsdHMpLl9fbmFtZV9ffSIKICAgIGFzc2VydCByZXN1bHRzID09IEVYUEVDVEVEX1JFU1VMVFMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCB7cmVzdWx0cyFyfSwgZXhwZWN0ZWQge0VYUEVDVEVEX1JFU1VMVFMhcn0uIEEgY29ycmVjdCBjbGllbnQgcnVucyB0aGUgIgogICAgICAgIGYidGhyZWUtc3RlcCBhc3luYyBmbG93IChQT1NUIC8xL3F1ZXJpZXMgLT4gUE9TVCAvMS9xdWVyeV9yZXN1bHRzIC0+IHBvbGwgR0VUICIKICAgICAgICBmIi8xL3F1ZXJ5X3Jlc3VsdHMve3tkYXRhc2V0fX0ve3tpZH19IHVudGlsIGNvbXBsZXRlKSBhbmQgcmV0dXJucyBkYXRhLnJlc3VsdHMuICIKICAgICAgICBmIlNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0hvbmV5Y29tYi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" + }, + { + "vendor": "honeycomb:get_dataset", + "fn": "get_dataset", + "testFile": "test_honeycomb_get_dataset.py", + "signature": "def get_dataset(base_url: str, api_key: str, dataset_slug: str) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgaG9uZXljb21iLWdldC1kYXRhc2V0IChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBIb25leWNvbWIncyBDVVJSRU5UIChBUEkgdjEpIHNpbmdsZS0KZGF0YXNldCBjb250cmFjdCBvbiB0d28gc2VhcmNoLW5lY2Vzc2FyeSBheGVzOgogIC0gYXV0aCAgICAgIFgtSG9uZXljb21iLVRlYW06IDxrZXk+ICAgKGN1c3RvbSBoZWFkZXIsIE5PIEF1dGhvcml6YXRpb246IEJlYXJlciBzY2hlbWUpCiAgLSBlbmRwb2ludCAgLzEvZGF0YXNldHMve3NsdWd9ICAgICAgICAodGhlIEFQSSB2ZXJzaW9uIHByZWZpeCBpcyB0aGUgbGl0ZXJhbCAiMSIpCgpBIGZyb20tbWVtb3J5IGNsaWVudCB0aGF0IHJlYWNoZXMgZm9yIEF1dGhvcml6YXRpb246IEJlYXJlciAodGhlIG5lYXItdW5pdmVyc2FsIGRlZmF1bHQpIG9yCmd1ZXNzZXMgL3YxfC9hcGkvdjF8L2FwaS8xIGhpdHMgNDAxIC8gNDA0IGFuZCBGQUlMUy4gQW4gZW1wdHkgc29sdXRpb24gRkFJTFMuIE9ubHkgYSBjbGllbnQKYnVpbHQgZnJvbSB0aGUgY3VycmVudCBIb25leWNvbWIgZG9jcyAoY3VzdG9tIGhlYWRlciArIC8xLyBwcmVmaXgpIFBBU1NFUy4gVGhlIGRhdGFzZXQgbmFtZSBpcwpzbHVnLWRlcml2ZWQgYW5kIG9ubHkgb2JzZXJ2YWJsZSBieSBhY3R1YWxseSBjYWxsaW5nIHRoZSBtb2NrLCBzbyB0aGUgYW5zd2VyIGNhbm5vdCBiZSBoYXJkY29kZWQuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZG9jcy5ob25leWNvbWIuaW8vYXBpL2F1dGgKICBodHRwczovL2FwaS1kb2NzLmhvbmV5Y29tYi5pby9hcGkvZGF0YXNldHMKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUga2V5IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBYLUhvbmV5Y29tYi1UZWFtIEhFQURFUiAodjEgY29udHJhY3QpLgpFWFBFQ1RFRF9LRVkgPSAiaGNhaWNfaG9uZXljb21iX2NvbmZpZ19rZXlfYWJjMTIzIgpEQVRBU0VUX1NMVUcgPSAiY2hlY2tvdXQtc2VydmljZSIKCgpjbGFzcyBfTW9ja0hvbmV5Y29tYihCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgICMgQ2xhc3MtbGV2ZWwgcmVxdWVzdCBsb2cgc28gdGhlIHRlc3QgY2FuIHJlcG9ydCB3aGF0IHBhdGgvYXV0aCB0aGUgY2xpZW50IGFjdHVhbGx5IHVzZWQuCiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6ICAjIHNpbGVuY2Ugc3RkZXJyIGFjY2VzcyBsb2cKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIGhlYWRlcnNfbG93ZXIgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX3RlYW1faGVhZGVyIjogIngtaG9uZXljb21iLXRlYW0iIGluIGhlYWRlcnNfbG93ZXIsCiAgICAgICAgICAgICJoYXNfYmVhcmVyIjogImF1dGhvcml6YXRpb24iIGluIGhlYWRlcnNfbG93ZXIsCiAgICAgICAgICAgICJxdWVyeV9hcGlfa2V5IjogImFwaV9rZXkiIGluIHBhcnNlX3FzKHBhcnNlZC5xdWVyeSksCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB2MSBmZXRjaGVzIGEgZGF0YXNldCBhdCAvMS9kYXRhc2V0cy97c2x1Z30uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi8xL2RhdGFzZXRzLyhbXi9dKykiLCBwYXRoKQogICAgICAgIGlmIG5vdCBtOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJlcnJvciI6ICJub3QgZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvcl9pbmZvIjogIkhvbmV5Y29tYiBBUEkgdjEgZmV0Y2hlcyBhIGRhdGFzZXQgYXQgLzEvZGF0YXNldHMve3NsdWd9In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IHYxIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUhvbmV5Y29tYi1UZWFtIGhlYWRlcjsgQmVhcmVyIGlzIE5PVCBob25vcmVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIlgtSG9uZXljb21iLVRlYW0iKSAhPSBFWFBFQ1RFRF9LRVk6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiSG9uZXljb21iIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUhvbmV5Y29tYi1UZWFtIGhlYWRlciJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc2x1ZyA9IG0uZ3JvdXAoMSkKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAibmFtZSI6IGYiU2VydmljZSB7c2x1Z30iLAogICAgICAgICAgICAic2x1ZyI6IHNsdWcsCiAgICAgICAgICAgICJkZXNjcmlwdGlvbiI6IGYidGVsZW1ldHJ5IGZvciB7c2x1Z30iLAogICAgICAgICAgICAiZXhwYW5kX2pzb25fZGVwdGgiOiAyLAogICAgICAgICAgICAibGFzdF93cml0dGVuX2F0IjogIjIwMjYtMDctMDFUMDA6MDA6MDBaIiwKICAgICAgICAgICAgImNyZWF0ZWRfYXQiOiAiMjAyNS0wMS0wMVQwMDowMDowMFoiLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0hvbmV5Y29tYi5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrSG9uZXljb21iKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RhdGFzZXRfYnlfc2x1Zyhtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfZGF0YXNldCIpLCAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgZ2V0X2RhdGFzZXQoYmFzZV91cmwsIGFwaV9rZXksIGRhdGFzZXRfc2x1ZykiCgogICAgZGF0YXNldCA9IG1vZHVsZS5nZXRfZGF0YXNldChtb2NrX3NlcnZlciwgRVhQRUNURURfS0VZLCBEQVRBU0VUX1NMVUcpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoZGF0YXNldCwgZGljdCksIGYiZXhwZWN0ZWQgYSBkYXRhc2V0IGRpY3QsIGdvdCB7dHlwZShkYXRhc2V0KS5fX25hbWVfX30iCiAgICBhc3NlcnQgZGF0YXNldC5nZXQoInNsdWciKSA9PSBEQVRBU0VUX1NMVUcgYW5kIGRhdGFzZXQuZ2V0KCJuYW1lIikgPT0gZiJTZXJ2aWNlIHtEQVRBU0VUX1NMVUd9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkYXRhc2V0IXJ9OyBhIGNvcnJlY3QgdjEgY2xpZW50IEdFVHMgLzEvZGF0YXNldHMve0RBVEFTRVRfU0xVR30gd2l0aCB0aGUgIgogICAgICAgIGYiWC1Ib25leWNvbWItVGVhbSBoZWFkZXIuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0hvbmV5Y29tYi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" + }, + { + "vendor": "boldsign:list_all_documents", + "fn": "list_all_documents", + "testFile": "test_boldsign_client.py", + "signature": "def list_all_documents(base_url: str, api_key: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYm9sZHNpZ24tbGlzdC1hbGwtZG9jdW1lbnRzIChoYXJkKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUgdGltZQooYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdCB0aGUgYWdlbnQncwpzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgQm9sZFNpZ24ncyBDVVJSRU5UIERvY3VtZW50cy1saXN0CmNvbnRyYWN0OgogIC0gZW5kcG9pbnQgcGF0aCAgIEdFVCAvdjEvZG9jdW1lbnQvbGlzdD9wYWdlPXtufSZwYWdlU2l6ZT17bn0gICAoc2luZ3VsYXIgImRvY3VtZW50IiArIC9saXN0OwogICAgICAgICAgICAgICAgICAgIE5PVCB0aGUgUkVTVC1ndWVzcyAvdjEvZG9jdW1lbnRzKQogIC0gYXV0aCAgICAgICAgICAgIFgtQVBJLUtFWTogPGtleT4gcmVxdWVzdCBoZWFkZXIgICAgICAgICAgICAgICAoTk9UIEF1dGhvcml6YXRpb246IEJlYXJlciA8a2V5PikKICAtIHBhZ2luYXRpb24gICAgICAxLWluZGV4ZWQgYHBhZ2VgIHF1ZXJ5IHBhcmFtOyB0aGUgcmVzcG9uc2UgaXMKICAgICAgICAgICAgICAgICAgICB7ICJwYWdlRGV0YWlscyI6IHsgInBhZ2UiLCAicGFnZVNpemUiLCAidG90YWxSZWNvcmRzQ291bnQiLCAidG90YWxQYWdlcyIsIC4uLiB9LAogICAgICAgICAgICAgICAgICAgICAgInJlc3VsdCI6IFsgPGRvY3VtZW50PiwgLi4uIF0gfQogICAgICAgICAgICAgICAgICAgIFRoZSBjYWxsZXIgcmVhZHMgdGhlIGByZXN1bHRgIGFycmF5IChOT1QgYGRhdGFgL2Bkb2N1bWVudHNgKSBhbmQgZm9sbG93cwogICAgICAgICAgICAgICAgICAgIHBhZ2UvcGFnZURldGFpbHMudG90YWxQYWdlcyB0byB0aGUgZW5kLiBUaGUgc2VydmVyIGVuZm9yY2VzIGl0cyBvd24gcGFnZQogICAgICAgICAgICAgICAgICAgIHNpemUgKDMpLCBzbyBhIGNsaWVudCB0aGF0IHN0b3BzIGFmdGVyIHBhZ2UgMSBjb2xsZWN0cyBvbmx5IDMgb2YgNy4KCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIEJvbGRTaWduIGRvY3MgY29sbGVjdHMgYWxsIDcgZG9jdW1lbnRzIGFjcm9zcyAzIHBhZ2VzIGFuZCBwYXNzZXMuCkEgY2xpZW50IGZyb20gc3RhbGUvZ3Vlc3NlZCBtZW1vcnkgKEJlYXJlciBhdXRoLCAvdjEvZG9jdW1lbnRzIHBhdGgsIGEgYGRhdGFgIGVudmVsb3BlLCBjdXJzb3IKb3IgYG5leHRgLWxpbmsgcGFnaW5hdGlvbiwgb3IgZmlyc3QtcGFnZS1vbmx5KSBoaXRzIDQwMSAvIDQwNCAvIHN0b3BzIGVhcmx5IGFuZCBGQUlMUy4gQW4gZW1wdHkKc29sdXRpb24gRkFJTFMuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZGV2ZWxvcGVycy5ib2xkc2lnbi5jb20vYXV0aGVudGljYXRpb24vYXBpLWtleS8KICBodHRwczovL2RldmVsb3BlcnMuYm9sZHNpZ24uY29tL2RvY3VtZW50cy9saXN0LWRvY3VtZW50cy8KIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9LRVkgPSAiYnNfbGl2ZV9rZXlfYWJjMTIzIgoKIyBIaWRkZW4gZGF0YXNldDogNyBkb2N1bWVudHMuIE9ubHkgb2JzZXJ2YWJsZSBieSBjYWxsaW5nIHRoZSBtb2NrIGFuZCBwYWdpbmF0aW5nIHRvIHRoZSBlbmQsCiMgc28gdGhlIGFnZW50IGNhbm5vdCBoYXJkY29kZSB0aGUgYW5zd2VyLiBTZXJ2ZXIgcGFnZSBzaXplIGlzIDMgLT4gMyBwYWdlcy4KRE9DVU1FTlRTID0gWwogICAgeyJkb2N1bWVudElkIjogZiJkb2Mte2k6MDRkfSIsICJtZXNzYWdlVGl0bGUiOiBmIkNvbnRyYWN0IHtpfSIsICJzdGF0dXMiOiAiSW5Qcm9ncmVzcyJ9CiAgICBmb3IgaSBpbiByYW5nZSgxLCA4KQpdClBBR0VfU0laRSA9IDMKCgpjbGFzcyBfTW9ja0JvbGRTaWduKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOgogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgaGVhZGVyX2tleXMgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX2FwaV9rZXlfaGVhZGVyIjogIngtYXBpLWtleSIgaW4gaGVhZGVyX2tleXMsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uX2hlYWRlciI6IHNlbGYuaGVhZGVycy5nZXQoIkF1dGhvcml6YXRpb24iKSwKICAgICAgICAgICAgInBhZ2UiOiBxcy5nZXQoInBhZ2UiLCBbTm9uZV0pWzBdLAogICAgICAgIH0pCgogICAgICAgICMgKDEpIGVuZHBvaW50IGF4aXM6IEJvbGRTaWduIGxpc3RzIGRvY3VtZW50cyBhdCAvdjEvZG9jdW1lbnQvbGlzdC4KICAgICAgICBpZiBwYXRoICE9ICIvdjEvZG9jdW1lbnQvbGlzdCI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImVycm9yIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiQm9sZFNpZ24gbGlzdHMgZG9jdW1lbnRzIGF0IC92MS9kb2N1bWVudC9saXN0In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IEJvbGRTaWduIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUFQSS1LRVkgaGVhZGVyLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIlgtQVBJLUtFWSIpICE9IEVYUEVDVEVEX0tFWToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsiZXJyb3IiOiAidW5hdXRob3JpemVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJCb2xkU2lnbiBhdXRoZW50aWNhdGVzIHZpYSB0aGUgWC1BUEktS0VZIHJlcXVlc3QgaGVhZGVyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgzKSBwYWdpbmF0aW9uIGF4aXM6IDEtaW5kZXhlZCBwYWdlOyBzZXJ2ZXIgZW5mb3JjZXMgaXRzIG93biBwYWdlIHNpemUuCiAgICAgICAgcGFnZV9yYXcgPSBxcy5nZXQoInBhZ2UiLCBbIjEiXSlbMF0KICAgICAgICB0cnk6CiAgICAgICAgICAgIHBhZ2UgPSBpbnQocGFnZV9yYXcpCiAgICAgICAgZXhjZXB0IChUeXBlRXJyb3IsIFZhbHVlRXJyb3IpOgogICAgICAgICAgICBwYWdlID0gMQogICAgICAgIGlmIHBhZ2UgPCAxOgogICAgICAgICAgICBwYWdlID0gMQogICAgICAgIHN0YXJ0ID0gKHBhZ2UgLSAxKSAqIFBBR0VfU0laRQogICAgICAgIGNodW5rID0gRE9DVU1FTlRTW3N0YXJ0OnN0YXJ0ICsgUEFHRV9TSVpFXQogICAgICAgIHRvdGFsX3BhZ2VzID0gKGxlbihET0NVTUVOVFMpICsgUEFHRV9TSVpFIC0gMSkgLy8gUEFHRV9TSVpFCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgInBhZ2VEZXRhaWxzIjogewogICAgICAgICAgICAgICAgInBhZ2UiOiBwYWdlLAogICAgICAgICAgICAgICAgInBhZ2VTaXplIjogUEFHRV9TSVpFLAogICAgICAgICAgICAgICAgInRvdGFsUmVjb3Jkc0NvdW50IjogbGVuKERPQ1VNRU5UUyksCiAgICAgICAgICAgICAgICAidG90YWxQYWdlcyI6IHRvdGFsX3BhZ2VzLAogICAgICAgICAgICAgICAgInNvcnRlZENvbHVtbiI6ICJjcmVhdGVkRGF0ZSIsCiAgICAgICAgICAgICAgICAic29ydERpcmVjdGlvbiI6ICJkZXNjZW5kaW5nIiwKICAgICAgICAgICAgfSwKICAgICAgICAgICAgInJlc3VsdCI6IGNodW5rLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0JvbGRTaWduLnJlcXVlc3RfbG9nID0gW10KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tCb2xkU2lnbikKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9kb2N1bWVudHNfYWNyb3NzX3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2RvY3VtZW50cyIpLCAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgbGlzdF9hbGxfZG9jdW1lbnRzKGJhc2VfdXJsLCBhcGlfa2V5KSIKCiAgICByZXN1bHQgPSBtb2R1bGUubGlzdF9hbGxfZG9jdW1lbnRzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9LRVkpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UocmVzdWx0LCBsaXN0KSwgZiJleHBlY3RlZCBhIGxpc3Qgb2YgZG9jdW1lbnRzLCBnb3Qge3R5cGUocmVzdWx0KS5fX25hbWVfX30iCiAgICBpZHMgPSBzb3J0ZWQoZFsiZG9jdW1lbnRJZCJdIGZvciBkIGluIHJlc3VsdCBpZiBpc2luc3RhbmNlKGQsIGRpY3QpIGFuZCAiZG9jdW1lbnRJZCIgaW4gZCkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChkWyJkb2N1bWVudElkIl0gZm9yIGQgaW4gRE9DVU1FTlRTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBkb2N1bWVudCBpZHMge2lkc30sIGV4cGVjdGVkIHtleHBlY3RlZF9pZHN9LiBBIGNvcnJlY3QgY2xpZW50IHJlYWRzIHRoZSBgcmVzdWx0YCBhcnJheSAiCiAgICAgICAgZiJhbmQgZm9sbG93cyBwYWdlL3BhZ2VEZXRhaWxzLnRvdGFsUGFnZXMgcGFnaW5hdGlvbiB0byB0aGUgZW5kLiBTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tCb2xkU2lnbi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" + }, + { + "vendor": "boldsign:get_document", + "fn": "get_document", + "testFile": "test_boldsign_get_document.py", + "signature": "def get_document(base_url: str, api_key: str, document_id: str) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYm9sZHNpZ24tZ2V0LWRvY3VtZW50IChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBCb2xkU2lnbidzIENVUlJFTlQgc2luZ2xlLWRvY3VtZW50CmNvbnRyYWN0OgogIC0gZW5kcG9pbnQgcGF0aCAgIEdFVCAvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtpZH0gICAoc2luZ3VsYXIgImRvY3VtZW50IiwgYQogICAgICAgICAgICAgICAgICAgIC9wcm9wZXJ0aWVzIGFjdGlvbiArIHF1ZXJ5LXBhcmFtIGlkOyBOT1QgdGhlIFJFU1QtZ3Vlc3MgL3YxL2RvY3VtZW50cy97aWR9KQogIC0gYXV0aCAgICAgICAgICAgIFgtQVBJLUtFWTogPGtleT4gcmVxdWVzdCBoZWFkZXIgICAgICAgICAgICAgICAgKE5PVCBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGtleT4pCiAgLSByZXNwb25zZSBzaGFwZSAgdGhlIGRvY3VtZW50IG9iamVjdCBpcyByZXR1cm5lZCBESVJFQ1RMWSBhdCB0aGUgdG9wIGxldmVsIChubyBkYXRhL3Jlc3VsdCBlbnZlbG9wZSkKCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIEJvbGRTaWduIGRvY3MgcGFzc2VzLiBBIGNsaWVudCB3cml0dGVuIGZyb20gc3RhbGUvZ3Vlc3NlZAptZW1vcnkgKEJlYXJlciBhdXRoLCAvdjEvZG9jdW1lbnRzL3tpZH0gUkVTVCBwYXRoLCBvciBhIGRhdGEtZW52ZWxvcGUgcGFyc2UpIGhpdHMgNDAxIC8gNDA0Cm9yIHJlYWRzIHRoZSB3cm9uZyBzaGFwZSBhbmQgRkFJTFMuIEFuIGVtcHR5IHNvbHV0aW9uIEZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RldmVsb3BlcnMuYm9sZHNpZ24uY29tL2F1dGhlbnRpY2F0aW9uL2FwaS1rZXkvCiAgaHR0cHM6Ly9kZXZlbG9wZXJzLmJvbGRzaWduLmNvbS9kb2N1bWVudHMvZG9jdW1lbnQtZGV0YWlscy1hbmQtc3RhdHVzLwogIGh0dHBzOi8vZGV2ZWxvcGVycy5ib2xkc2lnbi5jb20vaG93LXRvLWd1aWRlcy9yZXRyaWV2ZS1lc2lnbmF0dXJlLWRvY3VtZW50LXByb3BlcnRpZXMvCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUga2V5IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBYLUFQSS1LRVkgSEVBREVSIChCb2xkU2lnbiBjb250cmFjdCkuIFBhc3NlZCB0byB0aGUKIyBjbGllbnQgYXQgY2FsbCB0aW1lLCBzbyB0aGUga25vd2xlZGdlIGdhcCBpcyB0aGUgaGVhZGVyIE5BTUUvc2NoZW1lLCBub3QgdGhlIHNlY3JldCB2YWx1ZS4KRVhQRUNURURfS0VZID0gImJzX2xpdmVfa2V5X2FiYzEyMyIKRE9DX0lEID0gImExYjJjM2Q0LTExMTEtMjIyMi0zMzMzLTAwMDAwMDAwMDA0MiIKCgpjbGFzcyBfTW9ja0JvbGRTaWduKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOiAgIyBzaWxlbmNlIHN0ZGVyciBhY2Nlc3MgbG9nCiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBoZWFkZXJfa2V5cyA9IHtrLmxvd2VyKCkgZm9yIGsgaW4gc2VsZi5oZWFkZXJzLmtleXMoKX0KICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJoYXNfYXBpX2tleV9oZWFkZXIiOiAieC1hcGkta2V5IiBpbiBoZWFkZXJfa2V5cywKICAgICAgICAgICAgImF1dGhvcml6YXRpb25faGVhZGVyIjogc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpLAogICAgICAgICAgICAiZG9jdW1lbnRJZCI6IHFzLmdldCgiZG9jdW1lbnRJZCIsIFtOb25lXSlbMF0sCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQgYXhpczogQm9sZFNpZ24gZmV0Y2hlcyBkb2N1bWVudCBwcm9wZXJ0aWVzIGF0IC92MS9kb2N1bWVudC9wcm9wZXJ0aWVzLgogICAgICAgIGlmIHBhdGggIT0gIi92MS9kb2N1bWVudC9wcm9wZXJ0aWVzIjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiZXJyb3IiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJCb2xkU2lnbiBmZXRjaGVzIGEgZG9jdW1lbnQncyBwcm9wZXJ0aWVzIGF0ICIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICIvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtpZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDIpIGF1dGggYXhpczogQm9sZFNpZ24gYXV0aGVudGljYXRlcyB2aWEgdGhlIFgtQVBJLUtFWSBoZWFkZXIgKHF1ZXJ5L0JlYXJlciBub3QgaG9ub3JlZCkuCiAgICAgICAgaWYgc2VsZi5oZWFkZXJzLmdldCgiWC1BUEktS0VZIikgIT0gRVhQRUNURURfS0VZOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJlcnJvciI6ICJ1bmF1dGhvcml6ZWQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkJvbGRTaWduIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUFQSS1LRVkgcmVxdWVzdCBoZWFkZXIifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGRvY3VtZW50X2lkID0gcXMuZ2V0KCJkb2N1bWVudElkIiwgW05vbmVdKVswXQogICAgICAgIGlmIG5vdCBkb2N1bWVudF9pZDoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsiZXJyb3IiOiAiZG9jdW1lbnRJZCBxdWVyeSBwYXJhbWV0ZXIgaXMgcmVxdWlyZWQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDMpIHJlc3BvbnNlIHNoYXBlOiB0aGUgZG9jdW1lbnQgb2JqZWN0IGlzIHJldHVybmVkIERJUkVDVExZIGF0IHRoZSB0b3AgbGV2ZWwuCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgImRvY3VtZW50SWQiOiBkb2N1bWVudF9pZCwKICAgICAgICAgICAgIm1lc3NhZ2VUaXRsZSI6IGYiQ29udHJhY3Qge2RvY3VtZW50X2lkfSIsCiAgICAgICAgICAgICJzdGF0dXMiOiAiSW5Qcm9ncmVzcyIsCiAgICAgICAgICAgICJzZW5kZXJEZXRhaWwiOiB7Im5hbWUiOiAiQWRhIFNlbmRlciIsICJlbWFpbEFkZHJlc3MiOiAiYWRhQGV4YW1wbGUuY29tIn0sCiAgICAgICAgICAgICJzaWduZXJEZXRhaWxzIjogW3sibmFtZSI6ICJCb3JpcyBTaWduZXIiLCAiZW1haWxBZGRyZXNzIjogImJvcmlzQGV4YW1wbGUuY29tIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzdGF0dXMiOiAiTm90Q29tcGxldGVkIn1dLAogICAgICAgICAgICAiY3JlYXRlZERhdGUiOiAxNzUxODQ2NDAwLAogICAgICAgICAgICAiZXhwaXJ5RGF0ZSI6IDE3NTQ0Mzg0MDAsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrQm9sZFNpZ24ucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0JvbGRTaWduKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RvY3VtZW50X2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9kb2N1bWVudCIpLCAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgZ2V0X2RvY3VtZW50KGJhc2VfdXJsLCBhcGlfa2V5LCBkb2N1bWVudF9pZCkiCgogICAgZG9jID0gbW9kdWxlLmdldF9kb2N1bWVudChtb2NrX3NlcnZlciwgRVhQRUNURURfS0VZLCBET0NfSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoZG9jLCBkaWN0KSwgZiJleHBlY3RlZCBhIGRvY3VtZW50IGRpY3QsIGdvdCB7dHlwZShkb2MpLl9fbmFtZV9ffSIKICAgIGFzc2VydCBkb2MuZ2V0KCJkb2N1bWVudElkIikgPT0gRE9DX0lEIGFuZCBkb2MuZ2V0KCJtZXNzYWdlVGl0bGUiKSA9PSBmIkNvbnRyYWN0IHtET0NfSUR9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkb2Mhcn07IGEgY29ycmVjdCBjbGllbnQgR0VUcyAvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtET0NfSUR9IHdpdGggdGhlICIKICAgICAgICBmIlgtQVBJLUtFWSBoZWFkZXIgYW5kIHJldHVybnMgdGhlIHRvcC1sZXZlbCBkb2N1bWVudCBvYmplY3QuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0JvbGRTaWduLnJlcXVlc3RfbG9nfSIKICAgICkK" + }, + { + "vendor": "finch:list_all_individuals", + "fn": "list_all_individuals", + "testFile": "test_finch_list_all.py", + "signature": "def list_all_individuals(base_url: str, access_token: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgZmluY2gtZGlyZWN0b3J5LXBhZ2luYXRlIChoYXJkKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUgdGltZQooYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdCB0aGUgYWdlbnQncwpzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgRmluY2gncyBDVVJSRU5UIGRpcmVjdG9yeSBjb250cmFjdCwKYWRkaW5nIHRoZSBPRkZTRVQtcGFnaW5hdGlvbiBheGlzIG9uIHRvcCBvZiB0aGUgbWVkaXVtIGxlYWY6CiAgLSBlbmRwb2ludCBwYXRoICAgICAgICAvZW1wbG95ZXIvZGlyZWN0b3J5CiAgLSBhdXRoICAgICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFjY2Vzc190b2tlbj4KICAtIHZlcnNpb24gaGVhZGVyICAgICAgIEZpbmNoLUFQSS1WZXJzaW9uOiA8WVlZWS1NTS1ERD4gICAoUkVRVUlSRUQ7IG1pc3NpbmcgLT4gNDAwKQogIC0gcGFnaW5hdGlvbiAgICAgICAgICAgb2Zmc2V0LWJhc2VkOiBgbGltaXRgICsgYG9mZnNldGAgcXVlcnkgcGFyYW1zOyB0aGUgcmVzcG9uc2UgY2FycmllcwogICAgICAgICAgICAgICAgICAgICAgICAgcGFnaW5nLmNvdW50ICh0b3RhbCkgYW5kIHBhZ2luZy5vZmZzZXQsIGFuZCB0aGVyZSBpcyBOTyBuZXh0L2hhc19tb3JlCiAgICAgICAgICAgICAgICAgICAgICAgICBjdXJzb3IgZmllbGQgLS0gdGhlIGNsaWVudCBtdXN0IGtlZXAgcGFnaW5nIHdoaWxlIG9mZnNldCtsZW4gPCBjb3VudAogIC0gcmVzcG9uc2UgZW52ZWxvcGUgICAgeyJwYWdpbmciOiB7ImNvdW50IiwgIm9mZnNldCJ9LCAiaW5kaXZpZHVhbHMiOiBbLi4uXX0KClRoZSBtb2NrIHJldHVybnMgYXQgbW9zdCBQQUdFX1NJWkUgaW5kaXZpZHVhbHMgcGVyIHJlc3BvbnNlIChzZXJ2ZXIgcGFnZSBzaXplKSwgc28gY29sbGVjdGluZwp0aGUgd2hvbGUgZGlyZWN0b3J5IFJFUVVJUkVTIG9mZnNldCBwYWdpbmF0aW9uIGRyaXZlbiBieSBwYWdpbmcuY291bnQuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbQpDVVJSRU5UIEZpbmNoIGRvY3MgY29sbGVjdHMgYWxsIDcgaW5kaXZpZHVhbHMgYWNyb3NzIDMgcGFnZXMuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtCmN1dG9mZiBtZW1vcnkgb21pdHMgdGhlIHZlcnNpb24gaGVhZGVyICg0MDApLCBndWVzc2VzIGEgL2VtcGxveWVlcyBwYXRoICg0MDQpLCByZWFkcyBhIGBkYXRhYAplbnZlbG9wZSwgb3IgYXNzdW1lcyBhIGBuZXh0YC9gaGFzX21vcmVgIGN1cnNvciBhbmQgc3RvcHMgYWZ0ZXIgcGFnZSAxICgzIG9mIDcpIGFuZCBGQUlMUy4KQW4gZW1wdHkgc29sdXRpb24gRkFJTFMuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZGV2ZWxvcGVyLnRyeWZpbmNoLmNvbS9hcGktcmVmZXJlbmNlL2RldmVsb3BtZW50LWd1aWRlcy9IZWFkZXJzCiAgaHR0cHM6Ly9kZXZlbG9wZXIudHJ5ZmluY2guY29tL2FwaS1yZWZlcmVuY2Uvb3JnYW5pemF0aW9uL2RpcmVjdG9yeQoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9UT0tFTiA9ICJmaW5jaF9hY2Nlc3NfdG9rZW5fYWJjMTIzIgoKIyBIaWRkZW4gZGF0YXNldDogNyBpbmRpdmlkdWFscywgb25seSBvYnNlcnZhYmxlIGJ5IGFjdHVhbGx5IGNhbGxpbmcgdGhlIG1vY2sgYW5kIHBhZ2luYXRpbmcuCklORElWSURVQUxTID0gWwogICAgewogICAgICAgICJpZCI6IGYiMjIyMjIyMjItMDAwMC00MDAwLTgwMDAtMDAwMDAwMDAwMDB7aX0iLAogICAgICAgICJmaXJzdF9uYW1lIjogZiJGaXJzdHtpfSIsCiAgICAgICAgIm1pZGRsZV9uYW1lIjogTm9uZSwKICAgICAgICAibGFzdF9uYW1lIjogZiJMYXN0e2l9IiwKICAgICAgICAiZGVwYXJ0bWVudCI6IHsibmFtZSI6ICJFbmdpbmVlcmluZyJ9LAogICAgICAgICJtYW5hZ2VyIjogTm9uZSwKICAgICAgICAiaXNfYWN0aXZlIjogVHJ1ZSwKICAgIH0KICAgIGZvciBpIGluIHJhbmdlKDEsIDgpCl0KUEFHRV9TSVpFID0gMyAgIyBzZXJ2ZXIgcGFnZSBzaXplOiBhdCBtb3N0IHRoaXMgbWFueSBpbmRpdmlkdWFscyBwZXIgcmVzcG9uc2UsIGZvcmNpbmcgcGFnaW5hdGlvbgoKCmNsYXNzIF9Nb2NrRmluY2goQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBoZWFkZXJfa2V5cyA9IHtrLmxvd2VyKCkgZm9yIGsgaW4gc2VsZi5oZWFkZXJzLmtleXMoKX0KICAgICAgICB2ZXJzaW9uID0gc2VsZi5oZWFkZXJzLmdldCgiRmluY2gtQVBJLVZlcnNpb24iKQogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJoYXNfdmVyc2lvbl9oZWFkZXIiOiAiZmluY2gtYXBpLXZlcnNpb24iIGluIGhlYWRlcl9rZXlzLAogICAgICAgICAgICAidmVyc2lvbiI6IHZlcnNpb24sCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImxpbWl0IjogcXMuZ2V0KCJsaW1pdCIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJvZmZzZXQiOiBxcy5nZXQoIm9mZnNldCIsIFtOb25lXSlbMF0sCiAgICAgICAgfSkKCiAgICAgICAgaWYgcGF0aCAhPSAiL2VtcGxveWVyL2RpcmVjdG9yeSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImNvZGUiOiA0MDQsICJuYW1lIjogIm5vdF9mb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiRmluY2ggbGlzdHMgdGhlIG9yZyBkaXJlY3RvcnkgYXQgR0VUIC9lbXBsb3llci9kaXJlY3RvcnkifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIGF1dGggIT0gZiJCZWFyZXIge0VYUEVDVEVEX1RPS0VOfSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImNvZGUiOiA0MDEsICJuYW1lIjogImludmFsaWRfYWNjZXNzX3Rva2VuIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJBdXRob3JpemF0aW9uIG11c3QgYmUgJ0JlYXJlciA8YWNjZXNzX3Rva2VuPicifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIHZlcnNpb24gaXMgTm9uZSBvciBub3QgcmUuZnVsbG1hdGNoKHIiXGR7NH0tXGR7Mn0tXGR7Mn0iLCB2ZXJzaW9uKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsiY29kZSI6IDQwMCwgIm5hbWUiOiAibWlzc2luZ192ZXJzaW9uX2hlYWRlciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiRmluY2ggcmVxdWlyZXMgdGhlIEZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciAoZS5nLiAyMDIwLTA5LTE3KSJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyBvZmZzZXQgcGFnaW5hdGlvbjogaG9ub3IgYG9mZnNldGAgKGRlZmF1bHQgMCk7IHNlcnZlciBjYXBzIHRoZSBwYWdlIGF0IFBBR0VfU0laRS4KICAgICAgICB0cnk6CiAgICAgICAgICAgIG9mZnNldCA9IGludChxcy5nZXQoIm9mZnNldCIsIFsiMCJdKVswXSkKICAgICAgICBleGNlcHQgKFR5cGVFcnJvciwgVmFsdWVFcnJvcik6CiAgICAgICAgICAgIG9mZnNldCA9IDAKICAgICAgICBpZiBvZmZzZXQgPCAwOgogICAgICAgICAgICBvZmZzZXQgPSAwCiAgICAgICAgdHJ5OgogICAgICAgICAgICByZXF1ZXN0ZWRfbGltaXQgPSBpbnQocXMuZ2V0KCJsaW1pdCIsIFtzdHIoUEFHRV9TSVpFKV0pWzBdKQogICAgICAgIGV4Y2VwdCAoVHlwZUVycm9yLCBWYWx1ZUVycm9yKToKICAgICAgICAgICAgcmVxdWVzdGVkX2xpbWl0ID0gUEFHRV9TSVpFCiAgICAgICAgcGFnZV9sZW4gPSBtYXgoMCwgbWluKFBBR0VfU0laRSwgcmVxdWVzdGVkX2xpbWl0KSkKCiAgICAgICAgcGFnZSA9IElORElWSURVQUxTW29mZnNldDpvZmZzZXQgKyBwYWdlX2xlbl0KICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAicGFnaW5nIjogeyJjb3VudCI6IGxlbihJTkRJVklEVUFMUyksICJvZmZzZXQiOiBvZmZzZXR9LAogICAgICAgICAgICAiaW5kaXZpZHVhbHMiOiBwYWdlLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0ZpbmNoLnJlcXVlc3RfbG9nID0gW10KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tGaW5jaCkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9pbmRpdmlkdWFsc19hY3Jvc3Nfb2Zmc2V0X3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2luZGl2aWR1YWxzIiksIFwKICAgICAgICAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgbGlzdF9hbGxfaW5kaXZpZHVhbHMoYmFzZV91cmwsIGFjY2Vzc190b2tlbikiCgogICAgcmVzdWx0ID0gbW9kdWxlLmxpc3RfYWxsX2luZGl2aWR1YWxzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBpbmRpdmlkdWFscywgZ290IHt0eXBlKHJlc3VsdCkuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGRbImlkIl0gZm9yIGQgaW4gcmVzdWx0IGlmIGlzaW5zdGFuY2UoZCwgZGljdCkgYW5kICJpZCIgaW4gZCkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChkWyJpZCJdIGZvciBkIGluIElORElWSURVQUxTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCB7bGVuKGlkcyl9IGluZGl2aWR1YWwgaWRzIHtpZHN9LCBleHBlY3RlZCB7bGVuKGV4cGVjdGVkX2lkcyl9IHtleHBlY3RlZF9pZHN9LiAiCiAgICAgICAgZiJBIGNvcnJlY3QgY2xpZW50IHBhZ2luYXRlcyBieSBgb2Zmc2V0YCB3aGlsZSBwYWdpbmcub2Zmc2V0K2xlbihpbmRpdmlkdWFscykgPCBwYWdpbmcuY291bnQgIgogICAgICAgIGYiKHRoZXJlIGlzIG5vIG5leHQvaGFzX21vcmUgY3Vyc29yKSwgc2VuZGluZyB0aGUgQmVhcmVyIHRva2VuIEFORCB0aGUgcmVxdWlyZWQgIgogICAgICAgIGYiRmluY2gtQVBJLVZlcnNpb24gaGVhZGVyIGVhY2ggcmVxdWVzdC4gU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrRmluY2gucmVxdWVzdF9sb2d9IgogICAgKQo=" + }, + { + "vendor": "finch:get_directory", + "fn": "get_directory", + "testFile": "test_finch_directory.py", + "signature": "def get_directory(base_url: str, access_token: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgZmluY2gtZGlyZWN0b3J5LXBhZ2UgKG1lZGl1bSkgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpJdCBzdGFuZHMgdXAgYSBMT0NBTCBtb2NrIHRoYXQgZmFpdGhmdWxseSBpbXBsZW1lbnRzIEZpbmNoJ3MgQ1VSUkVOVAooVW5pdmVyc2FsIEVtcGxveW1lbnQgQVBJKSBkaXJlY3RvcnkgY29udHJhY3QgZm9yIGEgc2luZ2xlIHJlc3BvbnNlOgogIC0gZW5kcG9pbnQgcGF0aCAgICAgICAgL2VtcGxveWVyL2RpcmVjdG9yeSAgICAgICAgKGJhc2UgaG9zdCBpcyBpbmplY3RlZDsgdGhlIHBhdGggaXMgZml4ZWQpCiAgLSBhdXRoICAgICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFjY2Vzc190b2tlbj4KICAtIHZlcnNpb24gaGVhZGVyICAgICAgIEZpbmNoLUFQSS1WZXJzaW9uOiA8WVlZWS1NTS1ERD4gICAoUkVRVUlSRUQ7IG1pc3NpbmcgLT4gNDAwKQogIC0gcmVzcG9uc2UgZW52ZWxvcGUgICAgeyJwYWdpbmciOiB7ImNvdW50IiwgIm9mZnNldCJ9LCAiaW5kaXZpZHVhbHMiOiBbLi4uXX0KCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIEZpbmNoIGRvY3Mgc2VuZHMgdGhlIEJlYXJlciB0b2tlbiBBTkQgdGhlIHJlcXVpcmVkCkZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciB0byAvZW1wbG95ZXIvZGlyZWN0b3J5IGFuZCByZWFkcyB0aGUgYGluZGl2aWR1YWxzYCBhcnJheS4KQSBjbGllbnQgd3JpdHRlbiBmcm9tIHN0YWxlL3ByZS1jdXRvZmYgbWVtb3J5IG9taXRzIHRoZSByZXF1aXJlZCB2ZXJzaW9uIGhlYWRlciAoNDAwKSwKZ3Vlc3NlcyBhIFJFU1QtaXNoIC9lbXBsb3llZXMgcGF0aCAoNDA0KSwgb3IgcmVhZHMgYSBgZGF0YWAvYHJlc3VsdHNgIGVudmVsb3BlLCBhbmQgRkFJTFMuCkFuIGVtcHR5IHNvbHV0aW9uIEZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RldmVsb3Blci50cnlmaW5jaC5jb20vYXBpLXJlZmVyZW5jZS9kZXZlbG9wbWVudC1ndWlkZXMvSGVhZGVycwogIGh0dHBzOi8vZGV2ZWxvcGVyLnRyeWZpbmNoLmNvbS9hcGktcmVmZXJlbmNlL29yZ2FuaXphdGlvbi9kaXJlY3RvcnkKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgdG9rZW4gdGhlIGNsaWVudCBtdXN0IHNlbmQgaW4gdGhlIEF1dGhvcml6YXRpb246IEJlYXJlciA8dG9rZW4+IGhlYWRlci4KRVhQRUNURURfVE9LRU4gPSAiZmluY2hfYWNjZXNzX3Rva2VuX2FiYzEyMyIKCiMgU21hbGwgZGlyZWN0b3J5IHJldHVybmVkIGluIGEgc2luZ2xlIHJlc3BvbnNlIChkZWZhdWx0cy10by1hbGw7IG5vIHBhZ2luYXRpb24gaW4gdGhpcyBsZWFmKS4KSU5ESVZJRFVBTFMgPSBbCiAgICB7CiAgICAgICAgImlkIjogZiIxMTExMTExMS0wMDAwLTQwMDAtODAwMC0wMDAwMDAwMDAwMHtpfSIsCiAgICAgICAgImZpcnN0X25hbWUiOiBmIkZpcnN0e2l9IiwKICAgICAgICAibWlkZGxlX25hbWUiOiBOb25lLAogICAgICAgICJsYXN0X25hbWUiOiBmIkxhc3R7aX0iLAogICAgICAgICJkZXBhcnRtZW50IjogeyJuYW1lIjogIkVuZ2luZWVyaW5nIn0sCiAgICAgICAgIm1hbmFnZXIiOiBOb25lLAogICAgICAgICJpc19hY3RpdmUiOiBUcnVlLAogICAgfQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgNCkKXQoKCmNsYXNzIF9Nb2NrRmluY2goQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICAjIENsYXNzLWxldmVsIHJlcXVlc3QgbG9nIHNvIHRoZSB0ZXN0IGNhbiByZXBvcnQgd2hhdCB0aGUgY2xpZW50IGFjdHVhbGx5IHNlbnQgb24gZmFpbHVyZS4KICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKTogICMgc2lsZW5jZSBzdGRlcnIgYWNjZXNzIGxvZwogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgaGVhZGVyX2tleXMgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdmVyc2lvbiA9IHNlbGYuaGVhZGVycy5nZXQoIkZpbmNoLUFQSS1WZXJzaW9uIikKICAgICAgICBhdXRoID0gc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpCiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX3ZlcnNpb25faGVhZGVyIjogImZpbmNoLWFwaS12ZXJzaW9uIiBpbiBoZWFkZXJfa2V5cywKICAgICAgICAgICAgInZlcnNpb24iOiB2ZXJzaW9uLAogICAgICAgICAgICAiYXV0aG9yaXphdGlvbiI6IGF1dGgsCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB0aGUgZGlyZWN0b3J5IGxpdmVzIGF0IC9lbXBsb3llci9kaXJlY3RvcnkgKGhvc3QgaXMgaW5qZWN0ZWQpLgogICAgICAgIGlmIHBhdGggIT0gIi9lbXBsb3llci9kaXJlY3RvcnkiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJjb2RlIjogNDA0LCAibmFtZSI6ICJub3RfZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkZpbmNoIGxpc3RzIHRoZSBvcmcgZGlyZWN0b3J5IGF0IEdFVCAvZW1wbG95ZXIvZGlyZWN0b3J5In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IEF1dGhvcml6YXRpb24gbXVzdCBiZSB0aGUgQmVhcmVyIGFjY2VzcyB0b2tlbi4KICAgICAgICBpZiBhdXRoICE9IGYiQmVhcmVyIHtFWFBFQ1RFRF9UT0tFTn0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJjb2RlIjogNDAxLCAibmFtZSI6ICJpbnZhbGlkX2FjY2Vzc190b2tlbiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiQXV0aG9yaXphdGlvbiBtdXN0IGJlICdCZWFyZXIgPGFjY2Vzc190b2tlbj4nIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgzKSB2ZXJzaW9uIGF4aXM6IEZpbmNoLUFQSS1WZXJzaW9uIGlzIGEgUkVRVUlSRUQgaGVhZGVyIChkYXRlLWZvcm1hdHRlZCkuCiAgICAgICAgaWYgdmVyc2lvbiBpcyBOb25lIG9yIG5vdCByZS5mdWxsbWF0Y2gociJcZHs0fS1cZHsyfS1cZHsyfSIsIHZlcnNpb24pOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJjb2RlIjogNDAwLCAibmFtZSI6ICJtaXNzaW5nX3ZlcnNpb25faGVhZGVyIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJGaW5jaCByZXF1aXJlcyB0aGUgRmluY2gtQVBJLVZlcnNpb24gaGVhZGVyIChlLmcuIDIwMjAtMDktMTcpIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAicGFnaW5nIjogeyJjb3VudCI6IGxlbihJTkRJVklEVUFMUyksICJvZmZzZXQiOiAwfSwKICAgICAgICAgICAgImluZGl2aWR1YWxzIjogSU5ESVZJRFVBTFMsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrRmluY2gucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0ZpbmNoKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RpcmVjdG9yeShtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfZGlyZWN0b3J5IiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfZGlyZWN0b3J5KGJhc2VfdXJsLCBhY2Nlc3NfdG9rZW4pIgoKICAgIHJlc3VsdCA9IG1vZHVsZS5nZXRfZGlyZWN0b3J5KG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBpbmRpdmlkdWFscywgZ290IHt0eXBlKHJlc3VsdCkuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGRbImlkIl0gZm9yIGQgaW4gcmVzdWx0IGlmIGlzaW5zdGFuY2UoZCwgZGljdCkgYW5kICJpZCIgaW4gZCkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChkWyJpZCJdIGZvciBkIGluIElORElWSURVQUxTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBpbmRpdmlkdWFsIGlkcyB7aWRzfSwgZXhwZWN0ZWQge2V4cGVjdGVkX2lkc30uICIKICAgICAgICBmIkEgY29ycmVjdCBjbGllbnQgR0VUcyAvZW1wbG95ZXIvZGlyZWN0b3J5IHdpdGggdGhlIEJlYXJlciB0b2tlbiBBTkQgdGhlIHJlcXVpcmVkICIKICAgICAgICBmIkZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciwgdGhlbiByZWFkcyB0aGUgYGluZGl2aWR1YWxzYCBhcnJheS4gIgogICAgICAgIGYiU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrRmluY2gucmVxdWVzdF9sb2d9IgogICAgKQo=" + }, + { + "vendor": "novu:sync_subscribers_to_deals", + "fn": "sync_subscribers_to_deals", + "testFile": "test_pipeline_sync_idempotent.py", + "signature": "def sync_subscribers_to_deals(novu_base_url: str, novu_api_key: str, pd_base_url: str, pd_api_token: str) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcGlwZWxpbmUtbm92dS10by1waXBlZHJpdmUtc3luYy1pZGVtcG90ZW50IChoYXJkKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUKdGltZSAoYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdAp0aGUgYWdlbnQncyBzb2x1dGlvbi9zb2x1dGlvbi5weS4KClNhbWUgVFdPIHZlbmRvciBtb2NrcyBhcyB0aGUgbWVkaXVtIGxlYWYgKE5vdnUgc291cmNlICsgUGlwZWRyaXZlIHNpbms7IHNlZSB0aGUKYXhlcyBiZWxvdyksIGJ1dCB0aGUgdGVzdCBjYWxscyB0aGUgcGlwZWxpbmUgVFdJQ0UgYWdhaW5zdCB0aGUgc2FtZSBQaXBlZHJpdmUKc3RvcmUuIEEgY29ycmVjdCBwaXBlbGluZSBpcyBJREVNUE9URU5UOiBpdCBkZXJpdmVzIGFscmVhZHktc3luY2VkIHN0YXRlIGZyb20KUGlwZWRyaXZlIElUU0VMRiDigJQgR0VUIC9hcGkvdjIvZGVhbHMsIGZvbGxvd2luZyBjdXJzb3IgcGFnaW5hdGlvbiB0byBjb21wbGV0aW9uCih0aGUgc3RvcmUgaG9sZHMgNyBkZWFscyA9IDMgcGFnZXMgYXQgc2VydmVyIHBhZ2Ugc2l6ZSAzLCBzbyBhIGNsaWVudCBzdGFsZSBvbgp0aGUgTElTVCBjb250cmFjdCBvbmx5IHNlZXMgcGFnZSAxIGFuZCByZS1jcmVhdGVzIHRoZSByZXN0KSDigJQgYW5kIHNraXBzCnN1YnNjcmliZXJzIHdob3NlIGVtYWlsIGFscmVhZHkgdGl0bGVzIGEgZGVhbC4KCiAgTk9WVSAoc291cmNlKTogICAgL3YyL3N1YnNjcmliZXJzICsgQXV0aG9yaXphdGlvbjogQXBpS2V5IDx0b2tlbj4gKyBjdXJzb3IKICAgICAgICAgICAgICAgICAgICBgYWZ0ZXJgL2BuZXh0YCAoe2RhdGEsIG5leHQsIHByZXZpb3VzLCB0b3RhbENvdW50LCB0b3RhbENvdW50Q2FwcGVkfSkKICBQSVBFRFJJVkUgKHNpbmspOiBQT1NUIC9hcGkvdjIvZGVhbHMgKEpTT04sIGB0aXRsZWAgcmVxdWlyZWQsIDIwMSkgYW5kCiAgICAgICAgICAgICAgICAgICAgR0VUIC9hcGkvdjIvZGVhbHMgKGN1cnNvciBgY3Vyc29yYCArIGFkZGl0aW9uYWxfZGF0YS5uZXh0X2N1cnNvciksCiAgICAgICAgICAgICAgICAgICAgYm90aCB4LWFwaS10b2tlbiBIRUFERVIgYXV0aCAobmV2ZXIgP2FwaV90b2tlbj0pCgpBc3NlcnRlZDogcnVuIDEgc3luY3MgYWxsIDcgKGNvcnJlY3QgdGl0bGVzLCBjb3JyZWN0IGF1dGgsIE5vdnUgY3Vyc29yIGZvbGxvd2VkKTsKcnVuIDIgcmV0dXJucyBzeW5jZWQgPT0gMCwgY3JlYXRlcyBOTyBuZXcgZGVhbCAoc3RvcmUgc3RpbGwgZXhhY3RseSA3KSwgYW5kCmFjdHVhbGx5IHJlLXJlYWQgdGhlIFBpcGVkcml2ZSBzdG9yZSAoPj0gMyBsaXN0IEdFVHMgaW4gcnVuIDIg4oCUIGFuIGluLXByb2Nlc3MKY2FjaGUgaW5zdGVhZCBvZiByZWFkaW5nIHRoZSBzaW5rIEZBSUxTKS4gQSBub24taWRlbXBvdGVudCBwaXBlbGluZSBkdXBsaWNhdGVzCjcgLT4gMTQgYW5kIEZBSUxTLiBTdGFsZSB2MS9CZWFyZXIvb2Zmc2V0IGNsaWVudHMgRkFJTCBhcyBpbiB0aGUgbWVkaXVtIGxlYWYuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZG9jcy5ub3Z1LmNvL2FwaS1yZWZlcmVuY2UvYXV0aGVudGljYXRpb24KICBodHRwczovL2RvY3Mubm92dS5jby9hcGktcmVmZXJlbmNlL3N1YnNjcmliZXJzL3NlYXJjaC1zdWJzY3JpYmVycwogIGh0dHBzOi8vcGlwZWRyaXZlLnJlYWRtZS5pby9kb2NzL2NvcmUtYXBpLWNvbmNlcHRzLWF1dGhlbnRpY2F0aW9uCiAgaHR0cHM6Ly9waXBlZHJpdmUucmVhZG1lLmlvL2RvY3MvY29yZS1hcGktY29uY2VwdHMtcGFnaW5hdGlvbgogIGh0dHBzOi8vcGlwZWRyaXZlLnJlYWRtZS5pby9kb2NzL3BpcGVkcml2ZS1hcGktdjItbWlncmF0aW9uLWd1aWRlCiIiIgoKaW1wb3J0IGJhc2U2NAppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgojIEdyYWRlci1zaWRlIHNlY3JldHMg4oCUIERJRkZFUkVOVCBmcm9tIHRoZSBsaXZlLXR3aW4gdG9rZW5zIHRoZSBhZ2VudCBvYnNlcnZlZC4KTk9WVV9UT0tFTiA9ICJudl9zeW5jX2dyYWRlcl9rZXlfMzE0MTU5IgpQRF9UT0tFTiA9ICJwZF9zeW5jX2dyYWRlcl90b2tlbl8yNzE4MjgiCgpTVUJTQ1JJQkVSUyA9IFsKICAgIHsiX2lkIjogZiJfaWRfe2l9IiwgInN1YnNjcmliZXJJZCI6IGYidXNlcl97aX0iLCAiZW1haWwiOiBmInVzZXJ7aX1AZ3JhZGVyY29ycC5leGFtcGxlIn0KICAgIGZvciBpIGluIHJhbmdlKDEsIDgpCl0KTk9WVV9QQUdFX1NJWkUgPSAzClBEX1BBR0VfU0laRSA9IDMKCgpkZWYgX2VuY29kZV9jdXJzb3Iob2Zmc2V0OiBpbnQpIC0+IHN0cjoKICAgIHJldHVybiBiYXNlNjQudXJsc2FmZV9iNjRlbmNvZGUoc3RyKG9mZnNldCkuZW5jb2RlKCkpLmRlY29kZSgpCgoKZGVmIF9kZWNvZGVfY3Vyc29yKGN1cnNvcjogc3RyKSAtPiBpbnQ6CiAgICByZXR1cm4gaW50KGJhc2U2NC51cmxzYWZlX2I2NGRlY29kZShjdXJzb3IuZW5jb2RlKCkpLmRlY29kZSgpKQoKCmNsYXNzIF9Nb2NrTm92dShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImFwaWtleV9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkFwaUtleSAiKSwKICAgICAgICAgICAgImJlYXJlcl9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkJlYXJlciAiKSwKICAgICAgICAgICAgImFmdGVyIjogcXMuZ2V0KCJhZnRlciIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJwYWdlIjogcXMuZ2V0KCJwYWdlIiwgW05vbmVdKVswXSwKICAgICAgICB9KQoKICAgICAgICBpZiBwYXRoICE9ICIvdjIvc3Vic2NyaWJlcnMiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJzdGF0dXNDb2RlIjogNDA0LCAibWVzc2FnZSI6ICJub3QgZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6ICJOb3Z1IEFQSSB2MiBsaXN0cyBzdWJzY3JpYmVycyBhdCAvdjIvc3Vic2NyaWJlcnMifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIGF1dGggIT0gZiJBcGlLZXkge05PVlVfVE9LRU59IjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsic3RhdHVzQ29kZSI6IDQwMSwgIm1lc3NhZ2UiOiAidW5hdXRob3JpemVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3IiOiAiTm92dSBhdXRoZW50aWNhdGVzIHZpYSBgQXV0aG9yaXphdGlvbjogQXBpS2V5IDxzZWNyZXQ+YCAobm90IEJlYXJlcikifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGFmdGVyID0gcXMuZ2V0KCJhZnRlciIsIFtOb25lXSlbMF0KICAgICAgICBpZiBhZnRlciBpcyBOb25lOgogICAgICAgICAgICBvZmZzZXQgPSAwCiAgICAgICAgZWxzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgb2Zmc2V0ID0gX2RlY29kZV9jdXJzb3IoYWZ0ZXIpCiAgICAgICAgICAgIGV4Y2VwdCBFeGNlcHRpb246CiAgICAgICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJzdGF0dXNDb2RlIjogNDAwLCAibWVzc2FnZSI6ICJpbnZhbGlkIGN1cnNvciJ9KQogICAgICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIHBhZ2UgPSBTVUJTQ1JJQkVSU1tvZmZzZXQ6b2Zmc2V0ICsgTk9WVV9QQUdFX1NJWkVdCiAgICAgICAgbmV4dF9vZmZzZXQgPSBvZmZzZXQgKyBOT1ZVX1BBR0VfU0laRQogICAgICAgIG5leHRfY3Vyc29yID0gX2VuY29kZV9jdXJzb3IobmV4dF9vZmZzZXQpIGlmIG5leHRfb2Zmc2V0IDwgbGVuKFNVQlNDUklCRVJTKSBlbHNlIE5vbmUKICAgICAgICBwcmV2X2N1cnNvciA9IF9lbmNvZGVfY3Vyc29yKG1heChvZmZzZXQgLSBOT1ZVX1BBR0VfU0laRSwgMCkpIGlmIG9mZnNldCA+IDAgZWxzZSBOb25lCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsiZGF0YSI6IHBhZ2UsICJuZXh0IjogbmV4dF9jdXJzb3IsICJwcmV2aW91cyI6IHByZXZfY3Vyc29yLAogICAgICAgICAgICAgICAgICAgICAgICAgInRvdGFsQ291bnQiOiBsZW4oU1VCU0NSSUJFUlMpLCAidG90YWxDb3VudENhcHBlZCI6IEZhbHNlfSkKCgpjbGFzcyBfTW9ja1BpcGVkcml2ZShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KICAgIGRlYWxzID0gW10KICAgIF9sb2NrID0gdGhyZWFkaW5nLkxvY2soKQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOgogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgX3JlY29yZChzZWxmLCBwYXJzZWQsICoqZXh0cmEpOgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIGVudHJ5ID0gewogICAgICAgICAgICAibWV0aG9kIjogc2VsZi5jb21tYW5kLAogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX2hlYWRlcl9hdXRoIjogIngtYXBpLXRva2VuIiBpbiB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9LAogICAgICAgICAgICAicXVlcnlfYXBpX3Rva2VuIjogImFwaV90b2tlbiIgaW4gcXMsCiAgICAgICAgfQogICAgICAgIGVudHJ5LnVwZGF0ZShleHRyYSkKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZChlbnRyeSkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgY3Vyc29yID0gcXMuZ2V0KCJjdXJzb3IiLCBbTm9uZV0pWzBdCiAgICAgICAgc2VsZi5fcmVjb3JkKHBhcnNlZCwgY3Vyc29yPWN1cnNvciwgc3RhcnQ9cXMuZ2V0KCJzdGFydCIsIFtOb25lXSlbMF0pCgogICAgICAgIGlmIHBhdGggIT0gIi9hcGkvdjIvZGVhbHMiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJzdWNjZXNzIjogRmFsc2UsICJlcnJvciI6ICJub3QgZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvcl9pbmZvIjogIlBpcGVkcml2ZSBBUEkgdjIgbGlzdHMgZGVhbHMgYXQgL2FwaS92Mi9kZWFscyJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgaWYgc2VsZi5oZWFkZXJzLmdldCgieC1hcGktdG9rZW4iKSAhPSBQRF9UT0tFTjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsic3VjY2VzcyI6IEZhbHNlLCAiZXJyb3IiOiAidW5hdXRob3JpemVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3JfaW5mbyI6ICJQaXBlZHJpdmUgQVBJIHYyIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSB4LWFwaS10b2tlbiBoZWFkZXIifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIG9mZnNldCA9IDAKICAgICAgICBpZiBjdXJzb3IgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIG9mZnNldCA9IF9kZWNvZGVfY3Vyc29yKGN1cnNvcikKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAgICAgICAgIHNlbGYuX2pzb24oNDAwLCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogImludmFsaWQgY3Vyc29yIn0pCiAgICAgICAgICAgICAgICByZXR1cm4KICAgICAgICB3aXRoIHR5cGUoc2VsZikuX2xvY2s6CiAgICAgICAgICAgIHNuYXBzaG90ID0gbGlzdCh0eXBlKHNlbGYpLmRlYWxzKQogICAgICAgIHBhZ2UgPSBzbmFwc2hvdFtvZmZzZXQ6b2Zmc2V0ICsgUERfUEFHRV9TSVpFXQogICAgICAgIG5leHRfb2Zmc2V0ID0gb2Zmc2V0ICsgUERfUEFHRV9TSVpFCiAgICAgICAgbmV4dF9jdXJzb3IgPSBfZW5jb2RlX2N1cnNvcihuZXh0X29mZnNldCkgaWYgbmV4dF9vZmZzZXQgPCBsZW4oc25hcHNob3QpIGVsc2UgTm9uZQogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7InN1Y2Nlc3MiOiBUcnVlLCAiZGF0YSI6IHBhZ2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAiYWRkaXRpb25hbF9kYXRhIjogeyJuZXh0X2N1cnNvciI6IG5leHRfY3Vyc29yfX0pCgogICAgZGVmIGRvX1BPU1Qoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKCiAgICAgICAgaWYgcGF0aCAhPSAiL2FwaS92Mi9kZWFscyI6CiAgICAgICAgICAgIHNlbGYuX3JlY29yZChwYXJzZWQsIHRpdGxlPU5vbmUpCiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBjcmVhdGVzIGEgZGVhbCBhdCBQT1NUIC9hcGkvdjIvZGVhbHMifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIngtYXBpLXRva2VuIikgIT0gUERfVE9LRU46CiAgICAgICAgICAgIHNlbGYuX3JlY29yZChwYXJzZWQsIHRpdGxlPU5vbmUpCiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBhdXRoZW50aWNhdGVzIHZpYSB0aGUgeC1hcGktdG9rZW4gaGVhZGVyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBsZW5ndGggPSBpbnQoc2VsZi5oZWFkZXJzLmdldCgiQ29udGVudC1MZW5ndGgiKSBvciAwKQogICAgICAgIHJhdyA9IHNlbGYucmZpbGUucmVhZChsZW5ndGgpIGlmIGxlbmd0aCA+IDAgZWxzZSBiIiIKICAgICAgICB0cnk6CiAgICAgICAgICAgIGJvZHkgPSBqc29uLmxvYWRzKHJhdy5kZWNvZGUoKSBvciAibnVsbCIpCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAgICAgYm9keSA9IE5vbmUKICAgICAgICB0aXRsZSA9IChib2R5IG9yIHt9KS5nZXQoInRpdGxlIikgaWYgaXNpbnN0YW5jZShib2R5LCBkaWN0KSBlbHNlIE5vbmUKICAgICAgICBzZWxmLl9yZWNvcmQocGFyc2VkLCB0aXRsZT10aXRsZSkKICAgICAgICBpZiBub3QgaXNpbnN0YW5jZSh0aXRsZSwgc3RyKSBvciBub3QgdGl0bGUuc3RyaXAoKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsic3VjY2VzcyI6IEZhbHNlLCAiZXJyb3IiOiAiYmFkIHJlcXVlc3QiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvcl9pbmZvIjogInRoZSBgdGl0bGVgIGZpZWxkIGlzIHJlcXVpcmVkIChKU09OIGJvZHkpIn0pCiAgICAgICAgICAgIHJldHVybgogICAgICAgIHdpdGggdHlwZShzZWxmKS5fbG9jazoKICAgICAgICAgICAgZGVhbCA9IHsiaWQiOiBsZW4odHlwZShzZWxmKS5kZWFscykgKyAxLCAidGl0bGUiOiB0aXRsZSwgInZhbHVlIjogMCwKICAgICAgICAgICAgICAgICAgICAiY3VycmVuY3kiOiAiVVNEIiwgInN0YXR1cyI6ICJvcGVuIn0KICAgICAgICAgICAgdHlwZShzZWxmKS5kZWFscy5hcHBlbmQoZGVhbCkKICAgICAgICBzZWxmLl9qc29uKDIwMSwgeyJzdWNjZXNzIjogVHJ1ZSwgImRhdGEiOiBkZWFsfSkKCgpkZWYgX3NlcnZlKGhhbmRsZXJfY2xzKToKICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgaGFuZGxlcl9jbHMpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpLnN0YXJ0KCkKICAgIHJldHVybiBzZXJ2ZXIsIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIHNlcnZpY2VzKCk6CiAgICBfTW9ja05vdnUucmVxdWVzdF9sb2cgPSBbXQogICAgX01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2cgPSBbXQogICAgX01vY2tQaXBlZHJpdmUuZGVhbHMgPSBbXQogICAgbm92dV9zZXJ2ZXIsIG5vdnVfdXJsID0gX3NlcnZlKF9Nb2NrTm92dSkKICAgIHBkX3NlcnZlciwgcGRfdXJsID0gX3NlcnZlKF9Nb2NrUGlwZWRyaXZlKQogICAgdHJ5OgogICAgICAgIHlpZWxkIG5vdnVfdXJsLCBwZF91cmwKICAgIGZpbmFsbHk6CiAgICAgICAgbm92dV9zZXJ2ZXIuc2h1dGRvd24oKQogICAgICAgIG5vdnVfc2VydmVyLnNlcnZlcl9jbG9zZSgpCiAgICAgICAgcGRfc2VydmVyLnNodXRkb3duKCkKICAgICAgICBwZF9zZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3Rfc3luY19pc19jb21wbGV0ZV9hbmRfaWRlbXBvdGVudChzZXJ2aWNlcyk6CiAgICBub3Z1X3VybCwgcGRfdXJsID0gc2VydmljZXMKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgInN5bmNfc3Vic2NyaWJlcnNfdG9fZGVhbHMiKSwgKAogICAgICAgICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBzeW5jX3N1YnNjcmliZXJzX3RvX2RlYWxzKG5vdnVfYmFzZV91cmwsIG5vdnVfYXBpX2tleSwgcGRfYmFzZV91cmwsIHBkX2FwaV90b2tlbikiCiAgICApCgogICAgIyBSdW4gMTogZW1wdHkgc2luayDigJQgZXZlcnkgc3Vic2NyaWJlciBiZWNvbWVzIGEgZGVhbC4KICAgIHJlc3VsdDEgPSBtb2R1bGUuc3luY19zdWJzY3JpYmVyc190b19kZWFscyhub3Z1X3VybCwgTk9WVV9UT0tFTiwgcGRfdXJsLCBQRF9UT0tFTikKICAgIGFzc2VydCBpc2luc3RhbmNlKHJlc3VsdDEsIGRpY3QpIGFuZCByZXN1bHQxLmdldCgic3luY2VkIikgPT0gbGVuKFNVQlNDUklCRVJTKSwgKAogICAgICAgIGYiZmlyc3QgcnVuOiBleHBlY3RlZCB7eydzeW5jZWQnOiB7bGVuKFNVQlNDUklCRVJTKX19fSwgZ290IHtyZXN1bHQxIXJ9LiAiCiAgICAgICAgZiJOb3Z1IHNhdzoge19Nb2NrTm92dS5yZXF1ZXN0X2xvZ307IFBpcGVkcml2ZSBzYXc6IHtfTW9ja1BpcGVkcml2ZS5yZXF1ZXN0X2xvZ30iCiAgICApCiAgICB0aXRsZXMgPSBzb3J0ZWQoZFsidGl0bGUiXSBmb3IgZCBpbiBfTW9ja1BpcGVkcml2ZS5kZWFscykKICAgIGV4cGVjdGVkX3RpdGxlcyA9IHNvcnRlZChzWyJlbWFpbCJdIGZvciBzIGluIFNVQlNDUklCRVJTKQogICAgYXNzZXJ0IHRpdGxlcyA9PSBleHBlY3RlZF90aXRsZXMsICgKICAgICAgICBmImZpcnN0IHJ1bjogUGlwZWRyaXZlIHN0b3JlIGhvbGRzIHRpdGxlcyB7dGl0bGVzfSwgZXhwZWN0ZWQge2V4cGVjdGVkX3RpdGxlc30uICIKICAgICAgICBmIk5vdnUgc2F3OiB7X01vY2tOb3Z1LnJlcXVlc3RfbG9nfTsgUGlwZWRyaXZlIHNhdzoge19Nb2NrUGlwZWRyaXZlLnJlcXVlc3RfbG9nfSIKICAgICkKICAgIG5vdnVfaGl0cyA9IFtyIGZvciByIGluIF9Nb2NrTm92dS5yZXF1ZXN0X2xvZyBpZiByWyJwYXRoIl0uc3RhcnRzd2l0aCgiL3YyL3N1YnNjcmliZXJzIildCiAgICBhc3NlcnQgbGVuKG5vdnVfaGl0cykgPj0gMyBhbmQgYWxsKHJbImFwaWtleV9zY2hlbWUiXSBmb3IgciBpbiBub3Z1X2hpdHMpLCAoCiAgICAgICAgZiJOb3Z1IGN1cnNvciBtdXN0IGJlIGZvbGxvd2VkIGFjcm9zcyAzIHBhZ2VzIHdpdGggQXBpS2V5IGF1dGg6IHtfTW9ja05vdnUucmVxdWVzdF9sb2d9IgogICAgKQoKICAgIGdldHNfYmVmb3JlID0gbGVuKFtyIGZvciByIGluIF9Nb2NrUGlwZWRyaXZlLnJlcXVlc3RfbG9nIGlmIHJbIm1ldGhvZCJdID09ICJHRVQiXSkKICAgIHBvc3RzX2JlZm9yZSA9IGxlbihbciBmb3IgciBpbiBfTW9ja1BpcGVkcml2ZS5yZXF1ZXN0X2xvZyBpZiByWyJtZXRob2QiXSA9PSAiUE9TVCJdKQoKICAgICMgUnVuIDI6IHNpbmsgYWxyZWFkeSBob2xkcyBldmVyeSBzdWJzY3JpYmVyIOKAlCBub3RoaW5nIG5ldyBtYXkgYmUgY3JlYXRlZC4KICAgIHJlc3VsdDIgPSBtb2R1bGUuc3luY19zdWJzY3JpYmVyc190b19kZWFscyhub3Z1X3VybCwgTk9WVV9UT0tFTiwgcGRfdXJsLCBQRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQyLCBkaWN0KSBhbmQgcmVzdWx0Mi5nZXQoInN5bmNlZCIpID09IDAsICgKICAgICAgICBmInNlY29uZCBydW4gbXVzdCBjcmVhdGUgbm90aGluZyBhbmQgcmV0dXJuIHt7J3N5bmNlZCc6IDB9fSwgZ290IHtyZXN1bHQyIXJ9LiAiCiAgICAgICAgZiJQaXBlZHJpdmUgc2F3OiB7X01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2d9IgogICAgKQogICAgdGl0bGVzX2FmdGVyID0gc29ydGVkKGRbInRpdGxlIl0gZm9yIGQgaW4gX01vY2tQaXBlZHJpdmUuZGVhbHMpCiAgICBhc3NlcnQgdGl0bGVzX2FmdGVyID09IGV4cGVjdGVkX3RpdGxlcywgKAogICAgICAgIGYic2Vjb25kIHJ1biBkdXBsaWNhdGVkIGRlYWxzOiBzdG9yZSBob2xkcyB7dGl0bGVzX2FmdGVyfSwgZXhwZWN0ZWQgc3RpbGwgZXhhY3RseSAiCiAgICAgICAgZiJ7ZXhwZWN0ZWRfdGl0bGVzfS4gRXhpc3Rpbmcgc3RhdGUgbXVzdCBiZSByZWFkIGZyb20gUGlwZWRyaXZlIChsaXN0IGRlYWxzLCBmb2xsb3dpbmcgIgogICAgICAgIGYiY3Vyc29yIHBhZ2luYXRpb24gdG8gY29tcGxldGlvbikgYW5kIGFscmVhZHktc3luY2VkIHRpdGxlcyBza2lwcGVkLiAiCiAgICAgICAgZiJQaXBlZHJpdmUgc2F3OiB7X01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2d9IgogICAgKQogICAgcG9zdHNfYWZ0ZXIgPSBsZW4oW3IgZm9yIHIgaW4gX01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2cgaWYgclsibWV0aG9kIl0gPT0gIlBPU1QiXSkKICAgIGFzc2VydCBwb3N0c19hZnRlciA9PSBwb3N0c19iZWZvcmUsICgKICAgICAgICBmInNlY29uZCBydW4gbXVzdCBQT1NUIG5vdGhpbmc7IHNhdyB7cG9zdHNfYWZ0ZXIgLSBwb3N0c19iZWZvcmV9IG5ldyBQT1NUczogIgogICAgICAgIGYie19Nb2NrUGlwZWRyaXZlLnJlcXVlc3RfbG9nfSIKICAgICkKICAgICMgVGhlIHNlY29uZCBydW4gbXVzdCBkZXJpdmUgc3RhdGUgZnJvbSB0aGUgU0lOSyBpdHNlbGY6IDcgc3RvcmVkIGRlYWxzIGF0CiAgICAjIHNlcnZlciBwYWdlIHNpemUgMyA9IDMgbGlzdCBHRVRzIHRvIGVudW1lcmF0ZS4gQW4gaW4tcHJvY2VzcyBjYWNoZSAobm8KICAgICMgcmUtcmVhZCkgb3IgYW4gb2Zmc2V0LXBhZ2luYXRlZCBsaXN0IChzdG9wcyBhZnRlciBwYWdlIDEpIGZhaWxzIGhlcmUuCiAgICBnZXRzX2FmdGVyID0gbGVuKFtyIGZvciByIGluIF9Nb2NrUGlwZWRyaXZlLnJlcXVlc3RfbG9nIGlmIHJbIm1ldGhvZCJdID09ICJHRVQiXSkKICAgIGFzc2VydCBnZXRzX2FmdGVyIC0gZ2V0c19iZWZvcmUgPj0gMywgKAogICAgICAgIGYic2Vjb25kIHJ1biBtdXN0IHJlLXJlYWQgdGhlIFBpcGVkcml2ZSBkZWFscyBsaXN0IHRvIGNvbXBsZXRpb24gIgogICAgICAgIGYiKD49IDMgY3Vyc29yIHBhZ2VzKSwgc2F3IHtnZXRzX2FmdGVyIC0gZ2V0c19iZWZvcmV9IEdFVHM6IHtfTW9ja1BpcGVkcml2ZS5yZXF1ZXN0X2xvZ30iCiAgICApCg==" + }, + { + "vendor": "cal-com:list_all_bookings", + "fn": "list_all_bookings", + "testFile": "test_calcom_list.py", + "signature": "def list_all_bookings(base_url: str, api_key: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgY2FsY29tLXYyLWxpc3QtYWxsLWJvb2tpbmdzIChoYXJkKSB3ZWItZ3JvdW5kZWQgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpJdCBzdGFuZHMgdXAgYSBMT0NBTCBtb2NrIHRoYXQgZmFpdGhmdWxseSBpbXBsZW1lbnRzIENhbC5jb20ncyBDVVJSRU5UIChBUEkgdjIpIGJvb2tpbmdzLWxpc3QKY29udHJhY3Q6CiAgLSBlbmRwb2ludCBwYXRoICAgICAgICAvdjIvYm9va2luZ3MgICAgICAgICAgICAgICAgICAgICAodjEgd2FzIC92MS9ib29raW5ncykKICAtIGF1dGggICAgICAgICAgICAgICAgIEF1dGhvcml6YXRpb246IEJlYXJlciA8YXBpX2tleT4gICAodjEgdXNlZCA/YXBpS2V5PTxrZXk+IHF1ZXJ5IHBhcmFtKQogIC0gYXBpLXZlcnNpb24gaGVhZGVyICAgY2FsLWFwaS12ZXJzaW9uOiAyMDI2LTA1LTAxICAgICAgIChQT1NULUNVVE9GRjsgYSBmcm9tLW1lbW9yeSBjbGllbnQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb21pdHMgaXQgb3Igc2VuZHMgYW4gb2xkZXIgZGF0ZSBhbmQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2lsZW50bHkgZ2V0cyB0aGUgbGVnYWN5IHJlc3BvbnNlIHNoYXBlKQogIC0gcmVzcG9uc2UgZW52ZWxvcGUgICAgeyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6Wy4uLl0sInBhZ2luYXRpb24iOnsibmV4dEN1cnNvciIsImhhc01vcmUifX0KICAgICAgICAgICAgICAgICAgICAgICAgIChsZWdhY3kgc2hhcGUgd2FzIGEgYmFyZSB7ImJvb2tpbmdzIjpbLi4uXX0gd2l0aCBubyBjdXJzb3IgcGFnaW5hdGlvbikKICAtIHBhZ2luYXRpb24gICAgICAgICAgIGN1cnNvci1iYXNlZDogcXVlcnkgYGN1cnNvcmAsIHJlc3BvbnNlIHBhZ2luYXRpb24ubmV4dEN1cnNvciAvIGhhc01vcmUKCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIENhbC5jb20gZG9jcyBjb2xsZWN0cyBhbGwgNyBib29raW5ncyBhY3Jvc3MgMyBjdXJzb3IgcGFnZXMgYW5kCnBhc3Nlcy4gQSBjbGllbnQgd3JpdHRlbiBmcm9tIHN0YWxlL3ByZS1jdXRvZmYgbWVtb3J5ICh2MSBwYXRoLCBxdWVyeS1wYXJhbSBhdXRoLCBtaXNzaW5nL29sZApjYWwtYXBpLXZlcnNpb24sIG9yIGJhcmUteyJib29raW5ncyJ9IHBhcnNpbmcpIGhpdHMgNDA0IC8gNDAxIC8gYSBsZWdhY3kgZW52ZWxvcGUgaXQgY2Fubm90CnBhcnNlIC8gc3RvcHMgYWZ0ZXIgcGFnZSAxIGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9jYWwuY29tL2RvY3MvYXBpLXJlZmVyZW5jZS92Mi9ib29raW5ncy9nZXQtYWxsLWJvb2tpbmdzCiAgaHR0cHM6Ly9jYWwuY29tL2RvY3MvYXBpLXJlZmVyZW5jZS92Mi9pbnRyb2R1Y3Rpb24KIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgojIFRoZSBBUEkga2V5IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBBdXRob3JpemF0aW9uOiBCZWFyZXIgSEVBREVSICh2MiBjb250cmFjdCkuCkVYUEVDVEVEX0FQSV9LRVkgPSAiY2FsX2xpdmVfa2V5X2FiYzEyMyIKCiMgVGhlIENVUlJFTlQgKHBvc3QtY3V0b2ZmKSBhcGktdmVyc2lvbiB0aGUgR0VUIC92Mi9ib29raW5ncyBlbmRwb2ludCByZXF1aXJlcy4gQSBjbGllbnQgdGhhdAojIG9taXRzIHRoaXMgaGVhZGVyIG9yIHNlbmRzIGFuIG9sZGVyIGRhdGUgaXMgZGVmYXVsdGVkIHRvIHRoZSBsZWdhY3kgcmVzcG9uc2Ugc2hhcGUuClJFUVVJUkVEX0FQSV9WRVJTSU9OID0gIjIwMjYtMDUtMDEiCgojIFRoZSBoaWRkZW4gZGF0YXNldC4gT25seSBvYnNlcnZhYmxlIGJ5IGFjdHVhbGx5IGNhbGxpbmcgdGhlIG1vY2sgYW5kIHBhZ2luYXRpbmcgdG8gdGhlIGVuZC4KIyA3IGJvb2tpbmdzLCBzZXJ2ZXIgcGFnZSBzaXplIDMgLT4gMyBjdXJzb3IgcGFnZXMuCkJPT0tJTkdTID0gWwogICAgeyJpZCI6IGksICJ1aWQiOiBmImJrX3tpfSIsICJ0aXRsZSI6IGYiTWVldGluZyB7aX0iLCAic3RhdHVzIjogImFjY2VwdGVkIn0KICAgIGZvciBpIGluIHJhbmdlKDEsIDgpCl0KUEFHRV9TSVpFID0gMwoKCmNsYXNzIF9Nb2NrQ2FsY29tKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgIyBDbGFzcy1sZXZlbCByZXF1ZXN0IGxvZyBzbyB0aGUgdGVzdCBjYW4gaW5zcGVjdCB3aGF0IHBhdGgvYXV0aC92ZXJzaW9uIHRoZSBjbGllbnQgdXNlZC4KICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKTogICMgc2lsZW5jZSBzdGRlcnIgYWNjZXNzIGxvZwogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgYXV0aCA9IHNlbGYuaGVhZGVycy5nZXQoIkF1dGhvcml6YXRpb24iKQogICAgICAgIGFwaV92ZXJzaW9uID0gc2VsZi5oZWFkZXJzLmdldCgiY2FsLWFwaS12ZXJzaW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgInF1ZXJ5X2FwaUtleSI6ICJhcGlLZXkiIGluIHFzLAogICAgICAgICAgICAiY2FsX2FwaV92ZXJzaW9uIjogYXBpX3ZlcnNpb24sCiAgICAgICAgICAgICJjdXJzb3IiOiBxcy5nZXQoImN1cnNvciIsIFtOb25lXSlbMF0sCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiBvbmx5IHRoZSB2MiBwYXRoIGV4aXN0cy4KICAgICAgICBpZiBwYXRoICE9ICIvdjIvYm9va2luZ3MiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJzdGF0dXMiOiAiZXJyb3IiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6IHsiY29kZSI6ICJOT1RfRk9VTkQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJDYWwuY29tIEFQSSB2MiBsaXN0cyBib29raW5ncyBhdCAvdjIvYm9va2luZ3MifX0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IHYyIHJlcXVpcmVzIEF1dGhvcml6YXRpb246IEJlYXJlciA8a2V5PjsgcXVlcnktcGFyYW0gYXBpS2V5IGlzIG5vdCBob25vcmVkLgogICAgICAgIGlmIGF1dGggIT0gZiJCZWFyZXIge0VYUEVDVEVEX0FQSV9LRVl9IjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsic3RhdHVzIjogImVycm9yIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3IiOiB7ImNvZGUiOiAiVU5BVVRIT1JJWkVEIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiQ2FsLmNvbSBBUEkgdjIgYXV0aGVudGljYXRlcyB2aWEgQXV0aG9yaXphdGlvbjogQmVhcmVyIDxhcGlfa2V5PiJ9fSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDMpIGFwaS12ZXJzaW9uIGF4aXM6IHdpdGhvdXQgdGhlIENVUlJFTlQgKHBvc3QtY3V0b2ZmKSBjYWwtYXBpLXZlcnNpb24gdGhlIGVuZHBvaW50CiAgICAgICAgIyBkZWZhdWx0cyB0byB0aGUgTEVHQUNZIHJlc3BvbnNlIHNoYXBlIChiYXJlIHsiYm9va2luZ3MiOlsuLi5dfSwgbm8gY3Vyc29yIHBhZ2luYXRpb24pLgogICAgICAgICMgSVNPIFlZWVktTU0tREQgZGF0ZXMgY29tcGFyZSBjb3JyZWN0bHkgYXMgc3RyaW5ncy4KICAgICAgICBpZiBhcGlfdmVyc2lvbiBpcyBOb25lIG9yIGFwaV92ZXJzaW9uIDwgUkVRVUlSRURfQVBJX1ZFUlNJT046CiAgICAgICAgICAgIHNlbGYuX2pzb24oMjAwLCB7ImJvb2tpbmdzIjogQk9PS0lOR1NbOlBBR0VfU0laRV19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoNCkgcGFnaW5hdGlvbiBheGlzOiBjdXJzb3ItYmFzZWQgYWdhaW5zdCB0aGUgY3VycmVudCBlbnZlbG9wZS4KICAgICAgICBjdXJzb3IgPSBxcy5nZXQoImN1cnNvciIsIFtOb25lXSlbMF0KICAgICAgICBvZmZzZXQgPSBpbnQoY3Vyc29yKSBpZiBjdXJzb3IgaXMgbm90IE5vbmUgYW5kIGN1cnNvci5pc2RpZ2l0KCkgZWxzZSAwCiAgICAgICAgcGFnZSA9IEJPT0tJTkdTW29mZnNldDpvZmZzZXQgKyBQQUdFX1NJWkVdCiAgICAgICAgbmV4dF9vZmZzZXQgPSBvZmZzZXQgKyBQQUdFX1NJWkUKICAgICAgICBoYXNfbW9yZSA9IG5leHRfb2Zmc2V0IDwgbGVuKEJPT0tJTkdTKQogICAgICAgIG5leHRfY3Vyc29yID0gc3RyKG5leHRfb2Zmc2V0KSBpZiBoYXNfbW9yZSBlbHNlIE5vbmUKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAic3RhdHVzIjogInN1Y2Nlc3MiLAogICAgICAgICAgICAiZGF0YSI6IHBhZ2UsCiAgICAgICAgICAgICJwYWdpbmF0aW9uIjogeyJuZXh0Q3Vyc29yIjogbmV4dF9jdXJzb3IsICJoYXNNb3JlIjogaGFzX21vcmV9LAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0NhbGNvbS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrQ2FsY29tKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfbGlzdHNfYWxsX2Jvb2tpbmdzX2Fjcm9zc19jdXJzb3JfcGFnZXMobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAibGlzdF9hbGxfYm9va2luZ3MiKSwgXAogICAgICAgICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBsaXN0X2FsbF9ib29raW5ncyhiYXNlX3VybCwgYXBpX2tleSkiCgogICAgcmVzdWx0ID0gbW9kdWxlLmxpc3RfYWxsX2Jvb2tpbmdzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9BUElfS0VZKQoKICAgIGFzc2VydCBpc2luc3RhbmNlKHJlc3VsdCwgbGlzdCksIGYiZXhwZWN0ZWQgYSBsaXN0IG9mIGJvb2tpbmdzLCBnb3Qge3R5cGUocmVzdWx0KS5fX25hbWVfX30iCiAgICBpZHMgPSBzb3J0ZWQoYlsiaWQiXSBmb3IgYiBpbiByZXN1bHQgaWYgaXNpbnN0YW5jZShiLCBkaWN0KSBhbmQgImlkIiBpbiBiKQogICAgZXhwZWN0ZWRfaWRzID0gW2JbImlkIl0gZm9yIGIgaW4gQk9PS0lOR1NdCiAgICBhc3NlcnQgaWRzID09IGV4cGVjdGVkX2lkcywgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIGJvb2tpbmcgaWRzIHtpZHN9LCBleHBlY3RlZCB7ZXhwZWN0ZWRfaWRzfS4gIgogICAgICAgIGYiQSBjb3JyZWN0IHYyIGNsaWVudCBzZW5kcyBBdXRob3JpemF0aW9uOiBCZWFyZXIsIGNhbC1hcGktdmVyc2lvbjoge1JFUVVJUkVEX0FQSV9WRVJTSU9OfSwgIgogICAgICAgIGYiYW5kIGZvbGxvd3MgcGFnaW5hdGlvbi5uZXh0Q3Vyc29yIHRvIHRoZSBlbmQuICIKICAgICAgICBmIlNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0NhbGNvbS5yZXF1ZXN0X2xvZ30iCiAgICApCg==" + }, + { + "vendor": "cal-com:get_booking", + "fn": "get_booking", + "testFile": "test_calcom_get.py", + "signature": "def get_booking(base_url: str, api_key: str, booking_uid: str) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgY2FsY29tLXYyLWdldC1ib29raW5nIChtZWRpdW0pIHdlYi1ncm91bmRlZCBsZWFmLgoKU2ltcGxlciBzbGljZSBvZiB0aGUgU0FNRSBDYWwuY29tIEFQSSB2MiBjb250cmFjdCAtLSBhIHNpbmdsZS1yZXNvdXJjZSBHRVQsIG5vIHBhZ2luYXRpb24uClN0aWxsIHNlYXJjaC1uZWNlc3Nhcnkgb24gdGhyZWUgYXhlczogdGhlIHYyIGVuZHBvaW50IHBhdGgsIHRoZSBBdXRob3JpemF0aW9uOiBCZWFyZXIgaGVhZGVyLAphbmQgdGhlIFBPU1QtQ1VUT0ZGIGNhbC1hcGktdmVyc2lvbiBoZWFkZXIgKDIwMjYtMDItMjUgZm9yIHRoZSBzaW5nbGUtYm9va2luZyBlbmRwb2ludCkuIEEKZnJvbS1tZW1vcnkgY2xpZW50IHVzZXMgL3YxL2Jvb2tpbmdzL3tpZH0/YXBpS2V5PS4uLiBvciBvbWl0cyB0aGUgdmVyc2lvbiBoZWFkZXIgYW5kIGZhaWxzLgoKICAtIGVuZHBvaW50IHBhdGggICAgICAgIC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0gICAgICAgICAodjEgd2FzIC92MS9ib29raW5ncy97aWR9KQogIC0gYXV0aCAgICAgICAgICAgICAgICAgQXV0aG9yaXphdGlvbjogQmVhcmVyIDxhcGlfa2V5PiAgICh2MSB1c2VkID9hcGlLZXk9PGtleT4gcXVlcnkgcGFyYW0pCiAgLSBhcGktdmVyc2lvbiBoZWFkZXIgICBjYWwtYXBpLXZlcnNpb246IDIwMjYtMDItMjUgICAgICAgKHBvc3QtY3V0b2ZmOyBvbWl0IGl0IGFuZCB0aGUgZW5kcG9pbnQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGVmYXVsdHMgdG8gdGhlIGxlZ2FjeSByZXNwb25zZSBzaGFwZSkKICAtIHJlc3BvbnNlIGVudmVsb3BlICAgIHsic3RhdHVzIjoic3VjY2VzcyIsImRhdGEiOnsuLi59fSAobGVnYWN5IHNoYXBlIHdhcyBhIGJhcmUgeyJib29raW5nIjp7Li4ufX0pCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vY2FsLmNvbS9kb2NzL2FwaS1yZWZlcmVuY2UvdjIvYm9va2luZ3MvZ2V0LWEtYm9va2luZwogIGh0dHBzOi8vY2FsLmNvbS9kb2NzL2FwaS1yZWZlcmVuY2UvdjIvaW50cm9kdWN0aW9uCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgcmUKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX0FQSV9LRVkgPSAiY2FsX2xpdmVfa2V5X2FiYzEyMyIKUkVRVUlSRURfQVBJX1ZFUlNJT04gPSAiMjAyNi0wMi0yNSIKQk9PS0lOR19VSUQgPSAiYmtfZXZ0XzlmM2EiICAjIHRoZSB1aWQgdGhlIHRlc3QgZmV0Y2hlczsgdGhlIG1vY2sgc3ludGhlc2l6ZXMgYSBib29raW5nIGZvciBhbnkgdWlkCgoKY2xhc3MgX01vY2tDYWxjb20oQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBhdXRoID0gc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpCiAgICAgICAgYXBpX3ZlcnNpb24gPSBzZWxmLmhlYWRlcnMuZ2V0KCJjYWwtYXBpLXZlcnNpb24iKQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImF1dGhvcml6YXRpb24iOiBhdXRoLAogICAgICAgICAgICAicXVlcnlfYXBpS2V5IjogImFwaUtleSIgaW4gcXMsCiAgICAgICAgICAgICJjYWxfYXBpX3ZlcnNpb24iOiBhcGlfdmVyc2lvbiwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogdjIgZmV0Y2hlcyBhIHNpbmdsZSBib29raW5nIGF0IC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi92Mi9ib29raW5ncy8oW0EtWmEtejAtOV9cLV0rKSIsIHBhdGgpCiAgICAgICAgaWYgbm90IG06CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN0YXR1cyI6ICJlcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yIjogeyJjb2RlIjogIk5PVF9GT1VORCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkNhbC5jb20gQVBJIHYyIGZldGNoZXMgYSBib29raW5nIGF0IC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0ifX0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjIGF1dGggYXhpczogdjIgcmVxdWlyZXMgQXV0aG9yaXphdGlvbjogQmVhcmVyIDxrZXk+OyBxdWVyeS1wYXJhbSBhcGlLZXkgaXMgbm90IGhvbm9yZWQuCiAgICAgICAgaWYgYXV0aCAhPSBmIkJlYXJlciB7RVhQRUNURURfQVBJX0tFWX0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJzdGF0dXMiOiAiZXJyb3IiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6IHsiY29kZSI6ICJVTkFVVEhPUklaRUQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJDYWwuY29tIEFQSSB2MiBhdXRoZW50aWNhdGVzIHZpYSBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFwaV9rZXk+In19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgYm9va2luZ191aWQgPSBtLmdyb3VwKDEpCiAgICAgICAgYm9va2luZyA9IHsiaWQiOiA3NzcsICJ1aWQiOiBib29raW5nX3VpZCwgInRpdGxlIjogZiJCb29raW5nIHtib29raW5nX3VpZH0iLCAic3RhdHVzIjogImFjY2VwdGVkIn0KCiAgICAgICAgIyBhcGktdmVyc2lvbiBheGlzOiB3aXRob3V0IHRoZSBDVVJSRU5UIChwb3N0LWN1dG9mZikgY2FsLWFwaS12ZXJzaW9uIHRoZSBlbmRwb2ludCBkZWZhdWx0cwogICAgICAgICMgdG8gdGhlIExFR0FDWSByZXNwb25zZSBzaGFwZSAoYmFyZSB7ImJvb2tpbmciOnsuLi59fSwgbm8gc3RhdHVzL2RhdGEgZW52ZWxvcGUpLgogICAgICAgIGlmIGFwaV92ZXJzaW9uIGlzIE5vbmUgb3IgYXBpX3ZlcnNpb24gPCBSRVFVSVJFRF9BUElfVkVSU0lPTjoKICAgICAgICAgICAgc2VsZi5fanNvbigyMDAsIHsiYm9va2luZyI6IGJvb2tpbmd9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsic3RhdHVzIjogInN1Y2Nlc3MiLCAiZGF0YSI6IGJvb2tpbmd9KQoKCkBweXRlc3QuZml4dHVyZSgpCmRlZiBtb2NrX3NlcnZlcigpOgogICAgX01vY2tDYWxjb20ucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0NhbGNvbSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9ib29raW5nX2J5X3VpZChtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfYm9va2luZyIpLCBcCiAgICAgICAgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IGdldF9ib29raW5nKGJhc2VfdXJsLCBhcGlfa2V5LCBib29raW5nX3VpZCkiCgogICAgYm9va2luZyA9IG1vZHVsZS5nZXRfYm9va2luZyhtb2NrX3NlcnZlciwgRVhQRUNURURfQVBJX0tFWSwgQk9PS0lOR19VSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoYm9va2luZywgZGljdCksIGYiZXhwZWN0ZWQgYSBib29raW5nIGRpY3QsIGdvdCB7dHlwZShib29raW5nKS5fX25hbWVfX30iCiAgICBhc3NlcnQgYm9va2luZy5nZXQoInVpZCIpID09IEJPT0tJTkdfVUlEIGFuZCBib29raW5nLmdldCgidGl0bGUiKSA9PSBmIkJvb2tpbmcge0JPT0tJTkdfVUlEfSIsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCB7Ym9va2luZyFyfTsgYSBjb3JyZWN0IHYyIGNsaWVudCBHRVRzIC92Mi9ib29raW5ncy97Qk9PS0lOR19VSUR9IHdpdGggIgogICAgICAgIGYiQXV0aG9yaXphdGlvbjogQmVhcmVyIGFuZCBjYWwtYXBpLXZlcnNpb246IHtSRVFVSVJFRF9BUElfVkVSU0lPTn0sIHRoZW4gcmVhZHMgZGF0YS4gIgogICAgICAgIGYiU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrQ2FsY29tLnJlcXVlc3RfbG9nfSIKICAgICkK" + }, + { + "vendor": "bunny:list_all_file_paths", + "fn": "list_all_file_paths", + "testFile": "test_bunny_list_all.py", + "signature": "def list_all_file_paths(base_url: str, storage_zone: str, access_key: str, path: str) -> list[str]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYnVubnktc3RvcmFnZS1saXN0LWFsbC1maWxlcyAoaGFyZCkgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpTYW1lIEJ1bm55Lm5ldCBFZGdlIFN0b3JhZ2UgY29udHJhY3QgYXMgdGhlIG1lZGl1bSBsZWFmLCBwbHVzIG9uZSBtb3JlIGF4aXM6IEJ1bm55J3MgbGlzdAplbmRwb2ludCBpcyBOT1QgcmVjdXJzaXZlLCBzbyBsaXN0aW5nIEVWRVJZIGZpbGUgcmVxdWlyZXMgZGVzY2VuZGluZyBpbnRvIGV2ZXJ5IGVudHJ5IHdob3NlCklzRGlyZWN0b3J5IGlzIHRydWUgYW5kIHJlLWxpc3RpbmcgaXQuIFRoZSBncmFkZSByZXF1aXJlczoKICAtIGF1dGggICAgICAgICAgICBBY2Nlc3NLZXk6IDxzdG9yYWdlX3pvbmVfcGFzc3dvcmQ+IEhFQURFUiAobm90IEJlYXJlcikKICAtIGVuZHBvaW50ICAgICAgICBHRVQge2Jhc2V9L3tzdG9yYWdlWm9uZU5hbWV9L3twYXRofS8gICAocGF0aC1zdHlsZSwgb25lIHJlcXVlc3QgcGVyIGRpcmVjdG9yeSkKICAtIHJlc3BvbnNlLXNoYXBlICBwYXJzZSBlYWNoIEpTT04tYXJyYXkgb2JqZWN0J3MgUGFzY2FsQ2FzZSBPYmplY3ROYW1lICsgSXNEaXJlY3RvcnkKICAtIHRyYXZlcnNhbCAgICAgICByZWN1cnNlIGludG8gSXNEaXJlY3Rvcnk9PXRydWUgZW50cmllcyBhbmQgYWNjdW11bGF0ZSB0aGUgZnVsbCByZWxhdGl2ZQogICAgICAgICAgICAgICAgICAgIHBhdGggb2YgZXZlcnkgSXNEaXJlY3Rvcnk9PWZhbHNlIGZpbGUKCkEgY29ycmVjdCBjdXJyZW50LWNvbnRyYWN0IGNsaWVudCB3YWxrcyB0aGUgdHJlZSBhbmQgcmV0dXJucyBhbGwgNSBmaWxlIHBhdGhzLiBBIGNsaWVudCB0aGF0CnVzZXMgQmVhcmVyIGF1dGggKDQwMSksIHBhcnNlcyBzbmFrZV9jYXNlIGZpZWxkIG5hbWVzIChlbXB0eSksIG9yIGRvZXMgbm90IHJlY3Vyc2UgKHJldHVybnMgb25seQp0aGUgc2luZ2xlIHRvcC1sZXZlbCBmaWxlKSBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9kb2NzLmJ1bm55Lm5ldC9yZWZlcmVuY2Uvc3RvcmFnZS1hcGkKICBodHRwczovL2RvY3MuYnVubnkubmV0L2RvY3MvZWRnZS1zdG9yYWdlLW92ZXJ2aWV3CiAgaHR0cHM6Ly9idW5ueWNkbnN0b3JhZ2UuZG9jcy5hcGlhcnkuaW8vCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9LRVkgPSAiYnVubnktc3otcHctOWYzYWMyMWUiCkVYUEVDVEVEX1pPTkUgPSAibWVkaWEtYXNzZXRzIgoKIyBUaGUgaGlkZGVuIGRpcmVjdG9yeSB0cmVlLCBrZXllZCBieSBkaXJlY3RvcnkgcGF0aCByZWxhdGl2ZSB0byB0aGUgem9uZSByb290ICgiIiA9IHJvb3QpLgojIEJ1bm55J3MgbGlzdCBlbmRwb2ludCByZXR1cm5zIG9ubHkgdGhlIGltbWVkaWF0ZSBjaGlsZHJlbiBvZiBvbmUgZGlyZWN0b3J5LCBzbyBhIGNsaWVudCBtdXN0CiMgaXNzdWUgb25lIHJlcXVlc3QgcGVyIGRpcmVjdG9yeSBhbmQgZm9sbG93IElzRGlyZWN0b3J5IHRvIHJlYWNoIGV2ZXJ5IGZpbGUuClRSRUUgPSB7CiAgICAiIjogWygiaW1hZ2VzIiwgVHJ1ZSksICgibm90ZXMudHh0IiwgRmFsc2UpLCAoImRhdGEiLCBUcnVlKV0sCiAgICAiaW1hZ2VzIjogWygibG9nby5wbmciLCBGYWxzZSksICgiaWNvbnMiLCBUcnVlKV0sCiAgICAiaW1hZ2VzL2ljb25zIjogWygiaG9tZS5zdmciLCBGYWxzZSksICgibWVudS5zdmciLCBGYWxzZSldLAogICAgImRhdGEiOiBbKCJyZWNvcmRzLmNzdiIsIEZhbHNlKV0sCn0KCiMgRXZlcnkgSXNEaXJlY3Rvcnk9PWZhbHNlIGxlYWYsIGFzIGEgZnVsbCByZWxhdGl2ZSBwYXRoIOKAlCB0aGUgZXhwZWN0ZWQgcmV0dXJuIG9mIGEgY29ycmVjdCBjbGllbnQuCkVYUEVDVEVEX0ZJTEVTID0gc29ydGVkKFsKICAgICJub3Rlcy50eHQiLAogICAgImltYWdlcy9sb2dvLnBuZyIsCiAgICAiaW1hZ2VzL2ljb25zL2hvbWUuc3ZnIiwKICAgICJpbWFnZXMvaWNvbnMvbWVudS5zdmciLAogICAgImRhdGEvcmVjb3Jkcy5jc3YiLApdKQoKCmRlZiBfbWFrZV9vYmplY3QoZGlyX3BhdGgsIG5hbWUsIGlzX2RpcmVjdG9yeSk6CiAgICBwYXJlbnQgPSAiLyIgKyBFWFBFQ1RFRF9aT05FICsgIi8iICsgKGRpcl9wYXRoICsgIi8iIGlmIGRpcl9wYXRoIGVsc2UgIiIpCiAgICByZXR1cm4gewogICAgICAgICJHdWlkIjogIjAwMDAwMDAwLTAwMDAtMDAwMC0wMDAwLTAwMDAwMDAwMDAwMCIsCiAgICAgICAgIlN0b3JhZ2Vab25lTmFtZSI6IEVYUEVDVEVEX1pPTkUsCiAgICAgICAgIlBhdGgiOiBwYXJlbnQsCiAgICAgICAgIk9iamVjdE5hbWUiOiBuYW1lLAogICAgICAgICJMZW5ndGgiOiAwIGlmIGlzX2RpcmVjdG9yeSBlbHNlIDIwNDgsCiAgICAgICAgIkxhc3RDaGFuZ2VkIjogIjIwMjYtMDctMDFUMDA6MDA6MDAuMDAwIiwKICAgICAgICAiU2VydmVySWQiOiAxLAogICAgICAgICJBcnJheU51bWJlciI6IDAsCiAgICAgICAgIklzRGlyZWN0b3J5IjogaXNfZGlyZWN0b3J5LAogICAgICAgICJVc2VySWQiOiAiMDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAwIiwKICAgICAgICAiQ29udGVudFR5cGUiOiAiIiBpZiBpc19kaXJlY3RvcnkgZWxzZSAiYXBwbGljYXRpb24vb2N0ZXQtc3RyZWFtIiwKICAgICAgICAiRGF0ZUNyZWF0ZWQiOiAiMjAyNi0wNy0wMVQwMDowMDowMC4wMDAiLAogICAgICAgICJTdG9yYWdlWm9uZUlkIjogMTIzNDU2LAogICAgICAgICJDaGVja3N1bSI6IE5vbmUgaWYgaXNfZGlyZWN0b3J5IGVsc2UgIjAiICogNjQsCiAgICAgICAgIlJlcGxpY2F0ZWRab25lcyI6ICIiLAogICAgfQoKCmNsYXNzIF9Nb2NrQnVubnlTdG9yYWdlKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOgogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBoZWFkZXJfa2V5cyA9IHtrLmxvd2VyKCkgZm9yIGsgaW4gc2VsZi5oZWFkZXJzLmtleXMoKX0KICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJoYXNfYWNjZXNza2V5X2hlYWRlciI6ICJhY2Nlc3NrZXkiIGluIGhlYWRlcl9rZXlzLAogICAgICAgICAgICAiaGFzX2F1dGhvcml6YXRpb25faGVhZGVyIjogImF1dGhvcml6YXRpb24iIGluIGhlYWRlcl9rZXlzLAogICAgICAgIH0pCgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIkFjY2Vzc0tleSIpICE9IEVYUEVDVEVEX0tFWToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsiSHR0cENvZGUiOiA0MDEsICJNZXNzYWdlIjogIlVuYXV0aG9yaXplZCJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgcGFydHMgPSBbcCBmb3IgcCBpbiBwYXJzZWQucGF0aC5zcGxpdCgiLyIpIGlmIHAgIT0gIiJdCiAgICAgICAgaWYgbm90IHBhcnRzIG9yIHBhcnRzWzBdICE9IEVYUEVDVEVEX1pPTkU6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7Ikh0dHBDb2RlIjogNDA0LCAiTWVzc2FnZSI6ICJOb3QgRm91bmQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGRpcl9wYXRoID0gIi8iLmpvaW4ocGFydHNbMTpdKQogICAgICAgIGlmIGRpcl9wYXRoIG5vdCBpbiBUUkVFOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJIdHRwQ29kZSI6IDQwNCwgIk1lc3NhZ2UiOiAiTm90IEZvdW5kIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBib2R5ID0gW19tYWtlX29iamVjdChkaXJfcGF0aCwgbmFtZSwgaXNfZGlyKSBmb3IgKG5hbWUsIGlzX2RpcikgaW4gVFJFRVtkaXJfcGF0aF1dCiAgICAgICAgc2VsZi5fanNvbigyMDAsIGJvZHkpCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0J1bm55U3RvcmFnZS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrQnVubnlTdG9yYWdlKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfbGlzdHNfYWxsX2ZpbGVzX3JlY3Vyc2l2ZWx5KG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2ZpbGVfcGF0aHMiKSwgKAogICAgICAgICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBsaXN0X2FsbF9maWxlX3BhdGhzKGJhc2VfdXJsLCBzdG9yYWdlX3pvbmUsIGFjY2Vzc19rZXksIHBhdGgpIgogICAgKQoKICAgIHJlc3VsdCA9IG1vZHVsZS5saXN0X2FsbF9maWxlX3BhdGhzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9aT05FLCBFWFBFQ1RFRF9LRVksICIiKQoKICAgIGFzc2VydCBpc2luc3RhbmNlKHJlc3VsdCwgbGlzdCksIGYiZXhwZWN0ZWQgYSBsaXN0IG9mIGZpbGUgcGF0aHMsIGdvdCB7dHlwZShyZXN1bHQpLl9fbmFtZV9ffSIKICAgIGdvdCA9IHNvcnRlZChzdHIoeCkuc3RyaXAoIi8iKSBmb3IgeCBpbiByZXN1bHQpCiAgICBhc3NlcnQgZ290ID09IEVYUEVDVEVEX0ZJTEVTLCAoCiAgICAgICAgZiJjbGllbnQgcmV0dXJuZWQgZmlsZSBwYXRocyB7Z290fSwgZXhwZWN0ZWQge0VYUEVDVEVEX0ZJTEVTfS4gIgogICAgICAgIGYiQSBjb3JyZWN0IGNsaWVudCBzZW5kcyB0aGUgQWNjZXNzS2V5IGhlYWRlciwgcmVhZHMgZWFjaCBlbnRyeSdzIFBhc2NhbENhc2UgT2JqZWN0TmFtZSArICIKICAgICAgICBmIklzRGlyZWN0b3J5LCBhbmQgcmVjdXJzZXMgaW50byBldmVyeSBkaXJlY3RvcnkgdG8gY29sbGVjdCBhbGwgZmlsZXMuICIKICAgICAgICBmIlNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0J1bm55U3RvcmFnZS5yZXF1ZXN0X2xvZ30iCiAgICApCg==" + }, + { + "vendor": "bunny:list_object_names", + "fn": "list_object_names", + "testFile": "test_bunny_list_directory.py", + "signature": "def list_object_names(base_url: str, storage_zone: str, access_key: str, path: str) -> list[str]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYnVubnktc3RvcmFnZS1saXN0LWRpcmVjdG9yeSAobWVkaXVtKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUgdGltZQooYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdCB0aGUgYWdlbnQncwpzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgQnVubnkubmV0J3MgRWRnZSBTdG9yYWdlCmRpcmVjdG9yeS1saXN0aW5nIGNvbnRyYWN0OgogIC0gYXV0aCAgICAgICAgICAgIEFjY2Vzc0tleTogPHN0b3JhZ2Vfem9uZV9wYXNzd29yZD4gSFRUUCBIRUFERVIKICAgICAgICAgICAgICAgICAgICAoTk9UIEF1dGhvcml6YXRpb24gLyBOT1QgQmVhcmVyOyBhIGZyb20tbWVtb3J5IGNsaWVudCBndWVzc2VzIEJlYXJlciBhbmQgNDAxcykKICAtIGVuZHBvaW50ICAgICAgICBHRVQge2Jhc2V9L3tzdG9yYWdlWm9uZU5hbWV9L3twYXRofS8gICAgICAgIChwYXRoLXN0eWxlLCB0cmFpbGluZyBzbGFzaCBsaXN0cyBhIGRpcikKICAtIHJlc3BvbnNlICAgICAgICBhIEpTT04gQVJSQVkgKG5vIGVudmVsb3BlKSBvZiBvYmplY3RzIHdob3NlIGtleXMgYXJlIFBhc2NhbENhc2U6CiAgICAgICAgICAgICAgICAgICAgT2JqZWN0TmFtZSwgSXNEaXJlY3RvcnksIExlbmd0aCwgUGF0aCwgU3RvcmFnZVpvbmVOYW1lLCBHdWlkLCBDaGVja3N1bSwKICAgICAgICAgICAgICAgICAgICBDb250ZW50VHlwZSwgTGFzdENoYW5nZWQsIERhdGVDcmVhdGVkLCBTdG9yYWdlWm9uZUlkLCBTZXJ2ZXJJZCwgVXNlcklkLAogICAgICAgICAgICAgICAgICAgIFJlcGxpY2F0ZWRab25lcwogICAgICAgICAgICAgICAgICAgIChhIGZyb20tbWVtb3J5IGNsaWVudCBndWVzc2VzIHNuYWtlX2Nhc2UvY2FtZWxDYXNlIGxpa2UgYG5hbWVgL2BvYmplY3RfbmFtZWAKICAgICAgICAgICAgICAgICAgICBvciBhbiBlbnZlbG9wZSBsaWtlIHsiZmlsZXMiOlsuLi5dfSBhbmQgcGFyc2VzIG5vdGhpbmcpCgpBIGNsaWVudCB3cml0dGVuIGZyb20gdGhlIENVUlJFTlQgQnVubnkgZG9jcyAoQWNjZXNzS2V5IGhlYWRlciArIFBhc2NhbENhc2UgT2JqZWN0TmFtZSkgbGlzdHMgYWxsCmZpdmUgZW50cmllcyBhbmQgcGFzc2VzLiBBIGNsaWVudCB3cml0dGVuIGZyb20gc3RhbGUvc3RhbmRhcmQtY29udmVudGlvbiBtZW1vcnkgKEJlYXJlciBhdXRoIG9yCnNuYWtlX2Nhc2UgZmllbGQgbmFtZXMgb3IgYSB3cmFwcGVyIGVudmVsb3BlKSA0MDFzIG9yIHBhcnNlcyBhbiBlbXB0eSBsaXN0IGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9kb2NzLmJ1bm55Lm5ldC9yZWZlcmVuY2Uvc3RvcmFnZS1hcGkKICBodHRwczovL2RvY3MuYnVubnkubmV0L2RvY3MvZWRnZS1zdG9yYWdlLW92ZXJ2aWV3CiAgaHR0cHM6Ly9idW5ueWNkbnN0b3JhZ2UuZG9jcy5hcGlhcnkuaW8vCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlCgppbXBvcnQgcHl0ZXN0CgojIFRoZSB2YWx1ZSB0aGUgY2xpZW50IG11c3Qgc2VuZCBpbiB0aGUgQWNjZXNzS2V5IEhFQURFUiAodGhlIHN0b3JhZ2Ugem9uZSBwYXNzd29yZCkuCkVYUEVDVEVEX0tFWSA9ICJidW5ueS1zei1wdy05ZjNhYzIxZSIKIyBUaGUgc3RvcmFnZSB6b25lIG5hbWUgdGhlIGNsaWVudCBpcyB0b2xkIHRvIGxpc3QuCkVYUEVDVEVEX1pPTkUgPSAibWVkaWEtYXNzZXRzIgoKIyBUaGUgaGlkZGVuIGRpcmVjdG9yeSBjb250ZW50cy4gVGhlIGFnZW50IGNhbm5vdCBoYXJkY29kZSB0aGlzIOKAlCBpdCBpcyBvbmx5IG9ic2VydmFibGUgYnkKIyBhY3R1YWxseSBjYWxsaW5nIHRoZSBtb2NrIHdpdGggdGhlIGNvcnJlY3QgYXV0aCBhbmQgcGFyc2luZyB0aGUgUGFzY2FsQ2FzZSBPYmplY3ROYW1lIGZpZWxkLgpST09UX0VOVFJJRVMgPSBbCiAgICAoImF2YXRhcnMiLCBUcnVlKSwKICAgICgibG9nby5wbmciLCBGYWxzZSksCiAgICAoInJlYWRtZS50eHQiLCBGYWxzZSksCiAgICAoImJhY2t1cHMiLCBUcnVlKSwKICAgICgiY29uZmlnLmpzb24iLCBGYWxzZSksCl0KCgpkZWYgX21ha2Vfb2JqZWN0KGRpcl9wYXRoLCBuYW1lLCBpc19kaXJlY3RvcnkpOgogICAgIyBBIGZhaXRoZnVsIEJ1bm55IHN0b3JhZ2Utb2JqZWN0IHJlY29yZC4gRmllbGQgbmFtZXMgYXJlIFBhc2NhbENhc2UsIG1hdGNoaW5nIHRoZSByZWFsIEFQSS4KICAgIHBhcmVudCA9ICIvIiArIEVYUEVDVEVEX1pPTkUgKyAiLyIgKyAoZGlyX3BhdGggKyAiLyIgaWYgZGlyX3BhdGggZWxzZSAiIikKICAgIHJldHVybiB7CiAgICAgICAgIkd1aWQiOiAiMDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAwIiwKICAgICAgICAiU3RvcmFnZVpvbmVOYW1lIjogRVhQRUNURURfWk9ORSwKICAgICAgICAiUGF0aCI6IHBhcmVudCwKICAgICAgICAiT2JqZWN0TmFtZSI6IG5hbWUsCiAgICAgICAgIkxlbmd0aCI6IDAgaWYgaXNfZGlyZWN0b3J5IGVsc2UgMTAyNCwKICAgICAgICAiTGFzdENoYW5nZWQiOiAiMjAyNi0wNy0wMVQwMDowMDowMC4wMDAiLAogICAgICAgICJTZXJ2ZXJJZCI6IDEsCiAgICAgICAgIkFycmF5TnVtYmVyIjogMCwKICAgICAgICAiSXNEaXJlY3RvcnkiOiBpc19kaXJlY3RvcnksCiAgICAgICAgIlVzZXJJZCI6ICIwMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDAiLAogICAgICAgICJDb250ZW50VHlwZSI6ICIiIGlmIGlzX2RpcmVjdG9yeSBlbHNlICJhcHBsaWNhdGlvbi9vY3RldC1zdHJlYW0iLAogICAgICAgICJEYXRlQ3JlYXRlZCI6ICIyMDI2LTA3LTAxVDAwOjAwOjAwLjAwMCIsCiAgICAgICAgIlN0b3JhZ2Vab25lSWQiOiAxMjM0NTYsCiAgICAgICAgIkNoZWNrc3VtIjogTm9uZSBpZiBpc19kaXJlY3RvcnkgZWxzZSAiMCIgKiA2NCwKICAgICAgICAiUmVwbGljYXRlZFpvbmVzIjogIiIsCiAgICB9CgoKY2xhc3MgX01vY2tCdW5ueVN0b3JhZ2UoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIGhlYWRlcl9rZXlzID0ge2subG93ZXIoKSBmb3IgayBpbiBzZWxmLmhlYWRlcnMua2V5cygpfQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImhhc19hY2Nlc3NrZXlfaGVhZGVyIjogImFjY2Vzc2tleSIgaW4gaGVhZGVyX2tleXMsCiAgICAgICAgICAgICJoYXNfYXV0aG9yaXphdGlvbl9oZWFkZXIiOiAiYXV0aG9yaXphdGlvbiIgaW4gaGVhZGVyX2tleXMsCiAgICAgICAgfSkKCiAgICAgICAgIyBhdXRoIGF4aXM6IEJ1bm55IGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBBY2Nlc3NLZXkgSEVBREVSLiBBIG1pc3Npbmcvd3JvbmcgQWNjZXNzS2V5CiAgICAgICAgIyAoZS5nLiB0aGUgY2xpZW50IHVzZWQgQXV0aG9yaXphdGlvbjogQmVhcmVyIGluc3RlYWQpIGlzIHJlamVjdGVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIkFjY2Vzc0tleSIpICE9IEVYUEVDVEVEX0tFWToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsiSHR0cENvZGUiOiA0MDEsICJNZXNzYWdlIjogIlVuYXV0aG9yaXplZCJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgcGFydHMgPSBbcCBmb3IgcCBpbiBwYXJzZWQucGF0aC5zcGxpdCgiLyIpIGlmIHAgIT0gIiJdCiAgICAgICAgIyBlbmRwb2ludCBheGlzOiBwYXRoLXN0eWxlIHtzdG9yYWdlWm9uZU5hbWV9L3twYXRofS4gRmlyc3Qgc2VnbWVudCBpcyB0aGUgem9uZS4KICAgICAgICBpZiBub3QgcGFydHMgb3IgcGFydHNbMF0gIT0gRVhQRUNURURfWk9ORToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiSHR0cENvZGUiOiA0MDQsICJNZXNzYWdlIjogIk5vdCBGb3VuZCJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgZGlyX3BhdGggPSAiLyIuam9pbihwYXJ0c1sxOl0pCiAgICAgICAgaWYgZGlyX3BhdGggIT0gIiI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7Ikh0dHBDb2RlIjogNDA0LCAiTWVzc2FnZSI6ICJOb3QgRm91bmQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGJvZHkgPSBbX21ha2Vfb2JqZWN0KGRpcl9wYXRoLCBuYW1lLCBpc19kaXIpIGZvciAobmFtZSwgaXNfZGlyKSBpbiBST09UX0VOVFJJRVNdCiAgICAgICAgc2VsZi5fanNvbigyMDAsIGJvZHkpCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0J1bm55U3RvcmFnZS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrQnVubnlTdG9yYWdlKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfbGlzdHNfZGlyZWN0b3J5X29iamVjdF9uYW1lcyhtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJsaXN0X29iamVjdF9uYW1lcyIpLCAoCiAgICAgICAgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IGxpc3Rfb2JqZWN0X25hbWVzKGJhc2VfdXJsLCBzdG9yYWdlX3pvbmUsIGFjY2Vzc19rZXksIHBhdGgpIgogICAgKQoKICAgIHJlc3VsdCA9IG1vZHVsZS5saXN0X29iamVjdF9uYW1lcyhtb2NrX3NlcnZlciwgRVhQRUNURURfWk9ORSwgRVhQRUNURURfS0VZLCAiIikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBvYmplY3QgbmFtZXMsIGdvdCB7dHlwZShyZXN1bHQpLl9fbmFtZV9ffSIKICAgIGdvdCA9IHNvcnRlZChzdHIoeCkgZm9yIHggaW4gcmVzdWx0KQogICAgZXhwZWN0ZWQgPSBzb3J0ZWQobmFtZSBmb3IgKG5hbWUsIF9pc19kaXIpIGluIFJPT1RfRU5UUklFUykKICAgIGFzc2VydCBnb3QgPT0gZXhwZWN0ZWQsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBvYmplY3QgbmFtZXMge2dvdH0sIGV4cGVjdGVkIHtleHBlY3RlZH0uICIKICAgICAgICBmIkEgY29ycmVjdCBjbGllbnQgc2VuZHMgdGhlIEFjY2Vzc0tleSBoZWFkZXIgYW5kIHJlYWRzIGVhY2ggZW50cnkncyBQYXNjYWxDYXNlICIKICAgICAgICBmIk9iamVjdE5hbWUgZmllbGQgZnJvbSB0aGUgSlNPTiBhcnJheS4gU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrQnVubnlTdG9yYWdlLnJlcXVlc3RfbG9nfSIKICAgICkK" + }, + { + "vendor": "pipedrive:list_all_deals", + "fn": "list_all_deals", + "testFile": "test_pipedrive_client.py", + "signature": "def list_all_deals(base_url: str, api_token: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcGlwZWRyaXZlLWRlYWxzLXYyIHdlYi1ncm91bmRlZCBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQKZ3JhZGUgdGltZSAoYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQKYWdhaW5zdCB0aGUgYWdlbnQncyBzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgUGlwZWRyaXZlJ3MgQ1VSUkVOVCAoQVBJIHYyKQpEZWFscy1saXN0IGNvbnRyYWN0OgogIC0gZW5kcG9pbnQgcGF0aCAgICAgICAgICAgIC9hcGkvdjIvZGVhbHMgICAgICAgICAgICAodjEgd2FzIC9hcGkvdjEvZGVhbHMgb3IgL3YxL2RlYWxzKQogIC0gYXV0aCAgICAgICAgICAgICAgICAgICAgIHgtYXBpLXRva2VuOiA8dG9rZW4+ICAgICAodjEvZnJvbS1tZW1vcnkgdXNlZCA/YXBpX3Rva2VuPTx0b2tlbj4pCiAgLSBwYWdpbmF0aW9uICAgICAgICAgICAgICAgY3Vyc29yICsgYWRkaXRpb25hbF9kYXRhLm5leHRfY3Vyc29yIChudWxsID0gZW5kKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICh2MSB1c2VkIHN0YXJ0L2xpbWl0ICsgYWRkaXRpb25hbF9kYXRhLnBhZ2luYXRpb24ubW9yZV9pdGVtc19pbl9jb2xsZWN0aW9uKQoKQSBjbGllbnQgd3JpdHRlbiBmcm9tIENVUlJFTlQgUGlwZWRyaXZlIGRvY3MgY29sbGVjdHMgYWxsIDcgZGVhbHMgYWNyb3NzIDMgY3Vyc29yCnBhZ2VzIGFuZCBwYXNzZXMuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtY3V0b2ZmIG1lbW9yeSAodjEgcGF0aCwgcXVlcnktcGFyYW0KYXV0aCwgb3Igb2Zmc2V0IHBhZ2luYXRpb24pIGhpdHMgNDA0IC8gNDAxIC8gc3RvcHMgYWZ0ZXIgcGFnZSAxIGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9waXBlZHJpdmUucmVhZG1lLmlvL2RvY3MvY29yZS1hcGktY29uY2VwdHMtcGFnaW5hdGlvbgogIGh0dHBzOi8vcGlwZWRyaXZlLnJlYWRtZS5pby9kb2NzL3BpcGVkcml2ZS1hcGktdjItbWlncmF0aW9uLWd1aWRlCiAgaHR0cHM6Ly9waXBlZHJpdmUucmVhZG1lLmlvL2RvY3MvY29yZS1hcGktY29uY2VwdHMtYXV0aGVudGljYXRpb24KIiIiCgppbXBvcnQgYmFzZTY0CmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCiMgVGhlIHRva2VuIHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSB4LWFwaS10b2tlbiBIRUFERVIgKHYyIGNvbnRyYWN0KS4KRVhQRUNURURfVE9LRU4gPSAicGRfbGl2ZV90b2tlbl9hYmMxMjMiCgojIFRoZSBoaWRkZW4gZGF0YXNldC4gVGhlIGFnZW50IGNhbm5vdCBoYXJkY29kZSB0aGlzIOKAlCBpdCBpcyBvbmx5IG9ic2VydmFibGUgYnkKIyBhY3R1YWxseSBjYWxsaW5nIHRoZSBtb2NrIGFuZCBwYWdpbmF0aW5nIHRvIHRoZSBlbmQuIDcgZGVhbHMsIHNlcnZlciBwYWdlIHNpemUgMy4KREVBTFMgPSBbCiAgICB7ImlkIjogaSwgInRpdGxlIjogZiJEZWFsIHtpfSIsICJ2YWx1ZSI6IGkgKiAxMDAwLCAiY3VycmVuY3kiOiAiVVNEIiwgInN0YXR1cyI6ICJvcGVuIn0KICAgIGZvciBpIGluIHJhbmdlKDEsIDgpCl0KUEFHRV9TSVpFID0gMwoKCmRlZiBfZW5jb2RlX2N1cnNvcihvZmZzZXQ6IGludCkgLT4gc3RyOgogICAgIyBPcGFxdWUgY3Vyc29yIHN0cmluZywgbWF0Y2hpbmcgUGlwZWRyaXZlJ3MgIm9wYXF1ZSBzdHJpbmcgdmFsdWUiIGNvbnRyYWN0LgogICAgcmV0dXJuIGJhc2U2NC51cmxzYWZlX2I2NGVuY29kZShzdHIob2Zmc2V0KS5lbmNvZGUoKSkuZGVjb2RlKCkKCgpkZWYgX2RlY29kZV9jdXJzb3IoY3Vyc29yOiBzdHIpIC0+IGludDoKICAgIHJldHVybiBpbnQoYmFzZTY0LnVybHNhZmVfYjY0ZGVjb2RlKGN1cnNvci5lbmNvZGUoKSkuZGVjb2RlKCkpCgoKY2xhc3MgX01vY2tQaXBlZHJpdmUoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICAjIEV2ZXJ5IGhhbmRsZXIgaW5zdGFuY2Ugc2hhcmVzIHRoZSBjbGFzcy1sZXZlbCByZXF1ZXN0IGxvZyBzbyB0aGUgdGVzdCBjYW4KICAgICMgaW5zcGVjdCB3aGF0IHBhdGgvYXV0aCB0aGUgYWdlbnQncyBjbGllbnQgYWN0dWFsbHkgdXNlZCAoZGlhZ25vc3RpYyBvbiBmYWlsKS4KICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKTogICMgc2lsZW5jZSBzdGRlcnIgYWNjZXNzIGxvZwogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX2hlYWRlcl9hdXRoIjogIngtYXBpLXRva2VuIiBpbiB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9LAogICAgICAgICAgICAicXVlcnlfYXBpX3Rva2VuIjogImFwaV90b2tlbiIgaW4gcXMsCiAgICAgICAgICAgICJjdXJzb3IiOiBxcy5nZXQoImN1cnNvciIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJzdGFydCI6IHFzLmdldCgic3RhcnQiLCBbTm9uZV0pWzBdLAogICAgICAgIH0pCgogICAgICAgICMgKDEpIGVuZHBvaW50LXZlcnNpb24gYXhpczogb25seSB0aGUgdjIgcGF0aCBleGlzdHMuCiAgICAgICAgaWYgcGF0aCAhPSAiL2FwaS92Mi9kZWFscyI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAidW5rbm93biBlbmRwb2ludCAoUGlwZWRyaXZlIEFQSSB2MiBsaXN0cyBkZWFscyBhdCAvYXBpL3YyL2RlYWxzKSJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMikgYXV0aCBheGlzOiB2MiByZXF1aXJlcyB0aGUgeC1hcGktdG9rZW4gSEVBREVSOyBxdWVyeS1wYXJhbSBhdXRoIGlzIG5vdCBob25vcmVkLgogICAgICAgIHRva2VuID0gc2VsZi5oZWFkZXJzLmdldCgieC1hcGktdG9rZW4iKQogICAgICAgIGlmIHRva2VuICE9IEVYUEVDVEVEX1RPS0VOOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJzdWNjZXNzIjogRmFsc2UsICJlcnJvciI6ICJ1bmF1dGhvcml6ZWQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvcl9pbmZvIjogIlBpcGVkcml2ZSBBUEkgdjIgYXV0aGVudGljYXRlcyB2aWEgdGhlIHgtYXBpLXRva2VuIGhlYWRlciJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMykgcGFnaW5hdGlvbiBheGlzOiBjdXJzb3ItYmFzZWQuIGBzdGFydGAvb2Zmc2V0IGlzIGlnbm9yZWQgKHYyIGhhcyBubyBvZmZzZXQpLgogICAgICAgIGN1cnNvciA9IHFzLmdldCgiY3Vyc29yIiwgW05vbmVdKVswXQogICAgICAgIGlmIGN1cnNvciBpcyBOb25lOgogICAgICAgICAgICBvZmZzZXQgPSAwCiAgICAgICAgZWxzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgb2Zmc2V0ID0gX2RlY29kZV9jdXJzb3IoY3Vyc29yKQogICAgICAgICAgICBleGNlcHQgRXhjZXB0aW9uOgogICAgICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsic3VjY2VzcyI6IEZhbHNlLCAiZXJyb3IiOiAiaW52YWxpZCBjdXJzb3IifSkKICAgICAgICAgICAgICAgIHJldHVybgoKICAgICAgICBwYWdlID0gREVBTFNbb2Zmc2V0Om9mZnNldCArIFBBR0VfU0laRV0KICAgICAgICBuZXh0X29mZnNldCA9IG9mZnNldCArIFBBR0VfU0laRQogICAgICAgIG5leHRfY3Vyc29yID0gX2VuY29kZV9jdXJzb3IobmV4dF9vZmZzZXQpIGlmIG5leHRfb2Zmc2V0IDwgbGVuKERFQUxTKSBlbHNlIE5vbmUKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAic3VjY2VzcyI6IFRydWUsCiAgICAgICAgICAgICJkYXRhIjogcGFnZSwKICAgICAgICAgICAgImFkZGl0aW9uYWxfZGF0YSI6IHsibmV4dF9jdXJzb3IiOiBuZXh0X2N1cnNvcn0sCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrUGlwZWRyaXZlLnJlcXVlc3RfbG9nID0gW10KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tQaXBlZHJpdmUpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWQgPSB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpCiAgICB0aHJlYWQuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgIyBGYWxsIGJhY2sgdG8gYW55IHNvbHV0aW9uLnB5IHVuZGVyIHRoZSByb290IChhZ2VudCBtYXkgbmVzdCBpdCkuCiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9kZWFsc19hY3Jvc3NfY3Vyc29yX3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2RlYWxzIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBsaXN0X2FsbF9kZWFscyhiYXNlX3VybCwgYXBpX3Rva2VuKSIKCiAgICByZXN1bHQgPSBtb2R1bGUubGlzdF9hbGxfZGVhbHMobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX1RPS0VOKQoKICAgIGFzc2VydCBpc2luc3RhbmNlKHJlc3VsdCwgbGlzdCksIGYiZXhwZWN0ZWQgYSBsaXN0IG9mIGRlYWxzLCBnb3Qge3R5cGUocmVzdWx0KS5fX25hbWVfX30iCiAgICBpZHMgPSBzb3J0ZWQoZFsiaWQiXSBmb3IgZCBpbiByZXN1bHQgaWYgaXNpbnN0YW5jZShkLCBkaWN0KSBhbmQgImlkIiBpbiBkKQogICAgZXhwZWN0ZWRfaWRzID0gW2RbImlkIl0gZm9yIGQgaW4gREVBTFNdCiAgICBhc3NlcnQgaWRzID09IGV4cGVjdGVkX2lkcywgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIGRlYWwgaWRzIHtpZHN9LCBleHBlY3RlZCB7ZXhwZWN0ZWRfaWRzfS4gIgogICAgICAgIGYiQSBjb3JyZWN0IHYyIGNsaWVudCBmb2xsb3dzIGFkZGl0aW9uYWxfZGF0YS5uZXh0X2N1cnNvciB0byB0aGUgZW5kLiAiCiAgICAgICAgZiJTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2d9IgogICAgKQo=" + }, + { + "vendor": "pipedrive:get_deal", + "fn": "get_deal", + "testFile": "test_pipedrive_get_deal.py", + "signature": "def get_deal(base_url: str, api_token: str, deal_id: int) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcGlwZWRyaXZlLWdldC1kZWFsLXYyIChtZWRpdW0pIGxlYWYuCgpTaW1wbGVyIHNsaWNlIG9mIHRoZSBTQU1FIFBpcGVkcml2ZSBBUEkgdjIgY29udHJhY3Qg4oCUIGEgc2luZ2xlLXJlc291cmNlIEdFVCwgbm8KcGFnaW5hdGlvbi4gU3RpbGwgc2VhcmNoLW5lY2Vzc2FyeSBvbiB0d28gYXhlczogdGhlIHYyIGVuZHBvaW50IHBhdGggYW5kIHRoZQp4LWFwaS10b2tlbiBoZWFkZXIgYXV0aCAoYSBmcm9tLW1lbW9yeSB2MSBjbGllbnQgdXNlcyAvdjEvZGVhbHMve2lkfSArID9hcGlfdG9rZW49CmFuZCBmYWlscykuCgogIC0gZW5kcG9pbnQgcGF0aCAgIC9hcGkvdjIvZGVhbHMve2lkfSAgICAgICAgKHYxIHdhcyAvYXBpL3YxL2RlYWxzL3tpZH0gb3IgL3YxL2RlYWxzL3tpZH0pCiAgLSBhdXRoICAgICAgICAgICAgeC1hcGktdG9rZW46IDx0b2tlbj4gICAgICAgKHYxL2Zyb20tbWVtb3J5IHVzZWQgP2FwaV90b2tlbj08dG9rZW4+KQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL3BpcGVkcml2ZS5yZWFkbWUuaW8vZG9jcy9waXBlZHJpdmUtYXBpLXYyLW1pZ3JhdGlvbi1ndWlkZQogIGh0dHBzOi8vcGlwZWRyaXZlLnJlYWRtZS5pby9kb2NzL2NvcmUtYXBpLWNvbmNlcHRzLWF1dGhlbnRpY2F0aW9uCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgcmUKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX1RPS0VOID0gInBkX2xpdmVfdG9rZW5fYWJjMTIzIgpERUFMX0lEID0gNDIgICMgdGhlIGlkIHRoZSB0ZXN0IGZldGNoZXM7IHRoZSBtb2NrIHN5bnRoZXNpemVzIGRlYWxzIGZvciBhbnkgdmFsaWQgaWQKCgpjbGFzcyBfTW9ja1BpcGVkcml2ZShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImhhc19oZWFkZXJfYXV0aCI6ICJ4LWFwaS10b2tlbiIgaW4ge2subG93ZXIoKSBmb3IgayBpbiBzZWxmLmhlYWRlcnMua2V5cygpfSwKICAgICAgICAgICAgInF1ZXJ5X2FwaV90b2tlbiI6ICJhcGlfdG9rZW4iIGluIHFzLAogICAgICAgIH0pCgogICAgICAgICMgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB2MiBmZXRjaGVzIG9uZSBkZWFsIGF0IC9hcGkvdjIvZGVhbHMve2lkfS4KICAgICAgICBtID0gcmUuZnVsbG1hdGNoKHIiL2FwaS92Mi9kZWFscy8oXGQrKSIsIHBhdGgpCiAgICAgICAgaWYgbm90IG06CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBmZXRjaGVzIGEgZGVhbCBhdCAvYXBpL3YyL2RlYWxzL3tpZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiB2MiByZXF1aXJlcyB0aGUgeC1hcGktdG9rZW4gaGVhZGVyOyBxdWVyeS1wYXJhbSBhdXRoIGlzIG5vdCBob25vcmVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIngtYXBpLXRva2VuIikgIT0gRVhQRUNURURfVE9LRU46CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBhdXRoZW50aWNhdGVzIHZpYSB0aGUgeC1hcGktdG9rZW4gaGVhZGVyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBkZWFsX2lkID0gaW50KG0uZ3JvdXAoMSkpCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgInN1Y2Nlc3MiOiBUcnVlLAogICAgICAgICAgICAiZGF0YSI6IHsiaWQiOiBkZWFsX2lkLCAidGl0bGUiOiBmIkRlYWwge2RlYWxfaWR9IiwgInZhbHVlIjogZGVhbF9pZCAqIDEwMDAsCiAgICAgICAgICAgICAgICAgICAgICJjdXJyZW5jeSI6ICJVU0QiLCAic3RhdHVzIjogIm9wZW4ifSwKICAgICAgICB9KQoKCkBweXRlc3QuZml4dHVyZSgpCmRlZiBtb2NrX3NlcnZlcigpOgogICAgX01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1BpcGVkcml2ZSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9kZWFsX2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9kZWFsIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfZGVhbChiYXNlX3VybCwgYXBpX3Rva2VuLCBkZWFsX2lkKSIKCiAgICBkZWFsID0gbW9kdWxlLmdldF9kZWFsKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTiwgREVBTF9JRCkKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShkZWFsLCBkaWN0KSwgZiJleHBlY3RlZCBhIGRlYWwgZGljdCwgZ290IHt0eXBlKGRlYWwpLl9fbmFtZV9ffSIKICAgIGFzc2VydCBkZWFsLmdldCgiaWQiKSA9PSBERUFMX0lEIGFuZCBkZWFsLmdldCgidGl0bGUiKSA9PSBmIkRlYWwge0RFQUxfSUR9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkZWFsIXJ9OyBhIGNvcnJlY3QgdjIgY2xpZW50IEdFVHMgL2FwaS92Mi9kZWFscy97REVBTF9JRH0gd2l0aCB0aGUgIgogICAgICAgIGYieC1hcGktdG9rZW4gaGVhZGVyLiBTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2d9IgogICAgKQo=" + } +] \ No newline at end of file diff --git a/examples/ablation-suite/fixtures/multi-contract/contracts.json b/examples/ablation-suite/fixtures/multi-contract/contracts.json new file mode 100644 index 00000000..ab9c7d39 --- /dev/null +++ b/examples/ablation-suite/fixtures/multi-contract/contracts.json @@ -0,0 +1,72 @@ +[ + { + "vendor": "finch:get_directory", + "fn": "get_directory", + "testFile": "test_finch_directory.py", + "signature": "def get_directory(base_url: str, access_token: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgZmluY2gtZGlyZWN0b3J5LXBhZ2UgKG1lZGl1bSkgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpJdCBzdGFuZHMgdXAgYSBMT0NBTCBtb2NrIHRoYXQgZmFpdGhmdWxseSBpbXBsZW1lbnRzIEZpbmNoJ3MgQ1VSUkVOVAooVW5pdmVyc2FsIEVtcGxveW1lbnQgQVBJKSBkaXJlY3RvcnkgY29udHJhY3QgZm9yIGEgc2luZ2xlIHJlc3BvbnNlOgogIC0gZW5kcG9pbnQgcGF0aCAgICAgICAgL2VtcGxveWVyL2RpcmVjdG9yeSAgICAgICAgKGJhc2UgaG9zdCBpcyBpbmplY3RlZDsgdGhlIHBhdGggaXMgZml4ZWQpCiAgLSBhdXRoICAgICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFjY2Vzc190b2tlbj4KICAtIHZlcnNpb24gaGVhZGVyICAgICAgIEZpbmNoLUFQSS1WZXJzaW9uOiA8WVlZWS1NTS1ERD4gICAoUkVRVUlSRUQ7IG1pc3NpbmcgLT4gNDAwKQogIC0gcmVzcG9uc2UgZW52ZWxvcGUgICAgeyJwYWdpbmciOiB7ImNvdW50IiwgIm9mZnNldCJ9LCAiaW5kaXZpZHVhbHMiOiBbLi4uXX0KCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIEZpbmNoIGRvY3Mgc2VuZHMgdGhlIEJlYXJlciB0b2tlbiBBTkQgdGhlIHJlcXVpcmVkCkZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciB0byAvZW1wbG95ZXIvZGlyZWN0b3J5IGFuZCByZWFkcyB0aGUgYGluZGl2aWR1YWxzYCBhcnJheS4KQSBjbGllbnQgd3JpdHRlbiBmcm9tIHN0YWxlL3ByZS1jdXRvZmYgbWVtb3J5IG9taXRzIHRoZSByZXF1aXJlZCB2ZXJzaW9uIGhlYWRlciAoNDAwKSwKZ3Vlc3NlcyBhIFJFU1QtaXNoIC9lbXBsb3llZXMgcGF0aCAoNDA0KSwgb3IgcmVhZHMgYSBgZGF0YWAvYHJlc3VsdHNgIGVudmVsb3BlLCBhbmQgRkFJTFMuCkFuIGVtcHR5IHNvbHV0aW9uIEZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RldmVsb3Blci50cnlmaW5jaC5jb20vYXBpLXJlZmVyZW5jZS9kZXZlbG9wbWVudC1ndWlkZXMvSGVhZGVycwogIGh0dHBzOi8vZGV2ZWxvcGVyLnRyeWZpbmNoLmNvbS9hcGktcmVmZXJlbmNlL29yZ2FuaXphdGlvbi9kaXJlY3RvcnkKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgdG9rZW4gdGhlIGNsaWVudCBtdXN0IHNlbmQgaW4gdGhlIEF1dGhvcml6YXRpb246IEJlYXJlciA8dG9rZW4+IGhlYWRlci4KRVhQRUNURURfVE9LRU4gPSAiZmluY2hfYWNjZXNzX3Rva2VuX2FiYzEyMyIKCiMgU21hbGwgZGlyZWN0b3J5IHJldHVybmVkIGluIGEgc2luZ2xlIHJlc3BvbnNlIChkZWZhdWx0cy10by1hbGw7IG5vIHBhZ2luYXRpb24gaW4gdGhpcyBsZWFmKS4KSU5ESVZJRFVBTFMgPSBbCiAgICB7CiAgICAgICAgImlkIjogZiIxMTExMTExMS0wMDAwLTQwMDAtODAwMC0wMDAwMDAwMDAwMHtpfSIsCiAgICAgICAgImZpcnN0X25hbWUiOiBmIkZpcnN0e2l9IiwKICAgICAgICAibWlkZGxlX25hbWUiOiBOb25lLAogICAgICAgICJsYXN0X25hbWUiOiBmIkxhc3R7aX0iLAogICAgICAgICJkZXBhcnRtZW50IjogeyJuYW1lIjogIkVuZ2luZWVyaW5nIn0sCiAgICAgICAgIm1hbmFnZXIiOiBOb25lLAogICAgICAgICJpc19hY3RpdmUiOiBUcnVlLAogICAgfQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgNCkKXQoKCmNsYXNzIF9Nb2NrRmluY2goQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICAjIENsYXNzLWxldmVsIHJlcXVlc3QgbG9nIHNvIHRoZSB0ZXN0IGNhbiByZXBvcnQgd2hhdCB0aGUgY2xpZW50IGFjdHVhbGx5IHNlbnQgb24gZmFpbHVyZS4KICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKTogICMgc2lsZW5jZSBzdGRlcnIgYWNjZXNzIGxvZwogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgaGVhZGVyX2tleXMgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdmVyc2lvbiA9IHNlbGYuaGVhZGVycy5nZXQoIkZpbmNoLUFQSS1WZXJzaW9uIikKICAgICAgICBhdXRoID0gc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpCiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX3ZlcnNpb25faGVhZGVyIjogImZpbmNoLWFwaS12ZXJzaW9uIiBpbiBoZWFkZXJfa2V5cywKICAgICAgICAgICAgInZlcnNpb24iOiB2ZXJzaW9uLAogICAgICAgICAgICAiYXV0aG9yaXphdGlvbiI6IGF1dGgsCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB0aGUgZGlyZWN0b3J5IGxpdmVzIGF0IC9lbXBsb3llci9kaXJlY3RvcnkgKGhvc3QgaXMgaW5qZWN0ZWQpLgogICAgICAgIGlmIHBhdGggIT0gIi9lbXBsb3llci9kaXJlY3RvcnkiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJjb2RlIjogNDA0LCAibmFtZSI6ICJub3RfZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkZpbmNoIGxpc3RzIHRoZSBvcmcgZGlyZWN0b3J5IGF0IEdFVCAvZW1wbG95ZXIvZGlyZWN0b3J5In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IEF1dGhvcml6YXRpb24gbXVzdCBiZSB0aGUgQmVhcmVyIGFjY2VzcyB0b2tlbi4KICAgICAgICBpZiBhdXRoICE9IGYiQmVhcmVyIHtFWFBFQ1RFRF9UT0tFTn0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJjb2RlIjogNDAxLCAibmFtZSI6ICJpbnZhbGlkX2FjY2Vzc190b2tlbiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiQXV0aG9yaXphdGlvbiBtdXN0IGJlICdCZWFyZXIgPGFjY2Vzc190b2tlbj4nIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgzKSB2ZXJzaW9uIGF4aXM6IEZpbmNoLUFQSS1WZXJzaW9uIGlzIGEgUkVRVUlSRUQgaGVhZGVyIChkYXRlLWZvcm1hdHRlZCkuCiAgICAgICAgaWYgdmVyc2lvbiBpcyBOb25lIG9yIG5vdCByZS5mdWxsbWF0Y2gociJcZHs0fS1cZHsyfS1cZHsyfSIsIHZlcnNpb24pOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJjb2RlIjogNDAwLCAibmFtZSI6ICJtaXNzaW5nX3ZlcnNpb25faGVhZGVyIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJGaW5jaCByZXF1aXJlcyB0aGUgRmluY2gtQVBJLVZlcnNpb24gaGVhZGVyIChlLmcuIDIwMjAtMDktMTcpIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAicGFnaW5nIjogeyJjb3VudCI6IGxlbihJTkRJVklEVUFMUyksICJvZmZzZXQiOiAwfSwKICAgICAgICAgICAgImluZGl2aWR1YWxzIjogSU5ESVZJRFVBTFMsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrRmluY2gucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0ZpbmNoKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RpcmVjdG9yeShtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfZGlyZWN0b3J5IiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfZGlyZWN0b3J5KGJhc2VfdXJsLCBhY2Nlc3NfdG9rZW4pIgoKICAgIHJlc3VsdCA9IG1vZHVsZS5nZXRfZGlyZWN0b3J5KG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBpbmRpdmlkdWFscywgZ290IHt0eXBlKHJlc3VsdCkuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGRbImlkIl0gZm9yIGQgaW4gcmVzdWx0IGlmIGlzaW5zdGFuY2UoZCwgZGljdCkgYW5kICJpZCIgaW4gZCkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChkWyJpZCJdIGZvciBkIGluIElORElWSURVQUxTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBpbmRpdmlkdWFsIGlkcyB7aWRzfSwgZXhwZWN0ZWQge2V4cGVjdGVkX2lkc30uICIKICAgICAgICBmIkEgY29ycmVjdCBjbGllbnQgR0VUcyAvZW1wbG95ZXIvZGlyZWN0b3J5IHdpdGggdGhlIEJlYXJlciB0b2tlbiBBTkQgdGhlIHJlcXVpcmVkICIKICAgICAgICBmIkZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciwgdGhlbiByZWFkcyB0aGUgYGluZGl2aWR1YWxzYCBhcnJheS4gIgogICAgICAgIGYiU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrRmluY2gucmVxdWVzdF9sb2d9IgogICAgKQo=" + }, + { + "vendor": "honeycomb:get_dataset", + "fn": "get_dataset", + "testFile": "test_honeycomb_get_dataset.py", + "signature": "def get_dataset(base_url: str, api_key: str, dataset_slug: str) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgaG9uZXljb21iLWdldC1kYXRhc2V0IChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBIb25leWNvbWIncyBDVVJSRU5UIChBUEkgdjEpIHNpbmdsZS0KZGF0YXNldCBjb250cmFjdCBvbiB0d28gc2VhcmNoLW5lY2Vzc2FyeSBheGVzOgogIC0gYXV0aCAgICAgIFgtSG9uZXljb21iLVRlYW06IDxrZXk+ICAgKGN1c3RvbSBoZWFkZXIsIE5PIEF1dGhvcml6YXRpb246IEJlYXJlciBzY2hlbWUpCiAgLSBlbmRwb2ludCAgLzEvZGF0YXNldHMve3NsdWd9ICAgICAgICAodGhlIEFQSSB2ZXJzaW9uIHByZWZpeCBpcyB0aGUgbGl0ZXJhbCAiMSIpCgpBIGZyb20tbWVtb3J5IGNsaWVudCB0aGF0IHJlYWNoZXMgZm9yIEF1dGhvcml6YXRpb246IEJlYXJlciAodGhlIG5lYXItdW5pdmVyc2FsIGRlZmF1bHQpIG9yCmd1ZXNzZXMgL3YxfC9hcGkvdjF8L2FwaS8xIGhpdHMgNDAxIC8gNDA0IGFuZCBGQUlMUy4gQW4gZW1wdHkgc29sdXRpb24gRkFJTFMuIE9ubHkgYSBjbGllbnQKYnVpbHQgZnJvbSB0aGUgY3VycmVudCBIb25leWNvbWIgZG9jcyAoY3VzdG9tIGhlYWRlciArIC8xLyBwcmVmaXgpIFBBU1NFUy4gVGhlIGRhdGFzZXQgbmFtZSBpcwpzbHVnLWRlcml2ZWQgYW5kIG9ubHkgb2JzZXJ2YWJsZSBieSBhY3R1YWxseSBjYWxsaW5nIHRoZSBtb2NrLCBzbyB0aGUgYW5zd2VyIGNhbm5vdCBiZSBoYXJkY29kZWQuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZG9jcy5ob25leWNvbWIuaW8vYXBpL2F1dGgKICBodHRwczovL2FwaS1kb2NzLmhvbmV5Y29tYi5pby9hcGkvZGF0YXNldHMKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUga2V5IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBYLUhvbmV5Y29tYi1UZWFtIEhFQURFUiAodjEgY29udHJhY3QpLgpFWFBFQ1RFRF9LRVkgPSAiaGNhaWNfaG9uZXljb21iX2NvbmZpZ19rZXlfYWJjMTIzIgpEQVRBU0VUX1NMVUcgPSAiY2hlY2tvdXQtc2VydmljZSIKCgpjbGFzcyBfTW9ja0hvbmV5Y29tYihCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgICMgQ2xhc3MtbGV2ZWwgcmVxdWVzdCBsb2cgc28gdGhlIHRlc3QgY2FuIHJlcG9ydCB3aGF0IHBhdGgvYXV0aCB0aGUgY2xpZW50IGFjdHVhbGx5IHVzZWQuCiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6ICAjIHNpbGVuY2Ugc3RkZXJyIGFjY2VzcyBsb2cKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIGhlYWRlcnNfbG93ZXIgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX3RlYW1faGVhZGVyIjogIngtaG9uZXljb21iLXRlYW0iIGluIGhlYWRlcnNfbG93ZXIsCiAgICAgICAgICAgICJoYXNfYmVhcmVyIjogImF1dGhvcml6YXRpb24iIGluIGhlYWRlcnNfbG93ZXIsCiAgICAgICAgICAgICJxdWVyeV9hcGlfa2V5IjogImFwaV9rZXkiIGluIHBhcnNlX3FzKHBhcnNlZC5xdWVyeSksCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB2MSBmZXRjaGVzIGEgZGF0YXNldCBhdCAvMS9kYXRhc2V0cy97c2x1Z30uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi8xL2RhdGFzZXRzLyhbXi9dKykiLCBwYXRoKQogICAgICAgIGlmIG5vdCBtOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJlcnJvciI6ICJub3QgZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvcl9pbmZvIjogIkhvbmV5Y29tYiBBUEkgdjEgZmV0Y2hlcyBhIGRhdGFzZXQgYXQgLzEvZGF0YXNldHMve3NsdWd9In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IHYxIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUhvbmV5Y29tYi1UZWFtIGhlYWRlcjsgQmVhcmVyIGlzIE5PVCBob25vcmVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIlgtSG9uZXljb21iLVRlYW0iKSAhPSBFWFBFQ1RFRF9LRVk6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiSG9uZXljb21iIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUhvbmV5Y29tYi1UZWFtIGhlYWRlciJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc2x1ZyA9IG0uZ3JvdXAoMSkKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAibmFtZSI6IGYiU2VydmljZSB7c2x1Z30iLAogICAgICAgICAgICAic2x1ZyI6IHNsdWcsCiAgICAgICAgICAgICJkZXNjcmlwdGlvbiI6IGYidGVsZW1ldHJ5IGZvciB7c2x1Z30iLAogICAgICAgICAgICAiZXhwYW5kX2pzb25fZGVwdGgiOiAyLAogICAgICAgICAgICAibGFzdF93cml0dGVuX2F0IjogIjIwMjYtMDctMDFUMDA6MDA6MDBaIiwKICAgICAgICAgICAgImNyZWF0ZWRfYXQiOiAiMjAyNS0wMS0wMVQwMDowMDowMFoiLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0hvbmV5Y29tYi5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrSG9uZXljb21iKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RhdGFzZXRfYnlfc2x1Zyhtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfZGF0YXNldCIpLCAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgZ2V0X2RhdGFzZXQoYmFzZV91cmwsIGFwaV9rZXksIGRhdGFzZXRfc2x1ZykiCgogICAgZGF0YXNldCA9IG1vZHVsZS5nZXRfZGF0YXNldChtb2NrX3NlcnZlciwgRVhQRUNURURfS0VZLCBEQVRBU0VUX1NMVUcpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoZGF0YXNldCwgZGljdCksIGYiZXhwZWN0ZWQgYSBkYXRhc2V0IGRpY3QsIGdvdCB7dHlwZShkYXRhc2V0KS5fX25hbWVfX30iCiAgICBhc3NlcnQgZGF0YXNldC5nZXQoInNsdWciKSA9PSBEQVRBU0VUX1NMVUcgYW5kIGRhdGFzZXQuZ2V0KCJuYW1lIikgPT0gZiJTZXJ2aWNlIHtEQVRBU0VUX1NMVUd9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkYXRhc2V0IXJ9OyBhIGNvcnJlY3QgdjEgY2xpZW50IEdFVHMgLzEvZGF0YXNldHMve0RBVEFTRVRfU0xVR30gd2l0aCB0aGUgIgogICAgICAgIGYiWC1Ib25leWNvbWItVGVhbSBoZWFkZXIuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0hvbmV5Y29tYi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" + }, + { + "vendor": "bunny:list_object_names", + "fn": "list_object_names", + "testFile": "test_bunny_list_directory.py", + "signature": "def list_object_names(base_url: str, storage_zone: str, access_key: str, path: str) -> list[str]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYnVubnktc3RvcmFnZS1saXN0LWRpcmVjdG9yeSAobWVkaXVtKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUgdGltZQooYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdCB0aGUgYWdlbnQncwpzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgQnVubnkubmV0J3MgRWRnZSBTdG9yYWdlCmRpcmVjdG9yeS1saXN0aW5nIGNvbnRyYWN0OgogIC0gYXV0aCAgICAgICAgICAgIEFjY2Vzc0tleTogPHN0b3JhZ2Vfem9uZV9wYXNzd29yZD4gSFRUUCBIRUFERVIKICAgICAgICAgICAgICAgICAgICAoTk9UIEF1dGhvcml6YXRpb24gLyBOT1QgQmVhcmVyOyBhIGZyb20tbWVtb3J5IGNsaWVudCBndWVzc2VzIEJlYXJlciBhbmQgNDAxcykKICAtIGVuZHBvaW50ICAgICAgICBHRVQge2Jhc2V9L3tzdG9yYWdlWm9uZU5hbWV9L3twYXRofS8gICAgICAgIChwYXRoLXN0eWxlLCB0cmFpbGluZyBzbGFzaCBsaXN0cyBhIGRpcikKICAtIHJlc3BvbnNlICAgICAgICBhIEpTT04gQVJSQVkgKG5vIGVudmVsb3BlKSBvZiBvYmplY3RzIHdob3NlIGtleXMgYXJlIFBhc2NhbENhc2U6CiAgICAgICAgICAgICAgICAgICAgT2JqZWN0TmFtZSwgSXNEaXJlY3RvcnksIExlbmd0aCwgUGF0aCwgU3RvcmFnZVpvbmVOYW1lLCBHdWlkLCBDaGVja3N1bSwKICAgICAgICAgICAgICAgICAgICBDb250ZW50VHlwZSwgTGFzdENoYW5nZWQsIERhdGVDcmVhdGVkLCBTdG9yYWdlWm9uZUlkLCBTZXJ2ZXJJZCwgVXNlcklkLAogICAgICAgICAgICAgICAgICAgIFJlcGxpY2F0ZWRab25lcwogICAgICAgICAgICAgICAgICAgIChhIGZyb20tbWVtb3J5IGNsaWVudCBndWVzc2VzIHNuYWtlX2Nhc2UvY2FtZWxDYXNlIGxpa2UgYG5hbWVgL2BvYmplY3RfbmFtZWAKICAgICAgICAgICAgICAgICAgICBvciBhbiBlbnZlbG9wZSBsaWtlIHsiZmlsZXMiOlsuLi5dfSBhbmQgcGFyc2VzIG5vdGhpbmcpCgpBIGNsaWVudCB3cml0dGVuIGZyb20gdGhlIENVUlJFTlQgQnVubnkgZG9jcyAoQWNjZXNzS2V5IGhlYWRlciArIFBhc2NhbENhc2UgT2JqZWN0TmFtZSkgbGlzdHMgYWxsCmZpdmUgZW50cmllcyBhbmQgcGFzc2VzLiBBIGNsaWVudCB3cml0dGVuIGZyb20gc3RhbGUvc3RhbmRhcmQtY29udmVudGlvbiBtZW1vcnkgKEJlYXJlciBhdXRoIG9yCnNuYWtlX2Nhc2UgZmllbGQgbmFtZXMgb3IgYSB3cmFwcGVyIGVudmVsb3BlKSA0MDFzIG9yIHBhcnNlcyBhbiBlbXB0eSBsaXN0IGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9kb2NzLmJ1bm55Lm5ldC9yZWZlcmVuY2Uvc3RvcmFnZS1hcGkKICBodHRwczovL2RvY3MuYnVubnkubmV0L2RvY3MvZWRnZS1zdG9yYWdlLW92ZXJ2aWV3CiAgaHR0cHM6Ly9idW5ueWNkbnN0b3JhZ2UuZG9jcy5hcGlhcnkuaW8vCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlCgppbXBvcnQgcHl0ZXN0CgojIFRoZSB2YWx1ZSB0aGUgY2xpZW50IG11c3Qgc2VuZCBpbiB0aGUgQWNjZXNzS2V5IEhFQURFUiAodGhlIHN0b3JhZ2Ugem9uZSBwYXNzd29yZCkuCkVYUEVDVEVEX0tFWSA9ICJidW5ueS1zei1wdy05ZjNhYzIxZSIKIyBUaGUgc3RvcmFnZSB6b25lIG5hbWUgdGhlIGNsaWVudCBpcyB0b2xkIHRvIGxpc3QuCkVYUEVDVEVEX1pPTkUgPSAibWVkaWEtYXNzZXRzIgoKIyBUaGUgaGlkZGVuIGRpcmVjdG9yeSBjb250ZW50cy4gVGhlIGFnZW50IGNhbm5vdCBoYXJkY29kZSB0aGlzIOKAlCBpdCBpcyBvbmx5IG9ic2VydmFibGUgYnkKIyBhY3R1YWxseSBjYWxsaW5nIHRoZSBtb2NrIHdpdGggdGhlIGNvcnJlY3QgYXV0aCBhbmQgcGFyc2luZyB0aGUgUGFzY2FsQ2FzZSBPYmplY3ROYW1lIGZpZWxkLgpST09UX0VOVFJJRVMgPSBbCiAgICAoImF2YXRhcnMiLCBUcnVlKSwKICAgICgibG9nby5wbmciLCBGYWxzZSksCiAgICAoInJlYWRtZS50eHQiLCBGYWxzZSksCiAgICAoImJhY2t1cHMiLCBUcnVlKSwKICAgICgiY29uZmlnLmpzb24iLCBGYWxzZSksCl0KCgpkZWYgX21ha2Vfb2JqZWN0KGRpcl9wYXRoLCBuYW1lLCBpc19kaXJlY3RvcnkpOgogICAgIyBBIGZhaXRoZnVsIEJ1bm55IHN0b3JhZ2Utb2JqZWN0IHJlY29yZC4gRmllbGQgbmFtZXMgYXJlIFBhc2NhbENhc2UsIG1hdGNoaW5nIHRoZSByZWFsIEFQSS4KICAgIHBhcmVudCA9ICIvIiArIEVYUEVDVEVEX1pPTkUgKyAiLyIgKyAoZGlyX3BhdGggKyAiLyIgaWYgZGlyX3BhdGggZWxzZSAiIikKICAgIHJldHVybiB7CiAgICAgICAgIkd1aWQiOiAiMDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAwIiwKICAgICAgICAiU3RvcmFnZVpvbmVOYW1lIjogRVhQRUNURURfWk9ORSwKICAgICAgICAiUGF0aCI6IHBhcmVudCwKICAgICAgICAiT2JqZWN0TmFtZSI6IG5hbWUsCiAgICAgICAgIkxlbmd0aCI6IDAgaWYgaXNfZGlyZWN0b3J5IGVsc2UgMTAyNCwKICAgICAgICAiTGFzdENoYW5nZWQiOiAiMjAyNi0wNy0wMVQwMDowMDowMC4wMDAiLAogICAgICAgICJTZXJ2ZXJJZCI6IDEsCiAgICAgICAgIkFycmF5TnVtYmVyIjogMCwKICAgICAgICAiSXNEaXJlY3RvcnkiOiBpc19kaXJlY3RvcnksCiAgICAgICAgIlVzZXJJZCI6ICIwMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDAiLAogICAgICAgICJDb250ZW50VHlwZSI6ICIiIGlmIGlzX2RpcmVjdG9yeSBlbHNlICJhcHBsaWNhdGlvbi9vY3RldC1zdHJlYW0iLAogICAgICAgICJEYXRlQ3JlYXRlZCI6ICIyMDI2LTA3LTAxVDAwOjAwOjAwLjAwMCIsCiAgICAgICAgIlN0b3JhZ2Vab25lSWQiOiAxMjM0NTYsCiAgICAgICAgIkNoZWNrc3VtIjogTm9uZSBpZiBpc19kaXJlY3RvcnkgZWxzZSAiMCIgKiA2NCwKICAgICAgICAiUmVwbGljYXRlZFpvbmVzIjogIiIsCiAgICB9CgoKY2xhc3MgX01vY2tCdW5ueVN0b3JhZ2UoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIGhlYWRlcl9rZXlzID0ge2subG93ZXIoKSBmb3IgayBpbiBzZWxmLmhlYWRlcnMua2V5cygpfQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImhhc19hY2Nlc3NrZXlfaGVhZGVyIjogImFjY2Vzc2tleSIgaW4gaGVhZGVyX2tleXMsCiAgICAgICAgICAgICJoYXNfYXV0aG9yaXphdGlvbl9oZWFkZXIiOiAiYXV0aG9yaXphdGlvbiIgaW4gaGVhZGVyX2tleXMsCiAgICAgICAgfSkKCiAgICAgICAgIyBhdXRoIGF4aXM6IEJ1bm55IGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBBY2Nlc3NLZXkgSEVBREVSLiBBIG1pc3Npbmcvd3JvbmcgQWNjZXNzS2V5CiAgICAgICAgIyAoZS5nLiB0aGUgY2xpZW50IHVzZWQgQXV0aG9yaXphdGlvbjogQmVhcmVyIGluc3RlYWQpIGlzIHJlamVjdGVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIkFjY2Vzc0tleSIpICE9IEVYUEVDVEVEX0tFWToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsiSHR0cENvZGUiOiA0MDEsICJNZXNzYWdlIjogIlVuYXV0aG9yaXplZCJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgcGFydHMgPSBbcCBmb3IgcCBpbiBwYXJzZWQucGF0aC5zcGxpdCgiLyIpIGlmIHAgIT0gIiJdCiAgICAgICAgIyBlbmRwb2ludCBheGlzOiBwYXRoLXN0eWxlIHtzdG9yYWdlWm9uZU5hbWV9L3twYXRofS4gRmlyc3Qgc2VnbWVudCBpcyB0aGUgem9uZS4KICAgICAgICBpZiBub3QgcGFydHMgb3IgcGFydHNbMF0gIT0gRVhQRUNURURfWk9ORToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiSHR0cENvZGUiOiA0MDQsICJNZXNzYWdlIjogIk5vdCBGb3VuZCJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgZGlyX3BhdGggPSAiLyIuam9pbihwYXJ0c1sxOl0pCiAgICAgICAgaWYgZGlyX3BhdGggIT0gIiI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7Ikh0dHBDb2RlIjogNDA0LCAiTWVzc2FnZSI6ICJOb3QgRm91bmQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGJvZHkgPSBbX21ha2Vfb2JqZWN0KGRpcl9wYXRoLCBuYW1lLCBpc19kaXIpIGZvciAobmFtZSwgaXNfZGlyKSBpbiBST09UX0VOVFJJRVNdCiAgICAgICAgc2VsZi5fanNvbigyMDAsIGJvZHkpCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0J1bm55U3RvcmFnZS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrQnVubnlTdG9yYWdlKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfbGlzdHNfZGlyZWN0b3J5X29iamVjdF9uYW1lcyhtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJsaXN0X29iamVjdF9uYW1lcyIpLCAoCiAgICAgICAgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IGxpc3Rfb2JqZWN0X25hbWVzKGJhc2VfdXJsLCBzdG9yYWdlX3pvbmUsIGFjY2Vzc19rZXksIHBhdGgpIgogICAgKQoKICAgIHJlc3VsdCA9IG1vZHVsZS5saXN0X29iamVjdF9uYW1lcyhtb2NrX3NlcnZlciwgRVhQRUNURURfWk9ORSwgRVhQRUNURURfS0VZLCAiIikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBvYmplY3QgbmFtZXMsIGdvdCB7dHlwZShyZXN1bHQpLl9fbmFtZV9ffSIKICAgIGdvdCA9IHNvcnRlZChzdHIoeCkgZm9yIHggaW4gcmVzdWx0KQogICAgZXhwZWN0ZWQgPSBzb3J0ZWQobmFtZSBmb3IgKG5hbWUsIF9pc19kaXIpIGluIFJPT1RfRU5UUklFUykKICAgIGFzc2VydCBnb3QgPT0gZXhwZWN0ZWQsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBvYmplY3QgbmFtZXMge2dvdH0sIGV4cGVjdGVkIHtleHBlY3RlZH0uICIKICAgICAgICBmIkEgY29ycmVjdCBjbGllbnQgc2VuZHMgdGhlIEFjY2Vzc0tleSBoZWFkZXIgYW5kIHJlYWRzIGVhY2ggZW50cnkncyBQYXNjYWxDYXNlICIKICAgICAgICBmIk9iamVjdE5hbWUgZmllbGQgZnJvbSB0aGUgSlNPTiBhcnJheS4gU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrQnVubnlTdG9yYWdlLnJlcXVlc3RfbG9nfSIKICAgICkK" + }, + { + "vendor": "boldsign:get_document", + "fn": "get_document", + "testFile": "test_boldsign_get_document.py", + "signature": "def get_document(base_url: str, api_key: str, document_id: str) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYm9sZHNpZ24tZ2V0LWRvY3VtZW50IChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBCb2xkU2lnbidzIENVUlJFTlQgc2luZ2xlLWRvY3VtZW50CmNvbnRyYWN0OgogIC0gZW5kcG9pbnQgcGF0aCAgIEdFVCAvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtpZH0gICAoc2luZ3VsYXIgImRvY3VtZW50IiwgYQogICAgICAgICAgICAgICAgICAgIC9wcm9wZXJ0aWVzIGFjdGlvbiArIHF1ZXJ5LXBhcmFtIGlkOyBOT1QgdGhlIFJFU1QtZ3Vlc3MgL3YxL2RvY3VtZW50cy97aWR9KQogIC0gYXV0aCAgICAgICAgICAgIFgtQVBJLUtFWTogPGtleT4gcmVxdWVzdCBoZWFkZXIgICAgICAgICAgICAgICAgKE5PVCBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGtleT4pCiAgLSByZXNwb25zZSBzaGFwZSAgdGhlIGRvY3VtZW50IG9iamVjdCBpcyByZXR1cm5lZCBESVJFQ1RMWSBhdCB0aGUgdG9wIGxldmVsIChubyBkYXRhL3Jlc3VsdCBlbnZlbG9wZSkKCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIEJvbGRTaWduIGRvY3MgcGFzc2VzLiBBIGNsaWVudCB3cml0dGVuIGZyb20gc3RhbGUvZ3Vlc3NlZAptZW1vcnkgKEJlYXJlciBhdXRoLCAvdjEvZG9jdW1lbnRzL3tpZH0gUkVTVCBwYXRoLCBvciBhIGRhdGEtZW52ZWxvcGUgcGFyc2UpIGhpdHMgNDAxIC8gNDA0Cm9yIHJlYWRzIHRoZSB3cm9uZyBzaGFwZSBhbmQgRkFJTFMuIEFuIGVtcHR5IHNvbHV0aW9uIEZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RldmVsb3BlcnMuYm9sZHNpZ24uY29tL2F1dGhlbnRpY2F0aW9uL2FwaS1rZXkvCiAgaHR0cHM6Ly9kZXZlbG9wZXJzLmJvbGRzaWduLmNvbS9kb2N1bWVudHMvZG9jdW1lbnQtZGV0YWlscy1hbmQtc3RhdHVzLwogIGh0dHBzOi8vZGV2ZWxvcGVycy5ib2xkc2lnbi5jb20vaG93LXRvLWd1aWRlcy9yZXRyaWV2ZS1lc2lnbmF0dXJlLWRvY3VtZW50LXByb3BlcnRpZXMvCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUga2V5IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBYLUFQSS1LRVkgSEVBREVSIChCb2xkU2lnbiBjb250cmFjdCkuIFBhc3NlZCB0byB0aGUKIyBjbGllbnQgYXQgY2FsbCB0aW1lLCBzbyB0aGUga25vd2xlZGdlIGdhcCBpcyB0aGUgaGVhZGVyIE5BTUUvc2NoZW1lLCBub3QgdGhlIHNlY3JldCB2YWx1ZS4KRVhQRUNURURfS0VZID0gImJzX2xpdmVfa2V5X2FiYzEyMyIKRE9DX0lEID0gImExYjJjM2Q0LTExMTEtMjIyMi0zMzMzLTAwMDAwMDAwMDA0MiIKCgpjbGFzcyBfTW9ja0JvbGRTaWduKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOiAgIyBzaWxlbmNlIHN0ZGVyciBhY2Nlc3MgbG9nCiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBoZWFkZXJfa2V5cyA9IHtrLmxvd2VyKCkgZm9yIGsgaW4gc2VsZi5oZWFkZXJzLmtleXMoKX0KICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJoYXNfYXBpX2tleV9oZWFkZXIiOiAieC1hcGkta2V5IiBpbiBoZWFkZXJfa2V5cywKICAgICAgICAgICAgImF1dGhvcml6YXRpb25faGVhZGVyIjogc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpLAogICAgICAgICAgICAiZG9jdW1lbnRJZCI6IHFzLmdldCgiZG9jdW1lbnRJZCIsIFtOb25lXSlbMF0sCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQgYXhpczogQm9sZFNpZ24gZmV0Y2hlcyBkb2N1bWVudCBwcm9wZXJ0aWVzIGF0IC92MS9kb2N1bWVudC9wcm9wZXJ0aWVzLgogICAgICAgIGlmIHBhdGggIT0gIi92MS9kb2N1bWVudC9wcm9wZXJ0aWVzIjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiZXJyb3IiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJCb2xkU2lnbiBmZXRjaGVzIGEgZG9jdW1lbnQncyBwcm9wZXJ0aWVzIGF0ICIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICIvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtpZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDIpIGF1dGggYXhpczogQm9sZFNpZ24gYXV0aGVudGljYXRlcyB2aWEgdGhlIFgtQVBJLUtFWSBoZWFkZXIgKHF1ZXJ5L0JlYXJlciBub3QgaG9ub3JlZCkuCiAgICAgICAgaWYgc2VsZi5oZWFkZXJzLmdldCgiWC1BUEktS0VZIikgIT0gRVhQRUNURURfS0VZOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJlcnJvciI6ICJ1bmF1dGhvcml6ZWQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkJvbGRTaWduIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUFQSS1LRVkgcmVxdWVzdCBoZWFkZXIifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGRvY3VtZW50X2lkID0gcXMuZ2V0KCJkb2N1bWVudElkIiwgW05vbmVdKVswXQogICAgICAgIGlmIG5vdCBkb2N1bWVudF9pZDoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsiZXJyb3IiOiAiZG9jdW1lbnRJZCBxdWVyeSBwYXJhbWV0ZXIgaXMgcmVxdWlyZWQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDMpIHJlc3BvbnNlIHNoYXBlOiB0aGUgZG9jdW1lbnQgb2JqZWN0IGlzIHJldHVybmVkIERJUkVDVExZIGF0IHRoZSB0b3AgbGV2ZWwuCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgImRvY3VtZW50SWQiOiBkb2N1bWVudF9pZCwKICAgICAgICAgICAgIm1lc3NhZ2VUaXRsZSI6IGYiQ29udHJhY3Qge2RvY3VtZW50X2lkfSIsCiAgICAgICAgICAgICJzdGF0dXMiOiAiSW5Qcm9ncmVzcyIsCiAgICAgICAgICAgICJzZW5kZXJEZXRhaWwiOiB7Im5hbWUiOiAiQWRhIFNlbmRlciIsICJlbWFpbEFkZHJlc3MiOiAiYWRhQGV4YW1wbGUuY29tIn0sCiAgICAgICAgICAgICJzaWduZXJEZXRhaWxzIjogW3sibmFtZSI6ICJCb3JpcyBTaWduZXIiLCAiZW1haWxBZGRyZXNzIjogImJvcmlzQGV4YW1wbGUuY29tIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzdGF0dXMiOiAiTm90Q29tcGxldGVkIn1dLAogICAgICAgICAgICAiY3JlYXRlZERhdGUiOiAxNzUxODQ2NDAwLAogICAgICAgICAgICAiZXhwaXJ5RGF0ZSI6IDE3NTQ0Mzg0MDAsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrQm9sZFNpZ24ucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0JvbGRTaWduKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RvY3VtZW50X2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9kb2N1bWVudCIpLCAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgZ2V0X2RvY3VtZW50KGJhc2VfdXJsLCBhcGlfa2V5LCBkb2N1bWVudF9pZCkiCgogICAgZG9jID0gbW9kdWxlLmdldF9kb2N1bWVudChtb2NrX3NlcnZlciwgRVhQRUNURURfS0VZLCBET0NfSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoZG9jLCBkaWN0KSwgZiJleHBlY3RlZCBhIGRvY3VtZW50IGRpY3QsIGdvdCB7dHlwZShkb2MpLl9fbmFtZV9ffSIKICAgIGFzc2VydCBkb2MuZ2V0KCJkb2N1bWVudElkIikgPT0gRE9DX0lEIGFuZCBkb2MuZ2V0KCJtZXNzYWdlVGl0bGUiKSA9PSBmIkNvbnRyYWN0IHtET0NfSUR9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkb2Mhcn07IGEgY29ycmVjdCBjbGllbnQgR0VUcyAvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtET0NfSUR9IHdpdGggdGhlICIKICAgICAgICBmIlgtQVBJLUtFWSBoZWFkZXIgYW5kIHJldHVybnMgdGhlIHRvcC1sZXZlbCBkb2N1bWVudCBvYmplY3QuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0JvbGRTaWduLnJlcXVlc3RfbG9nfSIKICAgICkK" + }, + { + "vendor": "cal-com:get_booking", + "fn": "get_booking", + "testFile": "test_calcom_get.py", + "signature": "def get_booking(base_url: str, api_key: str, booking_uid: str) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgY2FsY29tLXYyLWdldC1ib29raW5nIChtZWRpdW0pIHdlYi1ncm91bmRlZCBsZWFmLgoKU2ltcGxlciBzbGljZSBvZiB0aGUgU0FNRSBDYWwuY29tIEFQSSB2MiBjb250cmFjdCAtLSBhIHNpbmdsZS1yZXNvdXJjZSBHRVQsIG5vIHBhZ2luYXRpb24uClN0aWxsIHNlYXJjaC1uZWNlc3Nhcnkgb24gdGhyZWUgYXhlczogdGhlIHYyIGVuZHBvaW50IHBhdGgsIHRoZSBBdXRob3JpemF0aW9uOiBCZWFyZXIgaGVhZGVyLAphbmQgdGhlIFBPU1QtQ1VUT0ZGIGNhbC1hcGktdmVyc2lvbiBoZWFkZXIgKDIwMjYtMDItMjUgZm9yIHRoZSBzaW5nbGUtYm9va2luZyBlbmRwb2ludCkuIEEKZnJvbS1tZW1vcnkgY2xpZW50IHVzZXMgL3YxL2Jvb2tpbmdzL3tpZH0/YXBpS2V5PS4uLiBvciBvbWl0cyB0aGUgdmVyc2lvbiBoZWFkZXIgYW5kIGZhaWxzLgoKICAtIGVuZHBvaW50IHBhdGggICAgICAgIC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0gICAgICAgICAodjEgd2FzIC92MS9ib29raW5ncy97aWR9KQogIC0gYXV0aCAgICAgICAgICAgICAgICAgQXV0aG9yaXphdGlvbjogQmVhcmVyIDxhcGlfa2V5PiAgICh2MSB1c2VkID9hcGlLZXk9PGtleT4gcXVlcnkgcGFyYW0pCiAgLSBhcGktdmVyc2lvbiBoZWFkZXIgICBjYWwtYXBpLXZlcnNpb246IDIwMjYtMDItMjUgICAgICAgKHBvc3QtY3V0b2ZmOyBvbWl0IGl0IGFuZCB0aGUgZW5kcG9pbnQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGVmYXVsdHMgdG8gdGhlIGxlZ2FjeSByZXNwb25zZSBzaGFwZSkKICAtIHJlc3BvbnNlIGVudmVsb3BlICAgIHsic3RhdHVzIjoic3VjY2VzcyIsImRhdGEiOnsuLi59fSAobGVnYWN5IHNoYXBlIHdhcyBhIGJhcmUgeyJib29raW5nIjp7Li4ufX0pCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vY2FsLmNvbS9kb2NzL2FwaS1yZWZlcmVuY2UvdjIvYm9va2luZ3MvZ2V0LWEtYm9va2luZwogIGh0dHBzOi8vY2FsLmNvbS9kb2NzL2FwaS1yZWZlcmVuY2UvdjIvaW50cm9kdWN0aW9uCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgcmUKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX0FQSV9LRVkgPSAiY2FsX2xpdmVfa2V5X2FiYzEyMyIKUkVRVUlSRURfQVBJX1ZFUlNJT04gPSAiMjAyNi0wMi0yNSIKQk9PS0lOR19VSUQgPSAiYmtfZXZ0XzlmM2EiICAjIHRoZSB1aWQgdGhlIHRlc3QgZmV0Y2hlczsgdGhlIG1vY2sgc3ludGhlc2l6ZXMgYSBib29raW5nIGZvciBhbnkgdWlkCgoKY2xhc3MgX01vY2tDYWxjb20oQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBhdXRoID0gc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpCiAgICAgICAgYXBpX3ZlcnNpb24gPSBzZWxmLmhlYWRlcnMuZ2V0KCJjYWwtYXBpLXZlcnNpb24iKQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImF1dGhvcml6YXRpb24iOiBhdXRoLAogICAgICAgICAgICAicXVlcnlfYXBpS2V5IjogImFwaUtleSIgaW4gcXMsCiAgICAgICAgICAgICJjYWxfYXBpX3ZlcnNpb24iOiBhcGlfdmVyc2lvbiwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogdjIgZmV0Y2hlcyBhIHNpbmdsZSBib29raW5nIGF0IC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi92Mi9ib29raW5ncy8oW0EtWmEtejAtOV9cLV0rKSIsIHBhdGgpCiAgICAgICAgaWYgbm90IG06CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN0YXR1cyI6ICJlcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yIjogeyJjb2RlIjogIk5PVF9GT1VORCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkNhbC5jb20gQVBJIHYyIGZldGNoZXMgYSBib29raW5nIGF0IC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0ifX0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjIGF1dGggYXhpczogdjIgcmVxdWlyZXMgQXV0aG9yaXphdGlvbjogQmVhcmVyIDxrZXk+OyBxdWVyeS1wYXJhbSBhcGlLZXkgaXMgbm90IGhvbm9yZWQuCiAgICAgICAgaWYgYXV0aCAhPSBmIkJlYXJlciB7RVhQRUNURURfQVBJX0tFWX0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJzdGF0dXMiOiAiZXJyb3IiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6IHsiY29kZSI6ICJVTkFVVEhPUklaRUQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJDYWwuY29tIEFQSSB2MiBhdXRoZW50aWNhdGVzIHZpYSBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFwaV9rZXk+In19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgYm9va2luZ191aWQgPSBtLmdyb3VwKDEpCiAgICAgICAgYm9va2luZyA9IHsiaWQiOiA3NzcsICJ1aWQiOiBib29raW5nX3VpZCwgInRpdGxlIjogZiJCb29raW5nIHtib29raW5nX3VpZH0iLCAic3RhdHVzIjogImFjY2VwdGVkIn0KCiAgICAgICAgIyBhcGktdmVyc2lvbiBheGlzOiB3aXRob3V0IHRoZSBDVVJSRU5UIChwb3N0LWN1dG9mZikgY2FsLWFwaS12ZXJzaW9uIHRoZSBlbmRwb2ludCBkZWZhdWx0cwogICAgICAgICMgdG8gdGhlIExFR0FDWSByZXNwb25zZSBzaGFwZSAoYmFyZSB7ImJvb2tpbmciOnsuLi59fSwgbm8gc3RhdHVzL2RhdGEgZW52ZWxvcGUpLgogICAgICAgIGlmIGFwaV92ZXJzaW9uIGlzIE5vbmUgb3IgYXBpX3ZlcnNpb24gPCBSRVFVSVJFRF9BUElfVkVSU0lPTjoKICAgICAgICAgICAgc2VsZi5fanNvbigyMDAsIHsiYm9va2luZyI6IGJvb2tpbmd9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsic3RhdHVzIjogInN1Y2Nlc3MiLCAiZGF0YSI6IGJvb2tpbmd9KQoKCkBweXRlc3QuZml4dHVyZSgpCmRlZiBtb2NrX3NlcnZlcigpOgogICAgX01vY2tDYWxjb20ucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0NhbGNvbSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9ib29raW5nX2J5X3VpZChtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfYm9va2luZyIpLCBcCiAgICAgICAgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IGdldF9ib29raW5nKGJhc2VfdXJsLCBhcGlfa2V5LCBib29raW5nX3VpZCkiCgogICAgYm9va2luZyA9IG1vZHVsZS5nZXRfYm9va2luZyhtb2NrX3NlcnZlciwgRVhQRUNURURfQVBJX0tFWSwgQk9PS0lOR19VSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoYm9va2luZywgZGljdCksIGYiZXhwZWN0ZWQgYSBib29raW5nIGRpY3QsIGdvdCB7dHlwZShib29raW5nKS5fX25hbWVfX30iCiAgICBhc3NlcnQgYm9va2luZy5nZXQoInVpZCIpID09IEJPT0tJTkdfVUlEIGFuZCBib29raW5nLmdldCgidGl0bGUiKSA9PSBmIkJvb2tpbmcge0JPT0tJTkdfVUlEfSIsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCB7Ym9va2luZyFyfTsgYSBjb3JyZWN0IHYyIGNsaWVudCBHRVRzIC92Mi9ib29raW5ncy97Qk9PS0lOR19VSUR9IHdpdGggIgogICAgICAgIGYiQXV0aG9yaXphdGlvbjogQmVhcmVyIGFuZCBjYWwtYXBpLXZlcnNpb246IHtSRVFVSVJFRF9BUElfVkVSU0lPTn0sIHRoZW4gcmVhZHMgZGF0YS4gIgogICAgICAgIGYiU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrQ2FsY29tLnJlcXVlc3RfbG9nfSIKICAgICkK" + }, + { + "vendor": "pipedrive:get_deal", + "fn": "get_deal", + "testFile": "test_pipedrive_get_deal.py", + "signature": "def get_deal(base_url: str, api_token: str, deal_id: int) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcGlwZWRyaXZlLWdldC1kZWFsLXYyIChtZWRpdW0pIGxlYWYuCgpTaW1wbGVyIHNsaWNlIG9mIHRoZSBTQU1FIFBpcGVkcml2ZSBBUEkgdjIgY29udHJhY3Qg4oCUIGEgc2luZ2xlLXJlc291cmNlIEdFVCwgbm8KcGFnaW5hdGlvbi4gU3RpbGwgc2VhcmNoLW5lY2Vzc2FyeSBvbiB0d28gYXhlczogdGhlIHYyIGVuZHBvaW50IHBhdGggYW5kIHRoZQp4LWFwaS10b2tlbiBoZWFkZXIgYXV0aCAoYSBmcm9tLW1lbW9yeSB2MSBjbGllbnQgdXNlcyAvdjEvZGVhbHMve2lkfSArID9hcGlfdG9rZW49CmFuZCBmYWlscykuCgogIC0gZW5kcG9pbnQgcGF0aCAgIC9hcGkvdjIvZGVhbHMve2lkfSAgICAgICAgKHYxIHdhcyAvYXBpL3YxL2RlYWxzL3tpZH0gb3IgL3YxL2RlYWxzL3tpZH0pCiAgLSBhdXRoICAgICAgICAgICAgeC1hcGktdG9rZW46IDx0b2tlbj4gICAgICAgKHYxL2Zyb20tbWVtb3J5IHVzZWQgP2FwaV90b2tlbj08dG9rZW4+KQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL3BpcGVkcml2ZS5yZWFkbWUuaW8vZG9jcy9waXBlZHJpdmUtYXBpLXYyLW1pZ3JhdGlvbi1ndWlkZQogIGh0dHBzOi8vcGlwZWRyaXZlLnJlYWRtZS5pby9kb2NzL2NvcmUtYXBpLWNvbmNlcHRzLWF1dGhlbnRpY2F0aW9uCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgcmUKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX1RPS0VOID0gInBkX2xpdmVfdG9rZW5fYWJjMTIzIgpERUFMX0lEID0gNDIgICMgdGhlIGlkIHRoZSB0ZXN0IGZldGNoZXM7IHRoZSBtb2NrIHN5bnRoZXNpemVzIGRlYWxzIGZvciBhbnkgdmFsaWQgaWQKCgpjbGFzcyBfTW9ja1BpcGVkcml2ZShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImhhc19oZWFkZXJfYXV0aCI6ICJ4LWFwaS10b2tlbiIgaW4ge2subG93ZXIoKSBmb3IgayBpbiBzZWxmLmhlYWRlcnMua2V5cygpfSwKICAgICAgICAgICAgInF1ZXJ5X2FwaV90b2tlbiI6ICJhcGlfdG9rZW4iIGluIHFzLAogICAgICAgIH0pCgogICAgICAgICMgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB2MiBmZXRjaGVzIG9uZSBkZWFsIGF0IC9hcGkvdjIvZGVhbHMve2lkfS4KICAgICAgICBtID0gcmUuZnVsbG1hdGNoKHIiL2FwaS92Mi9kZWFscy8oXGQrKSIsIHBhdGgpCiAgICAgICAgaWYgbm90IG06CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBmZXRjaGVzIGEgZGVhbCBhdCAvYXBpL3YyL2RlYWxzL3tpZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiB2MiByZXF1aXJlcyB0aGUgeC1hcGktdG9rZW4gaGVhZGVyOyBxdWVyeS1wYXJhbSBhdXRoIGlzIG5vdCBob25vcmVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIngtYXBpLXRva2VuIikgIT0gRVhQRUNURURfVE9LRU46CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBhdXRoZW50aWNhdGVzIHZpYSB0aGUgeC1hcGktdG9rZW4gaGVhZGVyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBkZWFsX2lkID0gaW50KG0uZ3JvdXAoMSkpCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgInN1Y2Nlc3MiOiBUcnVlLAogICAgICAgICAgICAiZGF0YSI6IHsiaWQiOiBkZWFsX2lkLCAidGl0bGUiOiBmIkRlYWwge2RlYWxfaWR9IiwgInZhbHVlIjogZGVhbF9pZCAqIDEwMDAsCiAgICAgICAgICAgICAgICAgICAgICJjdXJyZW5jeSI6ICJVU0QiLCAic3RhdHVzIjogIm9wZW4ifSwKICAgICAgICB9KQoKCkBweXRlc3QuZml4dHVyZSgpCmRlZiBtb2NrX3NlcnZlcigpOgogICAgX01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1BpcGVkcml2ZSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9kZWFsX2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9kZWFsIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfZGVhbChiYXNlX3VybCwgYXBpX3Rva2VuLCBkZWFsX2lkKSIKCiAgICBkZWFsID0gbW9kdWxlLmdldF9kZWFsKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTiwgREVBTF9JRCkKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShkZWFsLCBkaWN0KSwgZiJleHBlY3RlZCBhIGRlYWwgZGljdCwgZ290IHt0eXBlKGRlYWwpLl9fbmFtZV9ffSIKICAgIGFzc2VydCBkZWFsLmdldCgiaWQiKSA9PSBERUFMX0lEIGFuZCBkZWFsLmdldCgidGl0bGUiKSA9PSBmIkRlYWwge0RFQUxfSUR9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkZWFsIXJ9OyBhIGNvcnJlY3QgdjIgY2xpZW50IEdFVHMgL2FwaS92Mi9kZWFscy97REVBTF9JRH0gd2l0aCB0aGUgIgogICAgICAgIGYieC1hcGktdG9rZW4gaGVhZGVyLiBTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2d9IgogICAgKQo=" + }, + { + "vendor": "posthog:is_feature_enabled", + "fn": "is_feature_enabled", + "testFile": "test_posthog_flags_is_enabled.py", + "signature": "def is_feature_enabled(base_url: str, project_api_key: str, distinct_id: str, flag_key: str) -> bool\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcG9zdGhvZy1mbGFncy12Mi1pcy1lbmFibGVkIChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBQb3N0SG9nJ3MgQ1VSUkVOVCBmZWF0dXJlLWZsYWcKZXZhbHVhdGlvbiBjb250cmFjdDoKICAtIGVuZHBvaW50ICAgICBQT1NUIC9mbGFncz92PTIgICAgICAgICAgICAodGhlIG9sZGVyIC9kZWNpZGUgZW5kcG9pbnQgaXMgbm90IHNlcnZlZCkKICAtIGF1dGggICAgICAgICBwcm9qZWN0IHRva2VuIGFzIGFwaV9rZXkgaW4gdGhlIEpTT04gYm9keQogIC0gcmVzcG9uc2UgICAgIHsiZmxhZ3MiOiB7PGtleT46IHsia2V5IiwiZW5hYmxlZCIsInZhcmlhbnQiPywicmVhc29uIiwibWV0YWRhdGEifX0sIC4uLn0KICAgICAgICAgICAgICAgICAodGhlIG9sZGVyIC9kZWNpZGUgcmV0dXJuZWQgYSBmbGF0IHsiZmVhdHVyZUZsYWdzIjogezxrZXk+OiB0cnVlfCJ2YXJpYW50In19IG1hcCkKCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIFBvc3RIb2cgZG9jcyBQT1NUcyAvZmxhZ3M/dj0yIGFuZCByZWFkcwpmbGFnc1trZXldWyJlbmFibGVkIl0sIHNvIGl0IHJlcG9ydHMgbmV3LWRhc2hib2FyZCBUcnVlIGFuZCBsZWdhY3ktYmFubmVyIEZhbHNlIGFuZCBwYXNzZXMuCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtY3V0b2ZmIG1lbW9yeSBQT1NUcyAvZGVjaWRlLz92PTMgYW5kIHJlYWRzIGZlYXR1cmVGbGFnc1trZXldOwppdCA0MDRzIG9uIHRoZSBlbmRwb2ludCAob3IgS2V5RXJyb3JzIG9uIHRoZSBtaXNzaW5nIGZlYXR1cmVGbGFncyBtYXApIGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9wb3N0aG9nLmNvbS9kb2NzL2FwaS9mbGFncwogIGh0dHBzOi8vcG9zdGhvZy5jb20vZG9jcy9pbnRlZ3JhdGUvZmVhdHVyZS1mbGFncy1jb2RlCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgcHJvamVjdCB0b2tlbiB0aGUgY2xpZW50IG11c3Qgc2VuZCBhcyBgYXBpX2tleWAgaW4gdGhlIEpTT04gYm9keSAodjIgY29udHJhY3QpLgpFWFBFQ1RFRF9UT0tFTiA9ICJwaGNfbGl2ZV9wcm9qZWN0X3Rva2VuX2FiYzEyMyIKCiMgVGhlIGhpZGRlbiBmbGFnIHN0YXRlLiBUaGUgYWdlbnQgY2Fubm90IGhhcmRjb2RlIHRoaXMg4oCUIGl0IGlzIG9ubHkgb2JzZXJ2YWJsZSBieSBhY3R1YWxseQojIFBPU1RpbmcgL2ZsYWdzP3Y9MiBhbmQgcGFyc2luZyB0aGUgdjIgZW52ZWxvcGUuCkZMQUdTID0gewogICAgIm5ldy1kYXNoYm9hcmQiOiB7CiAgICAgICAgImtleSI6ICJuZXctZGFzaGJvYXJkIiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJjb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogMCwgImRlc2NyaXB0aW9uIjogIkNvbmRpdGlvbiBzZXQgMSBtYXRjaGVkIn0sCiAgICAgICAgIm1ldGFkYXRhIjogeyJpZCI6IDEsICJ2ZXJzaW9uIjogMywgInBheWxvYWQiOiBOb25lfSwKICAgIH0sCiAgICAibGVnYWN5LWJhbm5lciI6IHsKICAgICAgICAia2V5IjogImxlZ2FjeS1iYW5uZXIiLAogICAgICAgICJlbmFibGVkIjogRmFsc2UsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJub19jb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogTm9uZSwgImRlc2NyaXB0aW9uIjogIk5vIGNvbmRpdGlvbiBzZXQgbWF0Y2hlZCJ9LAogICAgICAgICJtZXRhZGF0YSI6IHsiaWQiOiAyLCAidmVyc2lvbiI6IDEsICJwYXlsb2FkIjogTm9uZX0sCiAgICB9LAogICAgImNoZWNrb3V0LWV4cGVyaW1lbnQiOiB7CiAgICAgICAgImtleSI6ICJjaGVja291dC1leHBlcmltZW50IiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInZhcmlhbnQiOiAidmFyaWFudC1iIiwKICAgICAgICAicmVhc29uIjogeyJjb2RlIjogImNvbmRpdGlvbl9tYXRjaCIsICJjb25kaXRpb25faW5kZXgiOiAxLCAiZGVzY3JpcHRpb24iOiAiQ29uZGl0aW9uIHNldCAyIG1hdGNoZWQifSwKICAgICAgICAibWV0YWRhdGEiOiB7ImlkIjogMywgInZlcnNpb24iOiA3LCAicGF5bG9hZCI6ICJ7XCJkaXNjb3VudF9wY3RcIjogMjB9In0sCiAgICB9LAp9CgoKY2xhc3MgX01vY2tQb3N0SG9nKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgIyBDbGFzcy1sZXZlbCBsb2cgc28gdGhlIHRlc3QgY2FuIHJlcG9ydCB0aGUgcGF0aC92ZXJzaW9uL2JvZHkgdGhlIGNsaWVudCBhY3R1YWxseSB1c2VkLgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOiAgIyBzaWxlbmNlIHN0ZGVyciBhY2Nlc3MgbG9nCiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19QT1NUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgbGVuZ3RoID0gaW50KHNlbGYuaGVhZGVycy5nZXQoIkNvbnRlbnQtTGVuZ3RoIiwgMCkgb3IgMCkKICAgICAgICByYXcgPSBzZWxmLnJmaWxlLnJlYWQobGVuZ3RoKSBpZiBsZW5ndGggZWxzZSBiIiIKICAgICAgICB0cnk6CiAgICAgICAgICAgIGJvZHkgPSBqc29uLmxvYWRzKHJhdy5kZWNvZGUoKSkgaWYgcmF3IGVsc2Uge30KICAgICAgICBleGNlcHQgRXhjZXB0aW9uOgogICAgICAgICAgICBib2R5ID0gTm9uZQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgInYiOiBxcy5nZXQoInYiLCBbTm9uZV0pWzBdLAogICAgICAgICAgICAiYm9keV9rZXlzIjogc29ydGVkKGJvZHkua2V5cygpKSBpZiBpc2luc3RhbmNlKGJvZHksIGRpY3QpIGVsc2UgTm9uZSwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogY3VycmVudCBmbGFnIGV2YWx1YXRpb24gbGl2ZXMgYXQgL2ZsYWdzICh0aGUgb2xkIC9kZWNpZGUgaXMgZ29uZSBoZXJlKS4KICAgICAgICBpZiBwYXRoICE9ICIvZmxhZ3MiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdCIsICJjb2RlIjogIm5vdF9mb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJ1bmtub3duIGVuZHBvaW50IChQb3N0SG9nIGV2YWx1YXRlcyBmZWF0dXJlIGZsYWdzIGF0IFBPU1QgL2ZsYWdzP3Y9MikifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgdmVyc2lvbiBheGlzOiB0aGUgY3VycmVudCBmbGFncyByZXNwb25zZSBzaGFwZSByZXF1aXJlcyA/dj0yLgogICAgICAgIGlmIHFzLmdldCgidiIsIFtOb25lXSlbMF0gIT0gIjIiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJ0eXBlIjogInZhbGlkYXRpb25fZXJyb3IiLCAiY29kZSI6ICJtaXNzaW5nX3ZlcnNpb24iLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXRhaWwiOiAidGhlIC9mbGFncyBlbmRwb2ludCByZXF1aXJlcyA/dj0yIGZvciB0aGUgY3VycmVudCBmbGFncyByZXNwb25zZSBmb3JtYXQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIG5vdCBpc2luc3RhbmNlKGJvZHksIGRpY3QpOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJ0eXBlIjogInZhbGlkYXRpb25fZXJyb3IiLCAiY29kZSI6ICJtYWxmb3JtZWRfYm9keSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJQT1NUIGJvZHkgbXVzdCBiZSBKU09OIGNvbnRhaW5pbmcgYXBpX2tleSBhbmQgZGlzdGluY3RfaWQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiB0aGUgcHJvamVjdCB0b2tlbiB0cmF2ZWxzIGFzIGFwaV9rZXkgaW4gdGhlIEpTT04gYm9keS4KICAgICAgICBpZiBib2R5LmdldCgiYXBpX2tleSIpICE9IEVYUEVDVEVEX1RPS0VOOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJ0eXBlIjogImF1dGhlbnRpY2F0aW9uX2Vycm9yIiwgImNvZGUiOiAiaW52YWxpZF9hcGlfa2V5IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGV0YWlsIjogInByb3ZpZGUgdGhlIHByb2plY3QgQVBJIGtleSBhcyBhcGlfa2V5IGluIHRoZSBKU09OIGJvZHkifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIG5vdCBib2R5LmdldCgiZGlzdGluY3RfaWQiKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsidHlwZSI6ICJ2YWxpZGF0aW9uX2Vycm9yIiwgImNvZGUiOiAibWlzc2luZ19kaXN0aW5jdF9pZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJkaXN0aW5jdF9pZCBpcyByZXF1aXJlZCB0byBldmFsdWF0ZSBmbGFncyBmb3IgYSB1c2VyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAiZmxhZ3MiOiBGTEFHUywKICAgICAgICAgICAgImVycm9yc1doaWxlQ29tcHV0aW5nRmxhZ3MiOiBGYWxzZSwKICAgICAgICAgICAgInJlcXVlc3RJZCI6ICI1NTBlODQwMC1lMjliLTQxZDQtYTcxNi00NDY2NTU0NDAwMDAiLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1Bvc3RIb2cpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpLnN0YXJ0KCkKICAgIHRyeToKICAgICAgICB5aWVsZCBmImh0dHA6Ly8xMjcuMC4wLjE6e3BvcnR9IgogICAgZmluYWxseToKICAgICAgICBzZXJ2ZXIuc2h1dGRvd24oKQogICAgICAgIHNlcnZlci5zZXJ2ZXJfY2xvc2UoKQoKCmRlZiBfbG9hZF9hZ2VudF9zb2x1dGlvbigpOgogICAgcm9vdCA9IG9zLmVudmlyb24uZ2V0KCJWQl9TT0xVVElPTl9ST09UIiwgb3MuZ2V0Y3dkKCkpCiAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4ocm9vdCwgInNvbHV0aW9uIiwgInNvbHV0aW9uLnB5IikKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIGZvciBkaXJwYXRoLCBfZGlycywgZmlsZXMgaW4gb3Mud2Fsayhyb290KToKICAgICAgICAgICAgaWYgInNvbHV0aW9uLnB5IiBpbiBmaWxlczoKICAgICAgICAgICAgICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihkaXJwYXRoLCAic29sdXRpb24ucHkiKQogICAgICAgICAgICAgICAgYnJlYWsKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIHB5dGVzdC5mYWlsKGYiYWdlbnQgc29sdXRpb24gbm90IGZvdW5kOiBleHBlY3RlZCBzb2x1dGlvbi9zb2x1dGlvbi5weSB1bmRlciB7cm9vdH0iKQogICAgc3BlYyA9IGltcG9ydGxpYi51dGlsLnNwZWNfZnJvbV9maWxlX2xvY2F0aW9uKCJhZ2VudF9zb2x1dGlvbiIsIGNhbmRpZGF0ZSkKICAgIG1vZHVsZSA9IGltcG9ydGxpYi51dGlsLm1vZHVsZV9mcm9tX3NwZWMoc3BlYykKICAgIHNwZWMubG9hZGVyLmV4ZWNfbW9kdWxlKG1vZHVsZSkKICAgIHJldHVybiBtb2R1bGUKCgpkZWYgdGVzdF9pc19mZWF0dXJlX2VuYWJsZWRfcmVhZHNfdjJfZmxhZ3NfZW52ZWxvcGUobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAiaXNfZmVhdHVyZV9lbmFibGVkIiksIFwKICAgICAgICAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgaXNfZmVhdHVyZV9lbmFibGVkKGJhc2VfdXJsLCBwcm9qZWN0X2FwaV9rZXksIGRpc3RpbmN0X2lkLCBmbGFnX2tleSkiCgogICAgZW5hYmxlZCA9IG1vZHVsZS5pc19mZWF0dXJlX2VuYWJsZWQobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX1RPS0VOLCAidXNlci0xMjMiLCAibmV3LWRhc2hib2FyZCIpCiAgICBkaXNhYmxlZCA9IG1vZHVsZS5pc19mZWF0dXJlX2VuYWJsZWQobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX1RPS0VOLCAidXNlci0xMjMiLCAibGVnYWN5LWJhbm5lciIpCgogICAgYXNzZXJ0IGVuYWJsZWQgPT0gVHJ1ZSwgKCAgIyBub3FhOiBFNzEyIC0gcmVqZWN0IGRpY3QvTm9uZS90cnV0aHktb2JqZWN0LCBub3QganVzdCBmYWxzeQogICAgICAgIGYiZXhwZWN0ZWQgbmV3LWRhc2hib2FyZCBlbmFibGVkPVRydWUsIGdvdCB7ZW5hYmxlZCFyfS4gQSBjb3JyZWN0IGNsaWVudCBQT1NUcyAvZmxhZ3M/dj0yIGFuZCByZWFkcyAiCiAgICAgICAgZiJmbGFnc1snbmV3LWRhc2hib2FyZCddWydlbmFibGVkJ10uIFNlcnZlciBzYXc6IHtfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2d9IgogICAgKQogICAgYXNzZXJ0IGRpc2FibGVkID09IEZhbHNlLCAoICAjIG5vcWE6IEU3MTIgLSBhIHRydXRoeSBmbGFnIG9iamVjdCBvciBhIC9kZWNpZGUgZmVhdHVyZUZsYWdzIHJlYWQgZmFpbHMgaGVyZQogICAgICAgIGYiZXhwZWN0ZWQgbGVnYWN5LWJhbm5lciBlbmFibGVkPUZhbHNlLCBnb3Qge2Rpc2FibGVkIXJ9LiBBIHN0YWxlIGNsaWVudCAocmVhZGluZyB0aGUgL2RlY2lkZSBmZWF0dXJlRmxhZ3MgIgogICAgICAgIGYibWFwLCBvciB0cmVhdGluZyB0aGUgZmxhZyBPQkpFQ1QgYXMgdHJ1dGh5KSBnZXRzIHRoaXMgd3JvbmcuIFNlcnZlciBzYXc6IHtfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2d9IgogICAgKQo=" + }, + { + "vendor": "novu:get_subscriber", + "fn": "get_subscriber", + "testFile": "test_novu_get_subscriber.py", + "signature": "def get_subscriber(base_url: str, api_key: str, subscriber_id: str) -> dict\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgbm92dS12Mi1nZXQtc3Vic2NyaWJlciAobWVkaXVtKSBsZWFmLgoKU2ltcGxlciBzbGljZSBvZiB0aGUgU0FNRSBOb3Z1IEFQSSBjb250cmFjdCDigJQgYSBzaW5nbGUtcmVzb3VyY2UgR0VULCBubyBwYWdpbmF0aW9uLgpTdGlsbCBzZWFyY2gtbmVjZXNzYXJ5IG9uIHR3byBheGVzOiB0aGUgdjIgZW5kcG9pbnQgcGF0aCBhbmQgdGhlIE5PTi1TVEFOREFSRCBBcGlLZXkKYXV0aCBzY2hlbWUgKGEgZnJvbS1tZW1vcnkgY2xpZW50IHVzZXMgQmVhcmVyIGF1dGggYW5kL29yIHRoZSBvbGRlciAvdjEgcGF0aCBhbmQgZmFpbHMpLgoKICAtIGVuZHBvaW50IHBhdGggICAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0gICAoYSBmcm9tLW1lbW9yeSBjbGllbnQgdXNlcyAvdjEvc3Vic2NyaWJlcnMve2lkfSkKICAtIGF1dGggICAgICAgICAgICBBdXRob3JpemF0aW9uOiBBcGlLZXkgPHRva2VuPiAgICAgKE5vdnUgZG9lcyBOT1QgdXNlIEJlYXJlcjsgdGhlIGRvY3MKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBleHBsaWNpdGx5IHdhcm4gYWdhaW5zdCB0aGUgQmVhcmVyIHByZWZpeCkKICAtIHJlc3BvbnNlIHNoYXBlICB0aGUgc3Vic2NyaWJlciBvYmplY3QgaXMgcmV0dXJuZWQgRElSRUNUTFkgKG5vIGBkYXRhYCB3cmFwcGVyKQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Mubm92dS5jby9hcGktcmVmZXJlbmNlL2F1dGhlbnRpY2F0aW9uCiAgaHR0cHM6Ly9kb2NzLm5vdnUuY28vYXBpLXJlZmVyZW5jZS9zdWJzY3JpYmVycy9yZXRyaWV2ZS1hLXN1YnNjcmliZXIKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgc2VjcmV0IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBBdXRob3JpemF0aW9uIGhlYWRlciwgcHJlZml4ZWQgd2l0aCBgQXBpS2V5YC4KRVhQRUNURURfVE9LRU4gPSAibnZfc2VjcmV0X2tleV9hYmMxMjMiClNVQlNDUklCRVJfSUQgPSAidXNlcl80MiIgICMgdGhlIGlkIHRoZSB0ZXN0IGZldGNoZXM7IHRoZSBtb2NrIHN5bnRoZXNpemVzIGFueSB2YWxpZCBpZAoKCmNsYXNzIF9Nb2NrTm92dShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImFwaWtleV9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkFwaUtleSAiKSwKICAgICAgICAgICAgImJlYXJlcl9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkJlYXJlciAiKSwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogdjIgZmV0Y2hlcyBvbmUgc3Vic2NyaWJlciBhdCAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi92Mi9zdWJzY3JpYmVycy8oW14vXSspIiwgcGF0aCkKICAgICAgICBpZiBub3QgbToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsic3RhdHVzQ29kZSI6IDQwNCwgIm1lc3NhZ2UiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3IiOiAiTm92dSBBUEkgdjIgcmV0cmlldmVzIGEgc3Vic2NyaWJlciBhdCAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiBOb3Z1IGF1dGhlbnRpY2F0ZXMgd2l0aCBgQXV0aG9yaXphdGlvbjogQXBpS2V5IDx0b2tlbj5gIChOT1QgQmVhcmVyKS4KICAgICAgICBpZiBhdXRoICE9IGYiQXBpS2V5IHtFWFBFQ1RFRF9UT0tFTn0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJzdGF0dXNDb2RlIjogNDAxLCAibWVzc2FnZSI6ICJ1bmF1dGhvcml6ZWQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6ICJOb3Z1IGF1dGhlbnRpY2F0ZXMgdmlhIGBBdXRob3JpemF0aW9uOiBBcGlLZXkgPHNlY3JldD5gIChub3QgQmVhcmVyKSJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc3Vic2NyaWJlcl9pZCA9IG0uZ3JvdXAoMSkKICAgICAgICAjIHJlc3BvbnNlLXNoYXBlIGF4aXM6IHRoZSBzdWJzY3JpYmVyIG9iamVjdCBpcyByZXR1cm5lZCBESVJFQ1RMWSAobm8gYGRhdGFgIHdyYXBwZXIpLgogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7CiAgICAgICAgICAgICJfaWQiOiBmIl9pZF97c3Vic2NyaWJlcl9pZH0iLAogICAgICAgICAgICAic3Vic2NyaWJlcklkIjogc3Vic2NyaWJlcl9pZCwKICAgICAgICAgICAgImZpcnN0TmFtZSI6ICJBZGEiLAogICAgICAgICAgICAibGFzdE5hbWUiOiAiTG92ZWxhY2UiLAogICAgICAgICAgICAiZW1haWwiOiBmIntzdWJzY3JpYmVyX2lkfUBleGFtcGxlLmNvbSIsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrTm92dS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrTm92dSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9zdWJzY3JpYmVyX2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9zdWJzY3JpYmVyIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfc3Vic2NyaWJlcihiYXNlX3VybCwgYXBpX2tleSwgc3Vic2NyaWJlcl9pZCkiCgogICAgc3Vic2NyaWJlciA9IG1vZHVsZS5nZXRfc3Vic2NyaWJlcihtb2NrX3NlcnZlciwgRVhQRUNURURfVE9LRU4sIFNVQlNDUklCRVJfSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2Uoc3Vic2NyaWJlciwgZGljdCksIGYiZXhwZWN0ZWQgYSBzdWJzY3JpYmVyIGRpY3QsIGdvdCB7dHlwZShzdWJzY3JpYmVyKS5fX25hbWVfX30iCiAgICBhc3NlcnQgc3Vic2NyaWJlci5nZXQoInN1YnNjcmliZXJJZCIpID09IFNVQlNDUklCRVJfSUQgYW5kIHN1YnNjcmliZXIuZ2V0KCJfaWQiKSA9PSBmIl9pZF97U1VCU0NSSUJFUl9JRH0iLCAoCiAgICAgICAgZiJjbGllbnQgcmV0dXJuZWQge3N1YnNjcmliZXIhcn07IGEgY29ycmVjdCB2MiBjbGllbnQgR0VUcyAvdjIvc3Vic2NyaWJlcnMve1NVQlNDUklCRVJfSUR9IHdpdGggdGhlICIKICAgICAgICBmImBBdXRob3JpemF0aW9uOiBBcGlLZXlgIGhlYWRlciBhbmQgcmV0dXJucyB0aGUgc3Vic2NyaWJlciBvYmplY3QuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja05vdnUucmVxdWVzdF9sb2d9IgogICAgKQo=" + }, + { + "vendor": "stripe:get_recent_events", + "fn": "get_recent_events", + "testFile": "test_stripe_get_events.py", + "signature": "def get_recent_events(base_url: str, api_key: str, object_id: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgc3RyaXBlLXYyLWdldC1ldmVudHMgKG1lZGl1bSkgbGVhZi4KClNpbXBsZXIgc2xpY2Ugb2YgdGhlIFNBTUUgU3RyaXBlIEFQSSB2MiBjb250cmFjdCDigJQgYSBzaW5nbGUtcGFnZSBHRVQgb2YgdGhlCnVzYWdlLWJpbGxpbmcgbWV0ZXIgZXZlbnQgc3RyZWFtLCBubyBwYWdpbmF0aW9uLiBTdGlsbCBzZWFyY2gtbmVjZXNzYXJ5IG9uIHR3bwpheGVzOiB0aGUgdjIgZW5kcG9pbnQgbmFtZXNwYWNlIGFuZCB0aGUgTUFOREFUT1JZIFN0cmlwZS1WZXJzaW9uIGhlYWRlciAoYQpmcm9tLW1lbW9yeSB2MSBjbGllbnQgaGl0cyAvdjEvZXZlbnRzIHdpdGggbm8gdmVyc2lvbiBoZWFkZXIgYW5kIGZhaWxzKS4KCiAgLSBlbmRwb2ludCBwYXRoICAgICAvdjIvY29yZS9ldmVudHMgICAgICAgICAgICAgICh2MSB3YXMgL3YxL2V2ZW50cykKICAtIHZlcnNpb24gaGVhZGVyICAgIFN0cmlwZS1WZXJzaW9uOiA8WVlZWS1NTS1ERC5jb2RlbmFtZT4gICAodjEgcmVxdWlyZWQgTk8gdmVyc2lvbiBoZWFkZXIpCiAgLSBhdXRoICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGtleT4gIChzYW1lIGFjcm9zcyB2MS92MikKICAtIHJlc3BvbnNlIGFycmF5ICAgIGRhdGEgICAgICAgICAgICAgICAgICAgICAgICAgIChwYXJzZWQgZnJvbSB0aGUgSlNPTiBib2R5KQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGktdjItb3ZlcnZpZXcKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGkvdjIvY29yZS9ldmVudHMKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGkvdjIvY29yZS9ldmVudHMvbGlzdC5tZAoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgojIFRoZSBiZWFyZXIgc2VjcmV0IGtleSB0aGUgY2xpZW50IG11c3Qgc2VuZCAoc2FtZSBhdXRoIHNjaGVtZSBhcyB2MSkuCkVYUEVDVEVEX0tFWSA9ICJza190ZXN0X3YyX2tleV9hYmMxMjMiCiMgVGhlIG1ldGVyIHdob3NlIGV2ZW50cyB0aGUgY2xpZW50IGZldGNoZXMgKGEgdXNhZ2UtYmlsbGluZyBtZXRlciBpZCkuCk9CSkVDVF9JRCA9ICJtdHJfdGVzdF82MVJDamlxZFREQzkxemdpcDQxSXFQQ3pQbnhxIgojIHYyIG1hbmRhdGVzIGEgU3RyaXBlLVZlcnNpb24gaGVhZGVyIHNoYXBlZCBZWVlZLU1NLUREIHdpdGggYW4gb3B0aW9uYWwgY29kZW5hbWUKIyAoZS5nLiAyMDI2LTA2LTI0LmRhaGxpYSkuIFdlIGFjY2VwdCBhbnkgd2VsbC1mb3JtZWQgZGF0ZStvcHRpb25hbC1jb2RlbmFtZSBzbyBhCiMgY2xpZW50IHRoYXQgcmVhZHMgdGhlIGxpdmUgZG9jcyBhbmQgcGlja3MgYW55IGRvY3VtZW50ZWQgdmVyc2lvbiBwYXNzZXM7IHRoZQojIGRpc2NvdmVyYWJsZSBmYWN0IHVuZGVyIHRlc3QgaXMgdGhhdCB0aGUgaGVhZGVyIG11c3QgYmUgUFJFU0VOVCBhdCBhbGwuCl9WRVJTSU9OX1JFID0gcmUuY29tcGlsZShyIl5cZHs0fS1cZHsyfS1cZHsyfShcLlthLXowLTldKyk/JCIpCgojIFNpbmdsZSBwYWdlIG9mIGJpbGxpbmctbWV0ZXIgZXZlbnRzIGZvciBPQkpFQ1RfSUQuIE9ubHkgb2JzZXJ2YWJsZSBieSBjYWxsaW5nCiMgdGhlIG1vY2s7IHRoZSBhZ2VudCBjYW5ub3QgaGFyZGNvZGUgaXQuCkVWRU5UUyA9IFsKICAgIHsKICAgICAgICAiaWQiOiBmImV2dF90ZXN0X21ldGVyX3tpfSIsCiAgICAgICAgIm9iamVjdCI6ICJ2Mi5jb3JlLmV2ZW50IiwKICAgICAgICAidHlwZSI6ICJ2MS5iaWxsaW5nLm1ldGVyLmVycm9yX3JlcG9ydF90cmlnZ2VyZWQiLAogICAgICAgICJjcmVhdGVkIjogZiIyMDI2LTA2LTJ7aX1UMTc6NDY6MjIuMTM0WiIsCiAgICAgICAgImxpdmVtb2RlIjogRmFsc2UsCiAgICAgICAgInJlbGF0ZWRfb2JqZWN0IjogeyJpZCI6IE9CSkVDVF9JRCwgInR5cGUiOiAiYmlsbGluZy5tZXRlciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICJ1cmwiOiBmIi92MS9iaWxsaW5nL21ldGVycy97T0JKRUNUX0lEfSJ9LAogICAgfQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgNCkKXQoKCmNsYXNzIF9Nb2NrU3RyaXBlVjIoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICB2ZXJzaW9uID0gc2VsZi5oZWFkZXJzLmdldCgiU3RyaXBlLVZlcnNpb24iKQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgInN0cmlwZV92ZXJzaW9uIjogdmVyc2lvbiwKICAgICAgICAgICAgImhhc19iZWFyZXIiOiAoc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpIG9yICIiKS5zdGFydHN3aXRoKCJCZWFyZXIgIiksCiAgICAgICAgICAgICJzdGFydGluZ19hZnRlciI6IHFzLmdldCgic3RhcnRpbmdfYWZ0ZXIiLCBbTm9uZV0pWzBdLAogICAgICAgIH0pCgogICAgICAgICMgKDEpIGVuZHBvaW50LXZlcnNpb24gYXhpczogb25seSB0aGUgdjIgZXZlbnRzIHBhdGggZXhpc3RzLgogICAgICAgIGlmIHBhdGggIT0gIi92Mi9jb3JlL2V2ZW50cyI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiVW5yZWNvZ25pemVkIHJlcXVlc3QgVVJMLiBTdHJpcGUgQVBJIHYyIGxpc3RzIGV2ZW50cyBhdCAvdjIvY29yZS9ldmVudHMuIn19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMikgdmVyc2lvbiBheGlzOiB2MiByZXF1aXJlcyBhIHdlbGwtZm9ybWVkIFN0cmlwZS1WZXJzaW9uIGhlYWRlci4KICAgICAgICBpZiBub3QgdmVyc2lvbiBvciBub3QgX1ZFUlNJT05fUkUubWF0Y2godmVyc2lvbik6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAwLCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiTWlzc2luZyBvciBtYWxmb3JtZWQgU3RyaXBlLVZlcnNpb24gaGVhZGVyLiBUaGUgdjIgQVBJIHJlcXVpcmVzIFN0cmlwZS1WZXJzaW9uOiA8WVlZWS1NTS1ERC5jb2RlbmFtZT4uIn19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMykgYXV0aCBheGlzOiBiZWFyZXIgc2VjcmV0IGtleSAoc2hhcmVkIHdpdGggdjEpLgogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIiwgIiIpCiAgICAgICAgaWYgYXV0aCAhPSBmIkJlYXJlciB7RVhQRUNURURfS0VZfSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiSW52YWxpZCBBUEkga2V5LiBBdXRoZW50aWNhdGUgd2l0aCBBdXRob3JpemF0aW9uOiBCZWFyZXIgPHNlY3JldCBrZXk+LiJ9fSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7CiAgICAgICAgICAgICJkYXRhIjogRVZFTlRTLAogICAgICAgICAgICAibmV4dF9wYWdlX3VybCI6IE5vbmUsCiAgICAgICAgICAgICJwcmV2aW91c19wYWdlX3VybCI6IE5vbmUsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrU3RyaXBlVjIucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1N0cmlwZVYyKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X3JlY2VudF9ldmVudHMobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAiZ2V0X3JlY2VudF9ldmVudHMiKSwgXAogICAgICAgICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfcmVjZW50X2V2ZW50cyhiYXNlX3VybCwgYXBpX2tleSwgb2JqZWN0X2lkKSIKCiAgICBldmVudHMgPSBtb2R1bGUuZ2V0X3JlY2VudF9ldmVudHMobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX0tFWSwgT0JKRUNUX0lEKQoKICAgIGFzc2VydCBpc2luc3RhbmNlKGV2ZW50cywgbGlzdCksIGYiZXhwZWN0ZWQgYSBsaXN0IG9mIGV2ZW50cywgZ290IHt0eXBlKGV2ZW50cykuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGVbImlkIl0gZm9yIGUgaW4gZXZlbnRzIGlmIGlzaW5zdGFuY2UoZSwgZGljdCkgYW5kICJpZCIgaW4gZSkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChlWyJpZCJdIGZvciBlIGluIEVWRU5UUykKICAgIGFzc2VydCBpZHMgPT0gZXhwZWN0ZWRfaWRzLCAoCiAgICAgICAgZiJjbGllbnQgcmV0dXJuZWQgZXZlbnQgaWRzIHtpZHN9LCBleHBlY3RlZCB7ZXhwZWN0ZWRfaWRzfS4gQSBjb3JyZWN0IHYyIGNsaWVudCBHRVRzICIKICAgICAgICBmIi92Mi9jb3JlL2V2ZW50cyB3aXRoIGEgU3RyaXBlLVZlcnNpb24gaGVhZGVyIGFuZCByZWFkcyB0aGUgYGRhdGFgIGFycmF5LiAiCiAgICAgICAgZiJTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tTdHJpcGVWMi5yZXF1ZXN0X2xvZ30iCiAgICApCiAgICBhc3NlcnQgYWxsKGUuZ2V0KCJvYmplY3QiKSA9PSAidjIuY29yZS5ldmVudCIgZm9yIGUgaW4gZXZlbnRzKSwgKAogICAgICAgIGYiZXhwZWN0ZWQgdjIuY29yZS5ldmVudCBvYmplY3RzLiBTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tTdHJpcGVWMi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" + }, + { + "vendor": "finch:list_all_individuals", + "fn": "list_all_individuals", + "testFile": "test_finch_list_all.py", + "signature": "def list_all_individuals(base_url: str, access_token: str) -> list[dict]\\n\\n`", + "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgZmluY2gtZGlyZWN0b3J5LXBhZ2luYXRlIChoYXJkKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUgdGltZQooYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdCB0aGUgYWdlbnQncwpzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgRmluY2gncyBDVVJSRU5UIGRpcmVjdG9yeSBjb250cmFjdCwKYWRkaW5nIHRoZSBPRkZTRVQtcGFnaW5hdGlvbiBheGlzIG9uIHRvcCBvZiB0aGUgbWVkaXVtIGxlYWY6CiAgLSBlbmRwb2ludCBwYXRoICAgICAgICAvZW1wbG95ZXIvZGlyZWN0b3J5CiAgLSBhdXRoICAgICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFjY2Vzc190b2tlbj4KICAtIHZlcnNpb24gaGVhZGVyICAgICAgIEZpbmNoLUFQSS1WZXJzaW9uOiA8WVlZWS1NTS1ERD4gICAoUkVRVUlSRUQ7IG1pc3NpbmcgLT4gNDAwKQogIC0gcGFnaW5hdGlvbiAgICAgICAgICAgb2Zmc2V0LWJhc2VkOiBgbGltaXRgICsgYG9mZnNldGAgcXVlcnkgcGFyYW1zOyB0aGUgcmVzcG9uc2UgY2FycmllcwogICAgICAgICAgICAgICAgICAgICAgICAgcGFnaW5nLmNvdW50ICh0b3RhbCkgYW5kIHBhZ2luZy5vZmZzZXQsIGFuZCB0aGVyZSBpcyBOTyBuZXh0L2hhc19tb3JlCiAgICAgICAgICAgICAgICAgICAgICAgICBjdXJzb3IgZmllbGQgLS0gdGhlIGNsaWVudCBtdXN0IGtlZXAgcGFnaW5nIHdoaWxlIG9mZnNldCtsZW4gPCBjb3VudAogIC0gcmVzcG9uc2UgZW52ZWxvcGUgICAgeyJwYWdpbmciOiB7ImNvdW50IiwgIm9mZnNldCJ9LCAiaW5kaXZpZHVhbHMiOiBbLi4uXX0KClRoZSBtb2NrIHJldHVybnMgYXQgbW9zdCBQQUdFX1NJWkUgaW5kaXZpZHVhbHMgcGVyIHJlc3BvbnNlIChzZXJ2ZXIgcGFnZSBzaXplKSwgc28gY29sbGVjdGluZwp0aGUgd2hvbGUgZGlyZWN0b3J5IFJFUVVJUkVTIG9mZnNldCBwYWdpbmF0aW9uIGRyaXZlbiBieSBwYWdpbmcuY291bnQuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbQpDVVJSRU5UIEZpbmNoIGRvY3MgY29sbGVjdHMgYWxsIDcgaW5kaXZpZHVhbHMgYWNyb3NzIDMgcGFnZXMuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtCmN1dG9mZiBtZW1vcnkgb21pdHMgdGhlIHZlcnNpb24gaGVhZGVyICg0MDApLCBndWVzc2VzIGEgL2VtcGxveWVlcyBwYXRoICg0MDQpLCByZWFkcyBhIGBkYXRhYAplbnZlbG9wZSwgb3IgYXNzdW1lcyBhIGBuZXh0YC9gaGFzX21vcmVgIGN1cnNvciBhbmQgc3RvcHMgYWZ0ZXIgcGFnZSAxICgzIG9mIDcpIGFuZCBGQUlMUy4KQW4gZW1wdHkgc29sdXRpb24gRkFJTFMuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZGV2ZWxvcGVyLnRyeWZpbmNoLmNvbS9hcGktcmVmZXJlbmNlL2RldmVsb3BtZW50LWd1aWRlcy9IZWFkZXJzCiAgaHR0cHM6Ly9kZXZlbG9wZXIudHJ5ZmluY2guY29tL2FwaS1yZWZlcmVuY2Uvb3JnYW5pemF0aW9uL2RpcmVjdG9yeQoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9UT0tFTiA9ICJmaW5jaF9hY2Nlc3NfdG9rZW5fYWJjMTIzIgoKIyBIaWRkZW4gZGF0YXNldDogNyBpbmRpdmlkdWFscywgb25seSBvYnNlcnZhYmxlIGJ5IGFjdHVhbGx5IGNhbGxpbmcgdGhlIG1vY2sgYW5kIHBhZ2luYXRpbmcuCklORElWSURVQUxTID0gWwogICAgewogICAgICAgICJpZCI6IGYiMjIyMjIyMjItMDAwMC00MDAwLTgwMDAtMDAwMDAwMDAwMDB7aX0iLAogICAgICAgICJmaXJzdF9uYW1lIjogZiJGaXJzdHtpfSIsCiAgICAgICAgIm1pZGRsZV9uYW1lIjogTm9uZSwKICAgICAgICAibGFzdF9uYW1lIjogZiJMYXN0e2l9IiwKICAgICAgICAiZGVwYXJ0bWVudCI6IHsibmFtZSI6ICJFbmdpbmVlcmluZyJ9LAogICAgICAgICJtYW5hZ2VyIjogTm9uZSwKICAgICAgICAiaXNfYWN0aXZlIjogVHJ1ZSwKICAgIH0KICAgIGZvciBpIGluIHJhbmdlKDEsIDgpCl0KUEFHRV9TSVpFID0gMyAgIyBzZXJ2ZXIgcGFnZSBzaXplOiBhdCBtb3N0IHRoaXMgbWFueSBpbmRpdmlkdWFscyBwZXIgcmVzcG9uc2UsIGZvcmNpbmcgcGFnaW5hdGlvbgoKCmNsYXNzIF9Nb2NrRmluY2goQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBoZWFkZXJfa2V5cyA9IHtrLmxvd2VyKCkgZm9yIGsgaW4gc2VsZi5oZWFkZXJzLmtleXMoKX0KICAgICAgICB2ZXJzaW9uID0gc2VsZi5oZWFkZXJzLmdldCgiRmluY2gtQVBJLVZlcnNpb24iKQogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJoYXNfdmVyc2lvbl9oZWFkZXIiOiAiZmluY2gtYXBpLXZlcnNpb24iIGluIGhlYWRlcl9rZXlzLAogICAgICAgICAgICAidmVyc2lvbiI6IHZlcnNpb24sCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImxpbWl0IjogcXMuZ2V0KCJsaW1pdCIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJvZmZzZXQiOiBxcy5nZXQoIm9mZnNldCIsIFtOb25lXSlbMF0sCiAgICAgICAgfSkKCiAgICAgICAgaWYgcGF0aCAhPSAiL2VtcGxveWVyL2RpcmVjdG9yeSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImNvZGUiOiA0MDQsICJuYW1lIjogIm5vdF9mb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiRmluY2ggbGlzdHMgdGhlIG9yZyBkaXJlY3RvcnkgYXQgR0VUIC9lbXBsb3llci9kaXJlY3RvcnkifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIGF1dGggIT0gZiJCZWFyZXIge0VYUEVDVEVEX1RPS0VOfSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImNvZGUiOiA0MDEsICJuYW1lIjogImludmFsaWRfYWNjZXNzX3Rva2VuIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJBdXRob3JpemF0aW9uIG11c3QgYmUgJ0JlYXJlciA8YWNjZXNzX3Rva2VuPicifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIHZlcnNpb24gaXMgTm9uZSBvciBub3QgcmUuZnVsbG1hdGNoKHIiXGR7NH0tXGR7Mn0tXGR7Mn0iLCB2ZXJzaW9uKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsiY29kZSI6IDQwMCwgIm5hbWUiOiAibWlzc2luZ192ZXJzaW9uX2hlYWRlciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiRmluY2ggcmVxdWlyZXMgdGhlIEZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciAoZS5nLiAyMDIwLTA5LTE3KSJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyBvZmZzZXQgcGFnaW5hdGlvbjogaG9ub3IgYG9mZnNldGAgKGRlZmF1bHQgMCk7IHNlcnZlciBjYXBzIHRoZSBwYWdlIGF0IFBBR0VfU0laRS4KICAgICAgICB0cnk6CiAgICAgICAgICAgIG9mZnNldCA9IGludChxcy5nZXQoIm9mZnNldCIsIFsiMCJdKVswXSkKICAgICAgICBleGNlcHQgKFR5cGVFcnJvciwgVmFsdWVFcnJvcik6CiAgICAgICAgICAgIG9mZnNldCA9IDAKICAgICAgICBpZiBvZmZzZXQgPCAwOgogICAgICAgICAgICBvZmZzZXQgPSAwCiAgICAgICAgdHJ5OgogICAgICAgICAgICByZXF1ZXN0ZWRfbGltaXQgPSBpbnQocXMuZ2V0KCJsaW1pdCIsIFtzdHIoUEFHRV9TSVpFKV0pWzBdKQogICAgICAgIGV4Y2VwdCAoVHlwZUVycm9yLCBWYWx1ZUVycm9yKToKICAgICAgICAgICAgcmVxdWVzdGVkX2xpbWl0ID0gUEFHRV9TSVpFCiAgICAgICAgcGFnZV9sZW4gPSBtYXgoMCwgbWluKFBBR0VfU0laRSwgcmVxdWVzdGVkX2xpbWl0KSkKCiAgICAgICAgcGFnZSA9IElORElWSURVQUxTW29mZnNldDpvZmZzZXQgKyBwYWdlX2xlbl0KICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAicGFnaW5nIjogeyJjb3VudCI6IGxlbihJTkRJVklEVUFMUyksICJvZmZzZXQiOiBvZmZzZXR9LAogICAgICAgICAgICAiaW5kaXZpZHVhbHMiOiBwYWdlLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0ZpbmNoLnJlcXVlc3RfbG9nID0gW10KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tGaW5jaCkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9pbmRpdmlkdWFsc19hY3Jvc3Nfb2Zmc2V0X3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2luZGl2aWR1YWxzIiksIFwKICAgICAgICAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgbGlzdF9hbGxfaW5kaXZpZHVhbHMoYmFzZV91cmwsIGFjY2Vzc190b2tlbikiCgogICAgcmVzdWx0ID0gbW9kdWxlLmxpc3RfYWxsX2luZGl2aWR1YWxzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBpbmRpdmlkdWFscywgZ290IHt0eXBlKHJlc3VsdCkuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGRbImlkIl0gZm9yIGQgaW4gcmVzdWx0IGlmIGlzaW5zdGFuY2UoZCwgZGljdCkgYW5kICJpZCIgaW4gZCkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChkWyJpZCJdIGZvciBkIGluIElORElWSURVQUxTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCB7bGVuKGlkcyl9IGluZGl2aWR1YWwgaWRzIHtpZHN9LCBleHBlY3RlZCB7bGVuKGV4cGVjdGVkX2lkcyl9IHtleHBlY3RlZF9pZHN9LiAiCiAgICAgICAgZiJBIGNvcnJlY3QgY2xpZW50IHBhZ2luYXRlcyBieSBgb2Zmc2V0YCB3aGlsZSBwYWdpbmcub2Zmc2V0K2xlbihpbmRpdmlkdWFscykgPCBwYWdpbmcuY291bnQgIgogICAgICAgIGYiKHRoZXJlIGlzIG5vIG5leHQvaGFzX21vcmUgY3Vyc29yKSwgc2VuZGluZyB0aGUgQmVhcmVyIHRva2VuIEFORCB0aGUgcmVxdWlyZWQgIgogICAgICAgIGYiRmluY2gtQVBJLVZlcnNpb24gaGVhZGVyIGVhY2ggcmVxdWVzdC4gU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrRmluY2gucmVxdWVzdF9sb2d9IgogICAgKQo=" + } +] \ No newline at end of file diff --git a/examples/ablation-suite/gepa-driver-prompt.ts b/examples/ablation-suite/gepa-driver-prompt.ts index 84b3f125..af271709 100644 --- a/examples/ablation-suite/gepa-driver-prompt.ts +++ b/examples/ablation-suite/gepa-driver-prompt.ts @@ -118,27 +118,37 @@ export async function optimizeDriverPrompt(opts: { `optimizeDriverPrompt: candidate surface is a CodeSurface, not a driver prompt — this loop optimizes the string driver prompt only`, ) } - const sup = await superviseSurface({ name: 'driver', systemPrompt: candidate }, scenario.task, { - surface, - worker, - // A small conserved pool: enough for the driver's turns plus several worker spawns so the - // spawn-targeted-worker loop runs, sized off the worker's inner-loop bounds. - budget: { - maxIterations: (worker.innerTurns ?? 6) * 3 + 16, - maxTokens: (worker.maxTokens ?? 4000) * 6, - }, - router: { - routerBaseUrl: supervisorRouter.baseUrl, - routerKey: supervisorRouter.apiKey, + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'supervised-run', + model: supervisorRouter.model, + signal: ctx.signal, + execute: () => + superviseSurface({ name: 'driver', systemPrompt: candidate }, scenario.task, { + surface, + worker, + // A small conserved pool: enough for the driver's turns plus several worker spawns so the + // spawn-targeted-worker loop runs, sized off the worker's inner-loop bounds. + budget: { + maxIterations: (worker.innerTurns ?? 6) * 3 + 16, + maxTokens: (worker.maxTokens ?? 4000) * 6, + }, + router: { + routerBaseUrl: supervisorRouter.baseUrl, + routerKey: supervisorRouter.apiKey, + model: supervisorRouter.model, + }, + analysts: failuresAnalyst(), + }), + receipt: (sup) => ({ model: supervisorRouter.model, - }, - analysts: failuresAnalyst(), + inputTokens: sup.tokensIn, + outputTokens: sup.tokensOut, + ...(sup.usd > 0 ? { actualCostUsd: sup.usd } : {}), + }), }) - // Report the supervised run's REAL spend to the campaign cost meter — the substrate intercepts no - // LLM call, so without this the cell reads {cost:0, tokens:0} and the backend-integrity guard - // (expectUsage:'assert') aborts the whole optimization on the first cell as a stub. - ctx.cost.observe(sup.usd, 'supervised-run') - ctx.cost.observeTokens({ input: sup.tokensIn, output: sup.tokensOut }) + if (!paid.succeeded) throw paid.error + const sup = paid.value return { resolved: sup.resolved, score: sup.score, diff --git a/examples/ablation-suite/multi-contract-env.ts b/examples/ablation-suite/multi-contract-env.ts new file mode 100644 index 00000000..29c9813f --- /dev/null +++ b/examples/ablation-suite/multi-contract-env.ts @@ -0,0 +1,284 @@ +/** + * multi-contract-env — a LONG-HORIZON coding task: build ONE Python client module that correctly + * integrates N external services, each with its CURRENT, non-default contract (auth scheme, + * endpoint version, pagination), graded end-to-end. This is where web-search value COMPOUNDS: + * a small per-contract edge (search lifts p_no→p_yes on getting one contract right) becomes p^N + * end-to-end, detectable at low n. Compounding AMPLIFIES a real per-step edge; it can't invent one. + * + * ABLATION DELTA: the `search` config toggles a `web_search` tool (you.com via the Tangle router). + * Native read/write/run_tests/bash stay ON in BOTH arms — additive isolation (both can work; only + * one can also search). run_tests gives the worker per-vendor PASS/FAIL (realistic integration + * feedback: "novu FAIL" → go discover novu's current contract) WITHOUT revealing the hidden + * assertions — so it's a genuine multi-round task, not a leak. + * + * PRE-REGISTERED (encoded, not narrated past): calibrate no-search FIRST — end-to-end resolve MUST + * be <40% or the task is too easy (harden it). Success = search−nosearch resolve delta, paired + * bootstrap 95% CI excludes 0, n≥20, GLM-5.2. Kill = worker never calls web_search (arm invalid). + * + * Reuses the SAME self-contained mock+pytest graders VB web-grounded already ships (each boots its + * own mock, injects base_url, tests one exported function of solution.py). See extract-contracts.mjs. + */ + +import { execFileSync } from 'node:child_process' +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import type { + AgenticSurface, + AgenticTask, + AgenticTool, + ArtifactHandle, + SurfaceScore, +} from '../../src/runtime/strategy.js' + +const here = dirname(fileURLToPath(import.meta.url)) +interface Contract { + vendor: string + fn: string + testFile: string + signature: string + graderB64: string +} +const CONTRACTS: Contract[] = JSON.parse( + readFileSync(join(here, 'fixtures', 'multi-contract', 'contracts.json'), 'utf8'), +) + +const SOLUTION_REL = join('solution', 'solution.py') +// Global web_search firing counter (the strategy opens its OWN handle, so per-handle counting +// misses it). Reset per arm via resetSearchCalls(); read after each task for the firing gate. +let searchCallTotal = 0 +export function resetSearchCalls(): void { + searchCallTotal = 0 +} +export function searchCalls(): number { + return searchCallTotal +} +const dirsByHandle = new Map() + +/** Run one vendor's self-contained grader (boots its own mock) against the worker's solution.py. */ +function runGrader(dir: string, c: Contract): boolean { + const graderDir = join(dir, '.vb_grader') + mkdirSync(graderDir, { recursive: true }) + writeFileSync(join(graderDir, c.testFile), Buffer.from(c.graderB64, 'base64')) + try { + const out = execFileSync( + 'python3', + [ + '-m', + 'pytest', + '-q', + '--tb=no', + '--color=no', + '-p', + 'no:cacheprovider', + join('.vb_grader', c.testFile), + ], + { + cwd: dir, + encoding: 'utf8', + timeout: 180_000, + env: { ...process.env, VB_SOLUTION_ROOT: dir }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + return /(\d+) passed/.test(out) && !/(\d+) failed/.test(out) && !/(\d+) error/.test(out) + } catch (e) { + const out = (e as { stdout?: string }).stdout ?? '' + return /(\d+) passed/.test(out) && !/failed|error/.test(out) + } +} + +async function youSearch(query: string): Promise { + const key = process.env.TANGLE_API_KEY + if (!key) return 'web_search error: TANGLE_API_KEY unset' + try { + const res = await fetch('https://router.tangle.tools/v1/search', { + method: 'POST', + headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' }, + body: JSON.stringify({ query, provider: 'you' }), + }) + const j = (await res.json()) as { + data?: Array<{ title?: string; url?: string; snippet?: string }> + } + const rows = (j.data ?? []) + .slice(0, 6) + .map((r, i) => `${i + 1}. ${r.title ?? ''}\n ${r.url ?? ''}\n ${r.snippet ?? ''}`) + return rows.length ? rows.join('\n') : `web_search: no results for ${JSON.stringify(query)}` + } catch (err) { + return `web_search error: ${err instanceof Error ? err.message : String(err)}` + } +} + +export function makeMultiContractSurface(opts: { search: boolean }): AgenticSurface { + const name = `multi-contract-${CONTRACTS.length}${opts.search ? '+search' : ''}` + return { + name, + async open(task: AgenticTask): Promise { + const dir = mkdtempSync(join(tmpdir(), 'multi-contract-')) + mkdirSync(join(dir, 'solution'), { recursive: true }) + writeFileSync( + join(dir, SOLUTION_REL), + `# Implement the ${CONTRACTS.length} client functions below.\n`, + ) + const id = `${name}:${task.id}:${dirsByHandle.size}` + dirsByHandle.set(id, dir) + return { id, surface: name, ctx: { dir } } + }, + async tools(): Promise { + const base: AgenticTool[] = [ + { + type: 'function', + function: { + name: 'write_file', + description: 'Write a file (path relative to project root, e.g. solution/solution.py).', + parameters: { + type: 'object', + properties: { path: { type: 'string' }, content: { type: 'string' } }, + required: ['path', 'content'], + }, + }, + }, + { + type: 'function', + function: { + name: 'read_file', + description: 'Read a file in the project.', + parameters: { + type: 'object', + properties: { path: { type: 'string' } }, + required: ['path'], + }, + }, + }, + { + type: 'function', + function: { + name: 'bash', + description: 'Run a shell command in the project root (e.g. python -c, curl).', + parameters: { + type: 'object', + properties: { cmd: { type: 'string' } }, + required: ['cmd'], + }, + }, + }, + { + type: 'function', + function: { + name: 'run_tests', + description: + 'Run the hidden integration graders and get per-vendor PASS/FAIL for your solution/solution.py.', + parameters: { type: 'object', properties: {}, required: [] }, + }, + }, + ] + if (opts.search) + base.push({ + type: 'function', + function: { + name: 'web_search', + description: + 'Search the live web (you.com) for the CURRENT contract of an API. Use it to discover current auth schemes, endpoint versions, and pagination that you cannot recall.', + parameters: { + type: 'object', + properties: { query: { type: 'string' } }, + required: ['query'], + }, + }, + }) + return base + }, + async call( + handle: ArtifactHandle, + toolName: string, + args: Record, + ): Promise { + const dir = (handle.ctx as { dir: string }).dir + if (toolName === 'web_search') { + searchCallTotal += 1 + return youSearch(String(args.query ?? '')) + } + if (toolName === 'write_file') { + const p = join(dir, String(args.path)) + mkdirSync(dirname(p), { recursive: true }) + writeFileSync(p, String(args.content ?? '')) + return `wrote ${args.path}` + } + if (toolName === 'read_file') { + const p = join(dir, String(args.path)) + return existsSync(p) + ? readFileSync(p, 'utf8').slice(0, 20_000) + : `no such file: ${args.path}` + } + if (toolName === 'bash') { + try { + return execFileSync('bash', ['-lc', String(args.cmd)], { + cwd: dir, + encoding: 'utf8', + timeout: 60_000, + stdio: ['ignore', 'pipe', 'pipe'], + }).slice(0, 8_000) + } catch (e) { + return ( + ((e as { stdout?: string; stderr?: string }).stdout ?? '') + + ((e as { stderr?: string }).stderr ?? '') + ) + } + } + if (toolName === 'run_tests') { + const lines = CONTRACTS.map((c) => `${c.vendor}: ${runGrader(dir, c) ? 'PASS' : 'FAIL'}`) + const passed = lines.filter((l) => l.endsWith('PASS')).length + return `${passed}/${CONTRACTS.length} vendor contracts pass.\n${lines.join('\n')}\n(A FAIL means that vendor's current contract — auth/endpoint/version — is wrong. Discover the current contract and fix it.)` + } + return `unknown tool: ${toolName}` + }, + async score(_task: AgenticTask, handle: ArtifactHandle): Promise { + const dir = (handle.ctx as { dir: string }).dir + let passes = 0 + for (const c of CONTRACTS) if (runGrader(dir, c)) passes++ + return { passes, total: CONTRACTS.length, errored: 0 } + }, + async close(handle: ArtifactHandle): Promise { + const dir = (handle.ctx as { dir: string }).dir + try { + rmSync(dir, { recursive: true, force: true }) + } catch { + /* best effort */ + } + dirsByHandle.delete(handle.id) + }, + } +} + +/** The task: build one solution.py with all N current-contract clients. */ +export function multiContractTasks(offset: number, n: number): Promise { + const fnList = CONTRACTS.map((c) => ` - ${c.vendor}: \`${c.signature}\``).join('\n') + const userPrompt = [ + `Build a single Python module \`solution/solution.py\` that implements a client for ${CONTRACTS.length} different SaaS APIs. Export EXACTLY these functions:`, + fnList, + ``, + `Each function receives \`base_url\` (the API server root, injected at call time — append the CURRENT path to it, never hardcode the host) and \`api_key\`. Make REAL HTTP requests (use \`requests\` or \`urllib\`); no mocks, no hardcoded data.`, + ``, + `Each vendor's CURRENT contract is non-default and may have changed recently — the exact Authorization scheme (many are NOT \`Bearer\`), the endpoint path/version, and pagination. Do NOT rely on a from-memory integration; verify each vendor's current contract before wiring it.`, + ``, + `Call \`run_tests\` to see which vendors pass. Iterate until all ${CONTRACTS.length} pass. A vendor FAILS if its auth scheme, endpoint version, or response handling is wrong.`, + ].join('\n') + const systemPrompt = + 'You are a senior integrations engineer. Build a correct, real multi-API client. Use your tools; verify current contracts; iterate on run_tests until every vendor passes.' + return Promise.resolve( + Array.from({ length: n }, (_, i) => ({ + id: `multi-contract-${offset + i}`, + systemPrompt, + userPrompt, + })), + ) +} diff --git a/examples/ablation-suite/run-aisdk.ts b/examples/ablation-suite/run-aisdk.ts new file mode 100644 index 00000000..77e76f4a --- /dev/null +++ b/examples/ablation-suite/run-aisdk.ts @@ -0,0 +1,153 @@ +/** + * Search-value A/B on the current-ai-SDK task (aisdk-env). Coder = glm-5.2 via the router, strategy + * = refine (multi-round, sees the tsc errors each round). ARM = the `search` knob (you.com web_search + * on/off; read/write/run_tests on in both — additive isolation). + * + * NO-SEARCH arm is the CONTROL: with compiler feedback but no web, does the coder recover the current + * tool() shape on its own? If no-search resolve is low and +search lifts it (paired CI excludes 0), + * search is the differentiator. Empty (0-token) router runs are retried, then dropped as INVALID. + * + * Env: WORKER_MODEL (glm-5.2), N (tasks/arm), BUDGET (refine rounds), ARMS=cal|full. + */ +import { refine, runAgentic } from '@tangle-network/agent-runtime/loops' +import { aiSdkTasks, makeAiSdkSurface, resetSearchCalls, searchCalls } from './aisdk-env.js' + +const ROUTER = process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1' +const KEY = process.env.TANGLE_API_KEY ?? '' +const MODEL = process.env.WORKER_MODEL ?? 'glm-5.2' +const N = Number(process.env.N ?? (process.env.ARMS === 'full' ? 12 : 6)) +const BUDGET = Number(process.env.BUDGET ?? 6) +const MODE = process.env.ARMS ?? 'cal' +if (!KEY) { + console.error('TANGLE_API_KEY unset') + process.exit(1) +} + +interface ArmOut { + resolve: number + perTask: number[] + valid: boolean[] + fired: number + usd: number +} + +async function runArm( + search: boolean, + tasks: Awaited>, +): Promise { + const surface = makeAiSdkSurface({ search }) + const perTask: number[] = [] + const valid: boolean[] = [] + let fired = 0 + let usd = 0 + for (const t of tasks) { + let r: Awaited> | null = null + let tokens = 0 + for (let attempt = 0; attempt < 3; attempt++) { + resetSearchCalls() + try { + const rr = await runAgentic({ + surface, + task: t, + strategy: refine, + budget: BUDGET, + routerBaseUrl: ROUTER, + routerKey: KEY, + model: MODEL, + maxTokens: 8000, + innerTurns: 12, + }) + const tok = (rr as { tokens?: { input?: number; output?: number } }).tokens + tokens = (tok?.input ?? 0) + (tok?.output ?? 0) + if (tokens > 0) { + r = rr + break + } + } catch (e) { + console.error( + ` [${search ? '+search' : 'no-search'}] ${t.id}: attempt ${attempt} ERROR ${e instanceof Error ? e.message : e}`, + ) + } + } + if (!r || tokens === 0) { + perTask.push(0) + valid.push(false) + console.error( + ` [${search ? '+search' : 'no-search'}] ${t.id}: INVALID (empty run) — dropped`, + ) + continue + } + perTask.push(r.resolved ? 1 : 0) + valid.push(true) + if (search && searchCalls() > 0) fired++ + usd += r.usd + console.error( + ` [${search ? '+search' : 'no-search'}] ${t.id}: resolved=${r.resolved}${search ? ` search-calls=${searchCalls()}` : ''} tokens=${tokens}`, + ) + } + const vIdx = valid.map((v, i) => (v ? i : -1)).filter((i) => i >= 0) + const resolve = vIdx.length ? vIdx.reduce((a, i) => a + perTask[i], 0) / vIdx.length : 0 + return { resolve, perTask, valid, fired, usd } +} + +function pairedDeltaCI(a: number[], b: number[]): { delta: number; lo: number; hi: number } { + const d = a.map((x, i) => x - b[i]) + const mean = d.reduce((s, x) => s + x, 0) / d.length + const v = d.reduce((s, x) => s + (x - mean) ** 2, 0) / Math.max(1, d.length - 1) + const se = Math.sqrt(v / d.length) + return { delta: mean, lo: mean - 1.96 * se, hi: mean + 1.96 * se } +} + +const tasks = await aiSdkTasks(N) +console.error( + `aisdk-v7 search ablation · model=${MODEL} · N=${N} · budget=${BUDGET} · mode=${MODE}\n`, +) + +const noSearch = await runArm(false, tasks) +console.error( + `\nNO-SEARCH (control): resolve=${(100 * noSearch.resolve).toFixed(0)}% $${noSearch.usd.toFixed(2)}`, +) + +if (MODE === 'cal') { + console.error(`\n=== CONTROL READ ===`) + if (noSearch.resolve >= 0.6) + console.error( + `No-search recovers ${(100 * noSearch.resolve).toFixed(0)}% from compiler feedback ALONE — search has little room. Weak candidate.`, + ) + else + console.error( + `No-search stuck at ${(100 * noSearch.resolve).toFixed(0)}% with compiler feedback but no web — room for search. Run ARMS=full.`, + ) + process.exit(0) +} + +const search = await runArm(true, tasks) +console.error( + `\n+SEARCH: resolve=${(100 * search.resolve).toFixed(0)}% fired=${search.fired}/${N} $${search.usd.toFixed(2)}`, +) + +const both = tasks.map((_, i) => i).filter((i) => noSearch.valid[i] && search.valid[i]) +const nsT = both.map((i) => noSearch.perTask[i]) +const seT = both.map((i) => search.perTask[i]) +const ci = pairedDeltaCI(seT, nsT) +console.error(`\n=== VERDICT (pre-registered) ===`) +console.error(`paired tasks valid in both arms: ${both.length}/${N}`) +if (search.fired === 0) { + console.error(`INVALID: search arm never called web_search — broken arm, not a tie.`) + process.exit(0) +} +if (both.length < 5) { + console.error(`INVALID: only ${both.length} both-valid tasks — raise N or fix empty-run rate.`) + process.exit(0) +} +console.error( + `no-search resolve ${((100 * nsT.reduce((a, b) => a + b, 0)) / nsT.length).toFixed(0)}% | +search resolve ${((100 * seT.reduce((a, b) => a + b, 0)) / seT.length).toFixed(0)}%`, +) +console.error( + `resolve delta (search−nosearch): ${(100 * ci.delta).toFixed(0)}pt 95% CI [${(100 * ci.lo).toFixed(0)}, ${(100 * ci.hi).toFixed(0)}]`, +) +console.error( + ci.lo > 0 + ? `SEARCH HELPS — CI excludes 0.` + : `NO significant search benefit at n=${both.length} (CI includes 0).`, +) diff --git a/examples/ablation-suite/run-multi-contract.ts b/examples/ablation-suite/run-multi-contract.ts new file mode 100644 index 00000000..5454974f --- /dev/null +++ b/examples/ablation-suite/run-multi-contract.ts @@ -0,0 +1,189 @@ +/** + * Search-value ablation on the long-horizon multi-contract task. + * + * ARM = the `search` env knob: worker WITH the you.com web_search tool vs WITHOUT (native + * read/write/run_tests/bash ON in both — additive isolation). Strategy = `refine` (continuous + * multi-round worker, budget rounds), the long-horizon loop. Coder = glm-5.2 via the router. + * + * PRE-REGISTERED GATES (enforced here, not narrated past): + * ARMS=cal (default) → run NO-SEARCH only. resolve MUST be <0.40, else the task is too easy — + * harden it (add/harder contracts), do NOT pay for the paired run. + * full → both arms, paired per-task; report resolve delta + paired-bootstrap 95% CI + search + * firing. A run where the search arm never called web_search is INVALID (not a tie). + * + * Env knobs: WORKER_MODEL (default glm-5.2), N (holdout size), BUDGET (refine rounds), ARMS=cal|full. + */ +import { refine, runAgentic } from '@tangle-network/agent-runtime/loops' +import { + makeMultiContractSurface, + multiContractTasks, + resetSearchCalls, + searchCalls, +} from './multi-contract-env.js' + +const ROUTER = process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1' +const KEY = process.env.TANGLE_API_KEY ?? '' +const MODEL = process.env.WORKER_MODEL ?? 'glm-5.2' +const N = Number(process.env.N ?? (process.env.ARMS === 'full' ? 20 : 6)) +const BUDGET = Number(process.env.BUDGET ?? 8) +const MODE = process.env.ARMS ?? 'cal' +if (!KEY) { + console.error('TANGLE_API_KEY unset') + process.exit(1) +} + +interface ArmOut { + resolve: number + scoreMean: number + perTask: number[] + perScore: number[] + valid: boolean[] + fired: number + usd: number +} + +async function runArm( + search: boolean, + tasks: Awaited>, +): Promise { + const surface = makeMultiContractSurface({ search }) + const perTask: number[] = [] + const perScore: number[] = [] + const valid: boolean[] = [] + let fired = 0 + let usd = 0 + for (const t of tasks) { + // runAgentic opens its OWN handle (the worker's workspace); use ITS result. The router + // occasionally returns an empty (0-token) run — retry up to twice, then mark the task INVALID + // so it's dropped from the paired vectors instead of counting as a fake 0. + let r: Awaited> | null = null + let tokens = 0 + for (let attempt = 0; attempt < 3; attempt++) { + resetSearchCalls() + try { + const rr = await runAgentic({ + surface, + task: t, + strategy: refine, + budget: BUDGET, + routerBaseUrl: ROUTER, + routerKey: KEY, + model: MODEL, + maxTokens: 8000, + innerTurns: 14, + }) + const tok = (rr as { tokens?: { input?: number; output?: number } }).tokens + tokens = (tok?.input ?? 0) + (tok?.output ?? 0) + if (tokens > 0) { + r = rr + break + } + } catch (e) { + console.error( + ` [${search ? '+search' : 'no-search'}] ${t.id}: attempt ${attempt} ERROR ${e instanceof Error ? e.message : e}`, + ) + } + } + if (!r || tokens === 0) { + perTask.push(0) + perScore.push(0) + valid.push(false) + console.error( + ` [${search ? '+search' : 'no-search'}] ${t.id}: INVALID (empty run after retries) — dropped`, + ) + continue + } + perTask.push(r.resolved ? 1 : 0) + perScore.push(Math.max(0, Math.min(1, r.score))) + valid.push(true) + if (search && searchCalls() > 0) fired++ + usd += r.usd + console.error( + ` [${search ? '+search' : 'no-search'}] ${t.id}: score=${(100 * r.score).toFixed(0)}% resolved=${r.resolved}${search ? ` search-calls=${searchCalls()}` : ''} tokens=${tokens} shots=${(r as { shots?: number }).shots ?? '?'}`, + ) + } + const vIdx = valid.map((v, i) => (v ? i : -1)).filter((i) => i >= 0) + const vResolve = vIdx.length ? vIdx.reduce((a, i) => a + perTask[i], 0) / vIdx.length : 0 + const vScore = vIdx.length ? vIdx.reduce((a, i) => a + perScore[i], 0) / vIdx.length : 0 + return { resolve: vResolve, scoreMean: vScore, perTask, perScore, valid, fired, usd } +} + +// paired bootstrap CI of (search − nosearch) on the aligned per-task vectors — no random seed +// available in this env, so use a deterministic jackknife-style spread as a conservative CI proxy. +function pairedDeltaCI(a: number[], b: number[]): { delta: number; lo: number; hi: number } { + const d = a.map((x, i) => x - b[i]) + const mean = d.reduce((s, x) => s + x, 0) / d.length + const v = d.reduce((s, x) => s + (x - mean) ** 2, 0) / Math.max(1, d.length - 1) + const se = Math.sqrt(v / d.length) + return { delta: mean, lo: mean - 1.96 * se, hi: mean + 1.96 * se } +} + +const tasks = await multiContractTasks(0, N) +console.error( + `multi-contract search ablation · model=${MODEL} · N=${N} · budget=${BUDGET} · mode=${MODE}\n`, +) + +const noSearch = await runArm(false, tasks) +console.error( + `\nNO-SEARCH: resolve=${(100 * noSearch.resolve).toFixed(0)}% contracts-mean=${(100 * noSearch.scoreMean).toFixed(0)}% $${noSearch.usd.toFixed(2)}`, +) + +if (MODE === 'cal') { + console.error(`\n=== CALIBRATION GATE (pre-registered) ===`) + if (noSearch.resolve >= 0.4) { + console.error( + `FAIL: no-search resolve ${(100 * noSearch.resolve).toFixed(0)}% ≥ 40% — task TOO EASY, no room for search to win. HARDEN it (more/harder contracts) before the paired run. Do NOT report a search delta from this.`, + ) + } else if (noSearch.scoreMean >= 0.98) { + console.error( + `FAIL: contracts-mean ${(100 * noSearch.scoreMean).toFixed(0)}% — aces the contracts without search. Harden.`, + ) + } else { + console.error( + `PASS: no-search resolve ${(100 * noSearch.resolve).toFixed(0)}% < 40% (contracts-mean ${(100 * noSearch.scoreMean).toFixed(0)}%) — middle band, room for search. Run ARMS=full for the paired verdict.`, + ) + } + process.exit(0) +} + +const search = await runArm(true, tasks) +console.error( + `\n+SEARCH: resolve=${(100 * search.resolve).toFixed(0)}% contracts-mean=${(100 * search.scoreMean).toFixed(0)}% fired=${search.fired}/${N} $${search.usd.toFixed(2)}`, +) + +// paired on tasks VALID in BOTH arms only (drop empty-run flukes so the pairing is honest) +const both = tasks.map((_, i) => i).filter((i) => noSearch.valid[i] && search.valid[i]) +const nsT = both.map((i) => noSearch.perTask[i]) +const seT = both.map((i) => search.perTask[i]) +const nsS = both.map((i) => noSearch.perScore[i]) +const seS = both.map((i) => search.perScore[i]) +const ci = pairedDeltaCI(seT, nsT) +const sci = pairedDeltaCI(seS, nsS) +console.error(`\n=== VERDICT (pre-registered) ===`) +console.error(`paired tasks valid in both arms: ${both.length}/${N}`) +if (search.fired === 0) { + console.error( + `INVALID: search arm never called web_search (0/${N} fired) — not a tie, a broken arm.`, + ) + process.exit(0) +} +if (both.length < 5) { + console.error( + `INVALID: only ${both.length} both-valid tasks — too few for a verdict (raise N or fix empty-run rate).`, + ) + process.exit(0) +} +console.error( + `no-search resolve ${((100 * nsT.reduce((a, b) => a + b, 0)) / nsT.length).toFixed(0)}% score ${((100 * nsS.reduce((a, b) => a + b, 0)) / nsS.length).toFixed(0)}% | +search resolve ${((100 * seT.reduce((a, b) => a + b, 0)) / seT.length).toFixed(0)}% score ${((100 * seS.reduce((a, b) => a + b, 0)) / seS.length).toFixed(0)}%`, +) +console.error( + `resolve delta (search−nosearch): ${(100 * ci.delta).toFixed(0)}pt 95% CI [${(100 * ci.lo).toFixed(0)}, ${(100 * ci.hi).toFixed(0)}]`, +) +console.error( + `contract-mean delta: ${(100 * sci.delta).toFixed(0)}pt 95% CI [${(100 * sci.lo).toFixed(0)}, ${(100 * sci.hi).toFixed(0)}]`, +) +console.error( + ci.lo > 0 || sci.lo > 0 + ? `SEARCH HELPS — CI excludes 0.` + : `NO significant search benefit (CI includes 0). Honest null at n=${N}.`, +) diff --git a/examples/agentic-data-creation/agentic-data-creation.ts b/examples/agentic-data-creation/agentic-data-creation.ts index 46821925..1f171a4c 100644 --- a/examples/agentic-data-creation/agentic-data-creation.ts +++ b/examples/agentic-data-creation/agentic-data-creation.ts @@ -219,24 +219,34 @@ async function sampleSolverScore(args: { decide: () => 'done', } - const result = await runLoop({ - driver: fanout, - agentRun: solverSpec, - output: solverOutput, - validator, - task: { example, sampleIndex: 0 }, - ctx: { sandboxClient: solver, signal: args.signal }, - maxIterations: samples, - maxConcurrency: samples, - }) - - ledger.record({ - model: solverSpec.profile.name ?? channel, + const model = solverSpec.profile.model?.default ?? solverSpec.profile.name ?? channel + const paid = await ledger.runPaidCall({ channel, - usage: { inputTokens: result.tokenUsage.input, outputTokens: result.tokenUsage.output }, - actualCostUsd: result.costUsd, + phase: 'solver-sampling', + actor: `${channel}/sample-x${samples}`, + model, tags: { role: channel }, + signal: args.signal, + execute: (signal) => + runLoop({ + driver: fanout, + agentRun: solverSpec, + output: solverOutput, + validator, + task: { example, sampleIndex: 0 }, + ctx: { sandboxClient: solver, signal }, + maxIterations: samples, + maxConcurrency: samples, + }), + receipt: (result) => ({ + model, + inputTokens: result.tokenUsage.input, + outputTokens: result.tokenUsage.output, + ...(result.costUsd > 0 ? { actualCostUsd: result.costUsd } : {}), + }), }) + if (!paid.succeeded) throw paid.error + const result = paid.value const scored = result.iterations.filter((it) => it.verdict).map((it) => it.verdict?.score ?? 0) if (scored.length === 0) @@ -415,23 +425,34 @@ export async function createDataCreationLoop( }, } - const result = await runLoop({ - driver: challengerDriver(maxRetries, config.baseInstruction), - agentRun: challengerSpec, - output: challengerOutput, - validator, - task: { doc: config.doc, prompt: config.baseInstruction(config.doc) }, - ctx: { sandboxClient: config.challenger, signal: config.signal }, - maxIterations: maxRetries + 1, - }) - - cost.record({ - model: challengerSpec.profile.name ?? 'challenger', + const challengerModel = + challengerSpec.profile.model?.default ?? challengerSpec.profile.name ?? 'challenger' + const paid = await cost.runPaidCall({ channel: 'challenger', - usage: { inputTokens: result.tokenUsage.input, outputTokens: result.tokenUsage.output }, - actualCostUsd: result.costUsd, + phase: 'example-generation', + actor: 'challenger', + model: challengerModel, tags: { role: 'challenger' }, + signal: config.signal, + execute: (signal) => + runLoop({ + driver: challengerDriver(maxRetries, config.baseInstruction), + agentRun: challengerSpec, + output: challengerOutput, + validator, + task: { doc: config.doc, prompt: config.baseInstruction(config.doc) }, + ctx: { sandboxClient: config.challenger, signal }, + maxIterations: maxRetries + 1, + }), + receipt: (result) => ({ + model: challengerModel, + inputTokens: result.tokenUsage.input, + outputTokens: result.tokenUsage.output, + ...(result.costUsd > 0 ? { actualCostUsd: result.costUsd } : {}), + }), }) + if (!paid.succeeded) throw paid.error + const result = paid.value const plain = evaluations.get(0) if (plain) plainGaps.push(plain.gap) diff --git a/examples/agents-of-all-shapes/python-agno/agno_to_intelligence.py b/examples/agents-of-all-shapes/python-agno/agno_to_intelligence.py index d38bab52..c7f93a5a 100644 --- a/examples/agents-of-all-shapes/python-agno/agno_to_intelligence.py +++ b/examples/agents-of-all-shapes/python-agno/agno_to_intelligence.py @@ -19,8 +19,9 @@ import time import urllib.request -INTELLIGENCE_BASE = os.environ.get( - "INTELLIGENCE_BASE", "https://intelligence.tangle.tools/v1/otlp" +INTELLIGENCE_BASE = ( + os.environ.get("TANGLE_INTELLIGENCE_URL", "https://intelligence.tangle.tools").rstrip("/") + + "/v1/otlp" ) API_KEY = os.environ.get("TANGLE_API_KEY", "sk-tan-...") diff --git a/examples/agents-of-all-shapes/run.ts b/examples/agents-of-all-shapes/run.ts index 379af806..72e8fbea 100644 --- a/examples/agents-of-all-shapes/run.ts +++ b/examples/agents-of-all-shapes/run.ts @@ -9,7 +9,7 @@ * (`fromOtelSpans → analyzeRuns`). Prints the fleet `InsightReport` plus a * per-shape breakdown. * - * Optional hosted path: set TANGLE_API_KEY (and INTELLIGENCE_BASE) to also + * Optional hosted path: set TANGLE_API_KEY (and TANGLE_INTELLIGENCE_URL) to also * POST the spans to the hosted OTLP ingest for the dashboard. */ @@ -45,7 +45,8 @@ async function main() { // Optional: also ship to the hosted ingest for the dashboard. const apiKey = process.env.TANGLE_API_KEY if (apiKey) { - const endpoint = process.env.INTELLIGENCE_BASE ?? 'https://intelligence.tangle.tools/v1/otlp' + const base = process.env.TANGLE_INTELLIGENCE_URL ?? 'https://intelligence.tangle.tools' + const endpoint = `${base.replace(/\/+$/, '')}/v1/otlp` await shipToTangleOtlp(allSpans, { endpoint, apiKey }) console.log(`\nShipped ${allSpans.length} spans to ${endpoint} for the dashboard.`) } else { diff --git a/examples/agents-of-all-shapes/shapes.ts b/examples/agents-of-all-shapes/shapes.ts index 5eb2f039..5fe7682d 100644 --- a/examples/agents-of-all-shapes/shapes.ts +++ b/examples/agents-of-all-shapes/shapes.ts @@ -99,10 +99,10 @@ export function openAiCompatibleRuns(): AgentRun[] { * 3. Mastra agent (TypeScript). * * LIVE: Mastra emits OpenTelemetry natively. Configure its OTLP exporter to - * point at `${INTELLIGENCE_BASE}/v1/otlp` (hosted) OR collect the spans and + * point at `${TANGLE_INTELLIGENCE_URL}/v1/otlp` (hosted) OR collect the spans and * call `toInsightReport` in-process: * export const mastra = new Mastra({ telemetry: { enabled: true, - * export: { type: 'otlp', endpoint: `${INTELLIGENCE_BASE}/v1/otlp/v1/traces` } } }) + * export: { type: 'otlp', endpoint: `${TANGLE_INTELLIGENCE_URL}/v1/otlp/v1/traces` } } }) * Add a `score` attribute from your eval step. No Tangle SDK required. */ export function mastraRuns(): AgentRun[] { diff --git a/examples/coding-benchmark/dispatch.ts b/examples/coding-benchmark/dispatch.ts index 62432978..c6db033d 100644 --- a/examples/coding-benchmark/dispatch.ts +++ b/examples/coding-benchmark/dispatch.ts @@ -96,6 +96,7 @@ export function codingDispatch( // Author the tool surface onto the profile (one line). The substrate // materializes it into the harness's real config. const equippedProfile = withTools(profile, toolPreset) + const model = equippedProfile.model?.default ?? 'unknown' const cmds = checkCmds(scenario) // Route the scenario's fields by destination (agent-visible / develop-against / // grading-only / judge-only). The firewall reads these tags, not field names. @@ -136,18 +137,27 @@ export function codingDispatch( // visible test `develop-against`, so seeding it never trips this. agentContextParts.push(prompt) assertNoHiddenLeak(routedFields, agentContextParts.join('\n')) - const turn = round === 0 ? await run.start(prompt) : await run.resume(prompt) + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'sandbox-cell', + model, + signal: ctx.signal, + execute: () => (round === 0 ? run.start(prompt) : run.resume(prompt)), + receipt: (turn) => { + const usage = sumSandboxUsage(turn.events) + return { + model, + inputTokens: usage.input, + outputTokens: usage.output, + ...(usage.costUsd > 0 ? { actualCostUsd: usage.costUsd } : {}), + } + }, + }) + if (!paid.succeeded) throw paid.error + const turn = paid.value solution = turn.out.solution finalText = turn.events.map(eventText).filter(Boolean).join(' ').slice(0, 2000) - // Report usage so the integrity guard sees a real backend (not a stub). - // `extractLlmCallEvent` reads usage off EVERY backend event shape — the live - // sandbox's `done`/`result`/`llm_call` events all sum correctly here. - const usage = sumSandboxUsage(turn.events) - if (usage.input || usage.output) - ctx.cost.observeTokens({ input: usage.input, output: usage.output }) - if (usage.costUsd) ctx.cost.observe(usage.costUsd, 'sandbox-cell') - // Dev checks (visible example tests), IN THE BOX, this round. These (and only // these) steer the next round — the firewall keeps the held-out suite + rubric // out of the loop. `run.box` is a `SandboxInstance`; `CheckBox` is the minimal diff --git a/examples/improve/improve.ts b/examples/improve/improve.ts index 5f9d8f17..6647165f 100644 --- a/examples/improve/improve.ts +++ b/examples/improve/improve.ts @@ -46,9 +46,21 @@ export const agent = async ( _scenario: DemoScenario, ctx: DispatchContext, ): Promise => { - ctx.cost.observe(0.0001, 'example') - ctx.cost.observeTokens({ input: 1, output: 1 }) - return String(surface) + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'example', + model: 'deterministic-example', + maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, + execute: async () => String(surface), + receipt: () => ({ + model: 'deterministic-example', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.0001, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value } // Deterministic judge: the literal string `PROMOTED` scores 1.0, anything else 0.0 — no LLM. diff --git a/examples/intelligence-drop-in/README.md b/examples/intelligence-drop-in/README.md index 6d6f1d05..0434d354 100644 --- a/examples/intelligence-drop-in/README.md +++ b/examples/intelligence-drop-in/README.md @@ -3,8 +3,8 @@ Wrap any agent function in one line and every call ships a trace (a per-call record of what ran and what it cost) to Tangle Intelligence. Two things make this worth a look: -1. **It's genuinely one line** — `withTangleIntelligence(agent, opts)` preserves your agent's exact - call shape and adds tracing invisibly. +1. **It's genuinely one hook** — `withIntelligence(agent, opts)` wraps your agent, ships a RunRecord + per call, and receives the tenant's certified profile (fail-closed) — all invisibly. 2. **It's safe** — the tracing is best-effort. If Tangle Intelligence is down, your agent still answers. Telemetry never takes down the app. 3. **"Off" really means free** — turn intelligence off and you're billed only for the model call you'd @@ -36,9 +36,9 @@ and fails loudly if it isn't `0`. You get a number you can check, not a label yo ## The two APIs -- **`withTangleIntelligence(agent, { project, apiKey, endpoint })`** — the one-liner. Wrap any - `(input) => Promise` agent; each call fires off one trace in the background and returns as - soon as your agent does. +- **`withIntelligence(agent, { project, target, apiKey, baseUrl })`** — the one hook. Wrap any + `(input, applied) => Promise` agent; each call fires off one RunRecord in the background, + receives the certified profile, and returns as soon as your agent does. - **`createIntelligenceClient(...).traceRun(meta, fn)`** — the explicit version, for when you want to record extra detail and flush traces on demand. Used here so the example can flush and read the trace back to prove the zero. diff --git a/examples/intelligence-drop-in/intelligence-drop-in.ts b/examples/intelligence-drop-in/intelligence-drop-in.ts index 2db781c2..87b0eb72 100644 --- a/examples/intelligence-drop-in/intelligence-drop-in.ts +++ b/examples/intelligence-drop-in/intelligence-drop-in.ts @@ -17,7 +17,7 @@ import { createServer } from 'node:http' import { createIntelligenceClient, - withTangleIntelligence, + withIntelligence, } from '@tangle-network/agent-runtime/intelligence' async function supportAgent(input: { question: string }): Promise<{ answer: string }> { @@ -65,23 +65,25 @@ async function main() { }) await new Promise((r) => server.listen(0, r)) const port = (server.address() as { port: number }).port - const endpoint = `http://127.0.0.1:${port}/v1/otlp` + const baseUrl = `http://127.0.0.1:${port}` + const apiKey = process.env.TANGLE_API_KEY ?? 'sk-tan-demo' const project = 'support-agent' - // 1) THE ERGONOMICS — wrap any `(input) => Promise` in one line; it ships a trace best-effort - // (fire-and-forget: the call returns as soon as the agent does, the span flushes in the background). - const wrapped = withTangleIntelligence(supportAgent, { + // 1) THE ERGONOMICS — wrap any agent in one line; it receives the tenant's certified profile (fail-closed) + // and ships a RunRecord best-effort (the call returns as soon as the agent does, the span flushes async). + const wrapped = withIntelligence(async (input: { question: string }) => supportAgent(input), { project, - apiKey: process.env.TANGLE_API_KEY ?? 'sk-tan-demo', - endpoint, + apiKey, + baseUrl, }) const out = await wrapped({ question: 'how do I reset my password?' }) console.log(`1) wrapped an agent, got its answer: ${JSON.stringify(out.answer).slice(0, 48)}…`) - // 2) The agent still answers when the trace endpoint is dead — best-effort telemetry never breaks the app. - const onDead = withTangleIntelligence(supportAgent, { + // 2) The agent still answers when Intelligence is dead — best-effort telemetry never breaks the app. + const onDead = withIntelligence(async (input: { question: string }) => supportAgent(input), { project, - endpoint: 'http://127.0.0.1:1/v1/otlp', // nothing listening + apiKey, + baseUrl: 'http://127.0.0.1:1', // nothing listening }) const stillWorks = await onDead({ question: 'is intelligence down?' }) console.log(`2) survived a dead endpoint: ${stillWorks.answer.length > 0}`) @@ -91,7 +93,7 @@ async function main() { // intelligence spawn, so only the base inference stream costs anything (we record just that); the // span's intelligence_usd is 0 by construction — a `standard`-tier run would fill it itself. received.length = 0 - const off = createIntelligenceClient({ project, endpoint, effort: 'off' }) + const off = createIntelligenceClient({ project, apiKey, baseUrl, effort: 'off' }) await off.traceRun({ input: { q: 'cheap turn' } }, async (trace) => { trace.recordOutcome({ usage: { inferenceUsd: 0.0008 } }) return supportAgent({ question: 'cheap turn' }) diff --git a/examples/intelligence-recommend/README.md b/examples/intelligence-recommend/README.md index 9bc4919f..54f3b9cd 100644 --- a/examples/intelligence-recommend/README.md +++ b/examples/intelligence-recommend/README.md @@ -54,7 +54,7 @@ prompt after: ## Going live -To run this for real: point `createIntelligenceClient` at an OTLP trace collector (set -`INTELLIGENCE_OTLP_ENDPOINT`) so traces actually export; replace the two hand-written findings with a +To run this for real: give `createIntelligenceClient` a tenant `apiKey` (and, if not the prod plane, a +`baseUrl` / `TANGLE_INTELLIGENCE_URL`) so traces actually export; replace the two hand-written findings with a real trace-analyst agent's output; and drop the scripted proposer for a model-backed one (pass an `llm` instead of a `generator`) so the rewrite is reflected from the findings rather than pre-canned. diff --git a/examples/intelligence-recommend/intelligence-recommend.ts b/examples/intelligence-recommend/intelligence-recommend.ts index f1e55cbf..9738000a 100644 --- a/examples/intelligence-recommend/intelligence-recommend.ts +++ b/examples/intelligence-recommend/intelligence-recommend.ts @@ -8,7 +8,7 @@ * improve.ts verbatim, so this file shows only the NEW seam a Recommend mode runs in production: * * 1. OBSERVE — record a run's `LoopTraceEvent` stream as one trace (best-effort export; with no - * OTLP endpoint configured it is a no-op, so this runs offline with no credentials). + * tenant apiKey configured it is a no-op, so this runs offline with no credentials). * 2. ANALYZE — derive `AnalystFinding`s from that trace. In production a trace analyst reads the * spans; here we hand-derive two findings that CITE the recorded trace, to keep it offline. * 3. IMPROVE — feed those findings to `improve()` and print the gated candidate. @@ -24,7 +24,7 @@ import { agent, judge, profile, scenarios, scriptedWinner } from '../improve/imp // ── 1. OBSERVE — record a run's loop topology as one trace ────────────────── // A real run emits this `LoopTraceEvent` stream from the kernel; here a tiny two-event trace stands -// in. With no OTLP endpoint set, export is a no-op — offline, no credentials. +// in. With no tenant apiKey set, export is a no-op — offline, no credentials. const runId = 'run-001' const events: LoopTraceEvent[] = [ { diff --git a/examples/intelligence-webcode/README.md b/examples/intelligence-webcode/README.md index 3034fac7..e9800618 100644 --- a/examples/intelligence-webcode/README.md +++ b/examples/intelligence-webcode/README.md @@ -20,7 +20,7 @@ Three layers, all wrapped around each single benchmark cell (`intelligence-webco | Layer | What it does | |---|---| -| **Boundary** | `withTangleIntelligence(cell, { effort })` wraps the whole run. `effort` ranges `off · eco · standard · thorough · max`; `off` is the provable floor — instrumentation spend clamped to 0, the task still runs. | +| **Boundary** | `withIntelligence(cell, { effort })` wraps the whole run. `effort` ranges `off · eco · standard · thorough · max`; `off` is the provable floor — instrumentation spend clamped to 0, the task still runs. | | **Cost waterfall** | `createWaterfallCollector()` records cost per tool/phase during the run. The sum of its parts **is** the billed total, so there's no second tally that can drift from reality. | | **Trace export** | `createOtelExporter()` streams every step to your OTLP trace collector. It's a no-op until you set `OTEL_EXPORTER_OTLP_ENDPOINT`, so you can run without one. | diff --git a/examples/intelligence-webcode/intelligence-webcode.ts b/examples/intelligence-webcode/intelligence-webcode.ts index c00be403..44b41c5d 100644 --- a/examples/intelligence-webcode/intelligence-webcode.ts +++ b/examples/intelligence-webcode/intelligence-webcode.ts @@ -6,7 +6,7 @@ * waterfall, and the spans streamed to your trace collector. * * Three intelligence layers, all on one cell: - * 1. BOUNDARY (the bill + the control) — `withTangleIntelligence(cell, { project, effort })`. + * 1. BOUNDARY (the bill + the control) — `withIntelligence(cell, { project, effort })`. * `effort ∈ off | eco | standard | thorough | max`; `'off'` is the PROVABLE passthrough floor — * intelligence spend clamped to 0, the cell still runs. This is the one knob that gates spend. * 2. WATERFALL (the cost truth) — `createWaterfallCollector()` on the run. The sum of its spans IS the @@ -14,7 +14,7 @@ * 3. OTLP (the production trace pipe) — `createOtelExporter()` + `loopEventToOtelSpan`. Streams every * span to your OTLP/HTTP collector (set `OTEL_EXPORTER_OTLP_ENDPOINT`); a no-op when unset. * - * The intelligence attaches at TWO seams: the BOUNDARY wraps the whole cell (`withTangleIntelligence` + * The intelligence attaches at TWO seams: the BOUNDARY wraps the whole cell (`withIntelligence` * works over any async fn), and the INTERNAL trace rides `openSandboxRun`'s `hooks` (the only run verb * here that emits per-tool spans). Same pattern instruments `runProfileMatrix`'s dispatch wholesale. * @@ -30,7 +30,7 @@ import { loopEventToOtelSpan, type RuntimeHooks, } from '@tangle-network/agent-runtime' -import { type EffortTier, withTangleIntelligence } from '@tangle-network/agent-runtime/intelligence' +import { type EffortTier, withIntelligence } from '@tangle-network/agent-runtime/intelligence' import { type AgentRunSpec, createWaterfallCollector, @@ -62,7 +62,7 @@ interface CellResult { /** One instrumented cell: run a harness×model on a WebCode task in its own sandbox with the INTERNAL trace * collected (cost waterfall) AND streamed (OTLP), then score on the hidden tests. Wrapping this in - * `withTangleIntelligence` (below) adds the BOUNDARY layer. */ + * `withIntelligence` (below) adds the BOUNDARY layer. */ function instrumentedCell(client: SandboxClient): (input: CellInput) => Promise { return async ({ profile, task }) => { const harness = String(profile.metadata?.harness ?? 'opencode') @@ -145,9 +145,9 @@ function instrumentedCell(client: SandboxClient): (input: CellInput) => Promise< /** Run the WebCode grid × tasks with the full intelligence stack on every cell. */ export async function runIntelligenceWebcode(client: SandboxClient): Promise { - // LAYER 1 — the BOUNDARY: every cell runs under `withTangleIntelligence` — traced + billed, effort-gated. + // LAYER 1 — the BOUNDARY: every cell runs under `withIntelligence` — observed + billed, effort-gated. // `effort: 'off'` clamps intelligence spend to 0 (the provable passthrough floor) while still running. - const smartCell = withTangleIntelligence(instrumentedCell(client), { project, effort }) + const smartCell = withIntelligence(instrumentedCell(client), { project, effort }) const webcodeTasks = loadWebCodeTasks( process.env.LIMIT ? { limit: Number(process.env.LIMIT) } : {}, ) diff --git a/examples/webcode-matrix/webcode-matrix.ts b/examples/webcode-matrix/webcode-matrix.ts index 89cd073d..be324576 100644 --- a/examples/webcode-matrix/webcode-matrix.ts +++ b/examples/webcode-matrix/webcode-matrix.ts @@ -106,13 +106,23 @@ function webcodeDispatch( { agentRun, scenarioId: task.id, signal: ctx.signal }, { kind: 'events', fromEvents: () => ({ passed: false }) }, ) - const turn = await run.start(prompt) - // Report the run's real usage so the backend-integrity guard sees a real backend (not a stub). The - // ONE metering seam — sums usage across every backend event shape. - const usage = sumSandboxUsage(turn.events) - if (usage.input || usage.output) - ctx.cost.observeTokens({ input: usage.input, output: usage.output }) - if (usage.costUsd) ctx.cost.observe(usage.costUsd, 'webcode-cell') + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'webcode-cell', + model, + signal: ctx.signal, + execute: () => run.start(prompt), + receipt: (turn) => { + const usage = sumSandboxUsage(turn.events) + return { + model, + inputTokens: usage.input, + outputTokens: usage.output, + ...(usage.costUsd > 0 ? { actualCostUsd: usage.costUsd } : {}), + } + }, + }) + if (!paid.succeeded) throw paid.error // Grade with Exa's EXACT test_patch: drop it into the box, ensure pytest, run it, score on exit. A // missing language toolchain (an exotic per-task image not provisioned) surfaces as a failing test — diff --git a/package.json b/package.json index 07173544..262d4407 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.89.0", + "version": "0.94.11", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -64,6 +64,16 @@ "import": "./dist/platform.js", "default": "./dist/platform.js" }, + "./primeintellect": { + "types": "./dist/primeintellect/index.d.ts", + "import": "./dist/primeintellect/index.js", + "default": "./dist/primeintellect/index.js" + }, + "./candidate-execution": { + "types": "./dist/candidate-execution/index.d.ts", + "import": "./dist/candidate-execution/index.js", + "default": "./dist/candidate-execution/index.js" + }, "./mcp": { "types": "./dist/mcp/index.d.ts", "import": "./dist/mcp/index.js", @@ -93,16 +103,18 @@ "typecheck": "tsc --noEmit && pnpm run typecheck:examples", "typecheck:examples": "tsc --noEmit -p tsconfig.examples.json", "verify:package": "node scripts/verify-package-exports.mjs", + "verify:primeintellect": "pnpm build && node scripts/verify-primeintellect-v1.mjs", "docs:api": "typedoc && node scripts/gen-primitive-catalog.mjs", "docs:freshness": "node scripts/check-docs-freshness.mjs", "docs:check": "pnpm run docs:api && git diff --exit-code -- docs/api && pnpm run docs:freshness" }, "devDependencies": { "@biomejs/biome": "^2.4.15", - "@tangle-network/agent-eval": "^0.108.1", - "@tangle-network/agent-interface": "^0.19.0", + "@tangle-network/agent-eval": "0.117.1", + "@tangle-network/agent-interface": "^0.25.0", "@tangle-network/sandbox": "^0.9.7", "@types/node": "^25.9.3", + "@types/tar-stream": "3.1.4", "playwright": "^1.61.0", "tsup": "^8.0.0", "tsx": "^4.22.4", @@ -116,6 +128,7 @@ "minimumReleaseAgeExclude": [ "@tangle-network/agent-eval", "@tangle-network/agent-interface", + "@tangle-network/agent-profile-materialize", "@tangle-network/sandbox" ], "onlyBuiltDependencies": [ @@ -128,15 +141,12 @@ "license": "MIT", "packageManager": "pnpm@10.28.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.101.0 <1.0.0", - "@tangle-network/agent-interface": ">=0.14.0 <1.0.0", + "@tangle-network/agent-eval": ">=0.117.1 <0.118.0", + "@tangle-network/agent-interface": ">=0.25.0 <0.26.0", "@tangle-network/sandbox": ">=0.8.0 <1.0.0", "playwright": "^1.40.0" }, "peerDependenciesMeta": { - "@tangle-network/agent-interface": { - "optional": true - }, "@tangle-network/sandbox": { "optional": true }, @@ -145,6 +155,8 @@ } }, "dependencies": { - "@tangle-network/agent-knowledge": "^1.11.1" + "@tangle-network/agent-knowledge": "^1.12.1", + "@tangle-network/agent-profile-materialize": "0.3.2", + "tar-stream": "3.2.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bf4d276a..18e1045c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,24 +9,33 @@ importers: .: dependencies: '@tangle-network/agent-knowledge': - specifier: ^1.11.1 - version: 1.11.1(typescript@5.9.3) + specifier: ^1.12.1 + version: 1.12.1(typescript@5.9.3) + '@tangle-network/agent-profile-materialize': + specifier: 0.3.2 + version: 0.3.2 + tar-stream: + specifier: 3.2.0 + version: 3.2.0 devDependencies: '@biomejs/biome': specifier: ^2.4.15 version: 2.4.15 '@tangle-network/agent-eval': - specifier: ^0.108.1 - version: 0.108.1(typescript@5.9.3) + specifier: 0.117.1 + version: 0.117.1(typescript@5.9.3) '@tangle-network/agent-interface': - specifier: ^0.19.0 - version: 0.19.0 + specifier: ^0.25.0 + version: 0.25.0 '@tangle-network/sandbox': specifier: ^0.9.7 version: 0.9.7(viem@2.54.6(typescript@5.9.3)(zod@4.4.3)) '@types/node': specifier: ^25.9.3 version: 25.9.3 + '@types/tar-stream': + specifier: 3.1.4 + version: 3.1.4 playwright: specifier: ^1.61.0 version: 1.61.0 @@ -640,13 +649,15 @@ packages: '@tangle-network/agent-core@0.3.8': resolution: {integrity: sha512-bZfVpdiFjXbcQwSxSQABdtXEmUuapWChCcXrHkP6fAnwqEy9hWEfeZLlMZUy+1XaOfpTaMLUOEAwYhXfN018wQ==} - '@tangle-network/agent-eval@0.108.1': - resolution: {integrity: sha512-5T6XKtqcxB8OrR1c8V4EGZk+gQizxmVVQKLCkxTlnMh2lApV+e0EIbEUpKZzD9HmCQSgRY8a2tW3hCTFBuZAyg==} + '@tangle-network/agent-eval@0.115.1': + resolution: {integrity: sha512-Yd487u8v0c8/+On9kaYGASS88oLhOJ5p0zwRMGdbWRSWC1UODbIgk1Fqdyq6EI60v+lZ+yDg/7LHkIrNi4kjPA==} engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-interface@0.10.1': - resolution: {integrity: sha512-yehY/0EgKvu8lG6jIVoZCtMPLkj8VEWwasuAtuph2RaB9MKE5wuxRF647O6jw8KufNZ3aQ2UVVWpZ19dGCbs6w==} + '@tangle-network/agent-eval@0.117.1': + resolution: {integrity: sha512-mtBdGhGIC+nrPZbseLQo0Iako1LR66ZeIr2UjVBJcNj/lHYfib2jbhdWy3RmbQcHR0Cxdqqydn7pLJ5dSXVYtQ==} + engines: {node: '>=20'} + hasBin: true '@tangle-network/agent-interface@0.13.0': resolution: {integrity: sha512-CeTPGRLoXqpt0h+BCyFgZPkfU1zyRpWmqfD+85i/uk+uvbqxkfI+JprfKVf3tBsQuCgJPSjPt5qjdW8n3h2BVg==} @@ -654,14 +665,20 @@ packages: '@tangle-network/agent-interface@0.17.1': resolution: {integrity: sha512-B7dRJTo0HSUtgBCB1VMwkTFYkLUaRr/4BcRglrQuGhGUwOzKv1RYyMejOVh5M3a5AagY9N79f7GYbjcA3UmnIA==} - '@tangle-network/agent-interface@0.19.0': - resolution: {integrity: sha512-WCN1j9dVmPlCDAU9I1R6OvGdZ9uvKrxz+3gsh61DbKoLvnxxRQEUWuZ/JByzSZGq4J84zdzryr6aET7KTZ6z+w==} + '@tangle-network/agent-interface@0.22.0': + resolution: {integrity: sha512-7fsJhNdvTmOB1X9E2owl06jzyrqaN+jMkOPVKbK7dvNqQvf627PowCtL/edhUqEEv+K0FWtkwG3R3xqjX7VNZg==} + + '@tangle-network/agent-interface@0.25.0': + resolution: {integrity: sha512-bMKjyjk8bk2651HleiTEUOTPgNE2bWxCEl9VBVUyub/UkP87Y09byXOxeDEsNGc4TIT19Y5Uo0b+k1kRWXkMVQ==} - '@tangle-network/agent-knowledge@1.11.1': - resolution: {integrity: sha512-2vvNHHsb4TQFnY+SrL7rYaMhwTxTK8R/lD9IAteonmgwcE5amhpPQZqftw+UDMl1d6OBe4QX+Qv1pNmWQDXzqQ==} + '@tangle-network/agent-knowledge@1.12.1': + resolution: {integrity: sha512-j5riyIvz5C+ZBELTtNVtmUbcKAMpgTHMNmDQ12MhOIPJv5Y7AeX2oCWuF6c8aPqg1hOrsLqgqXSr1zkV0ePsrA==} engines: {node: '>=20'} hasBin: true + '@tangle-network/agent-profile-materialize@0.3.2': + resolution: {integrity: sha512-jCj1Hc/brPQ5eS7or9A+Klv0FAZcGtyFr7kx2cKfk+wCj0/4Di3k1pMjayrmRNgI/7Oc47gt6/t+qvdtZxBkAw==} + '@tangle-network/sandbox@0.9.7': resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} peerDependencies: @@ -706,6 +723,9 @@ packages: '@types/node@25.9.3': resolution: {integrity: sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg==} + '@types/tar-stream@3.1.4': + resolution: {integrity: sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==} + '@types/unist@3.0.3': resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} @@ -764,10 +784,55 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + b4a@1.8.1: + resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + balanced-match@4.0.4: resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} engines: {node: 18 || 20 || >=22} + bare-events@2.9.1: + resolution: {integrity: sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==} + peerDependencies: + bare-abort-controller: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + + bare-fs@4.7.4: + resolution: {integrity: sha512-y1kC+ffIx/tPLdTE693uNjHfzTfr+ravR5tvWlMXe25nELbkqV400S71qHDwbkAQ1FVEZobB1NFRzFbCCcyBCQ==} + engines: {bare: '>=1.16.0'} + peerDependencies: + bare-buffer: '*' + peerDependenciesMeta: + bare-buffer: + optional: true + + bare-path@3.1.1: + resolution: {integrity: sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==} + + bare-stream@2.13.3: + resolution: {integrity: sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==} + peerDependencies: + bare-abort-controller: '*' + bare-buffer: '*' + bare-events: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + bare-buffer: + optional: true + bare-events: + optional: true + + bare-url@2.4.5: + resolution: {integrity: sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==} + brace-expansion@5.0.6: resolution: {integrity: sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==} engines: {node: 18 || 20 || >=22} @@ -848,10 +913,16 @@ packages: eventemitter3@5.0.1: resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -874,8 +945,8 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] - hono@4.12.28: - resolution: {integrity: sha512-YwUvVpSF7m1yOblFPrU3Hbo8XhPheBoiyfGuII6z19LnOr6JpDnyyp7LFNrfV56wS8tpvtBFGRISHN02pDdLOA==} + hono@4.12.30: + resolution: {integrity: sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==} engines: {node: '>=16.9.0'} isows@1.0.7: @@ -1040,6 +1111,9 @@ packages: std-env@3.10.0: resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + streamx@2.28.0: + resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==} + strip-literal@3.1.0: resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} @@ -1048,6 +1122,15 @@ packages: engines: {node: '>=16 || 14 >=14.17'} hasBin: true + tar-stream@3.2.0: + resolution: {integrity: sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==} + + teex@1.0.1: + resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==} + + text-decoder@1.2.7: + resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} + thenify-all@1.6.0: resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} engines: {node: '>=0.8'} @@ -1456,9 +1539,9 @@ snapshots: '@shikijs/types': 3.23.0 '@shikijs/vscode-textmate': 10.0.2 - '@hono/node-server@2.0.8(hono@4.12.28)': + '@hono/node-server@2.0.8(hono@4.12.30)': dependencies: - hono: 4.12.28 + hono: 4.12.30 '@jridgewell/gen-mapping@0.3.13': dependencies: @@ -1616,14 +1699,14 @@ snapshots: '@tangle-network/agent-interface': 0.17.1 zod: 4.4.3 - '@tangle-network/agent-eval@0.108.1(typescript@5.9.3)': + '@tangle-network/agent-eval@0.115.1(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 19.0.45(zod@4.4.3) - '@hono/node-server': 2.0.8(hono@4.12.28) - '@tangle-network/agent-interface': 0.10.1 + '@hono/node-server': 2.0.8(hono@4.12.30) + '@tangle-network/agent-interface': 0.22.0 '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) - hono: 4.12.28 + hono: 4.12.30 zod: 4.4.3 transitivePeerDependencies: - '@mastra/core' @@ -1634,9 +1717,23 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-interface@0.10.1': + '@tangle-network/agent-eval@0.117.1(typescript@5.9.3)': dependencies: + '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) + '@ax-llm/ax': 19.0.45(zod@4.4.3) + '@hono/node-server': 2.0.8(hono@4.12.30) + '@tangle-network/agent-interface': 0.22.0 + '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) + hono: 4.12.30 zod: 4.4.3 + transitivePeerDependencies: + - '@mastra/core' + - '@modelcontextprotocol/sdk' + - ai + - bufferutil + - openai + - typescript + - utf-8-validate '@tangle-network/agent-interface@0.13.0': dependencies: @@ -1646,13 +1743,17 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.19.0': + '@tangle-network/agent-interface@0.22.0': + dependencies: + zod: 4.4.3 + + '@tangle-network/agent-interface@0.25.0': dependencies: zod: 4.4.3 - '@tangle-network/agent-knowledge@1.11.1(typescript@5.9.3)': + '@tangle-network/agent-knowledge@1.12.1(typescript@5.9.3)': dependencies: - '@tangle-network/agent-eval': 0.108.1(typescript@5.9.3) + '@tangle-network/agent-eval': 0.115.1(typescript@5.9.3) zod: 4.4.3 transitivePeerDependencies: - '@mastra/core' @@ -1663,6 +1764,10 @@ snapshots: - typescript - utf-8-validate + '@tangle-network/agent-profile-materialize@0.3.2': + dependencies: + '@tangle-network/agent-interface': 0.25.0 + '@tangle-network/sandbox@0.9.7(viem@2.54.6(typescript@5.9.3)(zod@4.4.3))': dependencies: '@tangle-network/agent-core': 0.3.8 @@ -1707,6 +1812,10 @@ snapshots: dependencies: undici-types: 7.24.6 + '@types/tar-stream@3.1.4': + dependencies: + '@types/node': 25.9.3 + '@types/unist@3.0.3': {} '@vitest/expect@3.2.4': @@ -1764,8 +1873,39 @@ snapshots: assertion-error@2.0.1: {} + b4a@1.8.1: {} + balanced-match@4.0.4: {} + bare-events@2.9.1: {} + + bare-fs@4.7.4: + dependencies: + bare-events: 2.9.1 + bare-path: 3.1.1 + bare-stream: 2.13.3(bare-events@2.9.1) + bare-url: 2.4.5 + fast-fifo: 1.3.2 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + bare-path@3.1.1: {} + + bare-stream@2.13.3(bare-events@2.9.1): + dependencies: + b4a: 1.8.1 + streamx: 2.28.0 + teex: 1.0.1 + optionalDependencies: + bare-events: 2.9.1 + transitivePeerDependencies: + - react-native-b4a + + bare-url@2.4.5: + dependencies: + bare-path: 3.1.1 + brace-expansion@5.0.6: dependencies: balanced-match: 4.0.4 @@ -1875,8 +2015,16 @@ snapshots: eventemitter3@5.0.1: {} + events-universal@1.0.1: + dependencies: + bare-events: 2.9.1 + transitivePeerDependencies: + - bare-abort-controller + expect-type@1.3.0: {} + fast-fifo@1.3.2: {} + fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 @@ -1893,7 +2041,7 @@ snapshots: fsevents@2.3.3: optional: true - hono@4.12.28: {} + hono@4.12.30: {} isows@1.0.7(ws@8.21.0): dependencies: @@ -2059,6 +2207,15 @@ snapshots: std-env@3.10.0: {} + streamx@2.28.0: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.7 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + strip-literal@3.1.0: dependencies: js-tokens: 9.0.1 @@ -2073,6 +2230,30 @@ snapshots: tinyglobby: 0.2.16 ts-interface-checker: 0.1.13 + tar-stream@3.2.0: + dependencies: + b4a: 1.8.1 + bare-fs: 4.7.4 + fast-fifo: 1.3.2 + streamx: 2.28.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + + teex@1.0.1: + dependencies: + streamx: 2.28.0 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + text-decoder@1.2.7: + dependencies: + b4a: 1.8.1 + transitivePeerDependencies: + - react-native-b4a + thenify-all@1.6.0: dependencies: thenify: 3.3.1 diff --git a/scripts/gen-primitive-catalog.mjs b/scripts/gen-primitive-catalog.mjs index 9516bb28..98dc85b3 100644 --- a/scripts/gen-primitive-catalog.mjs +++ b/scripts/gen-primitive-catalog.mjs @@ -69,8 +69,10 @@ const ownSurfaceLabels = { './analyst-loop': 'Analyst loop — trace findings on a running loop', './lifecycle': 'Artifact lifecycle — generate → measure → promote → compose', './knowledge': 'Knowledge orchestration — supervised KB updates', + './primeintellect': 'PrimeIntellect: Verifiers v1 package and trace adapter', './profiles': 'Built-in agent profiles', './platform': 'Platform glue', + './candidate-execution': 'Candidate execution — immutable prepare, run, grade, and receipt', './mcp': 'MCP servers — delegate / coordination / detached-session', } // ./loops is an intentional alias of ./runtime (same source) — list it once as ./loops, diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index ab2847bb..fff84292 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -1,10 +1,11 @@ import { spawnSync } from 'node:child_process' -import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') -const tempRoot = mkdtempSync(join(repoRoot, '.tmp-package-exports-')) +const tempRoot = mkdtempSync(join(tmpdir(), 'agent-runtime-package-exports-')) try { const packDir = join(tempRoot, 'pack') @@ -12,7 +13,19 @@ try { const appDir = join(tempRoot, 'app') mkdirSync(packDir, { recursive: true }) mkdirSync(unpackDir, { recursive: true }) - mkdirSync(join(appDir, 'node_modules', '@tangle-network'), { recursive: true }) + mkdirSync(appDir, { recursive: true }) + writeFileSync( + join(appDir, 'package.json'), + `${JSON.stringify( + { + name: 'agent-runtime-package-verification', + private: true, + type: 'module', + }, + null, + 2, + )}\n`, + ) run('pnpm', ['pack', '--pack-destination', packDir], repoRoot) const tarballs = run('find', [packDir, '-maxdepth', '1', '-name', '*.tgz', '-print'], repoRoot) @@ -32,9 +45,11 @@ try { const requiredExports = { '.': ['import', 'types'], './agent': ['import', 'types'], + './candidate-execution': ['import', 'types'], './intelligence': ['import', 'types'], './loops': ['import', 'types'], './environment-provider': ['import', 'types'], + './primeintellect': ['import', 'types'], './profiles': ['import', 'types'], './mcp': ['import', 'types'], } @@ -51,7 +66,61 @@ try { } } - symlinkSync(packageDir, join(appDir, 'node_modules', '@tangle-network', 'agent-runtime'), 'dir') + // Install into an empty app so dependency resolution uses only published package metadata. + run( + 'npm', + [ + 'install', + '--ignore-scripts', + '--no-package-lock', + '--no-save', + '--no-audit', + '--no-fund', + tarballs[0], + ], + appDir, + ) + run( + process.execPath, + [ + '--input-type=module', + '--eval', + ` + const { readFileSync } = await import('node:fs') + const packageJson = JSON.parse( + readFileSync('node_modules/@tangle-network/agent-runtime/package.json', 'utf8'), + ) + const subpaths = Object.keys(packageJson.exports).map((subpath) => + subpath === '.' ? packageJson.name : packageJson.name + subpath.slice(1), + ) + for (const subpath of subpaths) await import(subpath) + `, + ], + appDir, + ) + run( + process.execPath, + [ + '--input-type=module', + '--eval', + ` + const prime = await import('@tangle-network/agent-runtime/primeintellect') + for (const name of [ + 'createPrimeIntellectPackage', + 'writePrimeIntellectPackage', + 'readPrimeIntellectEpisodeContext', + 'createPrimeIntellectBackend', + 'runPrimeIntellectProgram', + 'parsePrimeIntellectTraces', + 'primeIntellectTraceToRunRecord', + 'importPrimeIntellectTraces', + ]) { + if (typeof prime[name] !== 'function') throw new Error('missing PrimeIntellect export ' + name) + } + `, + ], + appDir, + ) run( process.execPath, [ @@ -61,7 +130,8 @@ try { const intelligence = await import('@tangle-network/agent-runtime/intelligence') const expectedIntelligence = [ 'createIntelligenceClient', - 'withTangleIntelligence', + 'withIntelligence', + 'pullCertified', 'resolveEffort', 'isIntelligenceOff', 'defaultRedactor', @@ -76,6 +146,51 @@ try { ], appDir, ) + run( + process.execPath, + [ + '--input-type=module', + '--eval', + ` + const candidates = await import('@tangle-network/agent-runtime/candidate-execution') + for (const name of [ + 'buildAgentCandidateBundle', + 'sealAgentCandidateBundle', + 'verifyAgentCandidateBundle', + ]) { + if (typeof candidates[name] !== 'function') throw new Error('missing candidate export ' + name) + } + const bundle = candidates.buildAgentCandidateBundle({ + profile: { + kind: 'profile', + profile: { name: 'packed-consumer', harness: 'codex' }, + }, + code: { kind: 'disabled', reason: 'control' }, + execution: { + harness: 'codex', + harnessVersion: '1.0.0', + launch: { kind: 'container-command', executable: 'codex' }, + instructionDelivery: { kind: 'stdin-utf8' }, + cwd: { workspace: 'task', path: '.' }, + environment: { kind: 'evaluator-task-container' }, + isolation: { + network: 'disabled', + remoteIntegrations: 'disabled', + candidateSecrets: 'disabled', + }, + }, + memory: { mode: 'disabled' }, + lineage: { source: 'human' }, + }) + const verified = await candidates.verifyAgentCandidateBundle(bundle, { + artifacts: { read: async () => { throw new Error('unexpected artifact read') } }, + repositories: { resolve: async () => { throw new Error('unexpected repository read') } }, + }) + if (verified.bundle.digest !== bundle.digest) throw new Error('packed candidate digest drift') + `, + ], + appDir, + ) run( process.execPath, [ diff --git a/scripts/verify-primeintellect-v1.mjs b/scripts/verify-primeintellect-v1.mjs new file mode 100644 index 00000000..9955d191 --- /dev/null +++ b/scripts/verify-primeintellect-v1.mjs @@ -0,0 +1,192 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + createPrimeIntellectPackage, + writePrimeIntellectPackage, +} from '../dist/primeintellect/index.js' + +const root = mkdtempSync(join(tmpdir(), 'agent-runtime-primeintellect-v1-')) +const output = join(root, 'strict-exact-v1') +const referenceOutput = join(root, 'reference-v1') +const commandOutput = join(root, 'command-v1') + +try { + const bundle = createPrimeIntellectPackage({ + name: 'strict-exact-v1', + version: '1.0.0', + tasks: [ + { id: 'train', split: 'train', prompt: 'Is a subscription refundable?', answer: 'No' }, + { id: 'eval', split: 'eval', prompt: 'Is a final-sale item refundable?', answer: 'No' }, + ], + scoring: { kind: 'exact', normalization: 'trim-casefold' }, + runner: { + image: 'node:22-bookworm-slim', + command: ['node', 'runner.mjs'], + files: { 'runner.mjs': 'process.exit(0)\n' }, + }, + }) + await writePrimeIntellectPackage(bundle, output) + await writePrimeIntellectPackage( + createPrimeIntellectPackage({ + name: 'reference-v1', + version: '1.0.0', + tasks: [ + { id: 'train-ref', split: 'train', prompt: 'Practice reference?', answer: 'No' }, + { id: 'eval-ref', split: 'eval', prompt: 'Held-out reference?', answer: 'No' }, + ], + scoring: { kind: 'reference-judge', model: 'openai/gpt-5.4-nano' }, + runner: { + image: 'node:22-bookworm-slim', + command: ['node', 'runner.mjs'], + files: { 'runner.mjs': 'process.exit(0)\n' }, + }, + }), + referenceOutput, + ) + await writePrimeIntellectPackage( + createPrimeIntellectPackage({ + name: 'command-v1', + version: '1.0.0', + tasks: [ + { id: 'train-command', split: 'train', prompt: 'Practice command?' }, + { id: 'eval-command', split: 'eval', prompt: 'Held-out command?' }, + ], + scoring: { + kind: 'command', + command: ['python', 'scoring/score.py'], + files: { + 'score.py': + 'import json, sys\nrequest = json.load(sys.stdin)\nassert request["protocol"] == "tangle.primeintellect.score/v1"\nprint(json.dumps({"reward": 0.25, "metrics": {"custom": 0.75}}))\n', + }, + }, + runner: { + image: 'node:22-bookworm-slim', + command: ['node', 'runner.mjs'], + files: { 'runner.mjs': 'process.exit(0)\n' }, + }, + }), + commandOutput, + ) + const check = ` +import asyncio +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import verifiers.v1 as vf +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "reference-v1")) +sys.path.insert(0, str(ROOT / "command-v1")) +from command_v1 import TangleTaskset as CommandTaskset +from command_v1.taskset import TangleTasksetConfig as CommandTasksetConfig +from reference_v1 import TangleTaskset as ReferenceTaskset +from reference_v1.taskset import TangleTasksetConfig as ReferenceTasksetConfig +from strict_exact_v1 import TangleRuntimeHarness, TangleTaskset +from strict_exact_v1.harness import TangleRuntimeHarnessConfig +from strict_exact_v1.taskset import TangleTasksetConfig + +train = TangleTaskset(TangleTasksetConfig(id="strict-exact-v1", split="train")).load() +heldout = TangleTaskset(TangleTasksetConfig(id="strict-exact-v1", split="eval")).load() +assert [task.data.name for task in train] == ["train"] +assert [task.data.name for task in heldout] == ["eval"] + +class ScoreTrace: + def __init__(self, reply): + self.last_reply = reply + +task = heldout[0] +assert asyncio.run(task.task_reward(ScoreTrace("No"))) == 1.0 +assert asyncio.run(task.task_reward(ScoreTrace("No, but the policy allows it"))) == 0.0 +assert asyncio.run(task.task_reward(ScoreTrace("Yes, it is allowed"))) == 0.0 + +reference_task = ReferenceTaskset( + ReferenceTasksetConfig(id="reference-v1", split="eval") +).load()[0] +reference_trace = SimpleNamespace(last_reply="", transcript="") +assert asyncio.run(reference_task.task_reward(reference_trace)) == 0.0 + +class CommandTrace: + def __init__(self): + self.recorded = {} + def model_dump(self, **kwargs): + return {"id": "command-trace", "nodes": []} + def record_metrics(self, metrics): + self.recorded.update(metrics) + +command_task = CommandTaskset( + CommandTasksetConfig(id="command-v1", split="eval") +).load()[0] +command_trace = CommandTrace() +assert asyncio.run(command_task.task_reward(command_trace)) == 0.25 +assert command_trace.recorded == {"custom": 0.75} + +class Runtime: + def __init__(self): + self.env = None + self.files = {} + async def write(self, path, contents): + self.files[path] = contents + async def run(self, command, env): + return vf.ProgramResult(exit_code=0, stdout="", stderr="") + async def run_program(self, command, env): + self.env = env + return vf.ProgramResult(exit_code=0, stdout="", stderr="") + +runtime = Runtime() +harness = TangleRuntimeHarness(TangleRuntimeHarnessConfig(id="strict-exact-v1")) +asyncio.run(harness.setup(runtime)) +asyncio.run(harness.launch( + SimpleNamespace(model="openai/gpt-5.4-20260601"), + SimpleNamespace(task=SimpleNamespace(data=task.data)), + runtime, + "http://127.0.0.1:9000/v1", + "secret", + {}, +)) +public = json.loads(runtime.env["TANGLE_PRIME_TASK_JSON"]) +assert "answer" not in public +assert "reference" not in public +assert public["id"] == "eval" +assert runtime.files["runner.mjs"] == b"process.exit(0)\\n" +print(json.dumps({ + "train": len(train), + "eval": len(heldout), + "strict_scores": [1, 0, 0], + "reference_empty": 0, + "command_score": 0.25, + "command_metric": 0.75, +})) +` + const checkPath = join(output, 'verify.py') + writeFileSync(checkPath, check) + const result = spawnSync( + 'uv', + [ + 'run', + '--project', + output, + '--with', + 'verifiers @ git+https://github.com/PrimeIntellect-ai/verifiers.git@v0.2.0', + 'python', + checkPath, + ], + { cwd: output, encoding: 'utf8' }, + ) + if (result.status !== 0) { + throw new Error( + [ + 'PrimeIntellect Verifiers v0.2.0 integration check failed', + result.stdout.trim(), + result.stderr.trim(), + ] + .filter(Boolean) + .join('\n'), + ) + } + process.stdout.write(result.stdout) +} finally { + rmSync(root, { recursive: true, force: true }) +} diff --git a/skills/build-with-agent-runtime/SKILL.md b/skills/build-with-agent-runtime/SKILL.md index ab139294..5f533acb 100644 --- a/skills/build-with-agent-runtime/SKILL.md +++ b/skills/build-with-agent-runtime/SKILL.md @@ -92,14 +92,15 @@ to its native default (`HARNESS_NATIVE_MODEL`) — never silently dropped. | **Spawn N coding agents on isolated git worktrees, keep the one whose patch passes checks** | `worktreeFanout` + `createWorktreeCliExecutor` + `gateOnDeliverable(DeliverableSpec)` over a raw `WorktreePatchArtifact`, winner via `selectValidWinner` — `/loops` — NOT a hand-rolled spawn-loop / "coder" role | canonical-api §3.1 / §5 | | **Sandbox coding rollout** (fresh box/round, or persistent+resume) | `runLoop(options)` / `openSandboxRun(client, opts, deliverable)` — `/loops` | canonical-api §3.1 | | **Optimize a CODE surface** in a gated loop | `improvementDriver({ worktree, generator })` — root `.` | canonical-api §3.4 | -| **Optimize a PROMPT/config surface** (one call) — START HERE | `improve(profile, findings, { surface, gate })` — root `.` (the one pluggable RSI verb; picks the default proposer from `surface` — `gepaProposer` for prompt, `skillOptProposer` for skills — and wraps `selfImprove`; drop to `selfImprove({ agent, scenarios, judge, baselineSurface })` from `agent-eval/contract` only for the lower-level loop) | canonical-api §3.4 | +| **Freeze a measured profile/diff + code surface for execution** | `buildAgentCandidateBundle(...)` then `verifyAgentCandidateBundle(...)` — `/candidate-execution` | canonical-api §2 | +| **Optimize any agent/code surface** (one call) — START HERE | `improve(profile, findings, { surface, gate })` — root `.`; prompt, skill-document, and curated-memory surfaces have built-in proposers, config/whole-profile surfaces accept a proposer, and code gets isolated incumbent/candidate worktrees. Workflow and rollout-policy files use code plus agent-eval's `parameterSweepProposer` for JSON sweeps; drop to `selfImprove({ agent, scenarios, judge, baselineSurface })` only for lower-level control | canonical-api §3.4 | | **Gate: ship/hold a candidate** (campaign ctx) | `defaultProductionGate` / `heldOutGate` / `composeGate` — `agent-eval/contract`; `neutralizationGate` (footprint-matched PLACEBO gate — proves a held-out lift is CONTENT, not added prompt/mount footprint) — `agent-eval/campaign` | canonical-api §3.4 | | **Gate: ship/hold from a `BenchmarkReport`** (per-task cells) | `promotionGate({ report, incumbent, candidate })` — `/loops` | canonical-api §3.4 | | **Run the full multi-generation flywheel + certify** | `runStrategyEvolution(config)` — `/loops` | canonical-api §3.4 | | **Observe a run** (cost/time waterfall, OTLP) | `createWaterfallCollector()` — `/loops`; `createOtelExporter` attached via `composeRuntimeHooks(...)` — root `.` | canonical-api §2 | | **State any A/B claim** | `pairedLift` (bench) over `pairedBootstrap`/`heldoutSignificance` (substrate) | canonical-api §3.5 | -| **Observe/ship with billing-boundary** | `withTangleIntelligence(agent, { project, effort })` — `/intelligence` | canonical-api §2 | -| **Pull the certified profile from the Intelligence plane** (pull-by-default delivery: fold the gate-certified prompt onto the base surface) | `pullCertified` / `withCertifiedDelivery` / `composeCertifiedPrompt` — `/intelligence` | `src/intelligence/delivery.ts` | +| **Observe + deliver on a live agent with billing-boundary** | `withIntelligence(agent, { project, target, effort })` — `/intelligence` | canonical-api §2 | +| **Pull the certified profile from the Intelligence plane** (pull-by-default delivery: fold the gate-certified prompt onto the base surface) | `pullCertified` / `withIntelligence` / `composeCertifiedPrompt` — `/intelligence` | `src/intelligence/delivery.ts` | ## Do-NOT-reinvent — the traps this skill exists to stop @@ -177,26 +178,29 @@ sandbox coding rollouts against external benchmarks. The full when-which map is ## Observe / ship with the Intelligence SDK -One line wraps any agent with trace + billing boundary: -`withTangleIntelligence(agent, { project, effort })`, `effort ∈ +One hook wraps any agent with send + receive + billing boundary: +`withIntelligence(agent, { project, target, effort })`, `effort ∈ off|eco|standard|thorough|max` (`'off'` is the provable passthrough floor — -intelligence spend clamped to 0). It builds on `createOtelExporter` + -`loopEventToOtelSpan` — don't hand-roll a trace-wrapper or effort/tier config. -Verify the live subpath against `src/intelligence/index.ts`. +intelligence spend clamped to 0). It SENDs a typed `RunRecord` and RECEIVEs the +certified profile, building on `createOtelExporter` + `buildLoopOtelSpans` — +don't hand-roll a trace-wrapper, a second receive path, or effort/tier config. +Verify the live subpath against `src/intelligence/with-intelligence.ts`. Two operational facts every consumer must know: -- **Export is a silent no-op without an endpoint.** The export leg only ships - when `INTELLIGENCE_OTLP_ENDPOINT` (or `OTEL_EXPORTER_OTLP_ENDPOINT`) is set — - e.g. `https://intelligence.tangle.tools/v1/otlp`; absent, spans are dropped - best-effort with no error. The client's `doctor().exportConfigured` is the - check that export will actually ship. +- **Send is a silent no-op without a tenant apiKey.** The export leg only ships + when an `apiKey` (or `TANGLE_API_KEY`) is present — the ingest requires the + tenant Bearer. One `baseUrl` (`TANGLE_INTELLIGENCE_URL`, default + `https://intelligence.tangle.tools`) drives both the OTLP send (`/v1/otlp`) + and the receive pull. The client's `doctor().exportConfigured` is the check + that a send will actually land. - **Delivery pulls the certified profile from the plane.** `pullCertified` / - `withCertifiedDelivery` hit - `GET {TANGLE_INTELLIGENCE_URL|https://intelligence.tangle.tools}/v1/profiles/:target/composed` - with `Bearer TANGLE_API_KEY`; `withCertifiedDelivery` folds the certified - prompt onto the base surface, refreshes at most every 5 minutes, and is - fail-closed — a failed pull runs the agent on its base surface. + `withIntelligence` hit + `GET {baseUrl}/v1/profiles/:target/composed` with `Bearer TANGLE_API_KEY`; + the pull deserializes the typed `agentProfileDiffs` (surfaced as PROPOSALS, + never auto-applied) alongside the certified prompt. `withIntelligence` folds + the certified prompt onto the base surface, refreshes at most every 5 minutes, + and is fail-closed — a failed pull runs the agent on its base surface. ## Final check diff --git a/src/agent/surfaces.ts b/src/agent/surfaces.ts index c86e5210..ceee7ff8 100644 --- a/src/agent/surfaces.ts +++ b/src/agent/surfaces.ts @@ -13,8 +13,8 @@ * refuses to route those subjects rather than fabricating a target. */ -import { existsSync } from 'node:fs' -import { isAbsolute, join } from 'node:path' +import { existsSync, statSync } from 'node:fs' +import { isAbsolute, join, relative, resolve, sep } from 'node:path' import type { FindingSubject } from '@tangle-network/agent-eval' /** @@ -53,6 +53,22 @@ export interface AgentSurfaces { rag?: string /** Optional: single file defining the output schema (Zod / JSON Schema). */ outputSchema?: string + /** Optional: directory containing Agent Skill packages. */ + skills?: string + /** Optional: directory containing MCP server/tool configuration. */ + mcp?: string + /** Optional: directory containing hook definitions. */ + hooks?: string + /** Optional: directory containing subagent definitions. */ + subagents?: string + /** Optional: directory containing orchestration/workflow policies. */ + workflows?: string + /** Optional: single file containing rollout-policy settings. */ + rolloutPolicy?: string + /** Optional: single canonical AgentProfile file. */ + agentProfile?: string + /** Optional: source root for code findings. */ + code?: string } export interface ResolvedSurface { @@ -118,44 +134,88 @@ function candidatePathsForSubject( switch (subject.kind) { case 'knowledge.wiki': case 'knowledge.stale': - return [join(surfaces.knowledge, `${subject.slug}.md`)] + return optionalPath(safeJoin(surfaces.knowledge, `${subject.slug}.md`)) case 'knowledge.claim': // Claims land in a per-topic claims directory under the knowledge root. - return [join(surfaces.knowledge, 'claims', `${slugify(subject.topic)}.md`)] + return optionalPath(safeJoin(surfaces.knowledge, 'claims', `${slugify(subject.topic)}.md`)) case 'knowledge.raw': - return [join(surfaces.knowledge, 'raw', `${subject.sourceId}.md`)] + return optionalPath(safeJoin(surfaces.knowledge, 'raw', `${subject.sourceId}.md`)) case 'system-prompt': { const slug = slugify(subject.section) // Prefer flat layout for create-new (canonical); probe skill-dir layout // in case the existing repo (tax/legal/gtm/creative) uses // `
/SKILL.md` already. return [ - join(surfaces.systemPrompt, `${slug}.md`), - join(surfaces.systemPrompt, slug, 'SKILL.md'), - join(surfaces.systemPrompt, slug, 'index.md'), - ] + safeJoin(surfaces.systemPrompt, `${slug}.md`), + safeJoin(surfaces.systemPrompt, slug, 'SKILL.md'), + safeJoin(surfaces.systemPrompt, slug, 'index.md'), + ].filter((path): path is string => path !== null) + } + case 'skill': { + if (!surfaces.skills) return [] + return [ + safeJoin(surfaces.skills, subject.name, 'SKILL.md'), + safeJoin(surfaces.skills, `${subject.name}.md`), + ].filter((path): path is string => path !== null) } case 'tool-doc': if (subject.aspect) { - return [join(surfaces.tools, subject.tool, `${slugify(subject.aspect)}.md`)] + return optionalPath(safeJoin(surfaces.tools, subject.tool, `${slugify(subject.aspect)}.md`)) } // tool-doc default: `/README.md`; also probe `.md` for flat // tool-list repos. return [ - join(surfaces.tools, subject.tool, 'README.md'), - join(surfaces.tools, `${subject.tool}.md`), - ] + safeJoin(surfaces.tools, subject.tool, 'README.md'), + safeJoin(surfaces.tools, `${subject.tool}.md`), + ].filter((path): path is string => path !== null) case 'new-tool': - return [join(surfaces.tools, subject.name, 'README.md')] + return optionalPath(safeJoin(surfaces.tools, subject.name, 'README.md')) + case 'mcp': + if (!surfaces.mcp) return [] + return subject.tool + ? optionalPath(safeJoin(surfaces.mcp, subject.server, `${subject.tool}.md`)) + : [ + safeJoin(surfaces.mcp, `${subject.server}.json`), + safeJoin(surfaces.mcp, subject.server, 'README.md'), + ].filter((path): path is string => path !== null) + case 'hook': + if (!surfaces.hooks) return [] + return [ + safeJoin(surfaces.hooks, `${subject.name}.md`), + safeJoin(surfaces.hooks, `${subject.name}.json`), + ].filter((path): path is string => path !== null) + case 'subagent': + if (!surfaces.subagents) return [] + return [ + safeJoin(surfaces.subagents, `${subject.name}.md`), + safeJoin(surfaces.subagents, `${subject.name}.yaml`), + safeJoin(surfaces.subagents, `${subject.name}.json`), + ].filter((path): path is string => path !== null) + case 'workflow': + if (!surfaces.workflows) return [] + return [ + safeJoin(surfaces.workflows, `${subject.name}.md`), + safeJoin(surfaces.workflows, `${subject.name}.yaml`), + safeJoin(surfaces.workflows, `${subject.name}.json`), + ].filter((path): path is string => path !== null) + case 'rollout-policy': + return surfaces.rolloutPolicy ? [surfaces.rolloutPolicy] : [] + case 'agent-profile': + return surfaces.agentProfile ? [surfaces.agentProfile] : [] + case 'code': { + if (!surfaces.code) return [] + const path = safeJoin(surfaces.code, subject.path) + return path ? [path] : [] + } case 'rag': if (!surfaces.rag) return [] - return [join(surfaces.rag, subject.corpus, `${subject.docId}.md`)] + return optionalPath(safeJoin(surfaces.rag, subject.corpus, `${subject.docId}.md`)) case 'memory': if (!surfaces.memory) return [] - return [join(surfaces.memory, `${slugify(subject.key)}.json`)] + return optionalPath(safeJoin(surfaces.memory, `${slugify(subject.key)}.json`)) case 'scaffolding': if (!surfaces.scaffolding) return [] - return [join(surfaces.scaffolding, `${slugify(subject.concern)}.md`)] + return optionalPath(safeJoin(surfaces.scaffolding, `${slugify(subject.concern)}.md`)) case 'output-schema': if (!surfaces.outputSchema) return [] return [surfaces.outputSchema] @@ -170,6 +230,19 @@ function candidatePathsForSubject( } } +function safeJoin(root: string, ...children: string[]): string | null { + if (children.some((child) => child.includes('\0') || isAbsolute(child))) return null + const rootAbsolute = resolve(root) + const targetAbsolute = resolve(rootAbsolute, ...children) + const escaped = relative(rootAbsolute, targetAbsolute) + if (escaped === '..' || escaped.startsWith(`..${sep}`) || isAbsolute(escaped)) return null + return join(root, ...children) +} + +function optionalPath(path: string | null): string[] { + return path ? [path] : [] +} + function slugify(s: string): string { return ( s @@ -207,8 +280,22 @@ export function validateSurfaces( 'knowledge', ] const fileSurfaces: ReadonlyArray = ['rubric'] - const optionalDirSurfaces: ReadonlyArray = ['scaffolding', 'memory', 'rag'] - const optionalFileSurfaces: ReadonlyArray = ['outputSchema'] + const optionalDirSurfaces: ReadonlyArray = [ + 'scaffolding', + 'memory', + 'rag', + 'skills', + 'mcp', + 'hooks', + 'subagents', + 'workflows', + 'code', + ] + const optionalFileSurfaces: ReadonlyArray = [ + 'outputSchema', + 'rolloutPolicy', + 'agentProfile', + ] for (const key of dirSurfaces) { const p = surfaces[key] as string | undefined @@ -219,6 +306,8 @@ export function validateSurfaces( const abs = isAbsolute(p) ? p : join(repoRoot, p) if (!existsSync(abs)) { issues.push({ surface: key, path: p, reason: 'missing' }) + } else if (!statSync(abs).isDirectory()) { + issues.push({ surface: key, path: p, reason: 'not-directory' }) } } for (const key of fileSurfaces) { @@ -230,6 +319,8 @@ export function validateSurfaces( const abs = isAbsolute(p) ? p : join(repoRoot, p) if (!existsSync(abs)) { issues.push({ surface: key, path: p, reason: 'missing' }) + } else if (!statSync(abs).isFile()) { + issues.push({ surface: key, path: p, reason: 'not-file' }) } } for (const key of [...optionalDirSurfaces, ...optionalFileSurfaces]) { @@ -238,6 +329,13 @@ export function validateSurfaces( const abs = isAbsolute(p) ? p : join(repoRoot, p) if (!existsSync(abs)) { issues.push({ surface: key, path: p, reason: 'missing' }) + continue + } + const expectedDirectory = optionalDirSurfaces.includes(key) + if (expectedDirectory && !statSync(abs).isDirectory()) { + issues.push({ surface: key, path: p, reason: 'not-directory' }) + } else if (!expectedDirectory && !statSync(abs).isFile()) { + issues.push({ surface: key, path: p, reason: 'not-file' }) } } return issues diff --git a/src/candidate-execution/artifacts.ts b/src/candidate-execution/artifacts.ts new file mode 100644 index 00000000..bcca8faa --- /dev/null +++ b/src/candidate-execution/artifacts.ts @@ -0,0 +1,271 @@ +import { constants as fsConstants } from 'node:fs' +import { lstat, open, readdir, realpath } from 'node:fs/promises' +import { relative, resolve, sep } from 'node:path' + +import type { + AgentCandidateArtifactRef, + AgentCandidateCapturedArtifact, + AgentCandidateProfilePlanMaterialV1, + AgentCandidateWorkspaceManifestMaterialV1, + AgentCandidateWorkspaceSnapshotEvidence, +} from '@tangle-network/agent-interface' + +import { canonicalCandidateBytes, sha256Bytes } from './digest' +import type { AgentCandidateArtifactPort } from './types' + +export function artifactCacheKey(artifact: AgentCandidateCapturedArtifact): string { + return `${artifact.sha256}:${artifact.byteLength}` +} + +export async function readVerifiedArtifact( + artifact: AgentCandidateCapturedArtifact, + port: AgentCandidateArtifactPort, +): Promise { + const bytes = + 'content' in artifact + ? Buffer.from(artifact.content, 'base64') + : await port.read(artifact as AgentCandidateArtifactRef) + verifyBytes(bytes, artifact.sha256, artifact.byteLength, 'candidate artifact') + return Uint8Array.from(bytes) +} + +export function verifyBytes( + bytes: Uint8Array, + digest: string, + byteLength: number, + label: string, +): void { + if (bytes.byteLength !== byteLength) { + throw new Error(`${label} byte length ${bytes.byteLength} does not match ${byteLength}`) + } + const actual = sha256Bytes(bytes) + if (actual !== digest) { + throw new Error(`${label} digest ${actual} does not match ${digest}`) + } +} + +export async function verifyWorkspaceSnapshotArtifacts( + snapshot: AgentCandidateWorkspaceSnapshotEvidence, + port: AgentCandidateArtifactPort, +): Promise<{ manifest: Uint8Array; archive: Uint8Array }> { + const [manifest, archive] = await Promise.all([ + readVerifiedArtifact(snapshot.manifest, port), + readVerifiedArtifact(snapshot.archive, port), + ]) + const canonicalManifest = canonicalCandidateBytes(snapshot.material) + if (!Buffer.from(manifest).equals(Buffer.from(canonicalManifest))) { + throw new Error('workspace manifest artifact is not the exact canonical manifest material') + } + if (sha256Bytes(canonicalManifest) !== snapshot.digest) { + throw new Error('workspace snapshot digest does not match its canonical manifest material') + } + return { manifest, archive } +} + +export async function verifyMaterializedWorkspace( + root: string, + expected: AgentCandidateWorkspaceManifestMaterialV1, + options: { ignoredProtectedRootEntries?: readonly ('.git' | '.sidecar')[] } = {}, +): Promise { + const observed = await scanWorkspace(root, new Set(options.ignoredProtectedRootEntries ?? [])) + assertWorkspaceManifest(observed.manifest, expected) +} + +export async function captureMaterializedWorkspace( + root: string, + options: { + ignoredProtectedRootEntries?: readonly ('.git' | '.sidecar')[] + limits?: { + maxFiles: number + maxFileBytes: number + maxTotalFileBytes: number + } + } = {}, +): Promise<{ + manifest: AgentCandidateWorkspaceManifestMaterialV1 + files: ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> +}> { + const observed = await scanWorkspace( + root, + new Set(options.ignoredProtectedRootEntries ?? []), + options.limits, + ) + return { + manifest: observed.manifest, + files: observed.files.map((file) => ({ + path: file.path, + mode: file.mode, + bytes: Uint8Array.from(file.bytes), + })), + } +} + +/** Capture exact verified regular-file bytes for fresh isolated materialization. */ +export async function readMaterializedWorkspaceFiles( + root: string, + expected: AgentCandidateWorkspaceManifestMaterialV1, + options: { ignoredProtectedRootEntries?: readonly ('.git' | '.sidecar')[] } = {}, +): Promise> { + const observed = await captureMaterializedWorkspace(root, options) + assertWorkspaceManifest(observed.manifest, expected) + return observed.files.map((file) => + Object.freeze({ path: file.path, mode: file.mode, bytes: Uint8Array.from(file.bytes) }), + ) +} + +function assertWorkspaceManifest( + observed: AgentCandidateWorkspaceManifestMaterialV1, + expected: AgentCandidateWorkspaceManifestMaterialV1, +): void { + if (!Buffer.from(canonicalCandidateBytes(observed)).equals(canonicalCandidateBytes(expected))) { + throw new Error( + 'materialized workspace files, modes, or bytes do not match the signed manifest', + ) + } +} + +export async function verifyMaterializedProfileWorkspace( + root: string, + expected: AgentCandidateProfilePlanMaterialV1, +): Promise { + const observed = await scanWorkspace(root, new Set()) + const observedProfile = observed.manifest.files.map(({ path, mode, sha256 }) => ({ + relPath: path, + mode, + contentSha256: sha256, + })) + if ( + !Buffer.from(canonicalCandidateBytes(observedProfile)).equals( + canonicalCandidateBytes(expected.files), + ) + ) { + throw new Error('profile staging files, modes, or bytes do not match the signed profile plan') + } +} + +async function scanWorkspace( + root: string, + ignoredProtectedRootEntries: ReadonlySet, + limits?: { + maxFiles: number + maxFileBytes: number + maxTotalFileBytes: number + }, +): Promise<{ + manifest: AgentCandidateWorkspaceManifestMaterialV1 + files: Array<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> +}> { + const absoluteRoot = resolve(root) + const rootStats = await lstat(absoluteRoot) + if (!rootStats.isDirectory() || rootStats.isSymbolicLink()) { + throw new Error('workspace root must be a real directory') + } + if ((await realpath(absoluteRoot)) !== absoluteRoot) { + throw new Error('workspace root has a symlinked path component') + } + const files: AgentCandidateWorkspaceManifestMaterialV1['files'] = [] + const capturedFiles: Array<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> = [] + let totalBytes = 0 + + async function visit(directory: string): Promise { + const entries = await readdir(directory, { withFileTypes: true }) + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + for (const entry of entries) { + if (directory === absoluteRoot && ignoredProtectedRootEntries.has(entry.name)) { + continue + } + const absolute = resolve(directory, entry.name) + const relPath = relative(absoluteRoot, absolute).split(sep).join('/') + if (!relPath || relPath.startsWith('../') || relPath.includes('/../')) { + throw new Error(`workspace entry escapes root: ${relPath}`) + } + const stats = await lstat(absolute) + if (stats.isSymbolicLink()) { + throw new Error(`workspace contains a symlink: ${relPath}`) + } + if (stats.isDirectory()) { + await visit(absolute) + continue + } + if (!stats.isFile()) { + throw new Error(`workspace contains a non-regular entry: ${relPath}`) + } + if (limits && files.length >= limits.maxFiles) { + throw new Error('workspace exceeds maxFiles') + } + const descriptor = await open( + absolute, + fsConstants.O_RDONLY | + (typeof fsConstants.O_NOFOLLOW === 'number' ? fsConstants.O_NOFOLLOW : 0), + ) + try { + const openedStats = await descriptor.stat() + if (!openedStats.isFile()) { + throw new Error(`workspace contains a non-regular entry: ${relPath}`) + } + if (openedStats.nlink !== 1) { + throw new Error(`workspace contains a hard-linked file: ${relPath}`) + } + const mode = openedStats.mode & 0o777 + if (mode !== 0o644 && mode !== 0o755) { + throw new Error(`workspace file has unsupported mode ${mode.toString(8)}: ${relPath}`) + } + if (limits && openedStats.size > limits.maxFileBytes) { + throw new Error(`workspace file exceeds maxFileBytes: ${relPath}`) + } + const remainingBytes = limits ? limits.maxTotalFileBytes - totalBytes : undefined + if (remainingBytes !== undefined && openedStats.size > remainingBytes) { + throw new Error('workspace exceeds maxTotalFileBytes') + } + const bytes = await readBoundedFile( + descriptor, + limits ? Math.min(limits.maxFileBytes, remainingBytes ?? limits.maxFileBytes) : undefined, + relPath, + ) + totalBytes += bytes.byteLength + const supportedMode = mode as 0o644 | 0o755 + files.push({ + path: relPath, + mode: supportedMode, + sha256: sha256Bytes(bytes), + byteLength: bytes.byteLength, + }) + capturedFiles.push({ path: relPath, mode: supportedMode, bytes: Uint8Array.from(bytes) }) + } finally { + await descriptor.close() + } + } + } + + await visit(absoluteRoot) + files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + capturedFiles.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + return { + manifest: { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files, + }, + files: capturedFiles, + } +} + +async function readBoundedFile( + descriptor: Awaited>, + maxBytes: number | undefined, + path: string, +): Promise { + if (maxBytes === undefined) return await descriptor.readFile() + const chunks: Buffer[] = [] + let total = 0 + while (true) { + const remaining = maxBytes - total + const chunk = Buffer.allocUnsafe(Math.min(64 * 1024, remaining + 1)) + const { bytesRead } = await descriptor.read(chunk, 0, chunk.byteLength, null) + if (bytesRead === 0) break + total += bytesRead + if (total > maxBytes) throw new Error(`workspace file exceeds its capture limit: ${path}`) + chunks.push(chunk.subarray(0, bytesRead)) + } + return Buffer.concat(chunks, total) +} diff --git a/src/candidate-execution/benchmark-grader.ts b/src/candidate-execution/benchmark-grader.ts new file mode 100644 index 00000000..26981081 --- /dev/null +++ b/src/candidate-execution/benchmark-grader.ts @@ -0,0 +1,142 @@ +import type { BenchmarkEvaluation } from '@tangle-network/agent-eval' +import type { + AgentCandidateArtifactRef, + AgentCandidateTermination, +} from '@tangle-network/agent-interface' + +import { readVerifiedArtifact } from './artifacts' +import { immutableCandidateValue, sha256Bytes } from './digest' +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateOutputArtifactPort, + VerifiedAgentCandidateTaskOutcome, +} from './types' + +export interface BoundAgentCandidateBenchmarkRun { + readonly grader: { + readonly name: string + readonly version: string + readonly artifact: AgentCandidateArtifactRef + } + readonly evaluation: BenchmarkEvaluation + readonly evidence: Uint8Array +} + +/** + * Admit verified grader bytes to the evaluator runner and reject any result + * that is not bound to those bytes, the exact task outcome, and its raw output. + */ +export async function runBoundCandidateBenchmarkGrader(input: { + executionId: string + termination: AgentCandidateTermination + outcome: VerifiedAgentCandidateTaskOutcome + grader: AgentCandidateBenchmarkGraderPort + artifacts: AgentCandidateOutputArtifactPort + signal?: AbortSignal +}): Promise { + input.signal?.throwIfAborted() + const descriptor = snapshotGraderDescriptor(input.grader) + const implementationBytes = await readVerifiedArtifact(descriptor.artifact, input.artifacts) + if (implementationBytes.byteLength === 0) { + throw new Error('candidate benchmark grader implementation cannot be empty') + } + const expectedImplementationDigest = sha256Bytes(implementationBytes) + const termination = immutableCandidateValue(input.termination) + const run = input.grader.run + const result = await run( + Object.freeze({ + executionId: input.executionId, + termination, + outcome: input.outcome, + implementation: detachedImplementation(implementationBytes), + signal: input.signal ?? new AbortController().signal, + }), + ) + input.signal?.throwIfAborted() + assertExactRunnerResult(result) + + const evidence = Uint8Array.from(result.evidence) + const outputDigest = sha256Bytes(evidence) + if (result.binding.implementationDigest !== expectedImplementationDigest) { + throw new Error( + 'candidate benchmark grader executed implementation digest does not match its verified artifact', + ) + } + if (result.binding.taskOutcomeDigest !== input.outcome.evidence.digest) { + throw new Error( + 'candidate benchmark grader task outcome digest does not match verified outcome', + ) + } + if (result.binding.outputDigest !== outputDigest) { + throw new Error('candidate benchmark grader raw output digest does not match returned evidence') + } + + return Object.freeze({ + grader: descriptor, + evaluation: result.evaluation, + evidence, + }) +} + +function snapshotGraderDescriptor( + grader: AgentCandidateBenchmarkGraderPort, +): BoundAgentCandidateBenchmarkRun['grader'] { + if (!grader.name || !grader.version) { + throw new Error('candidate benchmark grader name and version must be non-empty') + } + return immutableCandidateValue({ + name: grader.name, + version: grader.version, + artifact: grader.artifact, + }) +} + +function detachedImplementation(bytes: Uint8Array): { + readonly byteLength: number + readonly bytes: Uint8Array +} { + const stored = Uint8Array.from(bytes) + return Object.freeze({ + byteLength: stored.byteLength, + get bytes(): Uint8Array { + return Uint8Array.from(stored) + }, + }) +} + +function assertExactRunnerResult( + value: unknown, +): asserts value is Awaited> { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('candidate benchmark grader result must be an object') + } + const result = value as Record + const keys = Object.keys(result).sort() + if ( + keys.length !== 3 || + keys[0] !== 'binding' || + keys[1] !== 'evaluation' || + keys[2] !== 'evidence' + ) { + throw new Error('candidate benchmark grader returned unknown or missing fields') + } + if (!(result.evidence instanceof Uint8Array)) { + throw new Error('candidate benchmark grader evidence must be bytes') + } + if ( + result.binding === null || + typeof result.binding !== 'object' || + Array.isArray(result.binding) + ) { + throw new Error('candidate benchmark grader binding must be an object') + } + const bindingKeys = Object.keys(result.binding).sort() + if ( + bindingKeys.length !== 3 || + bindingKeys[0] !== 'implementationDigest' || + bindingKeys[1] !== 'outputDigest' || + bindingKeys[2] !== 'taskOutcomeDigest' + ) { + throw new Error('candidate benchmark grader binding returned unknown or missing fields') + } +} diff --git a/src/candidate-execution/builder.ts b/src/candidate-execution/builder.ts new file mode 100644 index 00000000..4fb05fc2 --- /dev/null +++ b/src/candidate-execution/builder.ts @@ -0,0 +1,158 @@ +import type { CodeSurface } from '@tangle-network/agent-eval/campaign' +import { verifyCodeSurface } from '@tangle-network/agent-eval/campaign' +import type { + AgentCandidateCodeDisabled, + AgentCandidateCodeNoOp, + AgentCandidateExecution, + AgentCandidateGitHubRepository, + AgentCandidateKnowledge, + AgentCandidateLineage, + AgentCandidateMemoryPolicy, + AgentCandidateProfile, + AgentProfile, + AgentProfileDiff, +} from '@tangle-network/agent-interface' +import { type AgentCandidateBundleInput, sealAgentCandidateBundle } from './bundle' +import { canonicalCandidateDigest, embeddedCandidateArtifact } from './digest' +import { + applyExactAgentProfileDiff, + freezeGenericAgentCandidateProfile, + parseExactAgentProfile, + parseExactAgentProfileDiff, + parseExactCandidateProfile, +} from './profile' + +/** A complete profile that can be frozen without losing behavior. */ +export type AgentCandidateProfileSource = + | { + kind: 'profile' + profile: AgentProfile + } + | { + kind: 'profile-diffs' + base: AgentProfile + /** Applied in order. Each exact diff is content-addressed into lineage. */ + diffs: readonly AgentProfileDiff[] + } + | { + kind: 'candidate-profile' + /** Already converted to the closed, secret-free candidate profile contract. */ + profile: AgentCandidateProfile + } + +/** The only accepted path from an agent-eval code candidate to executable bytes. */ +export interface AgentCandidateCodeSurfaceSource { + kind: 'code-surface' + surface: CodeSurface + repository: AgentCandidateGitHubRepository + /** Optional parent directory used to resolve a relative `surface.worktreeRef`. */ + worktreeDir?: string +} + +/** Explicit control/no-op code or one finalized CodeSurface whose bytes must still verify. */ +export type AgentCandidateCodeSource = + | AgentCandidateCodeDisabled + | AgentCandidateCodeNoOp + | AgentCandidateCodeSurfaceSource + +/** Complete measured surfaces and execution policy compiled into one candidate bundle. */ +export interface BuildAgentCandidateBundleInput { + profile: AgentCandidateProfileSource + code: AgentCandidateCodeSource + execution: AgentCandidateExecution + knowledge?: AgentCandidateKnowledge + memory: AgentCandidateMemoryPolicy + /** `profileDiffIds` is derived from `profile`; callers cannot contradict it. */ + lineage: Omit +} + +/** + * Compile one measured profile/code candidate into the immutable execution + * contract. Code bytes are re-read and verified by agent-eval before they are + * embedded. The returned bundle is schema-validated, canonically digested, and + * deeply immutable; call `verifyAgentCandidateBundle` at the execution boundary + * to re-read external knowledge, memory, repository, and workspace artifacts. + */ +export function buildAgentCandidateBundle( + input: BuildAgentCandidateBundleInput, +): ReturnType { + if (Object.hasOwn(input.lineage, 'profileDiffIds')) { + throw new Error('profileDiffIds is derived from the profile source and cannot be supplied') + } + const compiledProfile = compileCandidateProfile(input.profile) + const profileDiffIds = compiledProfile.profileDiffIds + const bundle: AgentCandidateBundleInput = { + schemaVersion: 1, + kind: 'agent-candidate-bundle', + digestAlgorithm: 'rfc8785-sha256', + profile: compiledProfile.profile, + code: compileCandidateCode(input.code), + execution: input.execution, + ...(input.knowledge ? { knowledge: input.knowledge } : {}), + memory: input.memory, + lineage: { + ...input.lineage, + ...(profileDiffIds.length > 0 ? { profileDiffIds } : {}), + }, + } + return sealAgentCandidateBundle(bundle) +} + +function compileCandidateProfile(source: AgentCandidateProfileSource): { + profile: AgentCandidateProfile + profileDiffIds: string[] +} { + if (source.kind === 'candidate-profile') { + return { + profile: parseExactCandidateProfile(source.profile), + profileDiffIds: [], + } + } + + if (source.kind === 'profile') { + return { profile: freezeGenericAgentCandidateProfile(source.profile), profileDiffIds: [] } + } + + if (source.kind !== 'profile-diffs') { + throw new Error( + `unsupported candidate profile source: ${String((source as { kind?: unknown }).kind)}`, + ) + } + if (source.diffs.length === 0) { + throw new Error('profile-diffs source requires at least one AgentProfileDiff') + } + let profile = parseExactAgentProfile(source.base, 'base profile') + const profileDiffIds: string[] = [] + for (const [index, inputDiff] of source.diffs.entries()) { + const diff = parseExactAgentProfileDiff(inputDiff, `profile diff ${index}`) + profile = applyExactAgentProfileDiff(profile, diff, `profile diff ${index}`) + profileDiffIds.push(canonicalCandidateDigest(diff)) + } + return { profile: freezeGenericAgentCandidateProfile(profile), profileDiffIds } +} + +function compileCandidateCode(source: AgentCandidateCodeSource): AgentCandidateBundleInput['code'] { + if (source.kind === 'disabled' || source.kind === 'no-op') return source + if (source.kind !== 'code-surface') { + throw new Error( + `unsupported candidate code source: ${String((source as { kind?: unknown }).kind)}`, + ) + } + + const verified = verifyCodeSurface(source.surface, source.worktreeDir) + const patch = embeddedCandidateArtifact(verified.patchBytes) + if ( + patch.sha256 !== source.surface.patch.sha256 || + patch.byteLength !== source.surface.patch.byteLength + ) { + throw new Error('verified CodeSurface patch bytes do not match its content identity') + } + return { + kind: 'git-patch', + repository: source.repository, + baseCommit: source.surface.baseCommit, + baseTree: source.surface.baseTree, + candidateTree: source.surface.candidateTree, + patch: { format: 'git-diff-binary', artifact: patch }, + } +} diff --git a/src/candidate-execution/bundle.ts b/src/candidate-execution/bundle.ts new file mode 100644 index 00000000..d78e8f8b --- /dev/null +++ b/src/candidate-execution/bundle.ts @@ -0,0 +1,18 @@ +import type { AgentCandidateBundle } from '@tangle-network/agent-interface' +import { agentCandidateBundleSchema } from '@tangle-network/agent-interface' + +import { canonicalCandidateDigest, immutableCandidateValue, omitTopLevelDigest } from './digest' + +/** Exact candidate wire shape before the runtime computes its canonical digest. */ +export type AgentCandidateBundleInput = Omit + +/** Validate and content-address a candidate bundle before it crosses an approval boundary. */ +export function sealAgentCandidateBundle(input: AgentCandidateBundleInput): AgentCandidateBundle { + const digest = canonicalCandidateDigest(input) + const parsed = agentCandidateBundleSchema.parse({ ...input, digest }) + const parsedDigest = canonicalCandidateDigest(omitTopLevelDigest(parsed)) + if (parsedDigest !== digest) { + throw new Error('candidate bundle changed while validating its canonical wire shape') + } + return immutableCandidateValue(parsed) +} diff --git a/src/candidate-execution/claim-file-formats.ts b/src/candidate-execution/claim-file-formats.ts new file mode 100644 index 00000000..54b99463 --- /dev/null +++ b/src/candidate-execution/claim-file-formats.ts @@ -0,0 +1,34 @@ +import type { Sha256Digest } from '@tangle-network/agent-interface' + +import type { AgentCandidateExecutionClaim, AgentCandidateExecutionTerminalRecord } from './claim' + +export const CLAIM_FORMAT_VERSION = 7 +export const PENDING_FORMAT_VERSION = 1 +export const TERMINAL_FORMAT_VERSION = 3 +export const PHASE_FORMAT_VERSION = 1 + +export interface PersistedAgentCandidateExecutionClaim extends AgentCandidateExecutionClaim { + version: typeof CLAIM_FORMAT_VERSION + phase: 'claimed' + leaseDigest: Sha256Digest +} + +export interface PersistedAgentCandidateExecutionPending { + version: typeof PENDING_FORMAT_VERSION + kind: 'candidate-execution-pending-terminal' + terminal: AgentCandidateExecutionTerminalRecord +} + +export interface PersistedAgentCandidateExecutionPhase { + version: typeof PHASE_FORMAT_VERSION + kind: 'candidate-execution-phase' + executionId: string + attempt: number + executionPlanDigest: Sha256Digest + phase: 'candidate-may-run' +} + +export interface PersistedAgentCandidateExecutionTerminal { + version: typeof TERMINAL_FORMAT_VERSION + terminal: AgentCandidateExecutionTerminalRecord +} diff --git a/src/candidate-execution/claim-file-store.ts b/src/candidate-execution/claim-file-store.ts new file mode 100644 index 00000000..d4d9aeba --- /dev/null +++ b/src/candidate-execution/claim-file-store.ts @@ -0,0 +1,417 @@ +import { randomUUID } from 'node:crypto' +import { linkSync } from 'node:fs' +import { mkdir, open, unlink } from 'node:fs/promises' +import { join } from 'node:path' +import type { Sha256Digest } from '@tangle-network/agent-interface' + +import { + type AgentCandidateExecutionAttemptRecord, + type AgentCandidateExecutionAttemptRef, + type AgentCandidateExecutionClaim, + type AgentCandidateExecutionClaimResult, + type AgentCandidateExecutionClaimStore, + type AgentCandidateExecutionFinishResult, + type AgentCandidateExecutionLease, + type AgentCandidateExecutionPhase, + type AgentCandidateExecutionPhaseResult, + type AgentCandidateExecutionRecoveryEvidence, + type AgentCandidateExecutionStageResult, + type AgentCandidateExecutionTerminalRecord, + type AgentCandidateExecutionTerminalResult, + type AgentCandidateRetryRejection, + candidateClaimFileInternals, +} from './claim' +import { + CLAIM_FORMAT_VERSION, + PENDING_FORMAT_VERSION, + PHASE_FORMAT_VERSION, + TERMINAL_FORMAT_VERSION, +} from './claim-file-formats' +import { + assertRecoveryMatchesStaged, + assertTerminalAllowedInPhase, + assertTerminalMatchesClaim, + assertTerminalMatchesStaged, + recoveredTerminalRecord, + rejectedFinish, + rejectedStage, + requireStagedTerminal, + sealTerminalDigest, + terminalRecord, +} from './claim-terminal' +import { canonicalCandidateBytes } from './digest' + +const { + assertExpiredLease, + assertLease, + assertSameSlot, + assertUnexpiredLease, + attemptRecord, + claimSlot, + leaseDigest, + newLease, + readClaim, + readClaimIfPresent, + readTerminal, + readTerminalIfPresent, + readTransitionIfPresent, + rejectedExistingClaim, + rejectedRetry, + retryRejection, + sealAttemptRef, + sealClaim, + sealLease, +} = candidateClaimFileInternals + +export interface FileAgentCandidateExecutionClaimStoreOptions { + /** Evaluator-owned directory shared by every process allowed to execute candidates. */ + directory: string + /** Testable evaluator clock; defaults to `Date.now`. */ + now?: () => number +} + +/** Cross-process lifecycle implemented as fsynced, create-if-absent records. */ +export class FileAgentCandidateExecutionClaimStore implements AgentCandidateExecutionClaimStore { + private readonly directory: string + private readonly now: () => number + + constructor(options: FileAgentCandidateExecutionClaimStoreOptions) { + if (options.directory.length === 0) { + throw new Error('candidate execution claim directory must not be empty') + } + this.directory = options.directory + this.now = options.now ?? Date.now + } + + async tryClaim( + requested: AgentCandidateExecutionClaim, + ): Promise { + const claim = sealClaim(requested) + await mkdir(this.directory, { recursive: true }) + const claimPath = this.claimPath(claim) + const existing = await readClaimIfPresent(claimPath) + if (existing) { + assertSameSlot(existing.claim, claim, claimPath) + return rejectedExistingClaim(existing.claim, claim) + } + assertUnexpiredLease(claim.leaseExpiresAtMs, this.now()) + const retryFailure = await this.retryFailure(claim) + if (retryFailure) return rejectedRetry(claim, retryFailure) + + const lease = newLease(claim) + const acquired = await writeRecordIfAbsent(this.directory, claimPath, { + version: CLAIM_FORMAT_VERSION, + ...claim, + phase: 'claimed', + leaseDigest: leaseDigest(lease), + }) + if (acquired) return Object.freeze({ acquired: true, claim, lease }) + + const winner = await readClaim(claimPath) + assertSameSlot(winner.claim, claim, claimPath) + return rejectedExistingClaim(winner.claim, claim) + } + + async getAttempt( + requestedAttempt: AgentCandidateExecutionAttemptRef, + ): Promise { + return await this.storedAttempt(sealAttemptRef(requestedAttempt)) + } + + async markCandidateMayRun( + requestedLease: AgentCandidateExecutionLease, + ): Promise { + const lease = sealLease(requestedLease) + const claimPath = this.claimPath(lease) + const stored = await readClaim(claimPath) + assertSameSlot(stored.claim, lease, claimPath) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + const state = await this.transitionState(stored.claim) + if (state.phase === 'candidate-may-run') { + return Object.freeze({ marked: false, phase: 'candidate-may-run' }) + } + if (state.staged) { + throw new Error('candidate execution terminal was staged before candidate-may-run phase') + } + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + const marked = await writeRecordIfAbsent( + this.directory, + this.transitionPath(stored.claim, 1), + { + version: PHASE_FORMAT_VERSION, + kind: 'candidate-execution-phase', + executionId: stored.claim.executionId, + attempt: stored.claim.attempt, + executionPlanDigest: stored.claim.executionPlanDigest, + phase: 'candidate-may-run', + }, + this.ownerPublication(stored.claim), + ) + if (marked) return Object.freeze({ marked: true, phase: 'candidate-may-run' }) + const winner = await this.transitionState(stored.claim) + if (winner.phase === 'candidate-may-run') { + return Object.freeze({ marked: false, phase: 'candidate-may-run' }) + } + throw new Error('candidate execution terminal was staged before candidate-may-run phase') + } + + async stageTerminal( + requestedLease: AgentCandidateExecutionLease, + result: AgentCandidateExecutionTerminalResult, + ): Promise { + const lease = sealLease(requestedLease) + const claimPath = this.claimPath(lease) + const stored = await readClaim(claimPath) + assertSameSlot(stored.claim, lease, claimPath) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + const terminal = terminalRecord(stored.claim, result) + const state = await this.transitionState(stored.claim) + if (state.staged) return rejectedStage(state.staged, terminal) + assertTerminalAllowedInPhase(state.phase, terminal) + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + const transition = state.phase === 'claimed' ? 1 : 2 + const staged = await writeRecordIfAbsent( + this.directory, + this.transitionPath(stored.claim, transition), + { + version: PENDING_FORMAT_VERSION, + kind: 'candidate-execution-pending-terminal', + terminal, + }, + this.ownerPublication(stored.claim), + ) + if (staged) return Object.freeze({ staged: true, terminal }) + const winner = await this.transitionState(stored.claim) + if (!winner.staged) { + throw new Error('candidate execution phase transition won while staging terminal') + } + return rejectedStage(winner.staged, terminal) + } + + async finish( + requestedLease: AgentCandidateExecutionLease, + requestedTerminalDigest: Sha256Digest, + ): Promise { + const lease = sealLease(requestedLease) + const terminalDigest = sealTerminalDigest(requestedTerminalDigest) + const claimPath = this.claimPath(lease) + const stored = await readClaim(claimPath) + assertSameSlot(stored.claim, lease, claimPath) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + const state = await this.transitionState(stored.claim) + const staged = requireStagedTerminal(state.staged, terminalDigest) + const terminalPath = this.terminalPath(lease) + const existing = await readTerminalIfPresent(terminalPath) + if (existing) { + assertTerminalMatchesClaim(existing, stored.claim, terminalPath) + assertTerminalMatchesStaged(existing, staged, terminalPath) + return rejectedFinish(existing, terminalDigest) + } + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + const finished = await writeRecordIfAbsent( + this.directory, + terminalPath, + { + version: TERMINAL_FORMAT_VERSION, + terminal: staged, + }, + this.ownerPublication(stored.claim), + ) + if (finished) return Object.freeze({ finished: true, terminal: staged }) + const winner = await readTerminal(terminalPath) + assertSameSlot(winner, lease, terminalPath) + assertTerminalMatchesClaim(winner, stored.claim, terminalPath) + assertTerminalMatchesStaged(winner, staged, terminalPath) + return rejectedFinish(winner, terminalDigest) + } + + async recoverExpired( + requestedAttempt: AgentCandidateExecutionAttemptRef, + evidence: AgentCandidateExecutionRecoveryEvidence, + ): Promise { + const attempt = sealAttemptRef(requestedAttempt) + const record = await this.storedAttempt(attempt) + if (!record) throw new Error('candidate execution recovery does not name an acquired attempt') + const recovered = recoveredTerminalRecord(record.claim, record.phase, evidence) + if (record.staged) assertRecoveryMatchesStaged(record.staged, recovered) + const requestedDigest = record.staged?.terminalDigest ?? recovered.terminalDigest + if (record.terminal) return rejectedFinish(record.terminal, requestedDigest) + assertExpiredLease(record.claim.leaseExpiresAtMs, this.now()) + + let staged = record.staged + if (!staged) { + const transition = record.phase === 'claimed' ? 1 : 2 + const didStage = await writeRecordIfAbsent( + this.directory, + this.transitionPath(record.claim, transition), + { + version: PENDING_FORMAT_VERSION, + kind: 'candidate-execution-pending-terminal', + terminal: recovered, + }, + ) + if (didStage) { + staged = recovered + } else { + const winner = await this.transitionState(record.claim) + if (!winner.staged) { + throw new Error( + 'candidate execution recovery lost terminal staging to a phase transition', + ) + } + assertRecoveryMatchesStaged(winner.staged, recovered) + staged = winner.staged + } + } + + const terminalPath = this.terminalPath(attempt) + const didFinish = await writeRecordIfAbsent(this.directory, terminalPath, { + version: TERMINAL_FORMAT_VERSION, + terminal: staged, + }) + if (didFinish) return Object.freeze({ finished: true, terminal: staged }) + const winner = await readTerminal(terminalPath) + assertSameSlot(winner, attempt, terminalPath) + assertTerminalMatchesClaim(winner, record.claim, terminalPath) + assertTerminalMatchesStaged(winner, staged, terminalPath) + return rejectedFinish(winner, staged.terminalDigest) + } + + private async storedAttempt( + claim: Pick, + ): Promise { + const claimPath = this.claimPath(claim) + const stored = await readClaimIfPresent(claimPath) + if (!stored) return undefined + assertSameSlot(stored.claim, claim, claimPath) + const state = await this.transitionState(stored.claim) + const terminalPath = this.terminalPath(claim) + const terminal = await readTerminalIfPresent(terminalPath) + if (terminal) { + assertTerminalMatchesClaim(terminal, stored.claim, terminalPath) + if (!state.staged) { + throw new Error( + `candidate execution terminal record at ${terminalPath} has no staged outbox`, + ) + } + assertTerminalMatchesStaged(terminal, state.staged, terminalPath) + } + return attemptRecord(stored.claim, state.phase, state.staged, terminal) + } + + private async transitionState(claim: AgentCandidateExecutionClaim): Promise<{ + phase: AgentCandidateExecutionPhase + staged?: AgentCandidateExecutionTerminalRecord + }> { + const firstPath = this.transitionPath(claim, 1) + const first = await readTransitionIfPresent(firstPath, claim) + if (!first) return { phase: 'claimed' } + if (first.kind === 'pending') return { phase: 'claimed', staged: first.terminal } + const secondPath = this.transitionPath(claim, 2) + const second = await readTransitionIfPresent(secondPath, claim) + if (!second) return { phase: 'candidate-may-run' } + if (second.kind !== 'pending') { + throw new Error(`candidate execution transition at ${secondPath} repeats the phase marker`) + } + return { phase: 'candidate-may-run', staged: second.terminal } + } + + private async retryFailure( + claim: AgentCandidateExecutionClaim, + ): Promise { + if (claim.attempt === 1) return undefined + return retryRejection( + claim, + await this.storedAttempt({ + executionId: claim.executionId, + attempt: claim.attempt - 1, + }), + ) + } + + private ownerPublication(claim: AgentCandidateExecutionClaim): { + authorizePublish: () => true + } { + return { + authorizePublish: () => { + assertUnexpiredLease(claim.leaseExpiresAtMs, this.now()) + return true + }, + } + } + + private claimPath(claim: Pick): string { + return join(this.directory, `${claimSlot(claim)}.claim.json`) + } + + private terminalPath( + claim: Pick, + ): string { + return join(this.directory, `${claimSlot(claim)}.terminal.json`) + } + + private transitionPath( + claim: Pick, + ordinal: 1 | 2, + ): string { + return join(this.directory, `${claimSlot(claim)}.transition-${ordinal}.json`) + } +} + +async function writeRecordIfAbsent( + directory: string, + destination: string, + record: object, + options: { + /** Synchronous authorization checked after temp fsync, immediately before atomic publication. */ + authorizePublish?: () => true + } = {}, +): Promise { + const temporaryPath = join(directory, `.candidate-execution-${process.pid}-${randomUUID()}.tmp`) + const handle = await open(temporaryPath, 'wx', 0o600) + try { + await handle.writeFile( + Buffer.concat([Buffer.from(canonicalCandidateBytes(record)), Buffer.from('\n')]), + ) + await handle.sync() + } finally { + await handle.close() + } + + let written = false + try { + if (options.authorizePublish && options.authorizePublish() !== true) { + throw new Error('candidate execution record publication was not authorized') + } + // Authorization and the atomic filesystem call are one synchronous critical + // section so the event loop cannot advance an injected lease clock between them. + linkSync(temporaryPath, destination) + written = true + await syncDirectory(directory) + } catch (error) { + if (!isNodeError(error, 'EEXIST')) throw error + } finally { + await unlink(temporaryPath).catch((error: unknown) => { + if (!isNodeError(error, 'ENOENT')) throw error + }) + } + return written +} + +async function syncDirectory(directory: string): Promise { + const handle = await open(directory, 'r') + try { + await handle.sync() + } finally { + await handle.close() + } +} + +function isNodeError(error: unknown, code: string): boolean { + return ( + error !== null && + typeof error === 'object' && + 'code' in error && + (error as { code?: unknown }).code === code + ) +} diff --git a/src/candidate-execution/claim-plan.ts b/src/candidate-execution/claim-plan.ts new file mode 100644 index 00000000..8d8333a3 --- /dev/null +++ b/src/candidate-execution/claim-plan.ts @@ -0,0 +1,107 @@ +import type { Sha256Digest } from '@tangle-network/agent-interface' + +import { type AgentCandidateExecutionClaim, candidateClaimFileInternals } from './claim' +import { canonicalCandidateDigest } from './digest' +import { candidateExecutionOwnerWindowMs } from './execution-window' +import { assertPreparedCandidateIntegrity } from './prepared-state' +import type { PreparedAgentCandidateExecution } from './types' + +/** Extract the complete durable claim from a prepared execution. */ +export function candidateExecutionClaim( + prepared: PreparedAgentCandidateExecution, +): AgentCandidateExecutionClaim { + const state = assertPreparedCandidateIntegrity(prepared) + const material = prepared.executionPlan.value.material + const attempt = material.attempt + const nowMs = Date.now() + if (!Number.isSafeInteger(nowMs) || nowMs < 0) { + throw new Error('candidate execution claim-store clock returned an invalid timestamp') + } + const leaseExpiresAtMs = + nowMs + + candidateExecutionOwnerWindowMs( + material.limits.timeoutMs, + state.cleanupTimeoutMs, + state.resultTimeoutMs, + ) + if (!Number.isSafeInteger(leaseExpiresAtMs) || leaseExpiresAtMs <= 0) { + throw new Error('candidate execution leaseExpiresAtMs must be a positive safe timestamp') + } + if (leaseExpiresAtMs > state.reservationExpiresAtMs) { + throw new Error( + 'candidate preparation expires before its full execution and cleanup owner window', + ) + } + return candidateClaimFileInternals.sealClaim({ + executionId: prepared.executionId, + attempt: attempt.number, + maxAttempts: attempt.maxAttempts, + retryPolicy: attempt.retryPolicy, + bundleDigest: prepared.bundle.digest, + executionPlanDigest: prepared.executionPlan.value.digest, + retryLineageDigest: retryLineageDigest(prepared, state.resultTimeoutMs), + leaseExpiresAtMs, + resultTimeoutMs: state.resultTimeoutMs, + cleanup: { + preparationId: state.preparationId, + modelGrantDigest: state.modelReservation.digest, + resolvedModel: state.resolvedModel, + traceRunId: state.trace.runId, + cleanupTimeoutMs: state.cleanupTimeoutMs, + ...(state.memoryReservation + ? { + memory: { + accessDigest: state.memoryReservation.accessDigest, + effectiveNamespace: state.memoryReservation.effectiveNamespace, + }, + } + : {}), + }, + }) +} + +function retryLineageDigest( + prepared: PreparedAgentCandidateExecution, + resultTimeoutMs: number, +): Sha256Digest { + const material = prepared.executionPlan.value.material + return canonicalCandidateDigest({ + resultTimeoutMs, + executionPlan: { + ...material, + attempt: { ...material.attempt, number: 0 }, + model: { + ...material.model, + access: { + ...material.model.access, + grantDigest: `sha256:${'0'.repeat(64)}`, + }, + }, + memory: + material.memory.mode === 'disabled' + ? material.memory + : { + mode: 'isolated', + scope: 'task', + effectiveNamespace: 'candidate/retry-lineage-normalized', + reset: { + kind: 'fresh', + emptyStateDigest: material.memory.reset.emptyStateDigest, + }, + beforeState: { + digest: material.memory.beforeState.digest, + material: material.memory.beforeState.material, + manifest: { + sha256: material.memory.beforeState.manifest.sha256, + byteLength: material.memory.beforeState.manifest.byteLength, + }, + archive: { + sha256: material.memory.beforeState.archive.sha256, + byteLength: material.memory.beforeState.archive.byteLength, + }, + }, + ...(material.memory.seedDigest ? { seedDigest: material.memory.seedDigest } : {}), + }, + }, + }) +} diff --git a/src/candidate-execution/claim-terminal.ts b/src/candidate-execution/claim-terminal.ts new file mode 100644 index 00000000..d71b1a9c --- /dev/null +++ b/src/candidate-execution/claim-terminal.ts @@ -0,0 +1,516 @@ +import type { AgentCandidateArtifactRef, Sha256Digest } from '@tangle-network/agent-interface' +import { agentCandidateArtifactRefSchema } from '@tangle-network/agent-interface' + +import type { + AgentCandidateExecutionClaim, + AgentCandidateExecutionFailureClass, + AgentCandidateExecutionFinishResult, + AgentCandidateExecutionPhase, + AgentCandidateExecutionRecoveryEvidence, + AgentCandidateExecutionStageResult, + AgentCandidateExecutionTerminalRecord, + AgentCandidateExecutionTerminalResult, + AgentCandidateExecutionUsage, +} from './claim' +import { canonicalCandidateDigest, immutableCandidateValue } from './digest' +import { assertExactObjectKeys as assertExactKeys } from './exact-object' + +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/ + +export function terminalRecord( + claim: AgentCandidateExecutionClaim, + result: AgentCandidateExecutionTerminalResult, +): AgentCandidateExecutionTerminalRecord { + const terminal = sealTerminalResult(result) + const value = { + executionId: claim.executionId, + attempt: claim.attempt, + bundleDigest: claim.bundleDigest, + executionPlanDigest: claim.executionPlanDigest, + ...terminal, + } + return immutableCandidateValue({ + ...value, + terminalDigest: canonicalCandidateDigest(value), + }) as AgentCandidateExecutionTerminalRecord +} + +export function recoveredTerminalRecord( + claim: AgentCandidateExecutionClaim, + phase: AgentCandidateExecutionPhase, + evidence: AgentCandidateExecutionRecoveryEvidence, +): AgentCandidateExecutionTerminalRecord { + const recovered = sealRecoveryEvidence(evidence, claim) + return terminalRecord(claim, { + schemaVersion: 1, + status: 'failed', + failureClass: + recovered.failureClass === 'pre-model-infrastructure' && phase !== 'claimed' + ? 'unknown' + : recovered.failureClass, + usage: recovered.usage, + modelSettlement: recovered.modelSettlement, + ...(recovered.failureEvidence ? { failureEvidence: recovered.failureEvidence } : {}), + }) +} + +export function assertTerminalAllowedInPhase( + phase: AgentCandidateExecutionPhase, + terminal: AgentCandidateExecutionTerminalRecord, +): void { + if ( + phase === 'candidate-may-run' && + terminal.status === 'failed' && + terminal.failureClass === 'pre-model-infrastructure' + ) { + throw new Error('candidate execution crossed candidate-may-run before pre-model failure') + } + if ( + phase === 'claimed' && + (terminal.status === 'succeeded' || + (terminal.status === 'failed' && + (terminal.failureClass === 'execution' || + terminal.failureClass === 'post-model-infrastructure'))) + ) { + throw new Error('candidate execution terminal requires candidate-may-run phase') + } +} + +export function rejectedFinish( + existing: AgentCandidateExecutionTerminalRecord, + requestedTerminalDigest: Sha256Digest, +): AgentCandidateExecutionFinishResult { + return Object.freeze({ + finished: false, + terminal: existing, + exactReplay: existing.terminalDigest === requestedTerminalDigest, + }) +} + +export function rejectedStage( + existing: AgentCandidateExecutionTerminalRecord, + requested: AgentCandidateExecutionTerminalRecord, +): AgentCandidateExecutionStageResult { + return Object.freeze({ + staged: false, + terminal: existing, + exactReplay: existing.terminalDigest === requested.terminalDigest, + }) +} + +export function requireStagedTerminal( + staged: AgentCandidateExecutionTerminalRecord | undefined, + terminalDigest: Sha256Digest, +): AgentCandidateExecutionTerminalRecord { + if (!staged) throw new Error('candidate execution terminal has not been staged') + if (staged.terminalDigest !== terminalDigest) { + throw new Error('candidate execution terminal digest does not match staged outbox') + } + return staged +} + +export function sealTerminalDigest(value: Sha256Digest): Sha256Digest { + assertSha256Digest(value, 'terminalDigest') + return value +} + +export function assertRecoveryMatchesStaged( + staged: AgentCandidateExecutionTerminalRecord, + recovered: AgentCandidateExecutionTerminalRecord, +): void { + if ( + canonicalCandidateDigest(staged.usage) !== canonicalCandidateDigest(recovered.usage) || + canonicalCandidateDigest(staged.modelSettlement) !== + canonicalCandidateDigest(recovered.modelSettlement) + ) { + throw new Error('candidate execution recovery evidence does not match staged model evidence') + } +} + +export function sealTerminalRecordValue( + value: Record, + label: string, +): AgentCandidateExecutionTerminalRecord { + const status = requireTerminalStatus(value.status, label) + assertExactKeys( + value, + status === 'succeeded' + ? [ + 'executionId', + 'attempt', + 'bundleDigest', + 'executionPlanDigest', + 'terminalDigest', + 'schemaVersion', + 'status', + 'usage', + 'modelSettlement', + 'taskOutcome', + 'benchmarkResult', + 'runReceipt', + ] + : [ + 'executionId', + 'attempt', + 'bundleDigest', + 'executionPlanDigest', + 'terminalDigest', + 'schemaVersion', + 'status', + 'failureClass', + 'usage', + 'modelSettlement', + ...(value.failureEvidence ? ['failureEvidence'] : []), + ], + label, + ) + const identity = { + executionId: requireString(value.executionId, label, 'executionId'), + attempt: requireNumber(value.attempt, label, 'attempt'), + bundleDigest: requireString(value.bundleDigest, label, 'bundleDigest') as Sha256Digest, + executionPlanDigest: requireString( + value.executionPlanDigest, + label, + 'executionPlanDigest', + ) as Sha256Digest, + } + assertExecutionId(identity.executionId) + if (!Number.isSafeInteger(identity.attempt) || identity.attempt < 1) { + throw new Error(`${label} has invalid attempt`) + } + assertSha256Digest(identity.bundleDigest, 'bundleDigest') + assertSha256Digest(identity.executionPlanDigest, 'executionPlanDigest') + const result = sealTerminalResult( + status === 'succeeded' + ? { + schemaVersion: requireNumber(value.schemaVersion, label, 'schemaVersion') as 1, + status, + usage: requireObject( + value.usage, + label, + 'usage', + ) as unknown as AgentCandidateExecutionUsage, + modelSettlement: requireArtifactRef(value.modelSettlement, label, 'modelSettlement'), + taskOutcome: requireArtifactRef(value.taskOutcome, label, 'taskOutcome'), + benchmarkResult: requireArtifactRef(value.benchmarkResult, label, 'benchmarkResult'), + runReceipt: requireArtifactRef(value.runReceipt, label, 'runReceipt'), + } + : { + schemaVersion: requireNumber(value.schemaVersion, label, 'schemaVersion') as 1, + status, + failureClass: requireFailureClass(value.failureClass, label), + usage: requireObject( + value.usage, + label, + 'usage', + ) as unknown as AgentCandidateExecutionUsage, + modelSettlement: requireArtifactRef(value.modelSettlement, label, 'modelSettlement'), + ...(value.failureEvidence + ? { + failureEvidence: requireArtifactRef( + value.failureEvidence, + label, + 'failureEvidence', + ), + } + : {}), + }, + ) + const material = { ...identity, ...result } + const terminalDigest = requireString( + value.terminalDigest, + label, + 'terminalDigest', + ) as Sha256Digest + assertSha256Digest(terminalDigest, 'terminalDigest') + if (terminalDigest !== canonicalCandidateDigest(material)) { + throw new Error(`${label} has invalid terminalDigest`) + } + return immutableCandidateValue({ + ...material, + terminalDigest, + }) as AgentCandidateExecutionTerminalRecord +} + +export function assertTerminalMatchesClaim( + terminal: AgentCandidateExecutionTerminalRecord, + claim: AgentCandidateExecutionClaim, + path: string, +): void { + if ( + terminal.executionId !== claim.executionId || + terminal.attempt !== claim.attempt || + terminal.bundleDigest !== claim.bundleDigest || + terminal.executionPlanDigest !== claim.executionPlanDigest + ) { + throw new Error(`candidate execution terminal record at ${path} does not match its claim`) + } +} + +export function assertTerminalMatchesStaged( + terminal: AgentCandidateExecutionTerminalRecord, + staged: AgentCandidateExecutionTerminalRecord, + path: string, +): void { + if ( + terminal.terminalDigest !== staged.terminalDigest || + canonicalCandidateDigest(terminal) !== canonicalCandidateDigest(staged) + ) { + throw new Error(`candidate execution terminal record at ${path} differs from staged outbox`) + } +} + +function sealRecoveryEvidence( + evidence: AgentCandidateExecutionRecoveryEvidence, + claim: AgentCandidateExecutionClaim, +): AgentCandidateExecutionRecoveryEvidence { + assertExactKeys( + evidence, + [ + 'failureClass', + 'usage', + 'modelSettlement', + 'process', + 'model', + ...(evidence.failureEvidence ? ['failureEvidence'] : []), + ...(evidence.memory ? ['memory'] : []), + ], + 'candidate execution recovery evidence', + ) + assertFailureClass(evidence.failureClass) + const usage = sealUsage(evidence.usage) + if (evidence.failureClass === 'pre-model-infrastructure' && usage.modelCalls !== 0) { + throw new Error('pre-model infrastructure failure cannot contain model calls') + } + const modelSettlement = sealArtifactRef(evidence.modelSettlement, 'modelSettlement') + const failureEvidence = evidence.failureEvidence + ? sealArtifactRef(evidence.failureEvidence, 'failureEvidence') + : undefined + assertExactKeys( + evidence.process, + ['stopped', 'executionPlanDigest'], + 'candidate execution process closure evidence', + ) + if ( + evidence.process.stopped !== true || + evidence.process.executionPlanDigest !== claim.executionPlanDigest + ) { + throw new Error('candidate execution recovery does not prove the claimed process stopped') + } + assertExactKeys( + evidence.model, + ['closed', 'preparationId', 'grantDigest'], + 'candidate execution model closure evidence', + ) + if ( + evidence.model.closed !== true || + evidence.model.preparationId !== claim.cleanup.preparationId || + evidence.model.grantDigest !== claim.cleanup.modelGrantDigest + ) { + throw new Error('candidate execution recovery does not prove the claimed model grant closed') + } + if (claim.cleanup.memory) { + if (!evidence.memory) { + throw new Error('candidate execution recovery is missing memory closure evidence') + } + assertExactKeys( + evidence.memory, + ['closed', 'preparationId', 'accessDigest', 'effectiveNamespace'], + 'candidate execution memory closure evidence', + ) + if ( + evidence.memory.closed !== true || + evidence.memory.preparationId !== claim.cleanup.preparationId || + evidence.memory.accessDigest !== claim.cleanup.memory.accessDigest || + evidence.memory.effectiveNamespace !== claim.cleanup.memory.effectiveNamespace + ) { + throw new Error( + 'candidate execution recovery does not prove the claimed memory access closed', + ) + } + } else if (evidence.memory !== undefined) { + throw new Error('candidate execution recovery has unexpected memory closure evidence') + } + return Object.freeze({ + failureClass: evidence.failureClass, + usage, + modelSettlement, + ...(failureEvidence ? { failureEvidence } : {}), + process: Object.freeze({ ...evidence.process }), + model: Object.freeze({ ...evidence.model }), + ...(evidence.memory ? { memory: Object.freeze({ ...evidence.memory }) } : {}), + }) +} + +function sealTerminalResult( + result: AgentCandidateExecutionTerminalResult, +): AgentCandidateExecutionTerminalResult { + if (result.status !== 'succeeded' && result.status !== 'failed') { + throw new Error('candidate execution terminal status is invalid') + } + assertExactKeys( + result, + result.status === 'succeeded' + ? [ + 'schemaVersion', + 'status', + 'usage', + 'modelSettlement', + 'taskOutcome', + 'benchmarkResult', + 'runReceipt', + ] + : [ + 'schemaVersion', + 'status', + 'failureClass', + 'usage', + 'modelSettlement', + ...(result.failureEvidence ? ['failureEvidence'] : []), + ], + 'candidate execution terminal result', + ) + if (result.schemaVersion !== 1) { + throw new Error('candidate execution terminal schemaVersion must be 1') + } + const usage = sealUsage(result.usage) + const modelSettlement = sealArtifactRef(result.modelSettlement, 'modelSettlement') + if (result.status === 'succeeded') { + return Object.freeze({ + schemaVersion: 1, + status: 'succeeded', + usage, + modelSettlement, + taskOutcome: sealArtifactRef(result.taskOutcome, 'taskOutcome'), + benchmarkResult: sealArtifactRef(result.benchmarkResult, 'benchmarkResult'), + runReceipt: sealArtifactRef(result.runReceipt, 'runReceipt'), + }) + } + assertFailureClass(result.failureClass) + if (result.failureClass === 'pre-model-infrastructure' && usage.modelCalls !== 0) { + throw new Error('pre-model infrastructure failure cannot contain model calls') + } + return Object.freeze({ + schemaVersion: 1, + status: 'failed', + failureClass: result.failureClass, + usage, + modelSettlement, + ...(result.failureEvidence + ? { failureEvidence: sealArtifactRef(result.failureEvidence, 'failureEvidence') } + : {}), + }) +} + +function sealUsage(usage: AgentCandidateExecutionUsage): AgentCandidateExecutionUsage { + assertExactKeys( + usage, + [ + 'costUsdNanos', + 'inputTokens', + 'outputTokens', + 'cachedInputTokens', + 'reasoningTokens', + 'modelCalls', + ], + 'candidate execution terminal usage', + ) + for (const [field, value] of Object.entries(usage)) { + assertCount(value, `terminal usage ${field}`) + } + return Object.freeze({ + costUsdNanos: usage.costUsdNanos, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + cachedInputTokens: usage.cachedInputTokens, + reasoningTokens: usage.reasoningTokens, + modelCalls: usage.modelCalls, + }) +} + +function sealArtifactRef(ref: AgentCandidateArtifactRef, label: string): AgentCandidateArtifactRef { + const parsed = agentCandidateArtifactRefSchema.parse(ref) + if (!Number.isSafeInteger(parsed.byteLength)) { + throw new Error(`candidate execution terminal ${label} byteLength exceeds safe integer range`) + } + return immutableCandidateValue(parsed) +} + +function requireArtifactRef( + value: unknown, + path: string, + field: string, +): AgentCandidateArtifactRef { + return requireObject(value, path, field) as unknown as AgentCandidateArtifactRef +} + +function requireString(value: unknown, path: string, field: string): string { + if (typeof value !== 'string') { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value +} + +function requireNumber(value: unknown, path: string, field: string): number { + if (typeof value !== 'number') { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value +} + +function requireObject(value: unknown, path: string, field: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value as Record +} + +function requireTerminalStatus( + value: unknown, + path: string, +): AgentCandidateExecutionTerminalResult['status'] { + if (value !== 'succeeded' && value !== 'failed') { + throw new Error(`candidate execution terminal record at ${path} has invalid status`) + } + return value +} + +function requireFailureClass(value: unknown, path: string): AgentCandidateExecutionFailureClass { + try { + assertFailureClass(value) + return value + } catch (error) { + throw new Error(`candidate execution terminal record at ${path} has invalid failureClass`, { + cause: error, + }) + } +} + +function assertFailureClass(value: unknown): asserts value is AgentCandidateExecutionFailureClass { + if ( + value !== 'pre-model-infrastructure' && + value !== 'execution' && + value !== 'post-model-infrastructure' && + value !== 'unknown' + ) { + throw new Error('candidate execution failureClass is invalid') + } +} + +function assertExecutionId(value: unknown): asserts value is string { + if (typeof value !== 'string' || !/^[A-Za-z0-9._:-]{1,200}$/.test(value)) { + throw new Error('candidate execution claim executionId is invalid') + } +} + +function assertSha256Digest(value: string, field: string): void { + if (!SHA256_PATTERN.test(value)) { + throw new Error(`candidate execution claim ${field} must be a lowercase sha256 digest`) + } +} + +function assertCount(value: unknown, label: string): asserts value is number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new Error(`candidate execution ${label} must be a non-negative safe integer`) + } +} diff --git a/src/candidate-execution/claim.ts b/src/candidate-execution/claim.ts new file mode 100644 index 00000000..e31ac79d --- /dev/null +++ b/src/candidate-execution/claim.ts @@ -0,0 +1,946 @@ +/** Durable one-shot lifecycle for candidate execution attempts. */ + +import { createHash, randomBytes, timingSafeEqual } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import { + type AgentCandidateArtifactRef, + type AgentCandidateAttemptPolicy, + type AgentCandidateResolvedModel, + agentCandidateResolvedModelSchema, + type Sha256Digest, +} from '@tangle-network/agent-interface' +import { + CLAIM_FORMAT_VERSION, + PENDING_FORMAT_VERSION, + type PersistedAgentCandidateExecutionClaim, + type PersistedAgentCandidateExecutionPending, + type PersistedAgentCandidateExecutionPhase, + type PersistedAgentCandidateExecutionTerminal, + PHASE_FORMAT_VERSION, + TERMINAL_FORMAT_VERSION, +} from './claim-file-formats' +import { + assertRecoveryMatchesStaged, + assertTerminalAllowedInPhase, + assertTerminalMatchesClaim, + recoveredTerminalRecord, + rejectedFinish, + rejectedStage, + requireStagedTerminal, + sealTerminalDigest, + sealTerminalRecordValue, + terminalRecord, +} from './claim-terminal' +import { candidateCleanupTimeout, candidateResultTimeout } from './cleanup' +import { canonicalCandidateDigest, immutableCandidateValue } from './digest' +import { assertExactObjectKeys as assertExactKeys } from './exact-object' + +/** Non-secret identities a trusted recovery worker needs to close an abandoned attempt. */ +export interface AgentCandidateExecutionCleanupHandles { + readonly preparationId: string + readonly modelGrantDigest: Sha256Digest + readonly resolvedModel: AgentCandidateResolvedModel + readonly traceRunId: string + readonly cleanupTimeoutMs: number + readonly memory?: { + readonly accessDigest: Sha256Digest + readonly effectiveNamespace: string + } +} + +/** Immutable signed identity stored for one execution attempt. */ +export interface AgentCandidateExecutionClaim { + readonly executionId: string + readonly attempt: number + readonly maxAttempts: number + readonly retryPolicy: AgentCandidateAttemptPolicy['retryPolicy'] + readonly bundleDigest: Sha256Digest + readonly executionPlanDigest: Sha256Digest + /** Frozen plan identity with only attempt number and per-attempt grant identity normalized. */ + readonly retryLineageDigest: Sha256Digest + /** The winning lease stops authorizing a new terminal write at this instant. */ + readonly leaseExpiresAtMs: number + /** Frozen budget for task verification, executable grading, and receipt construction. */ + readonly resultTimeoutMs: number + /** Non-secret handles retained so an expired attempt can be closed and reconciled. */ + readonly cleanup: AgentCandidateExecutionCleanupHandles +} + +/** Secret capability required to finish the acquired attempt. */ +export interface AgentCandidateExecutionLease { + readonly executionId: string + readonly attempt: number + readonly token: string + readonly expiresAtMs: number +} + +/** Only the first class is retryable, and only when the closed model ledger has zero calls. */ +export type AgentCandidateExecutionFailureClass = + | 'pre-model-infrastructure' + | 'execution' + | 'post-model-infrastructure' + | 'unknown' + +/** Exact fixed-point usage proven by the closed evaluator model ledger. */ +export interface AgentCandidateExecutionUsage { + readonly costUsdNanos: number + readonly inputTokens: number + readonly outputTokens: number + readonly cachedInputTokens: number + readonly reasoningTokens: number + readonly modelCalls: number +} + +/** Evaluator-owned terminal facts staged durably before the terminal CAS. */ +export type AgentCandidateExecutionTerminalResult = + | { + readonly schemaVersion: 1 + readonly status: 'succeeded' + readonly usage: AgentCandidateExecutionUsage + readonly modelSettlement: AgentCandidateArtifactRef + readonly taskOutcome: AgentCandidateArtifactRef + readonly benchmarkResult: AgentCandidateArtifactRef + readonly runReceipt: AgentCandidateArtifactRef + } + | { + readonly schemaVersion: 1 + readonly status: 'failed' + readonly failureClass: AgentCandidateExecutionFailureClass + readonly usage: AgentCandidateExecutionUsage + readonly modelSettlement: AgentCandidateArtifactRef + readonly failureEvidence?: AgentCandidateArtifactRef + } + +/** Durable terminal record for one acquired execution attempt. */ +export type AgentCandidateExecutionTerminalRecord = AgentCandidateExecutionTerminalResult & { + readonly executionId: string + readonly attempt: number + readonly bundleDigest: Sha256Digest + readonly executionPlanDigest: Sha256Digest + /** RFC 8785 SHA-256 of this record with `terminalDigest` omitted. */ + readonly terminalDigest: Sha256Digest +} + +/** Monotonic durable phase: the second value means candidate code could have started. */ +export type AgentCandidateExecutionPhase = 'claimed' | 'candidate-may-run' + +/** Trusted, independently observed closure facts for one expired winning lease. */ +export interface AgentCandidateExecutionRecoveryEvidence { + readonly failureClass: AgentCandidateExecutionFailureClass + readonly usage: AgentCandidateExecutionUsage + readonly modelSettlement: AgentCandidateArtifactRef + readonly failureEvidence?: AgentCandidateArtifactRef + readonly process: { + readonly stopped: true + readonly executionPlanDigest: Sha256Digest + } + readonly model: { + readonly closed: true + readonly preparationId: string + readonly grantDigest: Sha256Digest + } + readonly memory?: { + readonly closed: true + readonly preparationId: string + readonly accessDigest: Sha256Digest + readonly effectiveNamespace: string + } +} + +export interface AgentCandidateExecutionAttemptRef { + readonly executionId: string + readonly attempt: number +} + +/** Persisted state available to a fresh trusted recovery worker after a crash. */ +export interface AgentCandidateExecutionAttemptRecord { + readonly claim: AgentCandidateExecutionClaim + readonly phase: AgentCandidateExecutionPhase + /** Durable outbox content written before the terminal compare-and-set. */ + readonly staged?: AgentCandidateExecutionTerminalRecord + readonly terminal?: AgentCandidateExecutionTerminalRecord +} + +/** Result of atomically claiming one execution attempt. */ +export type AgentCandidateExecutionClaimResult = + | { + readonly acquired: true + readonly claim: AgentCandidateExecutionClaim + readonly lease: AgentCandidateExecutionLease + } + | { + readonly acquired: false + readonly reason: 'already-claimed' + /** The durable winner already occupying this execution-attempt slot. */ + readonly claim: AgentCandidateExecutionClaim + /** True only when every signed claim field matches the durable winner. */ + readonly exactReplay: boolean + } + | { + readonly acquired: false + readonly reason: 'retry-not-eligible' + readonly claim: AgentCandidateExecutionClaim + readonly detail: AgentCandidateRetryRejection + } + +/** Result of atomically recording an attempt's terminal facts. */ +export type AgentCandidateExecutionFinishResult = + | { + readonly finished: true + readonly terminal: AgentCandidateExecutionTerminalRecord + } + | { + readonly finished: false + readonly terminal: AgentCandidateExecutionTerminalRecord + /** True when a repeated finish supplied the same terminal digest. */ + readonly exactReplay: boolean + } + +/** Result of durably staging the one immutable terminal outbox entry. */ +export type AgentCandidateExecutionStageResult = + | { + readonly staged: true + readonly terminal: AgentCandidateExecutionTerminalRecord + } + | { + readonly staged: false + readonly terminal: AgentCandidateExecutionTerminalRecord + readonly exactReplay: boolean + } + +/** Result of crossing the irreversible candidate-may-run boundary. */ +export type AgentCandidateExecutionPhaseResult = + | { readonly marked: true; readonly phase: 'candidate-may-run' } + | { readonly marked: false; readonly phase: 'candidate-may-run' } + +export type AgentCandidateRetryRejection = + | 'prior-attempt-missing' + | 'prior-attempt-running' + | 'prior-attempt-succeeded' + | 'prior-attempt-spent-model-calls' + | 'prior-attempt-not-pre-model-infrastructure' + | 'retry-lineage-mismatch' + +/** + * Atomic one-shot store for candidate execution attempts. + * + * Implementations must linearize both methods across every process sharing the + * store. Terminal publication is deliberately two-step: `stageTerminal` + * fsyncs the complete immutable outbox record, then `finish` publishes exactly + * those staged bytes by digest. A crash between the two leaves recoverable + * evidence rather than an ambiguous completed run. + */ +export interface AgentCandidateExecutionClaimStore { + tryClaim(claim: AgentCandidateExecutionClaim): Promise + getAttempt( + attempt: AgentCandidateExecutionAttemptRef, + ): Promise + /** Persist the point after which candidate code may have run. */ + markCandidateMayRun( + lease: AgentCandidateExecutionLease, + ): Promise + /** Fsync the complete terminal record into the durable outbox. */ + stageTerminal( + lease: AgentCandidateExecutionLease, + result: AgentCandidateExecutionTerminalResult, + ): Promise + /** Publish exactly the staged terminal identified by `terminalDigest`. */ + finish( + lease: AgentCandidateExecutionLease, + terminalDigest: Sha256Digest, + ): Promise + /** + * Write a failed terminal only after the lease expired and a trusted worker + * independently proved process death plus model and memory closure. + */ + recoverExpired( + attempt: AgentCandidateExecutionAttemptRef, + evidence: AgentCandidateExecutionRecoveryEvidence, + ): Promise +} + +interface StoredClaim { + claim: AgentCandidateExecutionClaim + leaseDigest: Sha256Digest + phase: AgentCandidateExecutionPhase + staged?: AgentCandidateExecutionTerminalRecord + terminal?: AgentCandidateExecutionTerminalRecord +} + +function attemptRecord( + claim: AgentCandidateExecutionClaim, + phase: AgentCandidateExecutionPhase, + staged?: AgentCandidateExecutionTerminalRecord, + terminal?: AgentCandidateExecutionTerminalRecord, +): AgentCandidateExecutionAttemptRecord { + return Object.freeze({ + claim, + phase, + ...(staged ? { staged } : {}), + ...(terminal ? { terminal } : {}), + }) +} + +export interface InMemoryAgentCandidateExecutionClaimStoreOptions { + /** Testable evaluator clock; defaults to `Date.now`. */ + now?: () => number +} + +/** Single-process lifecycle implementation. */ +export class InMemoryAgentCandidateExecutionClaimStore + implements AgentCandidateExecutionClaimStore +{ + private readonly claims = new Map() + private readonly now: () => number + + constructor(options: InMemoryAgentCandidateExecutionClaimStoreOptions = {}) { + this.now = options.now ?? Date.now + } + + async tryClaim( + requested: AgentCandidateExecutionClaim, + ): Promise { + const claim = sealClaim(requested) + const slot = claimSlot(claim) + const existing = this.claims.get(slot) + if (existing) return rejectedExistingClaim(existing.claim, claim) + assertUnexpiredLease(claim.leaseExpiresAtMs, this.now()) + + const retryRejection = retryRejectionFromMemory(this.claims, claim) + if (retryRejection) return rejectedRetry(claim, retryRejection) + + const lease = newLease(claim) + // No await may occur between the read and write: this is the linearization + // point for every caller sharing this store instance. + this.claims.set(slot, { + claim, + leaseDigest: leaseDigest(lease), + phase: 'claimed', + }) + return Object.freeze({ acquired: true, claim, lease }) + } + + async getAttempt( + requestedAttempt: AgentCandidateExecutionAttemptRef, + ): Promise { + const attempt = sealAttemptRef(requestedAttempt) + const stored = this.claims.get(claimSlot(attempt)) + return stored + ? attemptRecord(stored.claim, stored.phase, stored.staged, stored.terminal) + : undefined + } + + async markCandidateMayRun( + requestedLease: AgentCandidateExecutionLease, + ): Promise { + const lease = sealLease(requestedLease) + const stored = this.requireClaim(lease) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + if (stored.phase === 'candidate-may-run') { + return Object.freeze({ marked: false, phase: 'candidate-may-run' }) + } + if (stored.staged || stored.terminal) { + throw new Error('candidate execution terminal was staged before candidate-may-run phase') + } + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + stored.phase = 'candidate-may-run' + return Object.freeze({ marked: true, phase: 'candidate-may-run' }) + } + + async stageTerminal( + requestedLease: AgentCandidateExecutionLease, + result: AgentCandidateExecutionTerminalResult, + ): Promise { + const lease = sealLease(requestedLease) + const stored = this.requireClaim(lease) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + const terminal = terminalRecord(stored.claim, result) + if (stored.staged) return rejectedStage(stored.staged, terminal) + assertTerminalAllowedInPhase(stored.phase, terminal) + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + stored.staged = terminal + return Object.freeze({ staged: true, terminal }) + } + + async finish( + requestedLease: AgentCandidateExecutionLease, + requestedTerminalDigest: Sha256Digest, + ): Promise { + const lease = sealLease(requestedLease) + const terminalDigest = sealTerminalDigest(requestedTerminalDigest) + const stored = this.requireClaim(lease) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + const staged = requireStagedTerminal(stored.staged, terminalDigest) + if (stored.terminal) return rejectedFinish(stored.terminal, terminalDigest) + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + + // The terminal assignment is the in-memory finish linearization point and + // publishes the exact immutable object already present in the outbox. + stored.terminal = staged + return Object.freeze({ finished: true, terminal: staged }) + } + + async recoverExpired( + requestedAttempt: AgentCandidateExecutionAttemptRef, + evidence: AgentCandidateExecutionRecoveryEvidence, + ): Promise { + const attempt = sealAttemptRef(requestedAttempt) + const stored = this.requireClaim(attempt, 'candidate execution recovery') + const recovered = recoveredTerminalRecord(stored.claim, stored.phase, evidence) + if (stored.staged) assertRecoveryMatchesStaged(stored.staged, recovered) + const requestedDigest = stored.staged?.terminalDigest ?? recovered.terminalDigest + if (stored.terminal) return rejectedFinish(stored.terminal, requestedDigest) + assertExpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + const terminal = stored.staged ?? recovered + assertRecoveryMatchesStaged(terminal, recovered) + stored.staged ??= terminal + stored.terminal = terminal + return Object.freeze({ finished: true, terminal }) + } + + private requireClaim( + attempt: Pick, + operation = 'candidate execution lease', + ): StoredClaim { + const stored = this.claims.get(claimSlot(attempt)) + if (!stored) throw new Error(`${operation} does not name an acquired attempt`) + return stored + } +} + +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/ +const LEASE_TOKEN_PATTERN = /^candidate-execution-lease-v1\.[A-Za-z0-9_-]{43}$/ +const PREPARATION_ID_PATTERN = /^candidate-preparation-v1\.[A-Za-z0-9_-]{43}$/ + +function sealClaim(claim: AgentCandidateExecutionClaim): AgentCandidateExecutionClaim { + assertExactKeys( + claim, + [ + 'executionId', + 'attempt', + 'maxAttempts', + 'retryPolicy', + 'bundleDigest', + 'executionPlanDigest', + 'retryLineageDigest', + 'leaseExpiresAtMs', + 'resultTimeoutMs', + 'cleanup', + ], + 'candidate execution claim', + ) + assertExecutionId(claim.executionId) + if (!Number.isSafeInteger(claim.attempt) || claim.attempt < 1) { + throw new Error('candidate execution claim attempt must be a positive safe integer') + } + if (!Number.isSafeInteger(claim.maxAttempts) || claim.maxAttempts < 1) { + throw new Error('candidate execution claim maxAttempts must be a positive safe integer') + } + if (claim.attempt > claim.maxAttempts) { + throw new Error('candidate execution claim attempt exceeds maxAttempts') + } + if (!['none', 'pre-model-infrastructure-only'].includes(claim.retryPolicy)) { + throw new Error('candidate execution claim retryPolicy is invalid') + } + if (claim.retryPolicy === 'none' && claim.maxAttempts !== 1) { + throw new Error('candidate execution claim retryPolicy none requires maxAttempts 1') + } + assertSha256Digest(claim.bundleDigest, 'bundleDigest') + assertSha256Digest(claim.executionPlanDigest, 'executionPlanDigest') + assertSha256Digest(claim.retryLineageDigest, 'retryLineageDigest') + assertPositiveTimestamp(claim.leaseExpiresAtMs, 'leaseExpiresAtMs') + candidateResultTimeout(claim.resultTimeoutMs, claim.resultTimeoutMs) + const cleanup = sealCleanupHandles(claim.cleanup) + return Object.freeze({ + executionId: claim.executionId, + attempt: claim.attempt, + maxAttempts: claim.maxAttempts, + retryPolicy: claim.retryPolicy, + bundleDigest: claim.bundleDigest, + executionPlanDigest: claim.executionPlanDigest, + retryLineageDigest: claim.retryLineageDigest, + leaseExpiresAtMs: claim.leaseExpiresAtMs, + resultTimeoutMs: claim.resultTimeoutMs, + cleanup, + }) +} + +function sealCleanupHandles( + cleanup: AgentCandidateExecutionCleanupHandles, +): AgentCandidateExecutionCleanupHandles { + assertExactKeys( + cleanup, + cleanup.memory + ? [ + 'preparationId', + 'modelGrantDigest', + 'resolvedModel', + 'traceRunId', + 'cleanupTimeoutMs', + 'memory', + ] + : ['preparationId', 'modelGrantDigest', 'resolvedModel', 'traceRunId', 'cleanupTimeoutMs'], + 'candidate execution cleanup handles', + ) + if (!PREPARATION_ID_PATTERN.test(cleanup.preparationId)) { + throw new Error('candidate execution cleanup preparationId is invalid') + } + assertSha256Digest(cleanup.modelGrantDigest, 'cleanup modelGrantDigest') + const resolvedModel = immutableCandidateValue( + agentCandidateResolvedModelSchema.parse(cleanup.resolvedModel), + ) + assertBoundedIdentifier(cleanup.traceRunId, 'cleanup traceRunId', 512) + const cleanupTimeoutMs = candidateCleanupTimeout(cleanup.cleanupTimeoutMs) + const memory = cleanup.memory ? sealMemoryCleanupHandle(cleanup.memory) : undefined + return Object.freeze({ + preparationId: cleanup.preparationId, + modelGrantDigest: cleanup.modelGrantDigest, + resolvedModel, + traceRunId: cleanup.traceRunId, + cleanupTimeoutMs, + ...(memory ? { memory } : {}), + }) +} + +function sealMemoryCleanupHandle( + memory: NonNullable, +): NonNullable { + assertExactKeys( + memory, + ['accessDigest', 'effectiveNamespace'], + 'candidate execution memory cleanup handle', + ) + assertSha256Digest(memory.accessDigest, 'memory accessDigest') + assertBoundedIdentifier(memory.effectiveNamespace, 'memory effectiveNamespace', 1_024) + return Object.freeze({ + accessDigest: memory.accessDigest, + effectiveNamespace: memory.effectiveNamespace, + }) +} + +function sealAttemptRef( + attempt: AgentCandidateExecutionAttemptRef, +): AgentCandidateExecutionAttemptRef { + assertExactKeys(attempt, ['executionId', 'attempt'], 'candidate execution attempt reference') + assertExecutionId(attempt.executionId) + if (!Number.isSafeInteger(attempt.attempt) || attempt.attempt < 1) { + throw new Error('candidate execution attempt reference must have a positive safe attempt') + } + return Object.freeze({ executionId: attempt.executionId, attempt: attempt.attempt }) +} + +function sealLease(lease: AgentCandidateExecutionLease): AgentCandidateExecutionLease { + assertExactKeys( + lease, + ['executionId', 'attempt', 'token', 'expiresAtMs'], + 'candidate execution lease', + ) + if (lease.executionId.length === 0 || !Number.isSafeInteger(lease.attempt) || lease.attempt < 1) { + throw new Error('candidate execution lease identity is invalid') + } + if (!LEASE_TOKEN_PATTERN.test(lease.token)) { + throw new Error('candidate execution lease token is invalid') + } + assertPositiveTimestamp(lease.expiresAtMs, 'lease expiresAtMs') + return Object.freeze({ + executionId: lease.executionId, + attempt: lease.attempt, + token: lease.token, + expiresAtMs: lease.expiresAtMs, + }) +} + +function newLease(claim: AgentCandidateExecutionClaim): AgentCandidateExecutionLease { + return Object.freeze({ + executionId: claim.executionId, + attempt: claim.attempt, + token: `candidate-execution-lease-v1.${randomBytes(32).toString('base64url')}`, + expiresAtMs: claim.leaseExpiresAtMs, + }) +} + +function leaseDigest(lease: AgentCandidateExecutionLease): Sha256Digest { + return sha256(lease.token) +} + +function assertLease( + expectedDigest: Sha256Digest, + expectedExpiresAtMs: number, + lease: AgentCandidateExecutionLease, +): void { + const expected = Buffer.from(expectedDigest) + const actual = Buffer.from(leaseDigest(lease)) + if ( + expected.length !== actual.length || + !timingSafeEqual(expected, actual) || + lease.expiresAtMs !== expectedExpiresAtMs + ) { + throw new Error('candidate execution lease is invalid') + } +} + +function claimSlot(claim: Pick): string { + return createHash('sha256') + .update(JSON.stringify([claim.executionId, claim.attempt]), 'utf8') + .digest('hex') +} + +function retryRejectionFromMemory( + claims: ReadonlyMap, + claim: AgentCandidateExecutionClaim, +): AgentCandidateRetryRejection | undefined { + if (claim.attempt === 1) return undefined + const prior = claims.get( + claimSlot({ executionId: claim.executionId, attempt: claim.attempt - 1 }), + ) + return retryRejection(claim, prior) +} + +function retryRejection( + claim: AgentCandidateExecutionClaim, + prior: + | { + claim: AgentCandidateExecutionClaim + terminal?: AgentCandidateExecutionTerminalRecord + } + | undefined, +): AgentCandidateRetryRejection | undefined { + if (!prior) return 'prior-attempt-missing' + if ( + claim.retryPolicy !== 'pre-model-infrastructure-only' || + prior.claim.retryPolicy !== claim.retryPolicy || + prior.claim.maxAttempts !== claim.maxAttempts || + prior.claim.bundleDigest !== claim.bundleDigest || + prior.claim.retryLineageDigest !== claim.retryLineageDigest + ) { + return 'retry-lineage-mismatch' + } + if (!prior.terminal) return 'prior-attempt-running' + if (prior.terminal.status === 'succeeded') return 'prior-attempt-succeeded' + if (prior.terminal.usage.modelCalls !== 0) return 'prior-attempt-spent-model-calls' + if (prior.terminal.failureClass !== 'pre-model-infrastructure') { + return 'prior-attempt-not-pre-model-infrastructure' + } + return undefined +} + +function rejectedExistingClaim( + existing: AgentCandidateExecutionClaim, + requested: AgentCandidateExecutionClaim, +): AgentCandidateExecutionClaimResult { + return Object.freeze({ + acquired: false, + reason: 'already-claimed', + claim: existing, + exactReplay: canonicalCandidateDigest(existing) === canonicalCandidateDigest(requested), + }) +} + +function rejectedRetry( + claim: AgentCandidateExecutionClaim, + detail: AgentCandidateRetryRejection, +): AgentCandidateExecutionClaimResult { + return Object.freeze({ acquired: false, reason: 'retry-not-eligible', claim, detail }) +} + +async function readClaim(path: string): Promise { + const parsed = await readJsonObject(path, 'claim') + const record = parsed as Partial + if (record.version !== CLAIM_FORMAT_VERSION) { + throw new Error(`candidate execution claim at ${path} has unsupported version`) + } + assertExactKeys( + parsed, + [ + 'version', + 'executionId', + 'attempt', + 'maxAttempts', + 'retryPolicy', + 'bundleDigest', + 'executionPlanDigest', + 'retryLineageDigest', + 'leaseExpiresAtMs', + 'resultTimeoutMs', + 'cleanup', + 'phase', + 'leaseDigest', + ], + `candidate execution claim at ${path}`, + ) + const claim = sealClaim({ + executionId: requireString(record.executionId, path, 'executionId'), + attempt: requireNumber(record.attempt, path, 'attempt'), + maxAttempts: requireNumber(record.maxAttempts, path, 'maxAttempts'), + retryPolicy: requireRetryPolicy(record.retryPolicy, path), + bundleDigest: requireString(record.bundleDigest, path, 'bundleDigest') as Sha256Digest, + executionPlanDigest: requireString( + record.executionPlanDigest, + path, + 'executionPlanDigest', + ) as Sha256Digest, + retryLineageDigest: requireString( + record.retryLineageDigest, + path, + 'retryLineageDigest', + ) as Sha256Digest, + leaseExpiresAtMs: requireNumber(record.leaseExpiresAtMs, path, 'leaseExpiresAtMs'), + resultTimeoutMs: requireNumber(record.resultTimeoutMs, path, 'resultTimeoutMs'), + cleanup: requireObject( + record.cleanup, + path, + 'cleanup', + ) as unknown as AgentCandidateExecutionCleanupHandles, + }) + const persistedLeaseDigest = requireString( + record.leaseDigest, + path, + 'leaseDigest', + ) as Sha256Digest + assertSha256Digest(persistedLeaseDigest, 'leaseDigest') + if (record.phase !== 'claimed') { + throw new Error(`candidate execution claim at ${path} has invalid initial phase`) + } + return { claim, leaseDigest: persistedLeaseDigest, phase: 'claimed' } +} + +async function readClaimIfPresent(path: string): Promise { + try { + return await readClaim(path) + } catch (error) { + if (isMissingError(error)) return undefined + throw error + } +} + +async function readTerminal(path: string): Promise { + const parsed = await readJsonObject(path, 'terminal record') + const record = parsed as Partial + if (record.version !== TERMINAL_FORMAT_VERSION) { + throw new Error(`candidate execution terminal record at ${path} has unsupported version`) + } + assertExactKeys(parsed, ['version', 'terminal'], `candidate execution terminal record at ${path}`) + return sealTerminalRecordValue( + requireObject(record.terminal, path, 'terminal'), + `candidate execution terminal record at ${path}`, + ) +} + +async function readTerminalIfPresent( + path: string, +): Promise { + try { + return await readTerminal(path) + } catch (error) { + if (isMissingError(error)) return undefined + throw error + } +} + +type ReadCandidateExecutionTransition = + | { kind: 'phase' } + | { kind: 'pending'; terminal: AgentCandidateExecutionTerminalRecord } + +async function readTransitionIfPresent( + path: string, + claim: AgentCandidateExecutionClaim, +): Promise { + let parsed: Record + try { + parsed = await readJsonObject(path, 'transition record') + } catch (error) { + if (isMissingError(error)) return undefined + throw error + } + if (parsed.kind === 'candidate-execution-phase') { + const record = parsed as unknown as PersistedAgentCandidateExecutionPhase + if (record.version !== PHASE_FORMAT_VERSION) { + throw new Error(`candidate execution phase record at ${path} has unsupported version`) + } + assertExactKeys( + parsed, + ['version', 'kind', 'executionId', 'attempt', 'executionPlanDigest', 'phase'], + `candidate execution phase record at ${path}`, + ) + if ( + record.executionId !== claim.executionId || + record.attempt !== claim.attempt || + record.executionPlanDigest !== claim.executionPlanDigest || + record.phase !== 'candidate-may-run' + ) { + throw new Error(`candidate execution phase record at ${path} does not match its claim`) + } + return { kind: 'phase' } + } + if (parsed.kind === 'candidate-execution-pending-terminal') { + const record = parsed as unknown as PersistedAgentCandidateExecutionPending + if (record.version !== PENDING_FORMAT_VERSION) { + throw new Error(`candidate execution pending record at ${path} has unsupported version`) + } + assertExactKeys( + parsed, + ['version', 'kind', 'terminal'], + `candidate execution pending record at ${path}`, + ) + const terminal = sealTerminalRecordValue( + requireObject(record.terminal, path, 'terminal'), + `candidate execution pending record at ${path}`, + ) + assertTerminalMatchesClaim(terminal, claim, path) + return { kind: 'pending', terminal } + } + throw new Error(`candidate execution transition record at ${path} has invalid kind`) +} + +async function readJsonObject(path: string, kind: string): Promise> { + let parsed: unknown + try { + parsed = JSON.parse(await readFile(path, 'utf8')) + } catch (error) { + if (isMissingError(error)) throw error + throw new Error(`candidate execution ${kind} at ${path} is unreadable`, { cause: error }) + } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`candidate execution ${kind} at ${path} is not an object`) + } + return parsed as Record +} + +function assertSameSlot( + existing: Pick, + requested: Pick, + path: string, +): void { + if (existing.executionId !== requested.executionId || existing.attempt !== requested.attempt) { + throw new Error(`candidate execution record at ${path} does not match its claim slot`) + } +} + +function assertSha256Digest(value: string, field: string): void { + if (!SHA256_PATTERN.test(value)) { + throw new Error(`candidate execution claim ${field} must be a lowercase sha256 digest`) + } +} + +function requireString(value: unknown, path: string, field: string): string { + if (typeof value !== 'string') { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value +} + +function requireNumber(value: unknown, path: string, field: string): number { + if (typeof value !== 'number') { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value +} + +function requireObject(value: unknown, path: string, field: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value as Record +} + +function requireRetryPolicy( + value: unknown, + path: string, +): AgentCandidateAttemptPolicy['retryPolicy'] { + if (value !== 'none' && value !== 'pre-model-infrastructure-only') { + throw new Error(`candidate execution record at ${path} has invalid retryPolicy`) + } + return value +} + +function assertExecutionId(value: unknown): asserts value is string { + if (typeof value !== 'string' || !/^[A-Za-z0-9._:-]{1,200}$/.test(value)) { + throw new Error('candidate execution claim executionId is invalid') + } +} + +function assertBoundedIdentifier( + value: unknown, + label: string, + maxLength: number, +): asserts value is string { + if ( + typeof value !== 'string' || + value.length === 0 || + value.length > maxLength || + hasControlCharacter(value) + ) { + throw new Error(`candidate execution ${label} is invalid`) + } +} + +function hasControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index) + if (code < 0x20 || code === 0x7f) return true + } + return false +} + +function assertPositiveTimestamp(value: unknown, label: string): asserts value is number { + if (!Number.isSafeInteger(value) || (value as number) <= 0) { + throw new Error(`candidate execution ${label} must be a positive safe timestamp`) + } +} + +function assertClock(value: number): void { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error('candidate execution claim-store clock returned an invalid timestamp') + } +} + +function assertUnexpiredLease(expiresAtMs: number, nowMs: number): void { + assertClock(nowMs) + if (nowMs >= expiresAtMs) throw new Error('candidate execution lease has expired') +} + +function assertExpiredLease(expiresAtMs: number, nowMs: number): void { + assertClock(nowMs) + if (nowMs < expiresAtMs) throw new Error('candidate execution lease has not expired') +} + +function sha256(value: string): Sha256Digest { + return `sha256:${createHash('sha256').update(value, 'utf8').digest('hex')}` +} + +function isMissingError(error: unknown): boolean { + return isNodeError(error, 'ENOENT') +} + +function isNodeError(error: unknown, code: string): boolean { + return ( + error !== null && + typeof error === 'object' && + 'code' in error && + (error as { code?: unknown }).code === code + ) +} + +export const candidateClaimFileInternals = Object.freeze({ + assertExpiredLease, + assertLease, + assertSameSlot, + assertUnexpiredLease, + attemptRecord, + claimSlot, + leaseDigest, + newLease, + readClaim, + readClaimIfPresent, + readTerminal, + readTerminalIfPresent, + readTransitionIfPresent, + rejectedExistingClaim, + rejectedRetry, + retryRejection, + sealAttemptRef, + sealClaim, + sealLease, +}) + +export { candidateExecutionClaim } from './claim-plan' diff --git a/src/candidate-execution/cleanup.ts b/src/candidate-execution/cleanup.ts new file mode 100644 index 00000000..daea4b9e --- /dev/null +++ b/src/candidate-execution/cleanup.ts @@ -0,0 +1,114 @@ +export const DEFAULT_CANDIDATE_CLEANUP_TIMEOUT_MS = 30_000 +/** Largest delay Node schedules without clamping to one millisecond. */ +export const MAX_CANDIDATE_TIMER_INTERVAL_MS = 2_147_483_647 + +export function candidateCleanupTimeout(timeoutMs: number | undefined): number { + const effective = timeoutMs ?? DEFAULT_CANDIDATE_CLEANUP_TIMEOUT_MS + if ( + !Number.isSafeInteger(effective) || + effective <= 0 || + effective > MAX_CANDIDATE_TIMER_INTERVAL_MS + ) { + throw new Error('candidate cleanup timeout is outside the supported timer range') + } + return effective +} + +export function candidateCleanupDeadline(timeoutMs: number | undefined): number { + return Date.now() + candidateCleanupTimeout(timeoutMs) +} + +/** Freeze a separate result-construction budget; defaults to the task wall limit. */ +export function candidateResultTimeout( + timeoutMs: number | undefined, + taskTimeoutMs: number, +): number { + const effective = timeoutMs ?? taskTimeoutMs + if ( + !Number.isSafeInteger(effective) || + effective <= 0 || + effective > MAX_CANDIDATE_TIMER_INTERVAL_MS + ) { + throw new Error('candidate result timeout is outside the supported timer range') + } + return effective +} + +/** Bound an evaluator cleanup call while keeping late rejection observed. */ +export async function withinCandidateCleanupDeadline( + operation: () => Promise, + deadlineAtMs: number, + label: string, +): Promise { + const remainingMs = deadlineAtMs - Date.now() + if (remainingMs <= 0) throw new CandidateCleanupTimeoutError(label) + + const pending = Promise.resolve().then(operation) + void pending.catch(() => undefined) + let timer: ReturnType | undefined + try { + const result = await Promise.race([ + pending, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new CandidateCleanupTimeoutError(label)), remainingMs) + }), + ]) + // Exact-boundary completion is ambiguous under event-loop delay. + if (Date.now() >= deadlineAtMs) throw new CandidateCleanupTimeoutError(label) + return result + } finally { + if (timer) clearTimeout(timer) + } +} + +/** + * Bound cancellable scoring/result work. Every side-effecting port called by + * `operation` must honor the supplied signal before durable publication. + */ +export async function withinCandidateResultDeadline( + operation: (signal: AbortSignal) => Promise, + deadlineAtMs: number, + label: string, +): Promise { + const remainingMs = deadlineAtMs - Date.now() + if (remainingMs <= 0) throw new CandidateResultTimeoutError(label) + + const controller = new AbortController() + const timeoutError = new CandidateResultTimeoutError(label) + const pending = Promise.resolve().then(() => operation(controller.signal)) + void pending.catch(() => undefined) + let timer: ReturnType | undefined + try { + const result = await Promise.race([ + pending, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + controller.abort(timeoutError) + reject(timeoutError) + }, remainingMs) + }), + ]) + // Exact-boundary completion is ambiguous under event-loop delay. + if (Date.now() >= deadlineAtMs) { + controller.abort(timeoutError) + throw timeoutError + } + return result + } finally { + if (timer) clearTimeout(timer) + } +} + +export class CandidateCleanupTimeoutError extends Error { + constructor(label: string) { + super(`${label} did not complete before the evaluator cleanup deadline`) + this.name = 'CandidateCleanupTimeoutError' + } +} + +export class CandidateResultTimeoutError extends Error { + constructor(label: string) { + super(`${label} did not complete before the evaluator result deadline`) + this.name = 'CandidateResultTimeoutError' + } +} diff --git a/src/candidate-execution/digest.ts b/src/candidate-execution/digest.ts new file mode 100644 index 00000000..47fe0c7b --- /dev/null +++ b/src/candidate-execution/digest.ts @@ -0,0 +1,76 @@ +import { createHash } from 'node:crypto' +import { canonicalJson } from '@tangle-network/agent-eval' +import type { AgentCandidateEmbeddedArtifact, Sha256Digest } from '@tangle-network/agent-interface' + +import { contentAddress } from '../durable/spawn-journal' +import type { CanonicalCandidateDocument } from './types' + +export function sha256Bytes(bytes: Uint8Array): Sha256Digest { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + +export function canonicalCandidateBytes(value: unknown): Uint8Array { + return Buffer.from(canonicalJson(value), 'utf8') +} + +export function canonicalCandidateDigest(value: unknown): Sha256Digest { + return contentAddress(value) as Sha256Digest +} + +/** Returns a detached, deeply frozen JSON value with canonical number normalization. */ +export function immutableCandidateValue(value: T): T { + return deepFreezeCandidate( + JSON.parse(Buffer.from(canonicalCandidateBytes(value)).toString('utf8')) as T, + ) +} + +export function canonicalCandidateDocument( + valueWithoutDigest: Omit, +): CanonicalCandidateDocument { + const bytes = canonicalCandidateBytes(valueWithoutDigest) + const digest = canonicalCandidateDigest(valueWithoutDigest) + if (sha256Bytes(bytes) !== digest) { + throw new Error('canonical candidate serializers disagree on document digest') + } + const storedBytes = Uint8Array.from(bytes) + const value = immutableCandidateValue({ ...valueWithoutDigest, digest }) as T + return Object.freeze({ + value, + get bytes(): Uint8Array { + return Uint8Array.from(storedBytes) + }, + digest, + }) +} + +export function embeddedCandidateArtifact(bytes: Uint8Array): AgentCandidateEmbeddedArtifact { + return { + encoding: 'base64', + content: Buffer.from(bytes).toString('base64'), + sha256: sha256Bytes(bytes), + byteLength: bytes.byteLength, + } +} + +export function omitTopLevelDigest( + value: T, +): Omit { + const { digest: _digest, ...rest } = value + return rest +} + +export function deepFreezeCandidate(value: T, seen = new Set()): T { + if ( + value === null || + typeof value !== 'object' || + ArrayBuffer.isView(value) || + seen.has(value as object) + ) { + return value + } + seen.add(value as object) + for (const child of Object.values(value as Record)) { + deepFreezeCandidate(child, seen) + } + return Object.freeze(value) +} diff --git a/src/candidate-execution/dispose.ts b/src/candidate-execution/dispose.ts new file mode 100644 index 00000000..c1899013 --- /dev/null +++ b/src/candidate-execution/dispose.ts @@ -0,0 +1,88 @@ +import { candidateCleanupDeadline, withinCandidateCleanupDeadline } from './cleanup' +import { sealAgentCandidateModelSettlement } from './model-settlement' +import { + assertPreparedCandidateIntegrity, + beginPreparedCandidateDisposal, + consumePreparedCandidateExecution, +} from './prepared-state' +import type { PreparedAgentCandidateExecution } from './types' + +export interface DisposePreparedAgentCandidateOptions { + cleanupTimeoutMs?: number +} + +/** Revoke reservations held by a prepared candidate that will not be executed. */ +export async function disposePreparedAgentCandidateExecution( + prepared: PreparedAgentCandidateExecution, + options: DisposePreparedAgentCandidateOptions = {}, +): Promise<{ disposed: true }> { + const initialState = assertPreparedCandidateIntegrity(prepared) + const cleanupTimeoutMs = options.cleanupTimeoutMs ?? initialState.cleanupTimeoutMs + if (cleanupTimeoutMs > initialState.cleanupTimeoutMs) { + throw new Error('disposal cleanup timeout exceeds the frozen preparation bound') + } + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + const state = beginPreparedCandidateDisposal(prepared) + const cleanup: Array> = [] + + if (state.memory.mode === 'isolated') { + const reservation = state.memoryReservation + if (!reservation) throw new Error('isolated memory reservation is missing') + cleanup.push( + withinCandidateCleanupDeadline( + async () => { + const closed = await state.ports.memory.close({ + executionId: state.executionId, + preparationId: reservation.preparationId, + accessDigest: reservation.accessDigest, + effectiveNamespace: reservation.effectiveNamespace, + reason: 'abandoned', + }) + if (closed.closed !== true || Object.keys(closed).some((key) => key !== 'closed')) { + throw new Error('abandoned isolated memory access did not acknowledge closure') + } + }, + cleanupDeadlineAtMs, + 'isolated memory disposal', + ), + ) + } + + cleanup.push( + withinCandidateCleanupDeadline( + async () => { + const settlement = sealAgentCandidateModelSettlement( + await state.ports.models.settleGrant({ + executionId: state.executionId, + preparationId: state.preparationId, + grantDigest: state.modelReservation.digest, + resolved: state.resolvedModel, + reason: 'abandoned', + }), + { + preparationId: state.preparationId, + grantDigest: state.modelReservation.digest, + model: state.resolvedModel.model, + }, + ) + if (settlement.usage.modelCalls !== 0) { + throw new Error('unexecuted model reservation unexpectedly contains calls') + } + }, + cleanupDeadlineAtMs, + 'model reservation disposal', + ), + ) + + const results = await Promise.allSettled(cleanup) + const failures = results + .filter((result): result is PromiseRejectedResult => result.status === 'rejected') + .map((result) => result.reason) + if (failures.length > 0) { + consumePreparedCandidateExecution(prepared, 'disposal-failed') + throw new AggregateError(failures, 'candidate preparation disposal failed') + } + + consumePreparedCandidateExecution(prepared, 'disposed') + return Object.freeze({ disposed: true as const }) +} diff --git a/src/candidate-execution/exact-object.ts b/src/candidate-execution/exact-object.ts new file mode 100644 index 00000000..2eecc367 --- /dev/null +++ b/src/candidate-execution/exact-object.ts @@ -0,0 +1,21 @@ +/** Reject unknown fields while requiring every declared non-optional field. */ +export function assertExactObjectKeys( + value: unknown, + required: readonly string[], + label: string, + optional: readonly string[] = [], +): void { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + const allowed = new Set([...required, ...optional]) + if (allowed.size !== required.length + optional.length) { + throw new Error(`${label} exact-key contract contains duplicate fields`) + } + for (const key of Object.keys(value)) { + if (!allowed.has(key)) throw new Error(`${label} contains unknown field ${key}`) + } + for (const key of required) { + if (!(key in value)) throw new Error(`${label} is missing field ${key}`) + } +} diff --git a/src/candidate-execution/execute.ts b/src/candidate-execution/execute.ts new file mode 100644 index 00000000..e9e8339e --- /dev/null +++ b/src/candidate-execution/execute.ts @@ -0,0 +1,918 @@ +import type { TraceStore } from '@tangle-network/agent-eval' +import type { AgentCandidateTermination } from '@tangle-network/agent-interface' + +import type { + AgentCandidateExecutionClaimStore, + AgentCandidateExecutionFailureClass, + AgentCandidateExecutionLease, + AgentCandidateExecutionTerminalResult, +} from './claim' +import { candidateExecutionClaim } from './claim-plan' +import { + candidateCleanupDeadline, + candidateCleanupTimeout, + candidateResultTimeout, + withinCandidateCleanupDeadline, + withinCandidateResultDeadline, +} from './cleanup' +import { canonicalCandidateBytes } from './digest' +import { candidatePostRunWindowMs, candidateTerminalWindowMs } from './execution-window' +import { + sealAgentCandidateExecutorFinalCapture, + sealAgentCandidateProtectedRunCapture, +} from './executor-capture' +import { failedAgentCandidateRun, finalizeAgentCandidateRun } from './finalize' +import { + appendAuthoritativeModelSettlementSpans, + type SealedAgentCandidateModelSettlement, + sealAgentCandidateModelSettlement, +} from './model-settlement' +import { + type PersistedAgentCandidateModelSettlement, + persistCandidateBenchmarkResult, + persistCandidateModelSettlement, + persistVerifiedCandidateTaskOutcome, +} from './outcome-evidence' +import { persistCandidateOutputArtifact } from './output-artifacts' +import { + assertPreparedCandidateIntegrity, + assertPreparedCandidateWorkspaces, + beginPreparedCandidateClaim, + beginPreparedCandidateRun, + beginPreparedCandidateSettlement, + consumePreparedCandidateExecution, + markPreparedCandidateClaimed, + type PreparedCandidateState, +} from './prepared-state' +import { redactProtectedReason } from './protected-redaction' +import { ProtectedAgentCandidateTraceStore } from './protected-trace-store' +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateExecutorFinalCapture, + AgentCandidateExecutorPort, + AgentCandidateExecutorRequest, + AgentCandidateOutputArtifactPort, + AgentCandidateProtectedModelActivation, + AgentCandidateProtectedRunCapture, + AgentCandidateRunFinalization, + PreparedAgentCandidateExecution, +} from './types' + +export interface ExecutePreparedAgentCandidateOptions { + executor: AgentCandidateExecutorPort + grader: AgentCandidateBenchmarkGraderPort + outputArtifacts: AgentCandidateOutputArtifactPort + traceStore: TraceStore + /** Long-lived evaluator-owned store shared by every process that can run this benchmark. */ + claimStore: AgentCandidateExecutionClaimStore + /** Maximum time to prove process death and revoke protected access after a run ends. */ + cleanupTimeoutMs?: number + /** Maximum time for task verification, executable grading, and receipt construction. */ + resultTimeoutMs?: number +} + +/** Executes and finalizes one durably claimed candidate without exposing an unproven result. */ +export async function executePreparedAgentCandidate( + prepared: PreparedAgentCandidateExecution, + options: ExecutePreparedAgentCandidateOptions, +): Promise { + const initialState = assertPreparedCandidateIntegrity(prepared) + const cleanupTimeoutMs = candidateCleanupTimeout( + options.cleanupTimeoutMs ?? initialState.cleanupTimeoutMs, + ) + if (cleanupTimeoutMs > initialState.cleanupTimeoutMs) { + throw new Error('execution cleanup timeout exceeds the frozen preparation bound') + } + const resultTimeoutMs = candidateResultTimeout( + options.resultTimeoutMs ?? initialState.resultTimeoutMs, + initialState.resultTimeoutMs, + ) + if (resultTimeoutMs > initialState.resultTimeoutMs) { + throw new Error('execution result timeout exceeds the frozen preparation bound') + } + let state: PreparedCandidateState + try { + state = beginPreparedCandidateClaim(prepared) + } catch (error) { + return failedAgentCandidateRun(initialState, errorMessage(error)) + } + + try { + // Mutable staging is only a preparation check. The executor receives the + // detached file bytes sealed in private state, never these host paths. + await assertPreparedCandidateWorkspaces(state) + } catch (error) { + return await failBeforeActivation(prepared, state, error, 'failed', cleanupTimeoutMs) + } + + let acquired: Awaited> + try { + acquired = await options.claimStore.tryClaim(candidateExecutionClaim(prepared)) + } catch (error) { + return await failBeforeActivation(prepared, state, error, 'failed', cleanupTimeoutMs) + } + if (!acquired.acquired) { + return await failBeforeActivation( + prepared, + state, + new Error( + acquired.reason === 'retry-not-eligible' + ? `candidate execution retry is not eligible: ${acquired.detail}` + : 'candidate execution attempt is already claimed', + ), + 'replayed', + cleanupTimeoutMs, + ) + } + + markPreparedCandidateClaimed(prepared) + const postRunWindowMs = candidatePostRunWindowMs(cleanupTimeoutMs, resultTimeoutMs) + // A caller may shorten cleanup for this invocation, but that must never buy + // the candidate more execution time. The claim was frozen with the prepared + // cleanup window, so recover the original task deadline from that value. + const deadlineAtMs = + acquired.claim.leaseExpiresAtMs - + candidatePostRunWindowMs(state.cleanupTimeoutMs, state.resultTimeoutMs) + const requiredLeaseExpiry = deadlineAtMs + postRunWindowMs + if ( + Date.now() >= deadlineAtMs || + deadlineAtMs > state.reservationExpiresAtMs || + requiredLeaseExpiry > acquired.lease.expiresAtMs + ) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + new Error('candidate claim no longer covers its full execution and cleanup window'), + 'failed', + cleanupTimeoutMs, + 'pre-model-infrastructure', + ) + } + let activation: AgentCandidateProtectedModelActivation + try { + const activated = await withinCandidateCleanupDeadline( + () => + state.ports.models.activateGrant({ + executionId: state.executionId, + preparationId: state.preparationId, + grantDigest: state.modelReservation.digest, + resolved: state.resolvedModel, + deadlineAtMs, + }), + Math.min(deadlineAtMs, candidateCleanupDeadline(cleanupTimeoutMs)), + 'protected model activation', + ) + activation = Object.freeze({ env: Object.freeze({ ...activated.env }) }) + } catch (error) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + new Error('protected model activation failed', { cause: error }), + 'failed', + cleanupTimeoutMs, + 'pre-model-infrastructure', + ) + } + + let memoryActivation: { env: Readonly> } | undefined + if (state.memory.mode === 'isolated') { + try { + const reservation = state.memoryReservation + if (!reservation) throw new Error('isolated memory reservation is missing') + const activated = await withinCandidateCleanupDeadline( + () => + state.ports.memory.activate({ + executionId: state.executionId, + preparationId: reservation.preparationId, + accessDigest: reservation.accessDigest, + effectiveNamespace: reservation.effectiveNamespace, + deadlineAtMs, + }), + Math.min(deadlineAtMs, candidateCleanupDeadline(cleanupTimeoutMs)), + 'isolated memory activation', + ) + memoryActivation = Object.freeze({ env: Object.freeze({ ...activated.env }) }) + } catch (error) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + new Error('isolated memory activation failed', { cause: error }), + 'failed', + cleanupTimeoutMs, + 'pre-model-infrastructure', + activation, + ) + } + } + + let request: AgentCandidateExecutorRequest + try { + request = beginPreparedCandidateRun(prepared, activation, memoryActivation).request + } catch (error) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + error, + 'failed', + cleanupTimeoutMs, + 'pre-model-infrastructure', + activation, + memoryActivation, + ) + } + + try { + const phase = await options.claimStore.markCandidateMayRun(acquired.lease) + if (phase.phase !== 'candidate-may-run') { + throw new Error('candidate claim did not persist the candidate-may-run phase') + } + } catch (error) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + new Error('candidate execution phase persistence failed', { cause: error }), + 'failed', + cleanupTimeoutMs, + 'pre-model-infrastructure', + activation, + memoryActivation, + ) + } + if (Date.now() >= deadlineAtMs) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + new Error('candidate execution deadline elapsed while persisting its launch phase'), + 'failed', + cleanupTimeoutMs, + 'unknown', + activation, + memoryActivation, + ) + } + + const protectedValues = protectedEnvironmentValues(activation, memoryActivation) + const protectedTraceStore = new ProtectedAgentCandidateTraceStore( + options.traceStore, + protectedValues, + ) + const execution = await runAndStopExecutor( + options.executor, + request, + protectedTraceStore, + deadlineAtMs, + cleanupTimeoutMs, + ) + beginPreparedCandidateSettlement(prepared) + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + + const accessReason = + execution.kind === 'timeout' || execution.termination?.kind === 'timeout' + ? 'timeout' + : execution.kind === 'capture' + ? 'completed' + : 'failed' + const [memoryClose, settlementResult] = await Promise.all([ + closeMemoryAccess(state, accessReason, cleanupDeadlineAtMs), + settleModelGrant(state, accessReason, cleanupDeadlineAtMs), + ]) + if (!execution.processStopped || !settlementResult.settlement || !memoryClose.closed) { + consumePreparedCandidateExecution(prepared, 'failed') + return failedAgentCandidateRun( + state, + redactProtectedReason( + joinErrors( + execution.error, + !execution.processStopped + ? new Error('candidate process termination is not proven') + : undefined, + memoryClose.error, + settlementResult.error ?? + (!settlementResult.settlement ? new Error('model settlement failed') : undefined), + new Error('candidate claim remains recoverable until protected cleanup is proven'), + ), + protectedValues, + ), + execution.termination, + settlementResult.settlement?.usage ?? null, + ) + } + + let modelSettlement: PersistedAgentCandidateModelSettlement + try { + modelSettlement = await withinCandidateCleanupDeadline( + () => + persistCandidateModelSettlement( + state, + settlementResult.settlement as SealedAgentCandidateModelSettlement, + options.outputArtifacts, + ), + candidateCleanupDeadline(cleanupTimeoutMs), + 'candidate model settlement persistence', + ) + } catch (error) { + consumePreparedCandidateExecution(prepared, 'failed') + return failedAgentCandidateRun( + state, + redactProtectedReason( + joinErrors( + error, + new Error('candidate claim remains recoverable until settlement evidence is durable'), + ), + protectedValues, + ), + execution.termination, + settlementResult.settlement.usage, + ) + } + + let result: AgentCandidateRunFinalization + const failureClass: AgentCandidateExecutionFailureClass = + execution.kind === 'error' ? 'execution' : 'post-model-infrastructure' + if (execution.kind === 'error') { + result = failedAgentCandidateRun( + state, + redactProtectedReason(errorMessage(execution.error), protectedValues), + execution.termination, + settlementResult.settlement.usage, + ) + } else if (!execution.finalCapture.taskOutcome) { + result = failedAgentCandidateRun( + state, + 'candidate executor stopped without a captured task outcome', + execution.termination, + settlementResult.settlement.usage, + ) + } else { + const capture = + execution.kind === 'capture' + ? execution.capture + : { + executionId: state.executionId, + termination: execution.termination, + } + try { + const resultDeadlineAtMs = Math.min( + Date.now() + resultTimeoutMs, + acquired.lease.expiresAtMs - candidateTerminalWindowMs(cleanupTimeoutMs), + ) + result = await withinCandidateResultDeadline( + async (signal) => { + await appendAuthoritativeModelSettlementSpans( + options.traceStore, + state.trace.runId, + settlementResult.settlement as SealedAgentCandidateModelSettlement, + ) + const taskOutcome = await persistVerifiedCandidateTaskOutcome( + state, + execution.finalCapture.taskOutcome!, + options.outputArtifacts, + protectedValues, + signal, + ) + const benchmarkResult = await persistCandidateBenchmarkResult( + state, + capture.termination, + taskOutcome, + options.grader, + options.outputArtifacts, + protectedValues, + signal, + ) + return await finalizeAgentCandidateRun( + state, + capture, + options.traceStore, + settlementResult.settlement as SealedAgentCandidateModelSettlement, + { + finalCapture: execution.finalCapture, + modelSettlement, + taskOutcome, + benchmarkResult, + outputArtifacts: options.outputArtifacts, + }, + protectedValues, + protectedTraceStore.report(), + signal, + ) + }, + resultDeadlineAtMs, + 'candidate evidence finalization', + ) + } catch (error) { + result = failedAgentCandidateRun( + state, + redactProtectedReason(errorMessage(error), protectedValues), + execution.termination, + settlementResult.settlement.usage, + ) + } + } + + let terminal: AgentCandidateExecutionTerminalResult + try { + terminal = result.succeeded + ? { + schemaVersion: 1, + status: 'succeeded', + usage: settlementResult.settlement.fixedUsage, + modelSettlement: result.artifacts.modelSettlement, + taskOutcome: result.artifacts.taskOutcome, + benchmarkResult: result.artifacts.benchmarkResult, + runReceipt: result.artifacts.runReceipt, + } + : { + schemaVersion: 1, + status: 'failed', + failureClass, + usage: settlementResult.settlement.fixedUsage, + modelSettlement: modelSettlement.artifact, + failureEvidence: await withinCandidateCleanupDeadline( + () => + persistFailureEvidence( + state, + result.reason, + failureClass, + execution.termination, + options.outputArtifacts, + ), + acquired.lease.expiresAtMs, + 'candidate failure-evidence persistence', + ), + } + } catch (error) { + consumePreparedCandidateExecution(prepared, 'failed') + return failedAgentCandidateRun( + state, + redactProtectedReason( + joinErrors( + error, + new Error('candidate claim remains recoverable until terminal evidence is durable'), + ), + protectedValues, + ), + execution.termination, + settlementResult.settlement.usage, + ) + } + + const finished = await finishClaim( + options.claimStore, + acquired.lease, + terminal, + acquired.lease.expiresAtMs, + ) + if (finished) { + consumePreparedCandidateExecution(prepared, result.succeeded ? 'succeeded' : 'failed') + return result + } + + consumePreparedCandidateExecution(prepared, 'failed') + return failedAgentCandidateRun( + state, + 'candidate execution terminal record could not be persisted', + execution.termination, + settlementResult.settlement.usage, + ) +} + +type ExecutorOutcome = + | { + kind: 'capture' + capture: AgentCandidateProtectedRunCapture + termination: AgentCandidateTermination + finalCapture: AgentCandidateExecutorFinalCapture + processStopped: true + error?: undefined + } + | { + kind: 'timeout' + termination: AgentCandidateTermination & { kind: 'timeout' } + finalCapture: AgentCandidateExecutorFinalCapture + processStopped: true + error?: undefined + } + | { + kind: 'error' + error: unknown + termination?: AgentCandidateTermination + finalCapture?: AgentCandidateExecutorFinalCapture + processStopped: boolean + } + +async function runAndStopExecutor( + executor: AgentCandidateExecutorPort, + request: AgentCandidateExecutorRequest, + traceStore: TraceStore, + deadlineAtMs: number, + cleanupTimeoutMs: number, +): Promise { + const timeoutMs = request.hardLimits.timeoutMs + const timeoutError = new CandidateExecutionDeadlineError(timeoutMs) + if (Date.now() >= deadlineAtMs) { + return { + kind: 'error', + error: timeoutError, + termination: { kind: 'timeout', timeoutMs }, + processStopped: true, + } + } + const controller = new AbortController() + let timer: ReturnType | undefined + let timedOut = false + let capture: AgentCandidateProtectedRunCapture | undefined + let executionError: unknown + const executionPromise = Promise.resolve().then(() => + executor.execute(request, { + traceStore, + signal: controller.signal, + deadlineAtMs, + }), + ) + // A stopped process may still leave a buggy adapter promise pending. Attach a + // terminal handler now so the deadline path cannot create an unhandled rejection. + void executionPromise.catch(() => undefined) + const deadlinePromise = new Promise((_resolve, reject) => { + timer = setTimeout( + () => { + timedOut = true + controller.abort(timeoutError) + reject(timeoutError) + }, + Math.max(0, deadlineAtMs - Date.now()), + ) + }) + void deadlinePromise.catch(() => undefined) + try { + capture = sealAgentCandidateProtectedRunCapture( + await Promise.race([executionPromise, deadlinePromise]), + ) + } catch (error) { + executionError = error + } + + if (!timedOut && capture && capture.executionId !== request.executionId) { + executionError = new Error('candidate execution capture id does not match the request') + capture = undefined + } else if (!timedOut && capture && Date.now() >= deadlineAtMs) { + // Promise resolution at the boundary is ambiguous under event-loop delay. + // Fail closed unless completion is observed strictly before the deadline. + timedOut = true + executionError = timeoutError + capture = undefined + } else if (!timedOut && capture?.termination.kind === 'timeout') { + executionError = new Error('candidate executor cannot declare the runtime-owned timeout') + capture = undefined + } + + if (!capture || timedOut) controller.abort(executionError) + const stopReason = timedOut ? 'timeout' : capture ? 'completed' : 'failed' + let stopped: unknown + try { + stopped = await withinCandidateCleanupDeadline( + () => + executor.stopAndCapture( + { + executionId: request.executionId, + executionPlanDigest: request.executionPlan.value.digest, + }, + { + traceStore, + reason: stopReason, + signal: controller.signal, + deadlineAtMs, + }, + ), + Date.now() + cleanupTimeoutMs, + 'candidate process termination', + ) + if ( + !stopped || + typeof stopped !== 'object' || + (stopped as { stopped?: unknown }).stopped !== true + ) { + throw new Error('candidate executor did not acknowledge exact process termination') + } + } catch (stopError) { + if (timer) clearTimeout(timer) + controller.abort(stopError) + return { + kind: 'error', + error: new Error(joinErrors(executionError, stopError)), + ...(timedOut ? { termination: { kind: 'timeout', timeoutMs } } : {}), + processStopped: false, + } + } + + let finalCapture: AgentCandidateExecutorFinalCapture + try { + finalCapture = sealAgentCandidateExecutorFinalCapture(stopped) + } catch (captureError) { + if (timer) clearTimeout(timer) + controller.abort(captureError) + return { + kind: 'error', + error: new Error(joinErrors(executionError, captureError)), + ...(timedOut ? { termination: { kind: 'timeout', timeoutMs } } : {}), + processStopped: true, + } + } + + if (Date.now() >= deadlineAtMs) { + timedOut = true + executionError = timeoutError + capture = undefined + controller.abort(timeoutError) + } + if (timer) clearTimeout(timer) + + if (timedOut) { + return { + kind: 'timeout', + termination: { kind: 'timeout', timeoutMs }, + finalCapture, + processStopped: true, + } + } + if (executionError || !capture) { + return { + kind: 'error', + error: executionError ?? new Error('candidate executor returned no capture'), + finalCapture, + processStopped: true, + } + } + return { + kind: 'capture', + capture, + termination: capture.termination, + finalCapture, + processStopped: true, + } +} + +async function failBeforeActivation( + prepared: PreparedAgentCandidateExecution, + state: PreparedCandidateState, + error: unknown, + reason: 'failed' | 'replayed', + cleanupTimeoutMs: number, +): Promise { + beginPreparedCandidateSettlement(prepared) + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + const [memoryClose, settled] = await Promise.all([ + closeMemoryAccess(state, reason, cleanupDeadlineAtMs), + settleModelGrant(state, reason, cleanupDeadlineAtMs), + ]) + const cleanupProven = memoryClose.closed && settled.settlement !== undefined + consumePreparedCandidateExecution(prepared, cleanupProven ? 'failed' : 'cleanup-failed') + return failedAgentCandidateRun( + state, + redactProtectedReason( + joinErrors( + error, + memoryClose.error, + settled.error, + !cleanupProven + ? new Error('prepared access cleanup remains incomplete and may be retried by disposal') + : undefined, + ), + [], + ), + undefined, + settled.settlement?.usage ?? null, + ) +} + +async function failClaimedExecution( + prepared: PreparedAgentCandidateExecution, + state: PreparedCandidateState, + lease: AgentCandidateExecutionLease, + claimStore: AgentCandidateExecutionClaimStore, + outputArtifacts: AgentCandidateOutputArtifactPort, + error: unknown, + reason: 'failed', + cleanupTimeoutMs: number, + failureClass: AgentCandidateExecutionFailureClass, + activation?: AgentCandidateProtectedModelActivation, + memoryActivation?: { env: Readonly> }, +): Promise { + beginPreparedCandidateSettlement(prepared) + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + const [memoryClose, settled] = await Promise.all([ + closeMemoryAccess(state, reason, cleanupDeadlineAtMs), + settleModelGrant(state, reason, cleanupDeadlineAtMs), + ]) + const protectedValues = protectedEnvironmentValues(activation, memoryActivation) + const safeReason = redactProtectedReason( + joinErrors(error, memoryClose.error, settled.error), + protectedValues, + ) + let finishFailed = false + let persistenceFailure: unknown + if (settled.settlement && memoryClose.closed) { + try { + const modelSettlement = await withinCandidateCleanupDeadline( + () => persistCandidateModelSettlement(state, settled.settlement!, outputArtifacts), + lease.expiresAtMs, + 'candidate pre-run model-settlement persistence', + ) + const failureEvidence = await withinCandidateCleanupDeadline( + () => persistFailureEvidence(state, safeReason, failureClass, undefined, outputArtifacts), + lease.expiresAtMs, + 'candidate pre-run failure-evidence persistence', + ) + finishFailed = !(await finishClaim( + claimStore, + lease, + { + schemaVersion: 1, + status: 'failed', + failureClass, + usage: settled.settlement.fixedUsage, + modelSettlement: modelSettlement.artifact, + failureEvidence, + }, + lease.expiresAtMs, + )) + } catch (persistenceError) { + finishFailed = true + persistenceFailure = persistenceError + } + } + consumePreparedCandidateExecution(prepared, 'failed') + return failedAgentCandidateRun( + state, + redactProtectedReason( + joinErrors( + safeReason, + persistenceFailure, + memoryClose.error, + settled.error, + !memoryClose.closed || !settled.settlement + ? new Error('candidate claim remains recoverable until protected cleanup is proven') + : undefined, + finishFailed + ? new Error('candidate execution terminal record could not be persisted') + : undefined, + ), + protectedValues, + ), + undefined, + settled.settlement?.usage ?? null, + ) +} + +async function closeMemoryAccess( + state: PreparedCandidateState, + reason: 'completed' | 'failed' | 'timeout' | 'replayed', + cleanupDeadlineAtMs: number, +): Promise<{ closed: true; error?: undefined } | { closed: false; error: unknown }> { + if (state.memory.mode === 'disabled') return { closed: true } + try { + const reservation = state.memoryReservation + if (!reservation) throw new Error('isolated memory reservation is missing') + const closed = await withinCandidateCleanupDeadline( + () => + state.ports.memory.close({ + executionId: state.executionId, + preparationId: reservation.preparationId, + accessDigest: reservation.accessDigest, + effectiveNamespace: reservation.effectiveNamespace, + reason, + }), + cleanupDeadlineAtMs, + 'isolated memory closure', + ) + if (closed.closed !== true || Object.keys(closed).some((key) => key !== 'closed')) { + throw new Error('isolated memory access did not acknowledge closure') + } + return { closed: true } + } catch (error) { + return { closed: false, error } + } +} + +async function settleModelGrant( + state: PreparedCandidateState, + reason: 'completed' | 'failed' | 'timeout' | 'replayed', + cleanupDeadlineAtMs: number, +): Promise<{ + settlement?: SealedAgentCandidateModelSettlement + error?: unknown +}> { + try { + const value = await withinCandidateCleanupDeadline( + () => + state.ports.models.settleGrant({ + executionId: state.executionId, + preparationId: state.preparationId, + grantDigest: state.modelReservation.digest, + resolved: state.resolvedModel, + reason, + }), + cleanupDeadlineAtMs, + 'protected model settlement', + ) + return { + settlement: sealAgentCandidateModelSettlement(value, { + preparationId: state.preparationId, + grantDigest: state.modelReservation.digest, + model: state.resolvedModel.model, + }), + } + } catch (error) { + return { error: new Error('protected model settlement failed', { cause: error }) } + } +} + +async function finishClaim( + store: AgentCandidateExecutionClaimStore, + lease: AgentCandidateExecutionLease, + terminal: AgentCandidateExecutionTerminalResult, + deadlineAtMs?: number, +): Promise { + try { + const staged = await withinCandidateCleanupDeadline( + () => store.stageTerminal(lease, terminal), + deadlineAtMs ?? lease.expiresAtMs, + 'candidate terminal staging', + ) + if (!staged.staged && !staged.exactReplay) return false + const result = await withinCandidateCleanupDeadline( + () => store.finish(lease, staged.terminal.terminalDigest), + deadlineAtMs ?? lease.expiresAtMs, + 'candidate terminal publication', + ) + return result.finished || result.exactReplay + } catch { + return false + } +} + +async function persistFailureEvidence( + state: PreparedCandidateState, + reason: string, + failureClass: AgentCandidateExecutionFailureClass, + termination: AgentCandidateTermination | undefined, + outputArtifacts: AgentCandidateOutputArtifactPort, +) { + const bytes = canonicalCandidateBytes({ + schemaVersion: 1, + kind: 'agent-candidate-execution-failure', + executionId: state.executionId, + bundleDigest: state.bundle.digest, + executionPlanDigest: state.executionPlan.value.digest, + failureClass, + reason, + ...(termination ? { termination } : {}), + }) + return await persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'failure-evidence', + bytes, + }) +} + +function protectedEnvironmentValues( + activation?: AgentCandidateProtectedModelActivation, + memoryActivation?: { env: Readonly> }, +): string[] { + return [...Object.values(activation?.env ?? {}), ...Object.values(memoryActivation?.env ?? {})] +} + +function joinErrors(...errors: unknown[]): string { + return errors + .filter((error) => error !== undefined) + .map(errorMessage) + .join('; ') +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +class CandidateExecutionDeadlineError extends Error { + constructor(timeoutMs: number) { + super(`candidate execution reached its frozen ${timeoutMs}ms deadline`) + this.name = 'CandidateExecutionDeadlineError' + } +} diff --git a/src/candidate-execution/execution-window.ts b/src/candidate-execution/execution-window.ts new file mode 100644 index 00000000..960407d8 --- /dev/null +++ b/src/candidate-execution/execution-window.ts @@ -0,0 +1,41 @@ +import { MAX_CANDIDATE_TIMER_INTERVAL_MS } from './cleanup' + +export const CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS = 10_000 +const CANDIDATE_POST_RUN_CLEANUP_PHASES = 4 + +/** Process stop, access closure, model evidence, and task/result evidence. */ +export function candidatePostRunWindowMs( + cleanupTimeoutMs: number, + resultTimeoutMs: number, +): number { + const windowMs = + cleanupTimeoutMs * CANDIDATE_POST_RUN_CLEANUP_PHASES + + resultTimeoutMs + + CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS + if (!Number.isSafeInteger(windowMs) || windowMs > MAX_CANDIDATE_TIMER_INTERVAL_MS) { + throw new Error('candidate post-run window exceeds the supported timer range') + } + return windowMs +} + +/** Maximum owner lifetime from claim through process stop, access closure, and terminal write. */ +export function candidateExecutionOwnerWindowMs( + timeoutMs: number, + cleanupTimeoutMs: number, + resultTimeoutMs: number, +): number { + const windowMs = timeoutMs + candidatePostRunWindowMs(cleanupTimeoutMs, resultTimeoutMs) + if (!Number.isSafeInteger(windowMs) || windowMs > MAX_CANDIDATE_TIMER_INTERVAL_MS) { + throw new Error('candidate execution and cleanup window exceeds the supported timer range') + } + return windowMs +} + +/** Time reserved after scoring for failure evidence plus terminal publication. */ +export function candidateTerminalWindowMs(cleanupTimeoutMs: number): number { + const windowMs = cleanupTimeoutMs + CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS + if (!Number.isSafeInteger(windowMs) || windowMs > MAX_CANDIDATE_TIMER_INTERVAL_MS) { + throw new Error('candidate terminal window exceeds the supported timer range') + } + return windowMs +} diff --git a/src/candidate-execution/executor-capture.ts b/src/candidate-execution/executor-capture.ts new file mode 100644 index 00000000..d8a68cee --- /dev/null +++ b/src/candidate-execution/executor-capture.ts @@ -0,0 +1,90 @@ +import { + agentCandidateTerminationSchema, + agentCandidateWorkspaceManifestMaterialSchema, +} from '@tangle-network/agent-interface' +import { assertExactObjectKeys as assertExactKeys } from './exact-object' +import type { AgentCandidateExecutorFinalCapture, AgentCandidateProtectedRunCapture } from './types' + +/** Validate and detach the only candidate-authored fields accepted from execution. */ +export function sealAgentCandidateProtectedRunCapture( + value: unknown, +): AgentCandidateProtectedRunCapture { + const capture = requireRecord(value, 'candidate execution capture') + assertExactKeys(capture, ['executionId', 'termination'], 'candidate execution capture') + if (typeof capture.executionId !== 'string' || capture.executionId.length === 0) { + throw new Error('candidate execution capture has an invalid executionId') + } + return Object.freeze({ + executionId: capture.executionId, + termination: Object.freeze(agentCandidateTerminationSchema.parse(capture.termination)), + }) +} + +/** Validate, detach, and freeze evaluator-owned evidence captured after process death. */ +export function sealAgentCandidateExecutorFinalCapture( + value: unknown, +): AgentCandidateExecutorFinalCapture { + const capture = requireRecord(value, 'candidate final capture') + assertExactKeys(capture, ['stopped'], 'candidate final capture', ['taskOutcome', 'memoryAfter']) + if (capture.stopped !== true) { + throw new Error('candidate final capture does not prove process death') + } + + const taskOutcome = capture.taskOutcome ? sealTaskOutcomeCapture(capture.taskOutcome) : undefined + const memoryAfter = capture.memoryAfter ? sealMemoryCapture(capture.memoryAfter) : undefined + return Object.freeze({ + stopped: true, + ...(taskOutcome ? { taskOutcome } : {}), + ...(memoryAfter ? { memoryAfter: Object.freeze(memoryAfter) } : {}), + }) +} + +function sealMemoryCapture( + value: unknown, +): NonNullable { + const capture = requireRecord(value, 'candidate memory capture') + assertExactKeys(capture, ['afterState', 'archive'], 'candidate memory capture') + if (!(capture.archive instanceof Uint8Array)) { + throw new Error('candidate memory capture archive must be a byte array') + } + return Object.freeze({ + afterState: Object.freeze( + agentCandidateWorkspaceManifestMaterialSchema.parse(capture.afterState), + ), + archive: Uint8Array.from(capture.archive), + }) +} + +function sealTaskOutcomeCapture( + value: unknown, +): NonNullable { + const capture = requireRecord(value, 'candidate task capture') + assertExactKeys( + capture, + ['resultTree', 'afterState', 'archive', 'gitDiff'], + 'candidate task capture', + ) + if ( + typeof capture.resultTree !== 'string' || + !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(capture.resultTree) + ) { + throw new Error('candidate task capture resultTree is not a Git object id') + } + if (!(capture.archive instanceof Uint8Array) || !(capture.gitDiff instanceof Uint8Array)) { + throw new Error('candidate task capture archive and gitDiff must be byte arrays') + } + const afterState = agentCandidateWorkspaceManifestMaterialSchema.parse(capture.afterState) + return Object.freeze({ + resultTree: capture.resultTree, + afterState: Object.freeze(afterState), + archive: Uint8Array.from(capture.archive), + gitDiff: Uint8Array.from(capture.gitDiff), + }) +} + +function requireRecord(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + return value as Record +} diff --git a/src/candidate-execution/finalize.ts b/src/candidate-execution/finalize.ts new file mode 100644 index 00000000..d06ac49a --- /dev/null +++ b/src/candidate-execution/finalize.ts @@ -0,0 +1,336 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { Span, TraceStore } from '@tangle-network/agent-eval' +import { isLlmSpan, REDACTION_VERSION } from '@tangle-network/agent-eval' +import type { + AgentCandidateBenchmarkResultEvidence, + AgentCandidateMemoryReceipt, + AgentCandidateModelSettlementEvidence, + AgentCandidateRunReceiptV2, + AgentCandidateSpend, + AgentCandidateTermination, +} from '@tangle-network/agent-interface' +import { agentCandidateRunReceiptV2Schema } from '@tangle-network/agent-interface' + +import { readMaterializedWorkspaceFiles } from './artifacts' +import { canonicalCandidateBytes, canonicalCandidateDocument } from './digest' +import { + sealAgentCandidateExecutorFinalCapture, + sealAgentCandidateProtectedRunCapture, +} from './executor-capture' +import { + assertTraceMatchesModelSettlement, + type SealedAgentCandidateModelSettlement, + usdToNanos, +} from './model-settlement' +import { persistCandidateOutputArtifact } from './output-artifacts' +import type { PreparedCandidateState } from './prepared-state' +import { + assertNoProtectedBytes, + type ProtectedRedactionReport, + redactProtectedReason, + redactProtectedValue, +} from './protected-redaction' +import { + type AgentCandidateExecutorFinalCapture, + type AgentCandidateOutputArtifactPort, + type AgentCandidateProtectedRunCapture, + type AgentCandidateRunFinalization, + CANDIDATE_TRACE_TAGS, + type VerifiedAgentCandidateTaskOutcome, +} from './types' +import { + persistCandidateWorkspaceSnapshot, + provisionalCandidateWorkspaceSnapshot, +} from './workspace-snapshot' + +interface CandidateFinalizationEvidence { + finalCapture: AgentCandidateExecutorFinalCapture + modelSettlement: AgentCandidateModelSettlementEvidence & { + artifact: import('@tangle-network/agent-interface').AgentCandidateArtifactRef + } + taskOutcome: VerifiedAgentCandidateTaskOutcome + benchmarkResult: AgentCandidateBenchmarkResultEvidence & { + artifact: import('@tangle-network/agent-interface').AgentCandidateArtifactRef + } + outputArtifacts: AgentCandidateOutputArtifactPort +} + +/** Builds a candidate run receipt exclusively from protected trace and memory evidence. */ +export async function finalizeAgentCandidateRun( + state: PreparedCandidateState, + capture: AgentCandidateProtectedRunCapture, + traceStore: TraceStore, + settlement: SealedAgentCandidateModelSettlement, + evidence: CandidateFinalizationEvidence, + protectedValues: readonly string[], + writeRedactionReport?: ProtectedRedactionReport, + signal?: AbortSignal, +): Promise { + let termination: AgentCandidateTermination | undefined + try { + signal?.throwIfAborted() + const protectedCapture = sealAgentCandidateProtectedRunCapture(capture) + if (protectedCapture.executionId !== state.executionId) { + throw new Error('protected capture execution id does not match the prepared execution') + } + termination = protectedCapture.termination + if ( + termination.kind === 'timeout' && + termination.timeoutMs !== state.executionPlan.value.material.limits.timeoutMs + ) { + throw new Error('timeout termination does not match the frozen execution limit') + } + + const run = await traceStore.getRun(state.trace.runId) + if (!run) throw new Error(`protected trace run is missing: ${state.trace.runId}`) + if (run.status === 'running' || run.endedAt === undefined) { + throw new Error('protected trace run is not terminal') + } + assertTraceBindings(run.tags, state) + + const [spans, events, budget, artifacts] = await Promise.all([ + traceStore.spans({ runId: run.runId }), + traceStore.events({ runId: run.runId }), + traceStore.budget(run.runId), + traceStore.artifacts(run.runId), + ]) + const orderedSpans = [...spans].sort( + (a, b) => a.startedAt - b.startedAt || compareStrings(a.spanId, b.spanId), + ) + const orderedEvents = [...events].sort( + (a, b) => a.timestamp - b.timestamp || compareStrings(a.eventId, b.eventId), + ) + const orderedBudget = [...budget].sort( + (a, b) => a.timestamp - b.timestamp || compareStrings(a.dimension, b.dimension), + ) + const orderedArtifacts = [...artifacts].sort((a, b) => + compareStrings(a.artifactId, b.artifactId), + ) + + const modelSpans = orderedSpans.filter(isLlmSpan) + assertTraceMatchesModelSettlement(modelSpans, settlement) + const usage = settlement.usage + enforceLimits(state, run.startedAt, run.endedAt, orderedSpans, settlement) + const finalCapture = sealAgentCandidateExecutorFinalCapture(evidence.finalCapture) + const memory = await memoryReceipt( + state, + finalCapture, + evidence.outputArtifacts, + protectedValues, + signal, + ) + + const redacted = redactProtectedValue( + { + schemaVersion: 1, + run: { ...run, redactionVersion: REDACTION_VERSION }, + spans: orderedSpans, + events: orderedEvents, + budget: orderedBudget, + artifacts: orderedArtifacts, + }, + protectedValues, + ) + const combinedByRule = { ...(writeRedactionReport?.byRule ?? {}) } + for (const [rule, count] of Object.entries(redacted.report.byRule)) { + combinedByRule[rule] = (combinedByRule[rule] ?? 0) + count + } + const traceMaterial = { + ...(redacted.value as Record), + evaluatorLimits: { resultTimeoutMs: state.resultTimeoutMs }, + redaction: { + version: REDACTION_VERSION, + redactionCount: + (writeRedactionReport?.redactionCount ?? 0) + redacted.report.redactionCount, + byRule: combinedByRule, + }, + } + const traceBytes = canonicalCandidateBytes(traceMaterial) + assertNoProtectedBytes(traceBytes, protectedValues) + const traceArtifact = await persistCandidateOutputArtifact(evidence.outputArtifacts, { + executionId: state.executionId, + purpose: 'trace', + bytes: traceBytes, + signal, + }) + const trace = { + schemaVersion: 1 as const, + artifact: traceArtifact, + eventCount: + 1 + + orderedSpans.length + + orderedEvents.length + + orderedBudget.length + + orderedArtifacts.length, + modelCallCount: modelSpans.length, + } + const document = canonicalCandidateDocument({ + schemaVersion: 2, + kind: 'agent-candidate-run', + digestAlgorithm: 'rfc8785-sha256', + bundleDigest: state.bundle.digest, + materializationReceiptDigest: state.materializationReceipt.digest, + executionPlanDigest: state.executionPlan.value.digest, + memory, + usage, + modelUsage: { resolved: state.resolvedModel, usage }, + trace, + termination, + fixedUsage: settlement.fixedUsage, + modelSettlement: evidence.modelSettlement, + taskOutcome: evidence.taskOutcome.evidence, + benchmarkResult: evidence.benchmarkResult, + }) + agentCandidateRunReceiptV2Schema.parse(document.value) + const runReceipt = await persistCandidateOutputArtifact(evidence.outputArtifacts, { + executionId: state.executionId, + purpose: 'run-receipt', + bytes: document.bytes, + signal, + }) + return { + succeeded: true, + receipt: document, + artifacts: { + modelSettlement: evidence.modelSettlement.artifact, + taskOutcome: evidence.taskOutcome.evidence.artifact, + benchmarkResult: evidence.benchmarkResult.artifact, + runReceipt, + }, + } + } catch (error) { + return { + ...failedAgentCandidateRun( + state, + redactProtectedReason( + error instanceof Error ? error.message : String(error), + protectedValues, + ), + termination, + settlement.usage, + ), + } + } +} + +function assertTraceBindings( + tags: Record | undefined, + state: PreparedCandidateState, +): void { + const expected = state.trace.tags + for (const name of Object.values(CANDIDATE_TRACE_TAGS)) { + if (tags?.[name] !== expected[name]) { + throw new Error(`protected trace is not bound to prepared execution tag ${name}`) + } + } +} + +function enforceLimits( + state: PreparedCandidateState, + startedAt: number, + endedAt: number, + spans: Span[], + settlement: SealedAgentCandidateModelSettlement, +): void { + const limits = state.executionPlan.value.material.limits + const usage = settlement.usage + // The runtime-owned Date.now deadline decides when the process must stop. + // This separately rejects a receipt whose evaluator-owned trace claims a + // longer run; neither clock can make an over-limit execution admissible. + const wallMs = endedAt - startedAt + if (!Number.isFinite(wallMs) || wallMs < 0 || wallMs > limits.timeoutMs) { + throw new Error(`protected trace wall time ${wallMs} exceeds ${limits.timeoutMs}`) + } + const steps = spans.filter((span) => span.kind === 'tool').length + const checks: Array<[number, number, string]> = [ + [steps, limits.maxSteps, 'tool steps'], + [usage.modelCalls, limits.maxModelCalls, 'model calls'], + [usage.inputTokens, limits.maxInputTokens, 'input tokens'], + [usage.outputTokens, limits.maxOutputTokens, 'output tokens'], + ] + for (const [actual, limit, label] of checks) { + if (actual > limit) throw new Error(`protected ${label} ${actual} exceeds ${limit}`) + } + if (settlement.costUsdNanos > usdToNanos(limits.maxCostUsd, 'frozen maxCostUsd')) { + throw new Error(`protected cost USD ${usage.costUsd} exceeds ${limits.maxCostUsd}`) + } +} + +async function memoryReceipt( + state: PreparedCandidateState, + capture: AgentCandidateExecutorFinalCapture, + outputArtifacts: AgentCandidateOutputArtifactPort, + protectedValues: readonly string[], + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted() + if (state.memory.mode === 'disabled') { + if (capture.memoryAfter !== undefined) { + throw new Error('disabled memory cannot return an after-state') + } + return { mode: 'disabled' } + } + if (!capture.memoryAfter) throw new Error('isolated memory is missing its protected after-state') + const archive = Uint8Array.from(capture.memoryAfter.archive) + if (archive.byteLength === 0) throw new Error('isolated memory archive cannot be empty') + const provisional = provisionalCandidateWorkspaceSnapshot(capture.memoryAfter.afterState, archive) + const afterState = provisional.material + assertNoProtectedBytes(provisional.manifestBytes, protectedValues) + assertNoProtectedBytes(archive, protectedValues) + const root = await mkdtemp(join(tmpdir(), 'agent-candidate-memory-after-')) + try { + await state.ports.workspaces.materialize({ + role: 'memory', + snapshot: provisional.snapshot, + archive: Uint8Array.from(archive), + destination: root, + }) + const files = await readMaterializedWorkspaceFiles(root, afterState) + for (const file of files) assertNoProtectedBytes(file.bytes, protectedValues) + signal?.throwIfAborted() + } finally { + await rm(root, { recursive: true, force: true }) + } + const snapshot = await persistCandidateWorkspaceSnapshot(outputArtifacts, { + executionId: state.executionId, + material: afterState, + archive, + purpose: 'memory-after', + signal, + }) + return { + mode: 'isolated', + scope: 'task', + effectiveNamespace: state.memory.effectiveNamespace, + resetEvidenceDigest: state.memory.reset.evidence.sha256, + beforeStateDigest: state.memory.beforeState.digest, + afterState: snapshot, + } +} + +export function failedAgentCandidateRun( + state: PreparedCandidateState, + reason: string, + termination?: AgentCandidateTermination, + usage: AgentCandidateSpend | null = null, +): AgentCandidateRunFinalization & { succeeded: false } { + return { + succeeded: false, + reason, + partial: { + executionId: state.executionId, + bundleDigest: state.bundle.digest, + executionPlanDigest: state.executionPlan.value.digest, + materializationReceiptDigest: state.materializationReceipt.digest, + ...(termination ? { termination } : {}), + }, + usage, + } +} + +function compareStrings(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0 +} diff --git a/src/candidate-execution/git-materialize.ts b/src/candidate-execution/git-materialize.ts new file mode 100644 index 00000000..249b894a --- /dev/null +++ b/src/candidate-execution/git-materialize.ts @@ -0,0 +1,515 @@ +import { spawn } from 'node:child_process' +import { mkdir, mkdtemp, readFile, realpath, rm, stat } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +import type { + AgentCandidateCode, + AgentCandidateGitHubRepository, + AgentCandidateGitHubResource, + AgentCandidateWorkspaceManifestMaterialV1, +} from '@tangle-network/agent-interface' + +import { verifyBytes } from './artifacts' +import { canonicalCandidateBytes, sha256Bytes } from './digest' +import type { AgentCandidateRepositoryPort } from './types' + +interface GitResult { + stdout: Buffer + stderr: Buffer +} + +export async function verifyCandidateCode( + code: AgentCandidateCode, + repositories: AgentCandidateRepositoryPort, + patchBytes?: Uint8Array, +): Promise { + if (code.kind === 'disabled') return undefined + const repositoryRoot = await verifiedRepositoryRoot(code.repository, repositories) + await assertCommitAndBaseTree(repositoryRoot, code.baseCommit, code.baseTree) + + if (code.kind === 'no-op') { + await assertSafeTree(repositoryRoot, code.baseTree) + return code.baseTree + } + if (!patchBytes) throw new Error('git-patch candidate is missing verified patch bytes') + + const temporary = await mkdtemp(join(tmpdir(), 'agent-candidate-git-')) + try { + const indexFile = join(temporary, 'index') + await runCandidateGit(repositoryRoot, ['read-tree', code.baseTree], undefined, { + GIT_INDEX_FILE: indexFile, + }) + await runCandidateGit( + repositoryRoot, + ['apply', '--cached', '--binary', '--whitespace=nowarn', '-'], + patchBytes, + { GIT_INDEX_FILE: indexFile }, + ) + const candidateTree = ( + await runCandidateGit(repositoryRoot, ['write-tree'], undefined, { + GIT_INDEX_FILE: indexFile, + }) + ).stdout + .toString('utf8') + .trim() + if (candidateTree !== code.candidateTree) { + throw new Error( + `git patch materialized tree ${candidateTree} does not match ${code.candidateTree}`, + ) + } + await assertSafeTree(repositoryRoot, candidateTree) + return candidateTree + } finally { + await rm(temporary, { recursive: true, force: true }) + } +} + +export async function readCandidateGitHubResource( + resource: AgentCandidateGitHubResource, + repositories: AgentCandidateRepositoryPort, +): Promise { + const repositoryRoot = await verifiedRepositoryRoot(resource.repository, repositories) + const commitType = ( + await runCandidateGit(repositoryRoot, ['cat-file', '-t', resource.commit]) + ).stdout + .toString('utf8') + .trim() + if (commitType !== 'commit') { + throw new Error(`GitHub resource commit is not a commit object: ${resource.commit}`) + } + const listing = ( + await runCandidateGit(repositoryRoot, ['ls-tree', '-z', resource.commit, '--', resource.path]) + ).stdout + const entries = parseTreeEntries(listing) + if (entries.length !== 1 || entries[0]?.path !== resource.path) { + throw new Error(`GitHub resource path is not one exact file: ${resource.path}`) + } + const entry = entries[0] + if (!entry || entry.type !== 'blob' || (entry.mode !== '100644' && entry.mode !== '100755')) { + throw new Error(`GitHub resource path is not a regular Git blob: ${resource.path}`) + } + const bytes = (await runCandidateGit(repositoryRoot, ['cat-file', 'blob', entry.object])).stdout + verifyBytes(bytes, resource.sha256, resource.byteLength, `GitHub resource ${resource.path}`) + return Uint8Array.from(bytes) +} + +export async function verifyTaskCheckout( + taskRoot: string, + expected: { baseCommit: string; baseTree: string }, +): Promise { + const root = resolve(taskRoot) + const head = (await runCandidateGit(root, ['rev-parse', 'HEAD'])).stdout.toString('utf8').trim() + if (head !== expected.baseCommit) { + throw new Error(`task checkout HEAD ${head} does not match ${expected.baseCommit}`) + } + const tree = (await runCandidateGit(root, ['rev-parse', 'HEAD^{tree}'])).stdout + .toString('utf8') + .trim() + if (tree !== expected.baseTree) { + throw new Error(`task checkout base tree ${tree} does not match ${expected.baseTree}`) + } +} + +/** + * Apply an evaluator-captured binary diff in a detached object database and + * prove its result tree exactly matches the captured after-state manifest. + */ +export async function verifyTaskOutcomePatch(input: { + repositoryRoot: string + baseCommit: string + baseTree: string + resultTree: string + patch: Uint8Array + afterState: AgentCandidateWorkspaceManifestMaterialV1 +}): Promise<{ resultTree: string; resultCommit: string }> { + const repositoryRoot = resolve(input.repositoryRoot) + await verifyTaskCheckout(repositoryRoot, input) + const gitDir = resolve( + (await runCandidateGit(repositoryRoot, ['rev-parse', '--absolute-git-dir'])).stdout + .toString('utf8') + .trim(), + ) + if ((await realpath(gitDir)) !== gitDir || gitDir.includes(':')) { + throw new Error('task Git object store has an unsupported path') + } + await assertNoGitIndirection(repositoryRoot, gitDir, 'task repository') + + const temporary = await mkdtemp(join(tmpdir(), 'agent-candidate-task-outcome-')) + try { + const objectDirectory = join(temporary, 'objects') + const indexFile = join(temporary, 'index') + await mkdir(objectDirectory) + const gitEnvironment = { + GIT_INDEX_FILE: indexFile, + GIT_OBJECT_DIRECTORY: objectDirectory, + GIT_ALTERNATE_OBJECT_DIRECTORIES: join(gitDir, 'objects'), + } + await runCandidateGit(repositoryRoot, ['read-tree', input.baseTree], undefined, gitEnvironment) + if (input.patch.byteLength > 0) { + await runCandidateGit( + repositoryRoot, + ['apply', '--cached', '--binary', '--whitespace=nowarn', '-'], + input.patch, + gitEnvironment, + ) + } + const resultTree = ( + await runCandidateGit(repositoryRoot, ['write-tree'], undefined, gitEnvironment) + ).stdout + .toString('utf8') + .trim() + if (resultTree !== input.resultTree) { + throw new Error( + `task outcome patch materialized tree ${resultTree} does not match ${input.resultTree}`, + ) + } + await assertSafeTree(repositoryRoot, resultTree, gitEnvironment) + const observed = await workspaceManifestFromGitTree(repositoryRoot, resultTree, gitEnvironment) + if ( + !Buffer.from(canonicalCandidateBytes(observed)).equals( + Buffer.from(canonicalCandidateBytes(input.afterState)), + ) + ) { + throw new Error('task outcome after-state does not match the materialized result tree') + } + const resultCommit = ( + await runCandidateGit( + repositoryRoot, + ['commit-tree', resultTree, '-p', input.baseCommit], + Buffer.from('candidate task outcome\n', 'utf8'), + { + ...gitEnvironment, + GIT_AUTHOR_NAME: 'Tangle Evaluator', + GIT_AUTHOR_EMAIL: 'evaluator@tangle.tools', + GIT_AUTHOR_DATE: '2000-01-01T00:00:00Z', + GIT_COMMITTER_NAME: 'Tangle Evaluator', + GIT_COMMITTER_EMAIL: 'evaluator@tangle.tools', + GIT_COMMITTER_DATE: '2000-01-01T00:00:00Z', + }, + ) + ).stdout + .toString('utf8') + .trim() + const committedTree = ( + await runCandidateGit( + repositoryRoot, + ['rev-parse', `${resultCommit}^{tree}`], + undefined, + gitEnvironment, + ) + ).stdout + .toString('utf8') + .trim() + if (committedTree !== resultTree) { + throw new Error('task outcome evaluator commit does not bind the verified result tree') + } + return { resultTree, resultCommit } + } finally { + await rm(temporary, { recursive: true, force: true }) + } +} + +async function verifiedRepositoryRoot( + repository: AgentCandidateGitHubRepository, + repositories: AgentCandidateRepositoryPort, +): Promise { + const repositoryRoot = resolve(await repositories.resolve(repository)) + const rootStats = await stat(repositoryRoot) + if (!rootStats.isDirectory()) throw new Error('candidate repository path is not a directory') + + const origin = (await runCandidateGit(repositoryRoot, ['remote', 'get-url', 'origin'])).stdout + .toString('utf8') + .trim() + const actual = parseGitHubRemote(origin) + if (!actual || actual.owner !== repository.owner || actual.repo !== repository.repo) { + throw new Error( + `local repository origin ${origin || ''} does not match github.com/${repository.owner}/${repository.repo}`, + ) + } + + const gitDirText = ( + await runCandidateGit(repositoryRoot, ['rev-parse', '--absolute-git-dir']) + ).stdout + .toString('utf8') + .trim() + const gitDir = resolve(gitDirText) + await assertNoGitIndirection(repositoryRoot, gitDir, 'candidate repository') + return repositoryRoot +} + +export async function assertNoGitIndirection( + repositoryRoot: string, + gitDir: string, + label: string, +): Promise { + const replacements = ( + await runCandidateGit(repositoryRoot, ['for-each-ref', '--format=%(refname)', 'refs/replace']) + ).stdout + .toString('utf8') + .trim() + if (replacements) throw new Error(`${label} contains Git replace refs`) + const commonDir = resolve( + repositoryRoot, + (await runCandidateGit(repositoryRoot, ['rev-parse', '--git-common-dir'])).stdout + .toString('utf8') + .trim(), + ) + if ((await realpath(commonDir)) !== commonDir || commonDir.includes(':')) { + throw new Error(`${label} Git common directory has an unsupported path`) + } + for (const objectRoot of new Set([gitDir, commonDir])) { + for (const name of ['alternates', 'http-alternates']) { + const path = join(objectRoot, 'objects', 'info', name) + try { + const contents = await readFile(path, 'utf8') + if (contents.trim()) throw new Error(`${label} uses forbidden Git ${name}`) + } catch (error) { + if (!isNoEntry(error)) throw error + } + } + } +} + +async function assertCommitAndBaseTree( + repositoryRoot: string, + commit: string, + expectedTree: string, +): Promise { + const type = (await runCandidateGit(repositoryRoot, ['cat-file', '-t', commit])).stdout + .toString('utf8') + .trim() + if (type !== 'commit') throw new Error(`candidate base object is not a commit: ${commit}`) + const actualTree = ( + await runCandidateGit(repositoryRoot, ['rev-parse', `${commit}^{tree}`]) + ).stdout + .toString('utf8') + .trim() + if (actualTree !== expectedTree) { + throw new Error(`candidate base commit tree ${actualTree} does not match ${expectedTree}`) + } +} + +async function assertSafeTree( + repositoryRoot: string, + tree: string, + environment: Record = {}, +): Promise { + const listing = ( + await runCandidateGit( + repositoryRoot, + ['ls-tree', '-rz', '--full-tree', tree], + undefined, + environment, + ) + ).stdout + const entries = parseTreeEntries(listing) + if (entries.length === 0) throw new Error('candidate Git tree cannot be empty') + for (const entry of entries) { + if (entry.type !== 'blob' || (entry.mode !== '100644' && entry.mode !== '100755')) { + throw new Error( + `candidate Git tree contains a symlink, submodule, or non-blob: ${entry.path}`, + ) + } + assertSafeGitPath(entry.path) + } +} + +async function workspaceManifestFromGitTree( + repositoryRoot: string, + tree: string, + environment: Record, +): Promise { + const files = (await readCandidateGitTreeFiles(repositoryRoot, tree, environment)).map( + ({ path, mode, bytes }) => ({ + path, + mode, + sha256: sha256Bytes(bytes), + byteLength: bytes.byteLength, + }), + ) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files, + } +} + +export async function readCandidateGitTreeFiles( + repositoryRoot: string, + tree: string, + environment: Record = {}, + limits?: { maxFiles: number; maxFileBytes: number; maxTotalFileBytes: number }, +): Promise> { + const listing = ( + await runCandidateGit( + repositoryRoot, + ['ls-tree', '-rzl', '--full-tree', tree], + undefined, + environment, + ) + ).stdout + const files: Array<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> = [] + const entries = parseTreeEntries(listing) + if (limits && entries.length > limits.maxFiles) { + throw new Error('candidate Git tree exceeds maxFiles') + } + let totalBytes = 0 + for (const entry of entries) { + if (entry.type !== 'blob' || (entry.mode !== '100644' && entry.mode !== '100755')) { + throw new Error(`candidate Git tree contains a non-regular file: ${entry.path}`) + } + assertSafeGitPath(entry.path) + if (entry.size === undefined) { + throw new Error(`candidate Git tree is missing blob size: ${entry.path}`) + } + if (limits && entry.size > limits.maxFileBytes) { + throw new Error(`candidate Git tree file exceeds maxFileBytes: ${entry.path}`) + } + totalBytes += entry.size + if (limits && totalBytes > limits.maxTotalFileBytes) { + throw new Error('candidate Git tree exceeds maxTotalFileBytes') + } + const bytes = ( + await runCandidateGit( + repositoryRoot, + ['cat-file', 'blob', entry.object], + undefined, + environment, + ) + ).stdout + if (bytes.byteLength !== entry.size) { + throw new Error(`candidate Git blob size changed: ${entry.path}`) + } + files.push({ + path: entry.path, + mode: entry.mode === '100755' ? 0o755 : 0o644, + bytes: Uint8Array.from(bytes), + }) + } + return files.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0)) +} + +function parseTreeEntries(bytes: Uint8Array): Array<{ + mode: string + type: string + object: string + size?: number + path: string +}> { + const raw = Buffer.from(bytes) + const decoded = raw.toString('utf8') + if (!Buffer.from(decoded, 'utf8').equals(raw)) { + throw new Error('candidate Git tree contains a non-UTF-8 path') + } + const rows = decoded.split('\0').filter(Boolean) + return rows.map((row) => { + const tab = row.indexOf('\t') + const header = row.slice(0, tab).split(' ').filter(Boolean) + const path = row.slice(tab + 1) + const [mode, type, object, sizeText] = header + if (tab < 1 || !mode || !type || !object || !path) { + throw new Error('malformed Git tree entry') + } + const size = sizeText && /^\d+$/.test(sizeText) ? Number(sizeText) : undefined + if (size !== undefined && !Number.isSafeInteger(size)) { + throw new Error('candidate Git tree entry size exceeds the safe integer range') + } + return { mode, type, object, ...(size === undefined ? {} : { size }), path } + }) +} + +function assertSafeGitPath(path: string): void { + if ( + !path || + path.startsWith('/') || + path.includes('\\') || + path.includes('\0') || + hasControlCharacter(path) || + path.split('/').some((part) => !part || part === '.' || part === '..') || + path.split('/')[0]?.toLowerCase() === '.git' + ) { + throw new Error(`candidate Git tree contains an unsafe path: ${path}`) + } +} + +function hasControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index) + if (code < 0x20 || code === 0x7f) return true + } + return false +} + +function parseGitHubRemote(value: string): { owner: string; repo: string } | undefined { + const match = value.match( + /^(?:https?:\/\/github\.com\/|ssh:\/\/git@github\.com\/|git@github\.com:)([^/]+)\/([^/]+?)(?:\.git)?$/, + ) + if (!match?.[1] || !match[2]) return undefined + return { owner: match[1], repo: match[2] } +} + +export async function runCandidateGit( + repositoryRoot: string, + args: string[], + input?: Uint8Array, + extraEnv: Record = {}, +): Promise { + const env = Object.fromEntries( + Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_')), + ) as Record + Object.assign(env, { + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_NO_REPLACE_OBJECTS: '1', + LC_ALL: 'C', + ...extraEnv, + }) + const fullArgs = [ + '-c', + 'core.hooksPath=/dev/null', + '-c', + 'core.fsmonitor=false', + '-c', + 'core.untrackedCache=false', + '-c', + 'protocol.file.allow=never', + '-C', + repositoryRoot, + ...args, + ] + return await new Promise((resolveResult, reject) => { + const child = spawn('git', fullArgs, { env, stdio: ['pipe', 'pipe', 'pipe'] }) + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)) + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)) + child.on('error', reject) + child.on('close', (code, signal) => { + const out = Buffer.concat(stdout) + const err = Buffer.concat(stderr) + if (code !== 0) { + reject( + new Error( + `git ${args[0] ?? ''} failed (${signal ?? code}): ${err.toString('utf8').trim()}`, + ), + ) + return + } + resolveResult({ stdout: out, stderr: err }) + }) + if (input) child.stdin.end(input) + else child.stdin.end() + }) +} + +function isNoEntry(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code?: string }).code === 'ENOENT' + ) +} diff --git a/src/candidate-execution/index.ts b/src/candidate-execution/index.ts new file mode 100644 index 00000000..0b363fc0 --- /dev/null +++ b/src/candidate-execution/index.ts @@ -0,0 +1,119 @@ +export { + type AgentCandidateCodeSource, + type AgentCandidateCodeSurfaceSource, + type AgentCandidateProfileSource, + type BuildAgentCandidateBundleInput, + buildAgentCandidateBundle, +} from './builder' +export { + type AgentCandidateBundleInput, + sealAgentCandidateBundle, +} from './bundle' +export { + type AgentCandidateExecutionAttemptRecord, + type AgentCandidateExecutionAttemptRef, + type AgentCandidateExecutionClaim, + type AgentCandidateExecutionClaimResult, + type AgentCandidateExecutionClaimStore, + type AgentCandidateExecutionCleanupHandles, + type AgentCandidateExecutionFailureClass, + type AgentCandidateExecutionFinishResult, + type AgentCandidateExecutionLease, + type AgentCandidateExecutionPhase, + type AgentCandidateExecutionPhaseResult, + type AgentCandidateExecutionRecoveryEvidence, + type AgentCandidateExecutionStageResult, + type AgentCandidateExecutionTerminalRecord, + type AgentCandidateExecutionTerminalResult, + type AgentCandidateExecutionUsage, + type AgentCandidateRetryRejection, + InMemoryAgentCandidateExecutionClaimStore, +} from './claim' +export { + FileAgentCandidateExecutionClaimStore, + type FileAgentCandidateExecutionClaimStoreOptions, +} from './claim-file-store' +export { candidateExecutionClaim } from './claim-plan' +export { + type DisposePreparedAgentCandidateOptions, + disposePreparedAgentCandidateExecution, +} from './dispose' +export { + type ExecutePreparedAgentCandidateOptions, + executePreparedAgentCandidate, +} from './execute' +export { persistCandidateOutputArtifact } from './output-artifacts' +export { + type PrepareAgentCandidateExecutionOptions, + prepareAgentCandidateExecution, +} from './prepare' +export { + applyExactAgentProfileDiff, + parseExactAgentProfile, + parseExactAgentProfileDiff, +} from './profile' +export { + type AgentCandidateModelGrantActivateInput, + type AgentCandidateModelGrantClient, + type AgentCandidateModelGrantReservation, + type AgentCandidateModelGrantReserveInput, + type AgentCandidateModelGrantSettleInput, + type CreateProtectedAgentCandidateModelPortOptions, + createProtectedAgentCandidateModelPort, +} from './protected-model-port' +export { + type RecoverExpiredAgentCandidateOptions, + recoverExpiredAgentCandidateExecution, +} from './recover' +export { + type AgentCandidateArtifactPort, + type AgentCandidateBenchmarkGraderIdentity, + type AgentCandidateBenchmarkGraderPort, + type AgentCandidateContainerPort, + type AgentCandidateExecutionPorts, + type AgentCandidateExecutorFinalCapture, + type AgentCandidateExecutorMemoryCapture, + type AgentCandidateExecutorPort, + type AgentCandidateExecutorProfileFile, + type AgentCandidateExecutorRequest, + type AgentCandidateExecutorStopRequest, + type AgentCandidateExecutorTaskOutcomeCapture, + type AgentCandidateExecutorWorkspaceFile, + type AgentCandidateExecutorWorkspaceInput, + type AgentCandidateMemoryPort, + type AgentCandidateMemoryResetResult, + type AgentCandidateModelLimits, + type AgentCandidateModelPort, + type AgentCandidateOutputArtifactPort, + type AgentCandidateOutputPurpose, + type AgentCandidateProtectedModelActivation, + type AgentCandidateProtectedModelCall, + type AgentCandidateProtectedModelReservation, + type AgentCandidateProtectedModelSettlement, + type AgentCandidateProtectedRunCapture, + type AgentCandidateRepositoryPort, + type AgentCandidateRunFinalization, + type AgentCandidateTaskExecution, + type AgentCandidateVerificationPorts, + type AgentCandidateWorkspacePort, + CANDIDATE_TRACE_ENV, + CANDIDATE_TRACE_TAGS, + type CanonicalCandidateDocument, + type PreparedAgentCandidateExecution, + type PreparedAgentCandidateInstruction, + type PreparedAgentCandidateLaunch, + type PreparedAgentCandidateTrace, + type ResolvedAgentCandidateContainer, + type VerifiedAgentCandidate, + type VerifiedAgentCandidateTaskOutcome, +} from './types' +export { verifyAgentCandidateBundle } from './verify' +export { + type AgentCandidateWorkspaceArchiveLimits, + type CaptureAgentCandidateWorkspaceOptions, + type CapturedAgentCandidateWorkspace, + type CreateAgentCandidateWorkspacePortOptions, + captureAgentCandidateWorkspace, + captureAgentCandidateWorkspaceFiles, + createAgentCandidateWorkspacePort, +} from './workspace-archive' diff --git a/src/candidate-execution/model-settlement.ts b/src/candidate-execution/model-settlement.ts new file mode 100644 index 00000000..e188f4d2 --- /dev/null +++ b/src/candidate-execution/model-settlement.ts @@ -0,0 +1,281 @@ +import { isLlmSpan, type LlmSpan, type TraceStore } from '@tangle-network/agent-eval' +import type { AgentCandidateSpend } from '@tangle-network/agent-interface' +import type { AgentCandidateExecutionUsage } from './claim' +import { assertExactObjectKeys } from './exact-object' +import type { + AgentCandidateProtectedModelCall, + AgentCandidateProtectedModelSettlement, +} from './types' + +const USD_NANOS = 1_000_000_000 + +export interface SealedAgentCandidateModelSettlement { + readonly value: AgentCandidateProtectedModelSettlement + readonly usage: AgentCandidateSpend + readonly fixedUsage: AgentCandidateExecutionUsage + readonly costUsdNanos: number +} + +/** Validate and detach the evaluator gateway's terminal, revoked call ledger. */ +export function sealAgentCandidateModelSettlement( + settlement: AgentCandidateProtectedModelSettlement, + expected: { preparationId: string; grantDigest: string; model: string }, +): SealedAgentCandidateModelSettlement { + assertExactObjectKeys( + settlement, + ['preparationId', 'grantDigest', 'closed', 'calls'], + 'model settlement', + ) + if (settlement.closed !== true) throw new Error('protected model grant is not closed') + if (settlement.grantDigest !== expected.grantDigest) { + throw new Error('protected model settlement grant digest does not match the reservation') + } + if (settlement.preparationId !== expected.preparationId) { + throw new Error('protected model settlement preparation does not match the reservation') + } + if (!Array.isArray(settlement.calls)) { + throw new Error('protected model settlement calls must be an array') + } + + const callIds = new Set() + const spanIds = new Set() + let inputTokens = 0 + let outputTokens = 0 + let cachedInputTokens = 0 + let reasoningTokens = 0 + let hasCachedInput = false + let costUsdNanos = 0 + const calls = settlement.calls.map((source, index) => { + assertExactObjectKeys( + source, + [ + 'callId', + 'generationId', + 'traceSpanId', + 'status', + 'model', + 'startedAtMs', + 'endedAtMs', + 'inputTokens', + 'outputTokens', + 'cachedInputTokens', + 'reasoningTokens', + 'costUsdNanos', + ], + `model settlement call ${index}`, + ) + assertIdentifier(source.callId, `model settlement call ${index} callId`) + assertIdentifier(source.generationId, `model settlement call ${index} generationId`) + assertIdentifier(source.traceSpanId, `model settlement call ${index} traceSpanId`) + if (source.traceSpanId !== source.generationId) { + throw new Error(`model settlement call ${index} traceSpanId is not its router generationId`) + } + if (source.status !== 'succeeded' && source.status !== 'failed') { + throw new Error(`model settlement call ${index} has an invalid status`) + } + if (callIds.has(source.callId)) + throw new Error('protected model settlement has duplicate call ids') + if (spanIds.has(source.traceSpanId)) { + throw new Error('protected model settlement has duplicate trace span ids') + } + callIds.add(source.callId) + spanIds.add(source.traceSpanId) + if (source.model !== expected.model) { + throw new Error(`protected model settlement call ${index} has an unexpected model`) + } + assertTimestamp(source.startedAtMs, `model settlement call ${index} startedAtMs`) + assertTimestamp(source.endedAtMs, `model settlement call ${index} endedAtMs`) + if (source.endedAtMs < source.startedAtMs) { + throw new Error(`model settlement call ${index} ended before it started`) + } + assertCount(source.inputTokens, `model settlement call ${index} inputTokens`) + assertCount(source.outputTokens, `model settlement call ${index} outputTokens`) + assertCount(source.cachedInputTokens, `model settlement call ${index} cachedInputTokens`) + cachedInputTokens = safeAdd( + cachedInputTokens, + source.cachedInputTokens, + 'cached input token total', + ) + hasCachedInput = true + assertCount(source.reasoningTokens, `model settlement call ${index} reasoningTokens`) + reasoningTokens = safeAdd(reasoningTokens, source.reasoningTokens, 'reasoning token total') + assertCount(source.costUsdNanos, `model settlement call ${index} costUsdNanos`) + inputTokens = safeAdd(inputTokens, source.inputTokens, 'input token total') + outputTokens = safeAdd(outputTokens, source.outputTokens, 'output token total') + costUsdNanos = safeAdd(costUsdNanos, source.costUsdNanos, 'cost total') + return Object.freeze({ ...source }) + }) + + const usage = Object.freeze({ + costUsd: costUsdNanos / USD_NANOS, + inputTokens, + outputTokens, + ...(hasCachedInput ? { cachedInputTokens } : {}), + modelCalls: calls.length, + }) + const fixedUsage = Object.freeze({ + costUsdNanos, + inputTokens, + outputTokens, + cachedInputTokens, + reasoningTokens, + modelCalls: calls.length, + }) + return Object.freeze({ + value: Object.freeze({ + preparationId: settlement.preparationId, + grantDigest: settlement.grantDigest, + closed: true as const, + calls: Object.freeze(calls), + }), + usage, + fixedUsage, + costUsdNanos, + }) +} + +/** + * Append the only accepted LLM spans from the router's closed ledger. + * Candidate and executor code may write tool/process spans, but never model usage. + */ +export async function appendAuthoritativeModelSettlementSpans( + traceStore: TraceStore, + runId: string, + settlement: SealedAgentCandidateModelSettlement, +): Promise { + const run = await traceStore.getRun(runId) + if (!run) throw new Error(`protected trace run is missing before model settlement: ${runId}`) + if (run.status === 'running' || run.endedAt === undefined) { + throw new Error('protected trace run must be terminal before model spans are appended') + } + const existing = await traceStore.spans({ runId }) + if (existing.some(isLlmSpan)) { + throw new Error( + 'protected trace contains a model span not authored from the closed router ledger', + ) + } + const occupiedIds = new Set(existing.map((span) => span.spanId)) + for (const call of settlement.value.calls) { + if (occupiedIds.has(call.traceSpanId)) { + throw new Error( + `protected trace span identity collides with router generation ${call.generationId}`, + ) + } + await traceStore.appendSpan({ + runId, + spanId: call.traceSpanId, + kind: 'llm', + name: 'protected model call', + model: call.model, + messages: [], + startedAt: call.startedAtMs, + endedAt: call.endedAtMs, + status: call.status === 'succeeded' ? 'ok' : 'error', + inputTokens: call.inputTokens, + outputTokens: call.outputTokens, + cachedTokens: call.cachedInputTokens, + reasoningTokens: call.reasoningTokens, + costUsd: call.costUsdNanos / USD_NANOS, + attributes: { + 'tangle.protected_model.source': 'router-settlement', + 'tangle.router.call_id': call.callId, + 'tangle.router.generation_id': call.generationId, + }, + }) + } +} + +/** Match every protected trace span one-for-one against gateway call evidence. */ +export function assertTraceMatchesModelSettlement( + spans: readonly LlmSpan[], + settlement: SealedAgentCandidateModelSettlement, +): void { + if (spans.length !== settlement.value.calls.length) { + throw new Error( + `protected trace model calls ${spans.length} do not match model ledger ${settlement.value.calls.length}`, + ) + } + const byId = new Map(spans.map((span) => [span.spanId, span])) + if (byId.size !== spans.length) throw new Error('protected trace has duplicate model span ids') + for (const call of settlement.value.calls) { + const span = byId.get(call.traceSpanId) + if (!span) { + throw new Error(`protected trace is missing model ledger span ${call.traceSpanId}`) + } + assertTraceCall(span, call) + } +} + +export function usdToNanos(value: number, label: string): number { + if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be nonnegative`) + const nanos = Math.round(value * USD_NANOS) + if (!Number.isSafeInteger(nanos)) throw new Error(`${label} exceeds fixed-point range`) + return nanos +} + +function assertTraceCall(span: LlmSpan, call: AgentCandidateProtectedModelCall): void { + if (span.model !== call.model) { + throw new Error(`protected trace span ${span.spanId} model does not match model ledger`) + } + if ( + span.startedAt !== call.startedAtMs || + span.endedAt !== call.endedAtMs || + span.status !== (call.status === 'succeeded' ? 'ok' : 'error') + ) { + throw new Error( + `protected trace span ${span.spanId} timing or status does not match model ledger`, + ) + } + if ( + span.attributes?.['tangle.protected_model.source'] !== 'router-settlement' || + span.attributes?.['tangle.router.call_id'] !== call.callId || + span.attributes?.['tangle.router.generation_id'] !== call.generationId + ) { + throw new Error(`protected trace span ${span.spanId} lacks router settlement provenance`) + } + for (const [name, traced, settled] of [ + ['inputTokens', span.inputTokens, call.inputTokens], + ['outputTokens', span.outputTokens, call.outputTokens], + ['cachedInputTokens', span.cachedTokens ?? 0, call.cachedInputTokens], + ['reasoningTokens', span.reasoningTokens ?? 0, call.reasoningTokens], + ] as const) { + if (traced === undefined || traced !== settled) { + throw new Error( + `protected trace span ${span.spanId} ${name} ${traced} does not match model ledger ${settled}`, + ) + } + } + if (span.costUsd === undefined) { + throw new Error(`protected trace span ${span.spanId} is missing costUsd`) + } + const tracedCost = usdToNanos(span.costUsd, `protected trace span ${span.spanId} costUsd`) + if (tracedCost !== call.costUsdNanos) { + throw new Error( + `protected trace span ${span.spanId} costUsdNanos ${tracedCost} does not match model ledger ${call.costUsdNanos}`, + ) + } +} + +function assertIdentifier(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || value.length === 0 || value.length > 256) { + throw new Error(`${label} must be a non-empty bounded string`) + } +} + +function assertCount(value: unknown, label: string): asserts value is number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new Error(`${label} must be a nonnegative safe integer`) + } +} + +function assertTimestamp(value: unknown, label: string): asserts value is number { + if (!Number.isSafeInteger(value) || (value as number) <= 0) { + throw new Error(`${label} must be a positive safe integer`) + } +} + +function safeAdd(left: number, right: number, label: string): number { + const total = left + right + if (!Number.isSafeInteger(total)) throw new Error(`${label} exceeds safe integer range`) + return total +} diff --git a/src/candidate-execution/outcome-evidence.ts b/src/candidate-execution/outcome-evidence.ts new file mode 100644 index 00000000..3cca4e73 --- /dev/null +++ b/src/candidate-execution/outcome-evidence.ts @@ -0,0 +1,333 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { BenchmarkEvaluation } from '@tangle-network/agent-eval' +import { + type AgentCandidateArtifactRef, + type AgentCandidateBenchmarkResultEvidence, + type AgentCandidateModelSettlementEvidence, + type AgentCandidateResolvedModel, + type AgentCandidateTaskOutcomeEvidence, + type AgentCandidateTermination, + type AgentCandidateWorkspaceSnapshotEvidence, + agentCandidateBenchmarkResultEvidenceSchema, + agentCandidateModelSettlementEvidenceSchema, + agentCandidateTaskOutcomeEvidenceSchema, + type Sha256Digest, +} from '@tangle-network/agent-interface' + +import { readMaterializedWorkspaceFiles } from './artifacts' +import { runBoundCandidateBenchmarkGrader } from './benchmark-grader' +import { canonicalCandidateBytes, immutableCandidateValue, sha256Bytes } from './digest' +import { verifyTaskOutcomePatch } from './git-materialize' +import type { SealedAgentCandidateModelSettlement } from './model-settlement' +import { persistCandidateOutputArtifact } from './output-artifacts' +import type { PreparedCandidateState } from './prepared-state' +import { assertNoProtectedBytes } from './protected-redaction' +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateExecutorTaskOutcomeCapture, + AgentCandidateOutputArtifactPort, + VerifiedAgentCandidateTaskOutcome, +} from './types' +import { verifiedTaskOutcomeBrand } from './types' +import { + persistCandidateWorkspaceSnapshot, + provisionalCandidateWorkspaceSnapshot, +} from './workspace-snapshot' + +export type PersistedAgentCandidateModelSettlement = AgentCandidateModelSettlementEvidence & { + artifact: AgentCandidateArtifactRef +} + +export type PersistedAgentCandidateBenchmarkResult = AgentCandidateBenchmarkResultEvidence & { + artifact: AgentCandidateArtifactRef +} + +/** Persist the closed evaluator model ledger as canonical V2 receipt evidence. */ +export async function persistCandidateModelSettlement( + state: PreparedCandidateState, + settlement: SealedAgentCandidateModelSettlement, + outputArtifacts: AgentCandidateOutputArtifactPort, +): Promise { + return await persistCandidateModelSettlementEvidence( + { + executionId: state.executionId, + executionPlanDigest: state.executionPlan.value.digest, + resolvedModel: state.resolvedModel, + }, + settlement, + outputArtifacts, + ) +} + +/** Persist a closed model ledger when only durable recovery identity remains. */ +export async function persistCandidateModelSettlementEvidence( + identity: { + executionId: string + executionPlanDigest: Sha256Digest + resolvedModel: AgentCandidateResolvedModel + }, + settlement: SealedAgentCandidateModelSettlement, + outputArtifacts: AgentCandidateOutputArtifactPort, +): Promise { + const material = { + schemaVersion: 2 as const, + kind: 'agent-candidate-model-settlement-material' as const, + executionPlanDigest: identity.executionPlanDigest, + preparationId: settlement.value.preparationId, + grantDigest: settlement.value.grantDigest, + closed: true as const, + resolved: identity.resolvedModel, + calls: settlement.value.calls.map((call) => ({ + callId: call.callId, + generationId: call.generationId, + traceSpanId: call.traceSpanId, + status: call.status, + model: call.model, + startedAtMs: call.startedAtMs, + endedAtMs: call.endedAtMs, + inputTokens: call.inputTokens, + outputTokens: call.outputTokens, + cachedInputTokens: call.cachedInputTokens ?? 0, + reasoningTokens: call.reasoningTokens ?? 0, + costUsdNanos: call.costUsdNanos, + })), + usage: settlement.fixedUsage, + } + const bytes = canonicalCandidateBytes(material) + const digest = sha256Bytes(bytes) + const artifact = await persistCandidateOutputArtifact(outputArtifacts, { + executionId: identity.executionId, + purpose: 'model-settlement', + bytes, + }) + return immutableCandidateValue( + agentCandidateModelSettlementEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-model-settlement', + digest, + material, + artifact, + }), + ) as PersistedAgentCandidateModelSettlement +} + +/** Recompute the result tree from the patch, then persist its exact task evidence. */ +export async function persistVerifiedCandidateTaskOutcome( + state: PreparedCandidateState, + capture: AgentCandidateExecutorTaskOutcomeCapture, + outputArtifacts: AgentCandidateOutputArtifactPort, + protectedValues: readonly string[], + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted() + const patch = Uint8Array.from(capture.gitDiff) + const archive = Uint8Array.from(capture.archive) + if (archive.byteLength === 0) throw new Error('candidate task archive cannot be empty') + assertNoProtectedBytes(patch, protectedValues) + assertNoProtectedBytes(archive, protectedValues) + const provisional = provisionalCandidateWorkspaceSnapshot(capture.afterState, archive) + const afterState = provisional.material + const repository = state.executionPlan.value.material.task.repository + const verified = await verifyTaskOutcomePatch({ + repositoryRoot: state.roots.staging.taskRoot, + baseCommit: repository.baseCommit, + baseTree: repository.baseTree, + resultTree: capture.resultTree, + patch, + afterState, + }) + signal?.throwIfAborted() + assertNoProtectedBytes(provisional.manifestBytes, protectedValues) + await verifyTaskOutcomeArchive(state, provisional.snapshot, archive, protectedValues) + signal?.throwIfAborted() + const snapshot = await persistCandidateWorkspaceSnapshot(outputArtifacts, { + executionId: state.executionId, + material: afterState, + archive, + purpose: 'task', + signal, + }) + const gitDiff = await persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'task-patch', + bytes: patch, + signal, + }) + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-task-outcome-material' as const, + executionPlanDigest: state.executionPlan.value.digest, + baseRepository: { + identity: repository.identity, + rootIdentity: repository.rootIdentity, + commit: repository.baseCommit, + tree: repository.baseTree, + }, + resultRepository: { + identity: repository.identity, + rootIdentity: repository.rootIdentity, + commit: verified.resultCommit, + tree: verified.resultTree, + }, + afterState: snapshot, + gitDiff: { + format: 'git-diff-binary' as const, + artifact: gitDiff, + }, + } + const bytes = canonicalCandidateBytes(material) + const digest = sha256Bytes(bytes) + const artifact = await persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'task-outcome', + bytes, + signal, + }) + const evidence = immutableCandidateValue( + agentCandidateTaskOutcomeEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-task-outcome', + digest, + material, + artifact, + }), + ) as AgentCandidateTaskOutcomeEvidence & { artifact: AgentCandidateArtifactRef } + const storedPatch = Uint8Array.from(patch) + return Object.freeze({ + evidence, + get patch(): Uint8Array { + return Uint8Array.from(storedPatch) + }, + [verifiedTaskOutcomeBrand]: true as const, + }) +} + +/** Grade only a runtime-verified outcome and persist both raw and normalized evidence. */ +export async function persistCandidateBenchmarkResult( + state: PreparedCandidateState, + termination: AgentCandidateTermination, + outcome: VerifiedAgentCandidateTaskOutcome, + grader: AgentCandidateBenchmarkGraderPort, + outputArtifacts: AgentCandidateOutputArtifactPort, + protectedValues: readonly string[], + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted() + const frozenTermination = immutableCandidateValue(termination) + const graded = await runBoundCandidateBenchmarkGrader({ + executionId: state.executionId, + termination: frozenTermination, + outcome, + grader, + artifacts: outputArtifacts, + signal, + }) + signal?.throwIfAborted() + const evaluation = normalizeEvaluation(graded.evaluation, frozenTermination) + const rawEvidence = Uint8Array.from(graded.evidence) + if (rawEvidence.byteLength === 0) throw new Error('candidate benchmark evidence cannot be empty') + assertNoProtectedBytes(rawEvidence, protectedValues) + const evidenceRef = await persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'grader-evidence', + bytes: rawEvidence, + signal, + }) + const task = state.executionPlan.value.material.task + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-benchmark-result-material' as const, + executionPlanDigest: state.executionPlan.value.digest, + taskOutcomeDigest: outcome.evidence.digest, + benchmark: { + name: task.benchmark, + version: task.benchmarkVersion, + taskId: task.taskId, + splitDigest: task.splitDigest, + }, + grader: { + name: graded.grader.name, + version: graded.grader.version, + artifact: graded.grader.artifact, + }, + evidence: evidenceRef, + score: evaluation.score, + passed: evaluation.passed, + dimensions: evaluation.dimensions, + } + const bytes = canonicalCandidateBytes(material) + const digest = sha256Bytes(bytes) + const artifact = await persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'benchmark-result', + bytes, + signal, + }) + return immutableCandidateValue( + agentCandidateBenchmarkResultEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-benchmark-result', + digest, + material, + artifact, + }), + ) as PersistedAgentCandidateBenchmarkResult +} + +async function verifyTaskOutcomeArchive( + state: PreparedCandidateState, + snapshot: AgentCandidateWorkspaceSnapshotEvidence, + archive: Uint8Array, + protectedValues: readonly string[], +): Promise { + const root = await mkdtemp(join(tmpdir(), 'agent-candidate-task-archive-')) + try { + await state.ports.workspaces.materialize({ + role: 'task', + snapshot, + archive: Uint8Array.from(archive), + destination: root, + }) + const files = await readMaterializedWorkspaceFiles(root, snapshot.material) + for (const file of files) assertNoProtectedBytes(file.bytes, protectedValues) + } finally { + await rm(root, { recursive: true, force: true }) + } +} + +function normalizeEvaluation( + evaluation: BenchmarkEvaluation, + termination: AgentCandidateTermination, +): { score: number; passed: boolean; dimensions: Array<{ name: string; score: number }> } { + if (!evaluation || typeof evaluation !== 'object' || Array.isArray(evaluation)) { + throw new Error('candidate benchmark evaluation must be an object') + } + assertUnitScore(evaluation.score, 'candidate benchmark score') + if (evaluation.passed !== undefined && typeof evaluation.passed !== 'boolean') { + throw new Error('candidate benchmark passed must be boolean') + } + const cleanExit = termination.kind === 'exit' && termination.exitCode === 0 + const dimensions = Object.entries(evaluation.dimensions ?? {}) + .map(([name, score]) => { + if (!/^[a-z0-9]+(?:[._-][a-z0-9]+)*$/.test(name)) { + throw new Error(`candidate benchmark dimension is not normalized: ${name}`) + } + assertUnitScore(score, `candidate benchmark dimension ${name}`) + return { name, score: cleanExit ? score : 0 } + }) + .sort((left, right) => left.name.localeCompare(right.name)) + return { + score: cleanExit ? evaluation.score : 0, + passed: cleanExit && (evaluation.passed ?? evaluation.score > 0), + dimensions, + } +} + +function assertUnitScore(value: unknown, label: string): asserts value is number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`${label} must be finite and within [0, 1]`) + } +} diff --git a/src/candidate-execution/output-artifacts.ts b/src/candidate-execution/output-artifacts.ts new file mode 100644 index 00000000..af91e4b9 --- /dev/null +++ b/src/candidate-execution/output-artifacts.ts @@ -0,0 +1,39 @@ +import { + type AgentCandidateArtifactRef, + agentCandidateArtifactRefSchema, +} from '@tangle-network/agent-interface' + +import { verifyBytes } from './artifacts' +import { immutableCandidateValue, sha256Bytes } from './digest' +import type { AgentCandidateOutputArtifactPort, AgentCandidateOutputPurpose } from './types' + +/** Persist evaluator evidence, read it back, and bind the returned locator to the exact bytes. */ +export async function persistCandidateOutputArtifact( + port: AgentCandidateOutputArtifactPort, + input: { + executionId: string + purpose: AgentCandidateOutputPurpose + bytes: Uint8Array + signal?: AbortSignal + }, +): Promise { + input.signal?.throwIfAborted() + const bytes = Uint8Array.from(input.bytes) + const expectedDigest = sha256Bytes(bytes) + const ref = agentCandidateArtifactRefSchema.parse( + await port.put({ + executionId: input.executionId, + purpose: input.purpose, + bytes: Uint8Array.from(bytes), + ...(input.signal ? { signal: input.signal } : {}), + }), + ) + input.signal?.throwIfAborted() + if (ref.sha256 !== expectedDigest || ref.byteLength !== bytes.byteLength) { + throw new Error('candidate output locator does not identify the submitted bytes') + } + const stored = await port.read(ref) + input.signal?.throwIfAborted() + verifyBytes(stored, expectedDigest, bytes.byteLength, 'persisted candidate output') + return immutableCandidateValue(ref) +} diff --git a/src/candidate-execution/prepare.ts b/src/candidate-execution/prepare.ts new file mode 100644 index 00000000..3579e436 --- /dev/null +++ b/src/candidate-execution/prepare.ts @@ -0,0 +1,983 @@ +import { randomBytes } from 'node:crypto' +import { lstat, readdir } from 'node:fs/promises' +import { isAbsolute, posix, relative, resolve as resolveHostPath } from 'node:path' + +import type { + AgentCandidateConfigValue, + AgentCandidateEffectiveMemory, + AgentCandidateExecutionLimits, + AgentCandidateExecutionPlanEvidence, + AgentCandidateExecutionPlanMaterialV1, + AgentCandidateMaterializationReceipt, + AgentCandidateModelAccessNetwork, + AgentCandidateResolvedModel, + HarnessType, +} from '@tangle-network/agent-interface' +import { + agentCandidateContainerSchema, + agentCandidateExecutionLimitsSchema, + agentCandidateExecutionPlanEvidenceSchema, + agentCandidateExecutionPlanMaterialSchema, + agentCandidateMaterializationReceiptSchema, + agentCandidateModelAccessNetworkSchema, + agentCandidateWorkspaceSnapshotEvidenceSchema, + sha256DigestSchema, +} from '@tangle-network/agent-interface' +import { + applyAgentCandidateWorkspacePlan, + type HarnessId, + materializeCandidateProfile, +} from '@tangle-network/agent-profile-materialize' + +import { + readMaterializedWorkspaceFiles, + readVerifiedArtifact, + verifyMaterializedProfileWorkspace, + verifyMaterializedWorkspace, + verifyWorkspaceSnapshotArtifacts, +} from './artifacts' +import { + candidateCleanupDeadline, + candidateCleanupTimeout, + candidateResultTimeout, + MAX_CANDIDATE_TIMER_INTERVAL_MS, + withinCandidateCleanupDeadline, +} from './cleanup' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + canonicalCandidateDocument, + embeddedCandidateArtifact, + sha256Bytes, +} from './digest' +import { candidateExecutionOwnerWindowMs } from './execution-window' +import { verifyTaskCheckout } from './git-materialize' +import { sealAgentCandidateModelSettlement, usdToNanos } from './model-settlement' +import { createPreparedCandidateExecution } from './prepared-state' +import { assertCandidateProfileExecutionSupport } from './profile' +import { + type AgentCandidateExecutionPorts, + type AgentCandidateTaskExecution, + CANDIDATE_TRACE_ENV, + CANDIDATE_TRACE_TAGS, + type PreparedAgentCandidateExecution, + type ResolvedAgentCandidateContainer, + type VerifiedAgentCandidate, +} from './types' +import { + getVerifiedCandidateState, + verifiedArtifactBytes, + verifiedResourceTextByDigest, +} from './verify' + +const MATERIALIZER_HARNESSES = new Set([ + 'claude-code', + 'claude', + 'claudish', + 'nanoclaw', + 'codex', + 'opencode', + 'kimi-code', + 'kimi', + 'pi', + 'gemini', + 'hermes', + 'openclaw', +]) + +const MIN_RESERVATION_TTL_MS = 15 * 60_000 +const PREPARED_HOLD_MARGIN_MS = 5 * 60_000 + +export interface PrepareAgentCandidateExecutionOptions { + cleanupTimeoutMs?: number + /** Maximum time for task verification, executable grading, and receipt construction. */ + resultTimeoutMs?: number +} + +/** Materializes a verified candidate into one immutable evaluator-owned execution plan. */ +export async function prepareAgentCandidateExecution( + candidate: VerifiedAgentCandidate, + task: AgentCandidateTaskExecution, + ports: AgentCandidateExecutionPorts, + options: PrepareAgentCandidateExecutionOptions = {}, +): Promise { + const cleanupTimeoutMs = candidateCleanupTimeout(options.cleanupTimeoutMs) + const verifiedState = getVerifiedCandidateState(candidate) + assertSameVerificationPorts(verifiedState.ports, ports) + const bundle = candidate.bundle + assertCandidateProfileExecutionSupport(bundle.profile) + const harness = materializerHarness(bundle.execution.harness) + assertTaskInput(task, bundle.execution.instructionDelivery) + const resultTimeoutMs = candidateResultTimeout(options.resultTimeoutMs, task.limits.timeoutMs) + const ownerWindowMs = candidateExecutionOwnerWindowMs( + task.limits.timeoutMs, + cleanupTimeoutMs, + resultTimeoutMs, + ) + const reservationWindowMs = ownerWindowMs + PREPARED_HOLD_MARGIN_MS + if (reservationWindowMs > MAX_CANDIDATE_TIMER_INTERVAL_MS) { + throw new Error('candidate reservation window exceeds the supported timer range') + } + assertDisjointHostStagingRoots(task) + + const instructionBytes = Buffer.from(task.instruction, 'utf8') + const instructionDigest = sha256Bytes(instructionBytes) + + const taskArtifacts = await verifyWorkspaceSnapshotArtifacts(task.workspace, ports.artifacts) + await ports.workspaces.materialize({ + role: 'task', + snapshot: task.workspace, + archive: taskArtifacts.archive, + destination: task.stagingRoots.taskRoot, + }) + await verifyMaterializedWorkspace(task.stagingRoots.taskRoot, task.workspace.material, { + ignoredProtectedRootEntries: ['.git', '.sidecar'], + }) + await verifyTaskCheckout(task.stagingRoots.taskRoot, task.repository) + const taskExecutorFiles = await readMaterializedWorkspaceFiles( + task.stagingRoots.taskRoot, + task.workspace.material, + { ignoredProtectedRootEntries: ['.git', '.sidecar'] }, + ) + + let candidateArchive: Uint8Array | undefined + let candidateExecutorFiles: + | ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> + | undefined + if (bundle.execution.workspace) { + if (!task.stagingRoots.candidateRoot || !task.executionRoots.candidateRoot) { + throw new Error('active candidate execution requires host and container candidate roots') + } + candidateArchive = await verifiedArtifactBytes(candidate, bundle.execution.workspace.archive) + await ports.workspaces.materialize({ + role: 'candidate', + snapshot: bundle.execution.workspace, + archive: candidateArchive, + destination: task.stagingRoots.candidateRoot, + }) + await verifyMaterializedWorkspace( + task.stagingRoots.candidateRoot, + bundle.execution.workspace.material, + ) + candidateExecutorFiles = await readMaterializedWorkspaceFiles( + task.stagingRoots.candidateRoot, + bundle.execution.workspace.material, + ) + } else if (task.stagingRoots.candidateRoot || task.executionRoots.candidateRoot) { + throw new Error('disabled code cannot receive a candidate workspace root') + } + + await assertEmptyDirectory(task.stagingRoots.profileRoot) + const profileWorkspacePlan = materializeCandidateProfile(bundle.profile, harness, { + resolvedResources: verifiedResourceTextByDigest(candidate), + }) + const profileApplication = applyAgentCandidateWorkspacePlan( + profileWorkspacePlan, + task.stagingRoots.profileRoot, + bundle.execution.cwd.workspace, + ) + await verifyMaterializedProfileWorkspace( + task.stagingRoots.profileRoot, + profileApplication.profilePlan.material, + ) + const profilePlanBytes = await readVerifiedArtifact( + profileApplication.profilePlan.artifact, + ports.artifacts, + ) + if ( + !Buffer.from(profilePlanBytes).equals( + Buffer.from(canonicalCandidateBytes(profileApplication.profilePlan.material)), + ) + ) { + throw new Error('profile materializer did not capture exact canonical plan bytes') + } + + const container = await resolveContainer(candidate, task, ports) + const resolvedModel = await resolveModel(candidate, task, ports) + const preparationId = `candidate-preparation-v1.${randomBytes(32).toString('base64url')}` + const reservationExpiresAtMs = Date.now() + Math.max(MIN_RESERVATION_TTL_MS, reservationWindowMs) + const modelReservation = await withinCandidateCleanupDeadline( + () => + ports.models.reserveGrant({ + executionId: task.executionId, + preparationId, + expiresAtMs: reservationExpiresAtMs, + attempt: task.attempt, + bundleDigest: bundle.digest, + resolved: resolvedModel, + limits: modelLimits(task.limits), + }), + candidateCleanupDeadline(cleanupTimeoutMs), + 'protected model reservation', + ) + let preparedMemory: Awaited> | undefined + try { + validateProtectedModelReservation( + modelReservation, + task.limits, + preparationId, + reservationExpiresAtMs, + ) + preparedMemory = await prepareMemory( + candidate, + task, + ports, + preparationId, + reservationExpiresAtMs, + cleanupTimeoutMs, + ) + const memory = preparedMemory.value + const knowledge = bundle.knowledge + ? { + snapshotId: bundle.knowledge.snapshotId, + manifestDigest: bundle.knowledge.manifest.sha256, + manifest: await verifiedArtifactBytes(candidate, bundle.knowledge.manifest), + } + : undefined + + const baseLaunch = buildLaunch(candidate, task, profileApplication.flags) + const publicEnv = mergePublicEnvironment( + bundle.execution.env ?? {}, + profileApplication.env, + bundle.execution.instructionDelivery.kind === 'utf8-file' + ? { + [bundle.execution.instructionDelivery.env]: { + kind: 'public', + value: bundle.execution.instructionDelivery.path, + }, + } + : {}, + ) + const routes = modelRoutes(bundle.profile, task.model.requested) + const executionMaterial: AgentCandidateExecutionPlanMaterialV1 = { + schemaVersion: 1, + kind: 'agent-candidate-execution-plan-material', + bundleDigest: bundle.digest, + executionId: task.executionId, + attempt: task.attempt, + task: { + benchmark: task.benchmark, + benchmarkVersion: task.benchmarkVersion, + taskId: task.taskId, + splitDigest: task.splitDigest, + instruction: { + encoding: 'utf8', + sha256: instructionDigest, + byteLength: instructionBytes.byteLength, + delivery: bundle.execution.instructionDelivery, + }, + repository: task.repository, + workspace: task.workspace, + }, + workspaces: { + taskRoot: task.executionRoots.taskRoot, + ...(task.executionRoots.candidateRoot + ? { candidateRoot: task.executionRoots.candidateRoot } + : {}), + }, + codeKind: bundle.code.kind, + ...(bundle.execution.workspace ? { candidateWorkspace: bundle.execution.workspace } : {}), + profile: profileApplication.application, + harness: bundle.execution.harness, + harnessVersion: bundle.execution.harnessVersion, + container, + model: { + policy: 'single', + resolved: resolvedModel, + access: { + kind: 'evaluator-mediated', + grantDigest: modelReservation.digest, + network: modelReservation.network, + }, + routes, + }, + grader: task.grader, + launch: { + executable: baseLaunch.executable, + args: baseLaunch.args, + env: publicEnv, + cwd: bundle.execution.cwd, + }, + ...(bundle.knowledge ? { knowledgeManifestDigest: bundle.knowledge.manifest.sha256 } : {}), + memory, + limits: task.limits, + network: { mode: 'disabled' }, + } + agentCandidateExecutionPlanMaterialSchema.parse(executionMaterial) + const executionBytes = canonicalCandidateBytes(executionMaterial) + const executionDigest = canonicalCandidateDigest(executionMaterial) + if (sha256Bytes(executionBytes) !== executionDigest) { + throw new Error('execution plan canonical serializers disagree') + } + const executionPlan: AgentCandidateExecutionPlanEvidence = + agentCandidateExecutionPlanEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-execution-plan', + digest: executionDigest, + material: executionMaterial, + artifact: embeddedCandidateArtifact(executionBytes), + }) + + const entrypoint = candidateEntrypointReceipt(candidate) + const materializationReceipt = canonicalCandidateDocument( + { + schemaVersion: 1, + kind: 'agent-candidate-materialization', + digestAlgorithm: 'rfc8785-sha256', + bundleDigest: bundle.digest, + profilePlan: profileApplication.profilePlan, + executionPlan, + ...(bundle.execution.workspace ? { candidateWorkspace: bundle.execution.workspace } : {}), + codeKind: bundle.code.kind, + ...(candidate.materializedTree ? { materializedTree: candidate.materializedTree } : {}), + harness: bundle.execution.harness, + harnessVersion: bundle.execution.harnessVersion, + container, + resolvedModel, + ...(bundle.knowledge ? { knowledgeManifestDigest: bundle.knowledge.manifest.sha256 } : {}), + ...(entrypoint ? { entrypoint } : {}), + }, + ) + agentCandidateMaterializationReceiptSchema.parse(materializationReceipt.value) + + const traceRunId = `${task.executionId}:attempt-${task.attempt.number}:${canonicalCandidateDigest({ preparationId }).slice(7, 23)}` + const traceTags = { + [CANDIDATE_TRACE_TAGS.executionId]: task.executionId, + [CANDIDATE_TRACE_TAGS.bundleDigest]: bundle.digest, + [CANDIDATE_TRACE_TAGS.executionPlanDigest]: executionPlan.digest, + [CANDIDATE_TRACE_TAGS.materializationReceiptDigest]: materializationReceipt.digest, + } + const traceEnv = { + [CANDIDATE_TRACE_ENV.executionId]: task.executionId, + [CANDIDATE_TRACE_ENV.bundleDigest]: bundle.digest, + [CANDIDATE_TRACE_ENV.executionPlanDigest]: executionPlan.digest, + [CANDIDATE_TRACE_ENV.materializationReceiptDigest]: materializationReceipt.digest, + [CANDIDATE_TRACE_ENV.traceRunId]: traceRunId, + } + assertEnvironmentDisjoint(publicEnv, traceEnv) + + return createPreparedCandidateExecution({ + ports, + bundle, + executionId: task.executionId, + roots: { + execution: { ...task.executionRoots }, + staging: { ...task.stagingRoots }, + }, + profilePlan: { + value: profileApplication.profilePlan, + bytes: profilePlanBytes, + written: [...profileApplication.application.mountPaths], + }, + executionPlan: { value: executionPlan, bytes: executionBytes }, + materializationReceipt, + launch: { + executable: baseLaunch.executable, + args: baseLaunch.args.map((value) => value.value), + env: unwrapPublicEnvironment(publicEnv), + flags: profileApplication.flags.map((value) => value.value), + cwd: absoluteExecutionCwd(bundle.execution.cwd, task.executionRoots), + }, + instruction: { + bytes: Uint8Array.from(instructionBytes), + delivery: bundle.execution.instructionDelivery, + }, + resolvedModel, + preparationId, + reservationExpiresAtMs, + cleanupTimeoutMs, + resultTimeoutMs, + modelReservation: { + preparationId: modelReservation.preparationId, + digest: modelReservation.digest, + expiresAtMs: modelReservation.expiresAtMs, + enforcedLimits: modelReservation.enforcedLimits, + network: modelReservation.network, + }, + executorInputs: { + taskFiles: taskExecutorFiles, + ...(candidateExecutorFiles ? { candidateFiles: candidateExecutorFiles } : {}), + profileFiles: exactProfileExecutorFiles( + profileWorkspacePlan.files, + profileApplication.profilePlan.material.files, + ), + }, + ...(preparedMemory.accessDigest && preparedMemory.value.mode === 'isolated' + ? { + memoryReservation: { + preparationId, + accessDigest: preparedMemory.accessDigest, + expiresAtMs: reservationExpiresAtMs, + effectiveNamespace: preparedMemory.value.effectiveNamespace, + }, + } + : {}), + ...(knowledge ? { knowledge } : {}), + trace: { runId: traceRunId, tags: traceTags, env: traceEnv }, + memory, + }) + } catch (error) { + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + const cleanup: Array> = [] + if (preparedMemory?.value.mode === 'isolated') { + const accessDigest = preparedMemory.accessDigest + const effectiveNamespace = preparedMemory.value.effectiveNamespace + if (!accessDigest) throw new Error('isolated memory preparation is missing access identity') + cleanup.push( + withinCandidateCleanupDeadline( + async () => { + const closed = await ports.memory.close({ + executionId: task.executionId, + preparationId, + accessDigest, + effectiveNamespace, + reason: 'preparation-failed', + }) + if (closed.closed !== true || Object.keys(closed).some((key) => key !== 'closed')) { + throw new Error('failed preparation did not close isolated memory access') + } + }, + cleanupDeadlineAtMs, + 'failed preparation memory cleanup', + ), + ) + } + cleanup.push( + withinCandidateCleanupDeadline( + async () => { + const settlement = sealAgentCandidateModelSettlement( + await ports.models.settleGrant({ + executionId: task.executionId, + preparationId, + grantDigest: modelReservation.digest, + resolved: resolvedModel, + reason: 'preparation-failed', + }), + { + preparationId, + grantDigest: modelReservation.digest, + model: resolvedModel.model, + }, + ) + if (settlement.usage.modelCalls !== 0) { + throw new Error('failed preparation unexpectedly contains model calls') + } + }, + cleanupDeadlineAtMs, + 'failed preparation model cleanup', + ), + ) + const cleanupResults = await Promise.allSettled(cleanup) + const cleanupErrors = cleanupResults + .filter((result): result is PromiseRejectedResult => result.status === 'rejected') + .map((result) => result.reason) + if (cleanupErrors.length > 0) { + throw new Error( + `candidate preparation failed and protected access cleanup failed: ${cleanupErrors.map(errorMessage).join('; ')}`, + { cause: error }, + ) + } + throw error + } +} + +function assertSameVerificationPorts( + verified: AgentCandidateExecutionPorts | { artifacts: unknown; repositories: unknown }, + execution: AgentCandidateExecutionPorts, +): void { + if ( + verified.artifacts !== execution.artifacts || + verified.repositories !== execution.repositories + ) { + throw new Error( + 'prepare must use the same artifact and repository ports that verified the bundle', + ) + } +} + +function assertTaskInput( + task: AgentCandidateTaskExecution, + delivery: VerifiedAgentCandidate['bundle']['execution']['instructionDelivery'], +): void { + const requiredStrings: Array<[string, string]> = [ + ['executionId', task.executionId], + ['benchmark', task.benchmark], + ['benchmarkVersion', task.benchmarkVersion], + ['taskId', task.taskId], + ['repository identity', task.repository.identity], + ['repository root identity', task.repository.rootIdentity], + ] + for (const [name, value] of requiredStrings) { + if (!value.trim()) throw new Error(`${name} must be non-empty`) + } + if (!/^[A-Za-z0-9._:-]{1,200}$/.test(task.executionId)) { + throw new Error('executionId must be a stable filesystem-neutral identifier') + } + if (!task.instruction || !isWellFormedUnicode(task.instruction)) { + throw new Error('task instruction must be non-empty well-formed Unicode') + } + sha256DigestSchema.parse(task.splitDigest) + agentCandidateWorkspaceSnapshotEvidenceSchema.parse(task.workspace) + agentCandidateExecutionLimitsSchema.parse(task.limits) + if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(task.repository.baseCommit)) { + throw new Error('task repository base commit is not a full Git object id') + } + if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(task.repository.baseTree)) { + throw new Error('task repository base tree is not a full Git object id') + } + if (task.repository.baseCommit.length !== task.repository.baseTree.length) { + throw new Error('task repository Git object formats disagree') + } + for (const [name, root] of [ + ['execution task root', task.executionRoots.taskRoot], + ['execution candidate root', task.executionRoots.candidateRoot], + ['staging task root', task.stagingRoots.taskRoot], + ['staging candidate root', task.stagingRoots.candidateRoot], + ['staging profile root', task.stagingRoots.profileRoot], + ] as const) { + if (root === undefined) continue + const canonical = name.startsWith('execution') + ? posix.isAbsolute(root) && posix.normalize(root) === root + : isAbsolute(root) && resolveHostPath(root) === root + if (!canonical) throw new Error(`${name} must be a canonical absolute path`) + } + if ( + !Number.isInteger(task.attempt.number) || + !Number.isInteger(task.attempt.maxAttempts) || + task.attempt.number < 1 || + task.attempt.number > task.attempt.maxAttempts || + (task.attempt.retryPolicy === 'none' && task.attempt.maxAttempts !== 1) + ) { + throw new Error('task attempt policy is invalid') + } + const limits = task.limits + if ( + !Number.isInteger(limits.timeoutMs) || + limits.timeoutMs <= 0 || + limits.timeoutMs > MAX_CANDIDATE_TIMER_INTERVAL_MS || + !Number.isInteger(limits.maxSteps) || + limits.maxSteps <= 0 || + !Number.isInteger(limits.maxModelCalls) || + limits.maxModelCalls < 0 || + !Number.isInteger(limits.maxInputTokens) || + limits.maxInputTokens < 0 || + !Number.isInteger(limits.maxOutputTokens) || + limits.maxOutputTokens < 0 || + !Number.isFinite(limits.maxCostUsd) || + limits.maxCostUsd < 0 + ) { + throw new Error('task execution limits are invalid') + } + usdToNanos(limits.maxCostUsd, 'task maxCostUsd') + if (!task.model.requested.trim()) throw new Error('evaluator model request must be non-empty') + if (!task.grader.name.trim() || !task.grader.version.trim()) { + throw new Error('evaluator benchmark grader identity must be non-empty') + } + if (!Number.isInteger(task.grader.artifact.byteLength) || task.grader.artifact.byteLength <= 0) { + throw new Error('evaluator benchmark grader artifact must be non-empty') + } + sha256DigestSchema.parse(task.grader.artifact.sha256) + if (task.evaluatorTaskContainer) { + if ( + task.evaluatorTaskContainer.source !== 'evaluator-task-container' || + !task.evaluatorTaskContainer.image.trim() || + !task.evaluatorTaskContainer.platform.os.trim() || + !task.evaluatorTaskContainer.platform.architecture.trim() + ) { + throw new Error('evaluator task container evidence is incomplete') + } + sha256DigestSchema.parse(task.evaluatorTaskContainer.indexDigest) + sha256DigestSchema.parse(task.evaluatorTaskContainer.manifestDigest) + agentCandidateContainerSchema.parse({ + image: task.evaluatorTaskContainer.image, + indexDigest: task.evaluatorTaskContainer.indexDigest, + }) + } + if (delivery.kind === 'utf8-file') { + if ( + executionPathsOverlap(task.executionRoots.taskRoot, delivery.path) || + (task.executionRoots.candidateRoot !== undefined && + executionPathsOverlap(task.executionRoots.candidateRoot, delivery.path)) + ) { + throw new Error('task instruction file must remain outside execution workspaces') + } + } +} + +function executionPathsOverlap(left: string, right: string): boolean { + const a = posix.normalize(left) + const b = posix.normalize(right) + return ( + a === b || b.startsWith(a === '/' ? '/' : `${a}/`) || a.startsWith(b === '/' ? '/' : `${b}/`) + ) +} + +function assertDisjointHostStagingRoots(task: AgentCandidateTaskExecution): void { + const roots = [ + task.stagingRoots.taskRoot, + task.stagingRoots.candidateRoot, + task.stagingRoots.profileRoot, + ] + .filter((value): value is string => value !== undefined) + .map((value) => resolveHostPath(value)) + for (let left = 0; left < roots.length; left++) { + for (let right = left + 1; right < roots.length; right++) { + const a = roots[left] + const b = roots[right] + if (a && b && (a === b || isContainedPath(a, b) || isContainedPath(b, a))) { + throw new Error('host task, candidate, and profile staging roots must be disjoint') + } + } + } +} + +function isContainedPath(parent: string, child: string): boolean { + const path = relative(parent, child) + return path !== '' && !path.startsWith('..') && !isAbsolute(path) +} + +async function assertEmptyDirectory(path: string): Promise { + const stats = await lstat(path) + if (!stats.isDirectory() || stats.isSymbolicLink()) { + throw new Error('profile staging root must be a real directory') + } + if ((await readdir(path)).length !== 0) { + throw new Error('profile staging root must be empty before materialization') + } +} + +function materializerHarness(harness: HarnessType): HarnessId { + if (!MATERIALIZER_HARNESSES.has(harness)) { + throw new Error( + `sealed candidate profile materialization is unsupported for harness ${harness}`, + ) + } + return harness as HarnessId +} + +async function resolveContainer( + candidate: VerifiedAgentCandidate, + task: AgentCandidateTaskExecution, + ports: AgentCandidateExecutionPorts, +): Promise { + const environment = candidate.bundle.execution.environment + const pinned = environment.kind === 'pinned-container' ? environment.container : undefined + if (environment.kind === 'evaluator-task-container' && !task.evaluatorTaskContainer) { + throw new Error('evaluator-task-container candidate requires an evaluator-owned task image') + } + if (environment.kind === 'pinned-container' && task.evaluatorTaskContainer) { + throw new Error('pinned candidate containers cannot be replaced by a task image') + } + const resolved = await ports.containers.resolve({ + candidate: pinned, + evaluatorTaskContainer: task.evaluatorTaskContainer, + }) + if (resolved.source !== environment.kind) throw new Error('resolved container source drifted') + if (pinned && (resolved.image !== pinned.image || resolved.indexDigest !== pinned.indexDigest)) { + throw new Error('resolved pinned container does not match the candidate image index') + } + if ( + task.evaluatorTaskContainer && + JSON.stringify(resolved) !== JSON.stringify(task.evaluatorTaskContainer) + ) { + throw new Error('resolved task container does not match evaluator-owned image evidence') + } + return resolved +} + +async function resolveModel( + candidate: VerifiedAgentCandidate, + task: AgentCandidateTaskExecution, + ports: AgentCandidateExecutionPorts, +): Promise { + const hints = candidate.bundle.profile.model + if (hints?.default !== undefined && hints.default !== task.model.requested) { + throw new Error('candidate model preference conflicts with the evaluator-owned model') + } + if ( + hints?.reasoningEffort !== undefined && + hints.reasoningEffort !== task.model.reasoningEffort + ) { + throw new Error('candidate reasoning effort conflicts with the evaluator-owned effort') + } + const resolved = await ports.models.resolve({ + requested: task.model.requested, + harness: candidate.bundle.execution.harness, + reasoningEffort: task.model.reasoningEffort, + }) + if ( + resolved.requested !== task.model.requested || + resolved.reasoningEffort !== task.model.reasoningEffort + ) { + throw new Error('model resolver drifted from the evaluator-owned request') + } + return resolved +} + +async function prepareMemory( + candidate: VerifiedAgentCandidate, + task: AgentCandidateTaskExecution, + ports: AgentCandidateExecutionPorts, + preparationId: string, + expiresAtMs: number, + cleanupTimeoutMs: number, +): Promise<{ + value: AgentCandidateEffectiveMemory + accessDigest?: `sha256:${string}` +}> { + const policy = candidate.bundle.memory + if (policy.mode === 'disabled') return { value: { mode: 'disabled' } } + const seed = policy.seed ? await verifiedArtifactBytes(candidate, policy.seed) : undefined + const executionSegment = canonicalCandidateDigest({ executionId: task.executionId }).slice(7) + const taskSegment = canonicalCandidateDigest({ taskId: task.taskId }).slice(7) + const preparationSegment = canonicalCandidateDigest({ preparationId }).slice(7, 23) + const effectiveNamespace = `candidate/${candidate.bundle.digest.slice(7, 23)}/${executionSegment}/${taskSegment}/${preparationSegment}` + const reset = await withinCandidateCleanupDeadline( + () => + ports.memory.reset({ + executionId: task.executionId, + preparationId, + expiresAtMs, + effectiveNamespace, + ...(seed ? { seed } : {}), + ...(policy.seed ? { seedDigest: policy.seed.sha256 } : {}), + }), + candidateCleanupDeadline(cleanupTimeoutMs), + 'isolated memory reset', + ) + try { + if ( + reset.preparationId !== preparationId || + reset.expiresAtMs !== expiresAtMs || + !/^sha256:[a-f0-9]{64}$/.test(reset.accessDigest) + ) { + throw new Error('isolated memory reservation is not scoped to this preparation') + } + await readVerifiedArtifact(reset.evidence, ports.artifacts) + await verifyWorkspaceSnapshotArtifacts(reset.beforeState, ports.artifacts) + return { + value: { + mode: 'isolated', + scope: 'task', + effectiveNamespace, + reset: { + kind: 'fresh', + evidence: reset.evidence, + emptyStateDigest: reset.emptyStateDigest, + }, + beforeState: reset.beforeState, + ...(policy.seed ? { seedDigest: policy.seed.sha256 } : {}), + }, + accessDigest: reset.accessDigest, + } + } catch (error) { + try { + const closed = await withinCandidateCleanupDeadline( + () => + ports.memory.close({ + executionId: task.executionId, + preparationId, + accessDigest: reset.accessDigest, + effectiveNamespace, + reason: 'preparation-failed', + }), + candidateCleanupDeadline(cleanupTimeoutMs), + 'invalid isolated memory reset cleanup', + ) + if (closed.closed !== true || Object.keys(closed).some((key) => key !== 'closed')) { + throw new Error('invalid isolated memory reset did not acknowledge closure') + } + } catch (closeError) { + throw new Error('isolated memory preparation and cleanup both failed', { + cause: new AggregateError([error, closeError]), + }) + } + throw error + } +} + +function buildLaunch( + candidate: VerifiedAgentCandidate, + task: AgentCandidateTaskExecution, + profileFlags: AgentCandidateConfigValue[], +): { executable: string; args: AgentCandidateConfigValue[] } { + const launch = candidate.bundle.execution.launch + if (launch.kind === 'container-command') { + return { executable: launch.executable, args: [...(launch.args ?? []), ...profileFlags] } + } + const candidateRoot = task.executionRoots.candidateRoot + if (!candidateRoot) throw new Error('candidate entrypoint requires a container candidate root') + const entrypoint = posix.join(candidateRoot, launch.entrypoint) + const candidateArgs = launch.args ?? [] + if (launch.interpreter) { + return { + executable: launch.interpreter, + args: [{ kind: 'public', value: entrypoint }, ...candidateArgs, ...profileFlags], + } + } + return { executable: entrypoint, args: [...candidateArgs, ...profileFlags] } +} + +function mergePublicEnvironment( + ...records: Array> +): Record { + const output: Record = {} + for (const record of records) { + for (const [name, value] of Object.entries(record)) { + const previous = output[name] + if (previous && previous.value !== value.value) { + throw new Error(`candidate and profile environment disagree on ${name}`) + } + output[name] = value + } + } + return output +} + +function unwrapPublicEnvironment( + values: Record, +): Record { + return Object.fromEntries(Object.entries(values).map(([name, value]) => [name, value.value])) +} + +function modelRoutes( + profile: VerifiedAgentCandidate['bundle']['profile'], + requested: string, +): AgentCandidateExecutionPlanMaterialV1['model']['routes'] { + const routes: AgentCandidateExecutionPlanMaterialV1['model']['routes'] = [ + { kind: 'primary', requested }, + ] + if (profile.model?.small) routes.push({ kind: 'small', requested }) + for (const name of Object.keys(profile.modes ?? {}).sort()) { + if (profile.modes?.[name]?.model) routes.push({ kind: 'mode', name, requested }) + } + for (const name of Object.keys(profile.subagents ?? {}).sort()) { + if (profile.subagents?.[name]?.model) routes.push({ kind: 'subagent', name, requested }) + } + return routes +} + +function candidateEntrypointReceipt( + candidate: VerifiedAgentCandidate, +): { path: string; sha256: `sha256:${string}`; byteLength: number } | undefined { + const launch = candidate.bundle.execution.launch + const workspace = candidate.bundle.execution.workspace + if (launch.kind !== 'candidate-entrypoint' || !workspace) return undefined + const file = workspace.material.files.find((entry) => entry.path === launch.entrypoint) + if (!file) throw new Error('candidate entrypoint is absent from the verified workspace') + return { path: file.path, sha256: file.sha256, byteLength: file.byteLength } +} + +function absoluteExecutionCwd( + cwd: VerifiedAgentCandidate['bundle']['execution']['cwd'], + roots: AgentCandidateTaskExecution['executionRoots'], +): string { + const root = cwd.workspace === 'task' ? roots.taskRoot : roots.candidateRoot + if (!root) throw new Error('candidate cwd is missing its execution workspace root') + const absolute = cwd.path === '.' ? root : posix.join(root, cwd.path) + if (absolute !== root && !absolute.startsWith(`${root}/`)) { + throw new Error('candidate cwd escapes its execution workspace') + } + return absolute +} + +function validateProtectedModelReservation( + reservation: { + preparationId: string + digest: string + expiresAtMs: number + enforcedLimits: { + maxModelCalls: number + maxInputTokens: number + maxOutputTokens: number + maxCostUsd: number + } + network: AgentCandidateModelAccessNetwork + }, + expectedLimits: AgentCandidateExecutionLimits, + preparationId: string, + expiresAtMs: number, +): void { + if (!/^sha256:[a-f0-9]{64}$/.test(reservation.digest)) { + throw new Error('protected model reservation has an invalid identity digest') + } + if (reservation.preparationId !== preparationId || reservation.expiresAtMs !== expiresAtMs) { + throw new Error('protected model reservation is not scoped to this preparation') + } + const limits = modelLimits(expectedLimits) + if (canonicalCandidateDigest(reservation.enforcedLimits) !== canonicalCandidateDigest(limits)) { + throw new Error('protected model reservation does not enforce the frozen model limits') + } + const expectedNetworkMode = limits.maxModelCalls === 0 ? 'disabled' : 'gateway-only' + const network = agentCandidateModelAccessNetworkSchema.parse(reservation.network) + if (network.mode !== expectedNetworkMode) { + throw new Error('protected model reservation has the wrong network policy for its call limit') + } +} + +function assertEnvironmentDisjoint( + publicEnv: Record, + traceEnv: Record, +): void { + const seen = new Set(Object.keys(publicEnv)) + for (const name of Object.keys(traceEnv)) { + if (seen.has(name)) throw new Error(`evaluator environment binding collides with ${name}`) + seen.add(name) + } +} + +function modelLimits(limits: AgentCandidateExecutionLimits): { + maxModelCalls: number + maxInputTokens: number + maxOutputTokens: number + maxCostUsd: number +} { + return { + maxModelCalls: limits.maxModelCalls, + maxInputTokens: limits.maxInputTokens, + maxOutputTokens: limits.maxOutputTokens, + maxCostUsd: limits.maxCostUsd, + } +} + +function exactProfileExecutorFiles( + sourceFiles: ReadonlyArray<{ relPath: string; content: string; mode?: number }>, + expectedFiles: ReadonlyArray<{ relPath: string; mode: number; contentSha256: string }>, +): Array<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> { + const byPath = new Map(sourceFiles.map((file) => [file.relPath, file])) + if (byPath.size !== sourceFiles.length || sourceFiles.length !== expectedFiles.length) { + throw new Error('profile source files do not match the signed profile plan') + } + return expectedFiles.map((expected) => { + const source = byPath.get(expected.relPath) + const mode = source?.mode ?? 0o644 + const bytes = Buffer.from(source?.content ?? '', 'utf8') + if ( + !source || + (mode !== 0o644 && mode !== 0o755) || + mode !== expected.mode || + sha256Bytes(bytes) !== expected.contentSha256 + ) { + throw new Error('profile source files do not match the signed profile plan') + } + return { path: expected.relPath, mode, bytes: Uint8Array.from(bytes) } + }) +} + +function isWellFormedUnicode(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index) + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1) + if (!(next >= 0xdc00 && next <= 0xdfff)) return false + index++ + } else if (code >= 0xdc00 && code <= 0xdfff) { + return false + } + } + return true +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/src/candidate-execution/prepared-state.ts b/src/candidate-execution/prepared-state.ts new file mode 100644 index 00000000..87522785 --- /dev/null +++ b/src/candidate-execution/prepared-state.ts @@ -0,0 +1,608 @@ +import { + agentCandidateExecutionPlanEvidenceSchema, + agentCandidateMaterializationReceiptSchema, + agentCandidateProfilePlanEvidenceSchema, +} from '@tangle-network/agent-interface' + +import { verifyMaterializedProfileWorkspace, verifyMaterializedWorkspace } from './artifacts' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + canonicalCandidateDocument, + immutableCandidateValue, + omitTopLevelDigest, + sha256Bytes, +} from './digest' +import { verifyTaskCheckout } from './git-materialize' +import type { + AgentCandidateExecutionPorts, + AgentCandidateExecutorRequest, + AgentCandidateProtectedModelActivation, + AgentCandidateProtectedModelReservation, + PreparedAgentCandidateExecution, +} from './types' +import { CANDIDATE_TRACE_ENV, CANDIDATE_TRACE_TAGS, preparedCandidateBrand } from './types' + +export interface PreparedCandidateState { + ports: AgentCandidateExecutionPorts + bundle: PreparedAgentCandidateExecution['bundle'] + executionId: string + roots: PreparedAgentCandidateExecution['roots'] + profilePlan: PreparedAgentCandidateExecution['profilePlan'] + executionPlan: PreparedAgentCandidateExecution['executionPlan'] + materializationReceipt: PreparedAgentCandidateExecution['materializationReceipt'] + launch: PreparedAgentCandidateExecution['launch'] + instruction: PreparedAgentCandidateExecution['instruction'] + resolvedModel: PreparedAgentCandidateExecution['resolvedModel'] + preparationId: string + reservationExpiresAtMs: number + cleanupTimeoutMs: number + resultTimeoutMs: number + modelReservation: AgentCandidateProtectedModelReservation + executorInputs: { + taskFiles: ReadonlyArray<{ + path: string + mode: 0o644 | 0o755 + bytes: Uint8Array + }> + candidateFiles?: ReadonlyArray<{ + path: string + mode: 0o644 | 0o755 + bytes: Uint8Array + }> + profileFiles: ReadonlyArray<{ + path: string + mode: 0o644 | 0o755 + bytes: Uint8Array + }> + } + memoryReservation?: { + preparationId: string + accessDigest: `sha256:${string}` + expiresAtMs: number + effectiveNamespace: string + } + knowledge?: PreparedAgentCandidateExecution['knowledge'] + trace: PreparedAgentCandidateExecution['trace'] + memory: PreparedAgentCandidateExecution['memory'] +} + +const stateByExecution = new WeakMap() +const lifecycleByExecution = new WeakMap< + PreparedAgentCandidateExecution, + { + status: + | 'prepared' + | 'claiming' + | 'claimed' + | 'running' + | 'settling' + | 'disposing' + | 'succeeded' + | 'failed' + | 'disposal-failed' + | 'cleanup-failed' + | 'disposed' + } +>() + +export function createPreparedCandidateExecution( + input: PreparedCandidateState, +): PreparedAgentCandidateExecution { + const state = detachPreparedCandidateState(input) + assertPrivateCandidateIntegrity(state) + const prepared = Object.freeze({ + bundle: state.bundle, + executionId: state.executionId, + roots: state.roots, + profilePlan: evidenceView(state.profilePlan), + executionPlan: evidenceView(state.executionPlan), + materializationReceipt: state.materializationReceipt, + launch: state.launch, + instruction: bytesView(state.instruction, 'bytes'), + resolvedModel: state.resolvedModel, + ...(state.knowledge ? { knowledge: knowledgeView(state.knowledge) } : {}), + trace: state.trace, + memory: state.memory, + [preparedCandidateBrand]: true as const, + }) + stateByExecution.set(prepared, state) + lifecycleByExecution.set(prepared, { status: 'prepared' }) + return prepared +} + +export function getPreparedCandidateState( + prepared: PreparedAgentCandidateExecution, +): PreparedCandidateState { + const state = stateByExecution.get(prepared) + if (!state || prepared[preparedCandidateBrand] !== true) { + throw new Error('execution must come from prepareAgentCandidateExecution') + } + return state +} + +/** Revalidates the exact private bytes immediately before execution or finalization. */ +export function assertPreparedCandidateIntegrity( + prepared: PreparedAgentCandidateExecution, +): PreparedCandidateState { + const state = getPreparedCandidateState(prepared) + assertPrivateCandidateIntegrity(state) + return state +} + +/** Atomically reserves this in-memory prepared value before the first await. */ +export function beginPreparedCandidateClaim( + prepared: PreparedAgentCandidateExecution, +): PreparedCandidateState { + const state = assertPreparedCandidateIntegrity(prepared) + transitionLifecycle(prepared, ['prepared'], 'claiming') + return state +} + +/** Claim an unexecuted preparation for explicit resource disposal. */ +export function beginPreparedCandidateDisposal( + prepared: PreparedAgentCandidateExecution, +): PreparedCandidateState { + const state = assertPreparedCandidateIntegrity(prepared) + transitionLifecycle(prepared, ['prepared', 'disposal-failed', 'cleanup-failed'], 'disposing') + return state +} + +export function markPreparedCandidateClaimed(prepared: PreparedAgentCandidateExecution): void { + transitionLifecycle(prepared, ['claiming'], 'claimed') +} + +/** Reveal protected values only after the durable claim and workspace checks succeed. */ +export function beginPreparedCandidateRun( + prepared: PreparedAgentCandidateExecution, + modelAccess: AgentCandidateProtectedModelActivation, + memoryAccess?: { env: Readonly> }, +): { state: PreparedCandidateState; request: AgentCandidateExecutorRequest } { + const state = assertPreparedCandidateIntegrity(prepared) + assertProtectedEnvironment(state, modelAccess.env, memoryAccess?.env) + transitionLifecycle(prepared, ['claimed'], 'running') + const completeEnvironment = immutableCandidateValue({ + ...state.launch.env, + ...modelAccess.env, + ...(memoryAccess?.env ?? {}), + ...state.trace.env, + }) + const material = state.executionPlan.value.material + const request = Object.freeze({ + executionId: state.executionId, + inputs: Object.freeze({ + task: workspaceInputView(material.task.workspace, state.executorInputs.taskFiles), + ...(material.candidateWorkspace && state.executorInputs.candidateFiles + ? { + candidate: workspaceInputView( + material.candidateWorkspace, + state.executorInputs.candidateFiles, + ), + } + : {}), + profile: Object.freeze({ + files: Object.freeze( + state.executorInputs.profileFiles.map((file) => profileFileView(file)), + ), + }), + }), + roots: state.roots.execution, + profilePlan: evidenceView(state.profilePlan), + executionPlan: evidenceView(state.executionPlan), + materializationReceipt: state.materializationReceipt, + launch: immutableCandidateValue({ ...state.launch, env: completeEnvironment }), + instruction: bytesView(state.instruction, 'bytes'), + resolvedModel: state.resolvedModel, + hardLimits: Object.freeze({ timeoutMs: state.executionPlan.value.material.limits.timeoutMs }), + observedLimits: Object.freeze({ maxSteps: state.executionPlan.value.material.limits.maxSteps }), + ...(state.knowledge ? { knowledge: knowledgeView(state.knowledge) } : {}), + trace: state.trace, + memory: state.memory, + }) + return { state, request } +} + +export function consumePreparedCandidateExecution( + prepared: PreparedAgentCandidateExecution, + outcome: 'succeeded' | 'failed' | 'disposed' | 'disposal-failed' | 'cleanup-failed', +): void { + transitionLifecycle( + prepared, + outcome === 'disposed' || outcome === 'disposal-failed' ? ['disposing'] : ['settling'], + outcome, + ) +} + +export function beginPreparedCandidateSettlement(prepared: PreparedAgentCandidateExecution): void { + transitionLifecycle(prepared, ['claiming', 'claimed', 'running'], 'settling') +} + +/** Rechecks every mutable staging byte immediately before handing control to an executor. */ +export async function assertPreparedCandidateWorkspaces( + state: PreparedCandidateState, +): Promise { + const plan = state.executionPlan.value.material + await verifyMaterializedWorkspace(state.roots.staging.taskRoot, plan.task.workspace.material, { + ignoredProtectedRootEntries: ['.git', '.sidecar'], + }) + await verifyTaskCheckout(state.roots.staging.taskRoot, plan.task.repository) + await verifyMaterializedProfileWorkspace( + state.roots.staging.profileRoot, + state.profilePlan.value.material, + ) + if (plan.candidateWorkspace) { + const candidateRoot = state.roots.staging.candidateRoot + if (!candidateRoot) throw new Error('prepared candidate staging root is missing') + await verifyMaterializedWorkspace(candidateRoot, plan.candidateWorkspace.material) + } else if (state.roots.staging.candidateRoot !== undefined) { + throw new Error('disabled candidate unexpectedly has a staging root') + } +} + +function detachPreparedCandidateState(input: PreparedCandidateState): PreparedCandidateState { + const profilePlan = immutableCandidateValue( + agentCandidateProfilePlanEvidenceSchema.parse(input.profilePlan.value), + ) + const executionPlan = immutableCandidateValue( + agentCandidateExecutionPlanEvidenceSchema.parse(input.executionPlan.value), + ) + const receiptValue = immutableCandidateValue( + agentCandidateMaterializationReceiptSchema.parse(input.materializationReceipt.value), + ) + const materializationReceipt = canonicalCandidateDocument( + omitTopLevelDigest(receiptValue), + ) as PreparedAgentCandidateExecution['materializationReceipt'] + if (materializationReceipt.digest !== input.materializationReceipt.digest) { + throw new Error('materialization receipt digest changed while sealing prepared execution') + } + return Object.freeze({ + ports: input.ports, + bundle: input.bundle, + executionId: input.executionId, + roots: immutableCandidateValue(input.roots), + profilePlan: Object.freeze({ + value: profilePlan, + bytes: Uint8Array.from(input.profilePlan.bytes), + written: Object.freeze([...input.profilePlan.written]), + }), + executionPlan: Object.freeze({ + value: executionPlan, + bytes: Uint8Array.from(input.executionPlan.bytes), + }), + materializationReceipt, + launch: immutableCandidateValue(input.launch), + instruction: Object.freeze({ + bytes: Uint8Array.from(input.instruction.bytes), + delivery: immutableCandidateValue(input.instruction.delivery), + }), + resolvedModel: immutableCandidateValue(input.resolvedModel), + preparationId: input.preparationId, + reservationExpiresAtMs: input.reservationExpiresAtMs, + cleanupTimeoutMs: input.cleanupTimeoutMs, + resultTimeoutMs: input.resultTimeoutMs, + modelReservation: immutableCandidateValue(input.modelReservation), + executorInputs: Object.freeze({ + taskFiles: immutableExecutorFiles(input.executorInputs.taskFiles), + ...(input.executorInputs.candidateFiles + ? { candidateFiles: immutableExecutorFiles(input.executorInputs.candidateFiles) } + : {}), + profileFiles: Object.freeze( + input.executorInputs.profileFiles.map((file) => + Object.freeze({ ...file, bytes: Uint8Array.from(file.bytes) }), + ), + ), + }), + ...(input.memoryReservation + ? { memoryReservation: immutableCandidateValue(input.memoryReservation) } + : {}), + ...(input.knowledge + ? { + knowledge: Object.freeze({ + snapshotId: input.knowledge.snapshotId, + manifestDigest: input.knowledge.manifestDigest, + manifest: Uint8Array.from(input.knowledge.manifest), + }), + } + : {}), + trace: immutableCandidateValue(input.trace), + memory: immutableCandidateValue(input.memory), + }) +} + +function assertPrivateCandidateIntegrity(state: PreparedCandidateState): void { + const bundleMaterial = omitTopLevelDigest(state.bundle) + if (canonicalCandidateDigest(bundleMaterial) !== state.bundle.digest) { + throw new Error('prepared candidate bundle no longer matches its digest') + } + assertPlanEvidence(state.profilePlan.value, state.profilePlan.bytes, 'profile plan') + assertPlanEvidence(state.executionPlan.value, state.executionPlan.bytes, 'execution plan') + agentCandidateExecutionPlanEvidenceSchema.parse(state.executionPlan.value) + + const receipt = agentCandidateMaterializationReceiptSchema.parse( + state.materializationReceipt.value, + ) + const receiptBytes = canonicalCandidateBytes(omitTopLevelDigest(receipt)) + if ( + canonicalCandidateDigest(omitTopLevelDigest(receipt)) !== state.materializationReceipt.digest || + !Buffer.from(receiptBytes).equals(Buffer.from(state.materializationReceipt.bytes)) + ) { + throw new Error('prepared materialization receipt no longer matches its canonical bytes') + } + + const instruction = state.executionPlan.value.material.task.instruction + if ( + sha256Bytes(state.instruction.bytes) !== instruction.sha256 || + state.instruction.bytes.byteLength !== instruction.byteLength || + JSON.stringify(state.instruction.delivery) !== JSON.stringify(instruction.delivery) + ) { + throw new Error('prepared instruction no longer matches the signed execution plan') + } + if ( + !/^candidate-preparation-v1\.[A-Za-z0-9_-]{43}$/.test(state.preparationId) || + !Number.isSafeInteger(state.reservationExpiresAtMs) || + state.reservationExpiresAtMs <= 0 || + !Number.isSafeInteger(state.cleanupTimeoutMs) || + state.cleanupTimeoutMs <= 0 || + !Number.isSafeInteger(state.resultTimeoutMs) || + state.resultTimeoutMs <= 0 || + JSON.stringify(state.resolvedModel) !== + JSON.stringify(state.executionPlan.value.material.model.resolved) || + state.modelReservation.digest !== state.executionPlan.value.material.model.access.grantDigest || + state.modelReservation.preparationId !== state.preparationId || + state.modelReservation.expiresAtMs !== state.reservationExpiresAtMs || + canonicalCandidateDigest(state.modelReservation.network) !== + canonicalCandidateDigest(state.executionPlan.value.material.model.access.network) || + canonicalCandidateDigest(state.modelReservation.enforcedLimits) !== + canonicalCandidateDigest(modelLimits(state.executionPlan.value.material.limits)) + ) { + throw new Error('prepared model access no longer matches the signed execution plan') + } + assertExecutorInputs(state) + if ( + (state.memory.mode === 'isolated' && !state.memoryReservation) || + (state.memory.mode === 'disabled' && state.memoryReservation) + ) { + throw new Error('prepared memory reservation does not match the signed execution plan') + } + if ( + state.memory.mode === 'isolated' && + state.memoryReservation && + (state.memoryReservation.preparationId !== state.preparationId || + state.memoryReservation.expiresAtMs !== state.reservationExpiresAtMs || + state.memoryReservation.effectiveNamespace !== state.memory.effectiveNamespace) + ) { + throw new Error('prepared memory reservation identity no longer matches the execution') + } + if (JSON.stringify(state.memory) !== JSON.stringify(state.executionPlan.value.material.memory)) { + throw new Error('prepared memory no longer matches the signed execution plan') + } + + const expectedTags = { + [CANDIDATE_TRACE_TAGS.executionId]: state.executionId, + [CANDIDATE_TRACE_TAGS.bundleDigest]: state.bundle.digest, + [CANDIDATE_TRACE_TAGS.executionPlanDigest]: state.executionPlan.value.digest, + [CANDIDATE_TRACE_TAGS.materializationReceiptDigest]: state.materializationReceipt.digest, + } + const expectedTraceEnvironment = { + [CANDIDATE_TRACE_ENV.executionId]: state.executionId, + [CANDIDATE_TRACE_ENV.bundleDigest]: state.bundle.digest, + [CANDIDATE_TRACE_ENV.executionPlanDigest]: state.executionPlan.value.digest, + [CANDIDATE_TRACE_ENV.materializationReceiptDigest]: state.materializationReceipt.digest, + [CANDIDATE_TRACE_ENV.traceRunId]: state.trace.runId, + } + if ( + !state.trace.runId.startsWith( + `${state.executionId}:attempt-${state.executionPlan.value.material.attempt.number}:`, + ) || + canonicalCandidateDigest(state.trace.tags) !== canonicalCandidateDigest(expectedTags) || + canonicalCandidateDigest(state.trace.env) !== canonicalCandidateDigest(expectedTraceEnvironment) + ) { + throw new Error('prepared trace identity no longer matches the signed execution') + } +} + +function assertPlanEvidence( + evidence: + | PreparedAgentCandidateExecution['profilePlan']['value'] + | PreparedAgentCandidateExecution['executionPlan']['value'], + bytes: Uint8Array, + label: string, +): void { + const expected = canonicalCandidateBytes(evidence.material) + if ( + sha256Bytes(expected) !== evidence.digest || + !Buffer.from(expected).equals(Buffer.from(bytes)) || + evidence.artifact.sha256 !== evidence.digest || + evidence.artifact.byteLength !== bytes.byteLength || + !('content' in evidence.artifact) || + !Buffer.from(evidence.artifact.content, 'base64').equals(Buffer.from(bytes)) + ) { + throw new Error(`prepared ${label} no longer matches its canonical bytes`) + } +} + +function evidenceView(evidence: T): T { + const bytes = Uint8Array.from(evidence.bytes) + return Object.freeze({ + ...evidence, + get bytes(): Uint8Array { + return Uint8Array.from(bytes) + }, + }) +} + +function bytesView(value: T, key: 'bytes'): T { + const bytes = Uint8Array.from(value.bytes) + return Object.freeze({ + ...value, + get [key](): Uint8Array { + return Uint8Array.from(bytes) + }, + }) +} + +function knowledgeView( + knowledge: NonNullable, +): NonNullable { + const manifest = Uint8Array.from(knowledge.manifest) + return Object.freeze({ + snapshotId: knowledge.snapshotId, + manifestDigest: knowledge.manifestDigest, + get manifest(): Uint8Array { + return Uint8Array.from(manifest) + }, + }) +} + +function workspaceInputView( + snapshot: AgentCandidateExecutorRequest['inputs']['task']['snapshot'], + sourceFiles: PreparedCandidateState['executorInputs']['taskFiles'], +): AgentCandidateExecutorRequest['inputs']['task'] { + return Object.freeze({ + snapshot, + files: Object.freeze(sourceFiles.map((file) => profileFileView(file))), + }) +} + +function profileFileView( + source: PreparedCandidateState['executorInputs']['profileFiles'][number], +): AgentCandidateExecutorRequest['inputs']['profile']['files'][number] { + const bytes = Uint8Array.from(source.bytes) + return Object.freeze({ + path: source.path, + mode: source.mode, + get bytes(): Uint8Array { + return Uint8Array.from(bytes) + }, + }) +} + +function assertExecutorInputs(state: PreparedCandidateState): void { + const material = state.executionPlan.value.material + assertWorkspaceExecutorFiles(state.executorInputs.taskFiles, material.task.workspace.material) + if (material.candidateWorkspace) { + if (!state.executorInputs.candidateFiles) { + throw new Error('prepared candidate executor files are missing') + } + assertWorkspaceExecutorFiles( + state.executorInputs.candidateFiles, + material.candidateWorkspace.material, + ) + } else if (state.executorInputs.candidateFiles) { + throw new Error('disabled candidate has executor files') + } + + const expectedFiles = state.profilePlan.value.material.files + if (state.executorInputs.profileFiles.length !== expectedFiles.length) { + throw new Error('prepared profile executor files do not match the signed profile plan') + } + for (let index = 0; index < expectedFiles.length; index++) { + const expected = expectedFiles[index] + const actual = state.executorInputs.profileFiles[index] + if ( + !expected || + !actual || + actual.path !== expected.relPath || + actual.mode !== expected.mode || + sha256Bytes(actual.bytes) !== expected.contentSha256 + ) { + throw new Error('prepared profile executor files do not match the signed profile plan') + } + } +} + +function assertWorkspaceExecutorFiles( + actualFiles: PreparedCandidateState['executorInputs']['taskFiles'], + expected: PreparedAgentCandidateExecution['executionPlan']['value']['material']['task']['workspace']['material'], +): void { + if (actualFiles.length !== expected.files.length) { + throw new Error('prepared workspace executor files do not match the signed manifest') + } + for (let index = 0; index < expected.files.length; index++) { + const actual = actualFiles[index] + const planned = expected.files[index] + if ( + !actual || + !planned || + actual.path !== planned.path || + actual.mode !== planned.mode || + actual.bytes.byteLength !== planned.byteLength || + sha256Bytes(actual.bytes) !== planned.sha256 + ) { + throw new Error('prepared workspace executor files do not match the signed manifest') + } + } +} + +function immutableExecutorFiles( + files: PreparedCandidateState['executorInputs']['taskFiles'], +): ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> { + return Object.freeze( + files.map((file) => Object.freeze({ ...file, bytes: Uint8Array.from(file.bytes) })), + ) +} + +function modelLimits( + limits: PreparedAgentCandidateExecution['executionPlan']['value']['material']['limits'], +): AgentCandidateProtectedModelReservation['enforcedLimits'] { + return { + maxModelCalls: limits.maxModelCalls, + maxInputTokens: limits.maxInputTokens, + maxOutputTokens: limits.maxOutputTokens, + maxCostUsd: limits.maxCostUsd, + } +} + +function transitionLifecycle( + prepared: PreparedAgentCandidateExecution, + expected: ReadonlyArray< + | 'prepared' + | 'claiming' + | 'claimed' + | 'running' + | 'settling' + | 'disposing' + | 'succeeded' + | 'failed' + | 'disposal-failed' + | 'cleanup-failed' + | 'disposed' + >, + next: + | 'claiming' + | 'claimed' + | 'running' + | 'settling' + | 'disposing' + | 'succeeded' + | 'failed' + | 'disposal-failed' + | 'cleanup-failed' + | 'disposed', +): void { + const lifecycle = lifecycleByExecution.get(prepared) + if (!lifecycle || !expected.includes(lifecycle.status)) { + throw new Error(`prepared candidate execution is already ${lifecycle?.status ?? 'unknown'}`) + } + lifecycle.status = next +} + +function assertProtectedEnvironment( + state: PreparedCandidateState, + modelEnvironment: Readonly>, + memoryEnvironment: Readonly> | undefined, +): void { + const seen = new Set([...Object.keys(state.launch.env), ...Object.keys(state.trace.env)]) + for (const [name, value] of [ + ...Object.entries(modelEnvironment), + ...Object.entries(memoryEnvironment ?? {}), + ]) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || typeof value !== 'string' || value.length < 8) { + throw new Error('protected model activation contains an invalid environment binding') + } + if (seen.has(name)) { + throw new Error(`protected model activation collides with environment binding ${name}`) + } + seen.add(name) + } +} diff --git a/src/candidate-execution/profile.ts b/src/candidate-execution/profile.ts new file mode 100644 index 00000000..7f814121 --- /dev/null +++ b/src/candidate-execution/profile.ts @@ -0,0 +1,361 @@ +import type { + AgentCandidateConfigValue, + AgentCandidateProfile, + AgentCandidateResourceRef, + AgentProfile, + AgentProfileDiff, + AgentProfileMcpServer, + AgentProfileResourceRef, +} from '@tangle-network/agent-interface' +import { + agentCandidateProfileSchema, + agentProfileDiffSchema, + agentProfileSchema, + applyAgentProfileDiff, +} from '@tangle-network/agent-interface' + +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + embeddedCandidateArtifact, +} from './digest' + +const CANDIDATE_PROFILE_DIRECT_FIELDS = [ + 'name', + 'description', + 'version', + 'tags', + 'prompt', + 'harness', + 'permissions', + 'tools', + 'confidential', +] as const + +/** Convert only behavior-preserving generic profile fields into the closed candidate contract. */ +export function freezeGenericAgentCandidateProfile(input: AgentProfile): AgentCandidateProfile { + const profile = parseExactAgentProfile(input, 'profile') + if (profile.connections !== undefined) unsupportedProfileField('connections') + if (profile.metadata !== undefined) unsupportedProfileField('metadata') + if (profile.extensions !== undefined) unsupportedProfileField('extensions') + if (profile.model?.metadata !== undefined) unsupportedProfileField('model.metadata') + + const candidate: Record = {} + copyDirectProfileFields(candidate, profile as Record) + if (profile.model) { + const { metadata: _metadata, ...model } = profile.model + candidate.model = model + } + if (profile.mcp) candidate.mcp = freezeMcpServers(profile.mcp) + if (profile.subagents) { + candidate.subagents = Object.fromEntries( + Object.entries(profile.subagents).map(([name, subagent]) => { + if (subagent.metadata !== undefined) unsupportedProfileField(`subagents.${name}.metadata`) + const { metadata: _metadata, ...value } = subagent + return [name, value] + }), + ) + } + if (profile.resources) candidate.resources = freezeResources(profile.resources) + if (profile.hooks && Object.values(profile.hooks).some((commands) => commands.length > 0)) { + throw new Error( + 'generic AgentProfile hooks cannot be safely tokenized; use a candidate-profile source with executable/args', + ) + } + if (profile.hooks) candidate.hooks = profile.hooks + if (profile.modes) { + candidate.modes = Object.fromEntries( + Object.entries(profile.modes).map(([name, mode]) => { + if (mode.metadata !== undefined) unsupportedProfileField(`modes.${name}.metadata`) + const { metadata: _metadata, ...value } = mode + return [name, value] + }), + ) + } + const parsed = parseExactCandidateProfile(candidate) + assertCandidateProfileBinding(profile, parsed) + return parsed +} + +/** Prove the measured generic profile and sealed candidate profile describe the same behavior. */ +export function assertCandidateProfileBinding( + measuredInput: AgentProfile, + bundled: AgentCandidateProfile, +): void { + const measured = parseExactAgentProfile(measuredInput, 'proposal candidate profile') + if (measured.connections || measured.metadata || measured.extensions) { + throw new Error('proposal candidate profile contains fields unsupported by sealed candidates') + } + const normalized = candidateProfileAsAgentProfile(bundled) + if (canonicalCandidateDigest(measured) !== canonicalCandidateDigest(normalized)) { + throw new Error('proposal candidateProfile does not match candidateBundle.profile') + } +} + +/** Parse a complete profile without silently discarding unsupported fields. */ +export function parseExactAgentProfile(input: unknown, label: string): AgentProfile { + const parsed = agentProfileSchema.parse(input) as AgentProfile + assertCanonicalParse(input, parsed, label) + return parsed +} + +/** Parse a profile diff without silently discarding unsupported fields. */ +export function parseExactAgentProfileDiff(input: unknown, label: string): AgentProfileDiff { + const parsed = agentProfileDiffSchema.parse(input) as AgentProfileDiff + assertCanonicalParse(input, parsed, label) + return parsed +} + +/** Apply one exact diff and reject any value that cannot be preserved canonically. */ +export function applyExactAgentProfileDiff( + baseInput: unknown, + diffInput: unknown, + label: string, +): AgentProfile { + const base = parseExactAgentProfile(baseInput, `${label} base profile`) + const diff = parseExactAgentProfileDiff(diffInput, `${label} diff`) + const applied = omitUndefinedObjectFields(applyAgentProfileDiff(base, diff), label) + return parseExactAgentProfile(applied, `${label} result`) +} + +export function parseExactCandidateProfile(input: unknown): AgentCandidateProfile { + const parsed = agentCandidateProfileSchema.parse(input) + assertCanonicalParse(input, parsed, 'candidate profile') + return parsed +} + +/** Reject profile fields whose behavioral effect is not yet proven by candidate executors. */ +export function assertCandidateProfileExecutionSupport(profile: AgentCandidateProfile): void { + const fields = ['tools', 'permissions', 'modes', 'confidential'] as const + const unsupported = fields.filter((field) => hasEntries(profile[field])) + if (unsupported.length > 0) { + throw new Error( + `candidate execution cannot prove in-session effects for non-empty AgentProfile fields: ${unsupported.join(', ')}`, + ) + } +} + +function candidateProfileAsAgentProfile(candidate: AgentCandidateProfile): AgentProfile { + const output: Record = {} + copyDirectProfileFields(output, candidate as unknown as Record) + if (candidate.model) output.model = { ...candidate.model } + if (candidate.mcp) { + output.mcp = Object.fromEntries( + Object.entries(candidate.mcp).map(([name, server]) => [ + name, + { + ...server, + ...(server.args ? { args: server.args.map(publicValue) } : {}), + ...(server.env ? { env: mapPublicValues(server.env) } : {}), + }, + ]), + ) + } + if (candidate.subagents) output.subagents = cloneRecord(candidate.subagents) + if (candidate.modes) output.modes = cloneRecord(candidate.modes) + if (candidate.hooks) { + output.hooks = Object.fromEntries( + Object.entries(candidate.hooks).map(([event, hooks]) => [ + event, + hooks.map(({ executable, args, env, ...hook }) => ({ + ...hook, + command: [executable, ...(args ?? []).map(publicValue)].map(shellQuote).join(' '), + ...(env ? { env: mapPublicValues(env) } : {}), + })), + ]), + ) + } + if (candidate.resources) { + if (candidate.resources.failOnError !== true) { + throw new Error('proposal candidate profile contains fields unsupported by sealed candidates') + } + output.resources = { + failOnError: true, + ...(candidate.resources.files + ? { + files: candidate.resources.files.map((file) => ({ + ...file, + resource: publicResource(file.resource), + })), + } + : {}), + ...(candidate.resources.tools + ? { tools: candidate.resources.tools.map(publicResource) } + : {}), + ...(candidate.resources.skills + ? { skills: candidate.resources.skills.map(publicResource) } + : {}), + ...(candidate.resources.agents + ? { agents: candidate.resources.agents.map(publicResource) } + : {}), + ...(candidate.resources.commands + ? { commands: candidate.resources.commands.map(publicResource) } + : {}), + ...(candidate.resources.instructions !== undefined + ? { + instructions: + typeof candidate.resources.instructions === 'string' + ? candidate.resources.instructions + : publicResource(candidate.resources.instructions), + } + : {}), + } + } + return output as AgentProfile +} + +function omitUndefinedObjectFields(value: unknown, path: string): unknown { + if (Array.isArray(value)) { + return value.map((entry, index) => { + if (entry === undefined) { + throw new Error(`${path} produced an undefined array entry at ${index}`) + } + return omitUndefinedObjectFields(entry, `${path}[${index}]`) + }) + } + if (value === null || typeof value !== 'object') return value + return Object.fromEntries( + Object.entries(value as Record) + .filter(([, entry]) => entry !== undefined) + .map(([key, entry]) => [key, omitUndefinedObjectFields(entry, `${path}.${key}`)]), + ) +} + +function freezeMcpServers(servers: Record): Record { + return Object.fromEntries( + Object.entries(servers).map(([name, server]) => { + if (server.transport !== undefined && server.transport !== 'stdio') { + unsupportedProfileField(`mcp.${name}.transport=${server.transport}`) + } + if (server.url !== undefined) unsupportedProfileField(`mcp.${name}.url`) + if (server.headers !== undefined) unsupportedProfileField(`mcp.${name}.headers`) + if (server.metadata !== undefined) unsupportedProfileField(`mcp.${name}.metadata`) + return [ + name, + { + ...(server.transport ? { transport: server.transport } : {}), + ...(server.command ? { command: server.command } : {}), + ...(server.args ? { args: server.args.map(candidatePublicValue) } : {}), + ...(server.env ? { env: mapCandidatePublicValues(server.env) } : {}), + ...(server.cwd ? { cwd: server.cwd } : {}), + ...(server.enabled === undefined ? {} : { enabled: server.enabled }), + }, + ] + }), + ) +} + +function freezeResources(resources: NonNullable): unknown { + if (resources.failOnError !== true) { + throw new Error('candidate profile resources require failOnError: true') + } + return { + failOnError: true, + ...(resources.files + ? { + files: resources.files.map((file) => ({ + ...file, + resource: freezeResource(file.resource), + })), + } + : {}), + ...(resources.tools ? { tools: resources.tools.map(freezeResource) } : {}), + ...(resources.skills ? { skills: resources.skills.map(freezeResource) } : {}), + ...(resources.agents ? { agents: resources.agents.map(freezeResource) } : {}), + ...(resources.commands ? { commands: resources.commands.map(freezeResource) } : {}), + ...(resources.instructions === undefined + ? {} + : { + instructions: + typeof resources.instructions === 'string' + ? resources.instructions + : freezeResource(resources.instructions), + }), + } +} + +function freezeResource(resource: AgentProfileResourceRef): AgentCandidateResourceRef { + if (resource.kind === 'github') { + throw new Error( + 'generic GitHub profile resources do not carry byte identity; use a candidate-profile source with a pinned commit, digest, and byte length', + ) + } + const bytes = Buffer.from(resource.content, 'utf8') + return { + ...resource, + sha256: embeddedCandidateArtifact(bytes).sha256, + byteLength: bytes.byteLength, + } +} + +function copyDirectProfileFields( + target: Record, + source: Record, +): void { + for (const key of CANDIDATE_PROFILE_DIRECT_FIELDS) { + if (source[key] !== undefined) target[key] = source[key] + } +} + +function assertCanonicalParse(input: unknown, parsed: unknown, label: string): void { + if (!Buffer.from(canonicalCandidateBytes(input)).equals(canonicalCandidateBytes(parsed))) { + throw new Error(`${label} contains unsupported or non-canonical fields`) + } +} + +function publicResource(resource: AgentCandidateResourceRef): unknown { + if (resource.kind === 'inline') { + return { kind: 'inline', name: resource.name, content: resource.content } + } + return { + kind: 'github', + repository: `${resource.repository.owner}/${resource.repository.repo}`, + path: resource.path, + ref: resource.commit, + ...(resource.name ? { name: resource.name } : {}), + } +} + +function candidatePublicValue(value: string): { kind: 'public'; value: string } { + return { kind: 'public', value } +} + +function mapCandidatePublicValues( + values: Record, +): Record { + return Object.fromEntries( + Object.entries(values).map(([name, value]) => [name, candidatePublicValue(value)]), + ) +} + +function publicValue(value: AgentCandidateConfigValue): string { + return value.value +} + +function mapPublicValues( + values: Record, +): Record { + return Object.fromEntries(Object.entries(values).map(([key, value]) => [key, publicValue(value)])) +} + +function cloneRecord(values: Record): Record { + return Object.fromEntries( + Object.entries(values).map(([key, value]) => [key, { ...value }]), + ) as Record +} + +function hasEntries(value: unknown): boolean { + return value !== undefined && value !== null && typeof value === 'object' + ? Object.keys(value).length > 0 + : false +} + +function shellQuote(value: string): string { + return /^[A-Za-z0-9_./:@%+=,-]+$/.test(value) ? value : `'${value.replaceAll("'", `'"'"'`)}'` +} + +function unsupportedProfileField(path: string): never { + throw new Error( + `generic AgentProfile field ${path} is not representable in a sealed candidate profile`, + ) +} diff --git a/src/candidate-execution/protected-model-port.ts b/src/candidate-execution/protected-model-port.ts new file mode 100644 index 00000000..e69194e1 --- /dev/null +++ b/src/candidate-execution/protected-model-port.ts @@ -0,0 +1,504 @@ +import type { + AgentCandidateModelAccessNetwork, + AgentCandidateResolvedModel, + Sha256Digest, +} from '@tangle-network/agent-interface' + +import { canonicalCandidateDigest, immutableCandidateValue } from './digest' +import { assertExactObjectKeys } from './exact-object' +import { sealAgentCandidateModelSettlement, usdToNanos } from './model-settlement' +import type { + AgentCandidateModelLimits, + AgentCandidateModelPort, + AgentCandidateProtectedModelActivation, + AgentCandidateProtectedModelReservation, + AgentCandidateProtectedModelSettlement, +} from './types' + +export type AgentCandidateModelGrantReserveInput = Parameters< + AgentCandidateModelPort['reserveGrant'] +>[0] +export type AgentCandidateModelGrantActivateInput = Parameters< + AgentCandidateModelPort['activateGrant'] +>[0] +export type AgentCandidateModelGrantSettleInput = Parameters< + AgentCandidateModelPort['settleGrant'] +>[0] + +/** Secret-free response from the service's reservation endpoint. */ +export type AgentCandidateModelGrantReservation = AgentCandidateProtectedModelReservation + +/** + * Narrow transport contract for a service that owns scoped model credentials + * and the authoritative per-call usage ledger. + * + * An HTTP client can bind these methods to control-plane endpoints. Keeping + * transport out of the runtime prevents parent credentials, endpoint paths, + * and retry policy from becoming part of the portable candidate contract. + */ +export interface AgentCandidateModelGrantClient { + reserve(input: AgentCandidateModelGrantReserveInput): Promise + activate( + input: AgentCandidateModelGrantActivateInput, + ): Promise + settle( + input: AgentCandidateModelGrantSettleInput, + ): Promise +} + +export interface CreateProtectedAgentCandidateModelPortOptions { + client: AgentCandidateModelGrantClient + /** Catalog/snapshot resolution stays separate from credential issuance. */ + resolveModel: AgentCandidateModelPort['resolve'] + /** The only public DNS name candidate processes may reach for inference. */ + gatewayDomain: string + /** Exact environment names the activation endpoint must return, no more or fewer. */ + activationEnvNames: readonly string[] +} + +interface RememberedReservation { + requestDigest: Sha256Digest + responseDigest: Sha256Digest + executionId: string + preparationId: string + grantDigest: Sha256Digest + expiresAtMs: number + resolved: AgentCandidateResolvedModel + limits: AgentCandidateModelLimits + maxCostUsdNanos: number + activation: 'reserved' | 'activating' | 'activated' + settlementDigest?: Sha256Digest + settlementReason?: AgentCandidateModelGrantSettleInput['reason'] +} + +interface RememberedSettlement { + settlementDigest: Sha256Digest + settlementReason: AgentCandidateModelGrantSettleInput['reason'] + expiresAtMs: number +} + +const MAX_RECENT_SETTLEMENTS = 4_096 +const RECOVERED_SETTLEMENT_RETENTION_MS = 15 * 60 * 1_000 + +/** + * Bind a protected model-grant service to the immutable candidate runtime. + * + * The service remains the authority for expiry, admission, revocation, and + * metering. This adapter independently checks every response before allowing + * it to cross into candidate execution or durable receipt finalization. + */ +export function createProtectedAgentCandidateModelPort( + options: CreateProtectedAgentCandidateModelPortOptions, +): AgentCandidateModelPort { + const gatewayDomain = assertGatewayDomain(options.gatewayDomain) + const activationEnvNames = exactEnvironmentNames(options.activationEnvNames) + const reservations = new Map() + const recentSettlements = new Map() + + return { + resolve: async (input) => { + const request = immutableCandidateValue(input) + const resolved = validateResolvedModel(await options.resolveModel(request), request) + return immutableCandidateValue(resolved) + }, + + reserveGrant: async (input) => { + pruneExpiredState(reservations, recentSettlements, Date.now()) + const request = immutableCandidateValue(input) + validateReserveInput(request) + const requestDigest = canonicalCandidateDigest(request) + const key = reservationKey(request.executionId, request.preparationId) + if (recentSettlements.has(key)) { + throw new Error('protected model reservation is already settled') + } + const previous = reservations.get(key) + if (previous && previous.requestDigest !== requestDigest) { + throw new Error('protected model reservation retry changed immutable input') + } + + const reservation = validateReservation( + await options.client.reserve(request), + request, + gatewayDomain, + ) + const responseDigest = canonicalCandidateDigest(reservation) + if (recentSettlements.has(key)) { + throw new Error('protected model reservation completed after the grant was settled') + } + const recorded = previous ?? reservations.get(key) + if (recorded && recorded.requestDigest !== requestDigest) { + throw new Error('protected model reservation retry changed immutable input') + } + if (recorded && recorded.responseDigest !== responseDigest) { + throw new Error('protected model reservation retry returned different evidence') + } + if (!recorded) { + reservations.set(key, { + requestDigest, + responseDigest, + executionId: request.executionId, + preparationId: request.preparationId, + grantDigest: reservation.digest, + expiresAtMs: request.expiresAtMs, + resolved: immutableCandidateValue(request.resolved), + limits: immutableCandidateValue(request.limits), + maxCostUsdNanos: usdToNanos(request.limits.maxCostUsd, 'reserved maxCostUsd'), + activation: 'reserved', + }) + } + return reservation + }, + + activateGrant: async (input) => { + pruneExpiredState(reservations, recentSettlements, Date.now()) + const request = immutableCandidateValue(input) + const key = reservationKey(request.executionId, request.preparationId) + if (recentSettlements.has(key)) throw new Error('protected model grant is already settled') + const state = reservations.get(key) + if (!state) throw new Error('protected model grant was not reserved by this port') + assertGrantIdentity(state, request) + if (state.settlementDigest) throw new Error('protected model grant is already settled') + if (state.activation !== 'reserved') { + throw new Error('protected model grant activation is single-use') + } + if (!Number.isSafeInteger(request.deadlineAtMs) || request.deadlineAtMs <= 0) { + throw new Error('protected model activation deadline must be a positive safe integer') + } + if (request.deadlineAtMs <= Date.now()) { + throw new Error('protected model activation deadline must be in the future') + } + if (request.deadlineAtMs > state.expiresAtMs) { + throw new Error('protected model activation deadline exceeds the reserved expiry') + } + state.activation = 'activating' + try { + const activation = validateActivation( + await options.client.activate(request), + state.limits.maxModelCalls === 0 ? [] : activationEnvNames, + ) + if (recentSettlements.has(key) || reservations.get(key) !== state) { + throw new Error('protected model grant settled while activation was in flight') + } + state.activation = 'activated' + return activation + } catch (error) { + state.activation = 'reserved' + throw error + } + }, + + settleGrant: async (input) => { + const now = Date.now() + pruneExpiredState(reservations, recentSettlements, now) + const request = immutableCandidateValue(input) + const key = reservationKey(request.executionId, request.preparationId) + const state = reservations.get(key) + if (state) assertGrantIdentity(state, request) + const remembered = state ?? recentSettlements.get(key) + if (remembered?.settlementReason && remembered.settlementReason !== request.reason) { + throw new Error('protected model settlement retry changed the termination reason') + } + + const sealed = sealAgentCandidateModelSettlement(await options.client.settle(request), { + preparationId: request.preparationId, + grantDigest: request.grantDigest, + model: request.resolved.model, + }) + if (state) assertWithinReservedLimits(sealed.fixedUsage, state) + + const settlementDigest = canonicalCandidateDigest(sealed.value) + if (remembered?.settlementDigest && remembered.settlementDigest !== settlementDigest) { + throw new Error('protected model settlement retry returned a different final ledger') + } + if (remembered?.settlementReason && remembered.settlementReason !== request.reason) { + throw new Error('protected model settlement retry changed the termination reason') + } + const expiresAtMs = state?.expiresAtMs ?? now + RECOVERED_SETTLEMENT_RETENTION_MS + if (state) { + state.settlementDigest = settlementDigest + state.settlementReason = request.reason + } + reservations.delete(key) + rememberSettlement(recentSettlements, key, { + settlementDigest, + settlementReason: request.reason, + expiresAtMs, + }) + return sealed.value + }, + } +} + +function validateReserveInput(input: AgentCandidateModelGrantReserveInput): void { + if (!Number.isSafeInteger(input.expiresAtMs) || input.expiresAtMs <= 0) { + throw new Error('protected model reservation expiry must be a positive safe integer') + } + if (input.expiresAtMs <= Date.now()) { + throw new Error('protected model reservation expiry must be in the future') + } + for (const [name, value] of [ + ['maxModelCalls', input.limits.maxModelCalls], + ['maxInputTokens', input.limits.maxInputTokens], + ['maxOutputTokens', input.limits.maxOutputTokens], + ] as const) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`protected model reservation ${name} must be a nonnegative safe integer`) + } + } + usdToNanos(input.limits.maxCostUsd, 'reserved maxCostUsd') +} + +function validateReservation( + value: unknown, + expected: AgentCandidateModelGrantReserveInput, + gatewayDomain: string, +): AgentCandidateModelGrantReservation { + const source = exactRecord( + value, + ['preparationId', 'digest', 'expiresAtMs', 'enforcedLimits', 'network'], + 'protected model reservation response', + ) + if (source.preparationId !== expected.preparationId) { + throw new Error('protected model reservation response changed preparation identity') + } + if (!isSha256Digest(source.digest)) { + throw new Error('protected model reservation response has an invalid digest') + } + if (source.expiresAtMs !== expected.expiresAtMs) { + throw new Error('protected model reservation response changed expiry') + } + exactRecord( + source.enforcedLimits, + ['maxModelCalls', 'maxInputTokens', 'maxOutputTokens', 'maxCostUsd'], + 'protected model reservation enforced limits', + ) + if ( + canonicalCandidateDigest(source.enforcedLimits) !== canonicalCandidateDigest(expected.limits) + ) { + throw new Error('protected model reservation response changed enforced limits') + } + + const network = validateReservationNetwork( + source.network, + expected.limits.maxModelCalls, + gatewayDomain, + ) + return immutableCandidateValue({ + preparationId: source.preparationId, + digest: source.digest, + expiresAtMs: source.expiresAtMs, + enforcedLimits: source.enforcedLimits, + network, + }) as AgentCandidateModelGrantReservation +} + +function validateReservationNetwork( + value: unknown, + maxModelCalls: number, + gatewayDomain: string, +): AgentCandidateModelAccessNetwork { + if (maxModelCalls === 0) { + const source = exactRecord(value, ['mode'], 'protected model reservation network') + if (source.mode !== 'disabled') { + throw new Error('zero-call protected model reservation must disable gateway access') + } + return { mode: 'disabled' } + } + + const source = exactRecord(value, ['mode', 'domains'], 'protected model reservation network') + if (source.mode !== 'gateway-only') { + throw new Error('protected model reservation must allow only its model gateway') + } + if ( + !Array.isArray(source.domains) || + source.domains.length !== 1 || + source.domains[0] !== gatewayDomain + ) { + throw new Error('protected model reservation returned an unexpected gateway domain') + } + return { mode: 'gateway-only', domains: [gatewayDomain] } +} + +function validateActivation( + value: unknown, + expectedNames: readonly string[], +): AgentCandidateProtectedModelActivation { + const source = exactRecord(value, ['env'], 'protected model activation response') + const env = record(source.env, 'protected model activation environment') + const actualNames = Object.keys(env).sort() + if ( + actualNames.length !== expectedNames.length || + actualNames.some((name, index) => name !== expectedNames[index]) + ) { + throw new Error('protected model activation returned unexpected environment names') + } + const detached: Record = Object.create(null) as Record + for (const name of expectedNames) { + const value = env[name] + if (typeof value !== 'string' || value.length < 8 || value.length > 65_536) { + throw new Error(`protected model activation ${name} must be a non-empty protected value`) + } + detached[name] = value + } + return Object.freeze({ env: Object.freeze(detached) }) +} + +function validateResolvedModel( + value: unknown, + expected: Parameters[0], +): AgentCandidateResolvedModel { + const source = exactRecord( + value, + ['requested', 'provider', 'model', 'snapshot', 'reasoningEffort'], + 'resolved candidate model', + ) + if (source.requested !== expected.requested) { + throw new Error('model resolver changed the evaluator-owned request') + } + if (source.reasoningEffort !== expected.reasoningEffort) { + throw new Error('model resolver changed the evaluator-owned reasoning effort') + } + for (const name of ['requested', 'provider', 'model', 'snapshot'] as const) { + if (typeof source[name] !== 'string' || source[name].length === 0) { + throw new Error(`resolved candidate model ${name} must be non-empty`) + } + } + return source as unknown as AgentCandidateResolvedModel +} + +function assertGrantIdentity( + expected: RememberedReservation, + actual: AgentCandidateModelGrantActivateInput | AgentCandidateModelGrantSettleInput, +): void { + if ( + actual.executionId !== expected.executionId || + actual.preparationId !== expected.preparationId || + actual.grantDigest !== expected.grantDigest || + canonicalCandidateDigest(actual.resolved) !== canonicalCandidateDigest(expected.resolved) + ) { + throw new Error('protected model grant input does not match its immutable reservation') + } +} + +function assertWithinReservedLimits( + usage: { + modelCalls: number + inputTokens: number + outputTokens: number + costUsdNanos: number + }, + reservation: RememberedReservation, +): void { + for (const [name, actual, limit] of [ + ['model calls', usage.modelCalls, reservation.limits.maxModelCalls], + ['input tokens', usage.inputTokens, reservation.limits.maxInputTokens], + ['output tokens', usage.outputTokens, reservation.limits.maxOutputTokens], + ['cost USD nanos', usage.costUsdNanos, reservation.maxCostUsdNanos], + ] as const) { + if (actual > limit) { + throw new Error(`protected model settlement ${name} ${actual} exceeds reserved ${limit}`) + } + } +} + +function exactEnvironmentNames(values: readonly string[]): readonly string[] { + if (values.length === 0) { + throw new Error('protected model activation environment names must not be empty') + } + const names = [...values].sort() + if (new Set(names).size !== names.length) { + throw new Error('protected model activation environment names must be unique') + } + for (const name of names) { + if ( + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || + ['__proto__', 'constructor', 'prototype'].includes(name) + ) { + throw new Error(`protected model activation environment name is invalid: ${name}`) + } + } + return Object.freeze(names) +} + +function assertGatewayDomain(value: string): string { + if ( + value.length > 253 || + value !== value.toLowerCase() || + value.endsWith('.') || + value.includes(':') || + value === 'localhost' || + value.endsWith('.localhost') || + value === 'metadata.google' || + value === 'metadata.google.internal' + ) { + throw new Error('protected model gateway must be an exact lowercase public DNS name') + } + const labels = value.split('.') + if ( + labels.length < 2 || + !labels.every( + (label) => + label.length >= 1 && label.length <= 63 && /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(label), + ) || + !/[a-z]/.test(labels.at(-1) ?? '') + ) { + throw new Error('protected model gateway must be an exact lowercase public DNS name') + } + return value +} + +function reservationKey(executionId: string, preparationId: string): string { + return canonicalCandidateDigest({ executionId, preparationId }) +} + +function pruneExpiredState( + reservations: Map, + settlements: Map, + now: number, +): void { + for (const [key, value] of reservations) { + if (value.expiresAtMs <= now) reservations.delete(key) + } + for (const [key, value] of settlements) { + if (value.expiresAtMs <= now) settlements.delete(key) + } +} + +function rememberSettlement( + settlements: Map, + key: string, + value: RememberedSettlement, +): void { + settlements.delete(key) + settlements.set(key, value) + while (settlements.size > MAX_RECENT_SETTLEMENTS) { + const oldest = settlements.keys().next().value + if (oldest === undefined) return + settlements.delete(oldest) + } +} + +function isSha256Digest(value: unknown): value is Sha256Digest { + return typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value) +} + +function exactRecord( + value: unknown, + keys: readonly string[], + label: string, +): Record { + const source = record(value, label) + assertExactObjectKeys(source, keys, label) + return source +} + +function record(value: unknown, label: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) { + throw new Error(`${label} must be a plain object`) + } + return value as Record +} diff --git a/src/candidate-execution/protected-redaction.ts b/src/candidate-execution/protected-redaction.ts new file mode 100644 index 00000000..7392e909 --- /dev/null +++ b/src/candidate-execution/protected-redaction.ts @@ -0,0 +1,176 @@ +import { + DEFAULT_REDACTION_RULES, + REDACTION_VERSION, + type RedactionRule, + redactString, +} from '@tangle-network/agent-eval' + +export interface ProtectedRedactionReport { + version: string + redactionCount: number + byRule: Record +} + +/** Redact protected values at the first persistence boundary, including object keys and bytes. */ +export function redactProtectedValue( + value: T, + protectedValues: readonly string[], +): { value: T; report: ProtectedRedactionReport } { + const report: ProtectedRedactionReport = { + version: REDACTION_VERSION, + redactionCount: 0, + byRule: {}, + } + const rules = protectedRedactionRules(protectedValues) + const redacted = redactNode(value, protectedValues, rules, report) as T + assertNoProtectedEvidence(redacted, protectedValues) + return { value: redacted, report } +} + +export function redactProtectedReason(reason: string, protectedValues: readonly string[]): string { + try { + return redactProtectedValue(reason, protectedValues).value + } catch { + return 'candidate execution failed with a protected error' + } +} + +export function assertNoProtectedEvidence( + value: unknown, + protectedValues: readonly string[], +): void { + if (value instanceof Uint8Array) { + assertNoProtectedBytes(value, protectedValues) + return + } + if (typeof value === 'string') { + for (const protectedValue of protectedValueVariants(protectedValues)) { + if (value.includes(protectedValue)) { + throw new Error('protected value survived candidate evidence redaction') + } + } + if (decodedBase64ContainsProtectedValue(value, protectedValues)) { + throw new Error('base64 protected value survived candidate evidence redaction') + } + return + } + if (Array.isArray(value)) { + for (const entry of value) assertNoProtectedEvidence(entry, protectedValues) + return + } + if (value && typeof value === 'object') { + for (const [key, entry] of Object.entries(value)) { + assertNoProtectedEvidence(key, protectedValues) + assertNoProtectedEvidence(entry, protectedValues) + } + } +} + +export function assertNoProtectedBytes( + bytes: Uint8Array, + protectedValues: readonly string[], +): void { + if (containsProtectedBytes(bytes, protectedValues)) { + throw new Error('protected value survived candidate evidence byte redaction') + } +} + +function redactNode( + value: unknown, + protectedValues: readonly string[], + rules: readonly RedactionRule[], + report: ProtectedRedactionReport, +): unknown { + if (value instanceof Uint8Array) { + if (containsProtectedBytes(value, protectedValues)) { + recordRedaction(report, 'candidate-access-binary', 1) + return Uint8Array.from(Buffer.from('[redacted:candidate-access-binary]', 'utf8')) + } + return Uint8Array.from(value) + } + if (typeof value === 'string') { + if (decodedBase64ContainsProtectedValue(value, protectedValues)) { + recordRedaction(report, 'candidate-access-binary', 1) + return '[redacted:candidate-access-binary]' + } + const redacted = redactString(value, [...rules]) + for (const [rule, count] of Object.entries(redacted.report.byRule)) { + recordRedaction(report, rule, count) + } + return redacted.output + } + if (Array.isArray(value)) { + return value.map((entry) => redactNode(entry, protectedValues, rules, report)) + } + if (value && typeof value === 'object') { + const entries: Array<[string, unknown]> = [] + const seenKeys = new Set() + for (const [key, entry] of Object.entries(value)) { + const redactedKey = redactNode(key, protectedValues, rules, report) + if (typeof redactedKey !== 'string' || seenKeys.has(redactedKey)) { + throw new Error('protected evidence redaction produced an ambiguous object key') + } + seenKeys.add(redactedKey) + entries.push([redactedKey, redactNode(entry, protectedValues, rules, report)]) + } + return Object.fromEntries(entries) + } + return value +} + +function protectedRedactionRules(protectedValues: readonly string[]): RedactionRule[] { + const exactRules = protectedValueVariants(protectedValues) + .sort((left, right) => right.length - left.length || left.localeCompare(right)) + .map((value, index) => ({ + id: `candidate-access-${index}`, + pattern: new RegExp(escapeRegularExpression(value), 'g'), + replacement: '[redacted:candidate-access]', + })) + return [...exactRules, ...DEFAULT_REDACTION_RULES] +} + +function recordRedaction(report: ProtectedRedactionReport, rule: string, count: number): void { + if (count <= 0) return + report.redactionCount += count + report.byRule[rule] = (report.byRule[rule] ?? 0) + count +} + +function containsProtectedBytes(bytes: Uint8Array, protectedValues: readonly string[]): boolean { + const source = Buffer.from(bytes) + return protectedValueVariants(protectedValues).some((value) => + source.includes(Buffer.from(value, 'utf8')), + ) +} + +function decodedBase64ContainsProtectedValue( + value: string, + protectedValues: readonly string[], +): boolean { + if (value.length < 4 || value.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(value)) { + return false + } + try { + return containsProtectedBytes(Buffer.from(value, 'base64'), protectedValues) + } catch { + return false + } +} + +function protectedValueVariants(protectedValues: readonly string[]): string[] { + const variants = new Set() + for (const value of normalizedProtectedValues(protectedValues)) { + variants.add(value) + variants.add(Buffer.from(value, 'utf8').toString('base64')) + variants.add(Buffer.from(value, 'utf8').toString('base64url')) + variants.add(encodeURIComponent(value)) + } + return [...variants] +} + +function normalizedProtectedValues(values: readonly string[]): string[] { + return [...new Set(values.filter((value) => value.length > 0))] +} + +function escapeRegularExpression(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} diff --git a/src/candidate-execution/protected-trace-store.ts b/src/candidate-execution/protected-trace-store.ts new file mode 100644 index 00000000..1026a65f --- /dev/null +++ b/src/candidate-execution/protected-trace-store.ts @@ -0,0 +1,160 @@ +import { REDACTION_VERSION, type TraceStore } from '@tangle-network/agent-eval' + +import { type ProtectedRedactionReport, redactProtectedValue } from './protected-redaction' + +/** Trace-store proxy that removes live credentials before any write reaches durable storage. */ +export class ProtectedAgentCandidateTraceStore implements TraceStore { + private readonly aggregate: ProtectedRedactionReport = { + version: REDACTION_VERSION, + redactionCount: 0, + byRule: {}, + } + + constructor( + private readonly inner: TraceStore, + private readonly protectedValues: readonly string[], + ) {} + + report(): ProtectedRedactionReport { + return { + version: this.aggregate.version, + redactionCount: this.aggregate.redactionCount, + byRule: { ...this.aggregate.byRule }, + } + } + + async appendRun(run: Parameters[0]): Promise { + await this.inner.appendRun(this.redact(run)) + } + + async updateRun( + runId: Parameters[0], + patch: Parameters[1], + ): Promise { + await this.inner.updateRun(this.redact(runId), this.redact(patch)) + } + + async appendSpan(span: Parameters[0]): Promise { + if (span.kind === 'llm') { + throw new Error('candidate executors cannot author protected model spans') + } + await this.inner.appendSpan(this.redact(span)) + } + + async updateSpan( + spanId: Parameters[0], + patch: Parameters[1], + ): Promise { + if (patch.kind === 'llm') { + throw new Error('candidate executors cannot author protected model spans') + } + await this.inner.updateSpan(this.redact(spanId), this.redact(patch)) + } + + async appendEvent(event: Parameters[0]): Promise { + await this.inner.appendEvent(this.redact(event)) + } + + async appendArtifact(artifact: Parameters[0]): Promise { + await this.inner.appendArtifact(this.redact(artifact)) + } + + async appendBudgetEntry(entry: Parameters[0]): Promise { + await this.inner.appendBudgetEntry(this.redact(entry)) + } + + getRun(...args: Parameters): ReturnType { + return this.inner.getRun(...args) + } + + listRuns(...args: Parameters): ReturnType { + return this.inner.listRuns(...args) + } + + spans(...args: Parameters): ReturnType { + return this.inner.spans(...args) + } + + events(...args: Parameters): ReturnType { + return this.inner.events(...args) + } + + budget(...args: Parameters): ReturnType { + return this.inner.budget(...args) + } + + artifacts(...args: Parameters): ReturnType { + return this.inner.artifacts(...args) + } + + private redact(value: T): T { + const redacted = redactProtectedValue(value, this.protectedValues) + this.aggregate.version = redacted.report.version + this.aggregate.redactionCount += redacted.report.redactionCount + for (const [rule, count] of Object.entries(redacted.report.byRule)) { + this.aggregate.byRule[rule] = (this.aggregate.byRule[rule] ?? 0) + count + } + return redacted.value + } +} + +/** Recovery can read existing trace state but must never accept new unredactable writes. */ +export class RecoveryAgentCandidateTraceStore implements TraceStore { + constructor(private readonly inner: TraceStore) {} + + appendRun(): Promise { + return this.rejectWrite() + } + + updateRun(): Promise { + return this.rejectWrite() + } + + appendSpan(): Promise { + return this.rejectWrite() + } + + updateSpan(): Promise { + return this.rejectWrite() + } + + appendEvent(): Promise { + return this.rejectWrite() + } + + appendArtifact(): Promise { + return this.rejectWrite() + } + + appendBudgetEntry(): Promise { + return this.rejectWrite() + } + + getRun(...args: Parameters): ReturnType { + return this.inner.getRun(...args) + } + + listRuns(...args: Parameters): ReturnType { + return this.inner.listRuns(...args) + } + + spans(...args: Parameters): ReturnType { + return this.inner.spans(...args) + } + + events(...args: Parameters): ReturnType { + return this.inner.events(...args) + } + + budget(...args: Parameters): ReturnType { + return this.inner.budget(...args) + } + + artifacts(...args: Parameters): ReturnType { + return this.inner.artifacts(...args) + } + + private rejectWrite(): Promise { + return Promise.reject(new Error('expired candidate recovery cannot append trace evidence')) + } +} diff --git a/src/candidate-execution/recover.ts b/src/candidate-execution/recover.ts new file mode 100644 index 00000000..cff11369 --- /dev/null +++ b/src/candidate-execution/recover.ts @@ -0,0 +1,175 @@ +import type { TraceStore } from '@tangle-network/agent-eval' + +import type { + AgentCandidateExecutionAttemptRef, + AgentCandidateExecutionClaimStore, + AgentCandidateExecutionFinishResult, +} from './claim' +import { + candidateCleanupDeadline, + candidateCleanupTimeout, + withinCandidateCleanupDeadline, +} from './cleanup' +import { sealAgentCandidateExecutorFinalCapture } from './executor-capture' +import { sealAgentCandidateModelSettlement } from './model-settlement' +import { persistCandidateModelSettlementEvidence } from './outcome-evidence' +import { RecoveryAgentCandidateTraceStore } from './protected-trace-store' +import type { + AgentCandidateExecutionPorts, + AgentCandidateExecutorPort, + AgentCandidateOutputArtifactPort, +} from './types' + +export interface RecoverExpiredAgentCandidateOptions { + attempt: AgentCandidateExecutionAttemptRef + claimStore: AgentCandidateExecutionClaimStore + executor: AgentCandidateExecutorPort + traceStore: TraceStore + ports: Pick + outputArtifacts: AgentCandidateOutputArtifactPort + cleanupTimeoutMs?: number + /** Evaluator clock; must be the same clock used by the claim store. */ + now?: () => number +} + +/** Close an expired crashed attempt from persisted non-secret handles, then record failure. */ +export async function recoverExpiredAgentCandidateExecution( + options: RecoverExpiredAgentCandidateOptions, +): Promise { + const record = await options.claimStore.getAttempt(options.attempt) + if (!record) throw new Error('candidate execution recovery attempt is missing') + if (record.terminal) { + return Object.freeze({ finished: false, terminal: record.terminal, exactReplay: true }) + } + const cleanupTimeoutMs = candidateCleanupTimeout( + options.cleanupTimeoutMs ?? record.claim.cleanup.cleanupTimeoutMs, + ) + if (cleanupTimeoutMs > record.claim.cleanup.cleanupTimeoutMs) { + throw new Error('recovery cleanup timeout exceeds the frozen preparation bound') + } + + const now = options.now ?? Date.now + if (now() < record.claim.leaseExpiresAtMs) { + throw new Error('candidate execution lease has not expired') + } + + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + const controller = new AbortController() + controller.abort(new Error('recovering an expired candidate execution')) + const recoveryTraceStore = new RecoveryAgentCandidateTraceStore(options.traceStore) + const processClosure = withinCandidateCleanupDeadline( + async () => { + const stopped = await options.executor.stopAndCapture( + { + executionId: record.claim.executionId, + executionPlanDigest: record.claim.executionPlanDigest, + }, + { + traceStore: recoveryTraceStore, + reason: 'failed', + signal: controller.signal, + deadlineAtMs: record.claim.leaseExpiresAtMs, + }, + ) + sealAgentCandidateExecutorFinalCapture(stopped) + return { stopped: true as const } + }, + cleanupDeadlineAtMs, + 'expired candidate process termination', + ) + + const modelClosure = withinCandidateCleanupDeadline( + async () => + sealAgentCandidateModelSettlement( + await options.ports.models.settleGrant({ + executionId: record.claim.executionId, + preparationId: record.claim.cleanup.preparationId, + grantDigest: record.claim.cleanup.modelGrantDigest, + resolved: record.claim.cleanup.resolvedModel, + reason: 'failed', + }), + { + preparationId: record.claim.cleanup.preparationId, + grantDigest: record.claim.cleanup.modelGrantDigest, + model: record.claim.cleanup.resolvedModel.model, + }, + ), + cleanupDeadlineAtMs, + 'expired candidate model settlement', + ) + + const memoryClosure = record.claim.cleanup.memory + ? withinCandidateCleanupDeadline( + async () => { + const memory = record.claim.cleanup.memory + if (!memory) throw new Error('expired candidate memory handle is missing') + const closed = await options.ports.memory.close({ + executionId: record.claim.executionId, + preparationId: record.claim.cleanup.preparationId, + accessDigest: memory.accessDigest, + effectiveNamespace: memory.effectiveNamespace, + reason: 'failed', + }) + if (closed.closed !== true || Object.keys(closed).some((key) => key !== 'closed')) { + throw new Error('expired candidate memory did not acknowledge closure') + } + return { closed: true as const } + }, + cleanupDeadlineAtMs, + 'expired candidate memory closure', + ) + : undefined + + const operations = [processClosure, modelClosure, ...(memoryClosure ? [memoryClosure] : [])] + const outcomes = await Promise.allSettled(operations) + const failures = outcomes + .filter((outcome): outcome is PromiseRejectedResult => outcome.status === 'rejected') + .map((outcome) => outcome.reason) + if (failures.length > 0) { + throw new AggregateError(failures, 'expired candidate cleanup could not be proven') + } + + const model = await modelClosure + const modelSettlement = await withinCandidateCleanupDeadline( + () => + persistCandidateModelSettlementEvidence( + { + executionId: record.claim.executionId, + executionPlanDigest: record.claim.executionPlanDigest, + resolvedModel: record.claim.cleanup.resolvedModel, + }, + model, + options.outputArtifacts, + ), + cleanupDeadlineAtMs, + 'expired candidate model-settlement persistence', + ) + const memory = record.claim.cleanup.memory + return await options.claimStore.recoverExpired(options.attempt, { + failureClass: + record.phase === 'claimed' && model.fixedUsage.modelCalls === 0 + ? 'pre-model-infrastructure' + : 'unknown', + usage: model.fixedUsage, + modelSettlement: modelSettlement.artifact, + process: { + stopped: true, + executionPlanDigest: record.claim.executionPlanDigest, + }, + model: { + closed: true, + preparationId: record.claim.cleanup.preparationId, + grantDigest: record.claim.cleanup.modelGrantDigest, + }, + ...(memory + ? { + memory: { + closed: true, + preparationId: record.claim.cleanup.preparationId, + accessDigest: memory.accessDigest, + effectiveNamespace: memory.effectiveNamespace, + }, + } + : {}), + }) +} diff --git a/src/candidate-execution/types.ts b/src/candidate-execution/types.ts new file mode 100644 index 00000000..097bc62b --- /dev/null +++ b/src/candidate-execution/types.ts @@ -0,0 +1,577 @@ +import type { BenchmarkEvaluation, TraceStore } from '@tangle-network/agent-eval' +import type { + AgentCandidateArtifactRef, + AgentCandidateAttemptPolicy, + AgentCandidateBundle, + AgentCandidateCapturedArtifact, + AgentCandidateContainer, + AgentCandidateEffectiveMemory, + AgentCandidateExecutionLimits, + AgentCandidateExecutionPlanEvidence, + AgentCandidateGitHubRepository, + AgentCandidateInstructionDelivery, + AgentCandidateMaterializationReceipt, + AgentCandidateMemoryReceipt, + AgentCandidateModelAccessNetwork, + AgentCandidateOciPlatform, + AgentCandidateProfilePlanEvidence, + AgentCandidateResolvedModel, + AgentCandidateRunReceiptV2, + AgentCandidateSpend, + AgentCandidateTaskOutcomeEvidence, + AgentCandidateTermination, + AgentCandidateWorkspaceManifestMaterialV1, + AgentCandidateWorkspaceSnapshotEvidence, + ReasoningEffort, + Sha256Digest, +} from '@tangle-network/agent-interface' + +export const verifiedCandidateBrand: unique symbol = Symbol('verifiedAgentCandidate') +export const preparedCandidateBrand: unique symbol = Symbol('preparedAgentCandidate') +export const verifiedTaskOutcomeBrand: unique symbol = Symbol('verifiedTaskOutcome') + +/** Reads one content-addressed object from the closed S3/IPFS locator set. */ +export interface AgentCandidateArtifactPort { + read(ref: AgentCandidateArtifactRef): Promise +} + +export type AgentCandidateOutputPurpose = + | 'candidate-workspace-manifest' + | 'candidate-workspace-archive' + | 'task-manifest' + | 'task-archive' + | 'task-patch' + | 'task-outcome' + | 'memory-after-manifest' + | 'memory-after-archive' + | 'grader-evidence' + | 'benchmark-result' + | 'model-settlement' + | 'trace' + | 'run-receipt' + | 'failure-evidence' + +/** Durable content-addressed evidence store controlled only by the evaluator. */ +export interface AgentCandidateOutputArtifactPort extends AgentCandidateArtifactPort { + /** Must be idempotent for identical bytes and return only a durable S3/IPFS locator. */ + put(input: { + executionId: string + purpose: AgentCandidateOutputPurpose + bytes: Uint8Array + /** Abort must prevent durable publication when it happens before resolution. */ + signal?: AbortSignal + }): Promise +} + +/** Resolves a declared GitHub repository to an already-present local Git object store. */ +export interface AgentCandidateRepositoryPort { + resolve(repository: AgentCandidateGitHubRepository): Promise +} + +export interface AgentCandidateVerificationPorts { + artifacts: AgentCandidateArtifactPort + repositories: AgentCandidateRepositoryPort +} + +/** + * Materializes an already-verified workspace archive. + * + * The runtime independently scans every resulting byte, mode, and path against + * the signed manifest after this returns. Implementations may therefore unpack + * any archive encoding, or no-op when the exact workspace is already present. + */ +export interface AgentCandidateWorkspacePort { + materialize(input: { + role: 'task' | 'candidate' | 'memory' + snapshot: AgentCandidateWorkspaceSnapshotEvidence + archive: Uint8Array + destination: string + }): Promise +} + +export interface ResolvedAgentCandidateContainer { + source: 'pinned-container' | 'evaluator-task-container' + image: string + indexDigest: Sha256Digest + manifestDigest: Sha256Digest + platform: AgentCandidateOciPlatform +} + +export interface AgentCandidateContainerPort { + resolve(input: { + candidate: AgentCandidateContainer | undefined + evaluatorTaskContainer: ResolvedAgentCandidateContainer | undefined + }): Promise +} + +export interface AgentCandidateModelPort { + resolve(input: { + requested: string + harness: AgentCandidateBundle['execution']['harness'] + reasoningEffort: NonNullable['reasoningEffort'] + }): Promise + /** + * Reserve a stable access identity without creating a live credential. + * The reservation is scoped to `preparationId` and must automatically expire + * at `expiresAtMs`, even if this call returns ambiguously to the runtime. + */ + reserveGrant(input: { + executionId: string + preparationId: string + expiresAtMs: number + attempt: AgentCandidateAttemptPolicy + bundleDigest: Sha256Digest + resolved: AgentCandidateResolvedModel + limits: AgentCandidateModelLimits + }): Promise + /** Create the live scoped credential only after the execution attempt is durably claimed. */ + activateGrant(input: { + executionId: string + preparationId: string + grantDigest: Sha256Digest + resolved: AgentCandidateResolvedModel + deadlineAtMs: number + }): Promise + /** + * Atomically revoke the grant, drain in-flight calls, and return its immutable final ledger. + * This operation must be idempotent for the exact preparation and must also + * settle a reservation that was never activated. It must never affect a + * different preparation, even when both reservations report the same digest. + */ + settleGrant(input: { + executionId: string + preparationId: string + grantDigest: Sha256Digest + resolved: AgentCandidateResolvedModel + reason: 'completed' | 'failed' | 'timeout' | 'replayed' | 'preparation-failed' | 'abandoned' + }): Promise +} + +/** Limits mechanically enforced by the evaluator-owned model gateway. */ +export type AgentCandidateModelLimits = Pick< + AgentCandidateExecutionLimits, + 'maxModelCalls' | 'maxInputTokens' | 'maxOutputTokens' | 'maxCostUsd' +> + +export interface AgentCandidateBenchmarkGraderIdentity { + name: string + version: string + artifact: AgentCandidateArtifactRef +} + +export interface AgentCandidateProtectedModelReservation { + preparationId: string + digest: Sha256Digest + /** Evaluator service must expire and revoke this reservation at this epoch millisecond. */ + expiresAtMs: number + /** The gateway must stop calls before any one of these limits is exceeded. */ + enforcedLimits: AgentCandidateModelLimits + /** Exact public endpoint exception; every other candidate destination stays blocked. */ + network: AgentCandidateModelAccessNetwork +} + +export interface AgentCandidateProtectedModelActivation { + /** Injected only into the trusted executor after all pre-launch checks pass. */ + env: Readonly> +} + +/** One evaluator-gateway call in the final, revoked model-access ledger. */ +export interface AgentCandidateProtectedModelCall { + callId: string + /** Router-generated public response identity. */ + generationId: string + /** Exact protected agent-eval LLM span produced from the router ledger. */ + traceSpanId: string + status: 'succeeded' | 'failed' + model: string + startedAtMs: number + endedAtMs: number + inputTokens: number + outputTokens: number + cachedInputTokens: number + reasoningTokens: number + /** Integer billionths of one US dollar; avoids floating-point ledger drift. */ + costUsdNanos: number +} + +export interface AgentCandidateProtectedModelSettlement { + preparationId: string + grantDigest: Sha256Digest + closed: true + calls: readonly AgentCandidateProtectedModelCall[] +} + +export interface AgentCandidateMemoryResetResult { + preparationId: string + accessDigest: Sha256Digest + expiresAtMs: number + evidence: AgentCandidateCapturedArtifact + emptyStateDigest: Sha256Digest + beforeState: AgentCandidateWorkspaceSnapshotEvidence +} + +export interface AgentCandidateMemoryPort { + /** + * Reset and reserve exact task memory without returning live access. + * The service must scope the reservation to `preparationId`, automatically + * revoke it at `expiresAtMs`, and never reuse it for another preparation. + */ + reset(input: { + executionId: string + preparationId: string + expiresAtMs: number + effectiveNamespace: string + seed?: Uint8Array + seedDigest?: Sha256Digest + }): Promise + /** + * Create live scoped access only after the execution attempt is durably claimed. + * Activation must match the exact preparation/access pair and may not extend expiry. + */ + activate(input: { + executionId: string + preparationId: string + accessDigest: Sha256Digest + effectiveNamespace: string + deadlineAtMs: number + }): Promise<{ env: Readonly> }> + /** + * Revoke evaluator-owned access after process death or a failed preparation. + * Must be idempotent and concurrency-safe for the exact preparation/access + * pair and must never close a different preparation. + */ + close(input: { + executionId: string + preparationId: string + accessDigest: Sha256Digest + effectiveNamespace: string + reason: 'completed' | 'failed' | 'timeout' | 'replayed' | 'preparation-failed' | 'abandoned' + }): Promise<{ closed: true }> +} + +export interface AgentCandidateExecutionPorts extends AgentCandidateVerificationPorts { + workspaces: AgentCandidateWorkspacePort + containers: AgentCandidateContainerPort + models: AgentCandidateModelPort + memory: AgentCandidateMemoryPort +} + +export interface AgentCandidateTaskExecution { + executionId: string + benchmark: string + benchmarkVersion: string + taskId: string + splitDigest: Sha256Digest + /** Exact agent-visible task instruction. The runtime rejects malformed Unicode. */ + instruction: string + repository: { + identity: string + rootIdentity: string + baseCommit: string + baseTree: string + } + attempt: AgentCandidateAttemptPolicy + model: { + requested: string + reasoningEffort: ReasoningEffort + } + grader: AgentCandidateBenchmarkGraderIdentity + /** Absolute paths inside the evaluator-owned execution environment. */ + executionRoots: { + taskRoot: string + candidateRoot?: string + } + /** Host-side staging roots. These are verified but never signed as container paths. */ + stagingRoots: { + taskRoot: string + candidateRoot?: string + profileRoot: string + } + workspace: AgentCandidateWorkspaceSnapshotEvidence + evaluatorTaskContainer?: ResolvedAgentCandidateContainer + limits: AgentCandidateExecutionLimits +} + +export interface VerifiedAgentCandidate { + readonly bundle: AgentCandidateBundle + readonly materializedTree?: string + readonly [verifiedCandidateBrand]: true +} + +export interface CanonicalCandidateDocument { + readonly value: T + /** Canonical UTF-8 bytes of `value` with its top-level digest omitted. */ + readonly bytes: Uint8Array + readonly digest: Sha256Digest +} + +export interface PreparedAgentCandidateLaunch { + executable: string + /** Complete fixed argv, including profile materializer flags but excluding task delivery. */ + args: readonly string[] + env: Readonly> + /** Informational subset already present at the tail of `args`; executors must not append twice. */ + flags: readonly string[] + cwd: string +} + +export interface PreparedAgentCandidateInstruction { + bytes: Uint8Array + delivery: AgentCandidateInstructionDelivery +} + +export interface PreparedAgentCandidateTrace { + runId: string + tags: Readonly> + env: Readonly> +} + +export interface PreparedAgentCandidateExecution { + readonly bundle: AgentCandidateBundle + readonly executionId: string + readonly roots: { + execution: { + taskRoot: string + candidateRoot?: string + } + staging: { + taskRoot: string + candidateRoot?: string + profileRoot: string + } + } + readonly profilePlan: { + value: AgentCandidateProfilePlanEvidence + bytes: Uint8Array + written: readonly string[] + } + readonly executionPlan: { + value: AgentCandidateExecutionPlanEvidence + bytes: Uint8Array + } + readonly materializationReceipt: CanonicalCandidateDocument + readonly launch: PreparedAgentCandidateLaunch + readonly instruction: PreparedAgentCandidateInstruction + readonly resolvedModel: AgentCandidateResolvedModel + readonly knowledge?: { + snapshotId: string + manifestDigest: Sha256Digest + manifest: Uint8Array + } + readonly trace: PreparedAgentCandidateTrace + readonly memory: AgentCandidateEffectiveMemory + readonly [preparedCandidateBrand]: true +} + +export interface AgentCandidateProtectedRunCapture { + executionId: string + termination: AgentCandidateTermination +} + +/** Raw evaluator capture made only after the candidate process is dead. */ +export interface AgentCandidateExecutorTaskOutcomeCapture { + /** Claimed final tree. The runtime recomputes it independently from `gitDiff`. */ + resultTree: string + /** Complete evaluator-captured workspace description after candidate execution. */ + afterState: AgentCandidateWorkspaceManifestMaterialV1 + /** Reproducible workspace archive corresponding to `afterState`. */ + archive: Uint8Array + /** Exact binary patch from the signed task base to `afterState`. */ + gitDiff: Uint8Array +} + +/** Raw isolated-memory capture made only after access has been revoked. */ +export interface AgentCandidateExecutorMemoryCapture { + readonly afterState: AgentCandidateWorkspaceManifestMaterialV1 + readonly archive: Uint8Array +} + +/** Idempotent executor result after process death and trace drain. */ +export interface AgentCandidateExecutorFinalCapture { + readonly stopped: true + readonly taskOutcome?: AgentCandidateExecutorTaskOutcomeCapture + /** Required only when the prepared candidate uses isolated task memory. */ + readonly memoryAfter?: AgentCandidateExecutorMemoryCapture +} + +/** Branded task outcome that has survived independent patch and tree verification. */ +export interface VerifiedAgentCandidateTaskOutcome { + readonly evidence: AgentCandidateTaskOutcomeEvidence & { + readonly artifact: AgentCandidateArtifactRef + } + readonly patch: Uint8Array + readonly [verifiedTaskOutcomeBrand]: true +} + +/** + * Evaluator-owned executable grader, pinned by immutable implementation bytes. + * + * `run` is an isolation boundary, not an arbitrary scoring callback. The + * implementation admitted to that boundary is supplied by the runtime after + * artifact verification. Implementations must derive every returned binding + * digest from the bytes and task outcome they actually admitted, rather than + * copying an expected digest from ambient configuration. + */ +export interface AgentCandidateBenchmarkGraderPort { + readonly name: string + readonly version: string + readonly artifact: AgentCandidateArtifactRef + run(input: { + readonly executionId: string + readonly termination: AgentCandidateTermination + readonly outcome: VerifiedAgentCandidateTaskOutcome + /** Exact verified artifact bytes. Each read returns a detached copy. */ + readonly implementation: { + readonly byteLength: number + readonly bytes: Uint8Array + } + /** Frozen result deadline; runners must stop work and side effects when aborted. */ + readonly signal: AbortSignal + }): Promise<{ + readonly evaluation: BenchmarkEvaluation + /** Raw grader output needed to audit or reproduce the normalized result. */ + readonly evidence: Uint8Array + /** Runtime-checked binding between admitted code, task input, and raw output. */ + readonly binding: { + /** Digest computed from the implementation bytes admitted to execution. */ + readonly implementationDigest: Sha256Digest + /** Digest of the exact runtime-verified task outcome graded by this run. */ + readonly taskOutcomeDigest: Sha256Digest + /** Digest computed from `evidence` before it leaves the execution boundary. */ + readonly outputDigest: Sha256Digest + } + }> +} + +/** One detached request passed to the trusted environment-specific executor. */ +export interface AgentCandidateExecutorRequest { + readonly executionId: string + /** Immutable bytes from which the executor creates fresh isolated workspaces. */ + readonly inputs: { + readonly task: AgentCandidateExecutorWorkspaceInput + readonly candidate?: AgentCandidateExecutorWorkspaceInput + readonly profile: { + readonly files: readonly AgentCandidateExecutorProfileFile[] + } + } + readonly roots: PreparedAgentCandidateExecution['roots']['execution'] + readonly profilePlan: PreparedAgentCandidateExecution['profilePlan'] + readonly executionPlan: PreparedAgentCandidateExecution['executionPlan'] + readonly materializationReceipt: CanonicalCandidateDocument + readonly launch: PreparedAgentCandidateLaunch + readonly instruction: PreparedAgentCandidateInstruction + readonly resolvedModel: AgentCandidateResolvedModel + /** Mechanically enforced by the runtime plus executor process-death acknowledgement. */ + readonly hardLimits: Pick + /** Validity bound checked against protected traces; generic black-box executors cannot preempt it. */ + readonly observedLimits: Pick + readonly knowledge?: PreparedAgentCandidateExecution['knowledge'] + readonly trace: PreparedAgentCandidateTrace + readonly memory: AgentCandidateEffectiveMemory +} + +/** + * Executes one prepared request inside an evaluator-owned isolation boundary. + * + * `request.launch.env` is the complete allowlisted environment, including + * protected model, memory, and trace bindings. Implementations must not merge + * ambient host variables into it. The returned capture deliberately contains + * no candidate-authored usage or score fields. + */ +export interface AgentCandidateExecutorPort { + execute( + request: AgentCandidateExecutorRequest, + context: { + traceStore: TraceStore + /** Aborted by the runtime at the exact frozen wall-time deadline. */ + signal: AbortSignal + /** Absolute epoch-millisecond deadline owned by the runtime. */ + deadlineAtMs: number + }, + ): Promise + /** + * Kill any process/container still associated with the request, drain trace + * writes, and capture the final task workspace before teardown. + * The runtime calls this on success, failure, and timeout before model settlement. + * Implementations must be idempotent and concurrency-safe for this exact + * execution/plan pair because a fresh worker may repeat crash recovery. + */ + stopAndCapture( + request: AgentCandidateExecutorStopRequest, + context: { + traceStore: TraceStore + reason: 'completed' | 'failed' | 'timeout' + /** Aborted at the frozen execution deadline or evaluator cleanup deadline. */ + signal: AbortSignal + /** Absolute execution deadline; a later stop acknowledgement cannot produce success. */ + deadlineAtMs: number + }, + ): Promise +} + +/** Opaque process identity used for termination without re-exposing launch credentials. */ +export interface AgentCandidateExecutorStopRequest { + readonly executionId: string + readonly executionPlanDigest: Sha256Digest +} + +export interface AgentCandidateExecutorWorkspaceInput { + readonly snapshot: AgentCandidateWorkspaceSnapshotEvidence + readonly files: readonly AgentCandidateExecutorWorkspaceFile[] +} + +export interface AgentCandidateExecutorWorkspaceFile { + readonly path: string + readonly mode: 0o644 | 0o755 + readonly bytes: Uint8Array +} + +export interface AgentCandidateExecutorProfileFile { + readonly path: string + readonly mode: 0o644 | 0o755 + readonly bytes: Uint8Array +} + +export type AgentCandidateRunFinalization = + | { + succeeded: true + receipt: CanonicalCandidateDocument + artifacts: { + modelSettlement: AgentCandidateArtifactRef + taskOutcome: AgentCandidateArtifactRef + benchmarkResult: AgentCandidateArtifactRef + runReceipt: AgentCandidateArtifactRef + } + } + | { + succeeded: false + reason: string + partial: { + executionId: string + bundleDigest: Sha256Digest + executionPlanDigest: Sha256Digest + materializationReceiptDigest: Sha256Digest + termination?: AgentCandidateTermination + } + /** Independent evaluator-gateway usage, even when execution or trace capture failed. */ + usage: AgentCandidateSpend | null + } + +/** Protected trace tags that bind a run to one prepared candidate execution. */ +export const CANDIDATE_TRACE_TAGS = { + executionId: 'tangle.candidate.execution_id', + bundleDigest: 'tangle.candidate.bundle_digest', + executionPlanDigest: 'tangle.candidate.execution_plan_digest', + materializationReceiptDigest: 'tangle.candidate.materialization_receipt_digest', +} as const + +/** Environment keys used to propagate immutable candidate trace identity. */ +export const CANDIDATE_TRACE_ENV = { + executionId: 'TANGLE_CANDIDATE_EXECUTION_ID', + bundleDigest: 'TANGLE_CANDIDATE_BUNDLE_DIGEST', + executionPlanDigest: 'TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST', + materializationReceiptDigest: 'TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST', + traceRunId: 'TANGLE_TRACE_RUN_ID', +} as const + +export type PreparedMemoryReceipt = AgentCandidateMemoryReceipt diff --git a/src/candidate-execution/verify.ts b/src/candidate-execution/verify.ts new file mode 100644 index 00000000..befde8e2 --- /dev/null +++ b/src/candidate-execution/verify.ts @@ -0,0 +1,167 @@ +import type { + AgentCandidateBundle, + AgentCandidateCapturedArtifact, + AgentCandidateResourceRef, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { agentCandidateBundleSchema } from '@tangle-network/agent-interface' + +import { + artifactCacheKey, + readVerifiedArtifact, + verifyBytes, + verifyWorkspaceSnapshotArtifacts, +} from './artifacts' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + deepFreezeCandidate, + omitTopLevelDigest, +} from './digest' +import { readCandidateGitHubResource, verifyCandidateCode } from './git-materialize' +import { + type AgentCandidateVerificationPorts, + type VerifiedAgentCandidate, + verifiedCandidateBrand, +} from './types' + +interface VerifiedCandidateState { + ports: AgentCandidateVerificationPorts + artifactBytes: Map + resourceBytes: Map +} + +const verifiedCandidateState = new WeakMap() + +/** Verifies every digest, resource, workspace, and Git object in a candidate bundle. */ +export async function verifyAgentCandidateBundle( + input: unknown, + ports: AgentCandidateVerificationPorts, +): Promise { + const parsed = agentCandidateBundleSchema.parse(input) + const withoutDigest = omitTopLevelDigest(parsed) + const actualDigest = canonicalCandidateDigest(withoutDigest) + if (actualDigest !== parsed.digest) { + throw new Error(`candidate bundle digest ${parsed.digest} does not match ${actualDigest}`) + } + const canonicalBytes = canonicalCandidateBytes(withoutDigest) + verifyBytes(canonicalBytes, parsed.digest, canonicalBytes.byteLength, 'candidate bundle') + + const artifactBytes = new Map() + const readArtifact = async (artifact: AgentCandidateCapturedArtifact): Promise => { + const key = artifactCacheKey(artifact) + const existing = artifactBytes.get(key) + if (existing) return Uint8Array.from(existing) + const bytes = await readVerifiedArtifact(artifact, ports.artifacts) + artifactBytes.set(key, Uint8Array.from(bytes)) + return bytes + } + + let patchBytes: Uint8Array | undefined + if (parsed.code.kind === 'git-patch') { + patchBytes = await readArtifact(parsed.code.patch.artifact) + } + const materializedTree = await verifyCandidateCode(parsed.code, ports.repositories, patchBytes) + + const resourceBytes = new Map() + for (const resource of candidateResources(parsed)) { + const bytes = + resource.kind === 'inline' + ? Buffer.from(resource.content, 'utf8') + : await readCandidateGitHubResource(resource, ports.repositories) + verifyBytes( + bytes, + resource.sha256, + resource.byteLength, + `candidate resource ${resource.name ?? (resource.kind === 'github' ? resource.path : '')}`, + ) + resourceBytes.set(resourceKey(resource), Uint8Array.from(bytes)) + resourceBytes.set(resource.sha256, Uint8Array.from(bytes)) + } + + if (parsed.execution.workspace) { + const workspace = await verifyWorkspaceSnapshotArtifacts( + parsed.execution.workspace, + ports.artifacts, + ) + artifactBytes.set(artifactCacheKey(parsed.execution.workspace.manifest), workspace.manifest) + artifactBytes.set(artifactCacheKey(parsed.execution.workspace.archive), workspace.archive) + } + if (parsed.knowledge) await readArtifact(parsed.knowledge.manifest) + if (parsed.memory.mode === 'isolated' && parsed.memory.seed) + await readArtifact(parsed.memory.seed) + + deepFreezeCandidate(parsed) + const verified = Object.freeze({ + bundle: parsed, + ...(materializedTree === undefined ? {} : { materializedTree }), + [verifiedCandidateBrand]: true as const, + }) + verifiedCandidateState.set(verified, { ports, artifactBytes, resourceBytes }) + return verified +} + +export function getVerifiedCandidateState( + candidate: VerifiedAgentCandidate, +): VerifiedCandidateState { + const state = verifiedCandidateState.get(candidate) + if (!state || candidate[verifiedCandidateBrand] !== true) { + throw new Error('candidate must come from verifyAgentCandidateBundle') + } + return state +} + +export async function verifiedArtifactBytes( + candidate: VerifiedAgentCandidate, + artifact: AgentCandidateCapturedArtifact, +): Promise { + const state = getVerifiedCandidateState(candidate) + const key = artifactCacheKey(artifact) + const existing = state.artifactBytes.get(key) + if (existing) return Uint8Array.from(existing) + const bytes = await readVerifiedArtifact(artifact, state.ports.artifacts) + state.artifactBytes.set(key, Uint8Array.from(bytes)) + return bytes +} + +export function verifiedResourceBytes( + candidate: VerifiedAgentCandidate, + resource: AgentCandidateResourceRef, +): Uint8Array { + const bytes = getVerifiedCandidateState(candidate).resourceBytes.get(resourceKey(resource)) + if (!bytes) { + throw new Error( + `candidate resource was not verified: ${resource.name ?? (resource.kind === 'github' ? resource.path : '')}`, + ) + } + return Uint8Array.from(bytes) +} + +export function verifiedResourceTextByDigest( + candidate: VerifiedAgentCandidate, +): ReadonlyMap { + const output = new Map() + for (const resource of candidateResources(candidate.bundle)) { + const bytes = getVerifiedCandidateState(candidate).resourceBytes.get(resource.sha256) + if (!bytes) throw new Error(`candidate resource digest was not verified: ${resource.sha256}`) + output.set(resource.sha256, new TextDecoder('utf-8', { fatal: true }).decode(bytes)) + } + return output +} + +function candidateResources(bundle: AgentCandidateBundle): AgentCandidateResourceRef[] { + const resources = bundle.profile.resources + if (!resources) return [] + const output: AgentCandidateResourceRef[] = [] + for (const mount of resources.files ?? []) output.push(mount.resource) + output.push(...(resources.tools ?? [])) + output.push(...(resources.skills ?? [])) + output.push(...(resources.agents ?? [])) + output.push(...(resources.commands ?? [])) + if (typeof resources.instructions === 'object') output.push(resources.instructions) + return output +} + +function resourceKey(resource: AgentCandidateResourceRef): string { + return canonicalCandidateDigest(resource) +} diff --git a/src/candidate-execution/workspace-archive.ts b/src/candidate-execution/workspace-archive.ts new file mode 100644 index 00000000..3ca40a71 --- /dev/null +++ b/src/candidate-execution/workspace-archive.ts @@ -0,0 +1,986 @@ +import { randomUUID } from 'node:crypto' +import { constants as fsConstants } from 'node:fs' +import { + lstat, + mkdir, + mkdtemp, + open, + readdir, + readFile, + realpath, + rename, + rm, + writeFile, +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, isAbsolute, join, resolve, sep, win32 } from 'node:path' +import { isAnyArrayBuffer, isSharedArrayBuffer } from 'node:util/types' + +import type { + AgentCandidateCapturedArtifact, + AgentCandidateWorkspaceManifestMaterialV1, + AgentCandidateWorkspaceSnapshotEvidence, +} from '@tangle-network/agent-interface' +import { type Entry, extract, type Pack, pack } from 'tar-stream' + +import { captureMaterializedWorkspace, verifyBytes, verifyMaterializedWorkspace } from './artifacts' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + deepFreezeCandidate, + embeddedCandidateArtifact, + sha256Bytes, +} from './digest' +import { + assertNoGitIndirection, + readCandidateGitTreeFiles, + runCandidateGit, +} from './git-materialize' +import { persistCandidateOutputArtifact } from './output-artifacts' +import type { + AgentCandidateExecutorWorkspaceFile, + AgentCandidateOutputArtifactPort, + AgentCandidateWorkspacePort, +} from './types' + +const archiveKind = 'agent-candidate-workspace-archive' as const +const workspaceEntryPrefix = 'workspace/' +const repositoryMetadataEntry = 'metadata/repository.json' +const repositoryBundleEntry = 'metadata/repository.bundle' +const fixedTarTime = new Date(0) + +export interface AgentCandidateWorkspaceArchiveLimits { + maxArchiveBytes: number + maxEmbeddedArtifactBytes: number + maxFiles: number + maxFileBytes: number + maxTotalFileBytes: number + maxPathBytes: number + maxRepositoryBundleBytes: number +} + +const defaultLimits: AgentCandidateWorkspaceArchiveLimits = Object.freeze({ + maxArchiveBytes: 512 * 1024 * 1024, + maxEmbeddedArtifactBytes: 1024 * 1024, + maxFiles: 50_000, + maxFileBytes: 128 * 1024 * 1024, + maxTotalFileBytes: 256 * 1024 * 1024, + maxPathBytes: 4_096, + maxRepositoryBundleBytes: 128 * 1024 * 1024, +}) + +interface WorkspaceArchiveRepositoryV1 { + headCommit: string + headTree: string + bundle: { + sha256: `sha256:${string}` + byteLength: number + bytes: Uint8Array + } +} + +interface WorkspaceArchiveRepositoryMetadataV1 { + schemaVersion: 1 + kind: typeof archiveKind + headCommit: string + headTree: string + bundle: { + sha256: `sha256:${string}` + byteLength: number + } +} + +interface DecodedWorkspaceArchive { + files: Array<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> + repository?: WorkspaceArchiveRepositoryV1 +} + +export interface CaptureAgentCandidateWorkspaceOptions { + /** Include Git HEAD so task preparation can prove its exact commit and tree. */ + includeRepository?: boolean + limits?: Partial + /** Use the evaluator-owned artifact store when manifest or archive bytes should not be embedded. */ + artifactPersistence?: { + executionId: string + outputArtifacts: AgentCandidateOutputArtifactPort + signal?: AbortSignal + } +} + +export interface CreateAgentCandidateWorkspacePortOptions { + limits?: Partial +} + +export interface CapturedAgentCandidateWorkspace { + readonly snapshot: AgentCandidateWorkspaceSnapshotEvidence + /** Caller-owned bytes accepted by createAgentCandidateWorkspacePort. */ + readonly archive: Uint8Array +} + +/** Capture one exact regular-file workspace for immutable candidate execution. */ +export async function captureAgentCandidateWorkspace( + rootInput: string, + options: CaptureAgentCandidateWorkspaceOptions = {}, +): Promise { + const root = resolve(rootInput) + const limits = workspaceLimits(options.limits) + const captured = options.includeRepository + ? await captureRepository(root, limits) + : await captureMaterializedWorkspace(root, { + limits: { + maxFiles: limits.maxFiles, + maxFileBytes: limits.maxFileBytes, + maxTotalFileBytes: limits.maxTotalFileBytes, + }, + }) + assertWorkspaceFilesWithinLimits(captured.files, limits) + const repository = 'repository' in captured ? captured.repository : undefined + return captureWorkspaceFiles(captured.files, repository, limits, options.artifactPersistence) +} + +/** Capture detached files returned by a remote executor into the standard archive. */ +export async function captureAgentCandidateWorkspaceFiles( + input: readonly AgentCandidateExecutorWorkspaceFile[], + options: Omit = {}, +): Promise { + const limits = workspaceLimits(options.limits) + return captureWorkspaceFiles( + normalizeWorkspaceFiles(input, limits), + undefined, + limits, + options.artifactPersistence, + ) +} + +async function captureWorkspaceFiles( + files: readonly AgentCandidateExecutorWorkspaceFile[], + repository: WorkspaceArchiveRepositoryV1 | undefined, + limits: AgentCandidateWorkspaceArchiveLimits, + artifactPersistence: CaptureAgentCandidateWorkspaceOptions['artifactPersistence'], +): Promise { + const material = workspaceManifest(files) + const manifest = canonicalCandidateBytes(material) + const archive = await encodeWorkspaceArchive(files, repository, limits.maxArchiveBytes) + if ( + !artifactPersistence && + (manifest.byteLength > limits.maxEmbeddedArtifactBytes || + archive.byteLength > limits.maxEmbeddedArtifactBytes) + ) { + throw new Error( + 'candidate workspace artifacts exceed maxEmbeddedArtifactBytes; artifactPersistence is required', + ) + } + const manifestArtifact = await captureWorkspaceArtifact('manifest', manifest, artifactPersistence) + const archiveArtifact = await captureWorkspaceArtifact('archive', archive, artifactPersistence) + const snapshot = deepFreezeCandidate({ + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: canonicalCandidateDigest(material), + material, + manifest: manifestArtifact, + archive: archiveArtifact, + } satisfies AgentCandidateWorkspaceSnapshotEvidence) + return Object.freeze({ snapshot, archive }) +} + +async function captureWorkspaceArtifact( + kind: 'manifest' | 'archive', + bytes: Uint8Array, + persistence: CaptureAgentCandidateWorkspaceOptions['artifactPersistence'], +): Promise { + if (!persistence) return embeddedCandidateArtifact(bytes) + if (!persistence.executionId.trim()) { + throw new Error('candidate workspace artifact persistence requires an executionId') + } + return persistCandidateOutputArtifact(persistence.outputArtifacts, { + executionId: persistence.executionId, + purpose: kind === 'manifest' ? 'candidate-workspace-manifest' : 'candidate-workspace-archive', + bytes, + ...(persistence.signal ? { signal: persistence.signal } : {}), + }) +} + +/** Create the standard bounded materializer for candidate execution ports. */ +export function createAgentCandidateWorkspacePort( + options: CreateAgentCandidateWorkspacePortOptions = {}, +): AgentCandidateWorkspacePort { + const limits = workspaceLimits(options.limits) + const port: AgentCandidateWorkspacePort = { + materialize: async ({ role, snapshot, archive, destination }) => { + await materializeAgentCandidateWorkspace({ + role, + snapshot, + archive, + destination, + limits, + }) + }, + } + return Object.freeze(port) +} + +async function materializeAgentCandidateWorkspace(input: { + role: 'task' | 'candidate' | 'memory' + snapshot: AgentCandidateWorkspaceSnapshotEvidence + archive: Uint8Array + destination: string + limits: AgentCandidateWorkspaceArchiveLimits +}): Promise { + verifyBytes( + input.archive, + input.snapshot.archive.sha256, + input.snapshot.archive.byteLength, + 'candidate workspace archive', + ) + const decoded = await parseWorkspaceArchive(input.archive, input.limits) + if (decoded.repository && input.role !== 'task') { + throw new Error('only task workspaces may carry a Git repository') + } + assertArchiveMatchesSnapshot(decoded.files, input.snapshot) + const destination = resolve(input.destination) + await prepareEmptyDestination(destination) + const staging = await mkdtemp(`${destination}.materializing-`) + try { + await writeWorkspaceFiles(staging, decoded.files) + if (decoded.repository) { + await materializeRepository(decoded.repository, staging) + } + await verifyMaterializedWorkspace(staging, input.snapshot.material, { + ignoredProtectedRootEntries: decoded.repository ? ['.git'] : [], + }) + await rename(staging, destination) + } catch (error) { + await rm(staging, { recursive: true, force: true }) + await rm(destination, { recursive: true, force: true }) + throw error + } +} + +async function captureRepository( + root: string, + limits: AgentCandidateWorkspaceArchiveLimits, +): Promise<{ + files: Array<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> + repository: WorkspaceArchiveRepositoryV1 +}> { + const stats = await lstat(root) + if (!stats.isDirectory() || stats.isSymbolicLink() || (await realpath(root)) !== root) { + throw new Error('candidate repository root must be a real directory') + } + const topLevel = (await runCandidateGit(root, ['rev-parse', '--show-toplevel'])).stdout + .toString('utf8') + .trim() + if (resolve(topLevel) !== root) { + throw new Error('candidate repository capture must start at the Git worktree root') + } + const gitDir = resolve( + (await runCandidateGit(root, ['rev-parse', '--absolute-git-dir'])).stdout + .toString('utf8') + .trim(), + ) + if ((await realpath(gitDir)) !== gitDir || gitDir.includes(':')) { + throw new Error('candidate repository Git object store has an unsupported path') + } + await assertNoGitIndirection(root, gitDir, 'candidate repository') + const headCommit = (await runCandidateGit(root, ['rev-parse', 'HEAD'])).stdout + .toString('utf8') + .trim() + const headTree = (await runCandidateGit(root, ['rev-parse', 'HEAD^{tree}'])).stdout + .toString('utf8') + .trim() + assertGitObjectId(headCommit, 'HEAD') + assertGitObjectId(headTree, 'HEAD tree') + if (headCommit.length !== headTree.length) { + throw new Error('candidate repository HEAD and tree use different object formats') + } + const files = await readCandidateGitTreeFiles(root, headTree, {}, limits) + assertWorkspaceFilesWithinLimits(files, limits) + const reachableBytesText = ( + await runCandidateGit(root, ['rev-list', '--disk-usage', '--objects', 'HEAD']) + ).stdout + .toString('utf8') + .trim() + const reachableBytes = Number(reachableBytesText) + if (!Number.isSafeInteger(reachableBytes) || reachableBytes < 0) { + throw new Error('candidate repository reachable size is invalid') + } + if (reachableBytes > limits.maxRepositoryBundleBytes) { + throw new Error('candidate repository exceeds maxRepositoryBundleBytes') + } + const temporary = await mkdtemp(join(tmpdir(), 'agent-candidate-workspace-bundle-')) + const bundlePath = join(temporary, `${randomUUID()}.bundle`) + try { + await runCandidateGit(root, ['bundle', 'create', bundlePath, 'HEAD']) + const bundle = Uint8Array.from(await readFile(bundlePath)) + if (bundle.byteLength > limits.maxRepositoryBundleBytes) { + throw new Error('candidate repository bundle exceeds maxRepositoryBundleBytes') + } + return { + files, + repository: { + headCommit, + headTree, + bundle: { + sha256: sha256Bytes(bundle), + byteLength: bundle.byteLength, + bytes: bundle, + }, + }, + } + } finally { + await rm(temporary, { recursive: true, force: true }) + } +} + +async function encodeWorkspaceArchive( + files: ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }>, + repository: WorkspaceArchiveRepositoryV1 | undefined, + maxArchiveBytes: number, +): Promise { + const archive = pack() + const output = collectStream(archive, maxArchiveBytes, 'candidate workspace archive') + try { + await writeWorkspaceArchiveEntries(archive, files, repository) + archive.finalize() + return await output + } catch (error) { + archive.destroy(error instanceof Error ? error : new Error(String(error))) + await output.catch(() => undefined) + throw error + } +} + +async function verifyCanonicalWorkspaceArchive( + files: ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }>, + repository: WorkspaceArchiveRepositoryV1 | undefined, + expected: Uint8Array, + maxArchiveBytes: number, +): Promise { + const archive = pack() + const comparison = streamEqualsBytes( + archive, + expected, + maxArchiveBytes, + 'candidate workspace archive', + ) + try { + await writeWorkspaceArchiveEntries(archive, files, repository) + archive.finalize() + if (!(await comparison)) throw new Error('candidate workspace tar is not canonical') + } catch (error) { + archive.destroy(error instanceof Error ? error : new Error(String(error))) + await comparison.catch(() => undefined) + throw error + } +} + +async function writeWorkspaceArchiveEntries( + archive: Pack, + files: ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }>, + repository: WorkspaceArchiveRepositoryV1 | undefined, +): Promise { + for (const file of files) { + await writeTarEntry(archive, `${workspaceEntryPrefix}${file.path}`, file.mode, file.bytes) + } + if (!repository) return + const metadata = canonicalCandidateBytes({ + schemaVersion: 1, + kind: archiveKind, + headCommit: repository.headCommit, + headTree: repository.headTree, + bundle: { + sha256: repository.bundle.sha256, + byteLength: repository.bundle.byteLength, + }, + } satisfies WorkspaceArchiveRepositoryMetadataV1) + await writeTarEntry(archive, repositoryMetadataEntry, 0o600, metadata) + await writeTarEntry(archive, repositoryBundleEntry, 0o600, repository.bundle.bytes) +} + +async function parseWorkspaceArchive( + bytes: Uint8Array, + limits: AgentCandidateWorkspaceArchiveLimits, +): Promise { + if (bytes.byteLength > limits.maxArchiveBytes) { + throw new Error('candidate workspace archive exceeds maxArchiveBytes') + } + const parser = extract() + const files: DecodedWorkspaceArchive['files'] = [] + const observedEntryNames = new Set() + const observedPaths = new Set() + let totalBytes = 0 + let retainedEntryBytes = 0 + let previousPath: string | undefined + let repositoryMetadata: WorkspaceArchiveRepositoryMetadataV1 | undefined + let repositoryBundle: Uint8Array | undefined + const decoded = (async () => { + for await (const entry of parser) { + const name = entry.header.name + if ( + entry.header.type !== 'file' || + typeof name !== 'string' || + observedEntryNames.has(name) + ) { + throw new Error('candidate workspace tar contains an invalid entry') + } + observedEntryNames.add(name) + if (name.startsWith(workspaceEntryPrefix)) { + if (files.length >= limits.maxFiles) { + throw new Error('candidate workspace archive has an invalid file count') + } + const path = safeArchivePath(name.slice(workspaceEntryPrefix.length), limits.maxPathBytes) + if (entry.header.mode !== 0o644 && entry.header.mode !== 0o755) { + throw new Error(`candidate workspace archive has an unsupported mode: ${path}`) + } + assertRetainedArchiveSize(bytes.byteLength, retainedEntryBytes, entry.header.size, limits) + const fileBytes = await readTarEntry(entry, limits.maxFileBytes, `workspace file ${path}`) + retainedEntryBytes += fileBytes.byteLength + assertWorkspacePathOrder( + path, + previousPath, + observedPaths, + 'candidate workspace archive paths must be unique and sorted', + ) + previousPath = path + if (fileBytes.byteLength > limits.maxTotalFileBytes - totalBytes) { + throw new Error('candidate workspace archive exceeds maxTotalFileBytes') + } + totalBytes += fileBytes.byteLength + files.push({ path, mode: entry.header.mode, bytes: fileBytes }) + } else if (name === repositoryMetadataEntry) { + if (entry.header.mode !== 0o600 || repositoryMetadata) { + throw new Error('candidate workspace archive repository metadata is invalid') + } + assertRetainedArchiveSize(bytes.byteLength, retainedEntryBytes, entry.header.size, limits) + const metadataBytes = await readTarEntry( + entry, + limits.maxPathBytes, + 'candidate repository metadata', + ) + retainedEntryBytes += metadataBytes.byteLength + repositoryMetadata = parseRepositoryMetadata(metadataBytes, limits.maxRepositoryBundleBytes) + } else if (name === repositoryBundleEntry) { + if (entry.header.mode !== 0o600 || repositoryBundle) { + throw new Error('candidate workspace archive repository bundle is invalid') + } + assertRetainedArchiveSize(bytes.byteLength, retainedEntryBytes, entry.header.size, limits) + repositoryBundle = await readTarEntry( + entry, + limits.maxRepositoryBundleBytes, + 'candidate Git bundle', + ) + retainedEntryBytes += repositoryBundle.byteLength + } else { + throw new Error(`candidate workspace tar contains an unsupported entry: ${name}`) + } + } + if (Boolean(repositoryMetadata) !== Boolean(repositoryBundle)) { + throw new Error('candidate workspace archive repository evidence is incomplete') + } + const repository = + repositoryMetadata && repositoryBundle + ? repositoryFromTar(repositoryMetadata, repositoryBundle) + : undefined + return { files, ...(repository ? { repository } : {}) } + })() + parser.end(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)) + const result = await decoded + await verifyCanonicalWorkspaceArchive( + result.files, + result.repository, + bytes, + limits.maxArchiveBytes, + ) + return result +} + +function assertRetainedArchiveSize( + archiveBytes: number, + retainedEntryBytes: number, + entryBytes: number | undefined, + limits: AgentCandidateWorkspaceArchiveLimits, +): void { + if ( + !Number.isSafeInteger(entryBytes) || + entryBytes === undefined || + entryBytes < 0 || + entryBytes > limits.maxArchiveBytes - archiveBytes - retainedEntryBytes + ) { + throw new Error('candidate workspace archive exceeds maxArchiveBytes while decoding') + } +} + +function parseRepositoryMetadata( + bytes: Uint8Array, + maxBundleBytes: number, +): WorkspaceArchiveRepositoryMetadataV1 { + let value: unknown + try { + value = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) + } catch (error) { + throw new Error('candidate workspace archive repository metadata is invalid', { cause: error }) + } + if ( + !isRecord(value) || + Object.keys(value).sort().join(',') !== 'bundle,headCommit,headTree,kind,schemaVersion' || + value.schemaVersion !== 1 || + value.kind !== archiveKind || + typeof value.headCommit !== 'string' || + typeof value.headTree !== 'string' || + !isRecord(value.bundle) || + Object.keys(value.bundle).sort().join(',') !== 'byteLength,sha256' || + typeof value.bundle.sha256 !== 'string' || + !/^sha256:[a-f0-9]{64}$/.test(value.bundle.sha256) || + !Number.isSafeInteger(value.bundle.byteLength) || + (value.bundle.byteLength as number) <= 0 || + (value.bundle.byteLength as number) > maxBundleBytes + ) { + throw new Error('candidate workspace archive repository is invalid') + } + if (!Buffer.from(canonicalCandidateBytes(value)).equals(Buffer.from(bytes))) { + throw new Error('candidate workspace archive repository metadata is not canonical') + } + assertGitObjectId(value.headCommit, 'repository HEAD') + assertGitObjectId(value.headTree, 'repository HEAD tree') + if (value.headCommit.length !== value.headTree.length) { + throw new Error('candidate repository HEAD and tree use different object formats') + } + return { + schemaVersion: 1, + kind: archiveKind, + headCommit: value.headCommit, + headTree: value.headTree, + bundle: { + sha256: value.bundle.sha256 as `sha256:${string}`, + byteLength: value.bundle.byteLength as number, + }, + } +} + +function repositoryFromTar( + metadata: WorkspaceArchiveRepositoryMetadataV1, + bundle: Uint8Array, +): WorkspaceArchiveRepositoryV1 { + verifyBytes(bundle, metadata.bundle.sha256, metadata.bundle.byteLength, 'candidate Git bundle') + return { + headCommit: metadata.headCommit, + headTree: metadata.headTree, + bundle: { ...metadata.bundle, bytes: bundle }, + } +} + +async function writeTarEntry( + archive: Pack, + name: string, + mode: 0o600 | 0o644 | 0o755, + bytes: Uint8Array, +): Promise { + await new Promise((resolveEntry, rejectEntry) => { + archive.entry( + { + name, + mode, + uid: 0, + gid: 0, + size: bytes.byteLength, + mtime: fixedTarTime, + type: 'file', + uname: '', + gname: '', + }, + Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength), + (error) => (error ? rejectEntry(error) : resolveEntry()), + ) + }) +} + +async function readTarEntry(entry: Entry, maxBytes: number, label: string): Promise { + const size = entry.header.size + if (!Number.isSafeInteger(size) || size === undefined || size < 0 || size > maxBytes) { + throw new Error(`${label} has an invalid tar size`) + } + const bytes = await collectStream(entry, maxBytes, label) + if (bytes.byteLength !== size) throw new Error(`${label} differs from its tar size`) + return bytes +} + +async function collectStream( + stream: AsyncIterable, + maxBytes: number, + label: string, +): Promise { + const chunks: Buffer[] = [] + let totalBytes = 0 + for await (const chunk of stream) { + if (chunk.byteLength > maxBytes - totalBytes) { + const error = new Error(`${label} exceeds its size limit`) + if ('destroy' in stream && typeof stream.destroy === 'function') stream.destroy(error) + throw error + } + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + chunks.push(bytes) + totalBytes += bytes.byteLength + } + return Buffer.concat(chunks, totalBytes) +} + +async function streamEqualsBytes( + stream: AsyncIterable, + expected: Uint8Array, + maxBytes: number, + label: string, +): Promise { + const expectedBuffer = Buffer.from(expected.buffer, expected.byteOffset, expected.byteLength) + let equal = true + let offset = 0 + for await (const chunk of stream) { + if (chunk.byteLength > maxBytes - offset) { + const error = new Error(`${label} exceeds its size limit`) + if ('destroy' in stream && typeof stream.destroy === 'function') stream.destroy(error) + throw error + } + const observed = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength) + if ( + offset + observed.byteLength > expectedBuffer.byteLength || + !observed.equals(expectedBuffer.subarray(offset, offset + observed.byteLength)) + ) { + equal = false + } + offset += observed.byteLength + } + return equal && offset === expectedBuffer.byteLength +} + +async function prepareEmptyDestination(destination: string): Promise { + await mkdir(destination, { recursive: true, mode: 0o700 }) + const stats = await lstat(destination) + if ( + !stats.isDirectory() || + stats.isSymbolicLink() || + (await realpath(destination)) !== destination + ) { + throw new Error('candidate workspace destination must be a real directory') + } + if ((await readdir(destination)).length > 0) { + throw new Error('candidate workspace destination must be empty') + } +} + +async function writeWorkspaceFiles( + destination: string, + files: ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }>, +): Promise { + for (const file of files) { + const path = workspacePath(destination, file.path) + await mkdir(dirname(path), { recursive: true, mode: 0o700 }) + const descriptor = await open( + path, + fsConstants.O_WRONLY | + fsConstants.O_CREAT | + fsConstants.O_EXCL | + (typeof fsConstants.O_NOFOLLOW === 'number' ? fsConstants.O_NOFOLLOW : 0), + 0o600, + ) + try { + await descriptor.writeFile(file.bytes) + const stats = await descriptor.stat() + if (!stats.isFile() || stats.nlink !== 1) { + throw new Error(`candidate workspace file identity changed while writing: ${file.path}`) + } + await descriptor.chmod(file.mode) + } finally { + await descriptor.close() + } + } +} + +async function materializeRepository( + repository: WorkspaceArchiveRepositoryV1, + destination: string, +): Promise { + const temporary = await mkdtemp(join(tmpdir(), 'agent-candidate-workspace-bundle-')) + const bundlePath = join(temporary, `${randomUUID()}.bundle`) + try { + await writeFile(bundlePath, repository.bundle.bytes, { flag: 'wx', mode: 0o600 }) + await runCandidateGit(destination, [ + 'init', + '--quiet', + '--initial-branch=candidate', + `--object-format=${repository.headCommit.length === 64 ? 'sha256' : 'sha1'}`, + ]) + await runCandidateGit(destination, ['bundle', 'verify', bundlePath]) + await runCandidateGit(destination, ['bundle', 'unbundle', bundlePath]) + const objectType = ( + await runCandidateGit(destination, ['cat-file', '-t', repository.headCommit]) + ).stdout + .toString('utf8') + .trim() + if (objectType !== 'commit') throw new Error('candidate repository HEAD is not a commit') + const observedTree = ( + await runCandidateGit(destination, ['rev-parse', `${repository.headCommit}^{tree}`]) + ).stdout + .toString('utf8') + .trim() + if (observedTree !== repository.headTree) { + throw new Error('candidate repository bundle HEAD tree changed') + } + await runCandidateGit(destination, ['update-ref', '--no-deref', 'HEAD', repository.headCommit]) + await runCandidateGit(destination, ['read-tree', repository.headTree]) + const status = ( + await runCandidateGit(destination, ['status', '--porcelain=v1', '--untracked-files=all']) + ).stdout + .toString('utf8') + .trim() + if (status) throw new Error('candidate repository files do not exactly match HEAD') + } finally { + await rm(temporary, { recursive: true, force: true }) + } +} + +function workspaceManifest( + files: ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }>, +): AgentCandidateWorkspaceManifestMaterialV1 { + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files: files.map((file) => ({ + path: file.path, + mode: file.mode, + sha256: sha256Bytes(file.bytes), + byteLength: file.bytes.byteLength, + })), + } +} + +function assertArchiveMatchesSnapshot( + files: ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }>, + snapshot: AgentCandidateWorkspaceSnapshotEvidence, +): void { + const material = workspaceManifest(files) + const materialBytes = canonicalCandidateBytes(material) + verifyBytes( + materialBytes, + snapshot.manifest.sha256, + snapshot.manifest.byteLength, + 'candidate workspace manifest', + ) + if ( + canonicalCandidateDigest(material) !== snapshot.digest || + !Buffer.from(materialBytes).equals(Buffer.from(canonicalCandidateBytes(snapshot.material))) + ) { + throw new Error('candidate workspace archive does not match its snapshot manifest') + } +} + +function assertWorkspaceFilesWithinLimits( + files: ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }>, + limits: AgentCandidateWorkspaceArchiveLimits, +): void { + if (files.length > limits.maxFiles) { + throw new Error('candidate workspace exceeds maxFiles') + } + let totalBytes = 0 + let previousPath: string | undefined + const observedPaths = new Set() + for (const file of files) { + const path = safeArchivePath(file.path, limits.maxPathBytes) + assertWorkspacePathOrder( + path, + previousPath, + observedPaths, + 'candidate workspace paths must be unique and sorted', + ) + previousPath = path + if (file.mode !== 0o644 && file.mode !== 0o755) { + throw new Error(`candidate workspace file has unsupported mode: ${path}`) + } + if (file.bytes.byteLength > limits.maxFileBytes) { + throw new Error(`candidate workspace file exceeds maxFileBytes: ${path}`) + } + totalBytes += file.bytes.byteLength + if (totalBytes > limits.maxTotalFileBytes) { + throw new Error('candidate workspace exceeds maxTotalFileBytes') + } + } +} + +function normalizeWorkspaceFiles( + input: readonly AgentCandidateExecutorWorkspaceFile[], + limits: AgentCandidateWorkspaceArchiveLimits, +): AgentCandidateExecutorWorkspaceFile[] { + if (!Array.isArray(input)) { + throw new Error('candidate workspace files must be an array') + } + const fileCount = input.length + if (fileCount > limits.maxFiles) { + throw new Error('candidate workspace exceeds maxFiles') + } + let totalBytes = 0 + const files: AgentCandidateExecutorWorkspaceFile[] = [] + for (let index = 0; index < fileCount; index++) { + const inputFile = input[index] + if (!isRecord(inputFile) || Object.keys(inputFile).sort().join(',') !== 'bytes,mode,path') { + throw new Error(`candidate workspace file ${index} is invalid`) + } + const descriptors = Object.getOwnPropertyDescriptors(inputFile) + if ( + !('value' in (descriptors.path ?? {})) || + !('value' in (descriptors.mode ?? {})) || + !('value' in (descriptors.bytes ?? {})) + ) { + throw new Error(`candidate workspace file ${index} must use fixed values`) + } + const path = safeArchivePath(descriptors.path?.value, limits.maxPathBytes) + const mode = descriptors.mode?.value + const inputBytes = descriptors.bytes?.value + if (mode !== 0o644 && mode !== 0o755) { + throw new Error(`candidate workspace file has unsupported mode: ${path}`) + } + const view = inspectWorkspaceBytes(inputBytes, path) + if (view.byteLength > limits.maxFileBytes) { + throw new Error(`candidate workspace file exceeds maxFileBytes: ${path}`) + } + if (view.byteLength > limits.maxTotalFileBytes - totalBytes) { + throw new Error('candidate workspace exceeds maxTotalFileBytes') + } + if (view.byteLength > Math.max(0, limits.maxArchiveBytes - 1024)) { + throw new Error('candidate workspace archive exceeds maxArchiveBytes') + } + totalBytes += view.byteLength + files.push({ path, mode, bytes: detachWorkspaceBytes(view) }) + } + files.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0)) + assertWorkspaceFilesWithinLimits(files, limits) + return files +} + +function workspaceLimits( + overrides: Partial | undefined, +): AgentCandidateWorkspaceArchiveLimits { + const limits = { ...defaultLimits, ...overrides } + for (const [name, value] of Object.entries(limits)) { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`candidate workspace ${name} must be a positive safe integer`) + } + } + return Object.freeze(limits) +} + +function safeArchivePath(value: unknown, maxPathBytes: number): string { + if ( + typeof value !== 'string' || + !value || + !isWellFormedUnicode(value) || + Buffer.byteLength(value, 'utf8') > maxPathBytes || + isAbsolute(value) || + win32.isAbsolute(value) || + value.includes('\\') || + value.includes('\0') || + hasControlCharacter(value) || + value.split('/').some((part) => !part || part === '.' || part === '..') || + ['.git', '.sidecar'].includes(value.split('/')[0]?.toLowerCase() ?? '') + ) { + throw new Error(`candidate workspace archive contains an unsafe path: ${String(value)}`) + } + return value +} + +function assertWorkspacePathOrder( + path: string, + previousPath: string | undefined, + observedPaths: Set, + errorMessage: string, +): void { + if (previousPath !== undefined && previousPath >= path) throw new Error(errorMessage) + for (let slash = path.indexOf('/'); slash !== -1; slash = path.indexOf('/', slash + 1)) { + if (observedPaths.has(path.slice(0, slash))) throw new Error(errorMessage) + } + observedPaths.add(path) +} + +function inspectWorkspaceBytes( + input: unknown, + path: string, +): { buffer: ArrayBuffer; byteOffset: number; byteLength: number } { + let buffer: ArrayBufferLike + let byteOffset: number + let byteLength: number + try { + buffer = typedArrayBufferGetter.call(input as Uint8Array) as ArrayBufferLike + byteOffset = typedArrayByteOffsetGetter.call(input as Uint8Array) as number + byteLength = typedArrayByteLengthGetter.call(input as Uint8Array) as number + } catch { + throw new Error(`candidate workspace file bytes are invalid: ${path}`) + } + if (isSharedArrayBuffer(buffer)) { + throw new Error(`candidate workspace file uses shared bytes: ${path}`) + } + if (!isAnyArrayBuffer(buffer)) { + throw new Error(`candidate workspace file bytes are invalid: ${path}`) + } + return { buffer, byteOffset, byteLength } +} + +function detachWorkspaceBytes(input: { + buffer: ArrayBuffer + byteOffset: number + byteLength: number +}): Uint8Array { + const { buffer, byteOffset, byteLength } = input + const copy = new Uint8Array(byteLength) + Uint8Array.prototype.set.call(copy, new Uint8Array(buffer, byteOffset, byteLength)) + return copy +} + +function isWellFormedUnicode(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index) + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1) + if (!(next >= 0xdc00 && next <= 0xdfff)) return false + index++ + } else if (code >= 0xdc00 && code <= 0xdfff) { + return false + } + } + return true +} + +function workspacePath(root: string, relativePath: string): string { + const path = resolve(root, relativePath) + if (!path.startsWith(`${root}${sep}`)) { + throw new Error(`candidate workspace archive path escapes its destination: ${relativePath}`) + } + return path +} + +function assertGitObjectId(value: string, label: string): void { + if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(value)) { + throw new Error(`candidate repository ${label} is not a Git object id`) + } +} + +function hasControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index) + if (code < 0x20 || code === 0x7f) return true + } + return false +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +type TypedArrayGetter = (this: Uint8Array) => unknown + +function requireTypedArrayGetter(name: 'buffer' | 'byteOffset' | 'byteLength'): TypedArrayGetter { + const typedArrayPrototype = Object.getPrototypeOf(Uint8Array.prototype) as object + const getter = Object.getOwnPropertyDescriptor(typedArrayPrototype, name)?.get + if (!getter) throw new Error(`typed array intrinsic ${name} accessor is unavailable`) + return getter +} + +const typedArrayBufferGetter = requireTypedArrayGetter('buffer') +const typedArrayByteOffsetGetter = requireTypedArrayGetter('byteOffset') +const typedArrayByteLengthGetter = requireTypedArrayGetter('byteLength') diff --git a/src/candidate-execution/workspace-snapshot.ts b/src/candidate-execution/workspace-snapshot.ts new file mode 100644 index 00000000..cd5ac7f2 --- /dev/null +++ b/src/candidate-execution/workspace-snapshot.ts @@ -0,0 +1,87 @@ +import type { + AgentCandidateWorkspaceManifestMaterialV1, + AgentCandidateWorkspaceSnapshotEvidence, +} from '@tangle-network/agent-interface' +import { + agentCandidateWorkspaceManifestMaterialSchema, + agentCandidateWorkspaceSnapshotEvidenceSchema, +} from '@tangle-network/agent-interface' + +import { + canonicalCandidateBytes, + embeddedCandidateArtifact, + immutableCandidateValue, + sha256Bytes, +} from './digest' +import { persistCandidateOutputArtifact } from './output-artifacts' +import type { AgentCandidateOutputArtifactPort } from './types' + +export interface ProvisionalCandidateWorkspaceSnapshot { + readonly material: AgentCandidateWorkspaceManifestMaterialV1 + readonly manifestBytes: Uint8Array + readonly snapshot: AgentCandidateWorkspaceSnapshotEvidence +} + +/** + * Build materialization-only evidence without parsing a large base64 field. + * Every field is evaluator-derived, and callers must not persist or expose this + * provisional value as receipt evidence. + */ +export function provisionalCandidateWorkspaceSnapshot( + materialInput: AgentCandidateWorkspaceManifestMaterialV1, + archive: Uint8Array, +): ProvisionalCandidateWorkspaceSnapshot { + const material = immutableCandidateValue( + agentCandidateWorkspaceManifestMaterialSchema.parse(materialInput), + ) + const manifestBytes = canonicalCandidateBytes(material) + const snapshot: AgentCandidateWorkspaceSnapshotEvidence = Object.freeze({ + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: sha256Bytes(manifestBytes), + material, + manifest: Object.freeze(embeddedCandidateArtifact(manifestBytes)), + archive: Object.freeze(embeddedCandidateArtifact(archive)), + }) + return Object.freeze({ material, manifestBytes, snapshot }) +} + +/** Persist verified workspace bytes and construct final reference-only evidence. */ +export async function persistCandidateWorkspaceSnapshot( + port: AgentCandidateOutputArtifactPort, + input: { + executionId: string + material: AgentCandidateWorkspaceManifestMaterialV1 + archive: Uint8Array + purpose: 'task' | 'memory-after' + signal?: AbortSignal + }, +): Promise { + input.signal?.throwIfAborted() + const material = immutableCandidateValue(input.material) + const manifestBytes = canonicalCandidateBytes(material) + const [manifest, archive] = await Promise.all([ + persistCandidateOutputArtifact(port, { + executionId: input.executionId, + purpose: input.purpose === 'task' ? 'task-manifest' : 'memory-after-manifest', + bytes: manifestBytes, + ...(input.signal ? { signal: input.signal } : {}), + }), + persistCandidateOutputArtifact(port, { + executionId: input.executionId, + purpose: input.purpose === 'task' ? 'task-archive' : 'memory-after-archive', + bytes: input.archive, + ...(input.signal ? { signal: input.signal } : {}), + }), + ]) + return immutableCandidateValue( + agentCandidateWorkspaceSnapshotEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: sha256Bytes(manifestBytes), + material, + manifest, + archive, + }), + ) +} diff --git a/src/conversation/run-persona.test.ts b/src/conversation/run-persona.test.ts index 04c8f0a7..b3786c14 100644 --- a/src/conversation/run-persona.test.ts +++ b/src/conversation/run-persona.test.ts @@ -1,5 +1,9 @@ -import type { AgentProfile } from '@tangle-network/agent-eval' -import type { DispatchContext, Scenario } from '@tangle-network/agent-eval/campaign' +import { type AgentProfile, CostLedger } from '@tangle-network/agent-eval' +import type { + CampaignCostMeter, + DispatchContext, + Scenario, +} from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' import { createIterableBackend } from '../backends' import type { AgentExecutionBackend, RuntimeStreamEvent } from '../types' @@ -75,27 +79,35 @@ function fakePersonaDriver(saw: { prompt?: string; calls: number }): AgentExecut }) } -const PROFILE = {} as AgentProfile -const WORKER_PROFILE = { tag: 'worker' } as unknown as AgentProfile -const PERSONA_PROFILE = { tag: 'persona' } as unknown as AgentProfile +const PROFILE = { model: { default: 'fake' } } as AgentProfile +const WORKER_PROFILE = { + model: { default: 'fake' }, + metadata: { tag: 'worker' }, +} as AgentProfile +const PERSONA_PROFILE = { + model: { default: 'fake-persona' }, + metadata: { tag: 'persona' }, +} as AgentProfile -function fakeCtx(): DispatchContext & { - observed: { costUsd: number; tokensIn: number; tokensOut: number } +function fakeCtx(costCeilingUsd?: number): DispatchContext & { + ledger: CostLedger } { - const observed = { costUsd: 0, tokensIn: 0, tokensOut: 0 } + const ledger = new CostLedger(costCeilingUsd === undefined ? {} : { costCeilingUsd }) + const cost: CampaignCostMeter = { + runPaidCall(input) { + return ledger.runPaidCall({ + ...input, + channel: input.channel ?? 'agent', + phase: 'persona-test', + tags: { cellId: 'persona-cell' }, + }) + }, + } return { - observed, + ledger, signal: new AbortController().signal, - cost: { - observe(usd: number) { - observed.costUsd += usd - }, - observeTokens(t: { input: number; output: number }) { - observed.tokensIn += t.input - observed.tokensOut += t.output - }, - }, - } as unknown as DispatchContext & { observed: typeof observed } + cost, + } as unknown as DispatchContext & { ledger: CostLedger } } describe('runPersonaConversation', () => { @@ -159,8 +171,7 @@ describe('runPersonaConversation', () => { persona: { kind: 'profile', profile: PERSONA_PROFILE }, backendFor: (_profile, role) => role === 'worker' ? fakeWorker(workerSaw) : fakePersonaDriver(personaSaw), - systemPromptOf: (p) => - (p as { tag?: string }).tag === 'persona' ? 'PERSONA-PROMPT' : 'WORKER-PROMPT', + systemPromptOf: (p) => (p.metadata?.tag === 'persona' ? 'PERSONA-PROMPT' : 'WORKER-PROMPT'), maxTurns: 4, }) // alternate, persona leads: persona, worker, persona, worker. @@ -217,7 +228,50 @@ describe('runPersonaDispatch (matrix adapter)', () => { ctx, ) expect(artifact).toBe(2) - expect(ctx.observed.tokensIn).toBe(20) - expect(ctx.observed.costUsd).toBeCloseTo(0.04, 5) + expect(ctx.ledger.list()).toEqual([ + expect.objectContaining({ + actor: 'persona-conversation', + model: 'fake', + inputTokens: 20, + outputTokens: 10, + actualCostUsd: 0.04, + costUsd: 0.04, + }), + ]) + }) + + it('refuses a capped conversation before the worker runs when no hard maximum is supplied', async () => { + const saw = { calls: 0 } as { prompt?: string; calls: number } + const dispatch = runPersonaDispatch({ + backendFor: () => fakeWorker(saw), + systemPromptOf: () => 'SYS', + personaOf: (scenario) => ({ kind: 'scripted', turns: scenario.turns }), + artifactOf: () => 0, + }) + const ctx = fakeCtx(1) + + await expect( + dispatch(PROFILE, { id: 'bounded', kind: 'persona', turns: ['q1'] }, ctx), + ).rejects.toThrow(/hard maximumCharge before execution/) + expect(saw.calls).toBe(0) + expect(ctx.ledger.list()).toHaveLength(0) + }) + + it('admits a capped conversation with an executor-enforced maximum', async () => { + const dispatch = runPersonaDispatch({ + backendFor: () => fakeWorker({ calls: 0 }), + systemPromptOf: () => 'SYS', + personaOf: (scenario) => ({ kind: 'scripted', turns: scenario.turns }), + artifactOf: (transcript) => transcript.length, + maximumCharge: { externallyEnforcedMaximumUsd: 0.05 }, + }) + const ctx = fakeCtx(1) + + await expect( + dispatch(PROFILE, { id: 'bounded', kind: 'persona', turns: ['q1'] }, ctx), + ).resolves.toBe(2) + expect(ctx.ledger.list()).toEqual([ + expect.objectContaining({ maximumCostUsd: 0.05, actualCostUsd: 0.02, costUsd: 0.02 }), + ]) }) }) diff --git a/src/conversation/run-persona.ts b/src/conversation/run-persona.ts index 6ce66aea..326b6b09 100644 --- a/src/conversation/run-persona.ts +++ b/src/conversation/run-persona.ts @@ -15,7 +15,7 @@ * `dispatchWithSurface` bridges. */ -import type { AgentProfile } from '@tangle-network/agent-eval' +import type { AgentProfile, MaximumCharge } from '@tangle-network/agent-eval' import type { DispatchContext, ProfileDispatchFn, @@ -208,6 +208,11 @@ export interface RunPersonaConfig { maxTurns?: (scenario: TScenario) => number seed?: (scenario: TScenario) => string workerName?: string + /** Provider- or executor-enforced maximum for the whole worker conversation. + * Required before execution when the enclosing campaign is cost-capped. */ + maximumCharge?: + | MaximumCharge + | ((worker: AgentProfile, scenario: TScenario) => MaximumCharge | undefined) } /** @@ -224,18 +229,37 @@ export function runPersonaDispatch( scenario: TScenario, ctx: DispatchContext, ): Promise => { - const result = await runPersonaConversation({ - worker, - persona: config.personaOf(scenario), - backendFor: config.backendFor, - systemPromptOf: config.systemPromptOf, - maxTurns: config.maxTurns?.(scenario), - seed: config.seed?.(scenario), + const model = worker.model?.default ?? 'unknown' + const maximumCharge = + typeof config.maximumCharge === 'function' + ? config.maximumCharge(worker, scenario) + : config.maximumCharge + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'persona-conversation', + model, signal: ctx.signal, - workerName: config.workerName, + ...(maximumCharge ? { maximumCharge } : {}), + execute: (executionSignal) => + runPersonaConversation({ + worker, + persona: config.personaOf(scenario), + backendFor: config.backendFor, + systemPromptOf: config.systemPromptOf, + maxTurns: config.maxTurns?.(scenario), + seed: config.seed?.(scenario), + signal: executionSignal, + workerName: config.workerName, + }), + receipt: (result) => ({ + model, + inputTokens: result.tokensIn, + outputTokens: result.tokensOut, + ...(result.costUsd > 0 ? { actualCostUsd: result.costUsd } : {}), + }), }) - ctx.cost.observe(result.costUsd, 'persona-conversation') - ctx.cost.observeTokens({ input: result.tokensIn, output: result.tokensOut }) + if (!paid.succeeded) throw paid.error + const result = paid.value return config.artifactOf(result.transcript, scenario) } } diff --git a/src/durable/spawn-journal.ts b/src/durable/spawn-journal.ts index 475d7dc3..742d999c 100644 --- a/src/durable/spawn-journal.ts +++ b/src/durable/spawn-journal.ts @@ -453,6 +453,7 @@ function addJournalSpend(a: Spend, b: Spend): Spend { iterations: a.iterations + b.iterations, tokens: { input: a.tokens.input + b.tokens.input, output: a.tokens.output + b.tokens.output }, usd: a.usd + b.usd, + ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), ms: a.ms + b.ms, } } diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index 2f9577ed..6d63722c 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -33,16 +33,41 @@ */ import { spawnSync } from 'node:child_process' -import { existsSync, readFileSync } from 'node:fs' -import { join } from 'node:path' -import type { AnalystFinding } from '@tangle-network/agent-eval' -import { type LocalHarness, runLocalHarness } from '../mcp/local-harness' +import { createHash } from 'node:crypto' +import { existsSync, readFileSync, rmSync } from 'node:fs' +import { join, resolve, sep } from 'node:path' +import type { + AnalystFinding, + CostLedger, + CostReceipt, + CostReceiptInput, + MaximumCharge, +} from '@tangle-network/agent-eval' +import type { AgentProfile, ReasoningEffort } from '@tangle-network/agent-interface' +import { + applyWorkspacePlan, + materializeProfile, + type WorkspacePlan, + type WorkspacePlanReceipt, +} from '@tangle-network/agent-profile-materialize' +import { + type CodexExecutionEvidence, + type CodexTokenUsage, + harnessInvocation, + type LocalHarness, + type LocalHarnessResult, + runLocalHarness, +} from '../mcp/local-harness' import type { CandidateGenerator } from './improvement-driver' const RAW_TRACE_ANALYST_ID = 'raw-trace-distiller' const RAW_TRACE_AREA = 'raw-trace-context' const RAW_TRACE_DIAGNOSIS_PATH = '.improve/raw-trace-diagnosis.md' +/** Dedicated ephemeral root for generic author-profile files. Every declared + * file must live below this root so cleanup cannot alter candidate-owned files. */ +export const AGENTIC_PROFILE_RESOURCE_ROOT = '.agent-runtime-profile-resources' + /** Outcome of verifying a candidate worktree. `feedback` (compiler errors, * failing test output) is fed into the next shot when `ok` is false. */ export interface VerifyResult { @@ -55,9 +80,113 @@ export interface VerifyResult { * throw). */ export type Verifier = (worktreePath: string) => Promise | VerifyResult +export interface AgenticGeneratorShotReceipt { + readonly schemaVersion: 1 + readonly generation: number | null + readonly candidateIndex: number | null + /** One-based shot number within this candidate. */ + readonly shot: number + readonly maxShots: number + readonly harness: LocalHarness + readonly model: string | null + readonly reasoningEffort: ReasoningEffort | null + readonly promptSha256: `sha256:${string}` + readonly startedAt: string + readonly completedAt: string + readonly durationMs: number + readonly exitCode: number | null + readonly timedOut: boolean + readonly killedBySignal: NodeJS.Signals | null + readonly stdoutBytes: number | null + readonly stdoutSha256: `sha256:${string}` | null + readonly stderrBytes: number | null + readonly stderrSha256: `sha256:${string}` | null + readonly usage: CodexTokenUsage | null + /** Digest of the exact profile-file workspace plan applied for this shot. */ + readonly profileWorkspacePlanDigest: string | null + readonly profileWorkspaceFileCount: number + /** Shared run-ledger call id for this exact shot. */ + readonly costCallId: string | null + /** Whether dollars came from the provider, the pricing table, or are unknown. */ + readonly costBasis: 'provider-reported' | 'estimated-pricing' | 'unknown' + readonly costUsd: number | null + /** True only for a provider-reported amount, never for a pricing estimate. */ + readonly costUsdKnown: boolean + readonly evidence: CodexExecutionEvidence | null + readonly error: { readonly name: string; readonly message: string } | null +} + +/** Frozen exact harness result for an author shot: full streams, process state, + * token usage, and execution-policy evidence. + * The `onShotCompleted` callback receives `null` when execution failed before + * the harness returned. */ +export type AgenticGeneratorShotExecution = Readonly< + Omit & { + readonly usage?: Readonly + readonly evidence?: Readonly> & { + readonly readDeniedPaths: ReadonlyArray + readonly policy: Readonly + } + } +> + +/** Worktree decision emitted before a completed shot is retried, accepted, or + * discarded. The callback runs while `worktreePath` is still available, so + * callers can persist the exact diff. */ +export type AgenticGeneratorShotDisposition = + | { + readonly kind: 'clean' + readonly worktreePath: string + } + | { + readonly kind: 'rejected' + readonly worktreePath: string + readonly stage: 'raw-trace-evidence' | 'verification' + readonly feedback: string | null + } + | { + readonly kind: 'accepted' + readonly worktreePath: string + readonly verified: boolean + } + | { + readonly kind: 'setup-error' + readonly worktreePath: string + readonly stage: 'worktree-inspection' | 'raw-trace-evidence' | 'verification' + readonly error: { readonly name: string; readonly message: string } + } + export interface AgenticGeneratorOptions { /** Local coding harness to run in the worktree. Default `claude`. */ harness?: LocalHarness + /** Author profile rendered through the canonical harness mapper. Required + * for reproducible Codex so model and reasoning settings are explicit. */ + profile?: AgentProfile + /** Run Codex with isolated configuration, exact prompt evidence, and required + * terminal token usage. Requires `harness: 'codex'` and `profile`. */ + codexReproducible?: boolean + /** Absolute paths reproducible Codex must not read. A function can derive + * candidate-specific paths after the driver creates its worktree. */ + codexReadDeniedPaths?: ReadonlyArray | ((worktreePath: string) => ReadonlyArray) + /** Awaited once for every attempted author shot, including process failures. + * The second argument preserves the exact harness result, including stdout + * and stderr, before worktree inspection or verification can reject the + * shot. Throwing aborts the candidate so evidence persistence fails closed. */ + onShotCompleted?: ( + receipt: AgenticGeneratorShotReceipt, + execution: AgenticGeneratorShotExecution | null, + ) => void | Promise + /** Awaited after worktree inspection and before the shot is accepted, + * retried, or discarded. Throwing aborts the candidate. */ + onShotDisposition?: ( + receipt: AgenticGeneratorShotReceipt, + disposition: AgenticGeneratorShotDisposition, + ) => void | Promise + /** Optional hard upper bound passed to the run-wide CostLedger before each + * author shot. This MUST be enforced by the provider or executor; a planning + * estimate is not an admissible bound. Omit for an uncapped ledger. A capped + * ledger rejects before model dispatch when this is absent. */ + maximumCharge?: MaximumCharge /** Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). */ timeoutMs?: number /** Build the harness task prompt from the report + findings. Override for @@ -78,6 +207,21 @@ export interface AgenticGeneratorOptions { /** Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. */ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateGenerator { const harness = opts.harness ?? 'claude' + if (opts.codexReproducible && harness !== 'codex') { + throw new Error("agenticGenerator: codexReproducible requires harness 'codex'") + } + if (opts.codexReproducible && !opts.profile) { + throw new Error('agenticGenerator: codexReproducible requires an explicit author profile') + } + if (opts.codexReadDeniedPaths && !opts.codexReproducible) { + throw new Error('agenticGenerator: codexReadDeniedPaths requires codexReproducible') + } + if (opts.maximumCharge && !opts.codexReproducible) { + throw new Error('agenticGenerator: maximumCharge requires codexReproducible') + } + const profileResourcePlan = opts.codexReproducible + ? authorProfileResourcePlan(opts.profile as AgentProfile) + : null const buildPrompt = opts.buildPrompt ?? defaultBuildPrompt const run = opts.runHarness ?? runLocalHarness const dirty = opts.isDirty ?? worktreeDirty @@ -85,8 +229,32 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG return { kind: `agentic:${harness}`, - async generate({ worktreePath, report, findings, maxShots, signal }) { - const basePrompt = buildPrompt({ report, findings }) + // The seed repo + (in rawTraceContext mode) the raw-trace filesystem context + // are the change signal — an agentic coder proposes from them even when the + // distiller yielded zero findings. Without this, the improvementDriver's + // empty-findings guard short-circuits and generates ZERO candidates on the + // first (and, for a single-generation run, only) proposal round. + proposesWithoutFindings: true, + async generate({ + worktreePath, + report, + findings, + maxShots, + signal, + generation, + candidateIndex, + costLedger, + costPhase, + }) { + if (opts.codexReproducible && !costLedger) { + throw new Error( + 'agenticGenerator: reproducible Codex requires the run-wide CostLedger supplied by agent-eval 0.117+', + ) + } + const basePrompt = appendProfileResourcePaths( + buildPrompt({ report, findings }), + profileResourcePlan, + ) const needsRawTraceEvidence = requiresRawTraceEvidence(findings) const shots = Math.max(1, maxShots) // Feedback appended to the base prompt for the NEXT shot — empty on shot 0. @@ -94,23 +262,157 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG for (let shot = 0; shot < shots; shot++) { if (signal.aborted) break - await run({ + const taskPrompt = attemptNote ? `${basePrompt}\n\n${attemptNote}` : basePrompt + const invocation = opts.profile + ? harnessInvocation(harness, opts.profile, taskPrompt, { + dangerouslySkipPermissions: harness === 'claude', + ...(opts.codexReproducible ? { codexReproducible: true } : {}), + }) + : undefined + const exactPrompt = invocation?.prompt ?? taskPrompt + const readDeniedPaths = + typeof opts.codexReadDeniedPaths === 'function' + ? opts.codexReadDeniedPaths(worktreePath) + : opts.codexReadDeniedPaths + const startedAt = new Date() + let harnessResult: LocalHarnessResult | null = null + let profileWorkspaceReceipt: WorkspacePlanReceipt | null = null + let costReceipt: CostReceipt | null = null + let costCallId: string | null = null + let shotError: Error | null = null + try { + const execute = async (executionSignal: AbortSignal): Promise => { + harnessResult = await withAuthorProfileResources( + profileResourcePlan, + worktreePath, + async (receipt) => { + profileWorkspaceReceipt = receipt + const result = await run({ + harness, + cwd: worktreePath, + taskPrompt, + ...(invocation + ? { invocation: { command: invocation.command, args: invocation.args } } + : {}), + // The candidate worktree is isolated and must be editable without an + // interactive permission prompt. Other runLocalHarness callers remain + // permission-safe by default. + dangerouslySkipPermissions: harness === 'claude', + ...(opts.codexReproducible ? { codexReproducible: true } : {}), + ...(readDeniedPaths ? { codexReadDeniedPaths: readDeniedPaths } : {}), + timeoutMs: opts.timeoutMs, + signal: executionSignal, + }) + // Assign before profile cleanup so a cleanup failure still + // settles the model usage already returned by the harness. + harnessResult = result + return result + }, + ) + const failure = shotFailure(harnessResult, exactPrompt, opts.codexReproducible === true) + if (failure) throw failure + return harnessResult + } + + if (opts.codexReproducible) { + const ledger = costLedger as CostLedger + const model = opts.profile?.model?.default + if (!model) { + throw new Error('agenticGenerator: reproducible Codex requires profile.model.default') + } + const paid = await ledger.runPaidCall({ + channel: 'driver', + phase: costPhase ?? 'search.proposal', + actor: `agentic-generator:${harness}`, + model, + tags: { + generation: String(generation ?? -1), + candidateIndex: String(candidateIndex ?? -1), + shot: String(shot + 1), + }, + signal, + ...(opts.maximumCharge ? { maximumCharge: opts.maximumCharge } : {}), + execute, + receipt: (result) => costReceiptFromHarness(result, model), + receiptFromError: () => + harnessResult?.usage ? costReceiptFromHarness(harnessResult, model) : undefined, + }) + costCallId = paid.callId ?? null + costReceipt = paid.receipt ?? null + if (!paid.succeeded) throw paid.error + harnessResult = paid.value + } else { + harnessResult = await execute(signal) + } + } catch (cause) { + shotError = cause instanceof Error ? cause : new Error(String(cause)) + } + const execution = shotExecutionSnapshot(harnessResult) + const receipt = shotReceipt({ + generation, + candidateIndex, + shot, + maxShots: shots, harness, - cwd: worktreePath, - taskPrompt: attemptNote ? `${basePrompt}\n\n${attemptNote}` : basePrompt, - timeoutMs: opts.timeoutMs, - signal, + profile: opts.profile, + prompt: exactPrompt, + startedAt, + completedAt: new Date(), + result: execution, + profileWorkspaceReceipt, + costCallId, + costReceipt, + error: shotError, }) + await emitShotReceipt(opts.onShotCompleted, receipt, execution, shotError) + + if (!execution) { + throw new Error('agenticGenerator: author shot completed without a harness result') + } + + let worktreeChanged: boolean + try { + worktreeChanged = dirty(worktreePath) + } catch (cause) { + return rethrowShotSetupError( + opts.onShotDisposition, + receipt, + worktreePath, + 'worktree-inspection', + cause, + ) + } // The worktree IS the signal: no edits ⇒ tell the next shot to act. - if (!dirty(worktreePath)) { + if (!worktreeChanged) { + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'clean', + worktreePath, + }) attemptNote = EMPTY_TREE_NOTE continue } if (needsRawTraceEvidence) { - const problem = rawTraceEvidenceProblem(worktreePath, findings) + let problem: string | null + try { + problem = rawTraceEvidenceProblem(worktreePath, findings) + } catch (cause) { + return rethrowShotSetupError( + opts.onShotDisposition, + receipt, + worktreePath, + 'raw-trace-evidence', + cause, + ) + } if (problem) { + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'rejected', + worktreePath, + stage: 'raw-trace-evidence', + feedback: problem, + }) attemptNote = problem continue } @@ -119,12 +421,39 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG // Dirty: with no verifier the diff IS the candidate (we trust the diff, // not the harness's stdout). With a verifier the candidate must pass it. if (!verify) { + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'accepted', + worktreePath, + verified: false, + }) return { applied: true, summary: summarize(findings) } } - const result = await verify(worktreePath) + let result: VerifyResult + try { + result = await verify(worktreePath) + } catch (cause) { + return rethrowShotSetupError( + opts.onShotDisposition, + receipt, + worktreePath, + 'verification', + cause, + ) + } if (result.ok) { + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'accepted', + worktreePath, + verified: true, + }) return { applied: true, summary: summarize(findings) } } + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'rejected', + worktreePath, + stage: 'verification', + feedback: result.feedback ?? null, + }) // Dirty but failing — resume next shot atop these edits with the error. attemptNote = failureNote(result.feedback) } @@ -135,6 +464,300 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG } } +function authorProfileResourcePlan(profile: AgentProfile): WorkspacePlan | null { + const resources = profile.resources + if (!resources) return null + const unsupportedKinds = [ + resources.tools?.length ? 'tools' : null, + resources.skills?.length ? 'skills' : null, + resources.agents?.length ? 'agents' : null, + ].filter((kind): kind is string => kind !== null) + if (unsupportedKinds.length > 0) { + throw new Error( + `agenticGenerator: reproducible Codex author resources support files only; unsupported: ${unsupportedKinds.join(', ')}`, + ) + } + if (!resources.files || resources.files.length === 0) return null + + const plan = materializeProfile( + { + name: profile.name, + resources: { files: resources.files }, + }, + 'codex', + ) + if (plan.unsupported.length > 0) { + throw new Error( + `agenticGenerator: author profile files could not be materialized: ${plan.unsupported.map((item) => item.reason).join('; ')}`, + ) + } + if (Object.keys(plan.env).length > 0 || plan.flags.length > 0) { + throw new Error( + 'agenticGenerator: generic author profile files unexpectedly changed spawn values', + ) + } + + const virtualRoot = resolve('/', AGENTIC_PROFILE_RESOURCE_ROOT) + const seen = new Set() + for (const file of plan.files) { + const target = resolve('/', file.relPath) + if (!target.startsWith(`${virtualRoot}${sep}`)) { + throw new Error( + `agenticGenerator: author profile file must be below ${AGENTIC_PROFILE_RESOURCE_ROOT}: ${file.relPath}`, + ) + } + if (seen.has(target)) { + throw new Error(`agenticGenerator: duplicate author profile file path: ${file.relPath}`) + } + seen.add(target) + } + return plan +} + +function appendProfileResourcePaths(prompt: string, plan: WorkspacePlan | null): string { + if (!plan) return prompt + return [ + prompt, + '', + 'Profile resource files available for this shot:', + ...plan.files.map((file) => `- ${file.relPath}`), + ].join('\n') +} + +async function withAuthorProfileResources( + plan: WorkspacePlan | null, + worktreePath: string, + run: (receipt: WorkspacePlanReceipt | null) => Promise, +): Promise { + if (!plan) return run(null) + + const rootPath = resolve(worktreePath, AGENTIC_PROFILE_RESOURCE_ROOT) + if (existsSync(rootPath)) { + throw new Error( + `agenticGenerator: ephemeral author profile root already exists: ${AGENTIC_PROFILE_RESOURCE_ROOT}`, + ) + } + + let value: T | undefined + let primaryError: unknown + try { + const receipt = applyWorkspacePlan(plan, worktreePath) + value = await run(receipt) + } catch (cause) { + primaryError = cause + } + + let cleanupError: unknown + try { + rmSync(rootPath, { recursive: true, force: true }) + if (existsSync(rootPath)) { + throw new Error( + `agenticGenerator: ephemeral author profile root survived cleanup: ${AGENTIC_PROFILE_RESOURCE_ROOT}`, + ) + } + } catch (cause) { + cleanupError = cause + } + + if (primaryError !== undefined && cleanupError !== undefined) { + throw new AggregateError( + [primaryError, cleanupError], + 'agenticGenerator: author shot and profile resource cleanup both failed', + ) + } + if (primaryError !== undefined) throw primaryError + if (cleanupError !== undefined) throw cleanupError + return value as T +} + +async function emitShotReceipt( + callback: AgenticGeneratorOptions['onShotCompleted'], + receipt: AgenticGeneratorShotReceipt, + execution: AgenticGeneratorShotExecution | null, + primaryError: Error | null, +): Promise { + try { + await callback?.(receipt, execution) + } catch (callbackError) { + if (primaryError !== null) { + throw new AggregateError( + [primaryError, callbackError], + 'agenticGenerator: author shot failed and its receipt could not be persisted', + ) + } + throw callbackError + } + if (primaryError !== null) throw primaryError +} + +async function emitShotDisposition( + callback: AgenticGeneratorOptions['onShotDisposition'], + receipt: AgenticGeneratorShotReceipt, + disposition: AgenticGeneratorShotDisposition, +): Promise { + await callback?.(receipt, disposition) +} + +async function rethrowShotSetupError( + callback: AgenticGeneratorOptions['onShotDisposition'], + receipt: AgenticGeneratorShotReceipt, + worktreePath: string, + stage: Extract['stage'], + cause: unknown, +): Promise { + const error = cause instanceof Error ? cause : new Error(String(cause)) + try { + await emitShotDisposition(callback, receipt, { + kind: 'setup-error', + worktreePath, + stage, + error: { name: error.name, message: error.message }, + }) + } catch (callbackError) { + throw new AggregateError( + [cause, callbackError], + 'agenticGenerator: shot processing failed and its worktree disposition could not be persisted', + ) + } + throw cause +} + +function shotExecutionSnapshot( + result: LocalHarnessResult | null, +): AgenticGeneratorShotExecution | null { + if (result === null) return null + const usage = result.usage ? Object.freeze({ ...result.usage }) : undefined + const evidence = result.evidence + ? Object.freeze({ + ...result.evidence, + readDeniedPaths: Object.freeze([...result.evidence.readDeniedPaths]), + policy: Object.freeze({ ...result.evidence.policy }), + }) + : undefined + return Object.freeze({ + exitCode: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + killedBySignal: result.killedBySignal, + durationMs: result.durationMs, + timedOut: result.timedOut, + ...(usage ? { usage } : {}), + ...(evidence ? { evidence } : {}), + }) +} + +function shotReceipt(input: { + readonly generation: number | undefined + readonly candidateIndex: number | undefined + readonly shot: number + readonly maxShots: number + readonly harness: LocalHarness + readonly profile: AgentProfile | undefined + readonly prompt: string + readonly startedAt: Date + readonly completedAt: Date + readonly result: AgenticGeneratorShotExecution | null + readonly profileWorkspaceReceipt: WorkspacePlanReceipt | null + readonly costCallId: string | null + readonly costReceipt: CostReceipt | null + readonly error: unknown +}): AgenticGeneratorShotReceipt { + const error = input.error + ? { + name: input.error instanceof Error ? input.error.name : 'Error', + message: input.error instanceof Error ? input.error.message : String(input.error), + } + : null + const result = input.result + const costBasis = costBasisFor(input.costReceipt) + return { + schemaVersion: 1, + generation: input.generation ?? null, + candidateIndex: input.candidateIndex ?? null, + shot: input.shot + 1, + maxShots: input.maxShots, + harness: input.harness, + model: input.profile?.model?.default ?? null, + reasoningEffort: input.profile?.model?.reasoningEffort ?? null, + promptSha256: sha256(input.prompt), + startedAt: input.startedAt.toISOString(), + completedAt: input.completedAt.toISOString(), + durationMs: result?.durationMs ?? input.completedAt.getTime() - input.startedAt.getTime(), + exitCode: result?.exitCode ?? null, + timedOut: result?.timedOut ?? false, + killedBySignal: result?.killedBySignal ?? null, + stdoutBytes: result ? Buffer.byteLength(result.stdout) : null, + stdoutSha256: result ? sha256(result.stdout) : null, + stderrBytes: result ? Buffer.byteLength(result.stderr) : null, + stderrSha256: result ? sha256(result.stderr) : null, + usage: result?.usage ? { ...result.usage } : null, + profileWorkspacePlanDigest: input.profileWorkspaceReceipt?.workspacePlanDigest ?? null, + profileWorkspaceFileCount: input.profileWorkspaceReceipt?.written.length ?? 0, + costCallId: input.costCallId, + costBasis, + costUsd: costBasis === 'unknown' ? null : (input.costReceipt?.costUsd ?? null), + costUsdKnown: costBasis === 'provider-reported', + evidence: result?.evidence + ? { + ...result.evidence, + readDeniedPaths: [...result.evidence.readDeniedPaths], + policy: { ...result.evidence.policy }, + } + : null, + error, + } +} + +function costBasisFor(receipt: CostReceipt | null): AgenticGeneratorShotReceipt['costBasis'] { + if (receipt === null || receipt.costUnknown) return 'unknown' + return receipt.actualCostUsd === undefined ? 'estimated-pricing' : 'provider-reported' +} + +function shotFailure( + result: LocalHarnessResult, + exactPrompt: string, + codexReproducible: boolean, +): Error | null { + if (result.timedOut) { + return new Error('agenticGenerator: author shot timed out') + } + if (result.killedBySignal) { + return new Error(`agenticGenerator: author shot was killed by ${result.killedBySignal}`) + } + if (result.exitCode !== 0) { + return new Error(`agenticGenerator: author shot exited with code ${String(result.exitCode)}`) + } + if (!codexReproducible) return null + if (!result.usage || !result.evidence) { + return new Error( + 'agenticGenerator: reproducible Codex shot completed without usage or execution evidence', + ) + } + const expectedPromptSha256 = sha256(exactPrompt).slice('sha256:'.length) + if (result.evidence.requestedPromptSha256 !== expectedPromptSha256) { + return new Error( + 'agenticGenerator: reproducible Codex prompt evidence does not match the exact authored prompt', + ) + } + return null +} + +function costReceiptFromHarness(result: LocalHarnessResult, model: string): CostReceiptInput { + if (!result.usage) { + throw new Error('agenticGenerator: author shot did not report terminal token usage') + } + return { + model, + inputTokens: result.usage.inputTokens - result.usage.cachedInputTokens, + outputTokens: result.usage.outputTokens, + ...(result.usage.cachedInputTokens > 0 ? { cachedTokens: result.usage.cachedInputTokens } : {}), + } +} + +function sha256(value: string): `sha256:${string}` { + return `sha256:${createHash('sha256').update(value).digest('hex')}` +} + /** Turn the analyst's findings (+ optional report) into a concrete coder task. */ function defaultBuildPrompt(args: { report: unknown; findings: AnalystFinding[] }): string { const lines: string[] = [ diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 758770fe..898030cd 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -2,8 +2,8 @@ * `improve()` default-proposer resolution proof. * * The regression this guards: `improve()` maps each surface to a default - * `SurfaceProposer` — `prompt → gepaProposer`, `skills → skillOptProposer`. - * Both proposers are factories exported from `@tangle-network/agent-eval/campaign`. + * `SurfaceProposer` — `prompt → gepaProposer`, `skills → skillOptProposer`, + * `memory → memoryCurationProposer`. * If either import resolves to `undefined` (a substrate export drift), the facade * does not fail at module load — it fails at CALL time, the first time a caller * names that surface. So a green typecheck is not enough; this test drives the @@ -17,7 +17,18 @@ * sees a real backend rather than a silent-zero stub. */ -import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import { + gitWorktreeAdapter, + type Worktree, + type WorktreeAdapter, +} from '@tangle-network/agent-eval/campaign' +import type { + CodeSurface, + DispatchContext, + JudgeConfig, + Scenario, + SurfaceProposer, +} from '@tangle-network/agent-eval/contract' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { ConfigError } from '../errors' @@ -40,17 +51,45 @@ const judge: JudgeConfig<{ text: string }, Scenario> = { score: () => ({ dimensions: { q: 0.5 }, composite: 0.5, notes: '' }), } -// The agent reports a token-bearing cost so the backend-integrity guard treats -// it as a real backend. Without `ctx.cost.observeTokens`, the default -// `expectUsage: 'assert'` reads the cell as a silent-zero stub and throws. +const improvementJudge: JudgeConfig<{ text: string }, Scenario> = { + name: 'improvement-judge', + dimensions: [{ key: 'q', description: 'contains the measured improvement marker' }], + score: ({ artifact }) => { + const score = artifact.text.includes('improved') ? 1 : 0 + return { dimensions: { q: score }, composite: score, notes: '' } + }, +} + +async function paidStubCall( + ctx: DispatchContext, + actor: string, + execute: () => T | Promise, +): Promise { + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor, + model: 'stub-model', + maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, + execute: async () => execute(), + receipt: () => ({ + model: 'stub-model', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.0001, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +// The fake dispatch enters the same paid-call path as a real backend so the +// backend-integrity check sees its explicit token and cost receipt. async function stubAgent( surface: unknown, _scenario: Scenario, ctx: DispatchContext, ): Promise<{ text: string }> { - ctx.cost.observe(0.0001, 'stub-agent') - ctx.cost.observeTokens({ input: 1, output: 1 }) - return { text: String(surface) } + return paidStubCall(ctx, 'stub-agent', () => ({ text: String(surface) })) } const promptProfile = (): AgentProfile => ({ @@ -88,6 +127,7 @@ describe('improve() — default proposer resolution (substrate export drift guar scenarios, judge, agent: stubAgent, + skills: { document: '# Fixture skill\n\nCheck the result.\n' }, }) expect(result.gateDecision).toBe('hold') @@ -95,6 +135,157 @@ describe('improve() — default proposer resolution (substrate export drift guar expect(result.profile.resources?.skills).toEqual([]) }) + it("surface 'skills' fails loud when the default document optimizer receives only resource refs", async () => { + await expect( + improve(skillProfile(), [], { + surface: 'skills', + gate: 'none', + scenarios, + judge, + agent: stubAgent, + }), + ).rejects.toThrow(/requires opts\.skills\.document/) + }) + + it("surface 'memory' resolves memoryCurationProposer and requires a real baseline", async () => { + const result = await improve(promptProfile(), [], { + surface: 'memory', + gate: 'none', + scenarios, + judge, + agent: stubAgent, + memory: { document: '# Durable memory\n' }, + }) + + expect(result.gateDecision).toBe('hold') + expect(result.shipped).toBe(false) + await expect( + improve(promptProfile(), [], { + surface: 'memory', + gate: 'none', + scenarios, + judge, + agent: stubAgent, + }), + ).rejects.toThrow(/requires opts\.memory\.document/) + }) + + it("surface 'memory' curates seed findings and awaits persistence of the promoted document", async () => { + const baseline = '# Durable memory\n' + let writtenBack: string | null = null + const result = await improve(promptProfile(), [{ claim: 'improved verification lesson' }], { + surface: 'memory', + scenarios, + judge: improvementJudge, + agent: stubAgent, + memory: { + document: baseline, + writeBack: async (winner) => { + await Promise.resolve() + writtenBack = winner + }, + }, + promotionGate: { + name: 'test-ship', + decide: async () => ({ + decision: 'ship', + reasons: ['test candidate'], + contributingGates: [], + }), + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }) + + expect(result.shipped).toBe(true) + expect(writtenBack).toContain('improved verification lesson') + expect(result.profile).toEqual(promptProfile()) + }) + + it("surface 'memory' rejects a shipped non-text winner before persistence", async () => { + let writeCalls = 0 + const nonTextSurface: CodeSurface = { + kind: 'code', + worktreeRef: 'not-a-memory-document', + baseRef: 'main', + baseCommit: 'a'.repeat(40), + baseTree: 'b'.repeat(40), + candidateCommit: 'c'.repeat(40), + candidateTree: 'd'.repeat(40), + patch: { + format: 'git-diff-binary', + sha256: `sha256:${'e'.repeat(64)}`, + byteLength: 1, + }, + } + const surfaceKindJudge: JudgeConfig<{ text: string }, Scenario> = { + name: 'surface-kind-judge', + dimensions: [{ key: 'q', description: 'candidate is code-tier' }], + score: ({ artifact }) => { + const score = artifact.text === 'code' ? 1 : 0 + return { dimensions: { q: score }, composite: score, notes: '' } + }, + } + const malformedMemory: SurfaceProposer = { + kind: 'malformed-memory', + propose: async () => [ + { + surface: nonTextSurface, + label: 'wrong-tier', + rationale: 'test malformed winner', + }, + ], + } + await expect( + improve(promptProfile(), [], { + surface: 'memory', + scenarios, + judge: surfaceKindJudge, + agent: async (surface, _scenario, ctx) => { + return paidStubCall(ctx, 'stub-agent', () => ({ + text: typeof surface === 'string' ? 'text' : surface.kind, + })) + }, + memory: { + document: '# Durable memory\n', + writeBack: () => { + writeCalls += 1 + }, + }, + generator: malformedMemory, + promotionGate: { + name: 'test-ship', + decide: async () => ({ + decision: 'ship', + reasons: ['test candidate'], + contributingGates: [], + }), + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }), + ).rejects.toThrow(/winner must be text/) + expect(writeCalls).toBe(0) + }) + + it("surface 'memory' never writes back a held candidate", async () => { + let writeCalls = 0 + const result = await improve(promptProfile(), [], { + surface: 'memory', + gate: 'none', + scenarios, + judge, + agent: stubAgent, + memory: { + document: '# Durable memory\n', + writeBack: () => { + writeCalls += 1 + }, + }, + }) + + expect(result.shipped).toBe(false) + expect(writeCalls).toBe(0) + }) + it('a real runDir makes the loop durable: provenance lands on the filesystem', async () => { const { mkdtempSync, existsSync, rmSync } = await import('node:fs') const { tmpdir } = await import('node:os') @@ -149,7 +340,7 @@ describe('improve() — default proposer resolution (substrate export drift guar } const skillProfileWithRef = (): AgentProfile => ({ name: 'fixture-agent', - resources: { skills: [{ path: 'or-skills.md' } as never] }, + resources: { skills: [{ kind: 'github', path: 'or-skills.md' }] }, }) const result = await improve(skillProfileWithRef(), [], { @@ -157,9 +348,7 @@ describe('improve() — default proposer resolution (substrate export drift guar scenarios, judge: docJudge, agent: async (surface, _s, ctx) => { - ctx.cost.observe(0.0001, 'stub') - ctx.cost.observeTokens({ input: 1, output: 1 }) - return { doc: String(surface) } + return paidStubCall(ctx, 'stub', () => ({ doc: String(surface) })) }, generator: stubProposer as never, skills: { @@ -180,11 +369,130 @@ describe('improve() — default proposer resolution (substrate export drift guar } }) + it.each([ + { + surface: 'subagents' as const, + profile: { name: 'fixture-agent', subagents: {} }, + winner: JSON.stringify({ reviewer: { prompt: 'improved review instructions' } }), + read: (profile: AgentProfile) => profile.subagents?.reviewer?.prompt, + expected: 'improved review instructions', + }, + { + surface: 'agent-profile' as const, + profile: { name: 'fixture-agent', prompt: { systemPrompt: 'baseline' } }, + winner: JSON.stringify({ + name: 'fixture-agent', + prompt: { systemPrompt: 'improved whole profile' }, + }), + read: (profile: AgentProfile) => profile.prompt?.systemPrompt, + expected: 'improved whole profile', + }, + ])('applies a valid shipped $surface winner to the AgentProfile', async (fixture) => { + const result = await improve(fixture.profile, [{ finding: 'surface needs improvement' }], { + surface: fixture.surface, + scenarios, + judge: improvementJudge, + agent: stubAgent, + generator: { + kind: `stub-${fixture.surface}`, + propose: async () => [ + { surface: fixture.winner, label: 'candidate', rationale: 'test candidate' }, + ], + }, + promotionGate: { + name: 'test-ship', + decide: async () => ({ + decision: 'ship', + reasons: ['test candidate'], + contributingGates: [], + }), + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }) + + expect(result.shipped).toBe(true) + expect(fixture.read(result.profile)).toEqual(fixture.expected) + }) + + it('rejects a shipped config that cannot form a valid AgentProfile', async () => { + await expect( + improve( + { name: 'fixture-agent', subagents: {} }, + [{ finding: 'surface needs improvement' }], + { + surface: 'subagents', + scenarios, + judge: improvementJudge, + agent: stubAgent, + generator: { + kind: 'stub-invalid-subagent', + propose: async () => [ + { + surface: JSON.stringify({ reviewer: { prompt: 'improved', maxSteps: 'many' } }), + label: 'candidate', + rationale: 'invalid candidate', + }, + ], + }, + promotionGate: { + name: 'test-ship', + decide: async () => ({ + decision: 'ship', + reasons: ['exercise validation'], + contributingGates: [], + }), + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }, + ), + ).rejects.toThrow(/valid AgentProfile/) + }) + + it('forwards evaluation controls to the shared self-improvement loop', async () => { + const progressKinds: string[] = [] + let provenanceCalls = 0 + let decisionCalls = 0 + const result = await improve(promptProfile(), [{ finding: 'prompt needs improvement' }], { + surface: 'prompt', + scenarios, + judge: improvementJudge, + agent: stubAgent, + generator: { + kind: 'stub-controls', + propose: async () => [ + { surface: 'improved prompt', label: 'candidate', rationale: 'test candidate' }, + ], + }, + promotionGate: { + name: 'test-hold', + decide: async () => { + decisionCalls += 1 + return { decision: 'hold', reasons: ['test hold'], contributingGates: [] } + }, + }, + onProgress: (event) => progressKinds.push(event.kind), + onProvenance: () => { + provenanceCalls += 1 + }, + collectWorkerRecords: () => [], + expectUsage: 'assert', + captureSource: 'eval-run', + autoOnPromote: 'none', + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }) + + expect(result.gateDecision).toBe('hold') + expect(decisionCalls).toBe(1) + expect(provenanceCalls).toBe(1) + expect(progressKinds).toContain('baseline.completed') + expect(progressKinds).toContain('gate.decided') + }) + it('a surface with no zero-config default still fails loud with ConfigError', async () => { - // The default-proposer map covers prompt + skills only; the config surfaces - // (tools/mcp/hooks/code) require a caller-supplied generator. This is the + // Prompt, skills, and memory have defaults; config surfaces require a + // caller-supplied generator. This is the // designed boundary the proposer migration must NOT erase. - const configSurfaces: ImproveSurface[] = ['tools', 'mcp', 'hooks', 'code'] + const configSurfaces: ImproveSurface[] = ['tools', 'mcp', 'hooks', 'subagents', 'agent-profile'] for (const surface of configSurfaces) { await expect( improve(promptProfile(), [], { surface, gate: 'none', scenarios, judge, agent: stubAgent }), @@ -192,6 +500,53 @@ describe('improve() — default proposer resolution (substrate export drift guar } }) + it.each([ + 'text', + 'external-code', + ] as const)("surface 'code' rejects a top-level proposer before it can return $surfaceKind", async (surfaceKind) => { + let proposerCalls = 0 + const externalCode: CodeSurface = { + kind: 'code', + worktreeRef: '/tmp/externally-owned-code-surface', + baseRef: 'main', + baseCommit: 'a'.repeat(40), + baseTree: 'b'.repeat(40), + candidateCommit: 'c'.repeat(40), + candidateTree: 'd'.repeat(40), + patch: { + format: 'git-diff-binary', + sha256: `sha256:${'e'.repeat(64)}`, + byteLength: 1, + }, + } + const externalProposer: SurfaceProposer = { + kind: 'externally-owned-code-test', + async propose() { + proposerCalls += 1 + return [ + { + surface: surfaceKind === 'text' ? 'not-a-code-surface' : externalCode, + label: 'unsafe external result', + rationale: 'exercise code ownership validation', + }, + ] + }, + } + + await expect( + improve(promptProfile(), [], { + surface: 'code', + scenarios, + judge, + agent: stubAgent, + generator: externalProposer, + code: { repoRoot: '/tmp/must-not-be-opened' }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }), + ).rejects.toThrow(/forbids opts\.generator/) + expect(proposerCalls).toBe(0) + }) + it('the default generation distiller feeds real failures to the next proposal round', async () => { // Judge fails scenario 'b' with a distinctive reason; everything else is perfect. const failingJudge: JudgeConfig<{ text: string }, Scenario> = { @@ -221,17 +576,19 @@ describe('improve() — default proposer resolution (substrate export drift guar }) expect(typeof result.gateDecision).toBe('string') expect(findingsSeen.length).toBeGreaterThanOrEqual(2) - // Generation 1 proposes from the static seed; generation 2 must propose from the - // DISTILLED failures of generation 1's cells (scenario 'b' + the judge's reason). - expect(JSON.stringify(findingsSeen[0])).toContain('static-seed-finding') - const secondRound = JSON.stringify(findingsSeen[1]) - expect(secondRound).toContain('"scenario":"b"') - expect(secondRound).toContain('not a permutation') + // Current selfImprove analyzes the baseline before generation 1, so every + // proposal round starts from measured failures instead of wasting a round + // on the static seed. + for (const seen of findingsSeen) { + const round = JSON.stringify(seen) + expect(round).toContain('"scenario":"b"') + expect(round).toContain('not a permutation') + } }) it("surface 'code' + opts.code assembles the worktree pipeline and measures a candidate", async () => { const { execSync } = await import('node:child_process') - const { mkdtempSync, rmSync, writeFileSync } = await import('node:fs') + const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') const { tmpdir } = await import('node:os') const { join } = await import('node:path') const repoRoot = mkdtempSync(join(tmpdir(), 'improve-code-')) @@ -247,16 +604,22 @@ describe('improve() — default proposer resolution (substrate export drift guar // Byte-producer stub via the designed test seam: writes a change into the // candidate worktree; the driver finalizes it into a CodeSurface. let generatorCalls = 0 + const startingContents: string[] = [] const measured: unknown[] = [] const result = await improve(promptProfile(), [{ finding: 'module.txt is stale' }], { surface: 'code', scenarios, - judge, + judge: improvementJudge, agent: async (surface, _scenario, ctx) => { - ctx.cost.observe(0.0001, 'stub-agent') - ctx.cost.observeTokens({ input: 1, output: 1 }) - measured.push(surface) - return { text: 'ok' } + return paidStubCall(ctx, 'stub-agent', () => { + measured.push(surface) + if (typeof surface !== 'object' || surface === null || !('worktreeRef' in surface)) { + throw new Error('expected code surface') + } + return { + text: readFileSync(join(String(surface.worktreeRef), 'module.txt'), 'utf8'), + } + }) }, code: { repoRoot, @@ -264,25 +627,354 @@ describe('improve() — default proposer resolution (substrate export drift guar kind: 'stub', async generate({ worktreePath }) { generatorCalls += 1 - writeFileSync(join(worktreePath, 'module.txt'), 'improved contents\n') + const current = readFileSync(join(worktreePath, 'module.txt'), 'utf8') + startingContents.push(current) + writeFileSync( + join(worktreePath, 'module.txt'), + `${current}improved ${generatorCalls}\n`, + ) return { applied: true, summary: 'stub improvement' } }, }, }, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + promotionGate: { + name: 'test-hold', + decide: async () => ({ + decision: 'hold', + reasons: ['exercise non-promoted candidate retention'], + contributingGates: [], + }), + }, + budget: { generations: 2, populationSize: 1, holdoutFraction: 0.25 }, }) // The facade assembled a real proposer: the stub produced candidates, the // loop measured them (code surfaces reached the agent), and the gate decided. - expect(generatorCalls).toBeGreaterThanOrEqual(1) - expect(typeof result.gateDecision).toBe('string') - const sawCodeSurface = measured.some( + expect(generatorCalls).toBe(2) + expect(startingContents).toEqual(['baseline contents\n', 'baseline contents\nimproved 1\n']) + expect(result.gateDecision).toBe('hold') + const codeSurfaces = measured.filter( (m) => typeof m === 'object' && m !== null && 'worktreeRef' in (m as Record), ) - expect(sawCodeSurface).toBe(true) + expect(codeSurfaces.length).toBeGreaterThanOrEqual(2) + expect( + codeSurfaces.some((surface) => + String((surface as { worktreeRef: string }).worktreeRef).includes('incumbent-baseline'), + ), + ).toBe(true) // A code winner is a worktree ref, not a profile field — profile unchanged. expect(result.profile.prompt?.systemPrompt).toBe('be careful') + expect(result.shipped).toBe(false) + if (typeof result.raw.winner.surface === 'string') { + throw new Error('expected code winner') + } + expect(readFileSync(join(result.raw.winner.surface.worktreeRef, 'module.txt'), 'utf8')).toBe( + 'baseline contents\nimproved 1\n', + ) + expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength(2) + await result.dispose() + await result.dispose() + expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength(1) + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it("surface 'code' keeps the baseline worktree when a baseline-only run returns it", async () => { + const { execSync } = await import('node:child_process') + const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') + const { tmpdir } = await import('node:os') + const { join } = await import('node:path') + const repoRoot = mkdtempSync(join(tmpdir(), 'improve-code-baseline-')) + const git = (cmd: string) => execSync(`git ${cmd}`, { cwd: repoRoot, stdio: 'pipe' }) + try { + git('init -q -b main') + git('config user.email improve@test.local') + git('config user.name improve-test') + writeFileSync(join(repoRoot, 'module.txt'), 'baseline contents\n') + git('add module.txt') + git('commit -qm baseline') + + const result = await improve(promptProfile(), [], { + surface: 'code', + gate: 'none', + scenarios, + judge, + agent: async (surface, _scenario, ctx) => { + return paidStubCall(ctx, 'stub-agent', () => { + if (typeof surface !== 'object' || surface === null || !('worktreeRef' in surface)) { + throw new Error('expected code surface') + } + return { + text: readFileSync(join(String(surface.worktreeRef), 'module.txt'), 'utf8'), + } + }) + }, + code: { + repoRoot, + generator: { + kind: 'must-not-run', + async generate() { + throw new Error('baseline-only run must not call the generator') + }, + }, + }, + }) + + if (typeof result.raw.winner.surface === 'string') { + throw new Error('expected code winner') + } + expect(result.gateDecision).toBe('hold') + expect(readFileSync(join(result.raw.winner.surface.worktreeRef, 'module.txt'), 'utf8')).toBe( + 'baseline contents\n', + ) + expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength(2) + await result.dispose() + await result.dispose() + expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength(1) + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it.each([ + { mode: 'transient' as const, expectedWorktrees: 1, expectedErrors: 1 }, + { mode: 'persistent' as const, expectedWorktrees: 3, expectedErrors: 2 }, + ])('surface code retries $mode cleanup before rejecting without a result', async (fixture) => { + const { execSync } = await import('node:child_process') + const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') + const { tmpdir } = await import('node:os') + const { join } = await import('node:path') + const repoRoot = mkdtempSync(join(tmpdir(), `improve-code-${fixture.mode}-cleanup-`)) + const git = (cmd: string) => execSync(`git ${cmd}`, { cwd: repoRoot, stdio: 'pipe' }) + try { + git('init -q -b main') + git('config user.email improve@test.local') + git('config user.name improve-test') + writeFileSync(join(repoRoot, 'module.txt'), 'baseline contents\n') + git('add module.txt') + git('commit -qm baseline') + + const realWorktree = gitWorktreeAdapter({ repoRoot }) + let discardAttempts = 0 + const flakyWorktree: WorktreeAdapter = { + ...realWorktree, + async discard(worktree: Worktree) { + discardAttempts += 1 + if (fixture.mode === 'persistent' || discardAttempts === 1) { + throw new Error(`${fixture.mode} discard failure ${discardAttempts}`) + } + await realWorktree.discard(worktree) + }, + } + + let caught: unknown + try { + await improve(promptProfile(), [], { + surface: 'code', + scenarios, + judge: improvementJudge, + agent: async (surface, _scenario, ctx) => { + return paidStubCall(ctx, 'stub-agent', () => { + if (typeof surface !== 'object' || surface === null || !('worktreeRef' in surface)) { + throw new Error('expected code surface') + } + return { + text: readFileSync(join(String(surface.worktreeRef), 'module.txt'), 'utf8'), + } + }) + }, + code: { + repoRoot, + worktree: flakyWorktree, + generator: { + kind: 'cleanup-test', + async generate({ worktreePath }) { + writeFileSync(join(worktreePath, 'module.txt'), 'improved contents\n') + return { applied: true, summary: 'improve cleanup fixture' } + }, + }, + }, + promotionGate: { + name: 'test-hold', + decide: async () => ({ + decision: 'hold', + reasons: ['exercise cleanup recovery'], + contributingGates: [], + }), + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }) + } catch (cause) { + caught = cause + } + + const aggregate = + caught instanceof AggregateError + ? caught + : (caught as { cause?: unknown } | undefined)?.cause + expect(aggregate).toBeInstanceOf(AggregateError) + expect((aggregate as AggregateError).errors).toHaveLength(fixture.expectedErrors) + expect(discardAttempts).toBe(3) + expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength( + fixture.expectedWorktrees, + ) + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it.each([ + { + mode: 'transient' as const, + expectedWorktrees: 1, + expectedErrors: 2, + expectedDiscardAttempts: 3, + }, + { + mode: 'persistent' as const, + expectedWorktrees: 3, + expectedErrors: 3, + expectedDiscardAttempts: 6, + }, + ])('surface code retries $mode cleanup when selfImprove rejects', async (fixture) => { + const { execSync } = await import('node:child_process') + const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') + const { tmpdir } = await import('node:os') + const { join } = await import('node:path') + const repoRoot = mkdtempSync(join(tmpdir(), `improve-code-${fixture.mode}-rejection-`)) + const git = (cmd: string) => execSync(`git ${cmd}`, { cwd: repoRoot, stdio: 'pipe' }) + try { + git('init -q -b main') + git('config user.email improve@test.local') + git('config user.name improve-test') + writeFileSync(join(repoRoot, 'module.txt'), 'baseline contents\n') + git('add module.txt') + git('commit -qm baseline') + + const realWorktree = gitWorktreeAdapter({ repoRoot }) + let discardAttempts = 0 + const flakyWorktree: WorktreeAdapter = { + ...realWorktree, + async discard(worktree: Worktree) { + discardAttempts += 1 + if (fixture.mode === 'persistent' || discardAttempts === 1) { + throw new Error(`${fixture.mode} discard failure ${discardAttempts}`) + } + await realWorktree.discard(worktree) + }, + } + let caught: unknown + try { + await improve(promptProfile(), [], { + surface: 'code', + scenarios, + judge, + agent: async (surface, _scenario, ctx) => { + return paidStubCall(ctx, 'stub-agent', () => { + if (typeof surface !== 'object' || surface === null || !('worktreeRef' in surface)) { + throw new Error('expected code surface') + } + return { + text: readFileSync(join(String(surface.worktreeRef), 'module.txt'), 'utf8'), + } + }) + }, + code: { + repoRoot, + worktree: flakyWorktree, + generator: { + kind: 'rejecting-test', + async generate() { + throw new Error('candidate generation failed') + }, + }, + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }) + } catch (cause) { + caught = cause + } + + const aggregate = + caught instanceof AggregateError + ? caught + : (caught as { cause?: unknown } | undefined)?.cause + expect(aggregate).toBeInstanceOf(AggregateError) + expect((aggregate as AggregateError).errors).toHaveLength(fixture.expectedErrors) + expect(discardAttempts).toBe(fixture.expectedDiscardAttempts) + expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength( + fixture.expectedWorktrees, + ) + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it.each([ + { mode: 'transient' as const, expectedWorktrees: 1, expectedErrors: 2 }, + { mode: 'persistent' as const, expectedWorktrees: 2, expectedErrors: 3 }, + ])('surface code retries $mode cleanup when baseline finalization rejects', async (fixture) => { + const { execSync } = await import('node:child_process') + const { mkdtempSync, rmSync, writeFileSync } = await import('node:fs') + const { tmpdir } = await import('node:os') + const { join } = await import('node:path') + const repoRoot = mkdtempSync(join(tmpdir(), `improve-code-${fixture.mode}-finalize-`)) + const git = (cmd: string) => execSync(`git ${cmd}`, { cwd: repoRoot, stdio: 'pipe' }) + try { + git('init -q -b main') + git('config user.email improve@test.local') + git('config user.name improve-test') + writeFileSync(join(repoRoot, 'module.txt'), 'baseline contents\n') + git('add module.txt') + git('commit -qm baseline') + + const realWorktree = gitWorktreeAdapter({ repoRoot }) + let discardAttempts = 0 + const rejectingWorktree: WorktreeAdapter = { + ...realWorktree, + async finalize() { + throw new Error('baseline finalization failed') + }, + async discard(worktree: Worktree) { + discardAttempts += 1 + if (fixture.mode === 'persistent' || discardAttempts === 1) { + throw new Error(`${fixture.mode} discard failure ${discardAttempts}`) + } + await realWorktree.discard(worktree) + }, + } + + let caught: unknown + try { + await improve(promptProfile(), [], { + surface: 'code', + scenarios, + judge, + agent: stubAgent, + code: { + repoRoot, + worktree: rejectingWorktree, + generator: { + kind: 'unused', + async generate() { + throw new Error('must not generate before baseline finalization') + }, + }, + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }) + } catch (cause) { + caught = cause + } + + expect(caught).toBeInstanceOf(AggregateError) + expect((caught as AggregateError).errors).toHaveLength(fixture.expectedErrors) + expect(discardAttempts).toBe(2) + expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength( + fixture.expectedWorktrees, + ) } finally { rmSync(repoRoot, { recursive: true, force: true }) } diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 266c93f1..92243f90 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -12,12 +12,18 @@ * * - `surface: 'prompt'` → `gepaProposer` mutates `profile.prompt.systemPrompt`. * - `surface: 'skills'` → `skillOptProposer` mutates a skills document string. - * - `surface` ∈ {`tools`, `mcp`, `hooks`, `code`} → no zero-config default + * - `surface: 'memory'` → `memoryCurationProposer` curates a bounded durable + * lesson document supplied through `opts.memory`. + * - `surface: 'agent-profile'` → caller-supplied proposer mutates the complete + * canonical AgentProfile JSON in one candidate. + * - `surface` ∈ {`tools`, `mcp`, `hooks`, `subagents`, `agent-profile`} → no zero-config default * proposer exists (a code/config proposer needs caller-supplied wiring — a * worktree repo root, a candidate generator, a serializer). The facade * requires an explicit `opts.generator` for these and throws a `ConfigError` * otherwise. This is a designed boundary, not a missing default: there is - * no safe value the facade could invent for those seams. + * no safe value the facade could invent for those surfaces. Code instead + * requires `opts.code.repoRoot` and accepts only the runtime-owned + * `opts.code.generator` path so every isolated checkout can be released. * * Everything else (`scenarios`, `judge`, `agent`, `budget`, `llm`) passes * straight through to `selfImprove`. @@ -25,14 +31,17 @@ * @experimental */ +import { canonicalJson } from '@tangle-network/agent-eval' import { gepaProposer, gitWorktreeAdapter, + memoryCurationProposer, skillOptProposer, + type Worktree, + type WorktreeAdapter, } from '@tangle-network/agent-eval/campaign' import { - type DispatchContext, - type JudgeConfig, + type CodeSurface, type MutableSurface, type Scenario, type SelfImproveBudget, @@ -42,51 +51,51 @@ import { type SurfaceProposer, selfImprove, } from '@tangle-network/agent-eval/contract' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, agentProfileSchema } from '@tangle-network/agent-interface' import { ConfigError } from '../errors' import type { LocalHarness } from '../mcp/local-harness' import { assertModelAllowed } from '../runtime/supervise/model-policy' import { agenticGenerator, type Verifier } from './agentic-generator' -import { type CandidateGenerator, improvementDriver } from './improvement-driver' +import { + type CandidateGenerator, + improvementDriver, + type ManagedImprovementDriver, +} from './improvement-driver' import { rawTraceDistiller } from './raw-trace-distiller' -/** The agent-profile lever `improve` optimizes. Mirrors the AgentProfile-law - * profile levers; `code` is the implementation-tier surface. */ -export type ImproveSurface = 'prompt' | 'skills' | 'tools' | 'mcp' | 'hooks' | 'code' +/** The executable agent lever `improve` optimizes. Profile fields remain + * portable AgentProfile coordinates; implementation and orchestration files + * use the code surface so a winner can be sealed into an exact candidate. */ +export type ImproveSurface = + | 'prompt' + | 'skills' + | 'tools' + | 'mcp' + | 'hooks' + | 'subagents' + | 'agent-profile' + | 'memory' + | 'code' -export interface ImproveOptions { +export type ImproveOptions = Omit< + SelfImproveOptions, + 'analyzeGeneration' | 'baselineSurface' | 'findings' | 'gate' | 'proposer' +> & { /** Which profile lever to optimize. Default `'prompt'`. Selects the default * generator + the baseline-surface extraction shape. */ surface?: ImproveSurface - /** The `SurfaceProposer` that mutates the surface. When unset, the facade - * picks the default for `surface` (`gepaProposer` for prompt, `skillOptProposer` - * for skills); surfaces with no default REQUIRE this (fail-loud otherwise). */ + /** The `SurfaceProposer` that mutates a profile surface. When unset, the facade + * picks the default for prompt, skills, and memory; surfaces + * with no default REQUIRE this (fail-loud otherwise). Forbidden for code; + * use `code.generator` so the runtime owns candidate cleanup. */ generator?: SurfaceProposer /** Gate mode. `'holdout'` (default) runs the held-out promotion gate; * `'none'` is a baseline-only run (`budget.generations = 0`). */ gate?: 'holdout' | 'none' - /** Scenarios to evaluate against. Passthrough to `selfImprove`. */ - scenarios: TScenario[] - /** Judge that scores artifacts. Passthrough to `selfImprove`. */ - judge: JudgeConfig - /** The agent under improvement — same shape as `selfImprove.agent`: it takes - * the current surface + scenario + ctx and returns the artifact to judge. */ - agent: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise - /** Budget + loop shape. Passthrough; `gate: 'none'` forces `generations = 0`. */ - budget?: SelfImproveBudget - /** LLM config. Passthrough to `selfImprove` AND used to construct the default - * reflective proposer (`gepaProposer`/`skillOptProposer`) when `generator` is unset. */ - llm?: SelfImproveLlm /** Restrict the run to this subset of models. When set, the reflection model * (`llm.model`, or the default when unset) must be a member, or `improve()` throws * a `ConfigError` before the generator is built. Unset = unrestricted. */ allowedModels?: readonly string[] - /** Run directory passthrough to `selfImprove`. Pass a REAL path to make the loop - * durable: campaign cells + the loop provenance record land on the filesystem as - * they complete, so a multi-hour search survives a process/infra death instead of - * losing every generation with it (the default `mem://` run keeps everything - * in-process). */ - runDir?: string /** Per-generation findings producer passthrough (see selfImprove.analyzeGeneration). * DEFAULT: the built-in failure distiller — after each generation it turns the * worst-scoring/errored cells into structured findings ({ scenario, composite, @@ -105,13 +114,13 @@ export interface ImproveOptions { * (disabled). Equivalent to `analyzeGeneration: rawTraceDistiller()`; this flag * is the one-line enable. Default `false` (the distiller stays the default). */ rawTraceContext?: boolean - /** CODE-surface wiring with prompt-parity DX: name `surface: 'code'`, point at a - * repo, and the facade assembles the whole candidate pipeline — git worktrees + /** CODE-surface wiring: name `surface: 'code'`, point at a repo, and the + * facade assembles the whole candidate pipeline — an isolated incumbent plus git worktrees * (`gitWorktreeAdapter`) driven by `improvementDriver` with the full agentic * generator (a real coding harness edits each candidate worktree; a `verify` * hook gates candidates before they are ever measured). Ignored when - * `opts.generator` is supplied. Without either, `surface: 'code'` still fails - * loud — there is no safe zero-config repo to invent. */ + * `opts.generator` is supplied. Required for every code run because a real + * repository and base ref are necessary to measure the incumbent. */ code?: ImproveCodeOptions /** SKILLS-surface wiring for real skill-DOCUMENT optimization. Without this, * `surface: 'skills'` optimizes the profile's skills REFS array (file pointers) @@ -120,8 +129,13 @@ export interface ImproveOptions { * shipped winner (the profile ref points at a file the caller owns). This is * what makes skillOpt reachable through improve(). */ skills?: ImproveSkillsOptions - /** Storage passthrough to `selfImprove`; overrides the default chosen from `runDir`. */ - storage?: SelfImproveOptions['storage'] + /** MEMORY-surface wiring for a curated durable memory document. The default + * deterministic proposer deduplicates and ranks lessons from findings, then + * replaces its managed block instead of growing memory without bound. */ + memory?: ImproveMemoryOptions + /** Custom held-back-exam decision. The string `gate` above controls whether + * the exam runs; this callback controls how its evidence decides promotion. */ + promotionGate?: SelfImproveOptions['gate'] } export interface ImproveSkillsOptions { @@ -130,7 +144,14 @@ export interface ImproveSkillsOptions { /** Persist the shipped winner document (write the file the profile ref points at). * Called only on a ship verdict. When omitted, the winner is still returned in * `result.raw.winner.surface` for the caller to materialize. */ - writeBack?: (winnerDocument: string) => void + writeBack?: (winnerDocument: string) => void | Promise +} + +export interface ImproveMemoryOptions { + /** Current durable memory text used as the measured baseline. */ + document: string + /** Persist the promoted memory document. Never called on hold or error. */ + writeBack?: (winnerDocument: string) => void | Promise } export interface ImproveCodeOptions { @@ -140,6 +161,9 @@ export interface ImproveCodeOptions { baseRef?: string /** Directory worktrees are created under. Default `/.worktrees`. */ worktreeDir?: string + /** Git-compatible adapter override, primarily for tests. Candidate advancement + * still requires normal Git worktree and commit semantics. */ + worktree?: WorktreeAdapter /** Coding harness the agentic generator runs in each worktree. Default `claude`. */ harness?: LocalHarness /** Verify a candidate worktree before it becomes a measurable surface; failures @@ -162,8 +186,13 @@ export interface ImproveResult { lift: number /** The five-valued gate verdict from `selfImprove`. */ gateDecision: SelfImproveResult['gateDecision'] - /** Full `selfImprove` result for advanced inspection. */ + /** Full `selfImprove` result for advanced inspection. For code runs, + * `raw.winner.surface.worktreeRef` remains live after return whether the + * candidate shipped or held; call `dispose()` after consuming it. */ raw: SelfImproveResult + /** Release resources owned by this result. Idempotent; currently disposes + * the returned code worktree and is a no-op for profile-only surfaces. */ + dispose(): Promise } /** Default model id for the reflective drivers when `llm.model` is unset — a model the Tangle @@ -188,18 +217,21 @@ function defaultGeneratorFor( return gepaProposer({ llm: llmClientOptions(llm), model, target: 'agent system prompt' }) case 'skills': return skillOptProposer({ llm: llmClientOptions(llm), model, target: 'agent skill document' }) + case 'memory': + return memoryCurationProposer() default: return undefined } } /** Extract the baseline surface a driver mutates from the profile field that - * backs `surface`. `prompt`/`skills` are string surfaces; the config surfaces - * serialize the matching profile record. */ + * backs `surface`. Prompt, skills, and memory are text surfaces; config + * surfaces serialize the matching profile record. */ function baselineSurfaceFor( profile: AgentProfile, surface: ImproveSurface, skills?: ImproveSkillsOptions, + memory?: ImproveMemoryOptions, ): MutableSurface { switch (surface) { case 'prompt': @@ -214,11 +246,19 @@ function baselineSurfaceFor( return JSON.stringify(profile.mcp ?? {}) case 'hooks': return JSON.stringify(profile.hooks ?? {}) + case 'subagents': + return JSON.stringify(profile.subagents ?? {}) + case 'agent-profile': + return canonicalJson(profile) + case 'memory': + if (!memory) { + throw new ConfigError("improve(): surface 'memory' requires opts.memory.document") + } + return memory.document case 'code': - // A code surface is produced by the caller's generator from a worktree; - // the facade has no worktree ref to seed, so the baseline is the empty - // string (the driver opens its own worktree off `baseRef`). - return '' + throw new ConfigError( + 'improve(): code requires the isolated baseline created from opts.code.repoRoot', + ) } } @@ -233,8 +273,13 @@ function generationFailureDistiller( ): NonNullable['analyzeGeneration']> { const CAP = 12 return async (input) => { - const failures: Array<{ scenario: string; composite: number; notes: string; error?: string }> = - [] + const failures: Array<{ + scenario: string + composite: number + notes: string + claim?: string + error?: string + }> = [] for (const candidate of input.candidates) { for (const rawCell of candidate.campaign.cells) { const cell = rawCell as unknown as Record @@ -259,10 +304,12 @@ function generationFailureDistiller( .filter((n): n is string => typeof n === 'string' && n.length > 0) .join('; ') .slice(0, 1500) + const claim = notes || (error ? `Scenario ${scenario} failed: ${error.slice(0, 500)}` : '') failures.push({ scenario, composite: Number(composite.toFixed(3)), notes, + ...(claim ? { claim } : {}), ...(error ? { error: error.slice(0, 500) } : {}), }) } @@ -273,33 +320,136 @@ function generationFailureDistiller( } } -/** Assemble the code-surface proposer from `opts.code`: git worktrees + the - * improvement driver + (by default) the full agentic generator. Returns - * `undefined` when the surface is not `code` or no code options were given — - * the caller then falls through to the fail-loud ConfigError. */ -function codeProposerFor( - surface: ImproveSurface, - code: ImproveCodeOptions | undefined, -): SurfaceProposer | undefined { - if (surface !== 'code' || !code) return undefined - const generator = - code.generator ?? - agenticGenerator({ - ...(code.harness ? { harness: code.harness } : {}), - ...(code.verify ? { verify: code.verify } : {}), - ...(code.timeoutMs ? { timeoutMs: code.timeoutMs } : {}), - }) - return improvementDriver({ - worktree: gitWorktreeAdapter({ +/** Memory accumulates durable lessons, so keep the caller's seed findings while + * adding fresh judge failures. Curator proposers consume `claim`; the generic + * distiller retains the richer diagnostic fields for reflective proposers. */ +function memoryGenerationDistiller( + staticFindings: unknown[], +): NonNullable['analyzeGeneration']> { + const distillFailures = generationFailureDistiller(staticFindings) + return async (input) => { + const fresh = await distillFailures(input) + return fresh === staticFindings ? staticFindings : [...staticFindings, ...fresh] + } +} + +interface PreparedCodeRun { + baseline: CodeSurface + proposer: SurfaceProposer + cleanup(retainedWinner?: MutableSurface): Promise +} + +/** Preserve the primary failure while making two best-effort cleanup attempts. + * A failed first attempt is retained in the thrown AggregateError even when the + * retry succeeds, so callers can diagnose degraded cleanup without losing the + * error that caused cleanup to run. */ +async function rethrowAfterCleanup( + cause: unknown, + cleanup: () => Promise, + message: string, +): Promise { + const cleanupErrors: unknown[] = [] + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + await cleanup() + } catch (cleanupCause) { + cleanupErrors.push(cleanupCause) + continue + } + if (cleanupErrors.length === 0) throw cause + throw new AggregateError([cause, ...cleanupErrors], `${message}; the cleanup retry succeeded`) + } + throw new AggregateError([cause, ...cleanupErrors], message) +} + +async function discardPreparedBaseline( + worktree: WorktreeAdapter, + baselineWorktree: Worktree, + cause: unknown, +): Promise { + return rethrowAfterCleanup( + cause, + () => worktree.discard(baselineWorktree), + 'improve(): code preparation failed and its baseline worktree could not be cleaned', + ) +} + +function isCodeSurface(surface: MutableSurface | undefined): surface is CodeSurface { + return typeof surface === 'object' && surface !== null && surface.kind === 'code' +} + +/** Create a clean incumbent checkout and the candidate producer for a code run. */ +async function prepareCodeRun(code: ImproveCodeOptions): Promise { + const baseRef = code.baseRef ?? 'main' + const worktree = + code.worktree ?? + gitWorktreeAdapter({ repoRoot: code.repoRoot, ...(code.worktreeDir ? { worktreeDir: code.worktreeDir } : {}), - }), - generator, - ...(code.baseRef ? { baseRef: code.baseRef } : {}), - }) as SurfaceProposer + }) + const baselineWorktree = await worktree.create({ baseRef, label: 'incumbent-baseline' }) + try { + const baseline = await worktree.finalize(baselineWorktree, 'Incumbent code checkout') + let baselineDiscarded = false + const generator = + code.generator ?? + agenticGenerator({ + ...(code.harness ? { harness: code.harness } : {}), + ...(code.verify ? { verify: code.verify } : {}), + ...(code.timeoutMs ? { timeoutMs: code.timeoutMs } : {}), + }) + const managed: ManagedImprovementDriver = improvementDriver({ worktree, generator, baseRef }) + + return { + baseline, + proposer: managed, + async cleanup(retainedWinner) { + const errors: unknown[] = [] + const retainedWorktreeRef = isCodeSurface(retainedWinner) + ? retainedWinner.worktreeRef + : undefined + try { + await managed?.cleanup(retainedWorktreeRef ? [retainedWorktreeRef] : []) + } catch (cause) { + errors.push(cause) + } + if (!baselineDiscarded && retainedWorktreeRef !== baseline.worktreeRef) { + try { + await worktree.discard(baselineWorktree) + baselineDiscarded = true + } catch (cause) { + errors.push(cause) + } + } + if (errors.length > 0) { + throw new AggregateError(errors, 'improve(): failed to clean code improvement worktrees') + } + }, + } + } catch (cause) { + return discardPreparedBaseline(worktree, baselineWorktree, cause) + } } -/** Parse a JSON winner surface (`skills`/`tools`/`mcp`/`hooks`) with a typed, +function idempotentDispose(dispose: () => Promise): () => Promise { + let disposed = false + let inFlight: Promise | undefined + return async () => { + if (disposed) return + if (inFlight) return inFlight + inFlight = (async () => { + await dispose() + disposed = true + })() + try { + await inFlight + } finally { + inFlight = undefined + } + } +} + +/** Parse a JSON winner surface (`tools`/`mcp`/`hooks`/`subagents`/`agent-profile`) with a typed, * contextual error. A malformed generator output must fail loud here, not throw * a raw `SyntaxError` to the caller after a ship verdict. */ function parseWinnerJson(winner: string, surface: ImproveSurface): T { @@ -316,7 +466,7 @@ function parseWinnerJson(winner: string, surface: ImproveSurface): T { /** Apply a promoted winner surface back into the profile field for `surface`. * Returns a shallow copy; never mutates the input profile. */ -function applyWinnerToProfile( +export function applyImprovementWinnerToProfile( profile: AgentProfile, surface: ImproveSurface, winner: MutableSurface, @@ -326,23 +476,44 @@ function applyWinnerToProfile( // caller materializes it from `raw.winner.surface`; the returned profile is // unchanged for that lever. if (typeof winner !== 'string') return profile + let candidate: AgentProfile switch (surface) { case 'prompt': - return { ...profile, prompt: { ...profile.prompt, systemPrompt: winner } } + candidate = { ...profile, prompt: { ...profile.prompt, systemPrompt: winner } } + break case 'skills': - return { + candidate = { ...profile, resources: { ...profile.resources, skills: parseWinnerJson(winner, surface) }, } + break case 'tools': - return { ...profile, tools: parseWinnerJson(winner, surface) } + candidate = { ...profile, tools: parseWinnerJson(winner, surface) } + break case 'mcp': - return { ...profile, mcp: parseWinnerJson(winner, surface) } + candidate = { ...profile, mcp: parseWinnerJson(winner, surface) } + break case 'hooks': - return { ...profile, hooks: parseWinnerJson(winner, surface) } + candidate = { ...profile, hooks: parseWinnerJson(winner, surface) } + break + case 'subagents': + candidate = { ...profile, subagents: parseWinnerJson(winner, surface) } + break + case 'agent-profile': + candidate = parseWinnerJson(winner, surface) + break + case 'memory': + return profile case 'code': return profile } + const parsed = agentProfileSchema.safeParse(candidate) + if (!parsed.success) { + throw new ConfigError( + `improve(): the shipped '${surface}' winner does not produce a valid AgentProfile: ${parsed.error.message}`, + ) + } + return parsed.data } /** @@ -363,60 +534,140 @@ export async function improve( findings: unknown[], opts: ImproveOptions, ): Promise> { - const surface = opts.surface ?? 'prompt' - const gate = opts.gate ?? 'holdout' + const { + surface = 'prompt', + gate = 'holdout', + generator, + allowedModels, + rawTraceContext, + code, + skills, + memory, + promotionGate, + analyzeGeneration, + ...sharedOptions + } = opts - // Fail loud before the generator is built: the reflection model must be in the allowed subset - // (no-op when allowedModels is unset). - assertModelAllowed(opts.llm?.model ?? defaultReflectionModel, opts.allowedModels) + const parsedProfile = agentProfileSchema.safeParse(profile) + if (!parsedProfile.success) { + throw new ConfigError( + `improve(): input is not a valid AgentProfile: ${parsedProfile.error.message}`, + ) + } + if (surface === 'skills' && !generator && !skills) { + throw new ConfigError( + 'improve(): the default skills optimizer requires opts.skills.document; pass the skill text or an explicit generator that understands resource refs', + ) + } + if (surface === 'memory' && !memory) { + throw new ConfigError("improve(): surface 'memory' requires opts.memory.document") + } + if (surface === 'code' && generator) { + throw new ConfigError( + "improve(): surface 'code' forbids opts.generator because an external SurfaceProposer cannot transfer checkout ownership; pass opts.code.generator instead", + ) + } + const usesReflectionModel = !generator && (surface === 'prompt' || surface === 'skills') + if (usesReflectionModel) { + assertModelAllowed(sharedOptions.llm?.model ?? defaultReflectionModel, allowedModels) + } + let preparedCode: PreparedCodeRun | undefined + if (surface === 'code') { + if (!code) { + throw new ConfigError( + "improve(): surface 'code' requires opts.code.repoRoot so the incumbent can run from an isolated checkout", + ) + } + preparedCode = await prepareCodeRun(code) + } const proposer = - opts.generator ?? defaultGeneratorFor(surface, opts.llm) ?? codeProposerFor(surface, opts.code) + preparedCode?.proposer ?? generator ?? defaultGeneratorFor(surface, sharedOptions.llm) if (!proposer) { throw new ConfigError( - surface === 'code' - ? `improve(): surface 'code' needs either opts.generator or opts.code ({ repoRoot, ... }) — there is no safe zero-config repo to invent` - : `improve(): surface '${surface}' has no default generator — pass opts.generator (a SurfaceProposer) explicitly`, + `improve(): surface '${surface}' has no default generator — pass opts.generator (a SurfaceProposer) explicitly`, ) } const budget: SelfImproveBudget = - gate === 'none' ? { ...opts.budget, generations: 0 } : { ...opts.budget } + gate === 'none' ? { ...sharedOptions.budget, generations: 0 } : { ...sharedOptions.budget } - const raw = await selfImprove({ - agent: opts.agent, - scenarios: opts.scenarios, - judge: opts.judge, - baselineSurface: baselineSurfaceFor(profile, surface, opts.skills), - proposer, - budget, - llm: opts.llm, - findings, - ...(opts.runDir !== undefined ? { runDir: opts.runDir } : {}), - ...(opts.storage !== undefined ? { storage: opts.storage } : {}), - ...(opts.analyzeGeneration === null - ? {} - : { - analyzeGeneration: - opts.analyzeGeneration ?? - (opts.rawTraceContext - ? rawTraceDistiller({ fallbackFindings: findings }) - : generationFailureDistiller(findings)), - }), - }) + let raw: SelfImproveResult + try { + raw = await selfImprove({ + ...sharedOptions, + baselineSurface: + preparedCode?.baseline ?? baselineSurfaceFor(profile, surface, skills, memory), + proposer, + budget, + findings, + ...(promotionGate !== undefined ? { gate: promotionGate } : {}), + ...(analyzeGeneration === null + ? {} + : { + analyzeGeneration: + analyzeGeneration ?? + (rawTraceContext + ? rawTraceDistiller({ fallbackFindings: findings }) + : surface === 'memory' + ? memoryGenerationDistiller(findings) + : generationFailureDistiller(findings)), + }), + }) + } catch (cause) { + if (!preparedCode) throw cause + return rethrowAfterCleanup( + cause, + () => preparedCode.cleanup(), + 'improve(): code improvement failed and its worktrees could not be cleaned', + ) + } const shipped = raw.gateDecision === 'ship' + const winnerSurface = raw.winner.surface + if (preparedCode) { + try { + await preparedCode.cleanup(winnerSurface) + } catch (cleanupCause) { + try { + await preparedCode.cleanup() + } catch (finalCleanupCause) { + throw new AggregateError( + [cleanupCause, finalCleanupCause], + 'improve(): code result cleanup failed, including the final all-worktree retry', + ) + } + throw new AggregateError( + [cleanupCause], + 'improve(): code result cleanup failed; the final all-worktree retry succeeded', + ) + } + } + const dispose = idempotentDispose(async () => preparedCode?.cleanup()) // When a skill DOCUMENT was optimized, the winner is document text — persist it // via writeBack (the profile ref points at the caller's file, unchanged) rather // than parsing it as a refs array. Otherwise use the standard field write-back. - const usedSkillDocument = surface === 'skills' && opts.skills !== undefined - if (shipped && usedSkillDocument && typeof raw.winner.surface === 'string') { - opts.skills?.writeBack?.(raw.winner.surface) + const externalDocument = + surface === 'skills' && skills ? skills : surface === 'memory' && memory ? memory : undefined + if (shipped && externalDocument) { + if (typeof winnerSurface !== 'string') { + throw new ConfigError( + `improve(): the shipped '${surface}' winner must be text before it can be persisted`, + ) + } + await externalDocument.writeBack?.(winnerSurface) } const nextProfile = - shipped && !usedSkillDocument - ? applyWinnerToProfile(profile, surface, raw.winner.surface) + shipped && !externalDocument + ? applyImprovementWinnerToProfile(profile, surface, winnerSurface) : profile - return { profile: nextProfile, shipped, lift: raw.lift, gateDecision: raw.gateDecision, raw } + return { + profile: nextProfile, + shipped, + lift: raw.lift, + gateDecision: raw.gateDecision, + raw, + dispose, + } } diff --git a/src/improvement/improvement-driver.ts b/src/improvement/improvement-driver.ts index 2f9699a3..b3ea5c8a 100644 --- a/src/improvement/improvement-driver.ts +++ b/src/improvement/improvement-driver.ts @@ -20,13 +20,16 @@ * @experimental */ -import type { AnalystFinding } from '@tangle-network/agent-eval' -import type { - CodeSurface, - LabeledScenarioStore, - ProposeContext, - SurfaceProposer, - WorktreeAdapter, +import { spawnSync } from 'node:child_process' +import type { AnalystFinding, CostLedger } from '@tangle-network/agent-eval' +import { + type CodeSurface, + type LabeledScenarioStore, + type ProposeContext, + type SurfaceProposer, + verifyCodeSurface, + type Worktree, + type WorktreeAdapter, } from '@tangle-network/agent-eval/campaign' /** The byte-producing seam — the ONE thing that differs between the cheap @@ -35,8 +38,19 @@ import type { * adapter's `finalize`. */ export interface CandidateGenerator { kind: string + /** Whether this generator can produce a candidate from an EMPTY findings set + * and no phase-2 report — i.e. it draws its change signal from the repo and + * the raw-trace filesystem context on disk, not only from pre-summarized + * findings. An agentic coder (`agenticGenerator`) sets this: the seed repo + + * raw traces ARE the signal, so it must still run the full `populationSize` + * when the distiller yielded nothing (this is the meta-harness contract — the + * agent diagnoses from the raw traces itself). A patch-applier + * (`reflectiveGenerator`) leaves it unset — with no findings there is no + * patch to draft, so the driver short-circuits rather than spin up worktrees + * for a guaranteed no-op. Default `false`. */ + proposesWithoutFindings?: boolean generate(args: { - /** The candidate worktree — a fresh checkout of baseRef. Write changes here. */ + /** The candidate worktree — a clean checkout of the current incumbent. */ worktreePath: string /** Phase-2 research report (analyst findings + diff), opaque. */ report: unknown @@ -48,38 +62,68 @@ export interface CandidateGenerator { * reflective generator ignores it). */ maxShots: number signal: AbortSignal + /** Improvement-loop coordinates. Present when called through improvementDriver. */ + generation?: number + candidateIndex?: number + /** Shared run-wide paid-call account supplied by agent-eval 0.117+. */ + costLedger?: CostLedger + /** Receipt attribution phase supplied alongside `costLedger`. */ + costPhase?: string }): Promise<{ applied: boolean; summary: string }> } export interface ImprovementDriverOptions { worktree: WorktreeAdapter generator: CandidateGenerator - /** Base ref candidate worktrees fork from. Default `main`. */ + /** Root ref for first-generation/direct callers. Default `main`. + * Later code generations retain the incumbent's original root. */ baseRef?: string } +export interface ManagedImprovementDriver extends SurfaceProposer { + /** Remove every owned candidate except explicitly retained finalized winners. */ + cleanup(retainWorktreeRefs?: readonly string[]): Promise +} + /** The one reflective/agentic improvement proposer (`SurfaceProposer`): owns the candidate worktree lifecycle and delegates HOW a change is produced to a pluggable `CandidateGenerator`. */ -export function improvementDriver(opts: ImprovementDriverOptions): SurfaceProposer { +export function improvementDriver(opts: ImprovementDriverOptions): ManagedImprovementDriver { const baseRef = opts.baseRef ?? 'main' + const owned = new Map() return { kind: `improvement:${opts.generator.kind}`, async propose(ctx: ProposeContext) { const findings = resolveFindings(ctx) - // No signal to act on — propose nothing rather than spin up worktrees. - if (findings.length === 0 && ctx.report === undefined) return [] + // No findings AND no report AND a generator that can only act on findings + // (the reflective patch-applier) — propose nothing rather than spin up + // worktrees for a guaranteed no-op. An agentic coder draws its signal from + // the repo + raw traces on disk, so it opts in via `proposesWithoutFindings` + // and still runs the full populationSize even on an empty findings set — + // otherwise the FIRST generation (whose seed findings are empty and whose + // rawTraceDistiller has not run yet) would always generate ZERO candidates. + if ( + findings.length === 0 && + ctx.report === undefined && + !opts.generator.proposesWithoutFindings + ) { + return [] + } const surfaces: CodeSurface[] = [] + const incumbent = verifiedCodeIncumbent(ctx.currentSurface) + const proposalBaseRef = incumbent?.baseCommit ?? baseRef for (let i = 0; i < ctx.populationSize; i++) { if (ctx.signal.aborted) break const wt = await opts.worktree.create({ - baseRef, + baseRef: proposalBaseRef, label: `${opts.generator.kind}-gen${ctx.generation}-cand${i}`, }) + owned.set(wt.path, wt) // Once a worktree exists it MUST be accounted for: finalized into a // surface, or discarded. A throw from generate()/finalize() must not // leak the worktree + branch — discard best-effort, then rethrow loud. try { + if (incumbent) advanceToIncumbent(wt, incumbent) const { applied, summary } = await opts.generator.generate({ worktreePath: wt.path, report: ctx.report, @@ -87,20 +131,101 @@ export function improvementDriver(opts: ImprovementDriverOptions): SurfacePropos dataset: ctx.dataset, maxShots: ctx.maxImprovementShots ?? 1, signal: ctx.signal, + generation: ctx.generation, + candidateIndex: i, + ...(ctx.costLedger ? { costLedger: ctx.costLedger } : {}), + ...(ctx.costPhase ? { costPhase: ctx.costPhase } : {}), }) if (!applied) { await opts.worktree.discard(wt) + owned.delete(wt.path) continue } - surfaces.push(await opts.worktree.finalize(wt, summary)) + const surface = await opts.worktree.finalize(wt, summary) + surfaces.push(surface) + owned.delete(wt.path) + owned.set(surface.worktreeRef, wt) } catch (err) { - // Best-effort cleanup; never mask the original failure. - await opts.worktree.discard(wt).catch(() => {}) - throw err + const cleanupErrors: unknown[] = [] + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + await opts.worktree.discard(wt) + owned.delete(wt.path) + break + } catch (cause) { + cleanupErrors.push(cause) + } + } + if (cleanupErrors.length === 0) throw err + const failure = err instanceof Error ? err.message : String(err) + const cleanupSucceeded = !owned.has(wt.path) + throw new AggregateError( + [err, ...cleanupErrors], + cleanupSucceeded + ? `improvementDriver: ${failure}; candidate cleanup retry succeeded` + : `improvementDriver: ${failure}; candidate worktree could not be cleaned`, + ) } } return surfaces }, + async cleanup(retainWorktreeRefs = []) { + const retained = new Set(retainWorktreeRefs) + const errors: unknown[] = [] + for (const [worktreeRef, worktree] of owned) { + if (retained.has(worktreeRef)) continue + try { + await opts.worktree.discard(worktree) + owned.delete(worktreeRef) + } catch (cause) { + errors.push(cause) + } + } + if (errors.length > 0) { + throw new AggregateError(errors, 'improvementDriver: failed to discard candidate worktrees') + } + }, + } +} + +/** A code incumbent must still match the immutable identity that was measured. */ +function verifiedCodeIncumbent(surface: ProposeContext['currentSurface']): CodeSurface | undefined { + if (typeof surface !== 'object' || surface.kind !== 'code') return undefined + verifyCodeSurface(surface) + return surface +} + +/** Start at the root commit recorded by the incumbent, then fast-forward the + * fresh branch to the incumbent commit. The worktree stays clean for the + * generator while `finalize()` still emits one cumulative root-to-candidate + * patch that can be applied or rolled back independently of prior branches. */ +function advanceToIncumbent(worktree: Worktree, incumbent: CodeSurface): void { + if (worktree.baseCommit !== incumbent.baseCommit || worktree.baseTree !== incumbent.baseTree) { + throw new Error('improvementDriver: candidate worktree does not match incumbent base identity') + } + if (worktree.baseCommit === incumbent.candidateCommit) return + + const merge = spawnSync('git', ['merge', '--ff-only', incumbent.candidateCommit], { + cwd: worktree.path, + encoding: 'utf8', + }) + if (merge.error) { + throw new Error( + `improvementDriver: failed to start candidate from incumbent: ${merge.error.message}`, + ) + } + if (merge.status !== 0) { + throw new Error( + `improvementDriver: could not fast-forward candidate to incumbent ${incumbent.candidateCommit}: ${merge.stderr.trim()}`, + ) + } + + const head = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: worktree.path, + encoding: 'utf8', + }) + if (head.error || head.status !== 0 || head.stdout.trim() !== incumbent.candidateCommit) { + throw new Error('improvementDriver: candidate worktree did not reach the incumbent commit') } } diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 28f56598..0b916825 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -2,19 +2,20 @@ * `@tangle-network/agent-runtime` improvement — the CODE-surface proposer for * agent-eval's improvement loop. * - * The ONE entry point for optimization is agent-eval's `selfImprove` - * (`@tangle-network/agent-eval/contract`) — text/config surfaces, held-out gated, - * with `analyzeGeneration` for analyst-fed reflection and `analyzeRuns` / - * `fromOtelSpans` / `partitionRunsByAuthoringModel` for production intake + - * cohorting. This module supplies only the one genuinely runtime-specific piece: - * a CODE-surface `SurfaceProposer` you pass to `selfImprove` as `proposer`, which - * mutates a git worktree via a pluggable `CandidateGenerator`: + * The public entry point is `improve()`, a profile-aware facade over agent-eval's + * `selfImprove`. This module also supplies the runtime-specific code candidate + * producer, which mutates an isolated git worktree via a pluggable + * `CandidateGenerator`: * - `reflectiveGenerator` — cheap, no sandbox, applies pre-drafted patches * - `agenticGenerator` — full coding harness in the worktree, multi-shot */ export { + AGENTIC_PROFILE_RESOURCE_ROOT, type AgenticGeneratorOptions, + type AgenticGeneratorShotDisposition, + type AgenticGeneratorShotExecution, + type AgenticGeneratorShotReceipt, agenticGenerator, commandVerifier, type Verifier, @@ -22,8 +23,12 @@ export { } from './agentic-generator' export { mcpBuildPrompt, toolBuildPrompt } from './build-prompts' export { + applyImprovementWinnerToProfile, + type ImproveCodeOptions, + type ImproveMemoryOptions, type ImproveOptions, type ImproveResult, + type ImproveSkillsOptions, type ImproveSurface, improve, } from './improve' @@ -31,8 +36,15 @@ export { type CandidateGenerator, type ImprovementDriverOptions, improvementDriver, + type ManagedImprovementDriver, } from './improvement-driver' export { type McpServeSpec, mcpServeVerifier } from './mcp-serve-verifier' +export { + type AgentProfileDiffProposal, + type ProfileDiffProposerContext, + type ProfileDiffProposerOptions, + profileDiffProposer, +} from './profile-diff-proposer' export { type RawTraceDistillerOptions, rawTraceDistiller, diff --git a/src/improvement/mcp-serve-verifier.ts b/src/improvement/mcp-serve-verifier.ts index d23a36ae..0ca9e4e3 100644 --- a/src/improvement/mcp-serve-verifier.ts +++ b/src/improvement/mcp-serve-verifier.ts @@ -73,6 +73,8 @@ export function mcpServeVerifier(spec: McpServeSpec): Verifier { const failCandidate = (msg: string) => settle(() => resolve({ ok: false, feedback: withStderr(msg) })) const setupFault = (err: Error) => settle(() => reject(err)) + const failStdin = (err: Error) => + failCandidate(`writing to MCP server stdin failed: ${err.message}`) const send = (msg: Record): boolean => { try { @@ -80,7 +82,7 @@ export function mcpServeVerifier(spec: McpServeSpec): Verifier { return true } catch (err) { // EPIPE: the server died mid-handshake — a failed candidate, not a fault. - failCandidate(`writing to MCP server stdin failed: ${(err as Error).message}`) + failStdin(err as Error) return false } } @@ -100,6 +102,8 @@ export function mcpServeVerifier(spec: McpServeSpec): Verifier { // server crashed on boot); after we settle, our own SIGKILL fires here. failCandidate(`MCP server exited (code ${code}, signal ${signal}) before serving`) }) + // Pipe failures normally arrive asynchronously, outside send()'s try/catch. + child.stdin.on('error', failStdin) child.stderr.on('data', (d) => stderr.push(String(d))) const rl = createInterface({ input: child.stdout }) diff --git a/src/improvement/profile-diff-proposer.test.ts b/src/improvement/profile-diff-proposer.test.ts new file mode 100644 index 00000000..504bfe21 --- /dev/null +++ b/src/improvement/profile-diff-proposer.test.ts @@ -0,0 +1,147 @@ +import type { + ProposeContext, + ProposedCandidate, + SurfaceProposer, +} from '@tangle-network/agent-eval/campaign' +import { compositeProposer } from '@tangle-network/agent-eval/campaign' +import { defineAgentProfileDiff, defineInlineResource } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { profileDiffProposer } from './profile-diff-proposer' + +const context = (populationSize = 2): ProposeContext => ({ + currentSurface: JSON.stringify({ + name: 'fixture', + prompt: { systemPrompt: 'baseline' }, + resources: { failOnError: true }, + }), + history: [], + findings: [], + populationSize, + generation: 0, + signal: new AbortController().signal, +}) + +describe('profileDiffProposer', () => { + it('puts source-grounded profile diffs and generated mutations in one candidate pool', async () => { + const discovered = profileDiffProposer({ + proposeDiffs: () => [ + { + diff: defineAgentProfileDiff({ + schemaVersion: 1, + kind: 'agent-profile-diff', + id: 'github-review-skill', + title: 'Pinned review skill', + rationale: 'Existing skill covers the observed review failure.', + source: { + kind: 'frontier-author', + artifacts: ['https://github.com/example/skills/tree/0123456789abcdef/review'], + notes: ['MIT license verified by the research worker.'], + }, + set: { + resources: { + skills: [ + defineInlineResource( + 'review.SKILL.md', + '# Review\n\nRead the failing test before editing.', + ), + ], + failOnError: true, + }, + }, + }), + }, + ], + }) + const generated: SurfaceProposer = { + kind: 'generated', + propose: async () => [ + { + surface: JSON.stringify({ + name: 'fixture', + prompt: { systemPrompt: 'generated mutation' }, + resources: { failOnError: true }, + }), + label: 'generated prompt', + rationale: 'Mutation proposer output.', + }, + ], + } + + const candidates = await compositeProposer({ + proposers: [discovered, generated], + }).propose(context(2)) + + expect(candidates).toHaveLength(2) + const researched = candidates[0] as ProposedCandidate + expect(researched.label).toBe('agent-profile-diff:Pinned review skill') + expect(researched.rationale).toContain('Existing skill') + expect(researched.rationale).toContain( + 'https://github.com/example/skills/tree/0123456789abcdef/review', + ) + expect(researched.rationale).toMatch(/sha256:[a-f0-9]{64}/) + expect(JSON.parse(researched.surface as string)).toMatchObject({ + name: 'fixture', + prompt: { systemPrompt: 'baseline' }, + resources: { + skills: [ + { + kind: 'inline', + name: 'review.SKILL.md', + }, + ], + failOnError: true, + }, + }) + expect((candidates[1] as ProposedCandidate).label).toBe('generated:generated prompt') + }) + + it('rejects source output with fields the shared diff contract would discard', async () => { + const proposer = profileDiffProposer({ + proposeDiffs: () => [ + { + diff: { + schemaVersion: 1, + kind: 'agent-profile-diff', + set: { prompt: { systemPrompt: 'candidate' } }, + unknownField: true, + } as never, + }, + ], + }) + await expect(proposer.propose(context(1))).rejects.toThrow(/unsupported or non-canonical/) + }) + + it('drops no-op diffs and respects the shared population budget', async () => { + const proposer = profileDiffProposer({ + proposeDiffs: () => [ + { diff: defineAgentProfileDiff({ schemaVersion: 1, kind: 'agent-profile-diff' }) }, + { + diff: defineAgentProfileDiff({ + schemaVersion: 1, + kind: 'agent-profile-diff', + set: { prompt: { systemPrompt: 'baseline' } }, + }), + }, + { + diff: defineAgentProfileDiff({ + schemaVersion: 1, + kind: 'agent-profile-diff', + set: { prompt: { systemPrompt: 'first' } }, + }), + }, + { + diff: defineAgentProfileDiff({ + schemaVersion: 1, + kind: 'agent-profile-diff', + set: { prompt: { systemPrompt: 'first' } }, + }), + }, + ], + }) + const candidates = await proposer.propose(context(2)) + expect(candidates).toHaveLength(1) + expect( + JSON.parse((candidates[0] as ProposedCandidate).surface as string).prompt.systemPrompt, + ).toBe('first') + }) +}) diff --git a/src/improvement/profile-diff-proposer.ts b/src/improvement/profile-diff-proposer.ts new file mode 100644 index 00000000..8eed304c --- /dev/null +++ b/src/improvement/profile-diff-proposer.ts @@ -0,0 +1,108 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import type { + MutableSurface, + ProposeContext, + ProposedCandidate, + SurfaceProposer, +} from '@tangle-network/agent-eval/campaign' +import { + type AgentProfile, + type AgentProfileDiff, + changedAgentProfileAxes, +} from '@tangle-network/agent-interface' +import { canonicalCandidateDigest } from '../candidate-execution/digest' +import { + applyExactAgentProfileDiff, + parseExactAgentProfile, + parseExactAgentProfileDiff, +} from '../candidate-execution/profile' + +export interface AgentProfileDiffProposal { + diff: AgentProfileDiff + label?: string + rationale?: string +} + +export type ProfileDiffProposerContext = ProposeContext & { + profile: AgentProfile +} + +export interface ProfileDiffProposerOptions { + proposeDiffs( + context: ProfileDiffProposerContext, + ): Promise | readonly AgentProfileDiffProposal[] +} + +/** + * Turn exact AgentProfileDiffs from any source into full profile candidates for + * the shared optimization loop. Research, catalogs, humans, and trace miners + * differ only in `proposeDiffs`; measurement and promotion stay identical. + */ +export function profileDiffProposer( + options: ProfileDiffProposerOptions, +): SurfaceProposer { + return { + kind: 'agent-profile-diff', + async propose(ctx): Promise> { + if (typeof ctx.currentSurface !== 'string') { + throw new Error('profileDiffProposer requires a serialized AgentProfile surface') + } + const profile = parseSerializedProfile(ctx.currentSurface) + const proposals = await options.proposeDiffs({ + ...ctx, + profile, + }) + const candidates: ProposedCandidate[] = [] + const normalizedBaseline = applyExactAgentProfileDiff( + profile, + { schemaVersion: 1, kind: 'agent-profile-diff' }, + 'profile diff baseline', + ) + const seen = new Set([canonicalJson(normalizedBaseline)]) + for (const [index, proposal] of proposals.entries()) { + if (candidates.length >= ctx.populationSize || ctx.signal.aborted) break + const diff = parseExactAgentProfileDiff(proposal.diff, `profile diff proposal ${index}`) + const axes = changedAgentProfileAxes(diff) + if (axes.length === 0) continue + const candidate = applyExactAgentProfileDiff( + profile, + diff, + `profile diff candidate ${index}`, + ) + const surface = canonicalJson(candidate) + if (seen.has(surface)) continue + seen.add(surface) + candidates.push({ + surface, + label: proposal.label ?? diff.title ?? diff.id ?? `Change ${axes.join(', ')}`, + rationale: candidateRationale(proposal, diff, axes), + }) + } + return candidates + }, + } +} + +function candidateRationale( + proposal: AgentProfileDiffProposal, + diff: AgentProfileDiff, + axes: readonly string[], +): string { + const reason = proposal.rationale ?? diff.rationale ?? `Changes profile axes: ${axes.join(', ')}.` + const source = diff.source + ? [diff.source.kind, ...(diff.source.artifacts ?? [])].join(' · ') + : 'source unspecified' + return `${reason} Source: ${source}. Diff: ${canonicalCandidateDigest(diff)}.` +} + +function parseSerializedProfile(surface: string): AgentProfile { + let raw: unknown + try { + raw = JSON.parse(surface) + } catch (cause) { + throw new Error('profileDiffProposer current surface is not valid AgentProfile JSON', { + cause, + }) + } + return parseExactAgentProfile(raw, 'profileDiffProposer current profile') +} diff --git a/src/improvement/raw-trace-distiller.test.ts b/src/improvement/raw-trace-distiller.test.ts index f9097793..e503bc52 100644 --- a/src/improvement/raw-trace-distiller.test.ts +++ b/src/improvement/raw-trace-distiller.test.ts @@ -218,4 +218,31 @@ describe('rawTraceDistiller', () => { const findings = await rawTraceDistiller({ fallbackFindings: seed })(input) expect(findings).toEqual(seed) }) + + it('emits the default raw-trace instruction (not []) when the fallback seed is EMPTY on a clean generation', async () => { + // The runner passes findings:[] as the seed, so fallbackFindings is []. An + // empty fallback must NOT wipe the steering context to nothing — it must fall + // through to the default raw-trace instruction so the meta-harness discipline + // (and the agenticGenerator diagnosis-evidence gate keyed on the + // `raw-trace-context` area) stays armed for the next generation. + const input = { + generation: 0, + runDir: join(root, 'gen-0'), + candidates: [ + { + surfaceHash: 'perfect', + composite: 1, + campaign: { + runDir: join(root, 'gen-0', 'candidate-0'), + cells: [{ cellId: 's:0', scenarioId: 's', judgeScores: { j: { composite: 1 } } }], + }, + }, + ], + history: [], + } as unknown as AnalyzeInput + + const findings = await rawTraceDistiller({ fallbackFindings: [] })(input) + expect(findings).toHaveLength(1) + expect((findings[0] as unknown as { area: string }).area).toBe('raw-trace-context') + }) }) diff --git a/src/improvement/raw-trace-distiller.ts b/src/improvement/raw-trace-distiller.ts index 6e46f3d5..5d7db35b 100644 --- a/src/improvement/raw-trace-distiller.ts +++ b/src/improvement/raw-trace-distiller.ts @@ -111,22 +111,29 @@ export function rawTraceDistiller n + c.cells.length, 0) // A clean generation: keep the proposer's steering context rather than - // wiping it (parity with the default distiller's static-seed fallback). + // wiping it (parity with the default distiller's static-seed fallback). An + // EMPTY fallback array means there is no STATIC seed to preserve — it must NOT + // wipe the context to nothing. Fall through to the default raw-trace + // instruction so the meta-harness discipline stays live (the agent still + // inspects the on-disk traces next round, and the finding's `raw-trace-context` + // area keeps the agenticGenerator's diagnosis-evidence gate armed). A bare + // `??` would return the empty array and silently disable both. if (totalFailingCells === 0) { - return ( - options.fallbackFindings ?? [ - makeFinding({ - analyst_id: ANALYST_ID, - severity: 'info', - area: 'raw-trace-context', - confidence: 1, - claim: `Generation ${input.generation} had no failing cells. The full raw run traces are on disk under ${genRoot}.`, - recommended_action: `To keep improving, grep/cat the raw traces under ${genRoot} (per-cell spans.jsonl + cached-result.json) to find the weakest passing runs, then make a targeted harness-code edit.`, - evidence_refs: [{ kind: 'artifact', uri: genRoot }], - metadata: { generation: input.generation, runDir: genRoot, failingCells: 0 }, - }), - ] - ) + if (options.fallbackFindings && options.fallbackFindings.length > 0) { + return options.fallbackFindings + } + return [ + makeFinding({ + analyst_id: ANALYST_ID, + severity: 'info', + area: 'raw-trace-context', + confidence: 1, + claim: `Generation ${input.generation} had no failing cells. The full raw run traces are on disk under ${genRoot}.`, + recommended_action: `To keep improving, grep/cat the raw traces under ${genRoot} (per-cell spans.jsonl + cached-result.json) to find the weakest passing runs, then make a targeted harness-code edit.`, + evidence_refs: [{ kind: 'artifact', uri: genRoot }], + metadata: { generation: input.generation, runDir: genRoot, failingCells: 0 }, + }), + ] } const findings: AnalystFinding[] = [] diff --git a/src/index.ts b/src/index.ts index 2fde0664..9482c9b1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,6 +28,11 @@ export { createOpenAICompatibleBackend, createSandboxPromptBackend, } from './backends' +// ── Immutable candidate execution ───────────────────────────────────── +// One verified bundle → one exact per-task plan → one protected run receipt. +// This composes the shared profile materializer and agent-eval trace store; +// benchmark adapters supply only environment-specific artifact/container ports. +export * from './candidate-execution' export type { AuthSource, BackendCallPolicy, @@ -171,11 +176,13 @@ export type { OtelExportConfig, OtelExporter, OtelSpan, + RuntimeEventOtelOptions, } from './otel-export' // ── OTEL export + trace propagation + eval-run provenance ──────────── export { buildLoopOtelSpans, buildLoopSpanNodes, + buildRuntimeEventOtelSpans, createOtelExporter, exportEvalRuns, INTELLIGENCE_WIRE_VERSION, diff --git a/src/intelligence/capability.test.ts b/src/intelligence/capability.test.ts index 047c484a..855fb538 100644 --- a/src/intelligence/capability.test.ts +++ b/src/intelligence/capability.test.ts @@ -23,6 +23,9 @@ const provenance = { const wireProfile: CertifiedProfile = { target: 'support-agent', generatedAt: '2026-06-13T00:00:00.000Z', + agentProfileDiffs: [], + capabilities: [], + agentProfile: null, promptSurface: { surface: 'Confirm the invoice id before refunding.', surfaceHash: 'abc123', @@ -86,6 +89,9 @@ describe('manifestFromProfile', () => { target: 't', generatedAt: 'g', promptSurface: null, + agentProfileDiffs: [], + capabilities: [], + agentProfile: null, artifacts: { tool: [ { @@ -113,6 +119,9 @@ describe('manifestFromProfile', () => { target: 't', generatedAt: 'g', promptSurface: null, + agentProfileDiffs: [], + capabilities: [], + agentProfile: null, artifacts: { mcp: [ { @@ -136,6 +145,9 @@ describe('manifestFromProfile', () => { target: 't', generatedAt: 'g', promptSurface: null, + agentProfileDiffs: [], + capabilities: [], + agentProfile: null, artifacts: { notes: [ { @@ -181,6 +193,9 @@ describe('composeCertifiedProfile — prompt fold regression lock', () => { target: 't', generatedAt: 'g', promptSurface: null, + agentProfileDiffs: [], + capabilities: [], + agentProfile: null, artifacts: { skill: [ { @@ -227,6 +242,9 @@ describe('composeCertifiedProfile — prompt fold regression lock', () => { target: 't', generatedAt: 'g', promptSurface: null, + agentProfileDiffs: [], + capabilities: [], + agentProfile: null, artifacts: { notes: [ { diff --git a/src/intelligence/delivery.test.ts b/src/intelligence/delivery.test.ts index 273dfa9d..f87f1798 100644 --- a/src/intelligence/delivery.test.ts +++ b/src/intelligence/delivery.test.ts @@ -1,10 +1,10 @@ +import type { AgentProfileDiff } from '@tangle-network/agent-interface' import { describe, expect, it, vi } from 'vitest' import { type CertifiedProfile, composeCertifiedPrompt, createCertifiedPromptSource, pullCertified, - withCertifiedDelivery, } from './delivery' const CERTIFIED: CertifiedProfile = { @@ -28,6 +28,9 @@ const CERTIFIED: CertifiedProfile = { }, ], }, + agentProfileDiffs: [], + capabilities: [], + agentProfile: null, } function jsonResponse(body: unknown, status = 200): Response { @@ -55,13 +58,7 @@ describe('composeCertifiedPrompt', () => { }) it('returns base when the certified profile has no usable content', () => { - const empty: CertifiedProfile = { - target: 't', - generatedAt: 'x', - promptSurface: null, - artifacts: {}, - } - expect(composeCertifiedPrompt('BASE', empty)).toBe('BASE') + expect(composeCertifiedPrompt('BASE', { promptSurface: null, artifacts: {} })).toBe('BASE') }) }) @@ -84,6 +81,63 @@ describe('pullCertified', () => { expect(call[1].headers).toMatchObject({ authorization: 'Bearer k_test' }) }) + it('deserializes the typed agentProfileDiffs the composed endpoint returns', async () => { + const diff: AgentProfileDiff = { + schemaVersion: 1, + kind: 'agent-profile-diff', + set: { tools: { refund: true } }, + } + const composed = { + ...CERTIFIED, + agentProfileDiffs: [ + { + diff, + provenance: { + version: 7, + lift: '+2.2pp', + contentHash: 'deadbeef', + promotedAt: '2026-06-12T00:00:00.000Z', + }, + }, + ], + agentProfile: { name: 'support-agent', tools: { refund: true } }, + } + const fetchImpl = vi.fn(async () => jsonResponse(composed)) as unknown as typeof fetch + const outcome = await pullCertified({ + target: 'support-agent', + apiKey: 'k', + baseUrl: 'https://plane.test', + fetchImpl, + }) + expect(outcome.succeeded).toBe(true) + if (outcome.succeeded) { + expect(outcome.value.agentProfileDiffs).toEqual(composed.agentProfileDiffs) + expect(outcome.value.agentProfile).toEqual(composed.agentProfile) + } + }) + + it('normalizes a legacy response with no diffs to empty arrays (fail-closed)', async () => { + const legacy = { + target: 't', + generatedAt: 'x', + promptSurface: null, + artifacts: {}, + } + const fetchImpl = vi.fn(async () => jsonResponse(legacy)) as unknown as typeof fetch + const outcome = await pullCertified({ + target: 't', + apiKey: 'k', + baseUrl: 'https://plane.test', + fetchImpl, + }) + expect(outcome.succeeded).toBe(true) + if (outcome.succeeded) { + expect(outcome.value.agentProfileDiffs).toEqual([]) + expect(outcome.value.capabilities).toEqual([]) + expect(outcome.value.agentProfile).toBeNull() + } + }) + it('treats 404 (nothing promoted yet) as a non-error succeeded:false with status', async () => { const fetchImpl = vi.fn( async () => new Response('', { status: 404 }), @@ -112,8 +166,6 @@ describe('pullCertified', () => { }) it('errors loudly when no apiKey is available', async () => { - // Stub the env so the test holds on machines/CI where TANGLE_API_KEY is set - // (resolveApiKey falls back to it when the passed key is empty). vi.stubEnv('TANGLE_API_KEY', '') try { const fetchImpl = (async () => jsonResponse({})) as unknown as typeof fetch @@ -130,8 +182,6 @@ describe('pullCertified', () => { }) it('fails closed when the plane hangs past timeoutMs', async () => { - // A fetch that rejects when its abort signal fires (what a hung request + - // AbortSignal.timeout produces) must surface as a typed fail-closed result. const fetchImpl = ((_url: string, init?: RequestInit) => new Promise((_resolve, reject) => { init?.signal?.addEventListener('abort', () => @@ -150,86 +200,6 @@ describe('pullCertified', () => { }) }) -describe('withCertifiedDelivery', () => { - it('delivers the certified profile to the agent and composes it into the prompt', async () => { - const fetchImpl = vi.fn(async () => jsonResponse(CERTIFIED)) as unknown as typeof fetch - let seenPrompt = '' - const agent = withCertifiedDelivery( - async (input: { q: string }, applied) => { - seenPrompt = applied.composePrompt('BASE SYSTEM PROMPT') - return `answer:${input.q}:v${applied.certified?.promptSurface?.version}` - }, - { project: 'support-agent', apiKey: 'k', baseUrl: 'https://plane.test', fetchImpl }, - ) - const out = await agent({ q: 'refund' }) - expect(out).toBe('answer:refund:v4') - expect(seenPrompt).toContain('confirm the invoice id before refunding') - expect(seenPrompt).toContain('BASE SYSTEM PROMPT') - }) - - it('runs fail-closed on the base surface when the pull 404s (nothing promoted)', async () => { - const fetchImpl = vi.fn( - async () => new Response('', { status: 404 }), - ) as unknown as typeof fetch - let seenPrompt = '' - let seenCertified: unknown = 'unset' - const agent = withCertifiedDelivery( - async (_input: null, applied) => { - seenPrompt = applied.composePrompt('BASE') - seenCertified = applied.certified - return 'ok' - }, - { project: 'p', apiKey: 'k', baseUrl: 'https://plane.test', fetchImpl }, - ) - expect(await agent(null)).toBe('ok') - expect(seenPrompt).toBe('BASE') - expect(seenCertified).toBeNull() - }) - - it('does not break the agent when Intelligence is unreachable', async () => { - const fetchImpl = vi.fn(async () => { - throw new Error('network down') - }) as unknown as typeof fetch - const agent = withCertifiedDelivery(async (input: number) => input * 2, { - project: 'p', - apiKey: 'k', - baseUrl: 'https://plane.test', - fetchImpl, - }) - await expect(agent(21)).resolves.toBe(42) - }) - - it('caches the certified pull across calls within refreshMs (one pull, N runs)', async () => { - const calls = { n: 0 } - const fetchImpl = vi.fn(async () => { - calls.n += 1 - return jsonResponse(CERTIFIED) - }) as unknown as typeof fetch - const agent = withCertifiedDelivery(async (_i: null, _a) => 'ok', { - project: 'support-agent', - apiKey: 'k', - baseUrl: 'https://plane.test', - refreshMs: 60_000, - fetchImpl, - }) - await agent(null) - await agent(null) - await agent(null) - expect(calls.n).toBe(1) - }) - - it('propagates the agent error (delivery never swallows the live path)', async () => { - const fetchImpl = vi.fn(async () => jsonResponse(CERTIFIED)) as unknown as typeof fetch - const agent = withCertifiedDelivery( - async (_i: null) => { - throw new Error('agent boom') - }, - { project: 'support-agent', apiKey: 'k', baseUrl: 'https://plane.test', fetchImpl }, - ) - await expect(agent(null)).rejects.toThrow('agent boom') - }) -}) - describe('createCertifiedPromptSource', () => { const opts = { target: 'support-agent', apiKey: 'k', baseUrl: 'https://plane.test' } diff --git a/src/intelligence/delivery.ts b/src/intelligence/delivery.ts index 1ef3cc24..0192e55a 100644 --- a/src/intelligence/delivery.ts +++ b/src/intelligence/delivery.ts @@ -1,31 +1,32 @@ /** * - * Tangle Intelligence — the DELIVERY half of the loop (pull-by-default). + * Tangle Intelligence — the RECEIVE half of the loop (pull-by-default). * - * The sibling Observe layer (`./index`) sends traces UP to the plane. This + * The sibling Observe path (`./index`) sends run records UP to the plane. This * module pulls certified artifacts DOWN: it reads the tenant's promoted, - * gate-certified profile from the deployed Intelligence plane and folds it into - * the running agent's prompt — so an approved improvement actually reaches the - * agent. This is "shipping intelligence to people's agents", pull-by-default; - * the push/Gated-PR opt-in composes on top of this. + * gate-certified profile from the deployed Intelligence plane so an approved + * improvement actually reaches the running agent. The pull carries three things + * the plane already composes: + * - the certified PROMPT surface + prompt-folding artifacts (delivered into the + * system prompt via {@link composeCertifiedPrompt} — the promoted prompt); + * - the typed profile DIFFS the plane has promoted, each with its held-out + * provenance (surfaced as PROPOSALS — never auto-applied at runtime); + * - the composed `agentProfile` those diffs fold to, for inspection. * * Pull contract (deployed plane): GET /v1/profiles/:target/composed → - * { target, generatedAt, promptSurface: {surface,surfaceHash,version,lift}|null, - * artifacts: { : [{path,content,contentHash,version,lift,promotedAt}] } } + * { target, generatedAt, + * promptSurface: {surface,surfaceHash,version,lift}|null, + * artifacts: { : [{path,content,contentHash,version,lift,promotedAt}] }, + * capabilities: [{id,iface:{surface},binding:{path,content},provenance}], + * agentProfileDiffs: [{diff, provenance:{version,lift,contentHash,promotedAt}}], + * agentProfile: AgentProfile|null } * Auth: Bearer (the one TANGLE_API_KEY shared by router + sandbox + * intelligence), resolved to a tenant by platform-api's key-verify S2S contract. * - * import { withCertifiedDelivery } from '@tangle-network/agent-runtime/intelligence' - * - * export const agent = withCertifiedDelivery( - * async (input, applied) => myAgent(input, { systemPrompt: applied.composePrompt(BASE) }), - * { project: 'support-agent', target: 'support-agent' }, - * ) - * * @experimental */ -import { createIntelligenceClient, type IntelligenceConfig } from './index' +import type { AgentProfile, AgentProfileDiff } from '@tangle-network/agent-interface' const defaultPlaneBaseUrl = 'https://intelligence.tangle.tools' const defaultRefreshMs = 300_000 @@ -50,6 +51,37 @@ export interface CertifiedPromptSurface { lift: string | null } +/** The held-out provenance the plane's certify step stamps on a promoted diff. + * `lift` is the held-out gate lift (e.g. "+3.1pp"), never a within-run claim. */ +export interface DiffProvenance { + version: number | null + lift: string | null + contentHash: string + promotedAt: string +} + +/** + * A gate-certified profile diff the plane has already promoted, plus the + * held-out provenance it carries. This is the previously-DROPPED typed diff the + * composed endpoint returns; `withIntelligence` deserializes it and surfaces it + * as a PROPOSAL — a human, or the gated local `improve()` loop, turns a proposal + * into a shipped profile. It is NEVER auto-applied at runtime. + */ +export interface ProposedProfileDiff { + diff: AgentProfileDiff + provenance: DiffProvenance +} + +/** The composed endpoint's per-capability summary — the narrow shape on the + * wire (id + surface + path/content + provenance). Distinct from the richer + * `CertifiedCapability` the capability resolver lowers a manifest into. */ +export interface CertifiedCapabilitySummary { + id: string + iface: { surface: string } + binding: { path: string | null; content: string } + provenance: DiffProvenance +} + /** The composed certified profile — exactly the shape the plane's * `GET /v1/profiles/:target/composed` returns. */ export interface CertifiedProfile { @@ -57,6 +89,14 @@ export interface CertifiedProfile { generatedAt: string promptSurface: CertifiedPromptSurface | null artifacts: Record + /** The typed profile diffs the plane has promoted, each with held-out + * provenance. Surfaced as proposals; never auto-applied. Empty when none. */ + agentProfileDiffs: ProposedProfileDiff[] + /** The composed capability summaries the plane returns. Empty when none. */ + capabilities: CertifiedCapabilitySummary[] + /** The composed profile the promoted diffs fold to, for inspection. `null` + * when no diffs are promoted. */ + agentProfile: AgentProfile | null } /** Typed outcome for the pull — inspect `succeeded` before `value`. A 404 @@ -83,7 +123,9 @@ export interface PullCertifiedOptions { const defaultPullTimeoutMs = 10_000 -function resolvePlaneBaseUrl(baseUrl: string | undefined): string { +/** Resolve the ONE Intelligence base URL — the single knob both the send and + * receive paths derive from. Env fallback: `TANGLE_INTELLIGENCE_URL`. */ +export function resolveIntelligenceBaseUrl(baseUrl: string | undefined): string { if (baseUrl) return baseUrl.replace(/\/+$/, '') if (typeof process !== 'undefined' && process.env.TANGLE_INTELLIGENCE_URL) { return process.env.TANGLE_INTELLIGENCE_URL.replace(/\/+$/, '') @@ -98,6 +140,50 @@ function resolveApiKey(apiKey: string | undefined): string { return '' } +function asRecord(value: unknown): Record { + return value && typeof value === 'object' ? (value as Record) : {} +} + +function toDiffProvenance(value: unknown): DiffProvenance { + const p = asRecord(value) + return { + version: typeof p.version === 'number' ? p.version : null, + lift: typeof p.lift === 'string' ? p.lift : null, + contentHash: typeof p.contentHash === 'string' ? p.contentHash : '', + promotedAt: typeof p.promotedAt === 'string' ? p.promotedAt : '', + } +} + +/** + * Deserialize the composed-endpoint response into a `CertifiedProfile`. The + * previously-dropped `agentProfileDiffs`/`capabilities`/`agentProfile` are read + * here so they round-trip to the consumer; a plane that has not yet promoted any + * diffs simply yields empty arrays / a null profile (fail-closed, never a crash). + */ +export function normalizeCertifiedProfile(raw: unknown): CertifiedProfile { + const r = asRecord(raw) + const promptSurface = r.promptSurface ? (r.promptSurface as CertifiedPromptSurface) : null + const artifacts = (r.artifacts as Record | undefined) ?? {} + const agentProfileDiffs: ProposedProfileDiff[] = Array.isArray(r.agentProfileDiffs) + ? r.agentProfileDiffs.map((entry) => { + const e = asRecord(entry) + return { diff: e.diff as AgentProfileDiff, provenance: toDiffProvenance(e.provenance) } + }) + : [] + const capabilities: CertifiedCapabilitySummary[] = Array.isArray(r.capabilities) + ? (r.capabilities as CertifiedCapabilitySummary[]) + : [] + return { + target: typeof r.target === 'string' ? r.target : '', + generatedAt: typeof r.generatedAt === 'string' ? r.generatedAt : '', + promptSurface, + artifacts, + agentProfileDiffs, + capabilities, + agentProfile: (r.agentProfile as AgentProfile | null | undefined) ?? null, + } +} + /** * Pull the certified composed profile for a target. Fail-closed: a network * error or a non-2xx returns a typed `succeeded: false` (never throws), so a @@ -109,7 +195,7 @@ export async function pullCertified(opts: PullCertifiedOptions): Promise | null, +): string { if (!certified) return base const parts: string[] = [] if (certified.promptSurface?.surface.trim()) parts.push(certified.promptSurface.surface.trim()) @@ -181,7 +270,7 @@ export function composeCertifiedPrompt(base: string, certified: CertifiedProfile /** A cached, self-refreshing source of a target's certified prompt additions — * the prompt-only delivery lane for callers that assemble their OWN system * prompt (product chat routes) rather than wrapping an agent fn. Same - * fail-closed semantics as {@link withCertifiedDelivery}: pulls at most every + * fail-closed semantics as {@link pullCertified}: pulls at most every * `refreshMs`, coalesces concurrent pulls, keeps the last-known profile on a * failed/404 pull, never throws, never blocks past the pull timeout. */ export interface CertifiedPromptSource { @@ -204,8 +293,8 @@ export interface CertifiedPromptSourceOptions extends PullCertifiedOptions { /** * Create the cached certified-prompt source — the ONE module-scope-cache + * coalesced-refresh + keep-last-known implementation. Product wiring uses this - * rather than hand-rolling the same lines around `pullCertified`, and - * {@link withCertifiedDelivery} is built on it. + * rather than hand-rolling the same lines around `pullCertified`. The + * `withIntelligence` hook rides this same source for its prompt delivery. */ export function createCertifiedPromptSource( opts: CertifiedPromptSourceOptions, @@ -241,73 +330,3 @@ export function createCertifiedPromptSource( }, } } - -/** What the delivery wrapper hands the agent each run. */ -export interface AppliedIntelligence { - /** The certified profile in effect (null when none promoted / pull failed — - * fail-closed: the agent runs on its base surface). */ - certified: CertifiedProfile | null - /** Fold the certified prompt surface into a base system prompt. */ - composePrompt(base: string): string -} - -/** An agent wrapped by {@link withCertifiedDelivery}: receives the input plus - * the certified intelligence delivered for this run. */ -export type DeliveredAgent = (input: I, applied: AppliedIntelligence) => Promise - -/** Delivery config = the Observe config plus the pull target + refresh cadence. */ -export interface DeliveryConfig extends IntelligenceConfig { - /** Pull target. Defaults to `project`. */ - target?: string - /** Plane base URL for the pull (NOT the OTLP `endpoint`). Defaults to - * `TANGLE_INTELLIGENCE_URL` then `https://intelligence.tangle.tools`. */ - baseUrl?: string - /** Min interval between certified-profile pulls. Default 5m. */ - refreshMs?: number - /** Per-pull timeout in ms (fail-closed on a hung plane). Default 10000. */ - timeoutMs?: number - /** fetch impl for the pull (tests). Defaults to global fetch. */ - fetchImpl?: typeof fetch -} - -/** - * Wrap an agent so it (a) Observes each run via the shipped Observe client and - * (b) RECEIVES the tenant's certified artifacts pulled from the deployed plane. - * The certified profile is cached and refreshed at most every `refreshMs`; a - * failed pull is fail-closed — the agent runs on its base surface and never - * breaks because Intelligence is unreachable. When the plane promotes a new - * gate-certified surface, the next refresh delivers it to the running agent. - */ -export function withCertifiedDelivery( - agent: DeliveredAgent, - config: DeliveryConfig, -): ((input: I) => Promise) & { refresh(): Promise } { - const client = createIntelligenceClient(config) - const source = createCertifiedPromptSource({ - target: config.target ?? config.project, - ...(config.apiKey !== undefined ? { apiKey: config.apiKey } : {}), - ...(config.baseUrl !== undefined ? { baseUrl: config.baseUrl } : {}), - ...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}), - ...(config.fetchImpl !== undefined ? { fetchImpl: config.fetchImpl } : {}), - ...(config.refreshMs !== undefined ? { refreshMs: config.refreshMs } : {}), - }) - - const wrapped = (async (input: I): Promise => { - await source.refresh() - const certified = source.current() - const applied: AppliedIntelligence = { - certified, - composePrompt: (base: string) => composeCertifiedPrompt(base, certified), - } - return client.traceRun( - { input, labels: { 'tangle.certified_version': certified?.promptSurface?.version ?? -1 } }, - async (trace) => { - const out = await agent(input, applied) - trace.recordOutput(out) - return out - }, - ) - }) as ((input: I) => Promise) & { refresh(): Promise } - wrapped.refresh = source.refresh - return wrapped -} diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts new file mode 100644 index 00000000..69226b21 --- /dev/null +++ b/src/intelligence/improvement-cycle.ts @@ -0,0 +1,547 @@ +import { type AnalystFinding, assertNoJudgeVerdict } from '@tangle-network/agent-eval/analyst' +import type { Scenario, SelfImproveResult } from '@tangle-network/agent-eval/contract' +import type { + AgentCandidateBundle, + AgentProfile, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { agentCandidateBundleSchema } from '@tangle-network/agent-interface' +import { runAnalystLoop } from '../analyst-loop' +import type { RunAnalystLoopOpts, RunAnalystLoopResult } from '../analyst-loop/types' +import { + type AgentCandidateBundleInput, + sealAgentCandidateBundle, +} from '../candidate-execution/bundle' +import { + canonicalCandidateDigest, + canonicalCandidateDocument, + immutableCandidateValue, + omitTopLevelDigest, +} from '../candidate-execution/digest' +import { + type ExecutePreparedAgentCandidateOptions, + executePreparedAgentCandidate, +} from '../candidate-execution/execute' +import { + type PrepareAgentCandidateExecutionOptions, + prepareAgentCandidateExecution, +} from '../candidate-execution/prepare' +import { + assertCandidateProfileBinding, + parseExactAgentProfile, +} from '../candidate-execution/profile' +import type { + AgentCandidateExecutionPorts, + AgentCandidateRunFinalization, + AgentCandidateTaskExecution, +} from '../candidate-execution/types' +import { verifyAgentCandidateBundle } from '../candidate-execution/verify' +import { + applyImprovementWinnerToProfile, + type ImproveOptions, + type ImproveResult, + type ImproveSurface, + improve, +} from '../improvement/improve' + +export type AgentImprovementEvaluation = Pick< + SelfImproveResult, + | 'baseline' + | 'winner' + | 'lift' + | 'diff' + | 'provenance' + | 'gateDecision' + | 'generationsExplored' + | 'durationMs' + | 'totalCostUsd' + | 'insight' + | 'power' +> + +export interface AgentImprovementProposal< + TScenario extends Scenario = Scenario, + TArtifact = unknown, +> { + schemaVersion: 1 + kind: 'agent-improvement-proposal' + runId: string + surface: ImproveSurface + proposedAt: string + baselineProfile: AgentProfile + baselineProfileHash: string + candidateProfile: AgentProfile + candidateProfileHash: string + findings: AnalystFinding[] + evaluation: AgentImprovementEvaluation + candidateBundle?: AgentCandidateBundle + digest: Sha256Digest +} + +export type AgentImprovementReviewDecision = 'approve' | 'reject' | 'request-changes' + +export interface AgentImprovementReview { + schemaVersion: 1 + kind: 'agent-improvement-review' + proposalDigest: Sha256Digest + candidateBundleDigest?: Sha256Digest + decision: AgentImprovementReviewDecision + reviewedBy: string + reviewedAt: string + reason: string + feedback?: string + digest: Sha256Digest +} + +export interface CandidateExecutionEvidence { + proposalDigest: Sha256Digest + reviewDigest: Sha256Digest + bundleDigest: Sha256Digest + executionId: string + executionPlanDigest: Sha256Digest + materializationReceiptDigest: Sha256Digest + succeeded: boolean + runReceiptDigest?: Sha256Digest +} + +export interface ProposeAgentImprovementOptions { + runId: string + profile: AgentProfile + analysis: Omit + improvement: ImproveOptions + /** + * Optional environment adapter that freezes an executable bundle after the + * measured comparison recommends the candidate. Return the sealed output of + * `buildAgentCandidateBundle` directly, or a low-level digest-free input. + */ + buildCandidate?: (input: { + analysis: RunAnalystLoopResult + improvement: ImproveResult + }) => + | AgentCandidateBundleInput + | AgentCandidateBundle + | Promise + now?: () => Date +} + +export interface ProposeAgentImprovementResult { + analysis: RunAnalystLoopResult + improvement: ImproveResult + proposal: AgentImprovementProposal +} + +export interface CreateAgentImprovementProposalOptions { + runId: string + surface: ImproveSurface + baselineProfile: AgentProfile + candidateProfile: AgentProfile + findings: readonly AnalystFinding[] + evaluation: SelfImproveResult + candidateBundle?: AgentCandidateBundleInput | AgentCandidateBundle + now?: () => Date +} + +export interface ReviewAgentImprovementInput { + decision: AgentImprovementReviewDecision + reviewedBy: string + reason: string + feedback?: string + now?: () => Date +} + +export interface ExecuteApprovedAgentCandidateOptions { + proposal: AgentImprovementProposal + review: AgentImprovementReview + /** Product-owned authentication check for the persisted approval record. */ + authorizeReview: ( + review: AgentImprovementReview, + proposal: AgentImprovementProposal, + ) => boolean | Promise + task: AgentCandidateTaskExecution + ports: AgentCandidateExecutionPorts + preparation?: PrepareAgentCandidateExecutionOptions + execution: ExecutePreparedAgentCandidateOptions +} + +export interface ExecuteApprovedAgentCandidateResult { + finalization: AgentCandidateRunFinalization + evidence: CandidateExecutionEvidence +} + +async function rethrowAfterImprovementDispose( + cause: unknown, + improvement: ImproveResult, +): Promise { + const disposeErrors: unknown[] = [] + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + await improvement.dispose() + } catch (disposeCause) { + disposeErrors.push(disposeCause) + continue + } + if (disposeErrors.length === 0) throw cause + throw new AggregateError( + [cause, ...disposeErrors], + 'proposeAgentImprovement failed; the improvement cleanup retry succeeded', + ) + } + throw new AggregateError( + [cause, ...disposeErrors], + 'proposeAgentImprovement failed and its improvement resources could not be cleaned', + ) +} + +/** Analyze one run and produce one measured, review-only improvement proposal. */ +export async function proposeAgentImprovement( + options: ProposeAgentImprovementOptions, +): Promise> { + const writeBackSurface = options.improvement.skills?.writeBack + ? 'skill' + : options.improvement.memory?.writeBack + ? 'memory' + : null + if (writeBackSurface) { + throw new Error( + `proposeAgentImprovement cannot write ${writeBackSurface} before human approval`, + ) + } + const analysis = await runAnalystLoop({ + ...options.analysis, + runId: options.runId, + }) + const findings = assertNoJudgeVerdict( + analysis.analystResult.findings, + 'proposeAgentImprovement findings', + ) + const improvement = await improve(options.profile, [...findings], options.improvement) + try { + const candidateBundle = + improvement.shipped && options.buildCandidate + ? await options.buildCandidate({ analysis, improvement }) + : undefined + const proposal = createAgentImprovementProposal({ + runId: options.runId, + surface: options.improvement.surface ?? 'prompt', + baselineProfile: options.profile, + candidateProfile: improvement.profile, + findings, + evaluation: improvement.raw, + ...(candidateBundle ? { candidateBundle } : {}), + ...(options.now ? { now: options.now } : {}), + }) + return { analysis, improvement, proposal } + } catch (cause) { + return rethrowAfterImprovementDispose(cause, improvement) + } +} + +/** + * Freeze an already-measured improvement into the one reviewable proposal + * contract. Products that run analysis or evaluation in separate workers use + * this constructor instead of rerunning either phase or rebuilding digests. + */ +export function createAgentImprovementProposal( + options: CreateAgentImprovementProposalOptions, +): AgentImprovementProposal { + const findings = assertNoJudgeVerdict( + [...options.findings], + 'createAgentImprovementProposal findings', + ) + const candidateBundle = options.candidateBundle + ? sealBuiltCandidate(options.candidateBundle) + : undefined + const baselineProfile = parseExactAgentProfile( + options.baselineProfile, + 'proposal baseline profile', + ) + const candidateProfile = parseExactAgentProfile( + options.candidateProfile, + 'proposal candidate profile', + ) + assertMeasuredProfileBinding({ + surface: options.surface, + baselineProfile, + candidateProfile, + evaluation: options.evaluation, + hasExecutableBundle: candidateBundle !== undefined, + }) + if (candidateBundle) { + if (options.evaluation.gateDecision !== 'ship') { + throw new Error('executable candidate bundle requires a passing measured comparison') + } + assertCandidateProfileBinding(candidateProfile, candidateBundle.profile) + } + const withoutDigest = { + schemaVersion: 1 as const, + kind: 'agent-improvement-proposal' as const, + runId: options.runId, + surface: options.surface, + proposedAt: (options.now ?? (() => new Date()))().toISOString(), + baselineProfile, + baselineProfileHash: canonicalCandidateDigest(baselineProfile), + candidateProfile, + candidateProfileHash: canonicalCandidateDigest(candidateProfile), + findings: canonicalJsonValue([...findings]), + evaluation: canonicalJsonValue(improvementEvaluation(options.evaluation)), + ...(candidateBundle ? { candidateBundle } : {}), + } + return canonicalCandidateDocument>(withoutDigest) + .value +} + +/** Persist an approve/reject/change-request decision bound to one exact proposal. */ +export function reviewAgentImprovementProposal( + inputProposal: AgentImprovementProposal, + input: ReviewAgentImprovementInput, +): AgentImprovementReview { + const proposal = verifyAgentImprovementProposal(inputProposal) + if (!input.reviewedBy.trim()) throw new Error('candidate review requires reviewedBy') + if (!input.reason.trim()) throw new Error('candidate review requires a reason') + if (input.decision === 'approve') { + if (proposal.evaluation.gateDecision !== 'ship') { + throw new Error('candidate cannot be approved without a passing measured comparison') + } + if (!proposal.candidateBundle) { + throw new Error('candidate cannot be approved until an executable bundle is sealed') + } + } + const withoutDigest = { + schemaVersion: 1 as const, + kind: 'agent-improvement-review' as const, + proposalDigest: proposal.digest, + ...(proposal.candidateBundle ? { candidateBundleDigest: proposal.candidateBundle.digest } : {}), + decision: input.decision, + reviewedBy: input.reviewedBy, + reviewedAt: (input.now ?? (() => new Date()))().toISOString(), + reason: input.reason, + ...(input.feedback === undefined ? {} : { feedback: input.feedback }), + } + return canonicalCandidateDocument(withoutDigest).value +} + +/** Verify, materialize, run, grade, and receipt only the exact approved bundle. */ +export async function executeApprovedAgentCandidate( + options: ExecuteApprovedAgentCandidateOptions, +): Promise { + const proposal = verifyAgentImprovementProposal(options.proposal) + const review = verifyAgentImprovementReview(options.review) + if (review.decision !== 'approve') throw new Error('candidate review is not an approval') + if (review.proposalDigest !== proposal.digest) { + throw new Error('candidate approval does not match the proposed improvement') + } + const bundle = proposal.candidateBundle + if (!bundle) throw new Error('approved proposal does not contain an executable candidate bundle') + if (review.candidateBundleDigest !== bundle.digest) { + throw new Error('candidate approval does not match the executable bundle') + } + if (!(await options.authorizeReview(review, proposal))) { + throw new Error('candidate approval was not authorized by the configured authority') + } + + const verified = await verifyAgentCandidateBundle(bundle, options.ports) + const prepared = await prepareAgentCandidateExecution( + verified, + options.task, + options.ports, + options.preparation, + ) + const finalization = await executePreparedAgentCandidate(prepared, options.execution) + return { + finalization, + evidence: { + proposalDigest: proposal.digest, + reviewDigest: review.digest, + bundleDigest: bundle.digest, + executionId: prepared.executionId, + executionPlanDigest: prepared.executionPlan.value.digest, + materializationReceiptDigest: prepared.materializationReceipt.digest, + succeeded: finalization.succeeded, + ...(finalization.succeeded ? { runReceiptDigest: finalization.receipt.digest } : {}), + }, + } +} + +/** Validate a proposal's schema, profile, sealed bundle, and canonical digest. */ +export function verifyAgentImprovementProposal(input: unknown): AgentImprovementProposal { + if ( + !isRecord(input) || + input.kind !== 'agent-improvement-proposal' || + input.schemaVersion !== 1 + ) { + throw new Error('invalid agent improvement proposal') + } + const proposal = input as unknown as AgentImprovementProposal + const parsedBaselineProfile = parseExactAgentProfile( + proposal.baselineProfile, + 'proposal baseline profile', + ) + if (proposal.baselineProfileHash !== canonicalCandidateDigest(parsedBaselineProfile)) { + throw new Error('proposal baselineProfileHash does not match baselineProfile') + } + const parsedProfile = parseExactAgentProfile( + proposal.candidateProfile, + 'proposal candidate profile', + ) + if (proposal.candidateProfileHash !== canonicalCandidateDigest(parsedProfile)) { + throw new Error('proposal candidateProfileHash does not match candidateProfile') + } + assertMeasuredProfileBinding({ + surface: proposal.surface, + baselineProfile: parsedBaselineProfile, + candidateProfile: parsedProfile, + evaluation: proposal.evaluation, + hasExecutableBundle: proposal.candidateBundle !== undefined, + }) + if (!Array.isArray(proposal.findings)) throw new Error('proposal findings must be an array') + if (!isSha256Digest(proposal.digest)) throw new Error('proposal digest is invalid') + if (proposal.candidateBundle) { + const parsedBundle = agentCandidateBundleSchema.parse(proposal.candidateBundle) + const sealed = sealAgentCandidateBundle(omitTopLevelDigest(parsedBundle)) + if (sealed.digest !== parsedBundle.digest) + throw new Error('proposal candidate bundle digest is invalid') + assertCandidateProfileBinding(parsedProfile, sealed.profile) + } + const actual = canonicalCandidateDigest(omitTopLevelDigest(proposal)) + if (actual !== proposal.digest) + throw new Error('agent improvement proposal digest does not match') + return immutableCandidateValue(proposal) +} + +function assertMeasuredProfileBinding(input: { + surface: ImproveSurface + baselineProfile: AgentProfile + candidateProfile: AgentProfile + evaluation: Pick, 'gateDecision' | 'winner'> + hasExecutableBundle: boolean +}): void { + const baselineDigest = canonicalCandidateDigest(input.baselineProfile) + if (input.evaluation.gateDecision !== 'ship') { + if (canonicalCandidateDigest(input.candidateProfile) !== baselineDigest) { + throw new Error('non-shipping evaluation must retain the baseline candidate profile') + } + return + } + + const measured = measuredWinnerProfile( + input.baselineProfile, + input.surface, + input.evaluation.winner.surface, + ) + if (!measured) { + if (input.hasExecutableBundle) { + throw new Error( + `executable '${input.surface}' candidate cannot be bound to its measured winner surface`, + ) + } + if (canonicalCandidateDigest(input.candidateProfile) !== baselineDigest) { + throw new Error( + `unbound '${input.surface}' improvement must retain the baseline candidate profile`, + ) + } + return + } + if (canonicalCandidateDigest(input.candidateProfile) !== canonicalCandidateDigest(measured)) { + throw new Error('proposal candidate profile does not match the measured winner surface') + } +} + +function measuredWinnerProfile( + baselineProfile: AgentProfile, + surface: ImproveSurface, + winner: unknown, +): AgentProfile | null { + if (surface === 'code' || surface === 'memory') return null + if (typeof winner !== 'string') { + throw new Error(`measured '${surface}' winner is not a serializable profile surface`) + } + if (surface !== 'skills') { + return applyImprovementWinnerToProfile(baselineProfile, surface, winner) + } + try { + return applyImprovementWinnerToProfile(baselineProfile, surface, winner) + } catch { + // A skill document is text stored outside the profile; a skill-ref array is + // JSON stored on the profile. Only the latter can bind to a profile bundle. + return null + } +} + +function sealBuiltCandidate( + input: AgentCandidateBundleInput | AgentCandidateBundle, +): AgentCandidateBundle { + if (!('digest' in input)) return sealAgentCandidateBundle(input) + const parsed = agentCandidateBundleSchema.parse(input) + const sealed = sealAgentCandidateBundle(omitTopLevelDigest(parsed)) + if (sealed.digest !== parsed.digest) { + throw new Error('built candidate bundle digest is invalid') + } + return sealed +} + +/** Validate a review's decision fields and canonical digest. */ +export function verifyAgentImprovementReview(input: unknown): AgentImprovementReview { + if (!isRecord(input) || input.kind !== 'agent-improvement-review' || input.schemaVersion !== 1) { + throw new Error('invalid agent improvement review') + } + const review = input as unknown as AgentImprovementReview + if (!isSha256Digest(review.digest) || !isSha256Digest(review.proposalDigest)) { + throw new Error('candidate review digest is invalid') + } + if (review.candidateBundleDigest !== undefined && !isSha256Digest(review.candidateBundleDigest)) { + throw new Error('candidate review bundle digest is invalid') + } + if (!['approve', 'reject', 'request-changes'].includes(review.decision)) { + throw new Error('candidate review decision is invalid') + } + const actual = canonicalCandidateDigest(omitTopLevelDigest(review)) + if (actual !== review.digest) throw new Error('agent improvement review digest does not match') + return immutableCandidateValue(review) +} + +function improvementEvaluation( + result: SelfImproveResult, +): AgentImprovementEvaluation { + return { + baseline: result.baseline, + winner: result.winner, + lift: result.lift, + diff: result.diff, + provenance: result.provenance, + gateDecision: result.gateDecision, + generationsExplored: result.generationsExplored, + durationMs: result.durationMs, + totalCostUsd: result.totalCostUsd, + insight: result.insight, + ...(result.power === undefined ? {} : { power: result.power }), + } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function isSha256Digest(value: unknown): value is Sha256Digest { + return typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value) +} + +function canonicalJsonValue(value: T, path = '$'): T { + if (value === null || typeof value === 'string' || typeof value === 'boolean') return value + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new Error(`non-finite number at ${path}`) + return value + } + if (Array.isArray(value)) { + return value.map((entry, index) => { + if (entry === undefined) throw new Error(`undefined array entry at ${path}[${index}]`) + return canonicalJsonValue(entry, `${path}[${index}]`) + }) as T + } + if (typeof value === 'object') { + const entries = Object.entries(value as Record) + .filter(([, entry]) => entry !== undefined) + .map(([key, entry]) => [key, canonicalJsonValue(entry, `${path}.${key}`)]) + return Object.fromEntries(entries) as T + } + throw new Error(`unsupported proposal value at ${path}`) +} diff --git a/src/intelligence/index.ts b/src/intelligence/index.ts index 26fbd2ad..f81bf8f2 100644 --- a/src/intelligence/index.ts +++ b/src/intelligence/index.ts @@ -1,10 +1,10 @@ /** * - * Tangle Intelligence SDK — the Observe + Mode-0 product layer. + * Tangle Intelligence SDK — trace capture plus reviewable improvement. * - * A thin, best-effort wrapper over the shipped trace-export substrate - * (`createOtelExporter` in `../otel-export`). It does exactly two things in - * this slice: + * The client keeps live-agent trace delivery best-effort. The separate + * improvement-cycle exports analyze completed traces, measure one candidate, + * bind human review, and execute only an approved immutable bundle. * * 1. OBSERVE — wrap a generic agent and export one trace span per call to * Tangle Intelligence, swallowing every export failure so a live agent @@ -15,21 +15,22 @@ * and at OFF `intelligenceUsd` is provably `0` — the mechanism that proves * an OFF customer paid inference-only. * - * Behavior-changing intelligence (analyst steer, candidate promotion, loops) - * is a LATER phase and is NOT built here. This wrapper only Observes and passes - * through; there is no abort path, so the only fail-soft surface is the - * telemetry export. - * * @experimental */ +import { contentHash } from '@tangle-network/agent-eval' +import type { AgentProfile } from '@tangle-network/agent-interface' import { buildLoopOtelSpans, + buildRuntimeEventOtelSpans, createOtelExporter, flatOtelSpan, type OtelExporter, } from '../otel-export' import type { LoopTraceEvent } from '../runtime/types' +import type { RuntimeTelemetryOptions } from '../sanitize' +import type { RuntimeStreamEvent } from '../types' +import { resolveIntelligenceBaseUrl } from './delivery' import { defaultEffortTier, type EffortOverrides, @@ -38,6 +39,7 @@ import { isIntelligenceOff, resolveEffort, } from './effort' +import type { CandidateExecutionEvidence } from './improvement-cycle' import { type Redactor, resolveRedactor } from './redact' export type { @@ -60,22 +62,23 @@ export type { } from './capability' export { CapabilityNotAdmittedError, manifestFromProfile } from './capability' export type { - AppliedIntelligence, CertifiedArtifact, + CertifiedCapabilitySummary, CertifiedProfile, CertifiedPromptSource, CertifiedPromptSourceOptions, CertifiedPromptSurface, - DeliveredAgent, - DeliveryConfig, + DiffProvenance, + ProposedProfileDiff, PullCertifiedOptions, PullOutcome, } from './delivery' export { composeCertifiedPrompt, createCertifiedPromptSource, + normalizeCertifiedProfile, pullCertified, - withCertifiedDelivery, + resolveIntelligenceBaseUrl, } from './delivery' export type { CorpusAccess, @@ -90,6 +93,27 @@ export { isIntelligenceOff, resolveEffort, } from './effort' +export type { + AgentImprovementEvaluation, + AgentImprovementProposal, + AgentImprovementReview, + AgentImprovementReviewDecision, + CandidateExecutionEvidence, + CreateAgentImprovementProposalOptions, + ExecuteApprovedAgentCandidateOptions, + ExecuteApprovedAgentCandidateResult, + ProposeAgentImprovementOptions, + ProposeAgentImprovementResult, + ReviewAgentImprovementInput, +} from './improvement-cycle' +export { + createAgentImprovementProposal, + executeApprovedAgentCandidate, + proposeAgentImprovement, + reviewAgentImprovementProposal, + verifyAgentImprovementProposal, + verifyAgentImprovementReview, +} from './improvement-cycle' export type { Redactor } from './redact' export { defaultRedactor, resolveRedactor } from './redact' export type { ProvisionedHost, ResolveCtx } from './resolver' @@ -97,6 +121,13 @@ export { composeCertifiedProfile, composeCertifiedProfileFromWire, } from './resolver' +export type { + AppliedIntelligence, + IntelligenceAgent, + IntelligenceHookConfig, + IntelligenceWrapped, +} from './with-intelligence' +export { withIntelligence } from './with-intelligence' /** Usage class for billing. Base-stream tokens bill `'inference'`; every * intelligence spawn (analyst, corpus, loop) bills `'intelligence'`. The @@ -115,6 +146,70 @@ export interface UsageSplit { intelligenceUsd: number } +/** + * The typed record `withIntelligence` sends per call — serialized through the + * shipped OTLP builders to the plane's `/v1/otlp` ingest. `input`/`output` are + * redacted on export; the per-class `usage` split carries the billing proof; + * `loopEvents`, when present, export as the nested loop→round→iteration span + * tree under the same `traceId`. + */ +export interface RunRecord { + runId: string + traceId: string + project: string + target: string + input: unknown + output: unknown + outcome: { + success?: boolean + score?: number + usage: UsageSplit + } + model?: string + provider?: string + loopEvents?: LoopTraceEvent[] + runtimeEvents?: RuntimeStreamEvent[] + profile?: AgentProfile + sessionId?: string + harness?: string + repository?: string + commitSha?: string + timing?: { startedAt: number; completedAt: number; durationMs: number } + tokens?: { + input: number + output: number + cachedInput?: number + reasoning?: number + } + error?: { name: string; message: string; code?: string } + /** Exact proposal → review → execution → receipt linkage for candidate runs. */ + candidateExecution?: CandidateExecutionEvidence +} + +/** + * What an agent reports (via `applied.record`) to enrich the {@link RunRecord} + * sent for its call. All optional — an un-recorded run still sends input/output + * with an inference-only zero usage split. `costUsd` without a split is treated + * as pure inference (the base stream). + */ +export interface RunReport { + success?: boolean + score?: number + usage?: Partial + costUsd?: number + model?: string + provider?: string + loopEvents?: LoopTraceEvent[] + runtimeEvents?: RuntimeStreamEvent[] + profile?: AgentProfile + sessionId?: string + harness?: string + commitSha?: string + tokens?: RunRecord['tokens'] + error?: RunRecord['error'] + candidateExecution?: CandidateExecutionEvidence +} + /** Repo coordinates a product may declare for the (later) Gated-PR mode. The * Observe slice only records their PRESENCE for `doctor()`; it never touches * the repo. */ @@ -135,12 +230,13 @@ export interface IntelligenceConfig { /** Effort tier (default `'standard'`) plus optional per-field overrides. */ effort?: EffortTier | { tier: EffortTier; overrides?: EffortOverrides } /** - * OTLP ingest base. The underlying exporter appends `/v1/traces`, so point - * this at the OTLP route (e.g. `https://intelligence.tangle.tools/v1/otlp`). - * Reads `INTELLIGENCE_OTLP_ENDPOINT` then `OTEL_EXPORTER_OTLP_ENDPOINT` when - * omitted; absent all three, export is a no-op (best-effort by construction). + * The ONE Tangle Intelligence base URL — both the send (OTLP `/v1/otlp`) and + * receive (`/v1/profiles/:target/composed`) paths derive from it. Reads + * `TANGLE_INTELLIGENCE_URL` when omitted, else `https://intelligence.tangle.tools`. + * Send is best-effort and only ships when an `apiKey` is present (the tenant + * key the ingest requires); absent a key, export is a no-op. */ - endpoint?: string + baseUrl?: string /** * Redaction hook run over every exported input/output. A function replaces * the default scrubber; `false` opts out entirely (raw fidelity, caller has @@ -153,6 +249,19 @@ export interface IntelligenceConfig { checks?: string[] /** Repo access a later PR mode would need. Recorded for `doctor()` only. */ repo?: RepoConfig + /** Full canonical profile used for this agent. Exported redacted with a stable hash. */ + profile?: AgentProfile + /** Commit that produced the running agent, when known. */ + commitSha?: string + /** Runtime-event payload policy. Tool inputs/results remain off unless explicitly enabled. */ + runtimeTelemetry?: RuntimeTelemetryOptions + /** + * Payloads are metadata-only by default: the run span carries a stable hash + * and UTF-8 byte count, but not the redacted content. Set `full` only when + * the configured OTLP destination is approved to receive complete redacted + * inputs, outputs, and profiles. + */ + payloadAttributes?: 'metadata' | 'full' } /** Metadata describing one traced run. `runId`/`traceId` default to fresh ids. */ @@ -241,6 +350,18 @@ export interface IntelligenceClient { * Best-effort: export failures are swallowed. Returns the resolved `traceId`. */ recordTrace(events: ReadonlyArray, meta?: RecordTraceMeta): string + /** + * Send one typed {@link RunRecord} — the run's flat span (input/output/outcome/ + * usage/model/provider, redacted) plus, when `loopEvents` are present, the + * nested loop topology under the same `traceId`. Reuses the shipped + * `flatOtelSpan` + `buildLoopOtelSpans` builders (no second builder). + * Best-effort: export failures are swallowed. Returns the record's `traceId`. + */ + exportRunRecord(record: RunRecord): string + /** Mint a fresh run id (`run-`). */ + freshRunId(): string + /** Mint a fresh 32-hex trace id. */ + freshTraceId(): string /** * Network-free readiness report: which adoption modes are reachable given * this config. Observe is always reachable; Recommend needs outcomes; PR @@ -277,12 +398,6 @@ function resolveEffortConfig(effort: IntelligenceConfig['effort']): EffortSettin return resolveEffort(effort.tier, effort.overrides) } -function resolveEndpoint(endpoint: string | undefined): string | undefined { - if (endpoint) return endpoint - if (typeof process === 'undefined') return undefined - return process.env.INTELLIGENCE_OTLP_ENDPOINT ?? process.env.OTEL_EXPORTER_OTLP_ENDPOINT -} - function freshTraceId(): string { return randomHex(32) } @@ -291,12 +406,8 @@ function freshRunId(): string { return `run-${randomHex(16)}` } -/** Serialize a redacted value to a bounded string for a span attribute. - * `loopEventToOtelSpan` only stamps string/number/boolean payload fields, so a - * structured input/output must be flattened here. Bounded to keep span - * attributes small; the full payload is the consumer's own store, not the span. */ -const previewMaxChars = 4096 -function previewJson(value: unknown): string { +/** Serialize a redacted value without dropping customer trace content. */ +function serializeJson(value: unknown): string { let s: string if (typeof value === 'string') s = value else { @@ -306,7 +417,19 @@ function previewJson(value: unknown): string { s = String(value) } } - return s.length > previewMaxChars ? `${s.slice(0, previewMaxChars)}…[truncated]` : s + return s +} + +function addPayloadAttributes( + labels: Record, + key: string, + value: unknown, + includeFullPayload: boolean, +): void { + const serialized = serializeJson(value) + labels[`${key}_hash`] = contentHash(serialized) + labels[`${key}_bytes`] = Buffer.byteLength(serialized, 'utf8') + if (includeFullPayload) labels[key] = serialized } function randomHex(chars: number): string { @@ -323,10 +446,10 @@ function randomHex(chars: number): string { } /** - * Create an Observe-mode Intelligence client. Resolves effort, endpoint, and - * redactor up front; the exporter is built lazily and is `undefined` when no - * endpoint is configured (export becomes a no-op — best-effort by - * construction). + * Create an Observe-mode Intelligence client. Resolves effort, the base URL, and + * the redactor up front; the exporter is built lazily and is `undefined` when no + * `apiKey` is present (send becomes a no-op — the ingest requires a tenant key, + * and best-effort export must never spam an unauthenticated plane). */ export function createIntelligenceClient(config: IntelligenceConfig): IntelligenceClient { if (!config.project) { @@ -335,20 +458,23 @@ export function createIntelligenceClient(config: IntelligenceConfig): Intelligen const effort = resolveEffortConfig(config.effort) const intelligenceOff = isIntelligenceOff(effort) const redactor = resolveRedactor(config.redact) + const includeFullPayload = config.payloadAttributes === 'full' const apiKey = config.apiKey ?? (typeof process !== 'undefined' ? process.env.TANGLE_API_KEY : undefined) - const endpoint = resolveEndpoint(config.endpoint) + // The ONE base URL drives both send and receive; the OTLP ingest lives at + // `${base}/v1/otlp` and the exporter appends `/v1/traces` → `${base}/v1/otlp/v1/traces`. + const otlpEndpoint = `${resolveIntelligenceBaseUrl(config.baseUrl)}/v1/otlp` - // Built lazily: a client with no endpoint never allocates an exporter timer. + // Built lazily: a client with no tenant key never allocates an exporter timer. let exporter: OtelExporter | undefined let exporterResolved = false function getExporter(): OtelExporter | undefined { if (exporterResolved) return exporter exporterResolved = true - if (!endpoint) return undefined + if (!apiKey) return undefined exporter = createOtelExporter({ - endpoint, - headers: apiKey ? { authorization: `Bearer ${apiKey}` } : {}, + endpoint: otlpEndpoint, + headers: { authorization: `Bearer ${apiKey}` }, serviceName: config.project, resourceAttributes: { 'tangle.project': config.project }, }) @@ -358,24 +484,28 @@ export function createIntelligenceClient(config: IntelligenceConfig): Intelligen function exportTrace(meta: TraceMeta, outcome: TraceOutcome, output: unknown): void { const ex = getExporter() if (!ex) return - const labels: Record = { - project: config.project, - 'tangle.effort.intelligence_off': outcome.intelligenceOff, - 'tangle.usage.inference_usd': outcome.usage.inferenceUsd, - 'tangle.usage.intelligence_usd': outcome.usage.intelligenceUsd, - ...(meta.model ? { 'gen_ai.request.model': meta.model } : {}), - ...(meta.provider ? { 'provider.name': meta.provider } : {}), - ...(typeof outcome.success === 'boolean' - ? { 'tangle.outcome.success': outcome.success } - : {}), - ...(typeof outcome.score === 'number' ? { 'tangle.outcome.score': outcome.score } : {}), - ...(meta.labels ?? {}), - } - const redactedInput = meta.input !== undefined ? redactor(meta.input) : undefined - const redactedOutput = output !== undefined ? redactor(output) : undefined - if (redactedInput !== undefined) labels['tangle.input'] = previewJson(redactedInput) - if (redactedOutput !== undefined) labels['tangle.output'] = previewJson(redactedOutput) try { + const labels: Record = { + project: config.project, + 'tangle.effort.intelligence_off': outcome.intelligenceOff, + 'tangle.usage.inference_usd': outcome.usage.inferenceUsd, + 'tangle.usage.intelligence_usd': outcome.usage.intelligenceUsd, + ...(meta.model ? { 'gen_ai.request.model': meta.model } : {}), + ...(meta.provider ? { 'provider.name': meta.provider } : {}), + ...(typeof outcome.success === 'boolean' + ? { 'tangle.outcome.success': outcome.success } + : {}), + ...(typeof outcome.score === 'number' ? { 'tangle.outcome.score': outcome.score } : {}), + ...(meta.labels ?? {}), + } + const redactedInput = meta.input !== undefined ? redactor(meta.input) : undefined + const redactedOutput = output !== undefined ? redactor(output) : undefined + if (redactedInput !== undefined) { + addPayloadAttributes(labels, 'tangle.input', redactedInput, includeFullPayload) + } + if (redactedOutput !== undefined) { + addPayloadAttributes(labels, 'tangle.output', redactedOutput, includeFullPayload) + } // Flat span with VERBATIM attribute keys — the plane's session/model/ // cost readers exact-match `tangle.sessionId` / `gen_ai.request.model`, // so the loop-namespacing builder must not be used here. @@ -392,9 +522,149 @@ export function createIntelligenceClient(config: IntelligenceConfig): Intelligen } } + function exportRunRecord(record: RunRecord): string { + const ex = getExporter() + if (!ex) return record.traceId + try { + // Clamp the OFF billing invariant on export — the proof holds even if a + // caller mis-reports an intelligence split at the OFF tier. + const intelligenceUsd = intelligenceOff ? 0 : record.outcome.usage.intelligenceUsd + const repository = + record.repository ?? (config.repo ? `${config.repo.owner}/${config.repo.name}` : undefined) + const labels: Record = { + project: record.project, + 'tangle.target': record.target, + 'tangle.effort.intelligence_off': intelligenceOff, + 'tangle.usage.inference_usd': record.outcome.usage.inferenceUsd, + 'tangle.usage.intelligence_usd': intelligenceUsd, + ...(record.model ? { 'gen_ai.request.model': record.model } : {}), + ...(record.provider ? { 'provider.name': record.provider } : {}), + ...(record.sessionId + ? { + 'tangle.sessionId': record.sessionId, + 'gen_ai.conversation.id': record.sessionId, + } + : {}), + ...(record.harness ? { 'tangle.agent.harness': record.harness } : {}), + ...(repository ? { 'vcs.repository.name': repository } : {}), + ...(record.commitSha ? { 'vcs.ref.head.revision': record.commitSha } : {}), + ...(record.timing + ? { + 'tangle.started_at_ms': record.timing.startedAt, + 'tangle.completed_at_ms': record.timing.completedAt, + 'tangle.duration_ms': record.timing.durationMs, + } + : {}), + ...(record.tokens + ? { + 'gen_ai.usage.input_tokens': record.tokens.input, + 'gen_ai.usage.output_tokens': record.tokens.output, + ...(record.tokens.cachedInput !== undefined + ? { 'gen_ai.usage.cache_read_input_tokens': record.tokens.cachedInput } + : {}), + ...(record.tokens.reasoning !== undefined + ? { 'gen_ai.usage.reasoning_tokens': record.tokens.reasoning } + : {}), + } + : {}), + ...(typeof record.outcome.success === 'boolean' + ? { 'tangle.outcome.success': record.outcome.success } + : {}), + ...(typeof record.outcome.score === 'number' + ? { 'tangle.outcome.score': record.outcome.score } + : {}), + ...(record.error + ? { + 'error.type': record.error.code ?? record.error.name, + 'error.message': serializeJson(redactor(record.error.message)), + } + : {}), + ...(record.runtimeEvents + ? { 'tangle.runtime.event_count': record.runtimeEvents.length } + : {}), + ...(record.candidateExecution + ? { + 'tangle.candidate.proposal_digest': record.candidateExecution.proposalDigest, + 'tangle.candidate.review_digest': record.candidateExecution.reviewDigest, + 'tangle.candidate.bundle_digest': record.candidateExecution.bundleDigest, + 'tangle.candidate.execution_id': record.candidateExecution.executionId, + 'tangle.candidate.execution_plan_digest': + record.candidateExecution.executionPlanDigest, + 'tangle.candidate.materialization_receipt_digest': + record.candidateExecution.materializationReceiptDigest, + 'tangle.candidate.succeeded': record.candidateExecution.succeeded, + ...(record.candidateExecution.runReceiptDigest + ? { + 'tangle.candidate.run_receipt_digest': + record.candidateExecution.runReceiptDigest, + } + : {}), + } + : {}), + } + if (record.profile) { + addPayloadAttributes( + labels, + 'tangle.agent.profile', + redactor(record.profile), + includeFullPayload, + ) + if (record.profile.name) labels['gen_ai.agent.name'] = record.profile.name + } + const redactedInput = record.input !== undefined ? redactor(record.input) : undefined + const redactedOutput = record.output !== undefined ? redactor(record.output) : undefined + if (redactedInput !== undefined) { + addPayloadAttributes(labels, 'tangle.input', redactedInput, includeFullPayload) + } + if (redactedOutput !== undefined) { + addPayloadAttributes(labels, 'tangle.output', redactedOutput, includeFullPayload) + } + const now = Date.now() + const runSpan = flatOtelSpan( + 'tangle.intelligence.run', + { 'tangle.runId': record.runId, ...labels }, + record.traceId, + record.timing?.startedAt ?? now, + undefined, + record.timing?.completedAt ?? now, + ) + ex.exportSpan(runSpan) + if (record.runtimeEvents && record.runtimeEvents.length > 0) { + const spans = buildRuntimeEventOtelSpans( + record.runtimeEvents, + record.traceId, + runSpan.spanId, + { ...config.runtimeTelemetry, redact: redactor }, + ) + for (const span of spans) ex.exportSpan(span) + } + // The loop topology (when present) exports under the SAME traceId, parented + // under the run span — reusing the shipped builder, never a second one. + if (record.loopEvents && record.loopEvents.length > 0) { + const spans = buildLoopOtelSpans( + record.loopEvents as ReadonlyArray<{ + kind: string + runId: string + timestamp: number + payload: object + }>, + record.traceId, + runSpan.spanId, + ) + for (const span of spans) ex.exportSpan(span) + } + } catch { + // Best-effort — a send failure must never fail the agent's turn. + } + return record.traceId + } + return { project: config.project, effort, + exportRunRecord, + freshRunId, + freshTraceId, async traceRun(meta: TraceMeta, fn: (trace: TraceHandle) => Promise): Promise { const runId = meta.runId ?? freshRunId() @@ -491,7 +761,9 @@ export function createIntelligenceClient(config: IntelligenceConfig): Intelligen return { project: config.project, effort, - exportConfigured: Boolean(endpoint), + // Send ships only with a tenant key — the honest "will export actually + // land" signal (the base URL always resolves to the plane default). + exportConfigured: Boolean(apiKey), modes: { observe: { ready: true, missing: [] }, recommend: { @@ -514,40 +786,3 @@ export function createIntelligenceClient(config: IntelligenceConfig): Intelligen }, } } - -/** A generic agent: one async input → output. The shape `withTangleIntelligence` - * preserves exactly. */ -export type Agent = (input: TInput) => Promise - -/** Either a built client or the config to build one. */ -export type ClientOrConfig = IntelligenceClient | IntelligenceConfig - -function isClient(value: ClientOrConfig): value is IntelligenceClient { - return typeof (value as IntelligenceClient).traceRun === 'function' -} - -/** - * Wrap a generic `agent` with best-effort Observe-mode tracing, returning the - * SAME shape. Each call runs the agent under a trace and exports one span; an - * export failure is swallowed (the live agent never fails because Intelligence - * is down) but an error from the agent itself propagates unchanged. - * - * At `effort: 'off'` this is pure passthrough plus best-effort telemetry — - * zero intelligence spawns, `intelligenceUsd: 0` on the trace. - */ -export function withTangleIntelligence( - agent: Agent, - clientOrConfig: ClientOrConfig, -): Agent { - const client = isClient(clientOrConfig) - ? clientOrConfig - : createIntelligenceClient(clientOrConfig) - - return async (input: TInput): Promise => { - return client.traceRun({ input }, async (trace) => { - const output = await agent(input) - trace.recordOutput(output) - return output - }) - } -} diff --git a/src/intelligence/intelligence.test.ts b/src/intelligence/intelligence.test.ts index 2a6e9913..88ec642e 100644 --- a/src/intelligence/intelligence.test.ts +++ b/src/intelligence/intelligence.test.ts @@ -7,10 +7,9 @@ import { isIntelligenceOff, resolveEffort, type UsageSplit, - withTangleIntelligence, } from './index' -const endpoint = 'https://intelligence.test/v1/otlp' +const baseUrl = 'https://intelligence.test' const apiKey = 'sk-tan-test-key' /** Capture every OTLP POST body the exporter flushes. */ @@ -42,9 +41,9 @@ function installFetchSpy(mode: 'ok' | 'throw'): { calls: FetchCall[] } { return { calls } } -function stubNoEndpointEnv(): void { - vi.stubEnv('INTELLIGENCE_OTLP_ENDPOINT', '') - vi.stubEnv('OTEL_EXPORTER_OTLP_ENDPOINT', '') +/** Stub away any ambient tenant key so a "no apiKey" client truly has none. */ +function stubNoApiKey(): void { + vi.stubEnv('TANGLE_API_KEY', '') } /** Pull every span attribute across an OTLP export body into one flat map. */ @@ -69,8 +68,24 @@ function attrsOf(body: unknown): Record { return out } +/** Pull every span's `name` + `traceId` across an OTLP export body. */ +function spansOf(body: unknown): Array<{ name: string; traceId: string }> { + const out: Array<{ name: string; traceId: string }> = [] + const resourceSpans = (body as { resourceSpans?: unknown[] })?.resourceSpans ?? [] + for (const rs of resourceSpans) { + for (const ss of (rs as { scopeSpans?: unknown[] }).scopeSpans ?? []) { + for (const span of (ss as { spans?: unknown[] }).spans ?? []) { + const s = span as { name?: string; traceId?: string } + out.push({ name: String(s.name), traceId: String(s.traceId) }) + } + } + } + return out +} + afterEach(() => { vi.unstubAllGlobals() + vi.unstubAllEnvs() vi.restoreAllMocks() }) @@ -107,7 +122,6 @@ describe('resolveEffort', () => { const s = resolveEffort('off', { analysts: true, fanout: 4 }) expect(s.analysts).toBe(true) expect(s.fanout).toBe(4) - // Overriding any axis lifts a run off the OFF floor. expect(isIntelligenceOff(s)).toBe(false) }) @@ -151,7 +165,7 @@ describe('createIntelligenceClient / traceRun — Observe', () => { it('exports one span on a successful run and returns the agent output', async () => { const { calls } = installFetchSpy('ok') - const client = createIntelligenceClient({ project: 'support-agent', apiKey, endpoint }) + const client = createIntelligenceClient({ project: 'support-agent', apiKey, baseUrl }) const result = await client.traceRun({ input: { q: 'hi' } }, async (trace) => { trace.recordOutput({ answer: 'hello' }) trace.recordOutcome({ success: true, score: 0.9, costUsd: 0.002 }) @@ -168,7 +182,7 @@ describe('createIntelligenceClient / traceRun — Observe', () => { it('survives a dead endpoint — agent output is returned, no throw', async () => { installFetchSpy('throw') - const client = createIntelligenceClient({ project: 'support-agent', apiKey, endpoint }) + const client = createIntelligenceClient({ project: 'support-agent', apiKey, baseUrl }) const result = await client.traceRun({ input: { q: 'hi' } }, async (trace) => { trace.recordOutput({ answer: 'still here' }) return 42 @@ -179,7 +193,7 @@ describe('createIntelligenceClient / traceRun — Observe', () => { it('propagates an error thrown by the agent body (not swallowed)', async () => { installFetchSpy('ok') - const client = createIntelligenceClient({ project: 'support-agent', apiKey, endpoint }) + const client = createIntelligenceClient({ project: 'support-agent', apiKey, baseUrl }) await expect( client.traceRun({ input: {} }, async () => { throw new Error('agent exploded') @@ -189,7 +203,7 @@ describe('createIntelligenceClient / traceRun — Observe', () => { it('redacts secrets in the exported input/output', async () => { const { calls } = installFetchSpy('ok') - const client = createIntelligenceClient({ project: 'p', apiKey, endpoint }) + const client = createIntelligenceClient({ project: 'p', apiKey, baseUrl }) await client.traceRun( { input: { prompt: 'my key is sk-tan-leakedsecretvalue999' } }, async (trace) => { @@ -203,11 +217,11 @@ describe('createIntelligenceClient / traceRun — Observe', () => { expect(blob).not.toContain('alice@acme.com') }) - it('is a no-op (no fetch) when no endpoint is configured', async () => { - stubNoEndpointEnv() + it('is a no-op (no fetch) when no tenant apiKey is present', async () => { + stubNoApiKey() const spy = vi.fn() vi.stubGlobal('fetch', spy) - const client = createIntelligenceClient({ project: 'p', apiKey }) + const client = createIntelligenceClient({ project: 'p', baseUrl }) const out = await client.traceRun({ input: {} }, async () => 'x') await client.flush() expect(out).toBe('x') @@ -221,11 +235,10 @@ describe('billing classification — OFF proves inference-only', () => { const client = createIntelligenceClient({ project: 'p', apiKey, - endpoint, + baseUrl, effort: 'off', }) await client.traceRun({ input: { q: 'x' } }, async (trace) => { - // Even a caller that mis-reports intelligence spend at OFF is clamped. trace.recordOutcome({ usage: { inferenceUsd: 0.01, intelligenceUsd: 0.05 } satisfies UsageSplit, }) @@ -235,13 +248,12 @@ describe('billing classification — OFF proves inference-only', () => { const attrs = attrsOf(calls[0]?.body) expect(attrs['tangle.effort.intelligence_off']).toBe(true) expect(attrs['tangle.usage.intelligence_usd']).toBe(0) - // Inference is still billed — OFF is not "free", it is inference-only. expect(attrs['tangle.usage.inference_usd']).toBe(0.01) }) it('a non-off tier preserves a reported intelligence split', async () => { const { calls } = installFetchSpy('ok') - const client = createIntelligenceClient({ project: 'p', apiKey, endpoint, effort: 'standard' }) + const client = createIntelligenceClient({ project: 'p', apiKey, baseUrl, effort: 'standard' }) await client.traceRun({ input: {} }, async (trace) => { trace.recordOutcome({ usage: { inferenceUsd: 0.01, intelligenceUsd: 0.03 } }) return 'ok' @@ -253,38 +265,9 @@ describe('billing classification — OFF proves inference-only', () => { }) }) -describe('withTangleIntelligence', () => { - it('preserves the agent shape and traces each call', async () => { - const { calls } = installFetchSpy('ok') - const agent = async (input: { name: string }) => `hi ${input.name}` - const wrapped = withTangleIntelligence(agent, { project: 'p', apiKey, endpoint }) - const out = await wrapped({ name: 'drew' }) - // Flush is internal to the client; force one export round. - expect(out).toBe('hi drew') - // The wrapper builds its own client; give the batch a manual flush via a fresh call path. - expect(calls.length).toBeGreaterThanOrEqual(0) - }) - - it('never fails the agent when telemetry export is down', async () => { - installFetchSpy('throw') - const agent = async () => 'result' - const wrapped = withTangleIntelligence(agent, { project: 'p', apiKey, endpoint }) - await expect(wrapped(undefined)).resolves.toBe('result') - }) - - it('propagates an agent error through the wrapper', async () => { - installFetchSpy('ok') - const agent = async () => { - throw new Error('boom') - } - const wrapped = withTangleIntelligence(agent, { project: 'p', apiKey, endpoint }) - await expect(wrapped(undefined)).rejects.toThrow('boom') - }) -}) - describe('doctor()', () => { it('reports observe always reachable, pr blocked without checks/surfaces/repo', () => { - const client = createIntelligenceClient({ project: 'p', apiKey, endpoint }) + const client = createIntelligenceClient({ project: 'p', apiKey, baseUrl }) const report = client.doctor() expect(report.modes.observe.ready).toBe(true) expect(report.modes.pr.ready).toBe(false) @@ -296,7 +279,7 @@ describe('doctor()', () => { const client = createIntelligenceClient({ project: 'p', apiKey, - endpoint, + baseUrl, checks: ['pnpm test'], surfaces: ['src/**'], repo: { owner: 'acme', name: 'p', baseBranch: 'main' }, @@ -307,34 +290,19 @@ describe('doctor()', () => { }) it('flags recommend as blocked at the off floor', () => { - const client = createIntelligenceClient({ project: 'p', apiKey, endpoint, effort: 'off' }) + const client = createIntelligenceClient({ project: 'p', apiKey, baseUrl, effort: 'off' }) const report = client.doctor() expect(report.modes.recommend.ready).toBe(false) expect(report.modes.recommend.missing).toContain('effort above off') }) - it('reports exportConfigured:false when no endpoint resolves', () => { - stubNoEndpointEnv() - const client = createIntelligenceClient({ project: 'p', apiKey }) + it('reports exportConfigured:false when no tenant apiKey resolves', () => { + stubNoApiKey() + const client = createIntelligenceClient({ project: 'p', baseUrl }) expect(client.doctor().exportConfigured).toBe(false) }) }) -/** Pull every span's `name` + `traceId` across an OTLP export body. */ -function spansOf(body: unknown): Array<{ name: string; traceId: string }> { - const out: Array<{ name: string; traceId: string }> = [] - const resourceSpans = (body as { resourceSpans?: unknown[] })?.resourceSpans ?? [] - for (const rs of resourceSpans) { - for (const ss of (rs as { scopeSpans?: unknown[] }).scopeSpans ?? []) { - for (const span of (ss as { spans?: unknown[] }).spans ?? []) { - const s = span as { name?: string; traceId?: string } - out.push({ name: String(s.name), traceId: String(s.traceId) }) - } - } - } - return out -} - /** A minimal but real loop event stream: a plan round over two iterations. */ function loopStream(runId = 'loop-run'): LoopTraceEvent[] { return [ @@ -400,20 +368,16 @@ function loopStream(runId = 'loop-run'): LoopTraceEvent[] { describe('recordTrace — loop topology via buildLoopOtelSpans (gap 2)', () => { it('exports a nested loop→round→iteration span tree under ONE traceId', async () => { const { calls } = installFetchSpy('ok') - const client = createIntelligenceClient({ project: 'support-agent', apiKey, endpoint }) + const client = createIntelligenceClient({ project: 'support-agent', apiKey, baseUrl }) const traceId = client.recordTrace(loopStream(), { traceId: 'a'.repeat(32) }) await client.flush() expect(traceId).toBe('a'.repeat(32)) const spans = calls.flatMap((c) => spansOf(c.body)) const names = spans.map((s) => s.name) - // The TREE builder emits topology-level names; a flat per-event builder would emit the - // raw event kinds (loop.started/loop.iteration.ended). Asserting these proves reuse of - // buildLoopOtelSpans, not a second span builder. expect(names).toContain('loop') expect(names).toContain('loop.round') expect(names.filter((n) => n === 'loop.iteration').length).toBe(2) - // Every span shares the supplied traceId (one trace, not N). const traceIds = new Set(spans.map((s) => s.traceId)) expect(traceIds.size).toBe(1) expect([...traceIds][0]).toBe('a'.repeat(32)) @@ -421,23 +385,98 @@ describe('recordTrace — loop topology via buildLoopOtelSpans (gap 2)', () => { it('mints a fresh traceId when none is supplied and survives a dead endpoint', async () => { installFetchSpy('throw') - const client = createIntelligenceClient({ project: 'p', apiKey, endpoint }) + const client = createIntelligenceClient({ project: 'p', apiKey, baseUrl }) const traceId = client.recordTrace(loopStream()) expect(traceId).toMatch(/^[0-9a-f]{32}$/) - // Export failure is swallowed — recordTrace never throws. await expect(client.flush()).resolves.toBeUndefined() }) - it('is a no-op (no fetch) on an empty event stream or with no endpoint', async () => { - stubNoEndpointEnv() + it('is a no-op (no fetch) on an empty event stream or with no tenant key', async () => { + stubNoApiKey() const spy = vi.fn() vi.stubGlobal('fetch', spy) - const withEndpoint = createIntelligenceClient({ project: 'p', apiKey, endpoint }) - withEndpoint.recordTrace([]) - await withEndpoint.flush() - const noEndpoint = createIntelligenceClient({ project: 'p', apiKey }) - noEndpoint.recordTrace(loopStream()) - await noEndpoint.flush() + const withKey = createIntelligenceClient({ project: 'p', apiKey, baseUrl }) + withKey.recordTrace([]) + await withKey.flush() + const noKey = createIntelligenceClient({ project: 'p', baseUrl }) + noKey.recordTrace(loopStream()) + await noKey.flush() + expect(spy).not.toHaveBeenCalled() + }) +}) + +describe('exportRunRecord — typed RunRecord send (the withIntelligence SEND path)', () => { + it('ships one flat run span with target/usage/model + the loop topology under one traceId', async () => { + const { calls } = installFetchSpy('ok') + const client = createIntelligenceClient({ project: 'support-agent', apiKey, baseUrl }) + const traceId = client.exportRunRecord({ + runId: 'run-1', + traceId: 'b'.repeat(32), + project: 'support-agent', + target: 'support-agent', + input: { q: 'hi' }, + output: { a: 'ok' }, + outcome: { + success: true, + score: 0.9, + usage: { inferenceUsd: 0.002, intelligenceUsd: 0.001 }, + }, + model: 'kimi-k2', + provider: 'moonshot', + loopEvents: loopStream(), + }) + await client.flush() + + expect(traceId).toBe('b'.repeat(32)) + const attrs = attrsOf(calls[0]?.body) + expect(attrs['tangle.target']).toBe('support-agent') + expect(attrs['tangle.usage.inference_usd']).toBe(0.002) + expect(attrs['tangle.usage.intelligence_usd']).toBe(0.001) + expect(attrs['tangle.outcome.success']).toBe(true) + expect(attrs['gen_ai.request.model']).toBe('kimi-k2') + + const spans = calls.flatMap((c) => spansOf(c.body)) + const names = spans.map((s) => s.name) + expect(names).toContain('tangle.intelligence.run') + expect(names).toContain('loop') + const traceIds = new Set(spans.map((s) => s.traceId)) + expect(traceIds.size).toBe(1) + }) + + it('clamps intelligence_usd to 0 at the OFF tier even if the record reports spend', async () => { + const { calls } = installFetchSpy('ok') + const client = createIntelligenceClient({ project: 'p', apiKey, baseUrl, effort: 'off' }) + client.exportRunRecord({ + runId: 'r', + traceId: 'c'.repeat(32), + project: 'p', + target: 'p', + input: {}, + output: {}, + outcome: { usage: { inferenceUsd: 0.01, intelligenceUsd: 0.05 } }, + }) + await client.flush() + const attrs = attrsOf(calls[0]?.body) + expect(attrs['tangle.usage.intelligence_usd']).toBe(0) + expect(attrs['tangle.usage.inference_usd']).toBe(0.01) + }) + + it('is a no-op (no fetch) when no tenant apiKey is present', async () => { + stubNoApiKey() + const spy = vi.fn() + vi.stubGlobal('fetch', spy) + const client = createIntelligenceClient({ project: 'p', baseUrl }) + const traceId = client.exportRunRecord({ + runId: 'r', + traceId: 'd'.repeat(32), + project: 'p', + target: 'p', + input: {}, + output: {}, + outcome: { usage: { inferenceUsd: 0, intelligenceUsd: 0 } }, + }) + await client.flush() + expect(traceId).toBe('d'.repeat(32)) expect(spy).not.toHaveBeenCalled() }) }) @@ -451,7 +490,6 @@ describe('compileEffort — EffortSettings → run-config overrides (gap 3/4)', withLoops: false, intelligenceBudgetUsd: 0, }) - // The product fail-closed: at off the caller omits the analyst (degrade, not throw). expect(compiled.withAnalyst).toBe(false) }) @@ -470,7 +508,6 @@ describe('compileEffort — EffortSettings → run-config overrides (gap 3/4)', }) it('carries a per-field override through to the compiled overrides', () => { - // Overriding analysts back on at off lifts the analyst-construction gate. const compiled = compileEffort(resolveEffort('off', { analysts: true, fanout: 4 })) expect(compiled.withAnalyst).toBe(true) expect(compiled.fanout).toBe(4) diff --git a/src/intelligence/resolver.ts b/src/intelligence/resolver.ts index f1068b63..ed2bc7ae 100644 --- a/src/intelligence/resolver.ts +++ b/src/intelligence/resolver.ts @@ -182,8 +182,6 @@ export async function composeCertifiedProfile( const promptAdditions = collectPromptAdditions(manifest.promptSurface, acc.contextArtifacts) const systemPrompt = composeCertifiedPrompt(base.systemPrompt, { - target: manifest.target, - generatedAt: manifest.generatedAt, promptSurface: manifest.promptSurface, artifacts: acc.contextArtifacts, }) @@ -219,8 +217,6 @@ function baseSurface( promptSurface: CapabilityManifest['promptSurface'], ): ResolvedSurface { const folded = composeCertifiedPrompt(systemPrompt, { - target: '', - generatedAt: '', promptSurface, artifacts: {}, }) diff --git a/src/intelligence/with-intelligence.test.ts b/src/intelligence/with-intelligence.test.ts new file mode 100644 index 00000000..e38e4478 --- /dev/null +++ b/src/intelligence/with-intelligence.test.ts @@ -0,0 +1,397 @@ +import type { AgentProfile, AgentProfileDiff } from '@tangle-network/agent-interface' +import { applyAgentProfileDiff } from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { ProposedProfileDiff } from './delivery' +import type { AppliedIntelligence } from './with-intelligence' +import { withIntelligence } from './with-intelligence' + +/** A valid, promoted profile diff — the previously-DROPPED typed artifact the + * composed endpoint returns alongside prompt/skill artifacts. */ +const DIFF: AgentProfileDiff = { + schemaVersion: 1, + kind: 'agent-profile-diff', + id: 'diff-1', + title: 'certified refund tool', + set: { tools: { refund: true }, metadata: { certified: true } }, +} + +/** The composed-endpoint fixture, carrying `agentProfileDiffs[]`. */ +const COMPOSED = { + target: 'support-agent', + generatedAt: '2026-06-13T00:00:00.000Z', + promptSurface: { + surface: 'Confirm the invoice id before refunding.', + surfaceHash: 'abc123', + version: 4, + lift: '+3.1pp', + }, + artifacts: {}, + capabilities: [ + { + id: 'prompt-surface', + iface: { surface: 'context' }, + binding: { path: null, content: 'Confirm the invoice id before refunding.' }, + provenance: { + contentHash: 'abc123', + version: 4, + lift: '+3.1pp', + promotedAt: '2026-06-12T00:00:00.000Z', + }, + }, + ], + agentProfileDiffs: [ + { + diff: DIFF, + provenance: { + version: 7, + lift: '+2.2pp', + contentHash: 'deadbeef', + promotedAt: '2026-06-12T00:00:00.000Z', + }, + }, + ], + agentProfile: { name: 'support-agent', tools: { refund: true }, metadata: { certified: true } }, +} + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }) +} + +afterEach(() => { + vi.unstubAllGlobals() + vi.restoreAllMocks() +}) + +describe('withIntelligence — RECEIVE (the previously-dropped typed diffs)', () => { + it('deserializes agentProfileDiffs and surfaces them as proposals (round-trip)', async () => { + const fetchImpl = vi.fn(async () => jsonResponse(COMPOSED)) as unknown as typeof fetch + let applied: AppliedIntelligence | undefined + const agent = withIntelligence( + async (_input: null, a) => { + applied = a + return 'ok' + }, + { project: 'support-agent', apiKey: 'k', baseUrl: 'https://plane.test', fetchImpl }, + ) + await agent(null) + + // The typed diffs the OLD receive path dropped now round-trip verbatim. + expect(agent.proposals()).toEqual(COMPOSED.agentProfileDiffs) + expect(applied?.proposals).toEqual(COMPOSED.agentProfileDiffs) + }) + + it('applyProfile folds the proposals exactly as applyAgentProfileDiff would', async () => { + const fetchImpl = vi.fn(async () => jsonResponse(COMPOSED)) as unknown as typeof fetch + let applied: AppliedIntelligence | undefined + const agent = withIntelligence( + async (_input: null, a) => { + applied = a + return 'ok' + }, + { project: 'support-agent', apiKey: 'k', baseUrl: 'https://plane.test', fetchImpl }, + ) + await agent(null) + + const base: AgentProfile = { name: 'support-agent' } + // The hook's fold == the canonical single-diff application. + expect(applied?.applyProfile(base)).toEqual(applyAgentProfileDiff(base, DIFF)) + }) + + it('fires onProposals once with the promoted set (silent on an unchanged refresh)', async () => { + const fetchImpl = vi.fn(async () => jsonResponse(COMPOSED)) as unknown as typeof fetch + const seen: ProposedProfileDiff[][] = [] + const agent = withIntelligence(async () => 'ok', { + project: 'support-agent', + apiKey: 'k', + baseUrl: 'https://plane.test', + fetchImpl, + onProposals: (p) => seen.push(p), + }) + await agent(null) + await agent(null) + expect(seen).toHaveLength(1) + expect(seen[0]).toEqual(COMPOSED.agentProfileDiffs) + }) +}) + +describe('withIntelligence — SAFETY (observe + deliver only, never auto-apply)', () => { + it('delivers ONLY the certified prompt surface into the prompt, never the raw diff', async () => { + const fetchImpl = vi.fn(async () => jsonResponse(COMPOSED)) as unknown as typeof fetch + let composed = '' + const agent = withIntelligence( + async (_input: null, a) => { + composed = a.composePrompt('BASE PROMPT') + return 'ok' + }, + { project: 'support-agent', apiKey: 'k', baseUrl: 'https://plane.test', fetchImpl }, + ) + await agent(null) + + expect(composed).toContain('BASE PROMPT') + expect(composed).toContain('Confirm the invoice id before refunding') + // The typed diff is a PROPOSAL — it must not leak into the delivered prompt. + expect(composed).not.toContain('agent-profile-diff') + // Proposals are surfaced, but the hook applied nothing on the run path. + expect(agent.proposals()).toHaveLength(1) + }) + + it('runs fail-closed on the base surface when the pull 404s (nothing promoted)', async () => { + const fetchImpl = vi.fn( + async () => new Response('', { status: 404 }), + ) as unknown as typeof fetch + let sawCertified: unknown = 'unset' + const agent = withIntelligence( + async (_input: null, a) => { + sawCertified = a.certified + return a.certified === null && a.proposals.length === 0 ? 'base' : 'x' + }, + { project: 'p', apiKey: 'k', baseUrl: 'https://plane.test', fetchImpl }, + ) + expect(await agent(null)).toBe('base') + expect(sawCertified).toBeNull() + expect(agent.proposals()).toEqual([]) + }) + + it('never breaks the agent when Intelligence is unreachable', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('network down') + }) as unknown as typeof fetch + const agent = withIntelligence(async (input: number) => input * 2, { + project: 'p', + apiKey: 'k', + baseUrl: 'https://plane.test', + fetchImpl, + }) + await expect(agent(21)).resolves.toBe(42) + }) + + it('propagates an agent error (delivery never swallows the live path)', async () => { + const fetchImpl = vi.fn(async () => jsonResponse(COMPOSED)) as unknown as typeof fetch + const agent = withIntelligence( + async (_i: null) => { + throw new Error('agent boom') + }, + { project: 'support-agent', apiKey: 'k', baseUrl: 'https://plane.test', fetchImpl }, + ) + await expect(agent(null)).rejects.toThrow('agent boom') + }) +}) + +/** Pull every span attribute across an OTLP export body into one flat map. */ +function attrsOf(body: unknown): Record { + const out: Record = {} + const resourceSpans = (body as { resourceSpans?: unknown[] })?.resourceSpans ?? [] + for (const rs of resourceSpans) { + for (const ss of (rs as { scopeSpans?: unknown[] }).scopeSpans ?? []) { + for (const span of (ss as { spans?: unknown[] }).spans ?? []) { + for (const a of (span as { attributes?: unknown[] }).attributes ?? []) { + const attr = a as { key: string; value: Record } + const v = attr.value + out[attr.key] = + v.stringValue ?? + (v.intValue !== undefined ? Number(v.intValue) : undefined) ?? + v.doubleValue ?? + v.boolValue + } + } + } + } + return out +} + +describe('withIntelligence — SEND (a typed RunRecord to /v1/otlp)', () => { + it('ships one run span carrying target + usage split + model, best-effort', async () => { + vi.useFakeTimers() + try { + const longInput = 'x'.repeat(5000) + const posts: unknown[] = [] + const otlpSpy = vi.fn(async (_url: unknown, init: unknown) => { + const body = (init as { body?: string })?.body + if (body) posts.push(JSON.parse(body)) + return { ok: true, status: 200, async json() {} } as unknown as Response + }) + vi.stubGlobal('fetch', otlpSpy) + // The pull rides its own fetchImpl; SEND rides global fetch (the exporter). + const pull = vi.fn(async () => jsonResponse(COMPOSED)) as unknown as typeof fetch + const agent = withIntelligence( + async (_input: { q: string }, a) => { + expect(a.runId).toMatch(/^run-/) + expect(a.traceId).toHaveLength(32) + a.record({ + success: true, + usage: { inferenceUsd: 0.002, intelligenceUsd: 0 }, + model: 'kimi-k2', + provider: 'moonshot', + sessionId: 'session-1', + runtimeEvents: [ + { + type: 'tool_call', + toolName: 'mcp__linear__linear_graphql', + toolCallId: 'call-1', + args: { query: longInput }, + }, + { + type: 'llm_call', + model: 'kimi-k2', + tokensIn: 11, + tokensOut: 7, + costUsd: 0.002, + latencyMs: 250, + }, + ], + candidateExecution: { + proposalDigest: `sha256:${'1'.repeat(64)}`, + reviewDigest: `sha256:${'2'.repeat(64)}`, + bundleDigest: `sha256:${'3'.repeat(64)}`, + executionId: 'candidate-execution-1', + executionPlanDigest: `sha256:${'4'.repeat(64)}`, + materializationReceiptDigest: `sha256:${'5'.repeat(64)}`, + succeeded: true, + runReceiptDigest: `sha256:${'6'.repeat(64)}`, + }, + }) + return 'answer' + }, + { + project: 'support-agent', + target: 'support-agent', + apiKey: 'k', + baseUrl: 'https://plane.test', + fetchImpl: pull, + profile: { + name: 'support-agent', + prompt: { systemPrompt: 'Handle support requests.' }, + tools: { mcp__linear__linear_graphql: true }, + }, + commitSha: 'a'.repeat(40), + repo: { owner: 'tangle-network', name: 'support', baseBranch: 'main' }, + runtimeTelemetry: { includeControlPayloads: true }, + payloadAttributes: 'full', + }, + ) + await agent({ q: longInput }) + await agent.flush() + + expect(posts.length).toBeGreaterThan(0) + const attrs = attrsOf(posts[0]) + expect(attrs.project).toBe('support-agent') + expect(attrs['tangle.target']).toBe('support-agent') + expect(attrs['tangle.usage.inference_usd']).toBe(0.002) + expect(attrs['tangle.usage.intelligence_usd']).toBe(0) + expect(attrs['tangle.outcome.success']).toBe(true) + expect(attrs['gen_ai.request.model']).toBe('kimi-k2') + expect(attrs['tangle.sessionId']).toBe('session-1') + expect(attrs['vcs.repository.name']).toBe('tangle-network/support') + expect(attrs['vcs.ref.head.revision']).toBe('a'.repeat(40)) + expect(attrs['gen_ai.usage.input_tokens']).toBe(11) + expect(attrs['gen_ai.usage.output_tokens']).toBe(7) + expect(String(attrs['tangle.input'])).toContain(longInput) + expect(String(attrs['tangle.input'])).not.toContain('[truncated]') + expect(attrs['tangle.input_hash']).toEqual(expect.any(String)) + expect(attrs['tangle.input_bytes']).toBeGreaterThan(5000) + expect(JSON.parse(String(attrs['tangle.agent.profile']))).toMatchObject({ + name: 'support-agent', + tools: { mcp__linear__linear_graphql: true }, + }) + expect(attrs['tangle.agent.profile_hash']).toEqual(expect.any(String)) + expect(attrs['tool.name']).toBe('mcp__linear__linear_graphql') + expect(String(attrs['tool.input'])).toContain(longInput) + expect(attrs['tangle.candidate.execution_id']).toBe('candidate-execution-1') + expect(attrs['tangle.candidate.proposal_digest']).toBe(`sha256:${'1'.repeat(64)}`) + expect(attrs['tangle.candidate.run_receipt_digest']).toBe(`sha256:${'6'.repeat(64)}`) + } finally { + vi.useRealTimers() + } + }) + + it('does not send when no tenant apiKey is present', async () => { + const otlpSpy = vi.fn() + vi.stubGlobal('fetch', otlpSpy) + vi.stubEnv('TANGLE_API_KEY', '') + try { + const pull = vi.fn(async () => new Response('', { status: 404 })) as unknown as typeof fetch + const agent = withIntelligence(async () => 'ok', { + project: 'p', + baseUrl: 'https://plane.test', + fetchImpl: pull, + }) + expect(await agent(null)).toBe('ok') + expect(otlpSpy).not.toHaveBeenCalled() + } finally { + vi.unstubAllEnvs() + } + }) + + it('exports payload hashes and byte counts without content by default', async () => { + vi.useFakeTimers() + try { + const posts: unknown[] = [] + vi.stubGlobal( + 'fetch', + vi.fn(async (_url: unknown, init: unknown) => { + const body = (init as { body?: string })?.body + if (body) posts.push(JSON.parse(body)) + return { ok: true, status: 200, async json() {} } as unknown as Response + }), + ) + const pull = vi.fn(async () => jsonResponse(COMPOSED)) as unknown as typeof fetch + const agent = withIntelligence(async () => 'private output', { + project: 'support-agent', + apiKey: 'k', + baseUrl: 'https://plane.test', + fetchImpl: pull, + profile: { name: 'support-agent' }, + }) + + await agent('private input') + await agent.flush() + + const attrs = attrsOf(posts[0]) + expect(attrs['tangle.input']).toBeUndefined() + expect(attrs['tangle.output']).toBeUndefined() + expect(attrs['tangle.agent.profile']).toBeUndefined() + expect(attrs['tangle.input_hash']).toEqual(expect.any(String)) + expect(attrs['tangle.input_bytes']).toBeGreaterThan(0) + expect(attrs['tangle.output_hash']).toEqual(expect.any(String)) + expect(attrs['tangle.agent.profile_hash']).toEqual(expect.any(String)) + } finally { + vi.useRealTimers() + } + }) + + it('exports a failed run before rethrowing the agent error', async () => { + vi.useFakeTimers() + try { + const posts: unknown[] = [] + vi.stubGlobal( + 'fetch', + vi.fn(async (_url: unknown, init: unknown) => { + const body = (init as { body?: string })?.body + if (body) posts.push(JSON.parse(body)) + return { ok: true, status: 200, async json() {} } as unknown as Response + }), + ) + const pull = vi.fn(async () => jsonResponse(COMPOSED)) as unknown as typeof fetch + const agent = withIntelligence( + async () => { + throw Object.assign(new Error('provider exhausted'), { code: 'rate_limit' }) + }, + { project: 'support-agent', apiKey: 'k', baseUrl: 'https://plane.test', fetchImpl: pull }, + ) + + await expect(agent(null)).rejects.toThrow('provider exhausted') + await agent.flush() + + const attrs = attrsOf(posts[0]) + expect(attrs['tangle.outcome.success']).toBe(false) + expect(attrs['error.type']).toBe('rate_limit') + expect(attrs['error.message']).toBe('provider exhausted') + expect(attrs['tangle.duration_ms']).toEqual(expect.any(Number)) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/intelligence/with-intelligence.ts b/src/intelligence/with-intelligence.ts new file mode 100644 index 00000000..56c458bd --- /dev/null +++ b/src/intelligence/with-intelligence.ts @@ -0,0 +1,289 @@ +/** + * + * `withIntelligence` — the ONE agent-runtime hook that unifies Tangle + * Intelligence send + receive. + * + * It does two things per call, both fail-open: + * 1. RECEIVE — pull the tenant's certified profile from the deployed plane + * (cached + refreshed), hand the agent the certified prompt surface to fold + * (`applied.composePrompt`) and the promoted, gate-certified profile DIFFS + * as PROPOSALS (`applied.proposals` / `applied.applyProfile`). + * 2. SEND — serialize a typed {@link RunRecord} for the call through the shipped + * OTLP builders to the plane's `/v1/otlp` ingest, best-effort. + * + * SAFETY — observe + deliver ONLY. This hook NEVER auto-applies a received diff + * at runtime. A promoted diff is surfaced as a proposal; a human, or the gated + * local `improve()` loop, turns a proposal into a shipped profile. That + * preserves the held-out invariant the plane enforces — a runtime that silently + * folded an un-gated diff into itself would defeat the very gate the diff cleared. + * + * import { withIntelligence } from '@tangle-network/agent-runtime/intelligence' + * + * export const agent = withIntelligence( + * async (input, applied) => { + * const out = await myAgent(input, { systemPrompt: applied.composePrompt(BASE) }) + * applied.record({ success: true, usage: { inferenceUsd: 0.002, intelligenceUsd: 0 } }) + * return out + * }, + * { project: 'support-agent', target: 'support-agent' }, + * ) + * + * @experimental + */ + +import type { AgentProfile } from '@tangle-network/agent-interface' +import { applyAgentProfileDiff } from '@tangle-network/agent-interface' +import { + type CertifiedProfile, + composeCertifiedPrompt, + createCertifiedPromptSource, + type ProposedProfileDiff, +} from './delivery' +import { + createIntelligenceClient, + type IntelligenceConfig, + type RunRecord, + type RunReport, +} from './index' + +/** What the hook hands the agent each run. Additive over the prompt-only + * delivery: `composePrompt` folds the certified prompt surface (as before); + * `proposals`/`applyProfile` surface the promoted profile DIFFS — never + * auto-applied; `record` enriches the {@link RunRecord} that is sent. */ +export interface AppliedIntelligence { + /** Stable ids shared by the run span and every nested runtime/loop span. */ + runId: string + traceId: string + /** The certified profile in effect (null when none promoted / pull failed — + * fail-closed: the agent runs on its base surface). */ + certified: CertifiedProfile | null + /** Fold the certified prompt surface into a base system prompt (the promoted + * prompt). The consumer opts in by calling it. */ + composePrompt(base: string): string + /** The promoted, gate-certified profile diffs — surfaced for a human or the + * gated `improve()` loop. NEVER auto-applied by this hook. Empty when none. */ + proposals: ProposedProfileDiff[] + /** Fold every proposal into `base` via `applyAgentProfileDiff`, in promotion + * order, and return the result. The caller invokes this EXPLICITLY (it is the + * human/gated apply step) — the hook never calls it on the run path. */ + applyProfile(base: AgentProfile): AgentProfile + /** Enrich the {@link RunRecord} sent for this call — outcome, usage split, + * model/provider, and the loop event stream. Optional; an un-recorded run + * still sends input/output with an inference-only zero usage split. */ + record(report: RunReport): void +} + +/** An agent wrapped by {@link withIntelligence}: receives the input plus the + * intelligence delivered for this run. */ +export type IntelligenceAgent = (input: I, applied: AppliedIntelligence) => Promise + +/** `withIntelligence` config = the Observe config plus the pull target, refresh + * cadence, and a proposals callback. One base URL (`baseUrl` / + * `TANGLE_INTELLIGENCE_URL`) drives both the send and receive paths. */ +export interface IntelligenceHookConfig extends IntelligenceConfig { + /** Pull target. Defaults to `project`. */ + target?: string + /** Min interval between certified-profile pulls. Default 5m. */ + refreshMs?: number + /** Per-pull timeout in ms (fail-closed on a hung plane). Default 10000. */ + timeoutMs?: number + /** fetch impl for the pull (tests). Defaults to global fetch. */ + fetchImpl?: typeof fetch + /** Notified when a refresh delivers a NEW set of promoted proposals (by + * provenance content hash). Surfaces diffs without auto-applying them. */ + onProposals?: (proposals: ProposedProfileDiff[]) => void +} + +/** The wrapped agent — same `(input) => Promise` shape, plus a manual + * `refresh()` and a `proposals()` accessor for the currently-promoted diffs. */ +export type IntelligenceWrapped = ((input: I) => Promise) & { + refresh(): Promise + proposals(): ProposedProfileDiff[] + /** Flush buffered trace spans before a short-lived process exits. */ + flush(): Promise +} + +interface RuntimeEventSummary { + inferenceUsd: number + inputTokens: number + outputTokens: number + model?: string + sessionId?: string + success?: boolean + error?: RunRecord['error'] +} + +function summarizeRuntimeEvents( + events: NonNullable, +): RuntimeEventSummary { + const summary: RuntimeEventSummary = { inferenceUsd: 0, inputTokens: 0, outputTokens: 0 } + for (const event of events) { + if ('session' in event && event.session) summary.sessionId = event.session.id + if (event.type === 'llm_call') { + summary.model = event.model + summary.inferenceUsd += event.costUsd ?? 0 + summary.inputTokens += event.tokensIn ?? 0 + summary.outputTokens += event.tokensOut ?? 0 + } else if (event.type === 'backend_error') { + summary.success = false + summary.error = { + name: event.error?.kind ?? 'BackendError', + message: event.message, + ...(event.error?.status !== undefined ? { code: String(event.error.status) } : {}), + } + } else if (event.type === 'final') { + summary.success = event.status === 'completed' + if (event.error) { + summary.error = { + name: event.error.kind, + message: event.error.message, + ...(event.error.status !== undefined ? { code: String(event.error.status) } : {}), + } + } + } + } + return summary +} + +function runError(cause: unknown): NonNullable { + if (cause instanceof Error) { + const code = (cause as Error & { code?: unknown }).code + return { + name: cause.name || 'Error', + message: cause.message, + ...(typeof code === 'string' || typeof code === 'number' ? { code: String(code) } : {}), + } + } + return { name: 'Error', message: String(cause) } +} + +/** + * Wrap an agent so it (a) RECEIVES the tenant's certified profile — the prompt + * surface to fold and the promoted profile diffs as proposals — and (b) SENDS a + * typed {@link RunRecord} per call to the plane. The pull is cached and refreshed + * at most every `refreshMs`; a failed pull is fail-closed (the agent runs on its + * base surface, never breaks because Intelligence is unreachable). The send is + * best-effort — an export failure never fails the agent's turn — while an error + * thrown by the agent itself propagates unchanged. + */ +export function withIntelligence( + agent: IntelligenceAgent, + config: IntelligenceHookConfig, +): IntelligenceWrapped { + const client = createIntelligenceClient(config) + const target = config.target ?? config.project + const source = createCertifiedPromptSource({ + target, + ...(config.apiKey !== undefined ? { apiKey: config.apiKey } : {}), + ...(config.baseUrl !== undefined ? { baseUrl: config.baseUrl } : {}), + ...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}), + ...(config.fetchImpl !== undefined ? { fetchImpl: config.fetchImpl } : {}), + ...(config.refreshMs !== undefined ? { refreshMs: config.refreshMs } : {}), + }) + + const currentProposals = (): ProposedProfileDiff[] => source.current()?.agentProfileDiffs ?? [] + + // Fire `onProposals` only when the promoted set actually changes (keyed by + // provenance content hash) — a refresh that re-delivers the same diffs is silent. + let lastSignal = '' + function signalProposals(): void { + const proposals = currentProposals() + if (proposals.length === 0) return + const sig = proposals.map((p) => p.provenance.contentHash).join('|') + if (sig === lastSignal) return + lastSignal = sig + config.onProposals?.(proposals) + } + + async function refresh(): Promise { + await source.refresh() + signalProposals() + } + + const wrapped = (async (input: I): Promise => { + const runId = client.freshRunId() + const traceId = client.freshTraceId() + const startedAt = Date.now() + await refresh() + const certified = source.current() + const proposals = currentProposals() + const report: RunReport = {} + const applied: AppliedIntelligence = { + runId, + traceId, + certified, + composePrompt: (base: string) => composeCertifiedPrompt(base, certified), + proposals, + applyProfile: (base: AgentProfile) => + proposals.reduce((profile, p) => applyAgentProfileDiff(profile, p.diff), base), + record: (r: RunReport) => Object.assign(report, r), + } + + function exportCompleted(output: unknown, caught?: unknown): void { + const completedAt = Date.now() + const eventSummary = summarizeRuntimeEvents(report.runtimeEvents ?? []) + const error = report.error ?? (caught !== undefined ? runError(caught) : eventSummary.error) + const tokens = + report.tokens ?? + (eventSummary.inputTokens > 0 || eventSummary.outputTokens > 0 + ? { input: eventSummary.inputTokens, output: eventSummary.outputTokens } + : undefined) + const profile = report.profile ?? config.profile + const record: RunRecord = { + runId, + traceId, + project: config.project, + target, + input, + output, + outcome: { + success: + report.success ?? + (caught !== undefined ? false : (eventSummary.success ?? error === undefined)), + ...(report.score !== undefined ? { score: report.score } : {}), + usage: { + inferenceUsd: report.usage?.inferenceUsd ?? report.costUsd ?? eventSummary.inferenceUsd, + intelligenceUsd: report.usage?.intelligenceUsd ?? 0, + }, + }, + timing: { startedAt, completedAt, durationMs: completedAt - startedAt }, + ...((report.model ?? eventSummary.model) + ? { model: report.model ?? eventSummary.model } + : {}), + ...(report.provider !== undefined ? { provider: report.provider } : {}), + ...(report.loopEvents !== undefined ? { loopEvents: report.loopEvents } : {}), + ...(report.runtimeEvents !== undefined ? { runtimeEvents: report.runtimeEvents } : {}), + ...(profile !== undefined ? { profile } : {}), + ...((report.sessionId ?? eventSummary.sessionId) + ? { sessionId: report.sessionId ?? eventSummary.sessionId } + : {}), + ...((report.harness ?? profile?.harness) + ? { harness: report.harness ?? profile?.harness } + : {}), + ...((report.commitSha ?? config.commitSha) + ? { commitSha: report.commitSha ?? config.commitSha } + : {}), + ...(tokens !== undefined ? { tokens } : {}), + ...(error !== undefined ? { error } : {}), + ...(report.candidateExecution !== undefined + ? { candidateExecution: report.candidateExecution } + : {}), + } + client.exportRunRecord(record) + } + + try { + const output = await agent(input, applied) + exportCompleted(output) + return output + } catch (cause) { + exportCompleted(undefined, cause) + throw cause + } + }) as IntelligenceWrapped + + wrapped.refresh = refresh + wrapped.proposals = currentProposals + wrapped.flush = client.flush + return wrapped +} diff --git a/src/mcp/codex-diagnostics.ts b/src/mcp/codex-diagnostics.ts new file mode 100644 index 00000000..5408c3a6 --- /dev/null +++ b/src/mcp/codex-diagnostics.ts @@ -0,0 +1,188 @@ +import { constants } from 'node:fs' +import { open } from 'node:fs/promises' +import { stripVTControlCharacters } from 'node:util' + +/** Bounded, credential-redacted process context attached when reproducible Codex output fails + * validation. The process still fails closed; this only preserves enough evidence to diagnose it. */ +export interface CodexExecutionFailureDiagnostic { + exitCode: number | null + killedBySignal: NodeJS.Signals | null + timedOut: boolean + durationMs: number + stdout: string + stderr: string + stdoutTruncated: boolean + stderrTruncated: boolean +} + +/** Thrown when reproducible Codex exits without one valid terminal usage event. */ +export class CodexExecutionDiagnosticError extends Error { + readonly code = 'CODEX_EXECUTION_DIAGNOSTIC' + + constructor( + readonly reason: string, + readonly diagnostic: CodexExecutionFailureDiagnostic, + cause?: unknown, + ) { + super( + `runLocalHarness: reproducible Codex execution failed: ${reason}; diagnostic=${JSON.stringify(diagnostic)}`, + { cause }, + ) + this.name = 'CodexExecutionDiagnosticError' + } +} + +const CODEX_FAILURE_STREAM_MAX_CHARS = 4096 +const CODEX_AUTH_MAX_BYTES = 1024 * 1024 +const CODEX_REDACTED = '' +export const codexSensitiveEnvironmentName = /(TOKEN|KEY|SECRET|PASSWORD|CREDENTIAL|AUTH|COOKIE)/i + +export function createCodexExecutionDiagnosticError(opts: { + reason: string + exitCode: number | null + killedBySignal: NodeJS.Signals | null + timedOut: boolean + durationMs: number + stdout: string + stderr: string + codexHome: string | undefined + redactionValues: string[] + cause: unknown +}): CodexExecutionDiagnosticError { + const redact = (value: string) => + redactCodexDiagnostic(value, opts.codexHome, opts.redactionValues) + const boundedStdout = boundCodexDiagnostic(redact(opts.stdout)) + const boundedStderr = boundCodexDiagnostic(redact(opts.stderr)) + const diagnostic: CodexExecutionFailureDiagnostic = { + exitCode: opts.exitCode, + killedBySignal: opts.killedBySignal, + timedOut: opts.timedOut, + durationMs: opts.durationMs, + stdout: boundedStdout.value, + stderr: boundedStderr.value, + stdoutTruncated: boundedStdout.truncated, + stderrTruncated: boundedStderr.truncated, + } + const safeReason = boundCodexDiagnostic(redact(opts.reason), 1024).value + return new CodexExecutionDiagnosticError( + safeReason, + diagnostic, + redactCodexErrorCause(opts.cause, safeReason, redact), + ) +} + +export function redactCodexHome(value: string, codexHome: string | undefined): string { + return codexHome ? value.replaceAll(codexHome, '') : value +} + +export function collectCodexDiagnosticRedactionValues( + env: NodeJS.ProcessEnv, + authText: string, +): string[] { + const values = Object.entries(env) + .filter( + ([name, value]) => codexSensitiveEnvironmentName.test(name) && typeof value === 'string', + ) + .map(([, value]) => value as string) + try { + collectStringLeaves(JSON.parse(authText), values) + } catch { + // Invalid auth JSON is diagnosed by Codex; generic credential patterns still redact it. + } + const encoded = values + .filter((value) => value.length >= 8) + .flatMap((value) => [ + Buffer.from(value, 'utf8').toString('base64'), + Buffer.from(value, 'utf8').toString('base64url'), + encodeURIComponent(value), + JSON.stringify(value).slice(1, -1), + ]) + return [...new Set([...values, ...encoded])] +} + +export async function readCodexAuthRedactionText(path: string): Promise { + const handle = await open(path, constants.O_RDONLY | constants.O_NONBLOCK) + try { + const stats = await handle.stat() + if (!stats.isFile() || !Number.isSafeInteger(stats.size) || stats.size > CODEX_AUTH_MAX_BYTES) { + throw new Error('Codex auth.json must be a regular file no larger than 1 MiB') + } + const buffer = Buffer.alloc(stats.size) + let offset = 0 + while (offset < buffer.length) { + const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, offset) + if (bytesRead === 0) break + offset += bytesRead + } + return buffer.subarray(0, offset).toString('utf8') + } finally { + await handle.close() + } +} + +function boundCodexDiagnostic( + value: string, + maxChars = CODEX_FAILURE_STREAM_MAX_CHARS, +): { value: string; truncated: boolean } { + if (value.length <= maxChars) return { value, truncated: false } + const marker = '\n<... OUTPUT TRUNCATED ...>\n' + const contentChars = maxChars - marker.length + const headChars = Math.floor(contentChars / 2) + const tailChars = contentChars - headChars + return { + value: `${value.slice(0, headChars)}${marker}${value.slice(-tailChars)}`, + truncated: true, + } +} + +function redactCodexDiagnostic( + value: string, + codexHome: string | undefined, + protectedValues: ReadonlyArray, +): string { + let redacted = redactCodexHome(stripVTControlCharacters(value), codexHome) + const exactValues = [...new Set(protectedValues.filter((item) => item.length >= 6))].sort( + (a, b) => b.length - a.length, + ) + for (const secret of exactValues) redacted = redacted.replaceAll(secret, CODEX_REDACTED) + redacted = redacted + .replace(/(\bBearer\s+)[A-Za-z0-9._~+/=-]{6,}/gi, `$1${CODEX_REDACTED}`) + .replace(/\b(?:sk|rk|pk)-[A-Za-z0-9._-]{8,}\b/g, CODEX_REDACTED) + .replace(/\beyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\b/g, CODEX_REDACTED) + .replace( + /((?:["']?(?:api[-_]?key|access[-_]?token|refresh[-_]?token|id[-_]?token|client[-_]?secret|private[-_]?key|session[-_]?(?:id|token)|secret|password|passwd|authorization|credential|cookie)["']?)\s*[:=]\s*)(["'])[^"'\r\n]*\2/gi, + `$1$2${CODEX_REDACTED}$2`, + ) + .replace( + /((?:["']?(?:api[-_]?key|access[-_]?token|refresh[-_]?token|id[-_]?token|client[-_]?secret|private[-_]?key|session[-_]?(?:id|token)|secret|password|passwd|authorization|credential|cookie)["']?)\s*[:=]\s*)([^\s,;}]+)/gi, + `$1${CODEX_REDACTED}`, + ) + for (const secret of exactValues) redacted = redacted.replaceAll(secret, CODEX_REDACTED) + return redacted +} + +function redactCodexErrorCause( + cause: unknown, + safeReason: string, + redact: (value: string) => string, +): Error | undefined { + if (!(cause instanceof Error)) return undefined + const safeCause = new Error(safeReason) + safeCause.name = cause.name + if (cause.stack) safeCause.stack = boundCodexDiagnostic(redact(cause.stack)).value + return safeCause +} + +function collectStringLeaves(value: unknown, output: string[]): void { + if (typeof value === 'string') { + output.push(value) + return + } + if (Array.isArray(value)) { + for (const item of value) collectStringLeaves(item, output) + return + } + if (value !== null && typeof value === 'object') { + for (const item of Object.values(value)) collectStringLeaves(item, output) + } +} diff --git a/src/mcp/index.ts b/src/mcp/index.ts index 90356a25..6cfd0664 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -89,8 +89,20 @@ export { type FactJudgeVerdict, type KbGateResult, } from './kb-gate' -export type { LocalHarness, LocalHarnessResult, RunLocalHarnessOptions } from './local-harness' -export { runLocalHarness } from './local-harness' +export type { + CodexExecutionEvidence, + CodexExecutionFailureDiagnostic, + CodexExecutionPolicy, + CodexTokenUsage, + LocalHarness, + LocalHarnessResult, + RunLocalHarnessOptions, +} from './local-harness' +export { + CodexExecutionDiagnosticError, + parseCodexTokenUsage, + runLocalHarness, +} from './local-harness' export { mcpToolsForRuntimeMcp, mcpToolsForRuntimeMcpSubset } from './openai-tools' export type { JsonRpcMessage, diff --git a/src/mcp/local-harness.ts b/src/mcp/local-harness.ts index 51e6d6cf..34c451a6 100644 --- a/src/mcp/local-harness.ts +++ b/src/mcp/local-harness.ts @@ -18,14 +18,63 @@ */ import { type ChildProcess, spawn } from 'node:child_process' +import { createHash, randomUUID } from 'node:crypto' +import { constants, createReadStream } from 'node:fs' +import { + access, + chmod, + copyFile, + mkdir, + mkdtemp, + open, + readFile, + realpath, + rm, + symlink, + writeFile, +} from 'node:fs/promises' +import { homedir, tmpdir } from 'node:os' +import { basename, delimiter, dirname, isAbsolute, join, resolve, sep } from 'node:path' import type { AgentProfile } from '@tangle-network/agent-interface' +import { + codexSensitiveEnvironmentName, + collectCodexDiagnosticRedactionValues, + createCodexExecutionDiagnosticError, + readCodexAuthRedactionText, + redactCodexHome, +} from './codex-diagnostics' + +export type { CodexExecutionFailureDiagnostic } from './codex-diagnostics' +export { CodexExecutionDiagnosticError } from './codex-diagnostics' /** Local coding harness available inside the sandbox. */ export type LocalHarness = 'claude' | 'codex' | 'opencode' +type ReasoningEffort = NonNullable['reasoningEffort']> + +const codexReasoningEffort: Record = { + none: 'none', + minimal: 'minimal', + low: 'low', + medium: 'medium', + high: 'high', + xhigh: 'xhigh', + ultracode: 'xhigh', +} + +function codexReasoningArgs(reasoningEffort: ReasoningEffort): string[] { + const mapped = codexReasoningEffort[reasoningEffort] + if (mapped === undefined) { + throw new Error( + `harnessInvocation: unsupported Codex reasoning effort ${String(reasoningEffort)}`, + ) + } + return ['-c', `model_reasoning_effort="${mapped}"`] +} + /** * Default per-harness command + arg shape. `buildArgs` takes ONLY the task prompt and - * emits the prompt-only invocation (no model, no system prompt) — the historical shape + * emits the prompt-only invocation (no model, no system prompt) — the safe default shape * the in-process executor's `streamPrompt` drives. `modelArgs` maps a resolved model to * the harness's selector flag (every supported harness takes `-m `). The §1.5 * profile-aware mapper `harnessInvocation` composes these to thread the full @@ -38,17 +87,22 @@ const HARNESS_INVOCATIONS: Record< buildArgs: (taskPrompt: string) => string[] /** Map a resolved model to the harness's model-selector flag. */ modelArgs: (model: string) => string[] + /** Map portable reasoning effort when the harness exposes a native control. */ + reasoningArgs?: (reasoningEffort: ReasoningEffort) => string[] } > = { claude: { command: 'claude', - buildArgs: (taskPrompt) => ['--headless', '-p', taskPrompt], + // `-p` IS headless/print mode; the old `--headless` flag was removed from the CLI. + // Permission bypass is an explicit per-run opt-in below, never the public default. + buildArgs: (taskPrompt) => ['-p', taskPrompt], modelArgs: (model) => ['-m', model], }, codex: { command: 'codex', - buildArgs: (taskPrompt) => ['run', taskPrompt], + buildArgs: (taskPrompt) => ['exec', taskPrompt], modelArgs: (model) => ['-m', model], + reasoningArgs: codexReasoningArgs, }, opencode: { command: 'opencode', @@ -61,6 +115,60 @@ const HARNESS_INVOCATIONS: Record< export interface HarnessInvocation { command: string args: string[] + /** Exact profile-composed prompt carried by the invocation. */ + prompt: string +} + +export interface HarnessInvocationOptions { + /** Allow an unattended Claude process to edit its isolated candidate worktree. + * Ignored by harnesses that do not use Claude's permission prompt. */ + dangerouslySkipPermissions?: boolean + /** Run Codex with benchmark-safe process controls and JSONL usage output. + * Valid only for the Codex harness. */ + codexReproducible?: boolean +} + +const CODEX_REPRODUCIBLE_ARGS = [ + '--ephemeral', + '--ignore-rules', + '--json', + '-c', + 'approval_policy="never"', + '-c', + 'web_search="disabled"', + '-c', + 'project_doc_max_bytes=0', + '-c', + 'skills.include_instructions=false', + '-c', + 'include_apps_instructions=false', + '--disable', + 'tool_suggest', + '-c', + 'features.multi_agent_v2.root_agent_usage_hint_text=""', + '-c', + 'features.multi_agent_v2.subagent_usage_hint_text=""', + '-c', + 'features.multi_agent_v2.multi_agent_mode_hint_text=""', + '--strict-config', +] as const + +function buildHarnessArgs( + harness: LocalHarness, + taskPrompt: string, + options: HarnessInvocationOptions = {}, +): string[] { + const args = HARNESS_INVOCATIONS[harness].buildArgs(taskPrompt) + if (harness === 'claude' && options.dangerouslySkipPermissions) { + args.push('--dangerously-skip-permissions') + } + if (options.codexReproducible) { + if (harness !== 'codex') { + throw new Error('harnessInvocation: codexReproducible requires the Codex harness') + } + args.push(...CODEX_REPRODUCIBLE_ARGS) + } + return args } /** @@ -73,6 +181,7 @@ export interface HarnessInvocation { * default that prepends the system prompt above the task prompt (`\n\n`), * so the authored standing instructions reach EVERY harness (none of the three CLIs * expose a portable replace-system-prompt flag for a one-shot non-interactive run). + * - `profile.prompt.instructions[]` → appended prompt sections in authored order. * - `profile.model.default` → the harness's `-m ` selector. * * The task prompt alone is the floor; an empty/absent profile yields exactly the legacy @@ -82,26 +191,52 @@ export function harnessInvocation( harness: LocalHarness, profile: AgentProfile, taskPrompt: string, + options: HarnessInvocationOptions = {}, ): HarnessInvocation { const invocation = HARNESS_INVOCATIONS[harness] if (!invocation) { throw new Error(`harnessInvocation: unknown harness ${String(harness)}`) } + if (options.codexReproducible) { + const model = profile.model?.default + if (typeof model !== 'string' || model.trim().length === 0) { + throw new Error('harnessInvocation: codexReproducible requires profile.model.default') + } + if (profile.model?.reasoningEffort === undefined) { + throw new Error('harnessInvocation: codexReproducible requires profile.model.reasoningEffort') + } + } + const systemPrompt = profile.prompt?.systemPrompt - const composedPrompt = - typeof systemPrompt === 'string' && systemPrompt.trim().length > 0 - ? `${systemPrompt}\n\n${taskPrompt}` - : taskPrompt + const instructions = profile.prompt?.instructions + if ( + instructions !== undefined && + (!Array.isArray(instructions) || + instructions.some((instruction) => typeof instruction !== 'string')) + ) { + throw new Error('harnessInvocation: profile.prompt.instructions must be an array of strings') + } + const promptSections = [ + ...(typeof systemPrompt === 'string' && systemPrompt.trim().length > 0 ? [systemPrompt] : []), + ...(instructions?.filter((instruction) => instruction.trim().length > 0) ?? []), + taskPrompt, + ] + const composedPrompt = promptSections.join('\n\n') - const args = invocation.buildArgs(composedPrompt) + const args = buildHarnessArgs(harness, composedPrompt, options) const model = profile.model?.default if (typeof model === 'string' && model.length > 0) { args.push(...invocation.modelArgs(model)) } - return { command: invocation.command, args } + const reasoningEffort = profile.model?.reasoningEffort + if (reasoningEffort !== undefined && invocation.reasoningArgs) { + args.push(...invocation.reasoningArgs(reasoningEffort)) + } + + return { command: invocation.command, args, prompt: composedPrompt } } /** @experimental */ @@ -119,6 +254,15 @@ export interface RunLocalHarnessOptions { * is used unchanged. */ invocation?: { command?: string; args: ReadonlyArray } + /** Allow autonomous Claude edits without an interactive permission prompt. + * Use only when `cwd` is an isolated candidate worktree. */ + dangerouslySkipPermissions?: boolean + /** Isolate Codex from ambient configuration/instructions and require JSONL token usage. + * The invocation should come from `harnessInvocation(..., { codexReproducible: true })`. */ + codexReproducible?: boolean + /** Absolute host paths that reproducible Codex must not read. The normalized set is compiled + * into the controlled permission profile and its digest is returned in execution evidence. */ + codexReadDeniedPaths?: ReadonlyArray /** Wall-clock kill deadline (ms). Default 5 min. Subprocess SIGTERMed on expiry. */ timeoutMs?: number /** Caller cancellation. SIGTERM is sent on abort. */ @@ -136,8 +280,65 @@ export interface RunLocalHarnessOptions { cwd: string env: NodeJS.ProcessEnv stdio: 'pipe' + detached: boolean }, ) => ChildProcess + /** Test seam for locating the native Codex executable before it is staged in the worktree. */ + resolveCodexExecutable?: (command: string, env: NodeJS.ProcessEnv) => Promise +} + +/** Exact aggregate usage emitted by Codex's terminal `turn.completed` JSONL event. */ +export interface CodexTokenUsage { + inputTokens: number + cachedInputTokens: number + outputTokens: number + reasoningOutputTokens: number +} + +/** Isolation settings asserted before a reproducible Codex run is allowed to start. */ +export interface CodexExecutionPolicy { + sessionPersistence: 'ephemeral' + userConfig: false + rules: false + projectInstructions: false + skillInstructions: false + appInstructions: false + toolSuggestions: false + multiAgentInstructions: false + sandbox: 'workspace-write' + permissionProfile: 'agent_runtime_reproducible' + approvalPolicy: 'never' + shellNetwork: false + webSearch: false + serviceTier: 'default' + shellEnvironment: 'core-filtered' + loginShell: false + credentialsReadable: false + hostHomeReadable: false + procEnvironment: 'private-sanitized' + sensitiveEnvironmentNamesVisible: false + parentRepoRead: false + gitMetadata: false + temporaryDirectory: 'workspace-private' + stagedExecutable: 'static-elf-read-only' + callerReadDeniedPaths: 'enforced' + containerSockets: false +} + +/** Zero-model-call evidence for the exact Codex process about to run. */ +export interface CodexExecutionEvidence { + cliVersion: string + executableSha256: string + /** SHA-256 of the exact composed prompt argument proved present in the rendered prompt. */ + requestedPromptSha256: string + effectivePromptSha256: string + nonPromptArgsSha256: string + controlledConfigSha256: string + /** Sorted normalized paths compiled into the permission profile. */ + readDeniedPaths: string[] + readDeniedPathsSha256: string + readDeniedPathCount: number + policy: CodexExecutionPolicy } /** @experimental */ @@ -154,9 +355,14 @@ export interface LocalHarnessResult { durationMs: number /** Set when timeoutMs elapsed before exit. */ timedOut: boolean + /** Present for a reproducible Codex run; parsed from the real terminal JSONL event. */ + usage?: CodexTokenUsage + /** Present for reproducible Codex runs; generated and checked before model execution. */ + evidence?: CodexExecutionEvidence } const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000 +const processKillGraceMs = 250 /** * Spawn a local coding harness CLI as a subprocess + collect its output. @@ -177,93 +383,1015 @@ const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000 * * @experimental */ -export function runLocalHarness(options: RunLocalHarnessOptions): Promise { +export async function runLocalHarness( + options: RunLocalHarnessOptions, +): Promise { const { harness, cwd, taskPrompt } = options + if (options.codexReproducible && harness !== 'codex') { + throw new Error('runLocalHarness: codexReproducible requires the Codex harness') + } + if (options.codexReproducible && process.platform !== 'linux') { + throw new Error('runLocalHarness: codexReproducible currently requires Linux') + } + if (options.codexReadDeniedPaths !== undefined && !options.codexReproducible) { + throw new Error('runLocalHarness: codexReadDeniedPaths requires codexReproducible') + } const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS - const env = options.env ?? process.env const spawnImpl = options.spawn ?? spawn const invocation = HARNESS_INVOCATIONS[harness] if (!invocation) { - return Promise.reject(new Error(`runLocalHarness: unknown harness ${String(harness)}`)) + throw new Error(`runLocalHarness: unknown harness ${String(harness)}`) } const startedAt = Date.now() - const command = options.invocation?.command ?? invocation.command - const args = options.invocation ? [...options.invocation.args] : invocation.buildArgs(taskPrompt) + const requestedCommand = options.invocation?.command ?? invocation.command + const args = options.invocation + ? [...options.invocation.args] + : buildHarnessArgs(harness, taskPrompt, options) + if (options.codexReproducible) assertCodexReproducibleInvocation(requestedCommand, args) + + // We spawn the harness with `cwd`, but a harness that resolves its working + // directory from `$PWD` rather than `getcwd()` (opencode does; the others may) + // inherits the parent's stale `PWD` and edits the WRONG directory — its + // worktree diff then comes back empty and the delegation fails with a phantom + // "empty patch". Pin `PWD` to `cwd` so the harness edits where it was placed. + // `baseEnv` feeds both the codex-isolated and the plain spawn paths below, so + // pinning it here covers every runLocalHarness consumer. + const baseEnv: NodeJS.ProcessEnv = { ...(options.env ?? process.env), PWD: cwd } + const isolated = options.codexReproducible + ? await isolateCodexHome({ + baseEnv, + cwd, + command: requestedCommand, + readDeniedPaths: options.codexReadDeniedPaths ?? [], + resolveExecutable: options.resolveCodexExecutable ?? resolveCodexNativeExecutable, + }) + : { + env: baseEnv, + command: requestedCommand, + cleanup: async () => undefined, + controlledConfigSha256: '', + executableSha256: '', + readDeniedPathsSha256: '', + readDeniedPathCount: 0, + readDeniedPaths: [] as string[], + diagnosticRedactionValues: [] as string[], + ambientAuth: '', + isolatedAuth: '', + writeProbe: '', + stagedWriteProbe: '', + deniedProbePaths: [] as string[], + } + const env = isolated.env + const command = isolated.command + if (options.codexReproducible) assertCodexReproducibleInvocation(command, args) + + try { + const evidence = options.codexReproducible + ? await collectCodexExecutionEvidence({ + command, + args, + cwd, + env, + spawn: spawnImpl, + controlledConfigSha256: isolated.controlledConfigSha256, + executableSha256: isolated.executableSha256, + readDeniedPathsSha256: isolated.readDeniedPathsSha256, + readDeniedPathCount: isolated.readDeniedPathCount, + readDeniedPaths: isolated.readDeniedPaths, + ambientAuth: isolated.ambientAuth, + isolatedAuth: isolated.isolatedAuth, + writeProbe: isolated.writeProbe, + stagedWriteProbe: isolated.stagedWriteProbe, + deniedProbePaths: isolated.deniedProbePaths, + ...(options.signal ? { signal: options.signal } : {}), + }) + : undefined + return await new Promise((resolve, reject) => { + let child: ChildProcess + try { + child = spawnImpl(command, args, { + cwd, + env, + stdio: 'pipe', + detached: process.platform !== 'win32', + }) + } catch (err) { + reject(err instanceof Error ? err : new Error(String(err))) + return + } + + // The harness takes its task as an argv arg, not on stdin. Leaving stdin + // OPEN makes a non-TTY `opencode run` (and likely the other harnesses) + // BLOCK forever waiting on input — zero output, SIGTERM at the wall cap, + // empty patch -> "no candidate passed validation". Close stdin so the + // subprocess sees EOF and proceeds (the `cliExecutor` leaf does the same). + child.stdin?.end() + + let stdout = '' + let stderr = '' + let timedOut = false + let settled = false + let forceKillTimer: ReturnType | null = null + let terminationStarted = false + + const terminate = () => { + if (terminationStarted) return + terminationStarted = true + signalProcessTree(child, 'SIGTERM') + forceKillTimer = setTimeout(() => signalProcessTree(child, 'SIGKILL'), processKillGraceMs) + forceKillTimer.unref?.() + } + + const timer = + timeoutMs > 0 + ? setTimeout(() => { + timedOut = true + terminate() + }, timeoutMs) + : null + if (timer && typeof (timer as { unref?: () => void }).unref === 'function') { + ;(timer as { unref: () => void }).unref() + } + + const onAbort = () => { + terminate() + } + if (options.signal) { + if (options.signal.aborted) onAbort() + else options.signal.addEventListener('abort', onAbort, { once: true }) + } + + child.stdout?.on('data', (chunk) => { + stdout += String(chunk) + }) + child.stderr?.on('data', (chunk) => { + stderr += String(chunk) + }) + + const finalize = (result: LocalHarnessResult) => { + if (settled) return + settled = true + if (timer) clearTimeout(timer) + if (forceKillTimer) clearTimeout(forceKillTimer) + options.signal?.removeEventListener('abort', onAbort) + resolve(result) + } + + child.on('error', (err) => { + if (settled) return + settled = true + if (timer) clearTimeout(timer) + if (forceKillTimer) clearTimeout(forceKillTimer) + options.signal?.removeEventListener('abort', onAbort) + reject(err) + }) + + child.on('close', (code, signal) => { + const durationMs = Date.now() - startedAt + try { + const usage = options.codexReproducible ? parseCodexTokenUsage(stdout) : undefined + finalize({ + exitCode: code, + stdout, + stderr: redactCodexHome(stderr, env.CODEX_HOME), + killedBySignal: signal, + durationMs, + timedOut, + ...(usage ? { usage } : {}), + ...(evidence ? { evidence } : {}), + }) + } catch (err) { + if (settled) return + settled = true + if (timer) clearTimeout(timer) + if (forceKillTimer) clearTimeout(forceKillTimer) + options.signal?.removeEventListener('abort', onAbort) + const reason = err instanceof Error ? err.message : String(err) + reject( + options.codexReproducible + ? createCodexExecutionDiagnosticError({ + reason, + exitCode: code, + killedBySignal: signal, + timedOut, + durationMs, + stdout, + stderr, + codexHome: env.CODEX_HOME, + redactionValues: isolated.diagnosticRedactionValues, + cause: err, + }) + : err instanceof Error + ? err + : new Error(reason), + ) + } + }) + }) + } finally { + await isolated.cleanup() + } +} + +const CODEX_EXECUTION_POLICY: CodexExecutionPolicy = { + sessionPersistence: 'ephemeral', + userConfig: false, + rules: false, + projectInstructions: false, + skillInstructions: false, + appInstructions: false, + toolSuggestions: false, + multiAgentInstructions: false, + sandbox: 'workspace-write', + permissionProfile: 'agent_runtime_reproducible', + approvalPolicy: 'never', + shellNetwork: false, + webSearch: false, + serviceTier: 'default', + shellEnvironment: 'core-filtered', + loginShell: false, + credentialsReadable: false, + hostHomeReadable: false, + procEnvironment: 'private-sanitized', + sensitiveEnvironmentNamesVisible: false, + parentRepoRead: false, + gitMetadata: false, + temporaryDirectory: 'workspace-private', + stagedExecutable: 'static-elf-read-only', + callerReadDeniedPaths: 'enforced', + containerSockets: false, +} + +const FORBIDDEN_CODEX_PROMPT_MARKERS = [ + '', + '# AGENTS.md instructions', + '', + '', + '', + 'You are `/root`, the primary agent', +] as const + +async function collectCodexExecutionEvidence(opts: { + command: string + args: string[] + cwd: string + env: NodeJS.ProcessEnv + spawn: NonNullable + controlledConfigSha256: string + readDeniedPaths: string[] + executableSha256: string + readDeniedPathsSha256: string + readDeniedPathCount: number + ambientAuth: string + isolatedAuth: string + writeProbe: string + stagedWriteProbe: string + deniedProbePaths: string[] + signal?: AbortSignal +}): Promise { + assertCodexReproducibleInvocation(opts.command, opts.args) + const prompt = opts.args[1] + if (opts.args[0] !== 'exec' || typeof prompt !== 'string' || prompt.length === 0) { + throw new Error('runLocalHarness: reproducible Codex invocation must be codex exec ') + } + const cliVersion = (await runCodexProbe(opts, ['--version'])).stdout.trim() + if (!/^codex-cli \d+\.\d+\.\d+(?:[-+].+)?$/.test(cliVersion)) { + throw new Error( + `runLocalHarness: unexpected Codex CLI version output ${JSON.stringify(cliVersion)}`, + ) + } + + const hiddenEnvironmentChecks = [ + 'CODEX_HOME', + 'CODEX_ACCESS_TOKEN', + 'CODEX_API_KEY', + 'OPENAI_API_KEY', + ] + .map((name) => `test -z "\${${name}:-}"`) + .join(' && ') + const sandboxProof = [ + hiddenEnvironmentChecks, + 'test -r .', + '! test -r .git', + 'touch "$1"', + 'rm "$1"', + 'staged_dir=$(dirname "$2")', + 'test -x "$staged_dir/codex"', + '! chmod 700 "$staged_dir" 2>/dev/null', + '! rm "$staged_dir/codex" 2>/dev/null', + '! touch "$2" 2>/dev/null', + 'shift 2', + 'for path in "$@"; do ! test -r "$path" || exit 1; done', + 'touch "$TMPDIR/.agent-runtime-tmp-proof"', + 'rm "$TMPDIR/.agent-runtime-tmp-proof"', + ].join(' && ') + await runCodexProbe(opts, [ + 'sandbox', + '-P', + CODEX_PERMISSION_PROFILE, + '-C', + opts.cwd, + 'sh', + '-c', + sandboxProof, + 'sh', + opts.writeProbe, + opts.stagedWriteProbe, + ...opts.deniedProbePaths, + ]) + await runCodexProbe(opts, [ + 'sandbox', + '-P', + CODEX_PERMISSION_PROFILE, + '-C', + opts.cwd, + 'python3', + '-c', + `import os,re +pattern=re.compile(br'(TOKEN|KEY|SECRET|PASSWORD|CREDENTIAL|AUTH|COOKIE|CODEX_HOME)', re.I) +paths=('/proc/1/environ', '/proc/self/environ', f'/proc/{os.getppid()}/environ') +for path in dict.fromkeys(paths): + data=open(path,'rb').read().split(b'\\0') + names=(item.split(b'=',1)[0] for item in data if b'=' in item) + assert not any(pattern.search(name) for name in names), path`, + ]) + await runCodexProbe(opts, [ + 'sandbox', + '-P', + CODEX_PERMISSION_PROFILE, + '-C', + opts.cwd, + 'python3', + '-c', + 'import socket,sys\nfor path in sys.argv[1:]:\n s=socket.socket(socket.AF_UNIX)\n try: s.connect(path)\n except OSError: continue\n raise SystemExit(1)', + '/var/run/docker.sock', + '/run/docker.sock', + '/var/run/containerd/containerd.sock', + '/run/containerd/containerd.sock', + ]) - return new Promise((resolve, reject) => { + const debugArgs = codexPromptDebugArgs(opts.args, prompt) + const renderedPrompt = (await runCodexProbe(opts, debugArgs)).stdout + let parsedPrompt: unknown + try { + parsedPrompt = JSON.parse(renderedPrompt) + } catch { + throw new Error('runLocalHarness: codex debug prompt-input returned invalid JSON') + } + const promptEvidenceText = normalizeCodexDebugPromptNewlines(prompt) + if (!Array.isArray(parsedPrompt) || !jsonTextContains(parsedPrompt, promptEvidenceText)) { + throw new Error('runLocalHarness: Codex prompt evidence did not contain the exact task prompt') + } + const promptContextWithoutTask = JSON.stringify(redactJsonText(parsedPrompt, promptEvidenceText)) + for (const marker of FORBIDDEN_CODEX_PROMPT_MARKERS) { + if (promptContextWithoutTask.includes(marker)) { + throw new Error(`runLocalHarness: Codex prompt evidence contains forbidden marker ${marker}`) + } + } + for (const marker of ['', '']) { + if (!renderedPrompt.includes(marker)) { + throw new Error(`runLocalHarness: Codex prompt evidence is missing required marker ${marker}`) + } + } + + const nonPromptArgs = [ + 'codex', + ...opts.args.map((arg, index) => (index === 1 ? '' : arg)), + ] + return { + cliVersion, + executableSha256: opts.executableSha256, + requestedPromptSha256: sha256(prompt), + effectivePromptSha256: sha256(renderedPrompt), + nonPromptArgsSha256: sha256(JSON.stringify(nonPromptArgs)), + controlledConfigSha256: opts.controlledConfigSha256, + readDeniedPathsSha256: opts.readDeniedPathsSha256, + readDeniedPathCount: opts.readDeniedPathCount, + readDeniedPaths: [...opts.readDeniedPaths], + policy: { ...CODEX_EXECUTION_POLICY }, + } +} + +function assertCodexReproducibleInvocation(command: string, args: string[]): void { + const executable = basename(command) + if (executable !== 'codex' && executable !== 'codex.exe') { + throw new Error('runLocalHarness: reproducible Codex invocation must use the Codex executable') + } + const prompt = args[1] + const expectedPrefix = ['exec', prompt, ...CODEX_REPRODUCIBLE_ARGS] + const hasExactPrefix = expectedPrefix.every((arg, index) => args[index] === arg) + const tail = args.slice(expectedPrefix.length) + const validModel = + tail[0] === '-m' && + typeof tail[1] === 'string' && + tail[1].length > 0 && + !tail[1].startsWith('-') + const validReasoning = + tail[2] === '-c' && + typeof tail[3] === 'string' && + /^model_reasoning_effort="(?:none|minimal|low|medium|high|xhigh)"$/.test(tail[3]) + if (!hasExactPrefix || tail.length !== 4 || !validModel || !validReasoning) { + throw new Error( + 'runLocalHarness: reproducible Codex invocation does not match the required isolated argv', + ) + } +} + +function codexPromptDebugArgs(execArgs: string[], prompt: string): string[] { + const debugArgs = ['debug', 'prompt-input'] + for (let index = 2; index < execArgs.length; index += 1) { + const arg = execArgs[index] + const value = execArgs[index + 1] + if (arg === '-c' && value !== undefined) { + debugArgs.push('-c', value) + index += 1 + } else if (arg === '--disable' && value !== undefined) { + debugArgs.push('--disable', value) + index += 1 + } else if (arg === '-m' && value !== undefined) { + debugArgs.push('-c', `model=${JSON.stringify(value)}`) + index += 1 + } else if (arg === '-s' && value !== undefined) { + debugArgs.push('-c', `sandbox_mode=${JSON.stringify(value)}`) + index += 1 + } + } + debugArgs.push(prompt) + return debugArgs +} + +function runCodexProbe( + opts: { + command: string + cwd: string + env: NodeJS.ProcessEnv + spawn: NonNullable + signal?: AbortSignal + }, + args: string[], +): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { let child: ChildProcess try { - child = spawnImpl(command, args, { cwd, env, stdio: 'pipe' }) + child = opts.spawn(opts.command, args, { + cwd: opts.cwd, + env: opts.env, + stdio: 'pipe', + detached: process.platform !== 'win32', + }) } catch (err) { reject(err instanceof Error ? err : new Error(String(err))) return } - - // The harness takes its task as an argv arg, not on stdin. Leaving stdin - // OPEN makes a non-TTY `opencode run` (and likely the other harnesses) - // BLOCK forever waiting on input — zero output, SIGTERM at the wall cap, - // empty patch -> "no candidate passed validation". Close stdin so the - // subprocess sees EOF and proceeds (the `cliExecutor` leaf does the same). child.stdin?.end() - let stdout = '' let stderr = '' - let timedOut = false let settled = false - - const timer = - timeoutMs > 0 - ? setTimeout(() => { - timedOut = true - if (!child.killed) child.kill('SIGTERM') - }, timeoutMs) - : null - if (timer && typeof (timer as { unref?: () => void }).unref === 'function') { - ;(timer as { unref: () => void }).unref() - } - + let forceKillTimer: ReturnType | null = null + const timer = setTimeout(() => { + signalProcessTree(child, 'SIGTERM') + forceKillTimer = setTimeout(() => signalProcessTree(child, 'SIGKILL'), processKillGraceMs) + forceKillTimer.unref?.() + }, 10_000) + timer.unref?.() const onAbort = () => { - if (!child.killed) child.kill('SIGTERM') + signalProcessTree(child, 'SIGTERM') + if (!forceKillTimer) { + forceKillTimer = setTimeout(() => signalProcessTree(child, 'SIGKILL'), processKillGraceMs) + forceKillTimer.unref?.() + } } - if (options.signal) { - if (options.signal.aborted) onAbort() - else options.signal.addEventListener('abort', onAbort, { once: true }) + if (opts.signal) { + if (opts.signal.aborted) onAbort() + else opts.signal.addEventListener('abort', onAbort, { once: true }) } - child.stdout?.on('data', (chunk) => { stdout += String(chunk) }) child.stderr?.on('data', (chunk) => { stderr += String(chunk) }) - - const finalize = (result: LocalHarnessResult) => { - if (settled) return - settled = true - if (timer) clearTimeout(timer) - options.signal?.removeEventListener('abort', onAbort) - resolve(result) - } - child.on('error', (err) => { if (settled) return settled = true - if (timer) clearTimeout(timer) - options.signal?.removeEventListener('abort', onAbort) + clearTimeout(timer) + if (forceKillTimer) clearTimeout(forceKillTimer) + opts.signal?.removeEventListener('abort', onAbort) reject(err) }) - child.on('close', (code, signal) => { - finalize({ - exitCode: code, - stdout, - stderr, - killedBySignal: signal, - durationMs: Date.now() - startedAt, - timedOut, - }) + if (settled) return + settled = true + clearTimeout(timer) + if (forceKillTimer) clearTimeout(forceKillTimer) + opts.signal?.removeEventListener('abort', onAbort) + if (code !== 0) { + const safeStderr = redactCodexHome(stderr, opts.env.CODEX_HOME) + reject( + new Error( + `runLocalHarness: Codex evidence probe failed (exit ${String(code)}, signal ${String(signal)}): ${safeStderr.trim()}`, + ), + ) + return + } + resolve({ stdout, stderr }) }) }) } + +function sha256(value: string): string { + return createHash('sha256').update(value).digest('hex') +} + +/** Codex's debug renderer converts CRLF pairs to LF. No other whitespace is equivalent. */ +function normalizeCodexDebugPromptNewlines(value: string): string { + return value.replaceAll('\r\n', '\n') +} + +function jsonTextContains(value: unknown, needle: string): boolean { + if (typeof value === 'string') return value.includes(needle) + if (Array.isArray(value)) return value.some((item) => jsonTextContains(item, needle)) + if (isRecord(value)) return Object.values(value).some((item) => jsonTextContains(item, needle)) + return false +} + +function redactJsonText(value: unknown, text: string): unknown { + if (typeof value === 'string') return value.replaceAll(text, '') + if (Array.isArray(value)) return value.map((item) => redactJsonText(item, text)) + if (isRecord(value)) { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, redactJsonText(item, text)]), + ) + } + return value +} + +const CODEX_AUTH_ENV = ['CODEX_ACCESS_TOKEN', 'CODEX_API_KEY', 'OPENAI_API_KEY'] as const + +const CODEX_PERMISSION_PROFILE = 'agent_runtime_reproducible' + +async function isolateCodexHome(opts: { + baseEnv: NodeJS.ProcessEnv + cwd: string + command: string + readDeniedPaths: ReadonlyArray + resolveExecutable: (command: string, env: NodeJS.ProcessEnv) => Promise +}): Promise<{ + env: NodeJS.ProcessEnv + command: string + cleanup: () => Promise + controlledConfigSha256: string + executableSha256: string + readDeniedPaths: string[] + readDeniedPathsSha256: string + readDeniedPathCount: number + diagnosticRedactionValues: string[] + ambientAuth: string + isolatedAuth: string + writeProbe: string + stagedWriteProbe: string + deniedProbePaths: string[] +}> { + const isolatedHome = await mkdtemp(join(tmpdir(), 'agent-runtime-codex-')) + await chmod(isolatedHome, 0o700) + let workspaceTemp = '' + let stagedExecutableDir = '' + let ownsStagedExecutableDir = false + let writeProbe = '' + const cleanup = async () => { + if (ownsStagedExecutableDir) { + await chmod(stagedExecutableDir, 0o700).catch(() => undefined) + } + await Promise.all([ + ...(writeProbe ? [rm(writeProbe, { force: true })] : []), + ...(workspaceTemp ? [rm(workspaceTemp, { recursive: true, force: true })] : []), + ...(ownsStagedExecutableDir + ? [rm(stagedExecutableDir, { recursive: true, force: true })] + : []), + rm(isolatedHome, { recursive: true, force: true }), + ]) + } + try { + const readDeniedPaths = await normalizeReadDeniedPaths(opts.readDeniedPaths) + const candidateRoot = await realpath(opts.cwd) + writeProbe = join(candidateRoot, `.agent-runtime-write-proof-${randomUUID()}`) + const stagedPath = join(candidateRoot, '.agent-runtime-bin') + if ( + readDeniedPaths.some( + (path) => + isSameOrDescendant(candidateRoot, path) || + isSameOrDescendant(stagedPath, path) || + isSameOrDescendant(path, stagedPath), + ) + ) { + throw new Error( + 'runLocalHarness: codexReadDeniedPaths cannot contain the candidate root or staged executable', + ) + } + const nativeExecutable = await opts.resolveExecutable(opts.command, opts.baseEnv) + if (!(await isStaticElfExecutable(nativeExecutable))) { + throw new Error( + 'runLocalHarness: reproducible Codex mode requires a statically linked Linux Codex ELF', + ) + } + const sourceExecutableSha256 = await sha256File(nativeExecutable) + + stagedExecutableDir = stagedPath + try { + await mkdir(stagedExecutableDir, { mode: 0o700 }) + ownsStagedExecutableDir = true + } catch (err) { + throw new Error( + `runLocalHarness: candidate-private .agent-runtime-bin must not already exist: ${String((err as NodeJS.ErrnoException).code ?? err)}`, + ) + } + const stagedExecutable = join(stagedExecutableDir, 'codex') + await copyFile(nativeExecutable, stagedExecutable, constants.COPYFILE_FICLONE) + await chmod(stagedExecutable, 0o500) + const executableSha256 = await sha256File(stagedExecutable) + if ( + executableSha256 !== sourceExecutableSha256 || + !(await isStaticElfExecutable(stagedExecutable)) + ) { + throw new Error('runLocalHarness: staged Codex executable digest mismatch') + } + await chmod(stagedExecutableDir, 0o500) + + workspaceTemp = await mkdtemp(join(candidateRoot, '.agent-runtime-tmp-')) + await chmod(workspaceTemp, 0o700) + const ambientHome = resolve(opts.baseEnv.CODEX_HOME?.trim() || join(homedir(), '.codex')) + const ambientAuth = join(ambientHome, 'auth.json') + const isolatedAuth = join(isolatedHome, 'auth.json') + let authText = '' + try { + await access(ambientAuth, constants.R_OK) + authText = await readCodexAuthRedactionText(ambientAuth) + await symlink(ambientAuth, isolatedAuth) + } catch (err) { + throw new Error( + `runLocalHarness: reproducible Codex mode requires readable auth.json: ${String((err as NodeJS.ErrnoException).code ?? err)}`, + ) + } + const deniedHostRoots = [homedir(), opts.baseEnv.HOME?.trim(), ambientHome] + .filter((path): path is string => Boolean(path)) + .map((path) => resolve(path)) + .filter((path, index, paths) => paths.indexOf(path) === index) + const controlledConfig = codexControlledConfig({ + ambientTmp: opts.baseEnv.TMPDIR?.trim() || tmpdir(), + deniedHostRoots, + workspaceTemp, + stagedExecutableDir, + readDeniedPaths, + }) + await writeFile(join(isolatedHome, 'config.toml'), controlledConfig, { mode: 0o600 }) + const env = { ...opts.baseEnv } + for (const name of Object.keys(env)) { + if ( + codexSensitiveEnvironmentName.test(name) || + CODEX_AUTH_ENV.some((authName) => authName === name) + ) { + delete env[name] + } + } + env.CODEX_HOME = isolatedHome + const home = homedir() + const deniedProbePaths = [ + ...deniedHostRoots, + ambientAuth, + isolatedAuth, + join(home, '.claude', '.credentials.json'), + join(home, '.claude.json'), + join(home, '.local', 'share', 'opencode', 'auth.json'), + join(home, '.agents'), + join(home, 'Documents'), + join(home, 'Downloads'), + join(home, '.npm'), + join(home, 'code'), + ...readDeniedPaths, + ].filter((path, index, paths) => paths.indexOf(path) === index) + return { + env, + command: stagedExecutable, + cleanup, + controlledConfigSha256: sha256(controlledConfig), + executableSha256, + readDeniedPaths, + readDeniedPathsSha256: sha256(JSON.stringify(readDeniedPaths)), + readDeniedPathCount: readDeniedPaths.length, + diagnosticRedactionValues: collectCodexDiagnosticRedactionValues(opts.baseEnv, authText), + ambientAuth, + isolatedAuth, + writeProbe, + stagedWriteProbe: join(stagedExecutableDir, 'write-proof'), + deniedProbePaths, + } + } catch (err) { + await cleanup() + throw err + } +} + +function codexControlledConfig(opts: { + ambientTmp: string + deniedHostRoots: string[] + workspaceTemp: string + stagedExecutableDir: string + readDeniedPaths: string[] +}): string { + const rules = new Map() + const broadDeniedPaths = minimalPathRoots([ + ...opts.deniedHostRoots.map((path) => resolve(path)), + tmpdir(), + opts.ambientTmp, + '/proc', + '/run/docker', + '/run/containerd', + ]) + for (const path of broadDeniedPaths) { + rules.set(path, 'deny') + } + const workspaceRoot = dirname(opts.workspaceTemp) + for (const path of opts.readDeniedPaths) { + const alreadyDenied = broadDeniedPaths.some((root) => isSameOrDescendant(path, root)) + if (!alreadyDenied || isSameOrDescendant(path, workspaceRoot)) rules.set(path, 'deny') + } + if (!rules.has(opts.workspaceTemp)) rules.set(opts.workspaceTemp, 'write') + if (!rules.has(opts.stagedExecutableDir)) rules.set(opts.stagedExecutableDir, 'read') + const filesystemRules = [...rules] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([path, access]) => `${JSON.stringify(path)} = ${JSON.stringify(access)}`) + .join('\n') + return `default_permissions = "${CODEX_PERMISSION_PROFILE}" +approval_policy = "never" +allow_login_shell = false +service_tier = "default" + +[shell_environment_policy] +inherit = "core" +ignore_default_excludes = false +exclude = ["^CODEX_HOME$", "(?i).*TOKEN.*", "(?i).*KEY.*", "(?i).*SECRET.*", "(?i).*PASSWORD.*", "(?i).*CREDENTIAL.*", "(?i).*AUTH.*", "(?i).*COOKIE.*"] +set = { TMPDIR = ${JSON.stringify(opts.workspaceTemp)} } + +[permissions.${CODEX_PERMISSION_PROFILE}.filesystem] +":minimal" = "read" +${filesystemRules} +":slash_tmp" = "deny" + +[permissions.${CODEX_PERMISSION_PROFILE}.filesystem.":workspace_roots"] +"." = "write" +".git" = "deny" + +[permissions.${CODEX_PERMISSION_PROFILE}.network] +enabled = false + +[permissions.${CODEX_PERMISSION_PROFILE}.network.unix_sockets] +"/var/run/docker.sock" = "deny" +"/run/docker.sock" = "deny" +"/var/run/containerd/containerd.sock" = "deny" +"/run/containerd/containerd.sock" = "deny" +` +} + +function isSameOrDescendant(path: string, root: string): boolean { + const normalizedPath = resolve(path) + const normalizedRoot = resolve(root) + return normalizedPath === normalizedRoot || normalizedPath.startsWith(`${normalizedRoot}${sep}`) +} + +function minimalPathRoots(paths: ReadonlyArray): string[] { + const ordered = [...new Set(paths.map((path) => resolve(path)))].sort( + (a, b) => a.length - b.length || a.localeCompare(b), + ) + return ordered.filter( + (path, index) => !ordered.slice(0, index).some((root) => isSameOrDescendant(path, root)), + ) +} + +async function normalizeReadDeniedPaths(paths: ReadonlyArray): Promise { + if (!Array.isArray(paths)) { + throw new Error('runLocalHarness: codexReadDeniedPaths must be an array of absolute paths') + } + const normalized = new Set() + for (const path of paths) { + if (typeof path !== 'string' || path.length === 0 || path.includes('\0') || !isAbsolute(path)) { + throw new Error('runLocalHarness: codexReadDeniedPaths must contain absolute paths') + } + const absolute = resolve(path) + normalized.add(absolute) + try { + normalized.add(await realpath(absolute)) + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err + } + } + return [...normalized].sort() +} + +async function resolveCodexNativeExecutable( + command: string, + env: NodeJS.ProcessEnv, +): Promise { + const candidates = + command.includes(sep) || isAbsolute(command) + ? [resolve(command)] + : (env.PATH ?? '') + .split(delimiter) + .filter(Boolean) + .map((entry) => join(entry, command)) + const seen = new Set() + for (const candidate of candidates) { + let resolved: string + try { + await access(candidate, constants.X_OK) + resolved = await realpath(candidate) + } catch { + continue + } + if (seen.has(resolved)) continue + seen.add(resolved) + if (await isStaticElfExecutable(resolved)) return resolved + const packageRoot = await findCodexPackageRoot(resolved) + if (!packageRoot) continue + const bundled = await findBundledCodexExecutable(packageRoot) + if (bundled) return bundled + } + throw new Error( + 'runLocalHarness: could not resolve the native Codex executable from the requested command', + ) +} + +async function findCodexPackageRoot(file: string): Promise { + let cursor = dirname(file) + while (true) { + try { + const parsed = JSON.parse(await readFile(join(cursor, 'package.json'), 'utf8')) as { + name?: unknown + } + if (parsed.name === '@openai/codex') return cursor + } catch { + // Keep walking; PATH wrappers commonly live outside the package directory. + } + const parent = dirname(cursor) + if (parent === cursor) return undefined + cursor = parent + } +} + +async function findBundledCodexExecutable(packageRoot: string): Promise { + const target = linuxCodexTarget() + const candidates = [ + join( + packageRoot, + 'node_modules', + '@openai', + target.packageName, + 'vendor', + target.triple, + 'bin', + 'codex', + ), + join(packageRoot, 'vendor', target.triple, 'bin', 'codex'), + ] + for (const executable of candidates) { + if (await isStaticElfExecutable(executable)) return executable + } + return undefined +} + +function linuxCodexTarget(): { packageName: string; triple: string } { + if (process.platform !== 'linux') { + throw new Error('runLocalHarness: reproducible Codex mode requires Linux') + } + if (process.arch === 'x64') { + return { packageName: 'codex-linux-x64', triple: 'x86_64-unknown-linux-musl' } + } + if (process.arch === 'arm64') { + return { packageName: 'codex-linux-arm64', triple: 'aarch64-unknown-linux-musl' } + } + throw new Error(`runLocalHarness: unsupported Linux architecture ${process.arch}`) +} + +async function isStaticElfExecutable(path: string): Promise { + let handle: Awaited> | undefined + try { + await access(path, constants.X_OK) + handle = await open(path, 'r') + const header = Buffer.alloc(64) + const { bytesRead } = await handle.read(header, 0, header.length, 0) + if ( + bytesRead !== header.length || + !header.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46])) || + header[4] !== 2 || + header[5] !== 1 + ) { + return false + } + const programHeaderOffset = Number(header.readBigUInt64LE(32)) + const programHeaderSize = header.readUInt16LE(54) + const programHeaderCount = header.readUInt16LE(56) + const expectedMachine = process.arch === 'x64' ? 62 : process.arch === 'arm64' ? 183 : -1 + if ( + ![2, 3].includes(header.readUInt16LE(16)) || + header.readUInt16LE(18) !== expectedMachine || + !Number.isSafeInteger(programHeaderOffset) || + programHeaderOffset < 64 || + programHeaderSize < 56 || + programHeaderSize > 1024 || + programHeaderCount === 0 || + programHeaderCount > 1024 + ) { + return false + } + const tableSize = programHeaderSize * programHeaderCount + const table = Buffer.alloc(tableSize) + const tableRead = await handle.read(table, 0, tableSize, programHeaderOffset) + if (tableRead.bytesRead !== tableSize) return false + for (let index = 0; index < programHeaderCount; index += 1) { + const type = table.readUInt32LE(index * programHeaderSize) + if (type === 3) return false + } + return true + } catch { + return false + } finally { + await handle?.close() + } +} + +function sha256File(path: string): Promise { + return new Promise((resolveDigest, reject) => { + const hash = createHash('sha256') + const stream = createReadStream(path) + stream.on('error', reject) + stream.on('data', (chunk) => hash.update(chunk)) + stream.on('end', () => resolveDigest(hash.digest('hex'))) + }) +} + +function signalProcessTree(child: ChildProcess, signal: NodeJS.Signals): void { + if (process.platform !== 'win32' && typeof child.pid === 'number') { + try { + process.kill(-child.pid, signal) + return + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ESRCH') return + } + } + try { + child.kill(signal) + } catch { + // The process may have exited between the timer and signal delivery. + } +} + +/** Parse and validate the one terminal usage event emitted by `codex exec --json`. */ +export function parseCodexTokenUsage(stdout: string): CodexTokenUsage { + const completed: unknown[] = [] + for (const [index, line] of stdout.split(/\r?\n/).entries()) { + if (line.trim().length === 0) continue + let event: unknown + try { + event = JSON.parse(line) + } catch { + throw new Error(`runLocalHarness: Codex JSONL line ${index + 1} is not valid JSON`) + } + if (isRecord(event) && event.type === 'turn.completed') completed.push(event) + } + if (completed.length !== 1) { + throw new Error( + `runLocalHarness: expected exactly one Codex turn.completed usage event, received ${completed.length}`, + ) + } + const event = completed[0] + if (!isRecord(event) || !isRecord(event.usage)) { + throw new Error('runLocalHarness: Codex turn.completed event is missing usage') + } + const usage = event.usage + const inputTokens = naturalNumber(usage.input_tokens, 'input_tokens') + const cachedInputTokens = naturalNumber(usage.cached_input_tokens, 'cached_input_tokens') + const outputTokens = naturalNumber(usage.output_tokens, 'output_tokens') + const reasoningOutputTokens = naturalNumber( + usage.reasoning_output_tokens, + 'reasoning_output_tokens', + ) + if (cachedInputTokens > inputTokens) { + throw new Error('runLocalHarness: Codex cached_input_tokens exceeds input_tokens') + } + if (reasoningOutputTokens > outputTokens) { + throw new Error('runLocalHarness: Codex reasoning_output_tokens exceeds output_tokens') + } + return { inputTokens, cachedInputTokens, outputTokens, reasoningOutputTokens } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function naturalNumber(value: unknown, field: string): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`runLocalHarness: Codex usage.${field} must be a non-negative safe integer`) + } + return value +} diff --git a/src/mcp/worktree-harness.ts b/src/mcp/worktree-harness.ts index a277d2f1..4a5ab50c 100644 --- a/src/mcp/worktree-harness.ts +++ b/src/mcp/worktree-harness.ts @@ -7,22 +7,30 @@ * - `createWorktreeCliExecutor` — the `Scope`/`Supervisor` leaf `Executor`. * - `createInProcessExecutor` — the `runLoop` `SandboxClient` / coder-delegate path. * - * §1.5 by construction: the authored `profile.prompt.systemPrompt` + `profile.model.default` - * reach the harness through `harnessInvocation` HERE, so neither port can drop them — the exact - * bug that existed while the in-process path called `runLocalHarness` with only the task prompt. + * §1.5 by construction: prompt + model reach the direct invocation, while file-backed resources + * are lowered by the shared profile materializer and applied before spawn. Resource instructions + * join the direct prompt because reproducible Codex intentionally disables ambient project docs. * - * Lifecycle: `createWorktree` → `harnessInvocation` + `runLocalHarness` → `captureWorktreeDiff` - * (BEFORE checks, so the patch is the harness's output, not polluted by files a test run writes) - * → the configured test/typecheck commands in the live worktree → return the result + a `cleanup` - * the caller invokes at its own teardown point. A throw cleans up before propagating, so a failed - * run never leaks a worktree. + * Lifecycle: `createWorktree` → materialize profile inputs → `harnessInvocation` + + * `runLocalHarness` → `captureWorktreeDiff` excluding those inputs (BEFORE checks, so the patch is + * the harness's output, not polluted by files a test run writes) → configured checks → return the + * result + caller-owned `cleanup`. A throw cleans up before propagating. * * @experimental */ import { spawn } from 'node:child_process' +import { createHash } from 'node:crypto' import type { AgentProfile } from '@tangle-network/agent-interface' import { + applyWorkspacePlan, + materializeProfile, + type WorkspacePlan, + type WorkspacePlanReceipt, +} from '@tangle-network/agent-profile-materialize' +import { + type CodexExecutionPolicy, + type CodexTokenUsage, harnessInvocation, type LocalHarness, type LocalHarnessResult, @@ -48,6 +56,26 @@ export interface WorktreeCommandResult { output: string } +/** Proof of the profile inputs delivered before the worker process started. */ +export interface WorktreeProfileMaterializationReceipt { + /** Digest of the exact materializer plan: files, modes, environment, flags, and unsupported rows. */ + workspacePlanDigest: string + /** Repository-relative profile input files written into the worker worktree. */ + writtenPaths: string[] + /** Must be empty on a successful run because this path fails closed. */ + unsupported: WorkspacePlanReceipt['unsupported'] + /** Environment variable names added to the worker process. Values remain out of telemetry. */ + environmentNames: string[] + /** Exact additional CLI arguments emitted by the materializer. */ + flags: string[] + /** `resources.instructions` bypasses native project files so reproducible Codex cannot drop it. */ + resourceInstructions: { + delivery: 'none' | 'invocation-prompt' + sha256: string | null + byteLength: number + } +} + /** The canonical result of one worktree-harness run, projected by each port to its own shape. */ export interface WorktreeHarnessResult { /** The branch the worktree was cut on (`delegate/`). */ @@ -56,6 +84,11 @@ export interface WorktreeHarnessResult { patch: string /** Shortstat-derived change counts. */ stats: { filesChanged: number; insertions: number; deletions: number } + /** + * Exact profile materialization applied before the harness launched. + * Absent on transports that cannot return a materializer receipt; never fabricated. + */ + profileMaterialization?: WorktreeProfileMaterializationReceipt /** The harness subprocess outcome. */ harness: { name: LocalHarness | 'bridge' @@ -65,6 +98,28 @@ export interface WorktreeHarnessResult { durationMs: number stdout: string stderr: string + /** Exact Codex JSONL usage when reproducible mode is enabled. */ + usage?: CodexTokenUsage + /** Installed CLI version captured immediately before execution. */ + cliVersion?: string + /** SHA-256 of the native Codex executable staged read-only in the candidate worktree. */ + executableSha256?: string + /** SHA-256 of the exact composed prompt argument proved present in Codex's rendered prompt. */ + requestedPromptSha256?: string + /** SHA-256 of `codex debug prompt-input` output for the exact isolated prompt. */ + effectivePromptSha256?: string + /** SHA-256 of the exact executable + argv with prompt content replaced by ``. */ + nonPromptArgsSha256?: string + /** SHA-256 of the isolated config that fixes permissions and shell environment. */ + controlledConfigSha256?: string + /** SHA-256 of the normalized caller-supplied host read-denial paths. */ + readDeniedPathsSha256?: string + /** Sorted normalized caller-supplied host read-denial paths. */ + readDeniedPaths?: string[] + /** Number of normalized caller-supplied host read-denial paths. */ + readDeniedPathCount?: number + /** Explicit isolation claims checked before model execution. */ + executionPolicy?: CodexExecutionPolicy } /** Verification signals derived in the live worktree (present only when commands were given). */ checks?: { @@ -85,9 +140,13 @@ export type WorktreeCheckRunner = (opts: { export interface RunWorktreeHarnessOptions { /** Absolute path to the git checkout the worktree is cut from. */ repoRoot: string - /** The SUPERVISOR-AUTHORED profile — its systemPrompt + model reach the harness (§1.5). */ + /** + * Supervisor-authored prompt/model plus structural resources materialized into the worktree. + * `model.default` selects the one-shot model; `small`, `provider`, and `metadata` remain hints. + * Resource failures are always fatal here, regardless of `resources.failOnError`. + */ profile: AgentProfile - /** Which local harness CLI drives this run. */ + /** Local harness for this run. This explicit choice overrides `profile.harness`. */ harness: LocalHarness /** The per-task instruction handed to the harness (composed under the system prompt). */ taskPrompt: string @@ -101,6 +160,10 @@ export interface RunWorktreeHarnessOptions { typecheckCmd?: string /** Wall-clock cap per harness subprocess (ms). */ harnessTimeoutMs?: number + /** Run Codex in isolated, network-off JSONL mode and require real token usage. */ + codexReproducible?: boolean + /** Absolute host paths the reproducible Codex process must not read. */ + codexReadDeniedPaths?: ReadonlyArray /** Wall-clock cap per verification command (ms). Default = `harnessTimeoutMs` or 5 min. */ checkTimeoutMs?: number /** Cap on each check's captured output. Default 16k. */ @@ -115,7 +178,7 @@ export interface RunWorktreeHarnessOptions { runCommand?: WorktreeCheckRunner } -/** One worktree-harness run: the result + the worktree handle + a single-use `cleanup`. */ +/** One worktree-harness run: the result + the worktree handle + caller-owned `cleanup`. */ export interface WorktreeHarnessRun { worktree: WorktreeHandle result: WorktreeHarnessResult @@ -128,8 +191,8 @@ export interface WorktreeHarnessRun { const defaultCheckOutputCap = 16_000 /** - * Run the one worktree-harness operation. Fail-loud cleanup: any throw removes the worktree - * before propagating, so a failed run never leaks one (the caller cleans up the success path). + * Run the one worktree-harness operation. A failed run attempts cleanup before propagating; if + * cleanup also fails, both errors are preserved. The caller cleans up a successful run. */ export async function runWorktreeHarness( opts: RunWorktreeHarnessOptions, @@ -151,16 +214,44 @@ export async function runWorktreeHarness( worktree, repoRoot: opts.repoRoot, ...(opts.runGit ? { runGit: opts.runGit } : {}), - }).catch(() => undefined) + }) try { - // §1.5: the authored systemPrompt + model reach the harness (NOT the prompt-only path). - const { command, args } = harnessInvocation(opts.harness, opts.profile, opts.taskPrompt) + assertSupportedWorktreeProfile(opts.profile, opts.harness) + const resourceInstructions = resolveResourceInstructions(opts.profile) + const workspaceProfile = materializationOnlyProfile(opts.profile) + const plan = materializeProfile(workspaceProfile, opts.harness) + if (plan.unsupported.length > 0) { + throw new Error( + `runWorktreeHarness: profile cannot be materialized for ${opts.harness}: ${plan.unsupported + .map(({ dimension, reason }) => `${dimension}: ${reason}`) + .join('; ')}`, + ) + } + assertSafeMaterializedPaths(plan) + const applied = applyWorkspacePlan(plan, worktree.path) + if (applied.unsupported.length > 0) { + throw new Error('runWorktreeHarness: applied profile unexpectedly retained unsupported rows') + } + + // §1.5: the authored prompt + model reach the harness directly. Resource instructions use + // the same explicit channel because reproducible Codex deliberately disables native project + // instructions; the workspace projection therefore omits both prompt sources. + const invocationProfile = profileWithResourceInstructions(opts.profile, resourceInstructions) + const { command, args } = harnessInvocation(opts.harness, invocationProfile, opts.taskPrompt, { + // This helper created the candidate worktree above; autonomous Claude + // edits are permitted only inside that isolated checkout. + dangerouslySkipPermissions: opts.harness === 'claude', + ...(opts.codexReproducible ? { codexReproducible: true } : {}), + }) const harnessResult: LocalHarnessResult = await runHarness({ harness: opts.harness, cwd: worktree.path, taskPrompt: opts.taskPrompt, - invocation: { command, args }, + invocation: { command, args: [...args, ...applied.flags] }, + env: { ...process.env, ...applied.env }, + ...(opts.codexReproducible ? { codexReproducible: true } : {}), + ...(opts.codexReadDeniedPaths ? { codexReadDeniedPaths: opts.codexReadDeniedPaths } : {}), ...(opts.harnessTimeoutMs !== undefined ? { timeoutMs: opts.harnessTimeoutMs } : {}), ...(opts.signal ? { signal: opts.signal } : {}), }) @@ -168,6 +259,7 @@ export async function runWorktreeHarness( // Diff BEFORE checks — the patch is the harness's output, not whatever a test run left behind. const diff = await captureWorktreeDiff({ worktree, + excludePaths: applied.written, ...(opts.runGit ? { runGit: opts.runGit } : {}), }) @@ -185,6 +277,7 @@ export async function runWorktreeHarness( branch: worktree.branch, patch: diff.patch, stats: diff.stats, + profileMaterialization: profileMaterializationReceipt(applied, resourceInstructions), harness: { name: opts.harness, exitCode: harnessResult.exitCode, @@ -193,16 +286,184 @@ export async function runWorktreeHarness( durationMs: harnessResult.durationMs, stdout: harnessResult.stdout, stderr: harnessResult.stderr, + ...(harnessResult.usage ? { usage: harnessResult.usage } : {}), + ...(harnessResult.evidence + ? { + cliVersion: harnessResult.evidence.cliVersion, + executableSha256: harnessResult.evidence.executableSha256, + requestedPromptSha256: harnessResult.evidence.requestedPromptSha256, + effectivePromptSha256: harnessResult.evidence.effectivePromptSha256, + nonPromptArgsSha256: harnessResult.evidence.nonPromptArgsSha256, + controlledConfigSha256: harnessResult.evidence.controlledConfigSha256, + readDeniedPaths: [...harnessResult.evidence.readDeniedPaths], + readDeniedPathsSha256: harnessResult.evidence.readDeniedPathsSha256, + readDeniedPathCount: harnessResult.evidence.readDeniedPathCount, + executionPolicy: harnessResult.evidence.policy, + } + : {}), }, ...(checks ? { checks } : {}), } return { worktree, result, cleanup } } catch (err) { - await cleanup() + try { + await cleanup() + } catch (cleanupError) { + throw new AggregateError( + [err, cleanupError], + 'runWorktreeHarness: run failed and worktree cleanup also failed', + ) + } throw err } } +function resolveResourceInstructions(profile: AgentProfile): string | undefined { + const instructions = profile.resources?.instructions + if (instructions === undefined) return undefined + if (typeof instructions === 'string') + return instructions.trim().length > 0 ? instructions : undefined + if (instructions.kind === 'inline' && typeof instructions.content === 'string') { + return instructions.content.trim().length > 0 ? instructions.content : undefined + } + throw new Error( + 'runWorktreeHarness: resources.instructions must be a string or inline resource; remote refs require pre-resolution', + ) +} + +function profileWithResourceInstructions( + profile: AgentProfile, + resourceInstructions: string | undefined, +): AgentProfile { + let resources: AgentProfile['resources'] | undefined + if (profile.resources) { + const { instructions: _instructions, ...structuralResources } = profile.resources + resources = structuralResources + } + return { + ...profile, + ...(resources ? { resources } : {}), + ...(resourceInstructions === undefined + ? {} + : { + prompt: { + ...profile.prompt, + instructions: [...(profile.prompt?.instructions ?? []), resourceInstructions], + }, + }), + } +} + +function materializationOnlyProfile(profile: AgentProfile): AgentProfile { + const { + prompt: _prompt, + model: _model, + resources: profileResources, + ...structuralProfile + } = profile + let resources: AgentProfile['resources'] | undefined + if (profileResources) { + const { instructions: _instructions, ...fileBackedResources } = profileResources + resources = fileBackedResources + } + return { + ...structuralProfile, + ...(resources ? { resources } : {}), + } +} + +function assertSupportedWorktreeProfile(profile: AgentProfile, harness: LocalHarness): void { + // `profile.harness` is only a preference and the explicit run option wins. Model small/provider/ + // metadata fields are routing or descriptive hints; this fixed one-shot path only selects the + // concrete `model.default`. `resources.failOnError` never weakens this path's fail-closed policy. + const unsupportedAxes = [ + hasEntries(profile.tools) ? 'tools' : null, + hasEntries(profile.permissions) ? 'permissions' : null, + profile.connections && profile.connections.length > 0 ? 'connections' : null, + hasEntries(profile.confidential) ? 'confidential' : null, + hasEntries(profile.modes) ? 'modes' : null, + hasEntries(profile.extensions) ? 'extensions' : null, + ].filter((axis): axis is string => axis !== null) + if (profile.model?.reasoningEffort !== undefined && harness !== 'codex') { + unsupportedAxes.push('model.reasoningEffort') + } + for (const [name, server] of Object.entries(profile.mcp ?? {})) { + const path = `mcp[${JSON.stringify(name)}]` + if (server.enabled === false && harness !== 'opencode') { + unsupportedAxes.push(`${path}.enabled`) + } + if (hasEntries(server.headers) && harness === 'codex') { + unsupportedAxes.push(`${path}.headers`) + } + if (server.cwd !== undefined && harness === 'opencode') { + unsupportedAxes.push(`${path}.cwd`) + } + } + if (harness === 'claude') { + for (const [event, commands] of Object.entries(profile.hooks ?? {})) { + for (const [index, command] of commands.entries()) { + const path = `hooks[${JSON.stringify(event)}][${index}]` + if (hasEntries(command.env)) unsupportedAxes.push(`${path}.env`) + if (command.blocking !== undefined) unsupportedAxes.push(`${path}.blocking`) + } + } + } + for (const [name, subagent] of Object.entries(profile.subagents ?? {})) { + const path = `subagents[${JSON.stringify(name)}]` + if (hasEntries(subagent.permissions)) unsupportedAxes.push(`${path}.permissions`) + if (subagent.maxSteps !== undefined) unsupportedAxes.push(`${path}.maxSteps`) + if (hasEntries(subagent.tools) && harness !== 'claude') { + unsupportedAxes.push(`${path}.tools`) + } + } + if (unsupportedAxes.length > 0) { + throw new Error( + `runWorktreeHarness: profile requests unsupported worktree behavior: ${unsupportedAxes.join(', ')}`, + ) + } +} + +function hasEntries(value: object | undefined): boolean { + return value !== undefined && Object.keys(value).length > 0 +} + +function assertSafeMaterializedPaths(plan: WorkspacePlan): void { + for (const file of plan.files) { + if (file.relPath.split('/').some(isGitMetadataSegment)) { + throw new Error( + `runWorktreeHarness: profile file cannot target reserved Git metadata: ${file.relPath}`, + ) + } + } +} + +function isGitMetadataSegment(segment: string): boolean { + const windowsCanonical = segment.replace(/[ .]+$/u, '').toLowerCase() + return windowsCanonical === '.git' || windowsCanonical.startsWith('.git:') +} + +function profileMaterializationReceipt( + applied: WorkspacePlanReceipt, + resourceInstructions: string | undefined, +): WorktreeProfileMaterializationReceipt { + const instructionBytes = + resourceInstructions === undefined ? null : Buffer.from(resourceInstructions, 'utf8') + return { + workspacePlanDigest: applied.workspacePlanDigest, + writtenPaths: [...applied.written], + unsupported: [...applied.unsupported], + environmentNames: Object.keys(applied.env).sort(), + flags: [...applied.flags], + resourceInstructions: instructionBytes + ? { + delivery: 'invocation-prompt', + sha256: `sha256:${createHash('sha256').update(instructionBytes).digest('hex')}`, + byteLength: instructionBytes.byteLength, + } + : { delivery: 'none', sha256: null, byteLength: 0 }, + } +} + /** Run the configured test + typecheck commands in the live worktree, projecting exit codes into * `checks`. Returns `undefined` when neither was configured (so the result omits `checks`). */ export async function runWorktreeChecks(opts: { diff --git a/src/mcp/worktree.ts b/src/mcp/worktree.ts index 018e819f..d37579da 100644 --- a/src/mcp/worktree.ts +++ b/src/mcp/worktree.ts @@ -17,6 +17,7 @@ */ import { spawn } from 'node:child_process' +import { isAbsolute, win32 } from 'node:path' /** @experimental */ export interface WorktreeHandle { @@ -48,6 +49,12 @@ export interface DiffOptions { worktree: WorktreeHandle /** What to compare against. Default `worktree.baseSha`. */ baseRef?: string + /** + * Repository-relative input paths to omit from the captured worker patch. + * Paths are passed to Git with literal exclusion magic, so profile-provided + * `*`, `?`, `[` and `:` characters can never expand into broader pathspecs. + */ + excludePaths?: ReadonlyArray /** Test seam. */ runGit?: GitRunner } @@ -104,12 +111,19 @@ function ensureGitOk( result: { stdout: string; stderr: string; exitCode: number }, ): void { if (result.exitCode !== 0) { - throw new Error( - `worktree: git ${step} failed (exit ${result.exitCode}): ${result.stderr.slice(0, 400)}`, - ) + throw gitFailure(step, result) } } +function gitFailure( + step: string, + result: { stdout: string; stderr: string; exitCode: number }, +): Error { + return new Error( + `worktree: git ${step} failed (exit ${result.exitCode}): ${result.stderr.slice(0, 400)}`, + ) +} + /** Checkout a fresh git worktree for a delegation run on a new branch under `variantsDir`. @experimental */ export async function createWorktree(options: CreateWorktreeOptions): Promise { const variants = options.variantsDir ?? '.agent-worktrees' @@ -130,32 +144,81 @@ export async function createWorktree(options: CreateWorktreeOptions): Promise { const baseRef = options.baseRef ?? options.worktree.baseSha + const pathspecs = worktreeDiffPathspecs(options.excludePaths ?? []) // Stage everything (incl. NEW/untracked files) before diffing: a plain `git diff ` // omits untracked files, so a worker that delivers by CREATING a file (a fresh dossier, // a new module) would produce an empty diff and silently fail to compound. Staging into // the index and diffing `--cached` captures created files. The worktree is ephemeral, so // mutating its index has no observable side effect. - await runGitAsync(['add', '-A'], options.worktree.path, options.runGit) + const staged = await runGitAsync( + ['add', '-A', ...(pathspecs.length > 0 ? ['--', '.', ...pathspecs] : [])], + options.worktree.path, + options.runGit, + ) + ensureGitOk('add -A', staged) const patch = await runGitAsync( - ['diff', '--cached', baseRef], + ['diff', '--cached', baseRef, ...(pathspecs.length > 0 ? ['--', '.', ...pathspecs] : [])], options.worktree.path, options.runGit, ) - // No `ensureGitOk` here — diff returns 0 even when there are no changes. + ensureGitOk(`diff --cached ${baseRef}`, patch) // Stats: `git diff --shortstat` produces e.g. " 3 files changed, 42 insertions(+), 10 deletions(-)". const shortstat = await runGitAsync( - ['diff', '--cached', '--shortstat', baseRef], + [ + 'diff', + '--cached', + '--shortstat', + baseRef, + ...(pathspecs.length > 0 ? ['--', '.', ...pathspecs] : []), + ], options.worktree.path, options.runGit, ) + ensureGitOk(`diff --cached --shortstat ${baseRef}`, shortstat) const stats = parseShortstat(shortstat.stdout) return { patch: patch.stdout, stats } } +function worktreeDiffPathspecs(paths: ReadonlyArray): string[] { + const normalized = new Set() + for (const path of paths) { + if ( + typeof path !== 'string' || + path.length === 0 || + path.includes('\\') || + hasControlCharacter(path) || + isAbsolute(path) || + win32.isAbsolute(path) + ) { + throw new Error( + `worktree: excluded path must be a canonical repository-relative path: ${path}`, + ) + } + const segments = path.split('/') + if (segments.some((segment) => segment.length === 0 || segment === '.' || segment === '..')) { + throw new Error( + `worktree: excluded path must be a canonical repository-relative path: ${path}`, + ) + } + normalized.add(path) + } + return [...normalized] + .sort((left, right) => left.localeCompare(right)) + .map((path) => `:(top,exclude,literal)${path}`) +} + +function hasControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index) + if (code < 0x20 || code === 0x7f) return true + } + return false +} + function parseShortstat(text: string): DiffResult['stats'] { // `text` is the raw stdout of `git diff --shortstat`. Empty when no // changes. Parse defensively — the format is stable but we don't trust @@ -170,28 +233,39 @@ function parseShortstat(text: string): DiffResult['stats'] { return out } -/** Remove a git worktree and delete its branch; tolerates already-removed paths. @experimental */ +/** + * Remove a git worktree and delete its branch. Already-removed paths are harmless; every other + * Git failure rejects so callers cannot report a worktree as destroyed when cleanup failed. + * @experimental + */ export async function removeWorktree(options: RemoveWorktreeOptions): Promise { const force = options.force ?? true const args = ['worktree', 'remove'] if (force) args.push('--force') args.push(options.worktree.path) - const result = await runGitAsync(args, options.repoRoot, options.runGit) - // Don't ensureGitOk — partial-removal scenarios are tolerable; the - // worktree dir may already be gone (caller deleted it manually). - if (result.exitCode !== 0 && !/not a working tree/.test(result.stderr)) { - // Best-effort branch cleanup so the next run can reuse the runId. - await runGitAsync( - ['branch', '-D', options.worktree.branch], - options.repoRoot, - options.runGit, - ).catch(() => undefined) - } - // Always attempt branch removal — the worktree-remove sometimes leaves - // the branch behind even when the directory is gone. - await runGitAsync( + const removal = await runGitAsync(args, options.repoRoot, options.runGit) + const removalError = + removal.exitCode !== 0 && !/not a working tree/iu.test(removal.stderr) + ? gitFailure(`worktree remove ${options.worktree.path}`, removal) + : undefined + + // Worktree removal leaves its branch behind by design. Missing branches are repeat cleanup. + const branchRemoval = await runGitAsync( ['branch', '-D', options.worktree.branch], options.repoRoot, options.runGit, - ).catch(() => undefined) + ) + const branchError = + branchRemoval.exitCode !== 0 && !/branch .+ not found/iu.test(branchRemoval.stderr) + ? gitFailure(`branch -D ${options.worktree.branch}`, branchRemoval) + : undefined + + if (removalError && branchError) { + throw new AggregateError( + [removalError, branchError], + 'worktree: failed to remove worktree and branch', + ) + } + if (removalError) throw removalError + if (branchError) throw branchError } diff --git a/src/otel-export.ts b/src/otel-export.ts index 18119b8a..a3333997 100644 --- a/src/otel-export.ts +++ b/src/otel-export.ts @@ -9,6 +9,9 @@ * (which get converted to OTLP spans automatically). */ +import { type RuntimeTelemetryOptions, sanitizeRuntimeStreamEvent } from './sanitize' +import type { RuntimeStreamEvent } from './types' + export interface OtelExportConfig { /** OTLP endpoint. Reads OTEL_EXPORTER_OTLP_ENDPOINT env by default. */ endpoint?: string @@ -207,21 +210,121 @@ export function flatOtelSpan( traceId: string, timestampMs: number, parentSpanId?: string, + endTimestampMs = timestampMs, ): OtelSpan { - const ts = msToNs(timestampMs) + const start = msToNs(timestampMs) + const end = msToNs(Math.max(timestampMs, endTimestampMs)) return { traceId: padTraceId(traceId), spanId: generateSpanId(), parentSpanId: parentSpanId ? padSpanId(parentSpanId) : undefined, name, kind: 1, - startTimeUnixNano: ts, - endTimeUnixNano: ts, + startTimeUnixNano: start, + endTimeUnixNano: end, attributes: toAttributes(attributes), status: { code: 1 }, } } +export interface RuntimeEventOtelOptions extends RuntimeTelemetryOptions { + /** Final customer redactor applied after the schema-aware runtime sanitizer. */ + redact?: (value: unknown) => unknown +} + +function eventTimestampMs(event: RuntimeStreamEvent): number { + if ('timestamp' in event && typeof event.timestamp === 'string') { + const parsed = Date.parse(event.timestamp) + if (Number.isFinite(parsed)) return parsed + } + return Date.now() +} + +function serialized(value: unknown): string { + try { + return JSON.stringify(value) ?? String(value) + } catch { + return String(value) + } +} + +function mcpIdentity(toolName: string): { server?: string; tool?: string } { + if (!toolName.startsWith('mcp__')) return {} + const [, server, ...toolParts] = toolName.split('__') + return { + ...(server ? { server } : {}), + ...(toolParts.length > 0 ? { tool: toolParts.join('__') } : {}), + } +} + +/** Convert normalized runtime events into lossless, redacted child spans. */ +export function buildRuntimeEventOtelSpans( + events: ReadonlyArray, + traceId: string, + parentSpanId?: string, + options: RuntimeEventOtelOptions = {}, +): OtelSpan[] { + return events.map((event) => { + const sanitized = sanitizeRuntimeStreamEvent(event, options) + const safe = options.redact ? options.redact(sanitized) : sanitized + const record = + safe && typeof safe === 'object' && !Array.isArray(safe) + ? (safe as Record) + : { value: safe } + const attrs: Record = { + 'tangle.runtime.event_type': event.type, + 'tangle.runtime.event': serialized(record), + } + let name = `tangle.runtime.${event.type}` + + if (event.type === 'tool_call' || event.type === 'tool_result') { + name = `agent.${event.type}` + attrs['tool.name'] = event.toolName + if (event.toolCallId) attrs['tool.call_id'] = event.toolCallId + const mcp = mcpIdentity(event.toolName) + if (mcp.server) attrs['mcp.server'] = mcp.server + if (mcp.tool) attrs['mcp.tool.name'] = mcp.tool + const payload = event.type === 'tool_call' ? record.args : record.result + if (payload !== undefined) { + attrs[event.type === 'tool_call' ? 'tool.input' : 'tool.output'] = serialized(payload) + } + } else if (event.type === 'llm_call') { + name = 'gen_ai.client.inference' + attrs['gen_ai.request.model'] = event.model + if (event.tokensIn !== undefined) attrs['gen_ai.usage.input_tokens'] = event.tokensIn + if (event.tokensOut !== undefined) attrs['gen_ai.usage.output_tokens'] = event.tokensOut + if (event.costUsd !== undefined) attrs['tangle.cost.usd'] = event.costUsd + if (event.latencyMs !== undefined) attrs['tangle.latency_ms'] = event.latencyMs + if (event.finishReason !== undefined) + attrs['gen_ai.response.finish_reasons'] = event.finishReason + } else if (event.type === 'backend_error') { + attrs['error.type'] = event.error?.kind ?? 'backend' + attrs['error.message'] = event.message + } else if (event.type === 'final') { + attrs['tangle.outcome.status'] = event.status + attrs['tangle.outcome.reason'] = event.reason + if (event.error) { + attrs['error.type'] = event.error.kind + attrs['error.message'] = event.error.message + } + } + + const startMs = eventTimestampMs(event) + const endMs = + event.type === 'llm_call' && event.latencyMs !== undefined && Number.isFinite(event.latencyMs) + ? startMs + event.latencyMs + : startMs + const span = flatOtelSpan(name, attrs, traceId, startMs, parentSpanId, endMs) + if ( + event.type === 'backend_error' || + (event.type === 'final' && event.status !== 'completed') + ) { + span.status = { code: 2, message: attrs['error.message']?.toString() ?? event.type } + } + return span + }) +} + /** * Sink-neutral node in a reconstructed loop span tree. The root node's * `parentSpanId` is `undefined` — sinks decide how to parent it (the OTEL @@ -503,21 +606,27 @@ function parseHeadersFromEnv(): Record { } function toAttributes(record: Record): OtelAttribute[] { - return Object.entries(record).map(([key, value]) => ({ - key, - value: - typeof value === 'number' - ? Number.isInteger(value) - ? { intValue: value.toString() } - : { doubleValue: value } - : typeof value === 'boolean' - ? { boolValue: value } - : { stringValue: value }, - })) + return Object.entries(record).flatMap(([key, value]) => { + if (typeof value === 'number' && !Number.isFinite(value)) return [] + return [ + { + key, + value: + typeof value === 'number' + ? Number.isInteger(value) + ? { intValue: value.toString() } + : { doubleValue: value } + : typeof value === 'boolean' + ? { boolValue: value } + : { stringValue: value }, + }, + ] + }) } function msToNs(ms: number): string { - return (BigInt(Math.floor(ms)) * 1_000_000n).toString() + const safeMs = Number.isFinite(ms) ? ms : Date.now() + return (BigInt(Math.floor(safeMs)) * 1_000_000n).toString() } function padSpanId(id: string): string { @@ -595,7 +704,7 @@ export interface EvalRunEvent { export interface EvalRunsExportConfig { /** Bearer key — tenant is resolved server-side from it. Reads TANGLE_API_KEY. */ apiKey?: string - /** Intelligence base. Reads INTELLIGENCE_BASE env, else prod. */ + /** Intelligence base. Reads TANGLE_INTELLIGENCE_URL env, else prod. */ base?: string /** Idempotency-Key header (e.g. the runId) — safe retries + upsert. */ idempotencyKey?: string @@ -627,7 +736,7 @@ export async function exportEvalRuns( throw new Error('exportEvalRuns: apiKey required (pass config.apiKey or set TANGLE_API_KEY)') const base = config?.base ?? - (typeof process !== 'undefined' ? process.env.INTELLIGENCE_BASE : undefined) ?? + (typeof process !== 'undefined' ? process.env.TANGLE_INTELLIGENCE_URL : undefined) ?? DEFAULT_INTELLIGENCE_BASE const url = `${base.replace(/\/+$/, '')}/v1/ingest/eval-runs` const res = await fetch(url, { diff --git a/src/primeintellect/index.ts b/src/primeintellect/index.ts new file mode 100644 index 00000000..5f7e7ee3 --- /dev/null +++ b/src/primeintellect/index.ts @@ -0,0 +1,35 @@ +export { + createPrimeIntellectPackage, + type WritePrimeIntellectPackageOptions, + writePrimeIntellectPackage, +} from './package' +export { + createPrimeIntellectBackend, + type PrimeIntellectBackendOptions, + type RunPrimeIntellectProgramOptions, + readPrimeIntellectEpisodeContext, + runPrimeIntellectProgram, +} from './runner' +export { + importPrimeIntellectTraces, + type PrimeIntellectImportDefaults, + type PrimeIntellectTrace, + type PrimeIntellectTraceImportOptions, + parsePrimeIntellectTraces, + primeIntellectTraceToRunRecord, +} from './traces' +export type { + PrimeIntellectContent, + PrimeIntellectEpisodeContext, + PrimeIntellectJson, + PrimeIntellectMessage, + PrimeIntellectPackageBundle, + PrimeIntellectPackageManifest, + PrimeIntellectPackageOptions, + PrimeIntellectPublicTask, + PrimeIntellectRunner, + PrimeIntellectScoring, + PrimeIntellectSetupCommand, + PrimeIntellectSplit, + PrimeIntellectTask, +} from './types' diff --git a/src/primeintellect/package.ts b/src/primeintellect/package.ts new file mode 100644 index 00000000..9e5a6122 --- /dev/null +++ b/src/primeintellect/package.ts @@ -0,0 +1,413 @@ +import { createHash, randomUUID } from 'node:crypto' +import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises' +import { basename, dirname, isAbsolute, join, normalize, relative, resolve, sep } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import type { + PrimeIntellectJson, + PrimeIntellectPackageBundle, + PrimeIntellectPackageManifest, + PrimeIntellectPackageOptions, + PrimeIntellectScoring, + PrimeIntellectTask, +} from './types' +import { validatePrimeIntellectJson, validatePrimeIntellectPrompt } from './validation' + +const VERIFIERS_RANGE = '>=0.2.0,<0.3.0' as const +const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/ +const PACKAGE_NAME = /^[a-z][a-z0-9-]{0,62}$/ +const VERSION = /^\d+\.\d+\.\d+(?:[-+][a-zA-Z0-9.-]+)?$/ +const DEFAULT_MAX_TURNS = 16 +const DEFAULT_ROLLOUT_TIMEOUT = 3_600 +const DEFAULT_SCORING_TIMEOUT = 300 + +export interface WritePrimeIntellectPackageOptions { + /** Replace an existing generated package and restore it if the final swap fails. */ + replace?: boolean +} + +/** Build a complete PrimeIntellect Verifiers v1 package without writing to disk. */ +export function createPrimeIntellectPackage( + options: PrimeIntellectPackageOptions, +): PrimeIntellectPackageBundle { + const validated = validateOptions(options) + const moduleName = validated.name.replaceAll('-', '_') + const rows = validated.tasks.map((task, idx) => taskRow(task, idx)) + const runnerFiles = validated.runner.files ?? {} + const scoringFiles = validated.scoring.kind === 'command' ? (validated.scoring.files ?? {}) : {} + const files: Record = { + 'pyproject.toml': renderPyproject(validated, moduleName), + 'prime.eval.toml': renderPrimeConfig(validated, 'eval'), + 'prime.train.toml': renderPrimeConfig(validated, 'train'), + 'README.md': renderReadme(validated), + [`${moduleName}/__init__.py`]: renderInit(moduleName), + [`${moduleName}/taskset.py`]: renderTaskset(moduleName, validated.scoring), + [`${moduleName}/harness.py`]: renderHarness(moduleName), + [`${moduleName}/tasks.jsonl`]: `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, + [`${moduleName}/runner.json`]: `${JSON.stringify( + { + command: validated.runner.command, + files: runnerFiles, + setup: validated.runner.setup ?? [], + forwardEnv: validated.runner.forwardEnv ?? [], + }, + null, + 2, + )}\n`, + } + for (const [path, contents] of Object.entries(scoringFiles)) { + files[`${moduleName}/scoring/${path}`] = contents + } + + const filesSha256 = Object.fromEntries( + Object.entries(files) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([path, contents]) => [path, sha256(contents)]), + ) + const manifest: PrimeIntellectPackageManifest = { + schema: 'tangle.primeintellect.package/v1', + name: validated.name, + moduleName, + version: validated.version, + verifiers: VERIFIERS_RANGE, + taskCount: validated.tasks.length, + splits: { + train: validated.tasks.filter((task) => task.split === 'train').length, + eval: validated.tasks.filter((task) => task.split === 'eval').length, + }, + taskIdsSha256: sha256( + validated.tasks + .map((task) => `${task.split}:${task.id}`) + .sort() + .join('\n'), + ), + filesSha256, + } + files['manifest.json'] = `${JSON.stringify(manifest, null, 2)}\n` + return { manifest, files: Object.freeze(files) } +} + +/** Write a bundle through a sibling temporary directory, then rename it into place. */ +export async function writePrimeIntellectPackage( + bundle: PrimeIntellectPackageBundle, + outputDirectory: string, + options: WritePrimeIntellectPackageOptions = {}, +): Promise { + const output = resolve(outputDirectory) + const parent = dirname(output) + await mkdir(parent, { recursive: true }) + const replacing = await pathExists(output) + if (replacing) { + if (!options.replace) throw new Error(`PrimeIntellect output already exists: ${output}`) + await assertGeneratedPackage(output) + } + + const temporary = join(parent, `.${basename(output)}.${randomUUID()}.tmp`) + const backup = replacing ? join(parent, `.${basename(output)}.${randomUUID()}.backup`) : undefined + try { + await mkdir(temporary) + for (const [path, contents] of Object.entries(bundle.files)) { + assertRelativePath(path, 'bundle file') + const target = resolve(temporary, path) + if (target !== temporary && !target.startsWith(`${temporary}${sep}`)) { + throw new Error(`bundle file escapes output directory: ${path}`) + } + await mkdir(dirname(target), { recursive: true }) + await writeFile(target, contents, 'utf8') + } + if (backup) await rename(output, backup) + try { + await rename(temporary, output) + } catch (error) { + if (backup) { + try { + await rename(backup, output) + } catch (restoreError) { + throw new AggregateError( + [error, restoreError], + `failed to install PrimeIntellect package and restore ${output}`, + ) + } + } + throw error + } + if (backup) await rm(backup, { recursive: true }) + } catch (error) { + await rm(temporary, { recursive: true, force: true }) + throw error + } + return output +} + +function validateOptions(options: PrimeIntellectPackageOptions): PrimeIntellectPackageOptions { + if (!PACKAGE_NAME.test(options.name)) { + throw new Error('PrimeIntellect package name must match /^[a-z][a-z0-9-]{0,62}$/') + } + if (!VERSION.test(options.version)) { + throw new Error('PrimeIntellect package version must be a numeric semantic version') + } + if (!Array.isArray(options.tasks) || options.tasks.length === 0) { + throw new Error('PrimeIntellect package requires tasks') + } + const tasks: readonly PrimeIntellectTask[] = options.tasks + const seen = new Set() + const seenInputs = new Map() + const splitCounts = { train: 0, eval: 0 } + for (const [index, task] of tasks.entries()) { + validateTask(task, index, options.scoring) + if (seen.has(task.id)) throw new Error(`duplicate PrimeIntellect task id: ${task.id}`) + seen.add(task.id) + const input = canonicalJson({ + prompt: task.prompt, + systemPrompt: task.systemPrompt ?? null, + metadata: task.metadata ?? {}, + }) + const duplicate = seenInputs.get(input) + if (duplicate) { + throw new Error( + `PrimeIntellect tasks ${duplicate.id} (${duplicate.split}) and ${task.id} (${task.split}) expose the same public input`, + ) + } + seenInputs.set(input, { id: task.id, split: task.split }) + splitCounts[task.split] += 1 + } + if (splitCounts.train === 0 || splitCounts.eval === 0) { + throw new Error('PrimeIntellect package requires non-empty, disjoint train and eval splits') + } + validateScoring(options.scoring) + validateCommand(options.runner.command, 'runner.command') + validateFiles(options.runner.files ?? {}, 'runner.files') + for (const [index, command] of (options.runner.setup ?? []).entries()) { + validateCommand(command, `runner.setup[${index}]`) + } + validateEnvNames(options.runner.forwardEnv ?? [], 'runner.forwardEnv') + if (typeof options.runner.image !== 'string' || options.runner.image.trim().length === 0) { + throw new Error('runner.image must be a non-empty container image') + } + if (/(^|:)latest$/i.test(options.runner.image)) { + throw new Error('runner.image must not use the mutable latest tag') + } + positiveInteger(options.maxTurns ?? DEFAULT_MAX_TURNS, 'maxTurns') + for (const [name, value] of [ + ['maxInputTokens', options.maxInputTokens], + ['maxOutputTokens', options.maxOutputTokens], + ['maxTotalTokens', options.maxTotalTokens], + ] as const) { + if (value !== undefined) positiveInteger(value, name) + } + positiveNumber(options.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT, 'rolloutTimeoutSeconds') + positiveNumber(options.scoringTimeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, 'scoringTimeoutSeconds') + return options +} + +function validateTask( + task: PrimeIntellectTask, + index: number, + scoring: PrimeIntellectScoring, +): void { + const path = `tasks[${index}]` + if (typeof task.id !== 'string' || task.id.trim().length === 0) { + throw new Error(`${path}.id must be a non-empty string`) + } + if (task.split !== 'train' && task.split !== 'eval') { + throw new Error(`${path}.split must be train or eval`) + } + const prompt = validatePrimeIntellectPrompt(task.prompt, `${path}.prompt`) + if (Array.isArray(prompt)) { + if (task.systemPrompt !== undefined && prompt.some((message) => message.role === 'system')) { + throw new Error(`${path} must not set systemPrompt and include a system message`) + } + } + if (task.systemPrompt !== undefined && typeof task.systemPrompt !== 'string') { + throw new Error(`${path}.systemPrompt must be a string`) + } + if (task.metadata !== undefined) { + validatePrimeIntellectJson(task.metadata, `${path}.metadata`) + } + if (scoring.kind !== 'command') { + const answers = Array.isArray(task.answer) ? task.answer : [task.answer] + if ( + answers.length === 0 || + answers.some((answer) => typeof answer !== 'string' || answer.length === 0) + ) { + throw new Error(`${path}.answer is required for ${scoring.kind} scoring`) + } + } +} + +function validateScoring(scoring: PrimeIntellectScoring): void { + if (scoring.kind === 'exact') return + if (scoring.kind === 'reference-judge') { + if (typeof scoring.model !== 'string' || scoring.model.length === 0) { + throw new Error('reference-judge scoring requires a model') + } + return + } + validateCommand(scoring.command, 'scoring.command') + validateFiles(scoring.files ?? {}, 'scoring.files') + validateEnvNames(scoring.forwardEnv ?? [], 'scoring.forwardEnv') + positiveNumber(scoring.timeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, 'scoring.timeoutSeconds') +} + +function validateCommand(command: readonly string[], path: string): void { + if (!Array.isArray(command) || command.length === 0) { + throw new Error(`${path} must be a non-empty argv array`) + } + for (const [index, argument] of command.entries()) { + if (typeof argument !== 'string' || argument.length === 0 || argument.includes('\0')) { + throw new Error(`${path}[${index}] must be a non-empty string without NUL bytes`) + } + } +} + +function validateFiles(files: Readonly>, path: string): void { + for (const [file, contents] of Object.entries(files)) { + assertRelativePath(file, path) + if (typeof contents !== 'string') throw new Error(`${path}.${file} must be a string`) + } +} + +function validateEnvNames(names: readonly string[], path: string): void { + const seen = new Set() + for (const [index, name] of names.entries()) { + if (!ENV_NAME.test(name)) throw new Error(`${path}[${index}] is not a valid environment name`) + if (seen.has(name)) throw new Error(`${path} contains duplicate name ${name}`) + seen.add(name) + } +} + +function assertRelativePath(path: string, label: string): void { + const normalized = normalize(path) + if ( + path.length === 0 || + path.includes('\0') || + isAbsolute(path) || + normalized === '..' || + normalized.startsWith(`..${sep}`) || + relative('.', normalized).startsWith('..') + ) { + throw new Error(`${label} contains unsafe path: ${path}`) + } +} + +function taskRow(task: PrimeIntellectTask, idx: number): Record { + return { + idx, + name: task.id, + prompt: task.prompt as PrimeIntellectJson, + system_prompt: task.systemPrompt ?? null, + split: task.split, + answer: task.answer ?? null, + metadata: task.metadata ?? {}, + } +} + +function renderPyproject(options: PrimeIntellectPackageOptions, moduleName: string): string { + const description = options.description ?? `PrimeIntellect tasks for ${options.name}` + return `[project]\nname = ${toml(options.name)}\nversion = ${toml(options.version)}\ndescription = ${toml(description)}\nrequires-python = ">=3.11,<3.14"\ndependencies = ["verifiers${VERIFIERS_RANGE}"]\n\n[build-system]\nrequires = ["hatchling"]\nbuild-backend = "hatchling.build"\n\n[tool.hatch.build.targets.wheel]\npackages = [${toml(moduleName)}]\n\n[tool.uv]\nprerelease = "allow"\n` +} + +function renderPrimeConfig(options: PrimeIntellectPackageOptions, split: 'train' | 'eval'): string { + const limits = [ + `max_turns = ${options.maxTurns ?? DEFAULT_MAX_TURNS}`, + options.maxInputTokens === undefined + ? undefined + : `max_input_tokens = ${options.maxInputTokens}`, + options.maxOutputTokens === undefined + ? undefined + : `max_output_tokens = ${options.maxOutputTokens}`, + options.maxTotalTokens === undefined + ? undefined + : `max_total_tokens = ${options.maxTotalTokens}`, + ].filter((line): line is string => line !== undefined) + return `${limits.join('\n')}\npush = false\n\n[timeout]\nrollout = ${options.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT}\nscoring = ${options.scoringTimeoutSeconds ?? DEFAULT_SCORING_TIMEOUT}\n\n[taskset]\nid = ${toml(options.name)}\nsplit = ${toml(split)}\n\n[harness]\nid = ${toml(options.name)}\nprogram = ${tomlArray(options.runner.command)}\nforward_env = ${tomlArray(options.runner.forwardEnv ?? [])}\n\n[harness.runtime]\ntype = "docker"\nimage = ${toml(options.runner.image)}\n` +} + +function renderInit(moduleName: string): string { + return `from ${moduleName}.harness import TangleRuntimeHarness\nfrom ${moduleName}.taskset import TangleTaskset\n\n__all__ = ["TangleRuntimeHarness", "TangleTaskset"]\n` +} + +function renderTaskset(moduleName: string, scoring: PrimeIntellectScoring): string { + const config = scoringConfig(scoring) + return `import asyncio\nimport json\nimport math\nimport os\nfrom importlib.resources import files\nfrom pathlib import Path\nfrom typing import Any, Literal\n\nimport verifiers.v1 as vf\n\nSCORING = json.loads(${pythonString(JSON.stringify(config))})\nPACKAGE_ROOT = Path(__file__).resolve().parent\n\n\nclass TangleTaskData(vf.TaskData):\n split: Literal["train", "eval"]\n answer: str | list[str] | None = None\n metadata: dict[str, Any] = {}\n\n\nclass TangleTaskConfig(vf.TaskConfig):\n scoring: Literal["exact", "reference-judge", "command"] = SCORING["kind"]\n normalization: Literal["none", "trim", "trim-casefold"] = SCORING.get("normalization", "trim")\n judge_model: str = SCORING.get("model", "openai/gpt-5.4-nano")\n judge_prompt: str | None = SCORING.get("prompt")\n judge_view: Literal["last_reply", "full_trace"] = SCORING.get("view", "last_reply")\n score_program: list[str] = SCORING.get("command", [])\n score_forward_env: list[str] = SCORING.get("forwardEnv", [])\n score_timeout_seconds: float = SCORING.get("timeoutSeconds", 300)\n\n\ndef _normalize(value: str, mode: str) -> str:\n if mode == "none":\n return value\n value = value.strip()\n return value.casefold() if mode == "trim-casefold" else value\n\n\nasync def _run_score_command(config: TangleTaskConfig, data: TangleTaskData, trace: vf.Trace) -> float:\n if not config.score_program:\n raise ValueError("command scoring requires score_program")\n safe_env = {\n key: os.environ[key]\n for key in ("PATH", "HOME", "TMPDIR", "LANG")\n if key in os.environ\n }\n safe_env.update(\n {key: os.environ[key] for key in config.score_forward_env if key in os.environ}\n )\n request = {\n "protocol": "tangle.primeintellect.score/v1",\n "task": data.model_dump(mode="json", exclude_none=True),\n "trace": trace.model_dump(mode="json", exclude_none=True),\n }\n process = await asyncio.create_subprocess_exec(\n *config.score_program,\n cwd=PACKAGE_ROOT,\n env=safe_env,\n stdin=asyncio.subprocess.PIPE,\n stdout=asyncio.subprocess.PIPE,\n stderr=asyncio.subprocess.PIPE,\n )\n payload = json.dumps(request, separators=(",", ":")).encode()\n try:\n stdout, stderr = await asyncio.wait_for(\n process.communicate(payload), timeout=config.score_timeout_seconds\n )\n except TimeoutError:\n process.kill()\n await process.communicate()\n raise RuntimeError(\n f"score command timed out after {config.score_timeout_seconds}s"\n )\n if process.returncode != 0:\n detail = (stderr or stdout).decode(errors="replace").strip()[-2000:]\n raise RuntimeError(f"score command exited {process.returncode}: {detail}")\n try:\n result = json.loads(stdout)\n except json.JSONDecodeError as error:\n raise ValueError(f"score command returned invalid JSON: {error}") from error\n if not isinstance(result, dict):\n raise ValueError("score command must return an object")\n reward = result.get("reward")\n if isinstance(reward, bool) or not isinstance(reward, (int, float)) or not math.isfinite(reward):\n raise ValueError("score command reward must be a finite number")\n metrics = result.get("metrics", {})\n if not isinstance(metrics, dict):\n raise ValueError("score command metrics must be an object")\n for name, value in metrics.items():\n if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):\n raise ValueError(f"score command metric {name!r} must be a finite number")\n trace.record_metrics(metrics)\n return float(reward)\n\n\nclass TangleTask(vf.Task[TangleTaskData, vf.State, TangleTaskConfig]):\n @vf.reward(weight=1.0)\n async def task_reward(self, trace: vf.Trace) -> float:\n if self.config.scoring == "command":\n return await _run_score_command(self.config, self.data, trace)\n if self.data.answer is None:\n raise ValueError(f"task {self.data.name!r} has no reference answer")\n if self.config.scoring == "reference-judge":\n judge = vf.ReferenceJudge(\n vf.ReferenceJudgeConfig(\n model=self.config.judge_model,\n prompt=self.config.judge_prompt,\n view=self.config.judge_view,\n )\n )\n return await judge.score(self.data, trace)\n expected = self.data.answer if isinstance(self.data.answer, list) else [self.data.answer]\n actual = _normalize(trace.last_reply, self.config.normalization)\n return float(any(actual == _normalize(answer, self.config.normalization) for answer in expected))\n\n\nclass TangleTasksetConfig(vf.TasksetConfig):\n split: Literal["train", "eval"] = "eval"\n task: TangleTaskConfig = TangleTaskConfig()\n\n\nclass TangleTaskset(vf.Taskset[TangleTask, TangleTasksetConfig]):\n def load(self) -> list[TangleTask]:\n resource = files(${pythonString(moduleName)}).joinpath("tasks.jsonl")\n tasks: list[TangleTask] = []\n with resource.open("r", encoding="utf-8") as handle:\n for line in handle:\n if not line.strip():\n continue\n row = json.loads(line)\n if row["split"] != self.config.split:\n continue\n tasks.append(TangleTask(TangleTaskData.model_validate(row), self.config.task))\n if not tasks:\n raise ValueError(f"task split {self.config.split!r} is empty")\n return tasks\n\n\n__all__ = ["TangleTaskset"]\n` +} + +function renderHarness(moduleName: string): string { + return `import json\nfrom importlib.resources import files\n\nimport verifiers.v1 as vf\n\nRUNNER = json.loads(files(${pythonString(moduleName)}).joinpath("runner.json").read_text(encoding="utf-8"))\n\n\nclass TangleRuntimeHarnessConfig(vf.HarnessConfig):\n program: list[str] = RUNNER["command"]\n setup_commands: list[list[str]] = RUNNER["setup"]\n forward_env: list[str] = RUNNER["forwardEnv"]\n\n\nclass TangleRuntimeHarness(vf.Harness[TangleRuntimeHarnessConfig]):\n APPENDS_SYSTEM_PROMPT = True\n SUPPORTS_MCP = True\n SUPPORTS_MESSAGE_PROMPT = True\n\n async def setup(self, runtime: vf.Runtime) -> None:\n for path, contents in RUNNER["files"].items():\n await runtime.write(path, contents.encode())\n for command in self.config.setup_commands:\n result = await runtime.run(command, self.config.resolved_env)\n if result.exit_code != 0:\n detail = (result.stderr or result.stdout).strip()[-2000:]\n raise RuntimeError(\n f"runner setup command {command[0]!r} exited {result.exit_code}: {detail}"\n )\n\n async def launch(\n self,\n ctx: vf.ModelContext,\n trace: vf.Trace,\n runtime: vf.Runtime,\n endpoint: str,\n secret: str,\n mcp_urls: dict[str, str],\n ) -> vf.ProgramResult:\n data = trace.task.data\n public_task = {\n "id": data.name or str(data.idx),\n "split": data.split,\n "prompt": data.prompt,\n "metadata": data.metadata,\n }\n if data.system_prompt is not None:\n public_task["systemPrompt"] = data.system_prompt\n env = {\n **self.config.resolved_env,\n "OPENAI_BASE_URL": endpoint,\n "OPENAI_API_KEY": secret,\n "OPENAI_MODEL": ctx.model,\n "TANGLE_PRIME_TASK_JSON": json.dumps(public_task, separators=(",", ":")),\n "TANGLE_PRIME_MCP_SERVERS_JSON": json.dumps(mcp_urls, separators=(",", ":")),\n }\n if not self.config.program:\n raise ValueError("Tangle runtime harness requires a program argv")\n return await runtime.run_program(self.config.program, env)\n\n\n__all__ = ["TangleRuntimeHarness"]\n` +} + +function scoringConfig(scoring: PrimeIntellectScoring): Record { + if (scoring.kind === 'exact') { + return { kind: scoring.kind, normalization: scoring.normalization ?? 'trim' } + } + if (scoring.kind === 'reference-judge') { + return { + kind: scoring.kind, + model: scoring.model, + prompt: scoring.prompt ?? null, + view: scoring.view ?? 'last_reply', + } + } + return { + kind: scoring.kind, + command: [...scoring.command], + forwardEnv: [...(scoring.forwardEnv ?? [])], + timeoutSeconds: scoring.timeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, + } +} + +function renderReadme(options: PrimeIntellectPackageOptions): string { + return `# ${options.name}\n\nPrimeIntellect Verifiers v1 tasks that run the caller's Tangle agent program.\n\n## Evaluate\n\n\`\`\`bash\nuv run eval @ prime.eval.toml --model \n\`\`\`\n\n## Train\n\nUse \`prime.train.toml\` as the environment config for Prime RL. The train and eval rows are disjoint and selected by \`taskset.split\`.\n\nThe runner receives only the prompt, metadata, intercepted model endpoint, and MCP URLs. Reference answers remain in the task process and are never written into the runner workspace or environment.\n` +} + +function toml(value: string): string { + return JSON.stringify(value) +} + +function tomlArray(values: readonly string[]): string { + return `[${values.map(toml).join(', ')}]` +} + +function pythonString(value: string): string { + return JSON.stringify(value) +} + +function positiveInteger(value: number, path: string): void { + if (!Number.isSafeInteger(value) || value <= 0) + throw new Error(`${path} must be a positive integer`) +} + +function positiveNumber(value: number, path: string): void { + if (!Number.isFinite(value) || value <= 0) throw new Error(`${path} must be positive`) +} + +async function pathExists(path: string): Promise { + try { + await stat(path) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false + throw error + } +} + +async function assertGeneratedPackage(output: string): Promise { + let manifest: unknown + try { + manifest = JSON.parse(await readFile(join(output, 'manifest.json'), 'utf8')) + } catch (error) { + throw new Error( + `refusing to replace ${output}: it is not a generated PrimeIntellect package (${error instanceof Error ? error.message : String(error)})`, + ) + } + if ( + manifest === null || + typeof manifest !== 'object' || + (manifest as Record).schema !== 'tangle.primeintellect.package/v1' + ) { + throw new Error(`refusing to replace ${output}: manifest schema does not match`) + } +} + +function sha256(value: string): string { + return createHash('sha256').update(value, 'utf8').digest('hex') +} diff --git a/src/primeintellect/primeintellect.test.ts b/src/primeintellect/primeintellect.test.ts new file mode 100644 index 00000000..ff893d15 --- /dev/null +++ b/src/primeintellect/primeintellect.test.ts @@ -0,0 +1,369 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { createPrimeIntellectPackage, writePrimeIntellectPackage } from './package' +import { + createPrimeIntellectBackend, + readPrimeIntellectEpisodeContext, + runPrimeIntellectProgram, +} from './runner' +import { + importPrimeIntellectTraces, + type PrimeIntellectTrace, + parsePrimeIntellectTraces, + primeIntellectTraceToRunRecord, +} from './traces' +import type { PrimeIntellectPackageOptions } from './types' + +function packageOptions(): PrimeIntellectPackageOptions { + return { + name: 'support-agent-v1', + version: '1.0.0', + tasks: [ + { + id: 'train-refund', + split: 'train', + prompt: 'Can this order be refunded?', + answer: 'No', + metadata: { policy: 'refunds' }, + }, + { + id: 'eval-refund', + split: 'eval', + prompt: 'Is a final-sale order refundable?', + answer: 'No', + }, + ], + scoring: { kind: 'exact', normalization: 'trim-casefold' }, + runner: { + image: 'node:22-bookworm-slim', + files: { 'runner.mjs': 'console.log("runner")\n' }, + command: ['node', 'runner.mjs'], + }, + } +} + +describe('PrimeIntellect v1 package', () => { + it('builds a split-safe package around the caller runtime program', () => { + const bundle = createPrimeIntellectPackage(packageOptions()) + + expect(bundle.manifest.splits).toEqual({ train: 1, eval: 1 }) + expect(bundle.manifest.verifiers).toBe('>=0.2.0,<0.3.0') + expect(bundle.files['prime.eval.toml']).toContain('max_turns = 16') + expect(bundle.files['prime.eval.toml']).toContain('type = "docker"') + expect(bundle.files['support_agent_v1/taskset.py']).toContain('import verifiers.v1 as vf') + expect(bundle.files['support_agent_v1/harness.py']).toContain('runtime.run_program') + expect(bundle.files['support_agent_v1/harness.py']).not.toContain('data.answer') + expect(bundle.files['support_agent_v1/harness.py']).not.toContain('reference') + }) + + it('requires both splits and rejects duplicate ids and unsafe runner files', () => { + const options = packageOptions() + expect(() => + createPrimeIntellectPackage({ ...options, tasks: options.tasks.slice(0, 1) }), + ).toThrow(/non-empty, disjoint train and eval splits/) + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [options.tasks[0]!, { ...options.tasks[1]!, id: options.tasks[0]!.id }], + }), + ).toThrow(/duplicate PrimeIntellect task id/) + expect(() => + createPrimeIntellectPackage({ + ...options, + runner: { ...options.runner, files: { '../answer.txt': 'secret' } }, + }), + ).toThrow(/unsafe path/) + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [options.tasks[0]!, { ...options.tasks[0]!, id: 'eval-copy', split: 'eval' }], + }), + ).toThrow(/expose the same public input/) + }) + + it('rejects malformed messages and duplicate system prompts', () => { + const options = packageOptions() + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [{ ...options.tasks[0]!, prompt: [{ role: 'user' }] as never }, options.tasks[1]!], + }), + ).toThrow(/content/) + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [ + { + ...options.tasks[0]!, + systemPrompt: 'First system prompt', + prompt: [{ role: 'system', content: 'Second system prompt' }], + }, + options.tasks[1]!, + ], + }), + ).toThrow(/must not set systemPrompt and include a system message/) + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [ + { + ...options.tasks[0]!, + prompt: [{ role: 'assistant', content: null, unsupported: true }] as never, + }, + options.tasks[1]!, + ], + }), + ).toThrow(/unsupported is not supported/) + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [ + { + ...options.tasks[0]!, + prompt: [ + { + role: 'assistant', + content: null, + provider_state: [{ signature: 'opaque-provider-state' }], + }, + ], + }, + options.tasks[1]!, + ], + }), + ).not.toThrow() + }) + + it('writes atomically and refuses an existing destination by default', async () => { + const root = await mkdtemp(join(tmpdir(), 'prime-package-test-')) + const output = join(root, 'generated') + try { + const bundle = createPrimeIntellectPackage(packageOptions()) + await writePrimeIntellectPackage(bundle, output) + expect(JSON.parse(await readFile(join(output, 'manifest.json'), 'utf8'))).toEqual( + bundle.manifest, + ) + await expect(writePrimeIntellectPackage(bundle, output)).rejects.toThrow(/already exists/) + await expect(writePrimeIntellectPackage(bundle, output, { replace: true })).resolves.toBe( + output, + ) + + const unrelated = join(root, 'user-owned') + await mkdir(unrelated) + await writeFile(join(unrelated, 'important.txt'), 'keep me', 'utf8') + await expect( + writePrimeIntellectPackage(bundle, unrelated, { replace: true }), + ).rejects.toThrow(/not a generated PrimeIntellect package/) + await expect(readFile(join(unrelated, 'important.txt'), 'utf8')).resolves.toBe('keep me') + } finally { + await rm(root, { recursive: true, force: true }) + } + }) +}) + +describe('PrimeIntellect runtime program contract', () => { + const env = { + TANGLE_PRIME_TASK_JSON: JSON.stringify({ + id: 'eval-refund', + split: 'eval', + prompt: 'Is this refundable?', + systemPrompt: 'Use the policy tools.', + metadata: { tenant: 'test' }, + }), + TANGLE_PRIME_MCP_SERVERS_JSON: JSON.stringify({ policy: 'http://127.0.0.1:3210/mcp' }), + OPENAI_MODEL: 'openai/gpt-5.4-20260601', + OPENAI_BASE_URL: 'http://127.0.0.1:9000/v1', + OPENAI_API_KEY: 'interception-secret', + } + + it('reads an answer-free episode and creates the existing backend', async () => { + const context = readPrimeIntellectEpisodeContext(env) + expect(context.task.id).toBe('eval-refund') + expect(context.mcpServers.policy).toBe('http://127.0.0.1:3210/mcp') + expect(createPrimeIntellectBackend(context).kind).toBe('primeintellect') + expect(createPrimeIntellectBackend(context, { kind: 'product-runtime' }).kind).toBe( + 'product-runtime', + ) + await expect( + runPrimeIntellectProgram(async (episode) => episode.task.id, { env }), + ).resolves.toBe('eval-refund') + }) + + it('fails if the task process exposes a private scoring field', () => { + expect(() => + readPrimeIntellectEpisodeContext({ + ...env, + TANGLE_PRIME_TASK_JSON: JSON.stringify({ + id: 'leaked', + split: 'eval', + prompt: 'Question', + answer: 'private', + }), + }), + ).toThrow(/exposed private field answer/) + }) + + it('rejects conflicting system prompts and malformed tool calls', () => { + expect(() => + readPrimeIntellectEpisodeContext({ + ...env, + TANGLE_PRIME_TASK_JSON: JSON.stringify({ + id: 'conflicting-system', + split: 'eval', + prompt: [{ role: 'system', content: 'In prompt' }], + systemPrompt: 'Alongside prompt', + }), + }), + ).toThrow(/must not set systemPrompt and include a system message/) + expect(() => + readPrimeIntellectEpisodeContext({ + ...env, + TANGLE_PRIME_TASK_JSON: JSON.stringify({ + id: 'bad-tool-call', + split: 'eval', + prompt: [{ role: 'assistant', tool_calls: [{ id: 'call-1', name: 'search' }] }], + }), + }), + ).toThrow(/arguments/) + }) +}) + +function primeTrace(): PrimeIntellectTrace { + return { + id: 'prime-trace-1', + task: { + type: 'TangleTask', + data: { idx: 1, name: 'eval-refund', split: 'eval', prompt: 'Question' }, + }, + nodes: [ + { + parent: null, + sampled: false, + usage: null, + }, + { + parent: 0, + sampled: true, + usage: { + prompt_tokens: 100, + completion_tokens: 20, + cached_input_tokens: 40, + reasoning_tokens: 5, + cost: 0.03, + }, + }, + { + parent: 1, + sampled: true, + usage: { + prompt_tokens: 50, + completion_tokens: 10, + cost: 0.02, + }, + }, + ], + rewards: { task_reward: 0.8 }, + metrics: { citation_precision: 0.75 }, + extra_usage: [{ prompt_tokens: 10, completion_tokens: 2, reasoning_tokens: 1, cost: 0.01 }], + is_completed: true, + stop_condition: 'agent_completed', + errors: [], + timing: { + start: 100, + generation: { start: 101, end: 104 }, + scoring: { start: 104, end: 105.5 }, + }, + } +} + +const importOptions = { + experimentId: 'prime-eval-1', + candidateId: 'support-agent-a', + seed: 42, + model: 'openai/gpt-5.4-20260601', + promptHash: 'a'.repeat(64), + configHash: 'b'.repeat(64), + commitSha: 'c'.repeat(40), +} + +describe('PrimeIntellect trace import', () => { + it('preserves score, metrics, turns, tokens, cost, duration, scenario, and split', () => { + const record = primeIntellectTraceToRunRecord(primeTrace(), importOptions) + + expect(record.splitTag).toBe('holdout') + expect(record.scenarioId).toBe('eval-refund') + expect(record.outcome.holdoutScore).toBe(0.8) + expect(record.outcome.raw).toMatchObject({ + reward: 0.8, + 'prime.turns': 2, + 'prime.branches': 1, + 'metric.citation_precision': 0.75, + }) + expect(record.tokenUsage).toEqual({ input: 200, output: 32, reasoning: 6, cached: 40 }) + expect(record.costUsd).toBeCloseTo(0.06) + expect(record.costProvenance?.kind).toBe('observed') + expect(record.costProvenance?.usd).toBeCloseTo(0.06) + expect(record.wallMs).toBe(5_500) + }) + + it('parses durable JSONL and refuses empty or malformed inputs', () => { + const jsonl = `${JSON.stringify(primeTrace())}\n` + expect(parsePrimeIntellectTraces(jsonl)).toHaveLength(1) + expect(importPrimeIntellectTraces(jsonl, importOptions)).toHaveLength(1) + expect(() => parsePrimeIntellectTraces('\n')).toThrow(/contains no traces/) + expect(() => parsePrimeIntellectTraces('{oops}\n')).toThrow(/line 1 is invalid JSON/) + }) + + it('marks partial cost as uncaptured while retaining the reported subtotal', () => { + const trace = primeTrace() + delete trace.nodes[2]!.usage!.cost + const record = primeIntellectTraceToRunRecord(trace, importOptions) + + expect(record.costUsd).toBe(0) + expect(record.costProvenance).toEqual({ kind: 'uncaptured', usd: null }) + expect(record.outcome.raw).toMatchObject({ + 'prime.cost_complete': 0, + 'prime.reported_cost_usd': 0.04, + }) + }) + + it('rejects malformed graph nodes and usage instead of undercounting them', () => { + expect(() => + parsePrimeIntellectTraces(`${JSON.stringify({ ...primeTrace(), nodes: [null] })}\n`), + ).toThrow(/nodes\[0\] must be an object/) + expect(() => + parsePrimeIntellectTraces( + `${JSON.stringify({ + ...primeTrace(), + nodes: [{ parent: 1, sampled: true, usage: null }], + })}\n`, + ), + ).toThrow(/parent must reference an earlier node/) + expect(() => + parsePrimeIntellectTraces( + `${JSON.stringify({ + ...primeTrace(), + nodes: [ + { + parent: null, + sampled: true, + usage: { prompt_tokens: -1, completion_tokens: 0 }, + }, + ], + })}\n`, + ), + ).toThrow(/prompt_tokens must be non-negative/) + expect(() => + parsePrimeIntellectTraces( + `${JSON.stringify({ ...primeTrace(), errors: [{ type: 'timeout' }] })}\n`, + ), + ).toThrow(/message must be a non-empty string/) + expect(() => + parsePrimeIntellectTraces( + `${JSON.stringify({ ...primeTrace(), timing: { start: 'now' } })}\n`, + ), + ).toThrow(/timing.start must be a finite number/) + }) +}) diff --git a/src/primeintellect/runner.ts b/src/primeintellect/runner.ts new file mode 100644 index 00000000..2cc78960 --- /dev/null +++ b/src/primeintellect/runner.ts @@ -0,0 +1,165 @@ +import { createOpenAICompatibleBackend } from '../backends' +import type { + PrimeIntellectEpisodeContext, + PrimeIntellectPublicTask, + PrimeIntellectSplit, +} from './types' +import { validatePrimeIntellectJsonObject, validatePrimeIntellectPrompt } from './validation' + +const ENV = { + task: 'TANGLE_PRIME_TASK_JSON', + model: 'OPENAI_MODEL', + baseUrl: 'OPENAI_BASE_URL', + apiKey: 'OPENAI_API_KEY', + mcpServers: 'TANGLE_PRIME_MCP_SERVERS_JSON', +} as const + +export interface RunPrimeIntellectProgramOptions { + env?: NodeJS.ProcessEnv +} + +export type PrimeIntellectBackendOptions = Omit< + Parameters[0], + 'apiKey' | 'baseUrl' | 'model' +> + +/** Read and validate the private process contract installed by the generated Prime harness. */ +export function readPrimeIntellectEpisodeContext( + env: NodeJS.ProcessEnv = process.env, +): PrimeIntellectEpisodeContext { + const rawTask = requiredEnv(env, ENV.task) + const rawMcp = env[ENV.mcpServers] ?? '{}' + const parsedTask = parseJson(rawTask, ENV.task) + const parsedMcp = parseJson(rawMcp, ENV.mcpServers) + const task = validatePublicTask(parsedTask) + const mcpServers = validateStringMap(parsedMcp, ENV.mcpServers) + + return { + protocol: 'tangle.primeintellect.episode/v1', + task, + model: { + name: requiredEnv(env, ENV.model), + baseUrl: requiredEnv(env, ENV.baseUrl), + apiKey: requiredEnv(env, ENV.apiKey), + }, + mcpServers, + } +} + +/** Build the existing runtime backend against Prime's intercepted model endpoint. */ +export function createPrimeIntellectBackend( + context: PrimeIntellectEpisodeContext, + options: PrimeIntellectBackendOptions = {}, +) { + return createOpenAICompatibleBackend({ + ...options, + apiKey: context.model.apiKey, + baseUrl: context.model.baseUrl, + model: context.model.name, + kind: options.kind ?? 'primeintellect', + }) +} + +/** + * Execute the caller's canonical runtime program inside a Prime rollout. + * The callback may call runPersonified, runAgentic, runLoop, or any product wrapper. + */ +export async function runPrimeIntellectProgram( + run: (context: PrimeIntellectEpisodeContext) => Promise, + options: RunPrimeIntellectProgramOptions = {}, +): Promise { + return run(readPrimeIntellectEpisodeContext(options.env)) +} + +function requiredEnv(env: NodeJS.ProcessEnv, name: string): string { + const value = env[name] + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`PrimeIntellect runner requires ${name}`) + } + return value +} + +function parseJson(value: string, name: string): unknown { + try { + return JSON.parse(value) + } catch (error) { + throw new Error( + `${name} must contain valid JSON: ${error instanceof Error ? error.message : String(error)}`, + ) + } +} + +function validatePublicTask(value: unknown): PrimeIntellectPublicTask { + const task = record(value, ENV.task) + for (const privateField of ['answer', 'reference', 'scoring', 'score']) { + if (privateField in task) { + throw new Error(`${ENV.task} exposed private field ${privateField}`) + } + } + for (const field of Object.keys(task)) { + if (!['id', 'split', 'prompt', 'systemPrompt', 'metadata'].includes(field)) { + throw new Error(`${ENV.task}.${field} is not supported`) + } + } + const id = nonEmptyString(task.id, `${ENV.task}.id`) + const split = validateSplit(task.split, `${ENV.task}.split`) + const prompt = validatePrimeIntellectPrompt(task.prompt, `${ENV.task}.prompt`) + const systemPrompt = optionalString(task.systemPrompt, `${ENV.task}.systemPrompt`) + if ( + systemPrompt !== undefined && + Array.isArray(prompt) && + prompt.some((message) => message.role === 'system') + ) { + throw new Error(`${ENV.task} must not set systemPrompt and include a system message`) + } + const metadata = + task.metadata === undefined + ? undefined + : validatePrimeIntellectJsonObject(task.metadata, `${ENV.task}.metadata`) + return { + id, + split, + prompt, + ...(systemPrompt !== undefined ? { systemPrompt } : {}), + ...(metadata !== undefined ? { metadata } : {}), + } +} + +function validateSplit(value: unknown, path: string): PrimeIntellectSplit { + if (value !== 'train' && value !== 'eval') { + throw new Error(`${path} must be train or eval`) + } + return value +} + +function validateStringMap(value: unknown, path: string): Readonly> { + const input = record(value, path) + const output: Record = {} + for (const [key, entry] of Object.entries(input)) { + if (typeof entry !== 'string' || entry.length === 0) { + throw new Error(`${path}.${key} must be a non-empty string`) + } + output[key] = entry + } + return output +} + +function record(value: unknown, path: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${path} must be an object`) + } + return value as Record +} + +function nonEmptyString(value: unknown, path: string): string { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`${path} must be a non-empty string`) + } + return value +} + +function optionalString(value: unknown, path: string): string | undefined { + if (value === undefined) return undefined + if (typeof value !== 'string') throw new Error(`${path} must be a string`) + return value +} diff --git a/src/primeintellect/traces.ts b/src/primeintellect/traces.ts new file mode 100644 index 00000000..003b9477 --- /dev/null +++ b/src/primeintellect/traces.ts @@ -0,0 +1,377 @@ +import { type RunRecord, validateRunRecord } from '@tangle-network/agent-eval' + +interface PrimeUsage { + prompt_tokens: number + completion_tokens: number + cached_input_tokens?: number | null + reasoning_tokens?: number | null + cost?: number | null +} + +interface PrimeTraceNode { + parent?: number | null + sampled?: boolean + usage?: PrimeUsage | null + message?: unknown +} + +interface PrimeTimeSpan { + start?: number + end?: number +} + +export interface PrimeIntellectTrace { + id: string + task: { + type: string + data: { + idx: number + name?: string | null + split?: 'train' | 'eval' + prompt?: unknown + system_prompt?: string | null + metadata?: Record + [key: string]: unknown + } + } + runtime?: unknown + nodes: PrimeTraceNode[] + rewards: Record + metrics: Record + info?: Record + extra_usage?: PrimeUsage[] + is_completed?: boolean + stop_condition?: string | null + errors?: Array<{ type: string; message: string; traceback?: string | null }> + timing?: { + start?: number + setup?: PrimeTimeSpan + generation?: PrimeTimeSpan + finalize?: PrimeTimeSpan + scoring?: PrimeTimeSpan + } +} + +export interface PrimeIntellectTraceImportOptions { + experimentId: string + candidateId: string + seed: number + /** Snapshot-pinned model id required by RunRecord validation. */ + model: string + promptHash: string + configHash: string + commitSha: string +} + +export type PrimeIntellectImportDefaults = PrimeIntellectTraceImportOptions + +/** Parse Prime's durable `traces.jsonl` and reject malformed rows with a line number. */ +export function parsePrimeIntellectTraces(jsonl: string): PrimeIntellectTrace[] { + const traces: PrimeIntellectTrace[] = [] + for (const [lineIndex, line] of jsonl.split('\n').entries()) { + if (!line.trim()) continue + let parsed: unknown + try { + parsed = JSON.parse(line) + } catch (error) { + throw new Error( + `PrimeIntellect trace line ${lineIndex + 1} is invalid JSON: ${ + error instanceof Error ? error.message : String(error) + }`, + ) + } + traces.push(validateTrace(parsed, lineIndex + 1)) + } + if (traces.length === 0) throw new Error('PrimeIntellect traces.jsonl contains no traces') + return traces +} + +/** Convert all Prime traces to agent-eval RunRecords while retaining one shared run config. */ +export function importPrimeIntellectTraces( + jsonl: string, + defaults: PrimeIntellectImportDefaults, +): RunRecord[] { + return parsePrimeIntellectTraces(jsonl).map((trace) => + primeIntellectTraceToRunRecord(trace, defaults), + ) +} + +/** Project one complete Prime trace into the common agent-eval analysis row. */ +export function primeIntellectTraceToRunRecord( + trace: PrimeIntellectTrace, + options: PrimeIntellectTraceImportOptions, +): RunRecord { + const split = trace.task.data.split + if (split !== 'train' && split !== 'eval') { + throw new Error(`PrimeIntellect trace ${trace.id} has no train/eval split`) + } + const reward = sumFinite(Object.values(trace.rewards), `trace ${trace.id} rewards`) + const usage = aggregateUsage([ + ...trace.nodes.map((node) => node.usage).filter((value): value is PrimeUsage => value != null), + ...(trace.extra_usage ?? []), + ]) + const errors = trace.errors ?? [] + const raw: Record = { + reward, + 'prime.turns': trace.nodes.filter((node) => node.sampled === true).length, + 'prime.branches': countBranches(trace.nodes), + 'prime.errors': errors.length, + 'prime.completed': trace.is_completed ? 1 : 0, + 'prime.cost_complete': usage.costComplete ? 1 : 0, + 'prime.reported_cost_usd': usage.reportedCostUsd, + } + for (const [name, value] of Object.entries(trace.rewards)) { + raw[`reward.${name}`] = finite(value, `trace ${trace.id} reward ${name}`) + } + for (const [name, value] of Object.entries(trace.metrics)) { + raw[`metric.${name}`] = finite(value, `trace ${trace.id} metric ${name}`) + } + + const record: RunRecord = { + runId: trace.id, + experimentId: options.experimentId, + candidateId: options.candidateId, + seed: options.seed, + model: options.model, + promptHash: options.promptHash, + configHash: options.configHash, + commitSha: options.commitSha, + wallMs: traceWallMs(trace), + costUsd: usage.costComplete ? usage.reportedCostUsd : 0, + costProvenance: usage.costComplete + ? { kind: 'observed', usd: usage.reportedCostUsd } + : { kind: 'uncaptured', usd: null }, + tokenUsage: { + input: usage.input, + output: usage.output, + ...(usage.reasoning !== undefined ? { reasoning: usage.reasoning } : {}), + ...(usage.cached !== undefined ? { cached: usage.cached } : {}), + }, + outcome: split === 'eval' ? { holdoutScore: reward, raw } : { searchScore: reward, raw }, + splitTag: split === 'eval' ? 'holdout' : 'search', + scenarioId: trace.task.data.name ?? String(trace.task.data.idx), + ...(errors[0] ? { failureMode: `primeintellect:${errors[0].type}:${errors[0].message}` } : {}), + } + return validateRunRecord(record) +} + +function validateTrace(value: unknown, line: number): PrimeIntellectTrace { + const trace = object(value, `PrimeIntellect trace line ${line}`) + nonEmptyString(trace.id, `PrimeIntellect trace line ${line}.id`) + const task = object(trace.task, `PrimeIntellect trace line ${line}.task`) + nonEmptyString(task.type, `PrimeIntellect trace line ${line}.task.type`) + const data = object(task.data, `PrimeIntellect trace line ${line}.task.data`) + if (!Number.isSafeInteger(data.idx) || (data.idx as number) < 0) { + throw new Error( + `PrimeIntellect trace line ${line}.task.data.idx must be a non-negative integer`, + ) + } + if (!Array.isArray(trace.nodes)) { + throw new Error(`PrimeIntellect trace line ${line}.nodes must be an array`) + } + for (const [index, rawNode] of trace.nodes.entries()) { + const node = object(rawNode, `PrimeIntellect trace line ${line}.nodes[${index}]`) + const parent = node.parent + if ( + parent !== undefined && + parent !== null && + (!Number.isSafeInteger(parent) || (parent as number) < 0 || (parent as number) >= index) + ) { + throw new Error( + `PrimeIntellect trace line ${line}.nodes[${index}].parent must reference an earlier node`, + ) + } + if (node.sampled !== undefined && typeof node.sampled !== 'boolean') { + throw new Error(`PrimeIntellect trace line ${line}.nodes[${index}].sampled must be boolean`) + } + if (node.usage !== undefined && node.usage !== null) { + validateUsage(node.usage, `PrimeIntellect trace line ${line}.nodes[${index}].usage`) + } + } + validateNumberMap(trace.rewards, `PrimeIntellect trace line ${line}.rewards`) + validateNumberMap(trace.metrics, `PrimeIntellect trace line ${line}.metrics`) + if (trace.extra_usage !== undefined && !Array.isArray(trace.extra_usage)) { + throw new Error(`PrimeIntellect trace line ${line}.extra_usage must be an array`) + } + for (const [index, usage] of (trace.extra_usage ?? []).entries()) { + validateUsage(usage, `PrimeIntellect trace line ${line}.extra_usage[${index}]`) + } + if (trace.errors !== undefined && !Array.isArray(trace.errors)) { + throw new Error(`PrimeIntellect trace line ${line}.errors must be an array`) + } + for (const [index, rawError] of (trace.errors ?? []).entries()) { + const error = object(rawError, `PrimeIntellect trace line ${line}.errors[${index}]`) + nonEmptyString(error.type, `PrimeIntellect trace line ${line}.errors[${index}].type`) + nonEmptyString(error.message, `PrimeIntellect trace line ${line}.errors[${index}].message`) + if ( + error.traceback !== undefined && + error.traceback !== null && + typeof error.traceback !== 'string' + ) { + throw new Error( + `PrimeIntellect trace line ${line}.errors[${index}].traceback must be a string or null`, + ) + } + } + if (trace.is_completed !== undefined && typeof trace.is_completed !== 'boolean') { + throw new Error(`PrimeIntellect trace line ${line}.is_completed must be boolean`) + } + if ( + trace.stop_condition !== undefined && + trace.stop_condition !== null && + typeof trace.stop_condition !== 'string' + ) { + throw new Error(`PrimeIntellect trace line ${line}.stop_condition must be a string or null`) + } + if (trace.timing !== undefined) validateTiming(trace.timing, line) + return value as PrimeIntellectTrace +} + +function validateTiming(value: unknown, line: number): void { + const timing = object(value, `PrimeIntellect trace line ${line}.timing`) + optionalFinite(timing.start, `PrimeIntellect trace line ${line}.timing.start`) + for (const phase of ['setup', 'generation', 'finalize', 'scoring']) { + if (timing[phase] === undefined) continue + const span = object(timing[phase], `PrimeIntellect trace line ${line}.timing.${phase}`) + optionalFinite(span.start, `PrimeIntellect trace line ${line}.timing.${phase}.start`) + optionalFinite(span.end, `PrimeIntellect trace line ${line}.timing.${phase}.end`) + } +} + +function aggregateUsage(usages: PrimeUsage[]): { + input: number + output: number + reasoning?: number + cached?: number + reportedCostUsd: number + costComplete: boolean +} { + let input = 0 + let output = 0 + let reasoning = 0 + let cached = 0 + let costUsd = 0 + let sawReasoning = false + let sawCached = false + let costsReported = 0 + for (const [index, usage] of usages.entries()) { + const prompt = nonNegative(usage.prompt_tokens, `usage[${index}].prompt_tokens`) + const completion = nonNegative(usage.completion_tokens, `usage[${index}].completion_tokens`) + const cachedInput = optionalNonNegative( + usage.cached_input_tokens, + `usage[${index}].cached_input_tokens`, + ) + const reasoningTokens = optionalNonNegative( + usage.reasoning_tokens, + `usage[${index}].reasoning_tokens`, + ) + const cost = optionalNonNegative(usage.cost, `usage[${index}].cost`) + input += prompt + (cachedInput ?? 0) + output += completion + if (cachedInput !== undefined) { + cached += cachedInput + sawCached = true + } + if (reasoningTokens !== undefined) { + reasoning += reasoningTokens + sawReasoning = true + } + if (cost !== undefined) { + costUsd += cost + costsReported += 1 + } + } + if (reasoning > output) { + throw new Error( + `PrimeIntellect reasoning token total ${reasoning} exceeds output total ${output}`, + ) + } + return { + input, + output, + ...(sawReasoning ? { reasoning } : {}), + ...(sawCached ? { cached } : {}), + reportedCostUsd: costUsd, + costComplete: usages.length > 0 && costsReported === usages.length, + } +} + +function validateUsage(value: unknown, path: string): void { + const usage = object(value, path) + nonNegative(usage.prompt_tokens, `${path}.prompt_tokens`) + nonNegative(usage.completion_tokens, `${path}.completion_tokens`) + optionalNonNegative(usage.cached_input_tokens, `${path}.cached_input_tokens`) + optionalNonNegative(usage.reasoning_tokens, `${path}.reasoning_tokens`) + optionalNonNegative(usage.cost, `${path}.cost`) +} + +function traceWallMs(trace: PrimeIntellectTrace): number { + const timing = trace.timing + if (!timing || typeof timing.start !== 'number' || !Number.isFinite(timing.start)) return 0 + const ends = [ + timing.setup?.end, + timing.generation?.end, + timing.finalize?.end, + timing.scoring?.end, + ].filter((value): value is number => typeof value === 'number' && Number.isFinite(value)) + if (ends.length === 0) return 0 + return Math.max(0, (Math.max(...ends) - timing.start) * 1000) +} + +function countBranches(nodes: PrimeTraceNode[]): number { + if (nodes.length === 0) return 0 + const parents = new Set( + nodes + .map((node) => node.parent) + .filter( + (parent): parent is number => Number.isSafeInteger(parent) && (parent as number) >= 0, + ), + ) + return nodes.reduce((count, _node, index) => count + (parents.has(index) ? 0 : 1), 0) +} + +function validateNumberMap(value: unknown, path: string): void { + const map = object(value, path) + for (const [key, entry] of Object.entries(map)) finite(entry, `${path}.${key}`) +} + +function sumFinite(values: unknown[], path: string): number { + return values.reduce((sum, value, index) => sum + finite(value, `${path}[${index}]`), 0) +} + +function finite(value: unknown, path: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`${path} must be a finite number`) + } + return value +} + +function nonNegative(value: unknown, path: string): number { + const parsed = finite(value, path) + if (parsed < 0) throw new Error(`${path} must be non-negative`) + return parsed +} + +function optionalNonNegative(value: unknown, path: string): number | undefined { + if (value === undefined || value === null) return undefined + return nonNegative(value, path) +} + +function optionalFinite(value: unknown, path: string): number | undefined { + if (value === undefined || value === null) return undefined + return finite(value, path) +} + +function object(value: unknown, path: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${path} must be an object`) + } + return value as Record +} + +function nonEmptyString(value: unknown, path: string): string { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`${path} must be a non-empty string`) + } + return value +} diff --git a/src/primeintellect/types.ts b/src/primeintellect/types.ts new file mode 100644 index 00000000..f3df50c6 --- /dev/null +++ b/src/primeintellect/types.ts @@ -0,0 +1,124 @@ +export type PrimeIntellectSplit = 'train' | 'eval' + +export type PrimeIntellectJson = + | null + | boolean + | number + | string + | PrimeIntellectJson[] + | { [key: string]: PrimeIntellectJson } + +export type PrimeIntellectContent = + | string + | Array<{ type: 'text'; text: string } | { type: 'image_url'; image_url: { url: string } }> + +export type PrimeIntellectMessage = + | { role: 'system' | 'user'; content: PrimeIntellectContent } + | { + role: 'assistant' + content?: string | null + reasoning_content?: string | null + tool_calls?: Array<{ id: string; name: string; arguments: string }> + provider_state?: Array> + } + | { + role: 'tool' + tool_call_id: string + content: PrimeIntellectContent + name?: string + } + +/** One immutable problem. References stay inside Prime's task process. */ +export interface PrimeIntellectTask { + id: string + split: PrimeIntellectSplit + prompt: string | PrimeIntellectMessage[] + systemPrompt?: string + answer?: string | string[] + metadata?: Record +} + +export type PrimeIntellectScoring = + | { + kind: 'exact' + normalization?: 'none' | 'trim' | 'trim-casefold' + } + | { + kind: 'reference-judge' + model: string + prompt?: string + view?: 'last_reply' | 'full_trace' + } + | { + kind: 'command' + command: readonly [string, ...string[]] + files?: Readonly> + forwardEnv?: readonly string[] + timeoutSeconds?: number + } + +export type PrimeIntellectSetupCommand = readonly [string, ...string[]] + +/** Files and commands that make the caller's real agent program runnable. */ +export interface PrimeIntellectRunner { + command: readonly [string, ...string[]] + files?: Readonly> + setup?: readonly PrimeIntellectSetupCommand[] + forwardEnv?: readonly string[] + /** Container image used by the generated eval config. */ + image: string +} + +export interface PrimeIntellectPackageOptions { + name: string + version: string + description?: string + tasks: readonly PrimeIntellectTask[] + scoring: PrimeIntellectScoring + runner: PrimeIntellectRunner + /** Prime-enforced model turn cap. Default 16. */ + maxTurns?: number + maxInputTokens?: number + maxOutputTokens?: number + maxTotalTokens?: number + rolloutTimeoutSeconds?: number + scoringTimeoutSeconds?: number +} + +export interface PrimeIntellectPackageManifest { + schema: 'tangle.primeintellect.package/v1' + name: string + moduleName: string + version: string + verifiers: '>=0.2.0,<0.3.0' + taskCount: number + splits: Record + taskIdsSha256: string + filesSha256: Record +} + +export interface PrimeIntellectPackageBundle { + manifest: PrimeIntellectPackageManifest + /** Relative package path to UTF-8 contents. */ + files: Readonly> +} + +/** The answer-free task exposed to the caller's runtime program. */ +export interface PrimeIntellectPublicTask { + id: string + split: PrimeIntellectSplit + prompt: string | PrimeIntellectMessage[] + systemPrompt?: string + metadata?: Record +} + +export interface PrimeIntellectEpisodeContext { + protocol: 'tangle.primeintellect.episode/v1' + task: PrimeIntellectPublicTask + model: { + name: string + baseUrl: string + apiKey: string + } + mcpServers: Readonly> +} diff --git a/src/primeintellect/validation.ts b/src/primeintellect/validation.ts new file mode 100644 index 00000000..53a60b26 --- /dev/null +++ b/src/primeintellect/validation.ts @@ -0,0 +1,156 @@ +import type { PrimeIntellectJson, PrimeIntellectMessage } from './types' + +/** Validate the Verifiers v1 prompt schema shared by package creation and runner input. */ +export function validatePrimeIntellectPrompt( + value: unknown, + path: string, +): string | PrimeIntellectMessage[] { + if (typeof value === 'string' && value.length > 0) return value + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${path} must be a non-empty string or message array`) + } + for (const [index, message] of value.entries()) { + validateMessage(message, `${path}[${index}]`) + } + return value as PrimeIntellectMessage[] +} + +export function validatePrimeIntellectJson(value: unknown, path: string): void { + if (value === null || ['string', 'boolean'].includes(typeof value)) return + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new Error(`${path} contains a non-finite number`) + return + } + if (Array.isArray(value)) { + value.forEach((entry, index) => { + validatePrimeIntellectJson(entry, `${path}[${index}]`) + }) + return + } + if (typeof value === 'object') { + for (const [key, entry] of Object.entries(value)) { + validatePrimeIntellectJson(entry, `${path}.${key}`) + } + return + } + throw new Error(`${path} is not JSON serializable`) +} + +export function validatePrimeIntellectJsonObject( + value: unknown, + path: string, +): Record { + const output = record(value, path) + validatePrimeIntellectJson(output, path) + return output as Record +} + +function validateMessage(value: unknown, path: string): void { + const message = record(value, path) + const role = message.role + if (!['system', 'user', 'assistant', 'tool'].includes(String(role))) { + throw new Error(`${path}.role is invalid`) + } + if (role === 'system' || role === 'user') { + assertOnlyKeys(message, ['role', 'content'], path) + validateContent(message.content, `${path}.content`) + } else if (role === 'assistant') { + assertOnlyKeys( + message, + ['role', 'content', 'reasoning_content', 'tool_calls', 'provider_state'], + path, + ) + optionalNullableString(message.content, `${path}.content`) + optionalNullableString(message.reasoning_content, `${path}.reasoning_content`) + if (message.tool_calls !== undefined) { + validateToolCalls(message.tool_calls, `${path}.tool_calls`) + } + if (message.provider_state !== undefined) { + validateProviderState(message.provider_state, `${path}.provider_state`) + } + } else { + assertOnlyKeys(message, ['role', 'tool_call_id', 'content', 'name'], path) + nonEmptyString(message.tool_call_id, `${path}.tool_call_id`) + if (message.name !== undefined && typeof message.name !== 'string') { + throw new Error(`${path}.name must be a string`) + } + validateContent(message.content, `${path}.content`) + } + validatePrimeIntellectJson(message, path) +} + +function validateToolCalls(value: unknown, path: string): void { + if (!Array.isArray(value)) throw new Error(`${path} must be an array`) + for (const [index, rawCall] of value.entries()) { + const call = record(rawCall, `${path}[${index}]`) + assertOnlyKeys(call, ['id', 'name', 'arguments'], `${path}[${index}]`) + for (const field of ['id', 'name', 'arguments']) { + nonEmptyString(call[field], `${path}[${index}].${field}`) + } + } +} + +function validateProviderState(value: unknown, path: string): void { + if (!Array.isArray(value)) throw new Error(`${path} must be an array`) + for (const [index, rawState] of value.entries()) { + const state = record(rawState, `${path}[${index}]`) + validatePrimeIntellectJson(state, `${path}[${index}]`) + } +} + +function validateContent(value: unknown, path: string): void { + if (typeof value === 'string') return + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${path} must be a string or non-empty content array`) + } + for (const [index, rawPart] of value.entries()) { + const part = record(rawPart, `${path}[${index}]`) + if (part.type === 'text' && typeof part.text === 'string') { + assertOnlyKeys(part, ['type', 'text'], `${path}[${index}]`) + continue + } + const image = part.image_url + if ( + part.type === 'image_url' && + image !== null && + typeof image === 'object' && + !Array.isArray(image) && + typeof (image as Record).url === 'string' + ) { + assertOnlyKeys(part, ['type', 'image_url'], `${path}[${index}]`) + assertOnlyKeys(image as Record, ['url'], `${path}[${index}].image_url`) + continue + } + throw new Error(`${path}[${index}] is not a supported text or image_url content part`) + } +} + +function optionalNullableString(value: unknown, path: string): void { + if (value !== undefined && value !== null && typeof value !== 'string') { + throw new Error(`${path} must be a string or null`) + } +} + +function nonEmptyString(value: unknown, path: string): string { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`${path} must be a non-empty string`) + } + return value +} + +function record(value: unknown, path: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${path} must be an object`) + } + return value as Record +} + +function assertOnlyKeys( + value: Record, + allowed: readonly string[], + path: string, +): void { + for (const key of Object.keys(value)) { + if (!allowed.includes(key)) throw new Error(`${path}.${key} is not supported`) + } +} diff --git a/src/runtime/define-leaderboard.test.ts b/src/runtime/define-leaderboard.test.ts index 2319ef2f..09100257 100644 --- a/src/runtime/define-leaderboard.test.ts +++ b/src/runtime/define-leaderboard.test.ts @@ -173,7 +173,7 @@ describe('defineLeaderboard', () => { 'moonshot/kimi-k2@2026-01-01', ] await expect(board().run([...snappedAxis, '--cases', 'case-alpha'])).rejects.toThrow( - /observeModel/, + /paid-call receipt/, ) const result = await board({ diff --git a/src/runtime/define-leaderboard.ts b/src/runtime/define-leaderboard.ts index 6095dbf4..b4359096 100644 --- a/src/runtime/define-leaderboard.ts +++ b/src/runtime/define-leaderboard.ts @@ -39,6 +39,7 @@ import { expandProfileAxes, type HarnessType, harnessAxisOf, + type MaximumCharge, } from '@tangle-network/agent-eval' import { type JudgeConfig, @@ -208,7 +209,7 @@ export interface LeaderboardSpec { * an out-of-family model expands to the `default` sentinel): the RunRecord * must pin a real snapshot-bearing model id, which only the dispatch — * reading the backend's usage/terminal events — can know. When this returns - * a value the default dispatch reports it via `ctx.cost.observeModel`; + * a value the default dispatch records it on the paid-call receipt; * in-family cells (concrete declared model) never need it. */ resolveModel?: (events: readonly SandboxEvent[]) => string | undefined @@ -227,6 +228,11 @@ export interface LeaderboardSpec { shots?: number /** Replicates per cell (`--reps`). Default 1. */ reps?: number + /** Provider- or executor-enforced maximum for one cell dispatch. Required + * before execution when `matrix.costCeiling` is configured. */ + maximumCharge?: + | MaximumCharge + | ((profile: AgentProfile, scenario: LeaderboardScenario) => MaximumCharge | undefined) /** Passthrough overrides spread onto the final `runProfileMatrix` call * (e.g. `maxConcurrency`, `costCeiling`, `integrity`, `storage`) — spread * LAST, so anything the facade wired can be overridden. */ @@ -470,8 +476,9 @@ export function defineLeaderboard( // per-cell resource cost) so the loop's finished iterations can be joined // with the campaign ctx: onCellEvents gets EVERY shot's outcome (a thrown // shot never reaches parse, so parse-time tapping would hide it), and a - // spec-resolved served model reaches ctx.cost.observeModel (the only - // channel that pins HARNESS_NATIVE_MODEL-snapped cells to a real model). + // the cost receipt records the spec-resolved served model (the only way to + // pin HARNESS_NATIVE_MODEL-snapped cells to a real model). + const maximumCharge = spec.maximumCharge const dispatch: ProfileDispatchFn, TArtifact> = spec.dispatch ?? ((profile, scenario, dispatchCtx) => { const cellDispatch = loopDispatch< @@ -482,6 +489,24 @@ export function defineLeaderboard( TArtifact >({ sandboxClient, + maximumCharge: + typeof maximumCharge === 'function' + ? (cellScenario, cellProfile) => maximumCharge(cellProfile, cellScenario) + : maximumCharge, + resolveCostModel: (result, _cellScenario, cellProfile) => { + if (!spec.resolveModel) return cellProfile.model?.default + const served = new Set( + result.iterations + .map((iteration) => spec.resolveModel?.(iteration.events)) + .filter((model): model is string => model !== undefined), + ) + if (served.size > 1) { + throw new Error( + `defineLeaderboard(${spec.name}): one cell reported multiple served models: ${[...served].join(', ')}`, + ) + } + return [...served][0] ?? cellProfile.model?.default + }, toLoopOptions: (cellScenario, cellProfile) => { // The cell's harness + model come off the profile's axis stamp set // by expandProfileAxes; the sandbox create override carries them to @@ -536,10 +561,6 @@ export function defineLeaderboard( ...(iter.error ? { error: iter.error.message } : {}), ...(iter.verdict ? { verdict: { score: iter.verdict.score } } : {}), }) - if (spec.resolveModel) { - const served = spec.resolveModel(iter.events) - if (served !== undefined) dispatchCtx.cost.observeModel?.(served) - } } // Same as loopDispatch's default: no winner → undefined artifact // (judges skip the cell; usage is still reported). diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 2921465a..b8b20e34 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -397,6 +397,35 @@ export { type StreamAgentTurnOptions, streamAgentTurn, } from './stream-agent-turn' +// The structural lever as a strategy-family member: k samples → select by task-visible checks +// (official above authored, crash lowest) → guarded repair steered by the checks' failure output. +// Measured +8..+21pp hidden-test lift (docs/design/structural-rollout-integration.md). +export { + type CheckExecChannel, + type CheckOutcome, + type CheckRunContext, + type CheckRunner, + type CheckSource, + type CheckSourceCtx, + canDisplace, + compareCheckOutcomes, + composeCheckSources, + defaultExtractCandidate, + defaultStructuralRolloutPolicy, + filterAuthoredAsserts, + modelAuthoredChecks, + officialChecksFromMeta, + type RepairStop, + resolveEntrySymbol, + type StructuralRolloutConfig, + type StructuralRolloutPolicy, + type StructuralRolloutResult, + sandboxCheckRunner, + selectBestIndex, + structuralRollout, + type VisibleCheck, + visibleCheckScore, +} from './structural-rollout' // The supervisor's intelligence: it AUTHORS each worker's profile (instructions + model) from a // SKILL (its own system prompt) — the optimizable self-improvement surface, not the plumbing. export { @@ -540,6 +569,7 @@ export { type WorktreeCliExecutorOptions, type WorktreeCommandResult, type WorktreePatchArtifact, + type WorktreeProfileMaterializationReceipt, } from './supervise/worktree-cli-executor' // The generic coding combinator: a fanout of authored harness profiles, each on its own // worktree-CLI leaf, each gated by the injected deliverable, winner via the shared valid-only diff --git a/src/runtime/loop-dispatch.ts b/src/runtime/loop-dispatch.ts index 63409ea3..70b9a720 100644 --- a/src/runtime/loop-dispatch.ts +++ b/src/runtime/loop-dispatch.ts @@ -29,7 +29,7 @@ // agent-eval's AgentProfile (the eval-harness unit of variation, `model: string`) // — NOT sandbox's AgentProfile. ProfileDispatchFn is keyed on the former. -import type { AgentProfile } from '@tangle-network/agent-eval' +import type { AgentProfile, CostReceiptInput, MaximumCharge } from '@tangle-network/agent-eval' import type { CampaignTraceWriter, DispatchContext, @@ -37,7 +37,6 @@ import type { ProfileDispatchFn, Scenario, } from '@tangle-network/agent-eval/campaign' -import { reportLoopUsage } from './report-usage' import { type RunLoopOptions, runLoop } from './run-loop' import type { LoopResult, LoopTraceEmitter, SandboxClient } from './types' @@ -72,6 +71,17 @@ export interface LoopDispatchOptions< forwardTrace?: boolean /** Cost-meter source label for the loop's spend. Default `'loop'`. */ costSource?: string + /** Provider- or executor-enforced maximum for this whole cell dispatch. + * Required by agent-eval before execution when the campaign is cost-capped. */ + maximumCharge?: + | MaximumCharge + | ((scenario: TScenario, profile: AgentProfile) => MaximumCharge | undefined) + /** Resolve the model actually served from the completed loop. */ + resolveCostModel?: ( + result: LoopResult, + scenario: TScenario, + profile: AgentProfile, + ) => string | undefined } /** Bridge a campaign `DispatchContext.trace` to a `LoopTraceEmitter` so every @@ -93,7 +103,16 @@ async function runLoopForCell { const loopOptions = opts.toLoopOptions(scenario, profile) - return runLoopWithCampaignContext(opts, loopOptions, ctx) + return runLoopWithCampaignContext(opts, loopOptions, ctx, { + model: profile.model?.default ?? modelFromLoopOptions(loopOptions), + maximumCharge: + typeof opts.maximumCharge === 'function' + ? opts.maximumCharge(scenario, profile) + : opts.maximumCharge, + resolveModel: opts.resolveCostModel + ? (result) => opts.resolveCostModel?.(result, scenario, profile) + : undefined, + }) } async function runLoopWithCampaignContext( @@ -105,21 +124,64 @@ async function runLoopWithCampaignContext( }, loopOptions: LoopOptionsForDispatch, ctx: DispatchContext, + cost: { + model: string + maximumCharge?: MaximumCharge + resolveModel?: (result: LoopResult) => string | undefined + }, ): Promise { - const result = await runLoop({ - ...loopOptions, - ctx: { - sandboxClient: opts.sandboxClient, - signal: ctx.signal, - traceEmitter: opts.forwardTrace === false ? undefined : campaignTraceToLoopEmitter(ctx.trace), - }, + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: opts.costSource ?? 'loop', + model: cost.model, + signal: ctx.signal, + ...(cost.maximumCharge ? { maximumCharge: cost.maximumCharge } : {}), + execute: (executionSignal) => + runLoop({ + ...loopOptions, + ctx: { + sandboxClient: opts.sandboxClient, + signal: executionSignal, + traceEmitter: + opts.forwardTrace === false ? undefined : campaignTraceToLoopEmitter(ctx.trace), + }, + }), + receipt: (result) => loopCostReceipt(result, cost.resolveModel?.(result) ?? cost.model), }) - reportLoopUsage(ctx.cost, result, opts.costSource ?? 'loop') + if (!paid.succeeded) { + throw paid.error + } + const result = paid.value const toArtifact = opts.toArtifact ?? ((r: LoopResult) => r.winner?.output as TArtifact) return toArtifact(result) } +function loopCostReceipt( + result: LoopResult, + model: string, +): CostReceiptInput { + return { + model, + inputTokens: result.tokenUsage.input, + outputTokens: result.tokenUsage.output, + ...(result.costUsd > 0 ? { actualCostUsd: result.costUsd } : {}), + } +} + +function modelFromLoopOptions( + options: LoopOptionsForDispatch, +): string { + const profiles = options.agentRun + ? [options.agentRun.profile] + : (options.agentRuns?.map((run) => run.profile) ?? []) + const models = new Set( + profiles.map((profile) => profile.model?.default).filter((model): model is string => !!model), + ) + if (models.size === 1) return [...models][0] as string + return models.size > 1 ? 'mixed' : 'unknown' +} + /** Options for adapting plain agent-eval campaign scenarios into runtime `runLoop` cells. */ export interface LoopCampaignDispatchOptions< Task, @@ -138,6 +200,13 @@ export interface LoopCampaignDispatchOptions< forwardTrace?: boolean /** Cost-meter source label for the loop's spend. Default `'loop'`. */ costSource?: string + /** Provider- or executor-enforced maximum for this whole cell dispatch. */ + maximumCharge?: MaximumCharge | ((scenario: TScenario) => MaximumCharge | undefined) + /** Resolve the model actually served from the completed loop. */ + resolveCostModel?: ( + result: LoopResult, + scenario: TScenario, + ) => string | undefined } /** @@ -148,7 +217,19 @@ export interface LoopCampaignDispatchOptions< export function loopCampaignDispatch( opts: LoopCampaignDispatchOptions, ): DispatchFn { - return (scenario, ctx) => runLoopWithCampaignContext(opts, opts.toLoopOptions(scenario), ctx) + return (scenario, ctx) => { + const loopOptions = opts.toLoopOptions(scenario) + return runLoopWithCampaignContext(opts, loopOptions, ctx, { + model: modelFromLoopOptions(loopOptions), + maximumCharge: + typeof opts.maximumCharge === 'function' + ? opts.maximumCharge(scenario) + : opts.maximumCharge, + resolveModel: opts.resolveCostModel + ? (result) => opts.resolveCostModel?.(result, scenario) + : undefined, + }) + } } /** diff --git a/src/runtime/personify/trajectory.ts b/src/runtime/personify/trajectory.ts index 8525f60e..9598d5c6 100644 --- a/src/runtime/personify/trajectory.ts +++ b/src/runtime/personify/trajectory.ts @@ -275,6 +275,7 @@ function addNodeSpend(a: Spend, b: Spend): Spend { iterations: a.iterations + b.iterations, tokens: { input: a.tokens.input + b.tokens.input, output: a.tokens.output + b.tokens.output }, usd: a.usd + b.usd, + ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), ms: a.ms + b.ms, } } @@ -284,6 +285,7 @@ function cloneSpend(spend: Spend): Spend { iterations: spend.iterations, tokens: { input: spend.tokens.input, output: spend.tokens.output }, usd: spend.usd, + ...(spend.usdKnown === false ? { usdKnown: false } : {}), ms: spend.ms, } } @@ -293,6 +295,7 @@ function addSpend(acc: Spend, delta: Spend): void { acc.iterations += delta.iterations addTokenUsage(acc.tokens, delta.tokens) acc.usd += delta.usd + if (delta.usdKnown === false) acc.usdKnown = false acc.ms += delta.ms } diff --git a/src/runtime/sandbox-events.ts b/src/runtime/sandbox-events.ts index d2b6c869..78397825 100644 --- a/src/runtime/sandbox-events.ts +++ b/src/runtime/sandbox-events.ts @@ -79,10 +79,11 @@ export function extractLlmCallEvent( * `openSandboxRun` cell. Folds `extractLlmCallEvent` over the stream (which reads usage off EVERY backend * event shape), so a `runProfileMatrix` dispatch can report it to `ctx.cost`: * - * const turn = await run.start(prompt) - * const u = sumSandboxUsage(turn.events) - * if (u.input || u.output) ctx.cost.observeTokens({ input: u.input, output: u.output }) - * if (u.costUsd) ctx.cost.observe(u.costUsd, 'sandbox-cell') + * receipt: (turn) => { + * const u = sumSandboxUsage(turn.events) + * return { model, inputTokens: u.input, outputTokens: u.output, + * ...(u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}) } + * } * * Without this a cell reads `{tokens:0, cost:0}` and the backend-integrity guard correctly aborts the * matrix as a stub. `agentRunName` is the fallback model label for cost-only events (default `'agent'`). diff --git a/src/runtime/strategy.ts b/src/runtime/strategy.ts index fb27cec3..b4e146be 100644 --- a/src/runtime/strategy.ts +++ b/src/runtime/strategy.ts @@ -306,17 +306,31 @@ async function consultAnalyst( const trajectory = compactTrajectory(messages) const analystModel = opts.analystModel ?? opts.model const chat = analystChat(opts, analystModel) + // With a trajectory, the analyst framing (instruction as system, behavior as the user + // turn) is the channel's shape. With NO trajectory (pre-task consults, e.g. authored + // check generation), that framing breaks: the user turn is then just the task, which + // reads as a solve request — measured on Llama-3-8B, the model ignores the system + // instruction and answers the task (authored-assert yield 6/18 reps vs 18/18 with the + // instruction and task fused into one user message, the proven rig shape). + const consultMessages = trajectory + ? [ + { role: 'system' as const, content: instruction }, + { + role: 'user' as const, + content: `TASK: ${task.userPrompt.slice(0, 1500)}\n\nTRAJECTORY:\n${trajectory}`, + }, + ] + : [ + { + role: 'user' as const, + content: `${instruction}\n\nTASK:\n${task.userPrompt.slice(0, 1500)}`, + }, + ] const res = await chat.chat({ model: analystModel, temperature: 0.2, maxTokens: 1024, - messages: [ - { role: 'system', content: instruction }, - { - role: 'user', - content: `TASK: ${task.userPrompt.slice(0, 1500)}\n\nTRAJECTORY:\n${trajectory}`, - }, - ], + messages: consultMessages, }) const usage = ( res as { diff --git a/src/runtime/structural-rollout.test.ts b/src/runtime/structural-rollout.test.ts new file mode 100644 index 00000000..dcfe5b96 --- /dev/null +++ b/src/runtime/structural-rollout.test.ts @@ -0,0 +1,399 @@ +import { describe, expect, it, vi } from 'vitest' +import { type AgenticSurface, type AgenticTask, runAgentic } from './strategy' +import { + type CheckOutcome, + type CheckRunner, + canDisplace, + compareCheckOutcomes, + defaultExtractCandidate, + filterAuthoredAsserts, + modelAuthoredChecks, + officialChecksFromMeta, + type StructuralRolloutResult, + sandboxCheckRunner, + selectBestIndex, + structuralRollout, + type VisibleCheck, + visibleCheckScore, +} from './structural-rollout' + +const outcome = ( + passedOfficial: number, + totalOfficial: number, + passedAuthored: number, + totalAuthored: number, + failureOutput = '', + crashed?: boolean, +): CheckOutcome => ({ + passedOfficial, + totalOfficial, + passedAuthored, + totalAuthored, + failureOutput, + ...(crashed !== undefined ? { crashed } : {}), +}) + +describe('selection order — official above authored, crash lowest', () => { + it('one passing official check outranks six passing authored guesses', () => { + const officialWins = outcome(1, 1, 0, 6) + const authoredSweep = outcome(0, 1, 6, 6) + expect(compareCheckOutcomes(officialWins, authoredSweep)).toBeGreaterThan(0) + expect(selectBestIndex([authoredSweep, officialWins])).toBe(1) + }) + + it('authored guesses only break official ties', () => { + const a = outcome(1, 2, 2, 6) + const b = outcome(1, 2, 3, 6) + expect(compareCheckOutcomes(b, a)).toBeGreaterThan(0) + expect(selectBestIndex([a, b])).toBe(1) + }) + + it('a crashed candidate ranks below one that ran and failed everything', () => { + const crashed = outcome(0, 0, 0, 0, 'SyntaxError', true) + const ranAndFailed = outcome(0, 1, 0, 6) + expect(compareCheckOutcomes(crashed, ranAndFailed)).toBeLessThan(0) + expect(selectBestIndex([crashed, ranAndFailed])).toBe(1) + expect(visibleCheckScore(crashed)).toBe(-1) + expect(visibleCheckScore(ranAndFailed)).toBe(0) + }) + + it('argmax breaks ties by FIRST index (the blind pick under zero coverage)', () => { + const noSignal = outcome(0, 0, 0, 0) + expect(selectBestIndex([noSignal, noSignal, noSignal])).toBe(0) + const tie = outcome(1, 2, 1, 2) + expect(selectBestIndex([outcome(0, 2, 2, 2), tie, tie])).toBe(1) + }) +}) + +describe('repair keep-best guard', () => { + it('never displaces with fewer official passes, even when authored/overall look better', () => { + const incumbent = outcome(2, 3, 0, 6) + const challenger = outcome(1, 3, 6, 6) + expect(canDisplace(challenger, incumbent)).toBe(false) + }) + + it('a crashed challenger never displaces', () => { + expect(canDisplace(outcome(0, 0, 0, 0, '', true), outcome(0, 3, 0, 6))).toBe(false) + }) + + it('requires a STRICT improvement in the selection order', () => { + const tie = outcome(2, 3, 1, 6) + expect(canDisplace(tie, tie)).toBe(false) + expect(canDisplace(outcome(2, 3, 2, 6), tie)).toBe(true) + expect(canDisplace(outcome(3, 3, 0, 6), tie)).toBe(true) + }) +}) + +describe('modelAuthoredChecks — the assert filter (visible info only, frozen per task)', () => { + const task: AgenticTask = { id: 't', systemPrompt: 's', userPrompt: 'Write add.' } + + it('keeps only single-line paren-balanced asserts mentioning the entry symbol, capped', async () => { + const reply = [ + 'Here are the tests:', + '```python', + 'assert add(1, 2) == 3', + 'print(add(1, 2))', // not an assert + 'assert add((1, 2) == 3', // unbalanced parens + 'assert sub(1) == 0', // wrong symbol + 'assert add(0, 0) == 0', + 'assert add(-1, 1) == 0', + 'assert add(2, 2) == 4', + 'assert add(3, 3) == 6', + 'assert add(4, 4) == 8', + 'assert add(5, 5) == 10', // over the cap of 6 + '```', + ].join('\n') + const consult = vi.fn(async () => reply) + const checks = await modelAuthoredChecks().generate(task, { + count: 6, + entrySymbol: 'add', + consult, + }) + expect(consult).toHaveBeenCalledOnce() + expect(checks.map((c) => c.code)).toEqual([ + 'assert add(1, 2) == 3', + 'assert add(0, 0) == 0', + 'assert add(-1, 1) == 0', + 'assert add(2, 2) == 4', + 'assert add(3, 3) == 6', + 'assert add(4, 4) == 8', + ]) + expect(checks.every((c) => c.kind === 'authored')).toBe(true) + }) + + it('skips authoring (no LLM call) when the budget is 0 or no entry symbol resolves', async () => { + const consult = vi.fn(async () => 'assert add(1) == 1') + expect(await modelAuthoredChecks().generate(task, { count: 0, entrySymbol: 'add', consult })) // + .toEqual([]) + expect(await modelAuthoredChecks().generate(task, { count: 6, consult })).toEqual([]) + expect(consult).not.toHaveBeenCalled() + }) + + it('filterAuthoredAsserts falls back to the raw reply when there is no fence', () => { + expect(filterAuthoredAsserts('assert add(1, 1) == 2\nnope', 'add', 6)).toEqual([ + 'assert add(1, 1) == 2', + ]) + }) +}) + +describe('officialChecksFromMeta', () => { + it('lifts string checks from task.meta as official; anything else means none', async () => { + const source = officialChecksFromMeta() + const withMeta: AgenticTask = { + id: 't', + systemPrompt: 's', + userPrompt: 'u', + meta: { visibleChecks: ['assert f() == 1', 42, ' '] }, + } + const bare: AgenticTask = { id: 't', systemPrompt: 's', userPrompt: 'u' } + const ctx = { count: 0, consult: async () => null } + expect(await source.generate(withMeta, ctx)).toEqual([ + { code: 'assert f() == 1', kind: 'official' }, + ]) + expect(await source.generate(bare, ctx)).toEqual([]) + }) +}) + +describe('sandboxCheckRunner', () => { + const task: AgenticTask = { id: 't', systemPrompt: 's', userPrompt: 'u' } + const checks: VisibleCheck[] = [ + { code: 'assert f() == 1', kind: 'official' }, + { code: 'assert f() != 2', kind: 'authored' }, + ] + + it('throws a clear error with no execution channel — never a silent 0', async () => { + await expect(sandboxCheckRunner().run('def f(): return 1', checks, { task })).rejects.toThrow( + /no execution channel/, + ) + }) + + it('short-circuits an EMPTY check set to a no-signal outcome (nothing to execute)', async () => { + expect(await sandboxCheckRunner().run('def f(): return 1', [], { task })).toEqual({ + passedOfficial: 0, + totalOfficial: 0, + passedAuthored: 0, + totalAuthored: 0, + failureOutput: '', + }) + }) + + it('parses the nonce sentinel and strips it from the repair feedback', async () => { + const box = { + async exec(command: string) { + // Decode the piped program to recover the per-call nonce, as the jail would see it. + const b64 = /printf '%s' '([A-Za-z0-9+/=]+)'/.exec(command)?.[1] ?? '' + const program = Buffer.from(b64, 'base64').toString('utf8') + const nonce = /SRCK-([0-9a-f]+)/.exec(program)?.[1] + expect(nonce).toBeTruthy() + expect(program).toContain('def f(): return 1') + return { + exitCode: 1, + stdout: `SRCK-${nonce} official=1/1 authored=0/1\nCHECK FAILED: assert f() != 2 -> AssertionError: `, + stderr: '', + } + }, + } + const result = await sandboxCheckRunner({ box }).run('def f(): return 1', checks, { task }) + expect(result).toMatchObject({ + passedOfficial: 1, + totalOfficial: 1, + passedAuthored: 0, + totalAuthored: 1, + }) + expect(result.failureOutput).toContain('CHECK FAILED: assert f() != 2') + expect(result.failureOutput).not.toContain('SRCK-') + }) + + it('no sentinel in the output ⇒ crashed (ranks below ran-and-failed), with the crash detail', async () => { + const box = { + async exec() { + return { exitCode: 1, stdout: '', stderr: 'SyntaxError: invalid syntax' } + }, + } + const result = await sandboxCheckRunner({ box }).run('def f(', checks, { task }) + expect(result.crashed).toBe(true) + expect(result.failureOutput).toContain('SyntaxError') + expect(result.totalOfficial).toBe(0) + }) +}) + +describe('defaultExtractCandidate', () => { + it('prefers the LAST submit_answer tool call over fenced content', () => { + const messages = [ + { role: 'assistant', content: '```python\ndef old(): pass\n```' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'c1', + function: { name: 'submit_answer', arguments: '{"answer":"def f(): return 1"}' }, + }, + ], + }, + { role: 'tool', content: 'submission 1 recorded' }, + ] + expect(defaultExtractCandidate(messages)).toBe('def f(): return 1') + }) + + it('prefers the fenced block containing a def over an echoed bare fence', () => { + const messages = [ + { + role: 'assistant', + content: + 'The failure was:\n```\nCHECK FAILED: ...\n```\nFixed:\n```python\ndef f():\n return 2\n```', + }, + ] + expect(defaultExtractCandidate(messages)).toBe('def f():\n return 2') + }) +}) + +describe('structuralRollout — the strategy, end to end (offline transport, fake runner)', () => { + it('validates the policy', () => { + expect(() => structuralRollout({ policy: { k: 0 } })).toThrow(/policy\.k/) + expect(() => structuralRollout({ policy: { repairRounds: -1 } })).toThrow( + /policy\.repairRounds/, + ) + expect(() => structuralRollout({ policy: { testgen: 1.5 } })).toThrow(/policy\.testgen/) + }) + + it('selects by frozen visible checks, repairs on failure output, and holds the official guard', async () => { + // The domain: a verifier-environment-shaped surface (submit_answer only); each + // shot's artifact scores 0 so nothing here can leak a hidden signal into selection — + // only the fake CheckRunner speaks. + let handles = 0 + const surface: AgenticSurface = { + name: 'inert', + async open() { + handles += 1 + return { id: `h${handles}`, surface: 'inert' } + }, + async tools() { + return [ + { + type: 'function' as const, + function: { + name: 'submit_answer', + parameters: { type: 'object', properties: { answer: { type: 'string' } } }, + }, + }, + ] + }, + async call(_handle, name) { + return name === 'submit_answer' ? 'submission recorded' : `ERROR: unknown tool ${name}` + }, + async score() { + return { passes: 0, total: 1, errored: 0 } + }, + async close() {}, + } + + // The offline model: each shot is two turns — a submit_answer tool call carrying the + // candidate (the recorded channel; a content-only final reply never enters the shot's + // conversation), then DONE. Samples produce A then B; repairs produce C then D. + const letters = ['A', 'B', 'C', 'D'] + const bodies: Array> = [] + let workerCall = 0 + const complete = async (body: Record) => { + bodies.push(body) + const shotIdx = Math.min(Math.floor(workerCall / 2), letters.length - 1) + const firstTurn = workerCall % 2 === 0 + workerCall += 1 + const message = firstTurn + ? { + content: '', + tool_calls: [ + { + id: `c${workerCall}`, + function: { + name: 'submit_answer', + arguments: JSON.stringify({ + answer: `def f():\n return "${letters[shotIdx]}"`, + }), + }, + }, + ], + } + : { content: 'DONE' } + return { + choices: [{ message }], + usage: { prompt_tokens: 10, completion_tokens: 5 }, + } + } + + const frozenChecks: VisibleCheck[] = [ + { code: 'assert f() == 1', kind: 'official' }, + { code: 'assert f() == 2', kind: 'official' }, + { code: 'assert f() == 3', kind: 'official' }, + { code: 'assert f() != 0', kind: 'authored' }, + { code: 'assert f() != 9', kind: 'authored' }, + ] + const generate = vi.fn(async () => frozenChecks) + + // Scripted visible-check outcomes, keyed on the candidate the shot produced: + // A (sample) official 2/3 — the argmax pick, with a failure report; + // B (sample) official 1/3; + // C (repair) official 1/3 but authored 2/2 — MUST be held out by the guard; + // D (repair) official 3/3 — displaces, repair stops on full pass. + const failureA = 'CHECK FAILED: assert f() == 3 -> AssertionError' + const runnerCalls: Array<{ candidate: string; checks: VisibleCheck[] }> = [] + const checkRunner: CheckRunner = { + async run(candidate, checks) { + runnerCalls.push({ candidate, checks }) + if (candidate.includes('"A"')) return outcome(2, 3, 0, 2, failureA) + if (candidate.includes('"B"')) return outcome(1, 3, 1, 2, 'CHECK FAILED: ...') + if (candidate.includes('"C"')) return outcome(1, 3, 2, 2, 'CHECK FAILED: ...') + return outcome(3, 3, 2, 2) + }, + } + + const result = await runAgentic({ + surface, + task: { id: 't1', systemPrompt: 'Solve it.', userPrompt: 'Write f.\n\ndef f():\n ...' }, + routerBaseUrl: 'http://offline.test/v1', + routerKey: 'k', + model: 'stub-model', + complete, + innerTurns: 2, + maxTokens: 64, + strategy: structuralRollout({ + policy: { k: 2, repairRounds: 2, testgen: 0 }, + checkSource: { generate }, + checkRunner, + }), + budget: 6, + }) + const rollout = result as unknown as StructuralRolloutResult & { mode: string } + + expect(rollout.mode).toBe('structuralRollout') + expect(rollout.shots).toBe(4) // 2 samples + 2 repairs + expect(rollout.repairStop).toBe('repaired-pass') + + // Checks were generated ONCE and the same frozen set fed every run. + expect(generate).toHaveBeenCalledOnce() + expect(runnerCalls).toHaveLength(4) + for (const c of runnerCalls) expect(c.checks).toBe(frozenChecks) + + // Selection receipts: A won the sample argmax, C was guard-held, D displaced. + expect(rollout.selection).toHaveLength(4) + expect(rollout.selection.map((r) => r.candidateIndex)).toEqual([0, 1, 2, 3]) + expect(rollout.selection[2]?.reason).toMatch(/fewer official checks/) + expect(rollout.selection.filter((r) => r.selected)).toHaveLength(1) + expect(rollout.selection[3]?.selected).toBe(true) + expect(rollout.selection.every((r) => r.selector === 'driver')).toBe(true) + + // Both repair shots were steered by the INCUMBENT's failure output (A's — C never + // displaced), riding the winner's carried conversation: their opening turns end with + // the steer as the last user message. + const steerTurns = bodies.filter((body) => { + const messages = body.messages as Array<{ role: string; content?: string }> + const last = messages[messages.length - 1] + return last?.role === 'user' && (last.content ?? '').includes('task-visible checks') + }) + expect(steerTurns).toHaveLength(2) + for (const body of steerTurns) { + const messages = body.messages as Array<{ role: string; content?: string }> + expect(messages[messages.length - 1]?.content).toContain(failureA) + } + }) +}) diff --git a/src/runtime/structural-rollout.ts b/src/runtime/structural-rollout.ts new file mode 100644 index 00000000..acd14685 --- /dev/null +++ b/src/runtime/structural-rollout.ts @@ -0,0 +1,683 @@ +/** + * structuralRollout — the measured structural lever as a fourth member of the + * sample/refine/sampleThenRefine strategy family: k independent samples, selection by + * TASK-VISIBLE checks only, then a guarded self-repair loop steered by the checks' + * failure output. Design: docs/design/structural-rollout-integration.md; measured basis + * (bench/src/hev-structural.mts, bench/src/mbpp-structural.mts): +8.5..+21.3pp hidden-test + * lift across Llama-3-8B/Qwen2.5-7B × HumanEval/MBPP, null only at saturation. + * + * Honesty invariants carried over from the proven rigs: + * - Visible checks are generated from task-visible information only, BEFORE any + * candidate exists, and FROZEN for every sample and repair round of the task. + * - OFFICIAL checks (shown in the task itself) rank lexicographically above + * model-AUTHORED guesses. This ordering is measured, not stylistic: authored guesses + * run 17–70% wrong depending on model × spec richness, and unweighted they flipped + * selection NEGATIVE on MBPP (6 noisy guesses outvoting the one reliable check). + * - A candidate that crashed before the checks could run ranks below one that ran and + * failed everything. + * - Repair sees ONLY the checks' failure output, and never displaces a candidate that + * passes more official checks with one that passes fewer (wrong visible examples + * poison repair at saturation — the glm /47,/116 regressions). + * + * Placement rule: this is an INFERENCE-TIME capability (it wraps the model call via the + * strategy seam). It does not belong in improve()/selfImprove (training-time); improve() + * may later tune `StructuralRolloutPolicy` as an optimizable surface. + */ + +import { randomBytes } from 'node:crypto' +import { + type AgenticTask, + defineStrategy, + type Strategy, + type StrategyCtx, + type StrategyResult, +} from './strategy' +import type { SelectionReceipt } from './types' + +type Msg = Record + +// ── Policy ──────────────────────────────────────────────────────────────────────── + +/** The rollout's compute recipe — promoted from the proven rigs' env vars (K/REPAIRS/ + * TESTGEN/DIVERSE/TEMPERATURE). Defaults are the measured sweet spot: repair value + * concentrates at low k (~+12pp at k=1, +1–3pp at k=5), so `k=5, repairRounds=2` is the + * full recipe and `k=1, repairRounds=2` the low-compute preset. */ +export interface StructuralRolloutPolicy { + /** Independent samples per task (selection breadth). */ + k: number + /** Repair shots after selection, each steered by the checks' failure output. */ + repairRounds: number + /** Model-authored visible checks requested per task; 0 disables authoring. */ + testgen: number + /** Per-slot strategy-lens prefixes on the k samples (attacks the all-k-fail bucket). + * Measured as a paired null (+0.6pp) — kept as an optional knob, off by default. */ + diverse?: boolean + /** Sampling temperature for every shot of this strategy; omitted ⇒ the worker default. */ + temperature?: number +} + +/** The measured default recipe: 5 samples, 2 guarded repair rounds, 6 authored checks. */ +export const defaultStructuralRolloutPolicy: StructuralRolloutPolicy = { + k: 5, + repairRounds: 2, + testgen: 6, +} + +function resolvePolicy(overrides?: Partial): StructuralRolloutPolicy { + const policy = { ...defaultStructuralRolloutPolicy, ...overrides } + if (!Number.isInteger(policy.k) || policy.k < 1) { + throw new Error(`structuralRollout: policy.k must be an integer >= 1, got ${policy.k}`) + } + if (!Number.isInteger(policy.repairRounds) || policy.repairRounds < 0) { + throw new Error( + `structuralRollout: policy.repairRounds must be an integer >= 0, got ${policy.repairRounds}`, + ) + } + if (!Number.isInteger(policy.testgen) || policy.testgen < 0) { + throw new Error( + `structuralRollout: policy.testgen must be an integer >= 0, got ${policy.testgen}`, + ) + } + return policy +} + +// ── Visible checks: the CheckSource seam (the only net-new seam of the design) ────── + +/** One task-visible executable check (e.g. a single-line Python assert). */ +export interface VisibleCheck { + code: string + /** 'official' = shown in the task itself (docstring example, shown assert); + * 'authored' = the model's own guess. Official outranks authored in selection. */ + kind: 'official' | 'authored' +} + +/** What a CheckSource composes with. `consult` is the strategy family's raw analyst + * channel (metered by the conserved pool, offline-injectable via `opts.complete`) — + * check authoring goes through it rather than a bespoke model client. */ +export interface CheckSourceCtx { + /** Authored-check budget for this task (`policy.testgen`). */ + count: number + /** The symbol authored checks must reference; undefined ⇒ authoring is skipped + * (no guesses beats guesses pinned to nothing). */ + entrySymbol?: string + /** One metered LLM call: instruction in, reply text out, null when the channel went + * down. The task's visible prompt is included by the channel itself. */ + consult(instruction: string): Promise +} + +/** Produces the task's visible checks. MUST derive them from agent-visible information + * only, before any candidate exists — the strategy freezes the returned set for every + * sample and repair round of the task. */ +export interface CheckSource { + generate(task: AgenticTask, ctx: CheckSourceCtx): Promise +} + +const authorInstruction = (count: number, entry: string) => + `Read the task below. Write exactly ${count} single-line assert statements that test the ` + + `function \`${entry}\`, based ONLY on the behavior the task itself describes. Each assert ` + + `must be one physical line of the form \`assert ${entry}(...) == expected\` (or a True/False ` + + 'check). Do NOT implement the function. Do NOT copy shown examples verbatim if you can test ' + + 'other cases too. Output ONLY the assert lines inside a single ```python code block.' + +/** The proven authored-assert filter (lifted from the rigs' generateTests): keep only + * single-line, paren-balanced asserts that reference the entry symbol — malformed lines + * are dropped here rather than poisoning every candidate's score identically. */ +export function filterAuthoredAsserts(reply: string, entrySymbol: string, count: number): string[] { + const fences = [...reply.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/gi)].map((m) => + (m[1] ?? '').trim(), + ) + const block = fences.length > 0 ? fences.join('\n') : reply + const balanced = (s: string) => { + let d = 0 + for (const ch of s) { + if (ch === '(' || ch === '[' || ch === '{') d += 1 + else if (ch === ')' || ch === ']' || ch === '}') d -= 1 + if (d < 0) return false + } + return d === 0 + } + return block + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.startsWith('assert ') && l.includes(entrySymbol) && balanced(l)) + .slice(0, count) +} + +/** Default authored-check source: one metered LLM call per task, before sampling, + * filtered through `filterAuthoredAsserts`. Returns [] (no signal, never a fabricated + * check) when the budget is 0, no entry symbol resolves, or the channel went down. */ +export function modelAuthoredChecks(overrides: { count?: number } = {}): CheckSource { + return { + async generate(_task, ctx): Promise { + const count = overrides.count ?? ctx.count + if (count <= 0 || !ctx.entrySymbol) return [] + const entry = ctx.entrySymbol + const reply = await ctx.consult(authorInstruction(count, entry)) + if (!reply) return [] + return filterAuthoredAsserts(reply, entry, count).map((code) => ({ + code, + kind: 'authored' as const, + })) + }, + } +} + +/** Official checks the surface stashed on the task (e.g. MBPP's shown assert). Reads + * `task.meta[key]` as a string array; anything else means no official checks. */ +export function officialChecksFromMeta(key = 'visibleChecks'): CheckSource { + return { + async generate(task): Promise { + const raw = task.meta?.[key] + if (!Array.isArray(raw)) return [] + return raw + .filter((c): c is string => typeof c === 'string' && c.trim().length > 0) + .map((code) => ({ code, kind: 'official' as const })) + }, + } +} + +/** Concatenate check sources (official first by convention — ordering does not affect + * scoring, which reads each check's `kind`). */ +export function composeCheckSources(...sources: CheckSource[]): CheckSource { + return { + async generate(task, ctx): Promise { + const all: VisibleCheck[] = [] + for (const source of sources) all.push(...(await source.generate(task, ctx))) + return all + }, + } +} + +/** The symbol authored checks are pinned to: `task.meta.entryPoint` when the surface + * provides it, else the LAST `def name(` in the visible prompt (a code-completion stub + * lists helpers first, the entry stub last). Undefined ⇒ authoring is skipped. */ +export function resolveEntrySymbol(task: AgenticTask): string | undefined { + const meta = task.meta?.entryPoint + if (typeof meta === 'string' && meta.trim().length > 0) return meta.trim() + const defs = [...task.userPrompt.matchAll(/(?:^|\n)\s*def\s+([A-Za-z_]\w*)\s*\(/g)] + const last = defs[defs.length - 1] + return last?.[1] +} + +// ── Check execution: the CheckRunner seam ──────────────────────────────────────────── + +/** How one candidate fared against the frozen visible checks, split by check kind. */ +export interface CheckOutcome { + passedOfficial: number + totalOfficial: number + passedAuthored: number + totalAuthored: number + /** The checks' failure report — the ONLY feedback the repair loop may see. */ + failureOutput: string + /** True when the candidate crashed before any check could run — ranks below a + * candidate that ran and failed everything. */ + crashed?: boolean +} + +/** Minimal exec channel the default runner needs. `SandboxInstance` (and therefore + * `ValidationCtx.box`) satisfies it structurally. */ +export interface CheckExecChannel { + exec( + command: string, + options?: { timeoutMs?: number }, + ): Promise<{ exitCode: number; stdout: string; stderr: string }> +} + +export interface CheckRunContext { + task: AgenticTask + /** Live exec channel for this run (`ValidationCtx.box` / a sandbox instance). */ + box?: CheckExecChannel + signal?: AbortSignal +} + +/** Executes the frozen checks against one candidate. Implementations MUST fail loud + * (throw) when they cannot execute — a silent zero poisons selection. */ +export interface CheckRunner { + run(candidate: string, checks: VisibleCheck[], ctx: CheckRunContext): Promise +} + +/** The check program (mirrors the rigs' visible-check judge): candidate executes at + * module level, then each check runs INDIVIDUALLY in try/except so one malformed line + * cannot zero the rest; the summary line carries a per-call NONCE so a candidate + * printing a forged summary cannot be parsed as the verdict. */ +function buildCheckProgram( + candidate: string, + official: string[], + authored: string[], + nonce: string, +): string { + const officialB64 = Buffer.from(JSON.stringify(official), 'utf8').toString('base64') + const authoredB64 = Buffer.from(JSON.stringify(authored), 'utf8').toString('base64') + return `${candidate}\n +import base64 as _b64, json as _json, sys as _sys +_official = _json.loads(_b64.b64decode("${officialB64}").decode("utf8")) +_authored = _json.loads(_b64.b64decode("${authoredB64}").decode("utf8")) +_lines = [] +def _run(_tests): + _att, _fail = 0, 0 + for _t in _tests: + _att += 1 + try: + exec(_t, dict(globals())) + except Exception as _e: + _fail += 1 + _lines.append("CHECK FAILED: %s -> %s: %s" % (_t.strip()[:200], type(_e).__name__, str(_e)[:200])) + return _att, _fail +_o_att, _o_fail = _run(_official) +_a_att, _a_fail = _run(_authored) +print("SRCK-${nonce} official=%d/%d authored=%d/%d" % (_o_att - _o_fail, _o_att, _a_att - _a_fail, _a_att)) +_sys.stdout.write("\\n".join(_lines)[-1500:]) +_sys.exit(0 if (_o_fail + _a_fail) == 0 and (_o_att + _a_att) > 0 else 1) +` +} + +/** Default CheckRunner backend: pipes the check program into `python3` over the sandbox + * exec channel (`ctx.box`, or one bound at construction). Never shells out to docker + * itself — the jail is the sandbox's concern. No channel ⇒ throws; it must never + * silently score 0. Empty check sets short-circuit to a no-signal outcome (nothing to + * execute, so no channel is required). */ +export function sandboxCheckRunner( + options: { box?: CheckExecChannel; python?: string; timeoutMs?: number } = {}, +): CheckRunner { + const python = options.python ?? 'python3' + const timeoutMs = options.timeoutMs ?? 20000 + return { + async run(candidate, checks, ctx): Promise { + if (checks.length === 0) { + return { + passedOfficial: 0, + totalOfficial: 0, + passedAuthored: 0, + totalAuthored: 0, + failureOutput: '', + } + } + const box = ctx.box ?? options.box + if (!box) { + throw new Error( + 'sandboxCheckRunner: no execution channel — bind one via sandboxCheckRunner({ box }) ' + + 'or CheckRunContext.box (ValidationCtx.box / a sandbox instance). Refusing to score ' + + 'without executing: a silent 0 would poison selection.', + ) + } + const nonce = randomBytes(8).toString('hex') + const official = checks.filter((c) => c.kind === 'official').map((c) => c.code) + const authored = checks.filter((c) => c.kind === 'authored').map((c) => c.code) + const program = buildCheckProgram(candidate, official, authored, nonce) + const b64 = Buffer.from(program, 'utf8').toString('base64') + const r = await box.exec(`printf '%s' '${b64}' | base64 -d | ${python} -`, { timeoutMs }) + const summary = new RegExp( + `SRCK-${nonce} official=(\\d+)/(\\d+) authored=(\\d+)/(\\d+)`, + ).exec(r.stdout) + if (!summary) { + // The candidate crashed (or hung) before the check scaffold could report. + const detail = + (r.stderr || r.stdout).slice(-1500) || + 'no output (crashed or timed out before the checks could run)' + return { + passedOfficial: 0, + totalOfficial: 0, + passedAuthored: 0, + totalAuthored: 0, + failureOutput: detail, + crashed: true, + } + } + // Strip the sentinel line from repair feedback — the model must never learn the + // summary format it could try to forge. + const failureOutput = r.stdout.replace(summary[0], '').slice(-1500).trim() + return { + passedOfficial: Number(summary[1]), + totalOfficial: Number(summary[2]), + passedAuthored: Number(summary[3]), + totalAuthored: Number(summary[4]), + failureOutput, + } + }, + } +} + +// ── Selection order (measured, not stylistic) ──────────────────────────────────────── + +const frac = (passed: number, total: number) => (total > 0 ? passed / total : 0) + +/** The selection order: crash < ran; then official pass-fraction; authored guesses only + * break ties. Returns > 0 when `a` outranks `b`. Strictly lexicographic — on MBPP, + * letting 6 noisy guesses outvote the one official check flipped selection negative. */ +export function compareCheckOutcomes(a: CheckOutcome, b: CheckOutcome): number { + const aCrashed = a.crashed === true + const bCrashed = b.crashed === true + if (aCrashed !== bCrashed) return aCrashed ? -1 : 1 + if (aCrashed) return 0 + const official = frac(a.passedOfficial, a.totalOfficial) - frac(b.passedOfficial, b.totalOfficial) + if (official !== 0) return official + return frac(a.passedAuthored, a.totalAuthored) - frac(b.passedAuthored, b.totalAuthored) +} + +/** Display scalar for receipts/reports (the rigs' `visibleScore` shape): crash = -1, + * else official fraction + 0.001 × authored fraction. Selection itself uses the exact + * lexicographic comparator, never this scalar. */ +export function visibleCheckScore(o: CheckOutcome): number { + if (o.crashed) return -1 + return frac(o.passedOfficial, o.totalOfficial) + 0.001 * frac(o.passedAuthored, o.totalAuthored) +} + +/** Argmax by `compareCheckOutcomes`, FIRST index wins ties (deterministic; with zero + * visible coverage every candidate ties at no-signal and index 0 is the blind pick). */ +export function selectBestIndex(outcomes: ReadonlyArray): number { + let best = 0 + for (let i = 1; i < outcomes.length; i += 1) { + if (compareCheckOutcomes(outcomes[i] as CheckOutcome, outcomes[best] as CheckOutcome) > 0) { + best = i + } + } + return best +} + +/** The repair keep-best guard: a challenger displaces the incumbent only when it is + * strictly better in the selection order AND passes at least as many official checks. + * The raw-count clause is deliberate belt-and-braces over the comparator (a custom + * runner can report shifted totals): repair must NEVER replace a candidate that passes + * more official checks with one that passes fewer. */ +export function canDisplace(challenger: CheckOutcome, incumbent: CheckOutcome): boolean { + if (challenger.crashed === true) return false + if (challenger.passedOfficial < incumbent.passedOfficial) return false + return compareCheckOutcomes(challenger, incumbent) > 0 +} + +const totalChecks = (o: CheckOutcome) => o.totalOfficial + o.totalAuthored + +const passesAllChecks = (o: CheckOutcome) => + o.crashed !== true && + totalChecks(o) > 0 && + o.passedOfficial === o.totalOfficial && + o.passedAuthored === o.totalAuthored + +// ── Candidate extraction ───────────────────────────────────────────────────────────── + +/** The candidate a shot produced, read from its conversation: the LAST `submit_answer` + * tool-call argument (verifier environments submit the artifact explicitly), else the + * latest assistant reply's fenced code block — preferring a block containing a `def`, + * because repair replies echo the failure report in a bare fence BEFORE the fixed code + * (the rigs' extractRepairCode lesson) — else the latest non-empty assistant text. */ +export function defaultExtractCandidate(messages: ReadonlyArray): string { + for (let i = messages.length - 1; i >= 0; i -= 1) { + const calls = messages[i]?.tool_calls as + | Array<{ function?: { name?: string; arguments?: string } }> + | undefined + if (!calls) continue + for (let j = calls.length - 1; j >= 0; j -= 1) { + const call = calls[j] + if (call?.function?.name !== 'submit_answer') continue + try { + const args = JSON.parse(call.function.arguments ?? '{}') as { answer?: unknown } + if (typeof args.answer === 'string' && args.answer.trim()) return args.answer.trim() + } catch { + /* malformed arguments — keep scanning */ + } + } + } + const contents: string[] = [] + for (const m of messages) { + if (m.role === 'assistant' && typeof m.content === 'string' && m.content.trim()) { + contents.push(m.content) + } + } + const fencesOf = (text: string) => + [...text.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/gi)].map((m) => (m[1] ?? '').trim()) + for (let i = contents.length - 1; i >= 0; i -= 1) { + const fences = fencesOf(contents[i] as string) + for (let j = fences.length - 1; j >= 0; j -= 1) { + if (/(^|\n)\s*def\s+\w+/.test(fences[j] as string)) return fences[j] as string + } + } + for (let i = contents.length - 1; i >= 0; i -= 1) { + const fences = fencesOf(contents[i] as string) + if (fences.length > 0) return fences[fences.length - 1] as string + } + return (contents[contents.length - 1] ?? '').trim() +} + +// ── The strategy ───────────────────────────────────────────────────────────────────── + +/** Per-slot approach lenses for `diverse` mode — copied from bench/src/directives.ts + * (the measured rig plumbing; a paired null there, kept as an optional knob). */ +const DIVERSE_LENSES: ReadonlyArray = [ + 'Answer directly and decisively from what you already know. State the single best answer without hedging.', + 'Decompose the question into the sub-facts it depends on. Establish each sub-fact explicitly, then compose them into the answer.', + 'Reason from first principles. Ignore the most obvious or popular guess; derive the answer from underlying facts and relationships.', + 'Name the most plausible WRONG answer and the trap that makes it tempting. Rule it out, then commit to the answer that survives.', +] + +function slotLens(slot: number): string { + const lens = DIVERSE_LENSES[slot % DIVERSE_LENSES.length] as string + const tag = + slot < DIVERSE_LENSES.length ? '' : ` (variant ${Math.floor(slot / DIVERSE_LENSES.length) + 1})` + return `${lens}${tag}` +} + +function repairSteer(outcome: CheckOutcome): string { + return [ + 'Your latest solution failed some of the task-visible checks.', + 'Result of running the visible checks against it:', + '```', + outcome.failureOutput.trim() || '(the code crashed before the checks could run)', + '```', + 'Fix the solution so the visible checks pass. Provide the COMPLETE corrected solution the', + 'same way you provided the original (same tool or format) — not a fragment or a diff.', + ].join('\n') +} + +function describeOutcome(label: string, o: CheckOutcome): string { + if (o.crashed) return `${label}: crashed before the checks could run` + return ( + `${label}: official ${o.passedOfficial}/${o.totalOfficial}, ` + + `authored ${o.passedAuthored}/${o.totalAuthored}` + ) +} + +export type RepairStop = + | 'already-passing' + | 'no-signal' + | 'repaired-pass' + | 'rounds-exhausted' + | 'no-candidates' + +/** The body's deliverable — a `StrategyResult` plus selection provenance. The extra + * fields ride through `defineStrategy`'s deliverable spread onto `AgenticRunResult` + * (score/resolved stay harness-verified, exactly as for every authored strategy). */ +export interface StructuralRolloutResult extends StrategyResult { + /** One receipt per scored candidate (k samples, then repairs), `SelectionReceipt` + * shaped like the kernel's (`types.ts`), selector 'driver'. */ + selection: SelectionReceipt[] + repairStop: RepairStop + officialChecks: number + authoredChecks: number +} + +export interface StructuralRolloutConfig { + /** Knobs; missing fields take the measured defaults (k=5, repairRounds=2, testgen=6). */ + policy?: Partial + /** Where the visible checks come from. Default: official checks from + * `task.meta.visibleChecks` composed with `modelAuthoredChecks()`. */ + checkSource?: CheckSource + /** How candidates are measured. Default `sandboxCheckRunner()` — it needs an exec + * channel (bind one to the runner, or pass `box` here) and fails loud without one. */ + checkRunner?: CheckRunner + /** Exec channel threaded into every check run of this strategy (a sandbox instance / + * `ValidationCtx.box`). The strategy seam itself carries no sandbox, so the caller + * who owns one supplies it here or binds it into the runner. */ + box?: CheckExecChannel + /** Candidate extraction from a shot's conversation. Default `defaultExtractCandidate`. */ + extractCandidate?: (messages: ReadonlyArray) => string +} + +/** + * Build the structuralRollout `Strategy`: k shots → score each by the frozen visible + * checks (official above authored, crash lowest) → argmax with first-index tie-break → + * up to `repairRounds` repair shots steered by the failure output, keep-best under the + * official-check guard. Authored via `defineStrategy`, so the deliverable score stays + * harness-verified and every shot is metered by the conserved pool. + * + * Budget note: `runAgentic`'s `budget` sizes the pool — pass at least + * `k + repairRounds + 1` so the samples, repairs, and the check-author consult all admit. + */ +export function structuralRollout(config: StructuralRolloutConfig = {}): Strategy { + const policy = resolvePolicy(config.policy) + const checkSource = + config.checkSource ?? composeCheckSources(officialChecksFromMeta(), modelAuthoredChecks()) + const checkRunner = config.checkRunner ?? sandboxCheckRunner() + const extract = config.extractCandidate ?? defaultExtractCandidate + + const inner = defineStrategy( + 'structuralRollout', + async (ctx: StrategyCtx): Promise => { + const { task, shot } = ctx + const progression: number[] = [] + const receipts: SelectionReceipt[] = [] + let completions = 0 + let shots = 0 + + // Freeze the visible checks BEFORE any candidate exists — every sample and repair + // round is measured against this same set. Authoring goes through the strategy's + // raw analyst channel, so its spend is metered and the offline transport applies. + const consult = async (instruction: string): Promise => { + const reply = await ctx.consult([], instruction) + completions += 1 + return reply + } + const entrySymbol = resolveEntrySymbol(task) + const checks = await checkSource.generate(task, { + count: policy.testgen, + ...(entrySymbol ? { entrySymbol } : {}), + consult, + }) + const officialChecks = checks.filter((c) => c.kind === 'official').length + const authoredChecks = checks.length - officialChecks + const runCtx: CheckRunContext = { task, ...(config.box ? { box: config.box } : {}) } + + interface Candidate { + index: number + messages: Msg[] + outcome: CheckOutcome + shotScore: number + shotResolved: boolean + } + + // k independent samples, each on its own artifact, scored by the frozen checks. + const candidates: Candidate[] = [] + for (let i = 0; i < policy.k; i += 1) { + const out = await shot(policy.diverse ? { steer: slotLens(i) } : undefined) + if (!out) break // pool starved — select among what settled + shots += 1 + completions += out.completions + progression.push(out.score) + const outcome = await checkRunner.run(extract(out.messages), checks, runCtx) + candidates.push({ + index: candidates.length, + messages: out.messages, + outcome, + shotScore: out.score, + shotResolved: out.total > 0 && out.passes === out.total, + }) + } + if (candidates.length === 0) { + return { + score: 0, + resolved: false, + completions, + progression, + shots, + selection: receipts, + repairStop: 'no-candidates', + officialChecks, + authoredChecks, + } + } + + // Argmax by the official-first order; first index wins ties (the blind pick when + // the task has zero visible coverage). + let best = candidates[selectBestIndex(candidates.map((c) => c.outcome))] as Candidate + for (const c of candidates) { + receipts.push({ + candidateIndex: c.index, + selected: false, + score: visibleCheckScore(c.outcome), + reason: describeOutcome('sample', c.outcome), + selector: 'driver', + }) + } + + // Guarded repair: steered ONLY by the checks' failure output, keep-best, and a + // repair never displaces a candidate that passes more official checks. + let seq = candidates.length + let repairStop: RepairStop = 'already-passing' + if (!passesAllChecks(best.outcome)) { + if (best.outcome.crashed !== true && totalChecks(best.outcome) === 0) { + repairStop = 'no-signal' // no visible checks ⇒ nothing honest to steer on + } else { + repairStop = 'rounds-exhausted' + for (let r = 0; r < policy.repairRounds; r += 1) { + const out = await shot({ messages: best.messages, steer: repairSteer(best.outcome) }) + if (!out) break + shots += 1 + completions += out.completions + progression.push(out.score) + const outcome = await checkRunner.run(extract(out.messages), checks, runCtx) + const displaced = canDisplace(outcome, best.outcome) + const label = displaced + ? 'repair (displaced the incumbent)' + : outcome.crashed !== true && outcome.passedOfficial < best.outcome.passedOfficial + ? 'repair (held out: passes fewer official checks than the incumbent)' + : 'repair (held out: no improvement)' + receipts.push({ + candidateIndex: seq, + selected: false, + score: visibleCheckScore(outcome), + reason: describeOutcome(label, outcome), + selector: 'driver', + }) + if (displaced) { + best = { + index: seq, + messages: out.messages, + outcome, + shotScore: out.score, + shotResolved: out.total > 0 && out.passes === out.total, + } + } + seq += 1 + if (passesAllChecks(best.outcome)) { + repairStop = 'repaired-pass' + break + } + } + } + } + + const winner = receipts.find((r) => r.candidateIndex === best.index) + if (winner) winner.selected = true + + return { + score: best.shotScore, + resolved: best.shotResolved, + completions, + progression, + shots, + selection: receipts, + repairStop, + officialChecks, + authoredChecks, + } + }, + ) + + if (policy.temperature === undefined) return inner + // The shot temperature is an AgenticOptions concern; the policy override threads in at + // the driver seam so the strategy stays a plain defineStrategy member. + return { + name: inner.name, + driver: (surface, task, opts, budget) => + inner.driver(surface, task, { ...opts, temperature: policy.temperature }, budget), + } +} diff --git a/src/runtime/supervise/budget.ts b/src/runtime/supervise/budget.ts index a842d938..b3c88eba 100644 --- a/src/runtime/supervise/budget.ts +++ b/src/runtime/supervise/budget.ts @@ -191,6 +191,11 @@ export function createBudgetPool(root: Budget, now: () => number = Date.now): Bu open.delete(ticket.id) const { tokens: rTokens, usd: rUsd, iterations: rIterations } = ticket.reserved + if (usdCapped && spent.usdKnown === false) { + throw new Error( + `budget pool: ticket ${ticket.id} reported unknown dollar cost under a dollar-capped budget`, + ) + } // Clamp actual spend to the reservation: a child must never commit more than it // reserved (that would overdraw the conserved pool). Over-spend is a fail-loud bug. @@ -235,6 +240,11 @@ export function createBudgetPool(root: Budget, now: () => number = Date.now): Bu } function observe(spend: Spend): void { + if (usdCapped && spend.usdKnown === false) { + throw new Error( + 'budget pool: cannot observe unknown dollar cost under a dollar-capped budget', + ) + } const tokens = totalTokens(spend.tokens) // Direct free → committed debit (no reservation ticket). `free` may go negative on overspend — // that is honest; the readout then reports exhaustion and the in-loop guard halts the driver. diff --git a/src/runtime/supervise/driver-executor.ts b/src/runtime/supervise/driver-executor.ts index 43ee3c57..6429f164 100644 --- a/src/runtime/supervise/driver-executor.ts +++ b/src/runtime/supervise/driver-executor.ts @@ -268,6 +268,7 @@ function sumSpend(settled: ReadonlyArray<{ spent: Spend }>): Spend { total.tokens.input += ev.spent.tokens.input total.tokens.output += ev.spent.tokens.output total.usd += ev.spent.usd + if (ev.spent.usdKnown === false) total.usdKnown = false total.ms += ev.spent.ms } return total @@ -284,6 +285,7 @@ function sumMetered(events: ReadonlyArray): Spend { total.tokens.input += ev.spend.tokens.input total.tokens.output += ev.spend.tokens.output total.usd += ev.spend.usd + if (ev.spend.usdKnown === false) total.usdKnown = false total.ms += ev.spend.ms } return total diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 67498071..0fa478bd 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -135,6 +135,10 @@ export interface CliWorktreeSeam { runId?: string baseRef?: string harnessTimeoutMs?: number + /** Isolated, network-off Codex execution with terminal JSONL usage capture. */ + codexReproducible?: boolean + /** Absolute host paths denied to reproducible Codex. */ + codexReadDeniedPaths?: ReadonlyArray testCmd?: string typecheckCmd?: string checkTimeoutMs?: number @@ -1514,6 +1518,8 @@ export const cliWorktreeExecutor: ExecutorFactory = (spec, ctx) => { ...(seam.runId ? { runId: seam.runId } : {}), ...(seam.baseRef ? { baseRef: seam.baseRef } : {}), ...(seam.harnessTimeoutMs !== undefined ? { harnessTimeoutMs: seam.harnessTimeoutMs } : {}), + ...(seam.codexReproducible ? { codexReproducible: true } : {}), + ...(seam.codexReadDeniedPaths ? { codexReadDeniedPaths: seam.codexReadDeniedPaths } : {}), ...(seam.testCmd !== undefined ? { testCmd: seam.testCmd } : {}), ...(seam.typecheckCmd !== undefined ? { typecheckCmd: seam.typecheckCmd } : {}), ...(seam.checkTimeoutMs !== undefined ? { checkTimeoutMs: seam.checkTimeoutMs } : {}), diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index 409dfe7c..d1a41ca3 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -764,6 +764,7 @@ function clampSpend(spend: Spend, budget: Budget): Spend { } : spend.tokens, usd: budget.maxUsd === undefined ? spend.usd : Math.min(spend.usd, budget.maxUsd), + ...(spend.usdKnown === false ? { usdKnown: false } : {}), ms: spend.ms, } } diff --git a/src/runtime/supervise/supervisor.ts b/src/runtime/supervise/supervisor.ts index d5715146..818f50b2 100644 --- a/src/runtime/supervise/supervisor.ts +++ b/src/runtime/supervise/supervisor.ts @@ -448,6 +448,7 @@ function accumulate(a: Spend, b: Spend): void { a.tokens.input += b.tokens.input a.tokens.output += b.tokens.output a.usd += b.usd + if (b.usdKnown === false) a.usdKnown = false a.ms += b.ms } @@ -458,6 +459,7 @@ function addSpend(a: Spend, b: Spend): Spend { iterations: a.iterations + b.iterations, tokens: { input: a.tokens.input + b.tokens.input, output: a.tokens.output + b.tokens.output }, usd: a.usd + b.usd, + ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), ms: a.ms + b.ms, } } diff --git a/src/runtime/supervise/trajectory-recorder.ts b/src/runtime/supervise/trajectory-recorder.ts index 47840af0..38012931 100644 --- a/src/runtime/supervise/trajectory-recorder.ts +++ b/src/runtime/supervise/trajectory-recorder.ts @@ -17,8 +17,8 @@ export interface TrajectoryAnalysis { /** Structured run summary (tool-call count, step order). Steps carry a single timestamp, so per-span * duration is 0; loop/waste detection keys on call PATTERNS + cross-span windows, not durations. */ readonly trajectory: Awaited> - /** Full-run repeated-call view (total occurrences + window) — catches a loop the online consecutive - * detector interleaves past. */ + /** Full-run repeated-call view (total occurrences + window) — allows one intervening call so it + * catches a loop the online consecutive detector interleaves past. */ readonly stuckLoop: Awaited> /** Wasted-vs-total tool-call ratio for the run. */ readonly toolWaste: Awaited> @@ -31,15 +31,47 @@ export async function analyzeTrace( ): Promise { const spans = await source.collect() const store = new InMemoryTraceStore() + const laneSpanId = `${runId}-session-lane` + if (spans.length > 0) { + const startedAt = Math.min(...spans.map((span) => span.startedAt)) + const endedAt = Math.max(...spans.map((span) => span.endedAt ?? span.startedAt)) + const agentSpanId = `${runId}-agent` + await store.appendSpan({ + spanId: agentSpanId, + runId, + kind: 'agent', + name: 'trace-source-agent', + startedAt, + endedAt, + }) + await store.appendSpan({ + spanId: laneSpanId, + parentSpanId: agentSpanId, + runId, + kind: 'custom', + name: 'trace-source-session', + startedAt, + endedAt, + }) + } // Re-stamp onto one runId so the runId-filtered analyzers see the whole trace regardless of the - // source's own id scheme. + // source's own id scheme. A TraceSource carries tool spans only, so attach + // them to one explicit session lane; 0.117's analyzer uses ancestry plus + // overlap times to distinguish serial repeats from parallel siblings. for (let i = 0; i < spans.length; i += 1) { const s = spans[i] - if (s) await store.appendSpan({ ...s, runId, spanId: `${runId}-t${i}` }) + if (s) { + await store.appendSpan({ + ...s, + runId, + spanId: `${runId}-t${i}`, + parentSpanId: laneSpanId, + }) + } } const [trajectory, stuckLoop, toolWaste] = await Promise.all([ buildTrajectory(store, runId), - stuckLoopView(store, { runId }), + stuckLoopView(store, { runId, maxInterveningToolCalls: 1 }), toolWasteView(store, { runId }), ]) return { trajectory, stuckLoop, toolWaste } diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 040ff7ee..489026ca 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -209,6 +209,9 @@ export interface Budget { export interface Spend { iterations: number tokens: LoopTokenUsage + /** Dollar accounting is known unless explicitly false. A false value must not be treated as $0 + * when enforcing a dollar-denominated comparison or limit. */ + usdKnown?: boolean usd: number ms: number } diff --git a/src/runtime/supervise/worktree-cli-executor.ts b/src/runtime/supervise/worktree-cli-executor.ts index d13d7a39..38485bdd 100644 --- a/src/runtime/supervise/worktree-cli-executor.ts +++ b/src/runtime/supervise/worktree-cli-executor.ts @@ -9,15 +9,13 @@ * This is a THIN adapter: the physical act (worktree → profile-aware harness invocation → diff → * checks → cleanup) lives ONCE in `runWorktreeHarness` (`../../mcp/worktree-harness`), shared with * the `runLoop`/coder-delegate `createInProcessExecutor`. This executor only projects that core's - * result onto the `Executor` port (artifact + spend) and owns the teardown point. The §1.5 payload - * (authored systemPrompt + model) reaches the harness inside the core, not here. + * result onto the `Executor` port (artifact + spend) and owns the teardown point. The complete + * profile delivery — direct prompt/model plus materialized file-backed resources — lives there. * - * Token accounting: a harness CLI does not surface usage, so this executor defaults to - * `budgetExempt: true` — its spend is NOT metered against the conserved pool and its iterations are - * EXCLUDED from the equal-k arms by construction (mirrors `cliExecutor`). The exemption is an - * explicit, documented `budgetExempt` option rather than a buried hardcode: set it `false` ONLY for - * a harness that genuinely surfaces real token/usd usage to meter into the pool — otherwise the - * executor would meter a fabricated zero, which the no-silent-zeros rule forbids. + * Token accounting: ordinary harness CLI runs remain `budgetExempt`. Reproducible Codex mode + * parses the CLI's terminal JSONL usage and is metered by default; an absent usage event fails the + * run instead of recording fabricated zero tokens. Codex does not report dollar cost, so that + * channel is explicitly marked unknown on the resulting `Spend`. * * @experimental */ @@ -34,11 +32,11 @@ import { type WorktreeCommandResult, type WorktreeHarnessResult, type WorktreeHarnessRun, + type WorktreeProfileMaterializationReceipt, } from '../../mcp/worktree-harness' -import { zeroTokenUsage } from '../util' import type { Executor, ExecutorResult, Spend } from './types' -export type { WorktreeCommandResult } +export type { WorktreeCommandResult, WorktreeProfileMaterializationReceipt } /** Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured * diff + the harness's run record + the derived checks). */ export type WorktreePatchArtifact = WorktreeHarnessResult @@ -47,9 +45,15 @@ export type WorktreePatchArtifact = WorktreeHarnessResult export interface WorktreeCliExecutorOptions { /** Absolute path to the git checkout the worktree is cut from. */ repoRoot: string - /** The SUPERVISOR-AUTHORED profile (the §1.5 payload: systemPrompt + model). */ + /** + * The supervisor-authored prompt/model plus materializable structural resources. + * `model.default` selects the one-shot model; `small`, `provider`, and `metadata` remain hints. + * Resource failures are fatal regardless of `resources.failOnError`. + * Tools, permissions, connections, confidential execution, modes, and extensions fail closed. + * Harness-specific nested controls that the pinned materializer cannot preserve also fail closed. + */ profile: AgentProfile - /** Which local harness CLI drives this leaf (`claude` | `codex` | `opencode`). */ + /** Local CLI for this leaf. This explicit choice overrides `profile.harness`. */ harness: LocalHarness /** The per-task instruction handed to the harness (composed under the system prompt). */ taskPrompt: string @@ -59,6 +63,12 @@ export interface WorktreeCliExecutorOptions { baseRef?: string /** Wall-clock cap per harness subprocess (ms). Default 5 min (the `runLocalHarness` default). */ harnessTimeoutMs?: number + /** Run Codex with an ephemeral session, isolated config/instructions, network disabled, and + * JSONL usage capture. Requires `harness: 'codex'`; metered by default. */ + codexReproducible?: boolean + /** Absolute host paths denied to reproducible Codex (for benchmark answer copies, credentials, + * or other task-specific ambient state). */ + codexReadDeniedPaths?: ReadonlyArray /** * Shell command run in the live worktree to derive the tests-PASS signal (e.g. `pnpm test`). * Its exit code becomes `artifact.checks.tests.passed`. Omit to skip (no signal derived). @@ -78,10 +88,9 @@ export interface WorktreeCliExecutorOptions { * outcomes without spawning a real shell. Defaults to a `/bin/sh -c` spawn in the worktree. */ runCommand?: WorktreeCheckRunner /** - * Exclude this leaf's spend from the conserved pool + equal-k arms. Defaults to `true` because a - * coding-harness CLI does not surface token usage, so metering it would record a fabricated zero - * (the no-silent-zeros rule forbids that). Set `false` ONLY for a harness that surfaces real - * token/usd usage worth metering — the executor would then debit the (real) spend it captures. + * Exclude this leaf's spend from accounting. Defaults to `true` for ordinary CLI runs and + * `false` for `codexReproducible`, which captures real token usage. A metered custom runner must + * likewise return `LocalHarnessResult.usage`. */ budgetExempt?: boolean } @@ -107,13 +116,23 @@ export function createWorktreeCliExecutor( if (typeof options.taskPrompt !== 'string' || options.taskPrompt.length === 0) { throw new ValidationError('createWorktreeCliExecutor: taskPrompt required') } + if (options.codexReproducible && options.harness !== 'codex') { + throw new ValidationError( + 'createWorktreeCliExecutor: codexReproducible requires harness "codex"', + ) + } + if (options.codexReproducible && options.budgetExempt === true) { + throw new ValidationError('createWorktreeCliExecutor: codexReproducible cannot be budgetExempt') + } + if (options.codexReadDeniedPaths !== undefined && !options.codexReproducible) { + throw new ValidationError( + 'createWorktreeCliExecutor: codexReadDeniedPaths requires codexReproducible', + ) + } const runId = options.runId ?? randomUUID() const controller = new AbortController() - // Default true: a harness CLI cannot account tokens, so the honest value is "exclude from the - // pool + equal-k" rather than meter a fabricated zero. An explicit `false` opts a real-usage - // harness into metering the spend it captures. - const budgetExempt = options.budgetExempt ?? true + const budgetExempt = options.budgetExempt ?? !options.codexReproducible let run: WorktreeHarnessRun | undefined let artifact: ExecutorResult | undefined @@ -137,6 +156,10 @@ export function createWorktreeCliExecutor( ...(options.harnessTimeoutMs !== undefined ? { harnessTimeoutMs: options.harnessTimeoutMs } : {}), + ...(options.codexReproducible ? { codexReproducible: true } : {}), + ...(options.codexReadDeniedPaths + ? { codexReadDeniedPaths: options.codexReadDeniedPaths } + : {}), ...(options.checkTimeoutMs !== undefined ? { checkTimeoutMs: options.checkTimeoutMs } : {}), ...(options.checkOutputCap !== undefined ? { checkOutputCap: options.checkOutputCap } : {}), ...(linked ? { signal: linked } : {}), @@ -145,13 +168,22 @@ export function createWorktreeCliExecutor( ...(options.runCommand ? { runCommand: options.runCommand } : {}), }) + const usage = run.result.harness.usage + if (!budgetExempt && !usage) { + const completed = run + run = undefined + await completed.cleanup() + throw new ValidationError( + 'createWorktreeCliExecutor: metered harness run returned no token usage', + ) + } const spent: Spend = { iterations: 1, - // The worktree-harness core surfaces no token/usd usage, so tokens/usd are a genuine zero - // (NOT a fabricated cost). When budgetExempt is true the pool ignores this spend entirely; - // when explicitly false the scope debits exactly this captured spend — the real iteration. - tokens: zeroTokenUsage(), + tokens: usage + ? { input: usage.inputTokens, output: usage.outputTokens } + : { input: 0, output: 0 }, usd: 0, + ...(usage ? { usdKnown: false } : {}), ms: Date.now() - started, } artifact = { outRef: contentAddress(run.result), out: run.result, spent } diff --git a/src/runtime/tool-loop.ts b/src/runtime/tool-loop.ts index a52725df..ee1ea7a7 100644 --- a/src/runtime/tool-loop.ts +++ b/src/runtime/tool-loop.ts @@ -164,8 +164,15 @@ export async function runBrainLoop(opts: { opts.hooks?.onUsage?.(r.usage) } if (r.content) lastText = r.content - if (r.toolCalls.length === 0) + if (r.toolCalls.length === 0) { + // The stopping reply is part of the conversation (the contract on `messages`: seed + + // every assistant/tool turn). Without this push, a shot that answers WITHOUT tools + // returns a transcript missing its own answer — depth continuation criticizes a + // solution the model can't see, and candidate extraction (structural-rollout's + // fenced-code fallback) reads an empty conversation on non-tool-calling models. + if (r.content) messages.push({ role: 'assistant', content: r.content }) return { final: lastText, turns: turn, toolCalls, toolTrace, usage, messages } + } // Record the assistant turn verbatim (content + the tool_calls it requested), then run each // call and fold the result back as a `tool` message. diff --git a/src/runtime/types.ts b/src/runtime/types.ts index 15d90073..6888fcc5 100644 --- a/src/runtime/types.ts +++ b/src/runtime/types.ts @@ -115,10 +115,8 @@ export interface OutputAdapter { parse(events: SandboxEvent[]): Output } -/** LLM token usage. Structurally matches agent-eval's `RunTokenUsage` / - * `CampaignTokenUsage` ({ input, output }) so a loop result maps straight - * onto `ctx.cost.observeTokens` in a `runProfileMatrix` dispatch — without - * which the backend-integrity guard reads the run as a stub. */ +/** LLM token usage. Structurally maps into agent-eval's paid-call receipt so a + * campaign dispatch settles real usage instead of appearing as a stub. */ export interface LoopTokenUsage { input: number output: number @@ -290,9 +288,8 @@ export interface LoopResult { durationMs: number /** Sum of every iteration's `costUsd`. */ costUsd: number - /** Sum of every iteration's token usage. Forward to - * `ctx.cost.observeTokens` in a `runProfileMatrix` dispatch so the - * integrity guard sees real LLM activity. */ + /** Sum of every iteration's token usage. `loopDispatch` commits it through + * the campaign's paid-call receipt. */ tokenUsage: LoopTokenUsage /** Domain-free run provenance for auditability: the mount manifest recorded * during `prepareBox` and the selection receipts for how the winner was diff --git a/tests/agent.test.ts b/tests/agent.test.ts index bca2e3f5..d1a52751 100644 --- a/tests/agent.test.ts +++ b/tests/agent.test.ts @@ -167,6 +167,14 @@ describe('resolveSubjectPath', () => { knowledge: '.agent-knowledge', personas: 'personas', rag: 'rag', + skills: 'skills', + mcp: 'mcp', + hooks: 'hooks', + subagents: 'agents', + workflows: 'workflows', + rolloutPolicy: 'rollout-policy.json', + agentProfile: 'agent-profile.json', + code: 'src', } it('routes system-prompt subject to /
.md', () => { @@ -195,6 +203,54 @@ describe('resolveSubjectPath', () => { expect(r?.repoRelativePath).toBe('tools/list_invoices/examples.md') }) + it.each([ + [{ kind: 'skill', name: 'linear-close' } as const, 'skills/linear-close/SKILL.md'], + [ + { kind: 'mcp', server: 'linear', tool: 'update_issue' } as const, + 'mcp/linear/update_issue.md', + ], + [{ kind: 'hook', name: 'pre-dispatch' } as const, 'hooks/pre-dispatch.md'], + [{ kind: 'subagent', name: 'reviewer' } as const, 'agents/reviewer.md'], + [{ kind: 'workflow', name: 'linear-task' } as const, 'workflows/linear-task.md'], + [{ kind: 'rollout-policy', field: 'k' } as const, 'rollout-policy.json'], + [{ kind: 'agent-profile', field: 'prompt.systemPrompt' } as const, 'agent-profile.json'], + [{ kind: 'code', path: 'workers/dispatch.ts' } as const, 'src/workers/dispatch.ts'], + ])('routes the %s finding to its declared surface', (subject, expected) => { + expect(resolveSubjectPath(subject, surfaces, tmpRoot)?.repoRelativePath).toBe(expected) + }) + + it('rejects a code finding that escapes its declared source root', () => { + expect( + resolveSubjectPath({ kind: 'code', path: '../../secrets.env' }, surfaces, tmpRoot), + ).toBeNull() + }) + + it('rejects raw-knowledge and RAG findings that escape their declared roots', () => { + expect( + resolveSubjectPath({ kind: 'knowledge.raw', sourceId: '../../secrets' }, surfaces, tmpRoot), + ).toBeNull() + expect( + resolveSubjectPath({ kind: 'rag', corpus: 'irs', docId: '../../secrets' }, surfaces, tmpRoot), + ).toBeNull() + }) + + it('rejects traversal in every profile surface derived from findings', () => { + const escaped = [ + { kind: 'skill', name: '../secrets' } as const, + { kind: 'tool-doc', tool: '../secrets' } as const, + { kind: 'new-tool', name: '../secrets' } as const, + { kind: 'mcp', server: '../secrets' } as const, + { kind: 'mcp', server: 'safe', tool: '../../secrets' } as const, + { kind: 'hook', name: '../secrets' } as const, + { kind: 'subagent', name: '../secrets' } as const, + { kind: 'workflow', name: '../secrets' } as const, + { kind: 'rag', corpus: '../../secrets', docId: 'entry' } as const, + ] + for (const subject of escaped) { + expect(resolveSubjectPath(subject, surfaces, tmpRoot)).toBeNull() + } + }) + it('returns null when subject targets an undeclared optional surface', () => { const noRag = { ...surfaces, rag: undefined } const r = resolveSubjectPath({ kind: 'rag', corpus: 'irs', docId: 'foo' }, noRag, tmpRoot) @@ -576,6 +632,25 @@ describe('validateSurfaces', () => { expect(flagged).toHaveLength(1) expect(flagged[0]!.surface).toBe('rag') }) + + it('distinguishes files from directories instead of treating existence as valid', () => { + writeFileSync(join(tmpRoot, 'not-a-directory'), 'file\n') + mkdirSync(join(tmpRoot, 'not-a-file')) + const issues = validateSurfaces( + { + systemPrompt: 'prompts', + tools: 'not-a-directory', + rubric: 'not-a-file', + knowledge: '.agent-knowledge', + personas: 'personas', + }, + tmpRoot, + ) + expect(issues).toEqual([ + { surface: 'tools', path: 'not-a-directory', reason: 'not-directory' }, + { surface: 'rubric', path: 'not-a-file', reason: 'not-file' }, + ]) + }) }) describe('AgentRunInvocation streaming contract', () => { diff --git a/tests/agentic-generator.test.ts b/tests/agentic-generator.test.ts index 5b07771c..6c85d9e3 100644 --- a/tests/agentic-generator.test.ts +++ b/tests/agentic-generator.test.ts @@ -1,11 +1,19 @@ import { execFileSync } from 'node:child_process' -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createHash } from 'node:crypto' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import type { AnalystFinding } from '@tangle-network/agent-eval' +import { type AnalystFinding, CostLedger } from '@tangle-network/agent-eval' import { gitWorktreeAdapter, type ProposeContext } from '@tangle-network/agent-eval/campaign' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { agenticGenerator, commandVerifier } from '../src/improvement/agentic-generator' +import { + AGENTIC_PROFILE_RESOURCE_ROOT, + type AgenticGeneratorShotDisposition, + type AgenticGeneratorShotExecution, + type AgenticGeneratorShotReceipt, + agenticGenerator, + commandVerifier, +} from '../src/improvement' import { improvementDriver } from '../src/improvement/improvement-driver' import type { LocalHarnessResult } from '../src/mcp/local-harness' @@ -70,6 +78,26 @@ const HARNESS_OK: LocalHarnessResult = { timedOut: false, } +const CODEX_USAGE = { + inputTokens: 120, + cachedInputTokens: 20, + outputTokens: 30, + reasoningOutputTokens: 10, +} + +const CODEX_EVIDENCE = { + cliVersion: 'codex-cli 1.0.0', + executableSha256: 'a'.repeat(64), + requestedPromptSha256: 'b'.repeat(64), + effectivePromptSha256: 'b'.repeat(64), + nonPromptArgsSha256: 'c'.repeat(64), + controlledConfigSha256: 'd'.repeat(64), + readDeniedPaths: ['/tmp/denied'], + readDeniedPathsSha256: 'e'.repeat(64), + readDeniedPathCount: 1, + policy: {}, +} as NonNullable + function ctx(findings: AnalystFinding[], maxShots = 1): ProposeContext { return { currentSurface: '', @@ -83,16 +111,619 @@ function ctx(findings: AnalystFinding[], maxShots = 1): ProposeContext { + it('pins the author profile and emits exact usage for every reproducible Codex shot', async () => { + const receipts: AgenticGeneratorShotReceipt[] = [] + const costLedger = new CostLedger() + const resourcePath = `${AGENTIC_PROFILE_RESOURCE_ROOT}/trace-analysis.md` + const profile = { + name: 'structural-author', + prompt: { + systemPrompt: 'AUTHOR SYSTEM', + instructions: ['Edit only the allowed implementation.'], + }, + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' as const }, + resources: { + files: [ + { + path: resourcePath, + resource: { + kind: 'inline' as const, + name: 'trace-analysis', + content: 'Repeated state reads caused stale edits.\n', + }, + }, + ], + }, + } + const runHarness = vi.fn( + async (options: { + cwd: string + taskPrompt: string + invocation?: { command?: string; args: ReadonlyArray } + codexReproducible?: boolean + codexReadDeniedPaths?: ReadonlyArray + }) => { + expect(options.codexReproducible).toBe(true) + expect(options.invocation?.command).toBe('codex') + expect(options.invocation?.args).toContain('gpt-5.4') + expect(options.invocation?.args.join('\n')).toContain('AUTHOR SYSTEM') + expect(options.taskPrompt).toContain(resourcePath) + expect(readFileSync(join(options.cwd, resourcePath), 'utf8')).toContain('stale edits') + expect(options.codexReadDeniedPaths).toEqual([`${options.cwd}/private-evidence`]) + writeFileSync(join(options.cwd, 'app.ts'), 'export const x = 2\n') + const prompt = options.invocation?.args[1] + if (!prompt) throw new Error('test invocation omitted the composed prompt') + const promptSha256 = createHash('sha256').update(prompt).digest('hex') + return { + ...HARNESS_OK, + usage: CODEX_USAGE, + evidence: { + ...CODEX_EVIDENCE, + requestedPromptSha256: promptSha256, + effectivePromptSha256: 'f'.repeat(64), + }, + } + }, + ) + const gen = agenticGenerator({ + harness: 'codex', + profile, + codexReproducible: true, + codexReadDeniedPaths: (worktreePath) => [`${worktreePath}/private-evidence`], + onShotCompleted: (receipt) => receipts.push(receipt), + runHarness: runHarness as never, + }) + + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'reproducible-receipt', + }) + const out = await gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 2, + signal: new AbortController().signal, + generation: 4, + candidateIndex: 2, + costLedger, + costPhase: 'search.proposal', + }) + + const [costReceipt] = costLedger.list() + expect(out.applied).toBe(true) + expect(receipts).toHaveLength(1) + expect(receipts[0]).toMatchObject({ + schemaVersion: 1, + generation: 4, + candidateIndex: 2, + shot: 1, + maxShots: 2, + harness: 'codex', + model: 'gpt-5.4', + reasoningEffort: 'xhigh', + usage: CODEX_USAGE, + profileWorkspacePlanDigest: expect.any(String), + profileWorkspaceFileCount: 1, + costCallId: expect.any(String), + costBasis: 'estimated-pricing', + costUsdKnown: false, + error: null, + }) + expect(receipts[0]?.costCallId).toBe(costReceipt?.callId) + expect(receipts[0]?.costUsd).toBeCloseTo(0.00045) + expect(receipts[0]?.promptSha256).toMatch(/^sha256:[a-f0-9]{64}$/) + expect(receipts[0]?.stdoutSha256).toBe( + `sha256:${createHash('sha256').update(HARNESS_OK.stdout).digest('hex')}`, + ) + expect(receipts[0]?.evidence?.readDeniedPathCount).toBe(1) + expect(receipts[0]?.evidence?.effectivePromptSha256).not.toBe( + receipts[0]?.evidence?.requestedPromptSha256, + ) + expect(existsSync(join(wt.path, AGENTIC_PROFILE_RESOURCE_ROOT))).toBe(false) + expect(git(['status', '--short'], wt.path)).toBe('M app.ts') + expect(costLedger.list()).toEqual([ + expect.objectContaining({ + channel: 'driver', + phase: 'search.proposal', + actor: 'agentic-generator:codex', + model: 'gpt-5.4', + tags: { generation: '4', candidateIndex: '2', shot: '1' }, + inputTokens: 100, + cachedTokens: 20, + outputTokens: 30, + costUsd: 0.00045, + costUnknown: false, + }), + ]) + }) + + it('exposes exact execution before a clean-tree shot is rejected', async () => { + const events: string[] = [] + const executions: AgenticGeneratorShotExecution[] = [] + const dispositions: AgenticGeneratorShotDisposition[] = [] + const exactExecution: LocalHarnessResult = { + exitCode: 0, + stdout: 'exact stdout\nwith a second line\n', + stderr: 'exact stderr\n', + killedBySignal: null, + durationMs: 4321, + timedOut: false, + usage: CODEX_USAGE, + evidence: CODEX_EVIDENCE, + } + const isDirty = vi.fn(() => { + events.push('dirty-check') + expect(events).toEqual(['callback-complete', 'dirty-check']) + return false + }) + const gen = agenticGenerator({ + runHarness: (async () => exactExecution) as never, + isDirty, + onShotCompleted: async (receipt, execution) => { + await Promise.resolve() + expect(execution).not.toBeNull() + if (!execution) throw new Error('expected a completed execution') + executions.push(execution) + expect(execution).toEqual(exactExecution) + expect(Object.isFrozen(execution)).toBe(true) + expect(Object.isFrozen(execution.usage)).toBe(true) + expect(Object.isFrozen(execution.evidence)).toBe(true) + expect(Object.isFrozen(execution.evidence?.readDeniedPaths)).toBe(true) + expect(Object.isFrozen(execution.evidence?.policy)).toBe(true) + expect(() => Object.assign(execution, { stdout: 'mutated' })).toThrow() + expect(receipt.stdoutBytes).toBe(Buffer.byteLength(execution.stdout)) + expect(receipt.stderrBytes).toBe(Buffer.byteLength(execution.stderr)) + expect(receipt.stdoutSha256).toBe( + `sha256:${createHash('sha256').update(execution.stdout).digest('hex')}`, + ) + expect(receipt.stderrSha256).toBe( + `sha256:${createHash('sha256').update(execution.stderr).digest('hex')}`, + ) + events.push('callback-complete') + }, + onShotDisposition: async (_receipt, disposition) => { + await Promise.resolve() + expect(disposition.kind).toBe('clean') + expect(disposition.worktreePath).toContain('exact-clean-shot') + dispositions.push(disposition) + events.push('disposition-complete') + }, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'exact-clean-shot', + }) + + const out = await gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }) + + expect(out.applied).toBe(false) + expect(executions).toEqual([exactExecution]) + expect(dispositions).toEqual([ + expect.objectContaining({ kind: 'clean', worktreePath: wt.path }), + ]) + expect(events).toEqual(['callback-complete', 'dirty-check', 'disposition-complete']) + expect(isDirty).toHaveBeenCalledTimes(1) + }) + + it('awaits shot evidence before verification can return a candidate', async () => { + let evidencePersisted = false + let dispositionPersisted = false + const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { + writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') + return HARNESS_OK + }) + const verify = vi.fn(() => { + expect(evidencePersisted).toBe(true) + return { ok: true } + }) + const gen = agenticGenerator({ + runHarness: runHarness as never, + onShotCompleted: async () => { + await Promise.resolve() + evidencePersisted = true + }, + onShotDisposition: async (_receipt, disposition) => { + await Promise.resolve() + expect(disposition).toEqual({ + kind: 'accepted', + worktreePath: expect.stringContaining('evidence-before-verify'), + verified: true, + }) + dispositionPersisted = true + }, + verify, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'evidence-before-verify', + }) + + const out = await gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }) + + expect(out.applied).toBe(true) + expect(dispositionPersisted).toBe(true) + expect(verify).toHaveBeenCalledTimes(1) + }) + + it('fails closed when shot evidence persistence throws', async () => { + const isDirty = vi.fn(() => false) + const gen = agenticGenerator({ + runHarness: (async () => HARNESS_OK) as never, + isDirty, + onShotCompleted: () => { + throw new Error('shot evidence persistence failed') + }, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'evidence-failure', + }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow('shot evidence persistence failed') + expect(isDirty).not.toHaveBeenCalled() + }) + + it('fails closed when worktree disposition persistence throws', async () => { + const gen = agenticGenerator({ + runHarness: (async () => HARNESS_OK) as never, + isDirty: () => false, + onShotDisposition: () => { + throw new Error('shot disposition persistence failed') + }, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'disposition-failure', + }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow('shot disposition persistence failed') + }) + + it('persists a setup-error disposition before rethrowing inspection failure', async () => { + const dispositions: AgenticGeneratorShotDisposition[] = [] + const gen = agenticGenerator({ + runHarness: (async () => HARNESS_OK) as never, + isDirty: () => { + throw new Error('git status unavailable') + }, + onShotDisposition: (_receipt, disposition) => { + dispositions.push(disposition) + }, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'inspection-error', + }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow('git status unavailable') + expect(dispositions).toEqual([ + { + kind: 'setup-error', + worktreePath: wt.path, + stage: 'worktree-inspection', + error: { name: 'Error', message: 'git status unavailable' }, + }, + ]) + }) + + it('rejects profile files outside the dedicated ephemeral root before dispatch', () => { + expect(() => + agenticGenerator({ + harness: 'codex', + profile: { + name: 'author', + model: { default: 'gpt-5.4' }, + resources: { + files: [ + { + path: 'research.md', + resource: { kind: 'inline', name: 'research', content: 'private evidence' }, + }, + ], + }, + }, + codexReproducible: true, + }), + ).toThrow(new RegExp(`must be below ${AGENTIC_PROFILE_RESOURCE_ROOT}`)) + }) + + it('emits a failed shot receipt before rethrowing a harness failure', async () => { + const receipts: unknown[] = [] + const executions: Array = [] + const gen = agenticGenerator({ + runHarness: (async () => { + throw new Error('author process failed') + }) as never, + onShotCompleted: (receipt, execution) => { + receipts.push(receipt) + executions.push(execution) + }, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'failed' }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow('author process failed') + expect(receipts).toHaveLength(1) + expect(executions).toEqual([null]) + expect(receipts[0]).toMatchObject({ + usage: null, + evidence: null, + error: { name: 'Error', message: 'author process failed' }, + }) + }) + + it('fails closed when reproducible Codex completes without token usage', async () => { + const receipts: unknown[] = [] + const costLedger = new CostLedger() + const gen = agenticGenerator({ + harness: 'codex', + profile: { + name: 'author', + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + }, + codexReproducible: true, + runHarness: (async ({ cwd }: { cwd: string }) => { + writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') + return { ...HARNESS_OK, evidence: CODEX_EVIDENCE } + }) as never, + onShotCompleted: (receipt) => receipts.push(receipt), + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'no-usage' }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + costLedger, + }), + ).rejects.toThrow(/without usage or execution evidence/) + expect(receipts).toHaveLength(1) + expect(receipts[0]).toMatchObject({ + usage: null, + costCallId: expect.any(String), + costBasis: 'unknown', + costUsd: null, + costUsdKnown: false, + error: { message: expect.stringMatching(/without usage or execution evidence/) }, + }) + expect(costLedger.list()).toEqual([ + expect.objectContaining({ + callId: expect.any(String), + usageUnknown: true, + costUnknown: true, + }), + ]) + }) + + it('refuses reproducible Codex before dispatch when the run-wide ledger is absent', async () => { + const runHarness = vi.fn() + const gen = agenticGenerator({ + harness: 'codex', + profile: { + name: 'author', + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + }, + codexReproducible: true, + runHarness: runHarness as never, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'no-ledger', + }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow(/requires the run-wide CostLedger/) + expect(runHarness).not.toHaveBeenCalled() + }) + + it('lets a capped ledger reject an unbounded author shot before model dispatch', async () => { + const receipts: AgenticGeneratorShotReceipt[] = [] + const costLedger = new CostLedger({ costCeilingUsd: 1 }) + const runHarness = vi.fn() + const gen = agenticGenerator({ + harness: 'codex', + profile: { + name: 'author', + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + }, + codexReproducible: true, + runHarness: runHarness as never, + onShotCompleted: (receipt) => receipts.push(receipt), + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'capped-without-maximum', + }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + costLedger, + }), + ).rejects.toThrow(/hard maximumCharge before execution/) + expect(runHarness).not.toHaveBeenCalled() + expect(costLedger.list()).toHaveLength(0) + expect(receipts).toEqual([ + expect.objectContaining({ + costCallId: expect.any(String), + costBasis: 'unknown', + costUsd: null, + costUsdKnown: false, + usage: null, + error: expect.objectContaining({ + message: expect.stringMatching(/hard maximumCharge before execution/), + }), + }), + ]) + }) + + it('records terminal usage and rejects partial edits from a failed author process', async () => { + const receipts: AgenticGeneratorShotReceipt[] = [] + const costLedger = new CostLedger() + const runHarness = vi.fn( + async (options: { cwd: string; invocation?: { args: ReadonlyArray } }) => { + expect( + readFileSync(join(options.cwd, AGENTIC_PROFILE_RESOURCE_ROOT, 'failure.md'), 'utf8'), + ).toBe('failure context\n') + writeFileSync(join(options.cwd, 'app.ts'), 'export const partial = true\n') + const prompt = options.invocation?.args[1] + if (!prompt) throw new Error('test invocation omitted the composed prompt') + return { + ...HARNESS_OK, + exitCode: 2, + usage: CODEX_USAGE, + evidence: { + ...CODEX_EVIDENCE, + requestedPromptSha256: createHash('sha256').update(prompt).digest('hex'), + }, + } + }, + ) + const gen = agenticGenerator({ + harness: 'codex', + profile: { + name: 'author', + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + resources: { + files: [ + { + path: `${AGENTIC_PROFILE_RESOURCE_ROOT}/failure.md`, + resource: { kind: 'inline', name: 'failure', content: 'failure context\n' }, + }, + ], + }, + }, + codexReproducible: true, + runHarness: runHarness as never, + onShotCompleted: (receipt) => receipts.push(receipt), + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'failed-partial-edit', + }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + generation: 1, + candidateIndex: 0, + costLedger, + }), + ).rejects.toThrow(/exited with code 2/) + expect(existsSync(join(wt.path, AGENTIC_PROFILE_RESOURCE_ROOT))).toBe(false) + expect(git(['status', '--short'], wt.path)).toBe('M app.ts') + expect(costLedger.list()).toEqual([ + expect.objectContaining({ + callId: expect.any(String), + inputTokens: 100, + cachedTokens: 20, + outputTokens: 30, + costUnknown: false, + }), + ]) + expect(receipts).toEqual([ + expect.objectContaining({ + exitCode: 2, + costCallId: expect.any(String), + costBasis: 'estimated-pricing', + costUsdKnown: false, + usage: CODEX_USAGE, + error: expect.objectContaining({ + message: 'agenticGenerator: author shot exited with code 2', + }), + }), + ]) + }) + it('returns applied when the harness changes the worktree', async () => { + const dispositions: AgenticGeneratorShotDisposition[] = [] // The harness "edits" by writing into its cwd (the worktree). We stub the // subprocess (the only process boundary) but use a REAL git dirty check. - const runHarness = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { - expect(taskPrompt).toContain('x should be 2') - expect(taskPrompt).toContain('set x to 2') - writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') - return HARNESS_OK + const runHarness = vi.fn( + async ({ + cwd, + taskPrompt, + dangerouslySkipPermissions, + }: { + cwd: string + taskPrompt: string + dangerouslySkipPermissions?: boolean + }) => { + expect(taskPrompt).toContain('x should be 2') + expect(taskPrompt).toContain('set x to 2') + expect(dangerouslySkipPermissions).toBe(true) + writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') + return HARNESS_OK + }, + ) + const gen = agenticGenerator({ + runHarness: runHarness as never, + onShotDisposition: (_receipt, disposition) => { + dispositions.push(disposition) + }, }) - const gen = agenticGenerator({ runHarness: runHarness as never }) const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'cand' }) const out = await gen.generate({ @@ -106,11 +737,18 @@ describe('agenticGenerator — runs a harness in the worktree', () => { expect(runHarness).toHaveBeenCalledTimes(1) expect(out.applied).toBe(true) expect(out.summary).toContain('x should be 2') + expect(dispositions).toEqual([{ kind: 'accepted', worktreePath: wt.path, verified: false }]) }) it('retries up to maxShots when the harness produces no change, then gives up', async () => { + const dispositions: AgenticGeneratorShotDisposition[] = [] const runHarness = vi.fn(async () => HARNESS_OK) // never edits the worktree - const gen = agenticGenerator({ runHarness: runHarness as never }) + const gen = agenticGenerator({ + runHarness: runHarness as never, + onShotDisposition: (_receipt, disposition) => { + dispositions.push(disposition) + }, + }) const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'noop' }) const out = await gen.generate({ @@ -123,6 +761,11 @@ describe('agenticGenerator — runs a harness in the worktree', () => { expect(runHarness).toHaveBeenCalledTimes(3) expect(out.applied).toBe(false) + expect(dispositions).toEqual([ + { kind: 'clean', worktreePath: wt.path }, + { kind: 'clean', worktreePath: wt.path }, + { kind: 'clean', worktreePath: wt.path }, + ]) }) it('stops retrying as soon as a shot produces a change', async () => { @@ -212,6 +855,7 @@ describe('agenticGenerator — verify-in-session loop', () => { it('feeds the verifier failure into the next shot, then ships when it passes', async () => { let shot = 0 const prompts: string[] = [] + const dispositions: AgenticGeneratorShotDisposition[] = [] const runHarness = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { prompts.push(taskPrompt) shot++ @@ -222,7 +866,13 @@ describe('agenticGenerator — verify-in-session loop', () => { const verify = vi.fn(() => shot === 1 ? { ok: false, feedback: 'TS2322: x must be 2' } : { ok: true }, ) - const gen = agenticGenerator({ runHarness: runHarness as never, verify }) + const gen = agenticGenerator({ + runHarness: runHarness as never, + verify, + onShotDisposition: (_receipt, disposition) => { + dispositions.push(disposition) + }, + }) const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'vresume' }) const out = await gen.generate({ @@ -240,6 +890,15 @@ describe('agenticGenerator — verify-in-session loop', () => { expect(prompts[1]).toContain('TS2322: x must be 2') // The first shot's prompt is the clean base — no failure note yet. expect(prompts[0]).not.toContain('verification FAILED') + expect(dispositions).toEqual([ + { + kind: 'rejected', + worktreePath: wt.path, + stage: 'verification', + feedback: 'TS2322: x must be 2', + }, + { kind: 'accepted', worktreePath: wt.path, verified: true }, + ]) }) it('discards (applied:false) a candidate that never verifies within maxShots', async () => { @@ -290,12 +949,18 @@ describe('agenticGenerator — raw-trace evidence discipline', () => { it('retries and discards a raw-trace candidate that edits code without citing inspected traces', async () => { const prompts: string[] = [] + const dispositions: AgenticGeneratorShotDisposition[] = [] const runHarness = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { prompts.push(taskPrompt) writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') return HARNESS_OK }) - const gen = agenticGenerator({ runHarness: runHarness as never }) + const gen = agenticGenerator({ + runHarness: runHarness as never, + onShotDisposition: (_receipt, disposition) => { + dispositions.push(disposition) + }, + }) const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'rt-miss' }) const out = await gen.generate({ @@ -311,6 +976,21 @@ describe('agenticGenerator — raw-trace evidence discipline', () => { expect(prompts[0]).toContain('Raw trace evidence requirement') expect(prompts[0]).toContain('.improve/raw-trace-diagnosis.md') expect(prompts[1]).toContain('raw-trace mode requires .improve/raw-trace-diagnosis.md') + expect(dispositions).toHaveLength(2) + expect(dispositions).toEqual([ + expect.objectContaining({ + kind: 'rejected', + worktreePath: wt.path, + stage: 'raw-trace-evidence', + feedback: expect.stringContaining('requires .improve/raw-trace-diagnosis.md'), + }), + expect.objectContaining({ + kind: 'rejected', + worktreePath: wt.path, + stage: 'raw-trace-evidence', + feedback: expect.stringContaining('requires .improve/raw-trace-diagnosis.md'), + }), + ]) }) it('rejects a raw-trace candidate that only writes the diagnosis artifact', async () => { diff --git a/tests/candidate-bundle-builder.test.ts b/tests/candidate-bundle-builder.test.ts new file mode 100644 index 00000000..985bc02c --- /dev/null +++ b/tests/candidate-bundle-builder.test.ts @@ -0,0 +1,390 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { canonicalJson } from '@tangle-network/agent-eval' +import { gitWorktreeAdapter } from '@tangle-network/agent-eval/campaign' +import type { + AgentCandidateArtifactRef, + AgentCandidateBundle, + AgentProfile, + AgentProfileDiff, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { afterEach, describe, expect, expectTypeOf, it } from 'vitest' + +import { + type BuildAgentCandidateBundleInput, + buildAgentCandidateBundle, + sealAgentCandidateBundle, + verifyAgentCandidateBundle, +} from '../src/index' +import { + candidateSha, + cleanupCandidateFixtures, + createCandidateExecutionFixture, +} from './helpers/candidate-execution-fixture' + +const temporaryRoots: string[] = [] + +afterEach(() => { + cleanupCandidateFixtures() + for (const root of temporaryRoots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +describe('public agent candidate bundle builder', () => { + it('binds profile diffs, verified CodeSurface bytes, knowledge, and memory into one executable candidate', async () => { + const fixture = createCandidateExecutionFixture(true) + const adapter = gitWorktreeAdapter({ + repoRoot: fixture.task.stagingRoots.taskRoot, + worktreeDir: temporaryRoot('candidate-builder-worktrees-'), + branchPrefix: 'candidate-builder-test', + }) + const worktree = await adapter.create({ baseRef: 'HEAD', label: 'candidate' }) + writeFileSync(join(worktree.path, 'source.ts'), 'export const value = 2\n') + const surface = await adapter.finalize(worktree, 'feat: candidate implementation') + + const base: AgentProfile = { + name: 'candidate', + prompt: { instructions: ['Inspect the repository, implement the fix, and run tests.'] }, + model: { default: 'provider/model', reasoningEffort: 'high' }, + harness: 'codex', + resources: { failOnError: true }, + } + const diff: AgentProfileDiff = { + schemaVersion: 1, + kind: 'agent-profile-diff', + id: 'add-review-skill', + source: { kind: 'optimizer', artifacts: ['trace:run-1'] }, + set: { + resources: { + failOnError: true, + skills: [ + { + kind: 'inline', + name: 'review/SKILL.md', + content: 'Read the failing test before editing.\n', + }, + ], + }, + }, + } + const stored = new Map() + const knowledgeManifest = storeArtifact(stored, 'knowledge/manifest.json', '{"version":1}\n') + const memorySeed = storeArtifact(stored, 'memory/seed.json', '{"entries":[]}\n') + const input: BuildAgentCandidateBundleInput = { + profile: { kind: 'profile-diffs', base, diffs: [diff] }, + code: { + kind: 'code-surface', + surface, + repository: { kind: 'github', owner: 'owner', repo: 'repo' }, + }, + execution: fixture.bundle.execution, + knowledge: { snapshotId: 'knowledge-snapshot-1', manifest: knowledgeManifest }, + memory: { mode: 'isolated', scope: 'task', seed: memorySeed }, + lineage: { + source: 'compound', + parentDigests: [ + fixture.bundle.lineage.parentDigests?.[0] ?? candidateSha('e'), + candidateSha('9'), + ], + runIds: ['optimizer-run-1'], + benchmark: fixture.bundle.lineage.benchmark, + spend: fixture.bundle.lineage.spend, + }, + } + + try { + const first = buildAgentCandidateBundle(input) + const second = buildAgentCandidateBundle(input) + expectTypeOf(first).toEqualTypeOf() + expect(first.digest).toBe(second.digest) + expect(Object.isFrozen(first)).toBe(true) + expect(first.code).toMatchObject({ + kind: 'git-patch', + baseCommit: surface.baseCommit, + baseTree: surface.baseTree, + candidateTree: surface.candidateTree, + patch: { + format: 'git-diff-binary', + artifact: { + sha256: surface.patch.sha256, + byteLength: surface.patch.byteLength, + }, + }, + }) + expect(first.profile.resources?.skills?.[0]).toMatchObject({ + kind: 'inline', + name: 'review/SKILL.md', + sha256: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), + }) + expect(first.lineage.profileDiffIds).toEqual([ + `sha256:${createHash('sha256').update(canonicalJson(diff)).digest('hex')}`, + ]) + + const verified = await verifyAgentCandidateBundle(first, { + repositories: fixture.ports.repositories, + artifacts: { + read: async (ref) => { + const bytes = stored.get(ref.sha256) + if (!bytes) throw new Error(`missing test artifact ${ref.sha256}`) + return Uint8Array.from(bytes) + }, + }, + }) + expect(verified.bundle.digest).toBe(first.digest) + expect(verified.materializedTree).toBe(surface.candidateTree) + } finally { + await adapter.discard(worktree) + } + }) + + it('rejects CodeSurface drift before sealing and exports the low-level sealer', async () => { + const fixture = createCandidateExecutionFixture(true) + const adapter = gitWorktreeAdapter({ + repoRoot: fixture.task.stagingRoots.taskRoot, + worktreeDir: temporaryRoot('candidate-builder-drift-'), + branchPrefix: 'candidate-builder-drift', + }) + const worktree = await adapter.create({ baseRef: 'HEAD', label: 'drift' }) + writeFileSync(join(worktree.path, 'source.ts'), 'export const value = 3\n') + const surface = await adapter.finalize(worktree, 'feat: drift candidate') + try { + expect(() => + buildAgentCandidateBundle({ + profile: { kind: 'profile', profile: simpleProfile() }, + code: { + kind: 'code-surface', + surface: { + ...surface, + patch: { ...surface.patch, sha256: `sha256:${'0'.repeat(64)}` }, + }, + repository: { kind: 'github', owner: 'owner', repo: 'repo' }, + }, + execution: fixture.bundle.execution, + memory: { mode: 'disabled' }, + lineage: { source: 'optimizer' }, + }), + ).toThrow(/CodeSurface|patch/i) + + const { digest: _digest, ...withoutDigest } = fixture.bundle + expect(sealAgentCandidateBundle(withoutDigest).digest).toMatch(/^sha256:[a-f0-9]{64}$/) + } finally { + await adapter.discard(worktree) + } + }) + + it('fails closed when a generic profile would lose behavior or byte identity', () => { + const fixture = createCandidateExecutionFixture(false) + const base = { + code: { kind: 'disabled', reason: 'control' } as const, + execution: fixture.bundle.execution, + memory: { mode: 'disabled' } as const, + lineage: { source: 'human' } as const, + } + expect(() => + buildAgentCandidateBundle({ + ...base, + profile: { + kind: 'profile', + profile: { ...simpleProfile(), metadata: { silentlyDropped: true } }, + }, + }), + ).toThrow(/metadata.*not representable/) + expect(() => + buildAgentCandidateBundle({ + ...base, + profile: { + kind: 'profile', + profile: { + ...simpleProfile(), + resources: { + failOnError: true, + skills: [ + { + kind: 'github', + repository: 'owner/repo', + ref: 'f'.repeat(40), + path: 'skills/review/SKILL.md', + }, + ], + }, + }, + }, + }), + ).toThrow(/GitHub profile resources do not carry byte identity/) + expect(() => + buildAgentCandidateBundle({ + ...base, + profile: { + kind: 'profile-diffs', + base: simpleProfile(), + diffs: [], + }, + }), + ).toThrow(/at least one AgentProfileDiff/) + expect(() => + buildAgentCandidateBundle({ + ...base, + profile: { kind: 'profile', profile: simpleProfile() }, + code: { kind: 'git-patch' } as unknown as BuildAgentCandidateBundleInput['code'], + }), + ).toThrow(/unsupported candidate code source/) + expect(() => + buildAgentCandidateBundle({ + ...base, + profile: { kind: 'profile', profile: simpleProfile() }, + lineage: { + source: 'human', + profileDiffIds: ['caller-controlled'], + } as unknown as BuildAgentCandidateBundleInput['lineage'], + }), + ).toThrow(/profileDiffIds.*derived/) + }) + + it('round-trips every currently representable generic profile surface', () => { + const fixture = createCandidateExecutionFixture(false) + const profile: AgentProfile = { + name: 'complete-candidate', + description: 'Exercises every closed profile surface.', + version: '1.0.0', + tags: ['coding', 'review'], + prompt: { + systemPrompt: 'Solve the task.', + instructions: ['Read the tests first.'], + }, + model: { + default: 'provider/model', + small: 'provider/model', + provider: 'provider', + reasoningEffort: 'high', + }, + harness: 'codex', + permissions: { shell: 'deny', files: { read: 'allow', write: 'ask' } }, + tools: { shell: false, editor: true }, + mcp: { + review: { + transport: 'stdio', + command: 'node', + args: ['scripts/review.mjs'], + env: { MODE: 'check' }, + cwd: 'tools', + enabled: true, + }, + }, + subagents: { + reviewer: { + description: 'Reviews the candidate.', + prompt: 'Find correctness defects.', + model: 'provider/model', + tools: { editor: true }, + permissions: { shell: 'deny' }, + maxSteps: 4, + }, + }, + resources: { + failOnError: true, + files: [ + { + path: 'notes/instructions.md', + resource: { kind: 'inline', name: 'instructions.md', content: 'Check invariants.\n' }, + }, + ], + tools: [{ kind: 'inline', name: 'review-tool.md', content: 'Review tool.\n' }], + skills: [{ kind: 'inline', name: 'review-skill.md', content: 'Review skill.\n' }], + agents: [{ kind: 'inline', name: 'review-agent.md', content: 'Review agent.\n' }], + commands: [{ kind: 'inline', name: 'review-command.md', content: 'Review command.\n' }], + instructions: { kind: 'inline', name: 'extra.md', content: 'Extra instructions.\n' }, + }, + hooks: { Stop: [] }, + modes: { + review: { + description: 'Review mode.', + model: 'provider/model', + prompt: 'Review only.', + tools: { editor: true }, + permissions: { shell: 'deny' }, + }, + }, + confidential: { tee: 'tdx', sealed: true }, + } + + const bundle = buildAgentCandidateBundle({ + profile: { kind: 'profile', profile }, + code: { kind: 'disabled', reason: 'control' }, + execution: fixture.bundle.execution, + memory: { mode: 'disabled' }, + lineage: { source: 'human' }, + }) + + expect(bundle.profile).toMatchObject({ + name: profile.name, + mcp: { review: { command: 'node' } }, + subagents: { reviewer: { model: 'provider/model' } }, + modes: { review: { model: 'provider/model' } }, + confidential: { tee: 'tdx', sealed: true }, + }) + }) + + it('accepts an already closed candidate profile for behavior generic profiles cannot encode', () => { + const fixture = createCandidateExecutionFixture(false) + const bundle = buildAgentCandidateBundle({ + profile: { + kind: 'candidate-profile', + profile: { + name: 'hooked-candidate', + harness: 'codex', + hooks: { + Stop: [ + { + executable: 'node', + args: [{ kind: 'public', value: 'scripts/check.mjs' }], + blocking: true, + }, + ], + }, + }, + }, + code: { kind: 'disabled', reason: 'control' }, + execution: fixture.bundle.execution, + memory: { mode: 'disabled' }, + lineage: { source: 'human' }, + }) + expect(bundle.profile.hooks?.Stop?.[0]).toEqual({ + executable: 'node', + args: [{ kind: 'public', value: 'scripts/check.mjs' }], + blocking: true, + }) + }) +}) + +function simpleProfile(): AgentProfile { + return { + name: 'candidate', + prompt: { instructions: ['Solve the task and run its tests.'] }, + model: { default: 'provider/model', reasoningEffort: 'high' }, + harness: 'codex', + } +} + +function temporaryRoot(prefix: string): string { + const root = mkdtempSync(join(tmpdir(), prefix)) + temporaryRoots.push(root) + return root +} + +function storeArtifact( + store: Map, + key: string, + value: string, +): AgentCandidateArtifactRef { + const bytes = Buffer.from(value, 'utf8') + const sha256 = `sha256:${createHash('sha256').update(bytes).digest('hex')}` as Sha256Digest + store.set(sha256, Uint8Array.from(bytes)) + return { + locator: { kind: 's3', bucket: 'candidate-test-artifacts', key }, + sha256, + byteLength: bytes.byteLength, + } +} diff --git a/tests/candidate-execution-claim.test.ts b/tests/candidate-execution-claim.test.ts new file mode 100644 index 00000000..0f139045 --- /dev/null +++ b/tests/candidate-execution-claim.test.ts @@ -0,0 +1,893 @@ +import { spawn } from 'node:child_process' +import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { resolve } from 'node:path' +import type { AgentCandidateArtifactRef } from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { + type AgentCandidateExecutionClaim, + type AgentCandidateExecutionClaimStore, + type AgentCandidateExecutionFailureClass, + type AgentCandidateExecutionLease, + type AgentCandidateExecutionRecoveryEvidence, + type AgentCandidateExecutionTerminalResult, + type AgentCandidateExecutionUsage, + InMemoryAgentCandidateExecutionClaimStore, +} from '../src/candidate-execution/claim' +import { FileAgentCandidateExecutionClaimStore } from '../src/candidate-execution/claim-file-store' +import { candidateExecutionClaim } from '../src/candidate-execution/claim-plan' +import { candidateExecutionOwnerWindowMs } from '../src/candidate-execution/execution-window' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + cleanupCandidateFixtures, + createCandidateExecutionFixture, +} from './helpers/candidate-execution-fixture' + +const temporaryDirectories: string[] = [] +const FUTURE_EXPIRY_MS = Date.now() + 60 * 60 * 1_000 + +afterEach(async () => { + vi.restoreAllMocks() + cleanupCandidateFixtures() + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ) +}) + +describe('candidate execution claim lifecycle', () => { + it('leases only the frozen execution and cleanup owner window', async () => { + const fixture = createCandidateExecutionFixture() + const cleanupTimeoutMs = 25 + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs }, + ) + const claimedAtMs = Date.now() + vi.spyOn(Date, 'now').mockReturnValue(claimedAtMs) + + const requested = candidateExecutionClaim(prepared) + const ownerWindowMs = candidateExecutionOwnerWindowMs( + fixture.task.limits.timeoutMs, + cleanupTimeoutMs, + fixture.task.limits.timeoutMs, + fixture.task.limits.timeoutMs, + ) + + expect(requested.leaseExpiresAtMs).toBe(claimedAtMs + ownerWindowMs) + expect(requested.cleanup.cleanupTimeoutMs).toBe(cleanupTimeoutMs) + expect(requested.resultTimeoutMs).toBe(fixture.task.limits.timeoutMs) + expect(ownerWindowMs).toBeLessThan(15 * 60_000) + }) + + it('rejects a preparation too near reservation expiry for the owner window', async () => { + const fixture = createCandidateExecutionFixture() + const cleanupTimeoutMs = 25 + let reservationExpiresAtMs = 0 + const reserveGrant = fixture.ports.models.reserveGrant + fixture.ports.models.reserveGrant = async (input) => { + reservationExpiresAtMs = input.expiresAtMs + return await reserveGrant(input) + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs }, + ) + const ownerWindowMs = candidateExecutionOwnerWindowMs( + fixture.task.limits.timeoutMs, + cleanupTimeoutMs, + fixture.task.limits.timeoutMs, + fixture.task.limits.timeoutMs, + ) + vi.spyOn(Date, 'now').mockReturnValue(reservationExpiresAtMs - ownerWindowMs + 1) + + expect(() => candidateExecutionClaim(prepared)).toThrow( + /full execution and cleanup owner window/, + ) + }) + + it('lets exactly one in-process caller acquire an attempt', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const results = await Promise.all(Array.from({ length: 32 }, () => store.tryClaim(claim()))) + + expect(results.filter((result) => result.acquired)).toHaveLength(1) + expect(results.filter((result) => !result.acquired)).toHaveLength(31) + expect( + results + .filter((result) => !result.acquired && result.reason === 'already-claimed') + .every((result) => result.exactReplay), + ).toBe(true) + }) + + it('does not call a different preparation an exact replay', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + await acquire(store, claim()) + + expect( + await store.tryClaim( + claim({ cleanup: { ...cleanupHandles(), preparationId: preparationId('q') } }), + ), + ).toMatchObject({ acquired: false, reason: 'already-claimed', exactReplay: false }) + }) + + it('requires stage before finish and the exact staged terminal digest', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + const result = failed() + + await expect(store.finish(acquired.lease, sha256('f'))).rejects.toThrow('has not been staged') + const staged = await store.stageTerminal(acquired.lease, result) + if (!staged.staged) throw new Error('test setup did not stage') + await expect(store.finish(acquired.lease, sha256('f'))).rejects.toThrow( + 'does not match staged outbox', + ) + const finished = await store.finish(acquired.lease, staged.terminal.terminalDigest) + + expect(finished).toEqual({ finished: true, terminal: staged.terminal }) + expect(finished.terminal).toEqual(staged.terminal) + }) + + it('requires the winning lease for phase, stage, and finish', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + const invalid = { ...acquired.lease, token: leaseToken('x') } + + await expect(store.markCandidateMayRun(invalid)).rejects.toThrow('lease is invalid') + await expect(store.stageTerminal(invalid, failed())).rejects.toThrow('lease is invalid') + const staged = await store.stageTerminal(acquired.lease, failed()) + await expect(store.finish(invalid, staged.terminal.terminalDigest)).rejects.toThrow( + 'lease is invalid', + ) + }) + + it('stages exactly once with exact-replay and mismatch evidence', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + const first = await store.stageTerminal(acquired.lease, failed()) + const replay = await store.stageTerminal(acquired.lease, failed()) + const mismatch = await store.stageTerminal( + acquired.lease, + failed(0, 'unknown', { failureEvidence: artifact('9') }), + ) + + expect(first).toMatchObject({ staged: true }) + expect(replay).toMatchObject({ staged: false, exactReplay: true }) + expect(mismatch).toMatchObject({ staged: false, exactReplay: false }) + }) + + it('persists every fixed usage field and durable success artifact reference', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + await store.markCandidateMayRun(acquired.lease) + const result = succeeded() + const terminal = await complete(store, acquired.lease, result) + const observed = await store.getAttempt({ executionId: 'execution-1', attempt: 1 }) + + expect(terminal.usage).toEqual(result.usage) + expect(terminal).toMatchObject({ + schemaVersion: 1, + modelSettlement: result.modelSettlement, + taskOutcome: result.taskOutcome, + benchmarkResult: result.benchmarkResult, + runReceipt: result.runReceipt, + terminalDigest: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), + }) + expect(observed).toMatchObject({ + phase: 'candidate-may-run', + staged: terminal, + terminal, + }) + }) + + it('persists the candidate-may-run phase monotonically before execution terminals', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + + expect(await store.getAttempt({ executionId: 'execution-1', attempt: 1 })).toMatchObject({ + phase: 'claimed', + }) + expect(await store.markCandidateMayRun(acquired.lease)).toEqual({ + marked: true, + phase: 'candidate-may-run', + }) + expect(await store.markCandidateMayRun(acquired.lease)).toEqual({ + marked: false, + phase: 'candidate-may-run', + }) + await expect(store.stageTerminal(acquired.lease, failed())).rejects.toThrow( + 'crossed candidate-may-run', + ) + await complete(store, acquired.lease, failed(0, 'execution')) + expect(await store.getAttempt({ executionId: 'execution-1', attempt: 1 })).toMatchObject({ + phase: 'candidate-may-run', + }) + }) + + it('refuses success unless candidate-may-run was persisted first', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + + await expect(store.stageTerminal(acquired.lease, succeeded())).rejects.toThrow( + 'requires candidate-may-run phase', + ) + }) + + it('keeps exact stage and finish replay idempotent after lease expiry', async () => { + let now = 1_000 + const store = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const acquired = await acquire(store, claim({ leaseExpiresAtMs: 1_100 })) + const staged = await store.stageTerminal(acquired.lease, failed()) + const finished = await store.finish(acquired.lease, staged.terminal.terminalDigest) + expect(finished.finished).toBe(true) + now = 1_100 + + expect(await store.stageTerminal(acquired.lease, failed())).toMatchObject({ + staged: false, + exactReplay: true, + }) + expect(await store.finish(acquired.lease, staged.terminal.terminalDigest)).toMatchObject({ + finished: false, + exactReplay: true, + }) + }) + + it('rejects new phase, stage, and finish writes after lease expiry', async () => { + let now = 2_000 + const phaseStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const phaseLease = await acquire(phaseStore, claim({ leaseExpiresAtMs: 2_100 })) + const finishStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const finishLease = await acquire(finishStore, claim({ leaseExpiresAtMs: 2_100 })) + const staged = await finishStore.stageTerminal(finishLease.lease, failed()) + now = 2_100 + + await expect(phaseStore.markCandidateMayRun(phaseLease.lease)).rejects.toThrow( + 'lease has expired', + ) + await expect(phaseStore.stageTerminal(phaseLease.lease, failed())).rejects.toThrow( + 'lease has expired', + ) + await expect( + finishStore.finish(finishLease.lease, staged.terminal.terminalDigest), + ).rejects.toThrow('lease has expired') + }) + + it.each([ + 'phase', + 'stage', + ] as const)('refuses file-backed %s publication when the lease expires during durable write preparation and lets recovery win', async (operation) => { + const directory = await tempDirectory() + const expiresAtMs = 2_600 + let now = 2_500 + let expireAfterNextRead = false + const store = new FileAgentCandidateExecutionClaimStore({ + directory, + now: () => { + const observed = now + if (expireAfterNextRead) { + expireAfterNextRead = false + now = expiresAtMs + } + return observed + }, + }) + const requested = retryClaim({ leaseExpiresAtMs: expiresAtMs }) + const acquired = await acquire(store, requested) + now = expiresAtMs - 1 + expireAfterNextRead = true + + await expect( + operation === 'phase' + ? store.markCandidateMayRun(acquired.lease) + : store.stageTerminal(acquired.lease, failed()), + ).rejects.toThrow('lease has expired') + const beforeRecovery = await store.getAttempt({ + executionId: requested.executionId, + attempt: requested.attempt, + }) + expect(beforeRecovery?.phase).toBe('claimed') + expect(beforeRecovery?.staged).toBeUndefined() + expect(beforeRecovery?.terminal).toBeUndefined() + + const recovered = await store.recoverExpired( + { executionId: requested.executionId, attempt: requested.attempt }, + recoveryEvidence(requested), + ) + expect(recovered).toMatchObject({ + finished: true, + terminal: { status: 'failed', failureClass: 'pre-model-infrastructure' }, + }) + }) + + it('refuses file-backed finish publication when the lease expires during durable write preparation and lets recovery win', async () => { + const directory = await tempDirectory() + const expiresAtMs = 2_900 + let now = 2_800 + let expireAfterNextRead = false + const store = new FileAgentCandidateExecutionClaimStore({ + directory, + now: () => { + const observed = now + if (expireAfterNextRead) { + expireAfterNextRead = false + now = expiresAtMs + } + return observed + }, + }) + const requested = retryClaim({ leaseExpiresAtMs: expiresAtMs }) + const acquired = await acquire(store, requested) + const staged = await store.stageTerminal(acquired.lease, failed()) + now = expiresAtMs - 1 + expireAfterNextRead = true + + await expect(store.finish(acquired.lease, staged.terminal.terminalDigest)).rejects.toThrow( + 'lease has expired', + ) + const beforeRecovery = await store.getAttempt({ + executionId: requested.executionId, + attempt: requested.attempt, + }) + expect(beforeRecovery?.staged).toEqual(staged.terminal) + expect(beforeRecovery?.terminal).toBeUndefined() + + const recovered = await store.recoverExpired( + { executionId: requested.executionId, attempt: requested.attempt }, + recoveryEvidence(requested), + ) + expect(recovered).toEqual({ finished: true, terminal: staged.terminal }) + }) + + it('recovers zero-call pre-model work only when candidate-may-run was never crossed', async () => { + let now = 3_000 + const store = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const requested = retryClaim({ leaseExpiresAtMs: 3_100, cleanup: cleanupHandles(true) }) + await acquire(store, requested) + now = 3_100 + + const recovered = await store.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + recoveryEvidence(requested), + ) + + expect(recovered).toMatchObject({ + finished: true, + terminal: { + status: 'failed', + failureClass: 'pre-model-infrastructure', + usage: { modelCalls: 0 }, + }, + }) + expect((await store.tryClaim(retryClaim({ attempt: 2 }))).acquired).toBe(true) + }) + + it('normalizes recovered pre-model evidence to unknown after candidate-may-run', async () => { + let now = 3_500 + const store = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const requested = retryClaim({ leaseExpiresAtMs: 3_600 }) + const acquired = await acquire(store, requested) + await store.markCandidateMayRun(acquired.lease) + now = 3_600 + + const recovered = await store.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + recoveryEvidence(requested), + ) + + expect(recovered).toMatchObject({ + terminal: { failureClass: 'unknown', usage: { modelCalls: 0 } }, + }) + expect(await store.tryClaim(retryClaim({ attempt: 2 }))).toMatchObject({ + acquired: false, + detail: 'prior-attempt-not-pre-model-infrastructure', + }) + }) + + it('promotes an exact staged terminal after a crash instead of replacing it', async () => { + let now = 4_000 + const directory = await tempDirectory() + const requested = claim({ leaseExpiresAtMs: 4_100 }) + const owner = new FileAgentCandidateExecutionClaimStore({ directory, now: () => now }) + const acquired = await acquire(owner, requested) + await owner.markCandidateMayRun(acquired.lease) + const staged = await owner.stageTerminal(acquired.lease, succeeded()) + expect( + (await owner.getAttempt({ executionId: 'execution-1', attempt: 1 }))?.terminal, + ).toBeUndefined() + now = 4_100 + + const recovery = new FileAgentCandidateExecutionClaimStore({ directory, now: () => now }) + const recovered = await recovery.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + recoveryEvidence(requested, { + failureClass: 'execution', + usage: staged.terminal.usage, + }), + ) + + expect(recovered).toEqual({ finished: true, terminal: staged.terminal }) + expect( + await recovery.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + recoveryEvidence(requested, { + failureClass: 'execution', + usage: staged.terminal.usage, + }), + ), + ).toMatchObject({ finished: false, exactReplay: true }) + }) + + it('refuses recovery whose model evidence differs from the staged outbox', async () => { + let now = 4_500 + const store = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const requested = claim({ leaseExpiresAtMs: 4_600 }) + const acquired = await acquire(store, requested) + await store.stageTerminal(acquired.lease, failed()) + now = 4_600 + + await expect( + store.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + recoveryEvidence(requested, { modelSettlement: artifact('8') }), + ), + ).rejects.toThrow('does not match staged model evidence') + }) + + it('requires exact process, model, and memory closure before recovery', async () => { + let now = 5_000 + const store = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const requested = claim({ leaseExpiresAtMs: 5_100, cleanup: cleanupHandles(true) }) + await acquire(store, requested) + const evidence = recoveryEvidence(requested) + await expect( + store.recoverExpired({ executionId: requested.executionId, attempt: 1 }, evidence), + ).rejects.toThrow('lease has not expired') + now = 5_100 + const { memory: _memory, ...withoutMemory } = evidence + + await expect( + store.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + { ...evidence, process: { ...evidence.process, stopped: false as true } }, + ), + ).rejects.toThrow('does not prove the claimed process stopped') + await expect( + store.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + { ...evidence, model: { ...evidence.model, grantDigest: sha256('f') } }, + ), + ).rejects.toThrow('does not prove the claimed model grant closed') + await expect( + store.recoverExpired({ executionId: requested.executionId, attempt: 1 }, withoutMemory), + ).rejects.toThrow('missing memory closure evidence') + }) + + it('unlocks retry only after a zero-call pre-model infrastructure terminal', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const first = await acquire(store, retryClaim()) + expect(await store.tryClaim(retryClaim({ attempt: 2 }))).toMatchObject({ + acquired: false, + detail: 'prior-attempt-running', + }) + await complete(store, first.lease, failed()) + expect((await store.tryClaim(retryClaim({ attempt: 2 }))).acquired).toBe(true) + }) + + it.each([ + 'execution', + 'post-model-infrastructure', + 'unknown', + ] as const)('rejects a zero-call %s terminal as non-retryable', async (failureClass) => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const first = await acquire(store, retryClaim()) + if (failureClass !== 'unknown') await store.markCandidateMayRun(first.lease) + await complete(store, first.lease, failed(0, failureClass)) + + expect(await store.tryClaim(retryClaim({ attempt: 2 }))).toMatchObject({ + acquired: false, + detail: 'prior-attempt-not-pre-model-infrastructure', + }) + }) + + it('rejects a pre-model terminal with paid calls', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, retryClaim()) + + await expect(store.stageTerminal(acquired.lease, failed(1))).rejects.toThrow( + 'pre-model infrastructure failure cannot contain model calls', + ) + }) + + it('rejects changed retry lineage and a missing prior attempt', async () => { + const missing = new InMemoryAgentCandidateExecutionClaimStore() + expect(await missing.tryClaim(retryClaim({ attempt: 2 }))).toMatchObject({ + acquired: false, + detail: 'prior-attempt-missing', + }) + const changed = new InMemoryAgentCandidateExecutionClaimStore() + const first = await acquire(changed, retryClaim()) + await complete(changed, first.lease, failed()) + + expect( + await changed.tryClaim(retryClaim({ attempt: 2, retryLineageDigest: sha256('f') })), + ).toMatchObject({ acquired: false, detail: 'retry-lineage-mismatch' }) + }) + + it('persists claim7, pending1, terminal3, phase, staged, and full usage across stores', async () => { + const directory = await tempDirectory() + const store = new FileAgentCandidateExecutionClaimStore({ directory }) + const acquired = await acquire(store, claim()) + await store.markCandidateMayRun(acquired.lease) + const terminal = await complete(store, acquired.lease, failed(2, 'execution')) + const reopened = new FileAgentCandidateExecutionClaimStore({ directory }) + const observed = await reopened.getAttempt({ executionId: 'execution-1', attempt: 1 }) + const files = await Promise.all( + (await readdir(directory)).map(async (name) => ({ + name, + text: await readFile(resolve(directory, name), 'utf8'), + })), + ) + + expect(observed).toMatchObject({ + phase: 'candidate-may-run', + staged: terminal, + terminal, + }) + expect(files.find(({ name }) => name.endsWith('.claim.json'))?.text).toContain('"version":7') + expect(files.find(({ name }) => name.includes('transition-2'))?.text).toContain('"version":1') + expect(files.find(({ name }) => name.endsWith('.terminal.json'))?.text).toContain('"version":3') + expect(files.map(({ text }) => text).join('\n')).not.toContain(acquired.lease.token) + }) + + it('finishes a staged file outbox after reopening without recomputing content', async () => { + const directory = await tempDirectory() + const owner = new FileAgentCandidateExecutionClaimStore({ directory }) + const acquired = await acquire(owner, claim()) + const staged = await owner.stageTerminal(acquired.lease, failed()) + const reopened = new FileAgentCandidateExecutionClaimStore({ directory }) + + const observed = await reopened.getAttempt({ executionId: 'execution-1', attempt: 1 }) + expect(observed).toMatchObject({ + staged: staged.terminal, + }) + expect(observed?.terminal).toBeUndefined() + expect(await reopened.finish(acquired.lease, staged.terminal.terminalDigest)).toEqual({ + finished: true, + terminal: staged.terminal, + }) + }) + + it('lets exactly one independent process acquire the same attempt', async () => { + const directory = await tempDirectory() + const results = await Promise.all( + Array.from({ length: 8 }, () => claimFromChildProcess(directory, claim())), + ) + + expect(results.filter((result) => result === 'acquired')).toHaveLength(1) + expect(results.filter((result) => result === 'already-claimed')).toHaveLength(7) + }) + + it('linearizes candidate-may-run across independent processes', async () => { + const directory = await tempDirectory() + const store = new FileAgentCandidateExecutionClaimStore({ directory }) + const acquired = await acquire(store, claim()) + const results = await Promise.all( + Array.from({ length: 8 }, () => markFromChildProcess(directory, acquired.lease)), + ) + + expect(results.filter((result) => result === 'marked')).toHaveLength(1) + expect(results.filter((result) => result === 'already-marked')).toHaveLength(7) + }) + + it('linearizes staging and finishing across independent processes', async () => { + const directory = await tempDirectory() + const store = new FileAgentCandidateExecutionClaimStore({ directory }) + const acquired = await acquire(store, retryClaim()) + const stageResults = await Promise.all( + Array.from({ length: 8 }, () => stageFromChildProcess(directory, acquired.lease, failed())), + ) + const record = await store.getAttempt({ executionId: 'execution-1', attempt: 1 }) + if (!record?.staged) throw new Error('cross-process stage is missing') + const finishResults = await Promise.all( + Array.from({ length: 8 }, () => + finishFromChildProcess( + directory, + acquired.lease, + record.staged?.terminalDigest ?? sha256('f'), + ), + ), + ) + + expect(stageResults.filter((result) => result === 'staged')).toHaveLength(1) + expect(stageResults.filter((result) => result === 'already-staged:exact')).toHaveLength(7) + expect(finishResults.filter((result) => result === 'finished')).toHaveLength(1) + expect(finishResults.filter((result) => result === 'already-finished:exact')).toHaveLength(7) + }, 15_000) + + it('linearizes expired recovery across independent processes', async () => { + const directory = await tempDirectory() + const requested = retryClaim({ leaseExpiresAtMs: 6_000 }) + const store = new FileAgentCandidateExecutionClaimStore({ directory, now: () => 5_900 }) + await acquire(store, requested) + const evidence = recoveryEvidence(requested) + const results = await Promise.all( + Array.from({ length: 8 }, () => + recoverFromChildProcess( + directory, + { executionId: requested.executionId, attempt: 1 }, + evidence, + 6_000, + ), + ), + ) + + expect(results.filter((result) => result === 'recovered')).toHaveLength(1) + expect(results.filter((result) => result === 'already-recovered:exact')).toHaveLength(7) + }) +}) + +function claim( + overrides: Partial = {}, +): AgentCandidateExecutionClaim { + return { + executionId: 'execution-1', + attempt: 1, + maxAttempts: 1, + retryPolicy: 'none', + bundleDigest: sha256('a'), + executionPlanDigest: sha256('b'), + retryLineageDigest: sha256('c'), + leaseExpiresAtMs: FUTURE_EXPIRY_MS, + resultTimeoutMs: 60_000, + cleanup: cleanupHandles(), + ...overrides, + } +} + +function retryClaim( + overrides: Partial = {}, +): AgentCandidateExecutionClaim { + return claim({ maxAttempts: 3, retryPolicy: 'pre-model-infrastructure-only', ...overrides }) +} + +function usage( + modelCalls = 0, + overrides: Partial = {}, +): AgentCandidateExecutionUsage { + return { + costUsdNanos: modelCalls * 123_456, + inputTokens: modelCalls * 101, + outputTokens: modelCalls * 29, + cachedInputTokens: modelCalls * 17, + reasoningTokens: modelCalls * 11, + modelCalls, + ...overrides, + } +} + +function failed( + modelCalls = 0, + failureClass: AgentCandidateExecutionFailureClass = 'pre-model-infrastructure', + overrides: Partial> = {}, +): Extract { + return { + schemaVersion: 1, + status: 'failed', + failureClass, + usage: usage(modelCalls), + modelSettlement: artifact('1'), + ...overrides, + } +} + +function succeeded(): Extract { + return { + schemaVersion: 1, + status: 'succeeded', + usage: usage(2), + modelSettlement: artifact('1'), + taskOutcome: artifact('2'), + benchmarkResult: artifact('3'), + runReceipt: artifact('4'), + } +} + +function artifact(character: string): AgentCandidateArtifactRef { + return { + locator: { kind: 's3', bucket: 'candidate-evidence', key: `${character}/artifact.json` }, + sha256: sha256(character), + byteLength: character.charCodeAt(0), + } +} + +function cleanupHandles(memory = false): AgentCandidateExecutionClaim['cleanup'] { + return { + preparationId: preparationId('p'), + modelGrantDigest: sha256('d'), + resolvedModel: { + requested: 'provider/model', + provider: 'provider', + model: 'model-snapshot', + snapshot: 'model-snapshot-2026-07-01', + reasoningEffort: 'high', + }, + traceRunId: 'execution-1:attempt-1:trace', + cleanupTimeoutMs: 30_000, + ...(memory + ? { + memory: { + accessDigest: sha256('e'), + effectiveNamespace: 'candidate/bundle/execution/task/preparation', + }, + } + : {}), + } +} + +function preparationId(character: string): string { + return `candidate-preparation-v1.${character.repeat(43)}` +} + +function recoveryEvidence( + requested: AgentCandidateExecutionClaim, + overrides: { + failureClass?: AgentCandidateExecutionFailureClass + usage?: AgentCandidateExecutionUsage + modelSettlement?: AgentCandidateArtifactRef + } = {}, +): AgentCandidateExecutionRecoveryEvidence { + return { + failureClass: overrides.failureClass ?? 'pre-model-infrastructure', + usage: overrides.usage ?? usage(), + modelSettlement: overrides.modelSettlement ?? artifact('1'), + process: { stopped: true, executionPlanDigest: requested.executionPlanDigest }, + model: { + closed: true, + preparationId: requested.cleanup.preparationId, + grantDigest: requested.cleanup.modelGrantDigest, + }, + ...(requested.cleanup.memory + ? { + memory: { + closed: true as const, + preparationId: requested.cleanup.preparationId, + accessDigest: requested.cleanup.memory.accessDigest, + effectiveNamespace: requested.cleanup.memory.effectiveNamespace, + }, + } + : {}), + } +} + +function sha256(character: string): `sha256:${string}` { + return `sha256:${character.repeat(64)}` +} + +function leaseToken(character: string): string { + return `candidate-execution-lease-v1.${character.repeat(43)}` +} + +async function acquire( + store: Pick, + requested: AgentCandidateExecutionClaim, +) { + const result = await store.tryClaim(requested) + if (!result.acquired) throw new Error(`test setup failed to acquire: ${result.reason}`) + return result +} + +async function complete( + store: Pick, + lease: AgentCandidateExecutionLease, + result: AgentCandidateExecutionTerminalResult, +) { + const staged = await store.stageTerminal(lease, result) + return (await store.finish(lease, staged.terminal.terminalDigest)).terminal +} + +async function tempDirectory(): Promise { + const directory = await mkdtemp(resolve(tmpdir(), 'candidate-execution-claim-')) + temporaryDirectories.push(directory) + return directory +} + +async function claimFromChildProcess( + directory: string, + requested: AgentCandidateExecutionClaim, +): Promise { + return runChild([ + storeSource(directory), + `const result = await store.tryClaim(${JSON.stringify(requested)})`, + "process.stdout.write(result.acquired ? 'acquired' : result.reason)", + ]) +} + +async function markFromChildProcess( + directory: string, + lease: AgentCandidateExecutionLease, +): Promise { + return runChild([ + storeSource(directory), + `const result = await store.markCandidateMayRun(${JSON.stringify(lease)})`, + "process.stdout.write(result.marked ? 'marked' : 'already-marked')", + ]) +} + +async function stageFromChildProcess( + directory: string, + lease: AgentCandidateExecutionLease, + result: AgentCandidateExecutionTerminalResult, +): Promise { + return runChild([ + storeSource(directory), + `const result = await store.stageTerminal(${JSON.stringify(lease)}, ${JSON.stringify(result)})`, + "process.stdout.write(result.staged ? 'staged' : 'already-staged:' + (result.exactReplay ? 'exact' : 'mismatch'))", + ]) +} + +async function finishFromChildProcess( + directory: string, + lease: AgentCandidateExecutionLease, + terminalDigest: string, +): Promise { + return runChild([ + storeSource(directory), + `const result = await store.finish(${JSON.stringify(lease)}, ${JSON.stringify(terminalDigest)})`, + "process.stdout.write(result.finished ? 'finished' : 'already-finished:' + (result.exactReplay ? 'exact' : 'mismatch'))", + ]) +} + +async function recoverFromChildProcess( + directory: string, + attempt: { executionId: string; attempt: number }, + evidence: AgentCandidateExecutionRecoveryEvidence, + nowMs: number, +): Promise { + return runChild([ + storeSource(directory, nowMs), + `const result = await store.recoverExpired(${JSON.stringify(attempt)}, ${JSON.stringify(evidence)})`, + "process.stdout.write(result.finished ? 'recovered' : 'already-recovered:' + (result.exactReplay ? 'exact' : 'mismatch'))", + ]) +} + +function storeSource(directory: string, nowMs?: number): string { + return `const store = new FileAgentCandidateExecutionClaimStore({ directory: ${JSON.stringify(directory)}${nowMs === undefined ? '' : `, now: () => ${nowMs}`} })` +} + +async function runChild(lines: readonly string[]): Promise { + const moduleUrl = new URL('../src/candidate-execution/claim-file-store.ts', import.meta.url).href + const source = [ + `import { FileAgentCandidateExecutionClaimStore } from ${JSON.stringify(moduleUrl)}`, + ...lines, + ].join('\n') + + return await new Promise((resolveChild, rejectChild) => { + const child = spawn( + process.execPath, + ['--import', 'tsx', '--input-type=module', '--eval', source], + { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + ) + let stdout = '' + let stderr = '' + child.stdout.setEncoding('utf8').on('data', (chunk: string) => { + stdout += chunk + }) + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { + stderr += chunk + }) + child.once('error', rejectChild) + child.once('close', (code) => { + if (code !== 0) { + rejectChild(new Error(`claim child exited ${code}: ${stderr}`)) + return + } + resolveChild(stdout) + }) + }) +} diff --git a/tests/candidate-execution-cleanup.test.ts b/tests/candidate-execution-cleanup.test.ts new file mode 100644 index 00000000..36b431c6 --- /dev/null +++ b/tests/candidate-execution-cleanup.test.ts @@ -0,0 +1,69 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { + CandidateCleanupTimeoutError, + CandidateResultTimeoutError, + candidateCleanupTimeout, + MAX_CANDIDATE_TIMER_INTERVAL_MS, + withinCandidateCleanupDeadline, + withinCandidateResultDeadline, +} from '../src/candidate-execution/cleanup' +import { + CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS, + candidateExecutionOwnerWindowMs, + candidatePostRunWindowMs, +} from '../src/candidate-execution/execution-window' + +describe('candidate cleanup timer bounds', () => { + afterEach(() => vi.useRealTimers()) + + it('accepts the Node timer boundary and rejects values that would clamp', () => { + expect(candidateCleanupTimeout(MAX_CANDIDATE_TIMER_INTERVAL_MS)).toBe( + MAX_CANDIDATE_TIMER_INTERVAL_MS, + ) + expect(() => candidateCleanupTimeout(MAX_CANDIDATE_TIMER_INTERVAL_MS + 1)).toThrow( + /supported timer range/, + ) + expect(() => candidateCleanupTimeout(Number.MAX_SAFE_INTEGER)).toThrow(/supported timer range/) + }) + + it('budgets every sequential post-run phase plus terminal persistence', () => { + expect(candidatePostRunWindowMs(25, 500)).toBe( + 25 * 4 + 500 + CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS, + ) + expect(candidateExecutionOwnerWindowMs(1_000, 25, 500)).toBe( + 1_000 + 25 * 4 + 500 + CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS, + ) + expect(() => candidatePostRunWindowMs(MAX_CANDIDATE_TIMER_INTERVAL_MS, 1)).toThrow( + /post-run window/, + ) + }) + + it('rejects results observed exactly at their frozen deadline', async () => { + vi.useFakeTimers({ now: 100 }) + await expect( + withinCandidateCleanupDeadline( + async () => { + vi.setSystemTime(110) + return 'ambiguous' + }, + 110, + 'cleanup', + ), + ).rejects.toBeInstanceOf(CandidateCleanupTimeoutError) + + let observedSignal: AbortSignal | undefined + await expect( + withinCandidateResultDeadline( + async (signal) => { + observedSignal = signal + vi.setSystemTime(120) + return 'ambiguous' + }, + 120, + 'result', + ), + ).rejects.toBeInstanceOf(CandidateResultTimeoutError) + expect(observedSignal?.aborted).toBe(true) + }) +}) diff --git a/tests/candidate-execution-core.test.ts b/tests/candidate-execution-core.test.ts new file mode 100644 index 00000000..5b3cb361 --- /dev/null +++ b/tests/candidate-execution-core.test.ts @@ -0,0 +1,244 @@ +import { execFileSync } from 'node:child_process' +import { + chmodSync, + linkSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { + AgentCandidateGitPatch, + AgentCandidateWorkspaceManifestMaterialV1, +} from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it } from 'vitest' + +import { + readVerifiedArtifact, + verifyMaterializedWorkspace, + verifyWorkspaceSnapshotArtifacts, +} from '../src/candidate-execution/artifacts' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + embeddedCandidateArtifact, +} from '../src/candidate-execution/digest' +import { verifyCandidateCode } from '../src/candidate-execution/git-materialize' + +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +function temporaryRoot(prefix: string): string { + const root = mkdtempSync(join(tmpdir(), prefix)) + roots.push(root) + return root +} + +function git(root: string, args: string[]): string { + return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim() +} + +function repositoryFixture(): { + root: string + baseCommit: string + baseTree: string + candidateTree: string + patch: Uint8Array +} { + const root = temporaryRoot('candidate-git-') + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'test@example.com']) + git(root, ['config', 'user.name', 'Test']) + git(root, ['config', 'core.hooksPath', '/dev/null']) + git(root, ['remote', 'add', 'origin', 'git@github.com:owner/repo.git']) + writeFileSync(join(root, 'index.ts'), 'export const value = 1\n') + git(root, ['add', 'index.ts']) + git(root, ['commit', '-m', 'base']) + const baseCommit = git(root, ['rev-parse', 'HEAD']) + const baseTree = git(root, ['rev-parse', 'HEAD^{tree}']) + writeFileSync(join(root, 'index.ts'), 'export const value = 2\n') + const patch = Buffer.from(execFileSync('git', ['diff', '--binary'], { cwd: root })) + git(root, ['add', 'index.ts']) + const candidateTree = git(root, ['write-tree']) + git(root, ['reset', '--hard', baseCommit]) + return { root, baseCommit, baseTree, candidateTree, patch } +} + +function codeFixture(fixture: ReturnType): AgentCandidateGitPatch { + return { + kind: 'git-patch', + repository: { kind: 'github', owner: 'owner', repo: 'repo' }, + baseCommit: fixture.baseCommit, + baseTree: fixture.baseTree, + candidateTree: fixture.candidateTree, + patch: { format: 'git-diff-binary', artifact: embeddedCandidateArtifact(fixture.patch) }, + } +} + +describe('candidate canonical bytes and artifacts', () => { + it('uses the existing stable content address for exact canonical bytes', () => { + const first = { z: [3, { b: true, a: 'x' }], a: -0 } + const second = { a: 0, z: [3, { a: 'x', b: true }] } + expect(canonicalCandidateBytes(first)).toEqual(canonicalCandidateBytes(second)) + expect(canonicalCandidateDigest(first)).toBe(canonicalCandidateDigest(second)) + }) + + it('rejects an artifact whose claimed hash does not match its bytes', async () => { + const artifact = embeddedCandidateArtifact(Buffer.from('actual')) + await expect( + readVerifiedArtifact( + { ...artifact, content: Buffer.from('forged').toString('base64') }, + { read: async () => new Uint8Array() }, + ), + ).rejects.toThrow(/digest|byte length/) + }) + + it('requires workspace manifest bytes to equal canonical material, not only a locator', async () => { + const material: AgentCandidateWorkspaceManifestMaterialV1 = { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files: [], + } + const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) + const archive = embeddedCandidateArtifact(Buffer.from('archive')) + await expect( + verifyWorkspaceSnapshotArtifacts( + { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest, + archive, + }, + { read: async () => new Uint8Array() }, + ), + ).resolves.toMatchObject({ archive: expect.any(Uint8Array) }) + await expect( + verifyWorkspaceSnapshotArtifacts( + { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest: embeddedCandidateArtifact(Buffer.from('{}')), + archive, + }, + { read: async () => new Uint8Array() }, + ), + ).rejects.toThrow(/canonical manifest/) + }) +}) + +describe('candidate Git identity', () => { + it('replays a binary-safe patch against the exact base tree', async () => { + const fixture = repositoryFixture() + await expect( + verifyCandidateCode( + codeFixture(fixture), + { resolve: async () => fixture.root }, + fixture.patch, + ), + ).resolves.toBe(fixture.candidateTree) + }) + + it('rejects candidate-tree drift and the wrong declared repository', async () => { + const fixture = repositoryFixture() + const code = codeFixture(fixture) + await expect( + verifyCandidateCode( + { ...code, candidateTree: fixture.baseTree }, + { resolve: async () => fixture.root }, + fixture.patch, + ), + ).rejects.toThrow(/does not match/) + await expect( + verifyCandidateCode( + { ...code, repository: { kind: 'github', owner: 'other', repo: 'repo' } }, + { resolve: async () => fixture.root }, + fixture.patch, + ), + ).rejects.toThrow(/does not match github/) + }) + + it('rejects a candidate Git tree containing symlinks', async () => { + const fixture = repositoryFixture() + symlinkSync('index.ts', join(fixture.root, 'linked.ts')) + git(fixture.root, ['add', 'linked.ts']) + const candidateTree = git(fixture.root, ['write-tree']) + const patch = Buffer.from( + execFileSync('git', ['diff', '--cached', '--binary', fixture.baseCommit], { + cwd: fixture.root, + }), + ) + const code = { + ...codeFixture(fixture), + candidateTree, + patch: { format: 'git-diff-binary' as const, artifact: embeddedCandidateArtifact(patch) }, + } + await expect( + verifyCandidateCode(code, { resolve: async () => fixture.root }, patch), + ).rejects.toThrow(/symlink|non-blob/) + }) +}) + +describe('materialized workspace identity', () => { + it('accepts only the exact regular files, modes, and bytes in the manifest', async () => { + const root = temporaryRoot('candidate-workspace-') + mkdirSync(join(root, '.git')) + writeFileSync(join(root, '.git', 'ignored'), 'Git internals are not uploaded') + writeFileSync(join(root, 'run.js'), 'ok\n', { mode: 0o755 }) + const bytes = Buffer.from('ok\n') + const material: AgentCandidateWorkspaceManifestMaterialV1 = { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files: [ + { + path: 'run.js', + mode: 0o755, + sha256: embeddedCandidateArtifact(bytes).sha256, + byteLength: bytes.byteLength, + }, + ], + } + await expect( + verifyMaterializedWorkspace(root, material, { ignoredProtectedRootEntries: ['.git'] }), + ).resolves.toBeUndefined() + await expect(verifyMaterializedWorkspace(root, material)).rejects.toThrow( + /do not match|unsupported mode/, + ) + rmSync(join(root, '.git'), { recursive: true }) + mkdirSync(join(root, '.sidecar')) + writeFileSync(join(root, '.sidecar', 'unsigned'), 'hidden executable bytes') + await expect(verifyMaterializedWorkspace(root, material)).rejects.toThrow( + /do not match|unsupported mode/, + ) + rmSync(join(root, '.sidecar'), { recursive: true }) + writeFileSync(join(root, 'extra.txt'), 'extra') + chmodSync(join(root, 'extra.txt'), 0o644) + await expect(verifyMaterializedWorkspace(root, material)).rejects.toThrow(/do not match/) + }) + + it('rejects symlink and hard-link ambiguity', async () => { + const root = temporaryRoot('candidate-workspace-links-') + writeFileSync(join(root, 'source'), 'x') + chmodSync(join(root, 'source'), 0o644) + symlinkSync('source', join(root, 'symlink')) + const material: AgentCandidateWorkspaceManifestMaterialV1 = { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files: [], + } + await expect(verifyMaterializedWorkspace(root, material)).rejects.toThrow(/symlink/) + rmSync(join(root, 'symlink')) + linkSync(join(root, 'source'), join(root, 'hardlink')) + await expect(verifyMaterializedWorkspace(root, material)).rejects.toThrow(/hard-linked/) + }) +}) diff --git a/tests/candidate-execution-dispose.test.ts b/tests/candidate-execution-dispose.test.ts new file mode 100644 index 00000000..89f7fe46 --- /dev/null +++ b/tests/candidate-execution-dispose.test.ts @@ -0,0 +1,108 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { InMemoryAgentCandidateExecutionClaimStore } from '../src/candidate-execution/claim' +import { disposePreparedAgentCandidateExecution } from '../src/candidate-execution/dispose' +import { executePreparedAgentCandidate } from '../src/candidate-execution/execute' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + candidateSha, + cleanupCandidateFixtures, + createCandidateExecutionFixture, + createCandidateOutputFixture, +} from './helpers/candidate-execution-fixture' + +afterEach(cleanupCandidateFixtures) + +describe('prepared candidate disposal', () => { + it('settles an unexecuted reservation exactly once and consumes the preparation', async () => { + const fixture = createCandidateExecutionFixture() + const reasons: string[] = [] + fixture.ports.models.settleGrant = async ({ preparationId, reason }) => { + reasons.push(`${preparationId}:${reason}`) + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + + await expect(disposePreparedAgentCandidateExecution(prepared)).resolves.toEqual({ + disposed: true, + }) + expect(reasons).toHaveLength(1) + expect(reasons[0]).toMatch(/candidate-preparation-v1\..+:abandoned/) + await expect( + executePreparedAgentCandidate(prepared, { + traceStore: {} as never, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + executor: {} as never, + ...createCandidateOutputFixture(), + }), + ).resolves.toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/already disposed/), + }) + }) + + it('retries exact idempotent cleanup when disposal transiently fails', async () => { + const fixture = createCandidateExecutionFixture() + let settlements = 0 + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settlements++ + if (settlements === 1) throw new Error('gateway unavailable') + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + + await expect( + disposePreparedAgentCandidateExecution(prepared, { cleanupTimeoutMs: 25 }), + ).rejects.toThrow(/disposal failed/) + await expect(disposePreparedAgentCandidateExecution(prepared)).resolves.toEqual({ + disposed: true, + }) + expect(settlements).toBe(2) + }) + + it('allows only one concurrent disposal attempt', async () => { + const fixture = createCandidateExecutionFixture() + let release!: () => void + const wait = new Promise((resolve) => { + release = resolve + }) + fixture.ports.models.settleGrant = async ({ preparationId }) => { + await wait + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const first = disposePreparedAgentCandidateExecution(prepared) + await expect(disposePreparedAgentCandidateExecution(prepared)).rejects.toThrow( + /already disposing/, + ) + release() + await expect(first).resolves.toEqual({ disposed: true }) + }) +}) diff --git a/tests/candidate-execution-execute.test.ts b/tests/candidate-execution-execute.test.ts new file mode 100644 index 00000000..545b5a99 --- /dev/null +++ b/tests/candidate-execution-execute.test.ts @@ -0,0 +1,1649 @@ +import { execFileSync } from 'node:child_process' +import { chmodSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { gzipSync } from 'node:zlib' + +import { InMemoryTraceStore, type TraceStore } from '@tangle-network/agent-eval' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { InMemoryAgentCandidateExecutionClaimStore } from '../src/candidate-execution/claim' +import { sha256Bytes } from '../src/candidate-execution/digest' +import { disposePreparedAgentCandidateExecution } from '../src/candidate-execution/dispose' +import { executePreparedAgentCandidate } from '../src/candidate-execution/execute' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import type { + AgentCandidateExecutorPort, + AgentCandidateExecutorRequest, +} from '../src/candidate-execution/types' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + captureAgentCandidateWorkspaceFiles, + createAgentCandidateWorkspacePort, +} from '../src/candidate-execution/workspace-archive' +import { + candidateBundle, + candidateSha, + cleanupCandidateFixtures, + createCandidateExecutionFixture, + createCandidateOutputFixture, + emptyCandidateSnapshot, + redigestCandidateBundle, + unchangedTaskOutcomeCapture, +} from './helpers/candidate-execution-fixture' + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + cleanupCandidateFixtures() +}) + +async function terminalTrace( + request: AgentCandidateExecutorRequest, + store: TraceStore, + endedAt = 200, +) { + await store.appendRun({ + runId: request.trace.runId, + scenarioId: 'candidate-execution', + startedAt: 100, + endedAt, + status: 'completed', + tags: { ...request.trace.tags }, + }) +} + +function options( + executor: AgentCandidateExecutorPort, + traceStore = new InMemoryTraceStore(), + claimStore = new InMemoryAgentCandidateExecutionClaimStore(), +) { + const outputs = createCandidateOutputFixture() + let request: AgentCandidateExecutorRequest | undefined + return { + executor: { + execute: async (...args: Parameters) => { + request = args[0] + return await executor.execute(...args) + }, + stopAndCapture: async (...args: Parameters) => { + const capture = await executor.stopAndCapture(...args) + if (capture.taskOutcome || !request) return capture + return { + ...capture, + taskOutcome: { + resultTree: request.executionPlan.value.material.task.repository.baseTree, + afterState: request.executionPlan.value.material.task.workspace.material, + archive: Buffer.from('fixture unchanged task archive', 'utf8'), + gitDiff: Buffer.alloc(0), + }, + } + }, + }, + traceStore, + claimStore, + ...outputs, + } +} + +describe('atomic prepared candidate execution', () => { + it('reveals credentials only to one trusted executor and returns a durable receipt', async () => { + const fixture = createCandidateExecutionFixture() + fixture.bundle = candidateBundle({ + env: { PUBLIC_MODE: { kind: 'public', value: 'fixture' } }, + }) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + expect(JSON.stringify(prepared)).not.toContain('MODEL_GATEWAY_TOKEN') + expect(JSON.stringify(prepared)).not.toContain('protected') + + const traceStore = new InMemoryTraceStore() + let observed: AgentCandidateExecutorRequest | undefined + let stopped = 0 + const executor: AgentCandidateExecutorPort = { + execute: async (request, context) => { + observed = request + expect(context.traceStore).not.toBe(traceStore) + expect(context.signal.aborted).toBe(false) + expect(context.deadlineAtMs).toBeGreaterThan(Date.now()) + expect(request.launch.env).toMatchObject({ + PUBLIC_MODE: 'fixture', + MODEL_GATEWAY_TOKEN: 'protected', + TANGLE_CANDIDATE_EXECUTION_ID: prepared.executionId, + }) + expect(request.launch.env.PATH).toBeUndefined() + expect(request.hardLimits).toEqual({ timeoutMs: fixture.task.limits.timeoutMs }) + expect(request.observedLimits).toEqual({ maxSteps: fixture.task.limits.maxSteps }) + expect(request.executionPlan.value.material.model.access.network).toEqual({ + mode: 'gateway-only', + domains: ['router.tangle.tools'], + }) + expect(Buffer.from(request.inputs.task.files[0]?.bytes ?? []).toString('utf8')).toBe( + 'export const value = 1\n', + ) + await terminalTrace(request, traceStore) + return { + executionId: request.executionId, + termination: { kind: 'exit', exitCode: 0 }, + } + }, + stopAndCapture: async () => { + stopped++ + if (!observed) throw new Error('candidate executor did not receive its request') + const changedBytes = Buffer.from('export const value = 2\n') + const taskRoot = fixture.task.stagingRoots.taskRoot + writeFileSync(join(taskRoot, 'source.ts'), changedBytes, { mode: 0o644 }) + execFileSync('git', ['add', 'source.ts'], { cwd: taskRoot }) + const resultTree = execFileSync('git', ['write-tree'], { + cwd: taskRoot, + encoding: 'utf8', + }).trim() + const gitDiff = execFileSync('git', ['diff', '--cached', '--binary', 'HEAD'], { + cwd: taskRoot, + }) + const captured = await captureAgentCandidateWorkspaceFiles( + observed.inputs.task.files.map((file) => ({ + path: file.path, + mode: file.mode, + bytes: file.path === 'source.ts' ? changedBytes : Uint8Array.from(file.bytes), + })), + ) + return { + stopped: true, + taskOutcome: { + resultTree, + afterState: captured.snapshot.material, + archive: captured.archive, + gitDiff, + }, + } + }, + } + + const executionOptions = options(executor, traceStore) + const result = await executePreparedAgentCandidate(prepared, executionOptions) + expect(observed).toBeDefined() + expect(stopped).toBe(1) + if (!result.succeeded) throw new Error(result.reason) + expect(result).toMatchObject({ + succeeded: true, + receipt: { + value: { + schemaVersion: 2, + usage: { modelCalls: 0, costUsd: 0 }, + fixedUsage: { modelCalls: 0, costUsdNanos: 0 }, + benchmarkResult: { + material: { + score: 1, + passed: true, + grader: { name: 'fixture-executable-grader', version: '1.0.0' }, + }, + }, + }, + }, + }) + await expect( + executionOptions.outputArtifacts.read(result.artifacts.runReceipt), + ).resolves.toEqual(result.receipt.bytes) + }) + + it('publishes a referenced final receipt for a task archive larger than 28,254,720 bytes', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const largeSource = Buffer.alloc(28_254_721) + let largeArchive: Uint8Array | undefined + fixture.ports.workspaces.materialize = createAgentCandidateWorkspacePort().materialize + const executionOptions = options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => { + const taskRoot = fixture.task.stagingRoots.taskRoot + writeFileSync(join(taskRoot, 'source.ts'), largeSource, { mode: 0o644 }) + execFileSync('git', ['add', 'source.ts'], { cwd: taskRoot }) + const resultTree = execFileSync('git', ['write-tree'], { + cwd: taskRoot, + encoding: 'utf8', + }).trim() + const gitDiff = execFileSync('git', ['diff', '--cached', '--binary', 'HEAD'], { + cwd: taskRoot, + }) + const captured = await captureAgentCandidateWorkspaceFiles( + [{ path: 'source.ts', mode: 0o644, bytes: largeSource }], + { limits: { maxEmbeddedArtifactBytes: 32 * 1024 * 1024 } }, + ) + largeArchive = captured.archive + return { + stopped: true, + taskOutcome: { + resultTree, + afterState: captured.snapshot.material, + archive: captured.archive, + gitDiff, + }, + } + }, + }, + traceStore, + ) + + const result = await executePreparedAgentCandidate(prepared, executionOptions) + if (!result.succeeded) throw new Error(result.reason) + if (!largeArchive) throw new Error('executor did not capture its large task archive') + const afterState = result.receipt.value.taskOutcome.material.afterState + const archive = afterState.archive + expect(archive).toMatchObject({ + byteLength: largeArchive.byteLength, + sha256: sha256Bytes(largeArchive), + locator: { kind: 's3' }, + }) + expect('content' in archive).toBe(false) + expect(afterState.manifest).toMatchObject({ locator: { kind: 's3' } }) + expect('content' in afterState.manifest).toBe(false) + expect(result.receipt.bytes.byteLength).toBeLessThan(100_000) + expect(result.receipt.value).toMatchObject({ + usage: { modelCalls: 0, inputTokens: 0, outputTokens: 0, costUsd: 0 }, + fixedUsage: { modelCalls: 0, inputTokens: 0, outputTokens: 0, costUsdNanos: 0 }, + trace: { modelCallCount: 0 }, + }) + const persistedArchive = await executionOptions.outputArtifacts.read(archive) + expect(persistedArchive.byteLength).toBe(largeArchive.byteLength) + expect(sha256Bytes(persistedArchive)).toBe(archive.sha256) + await expect( + executionOptions.outputArtifacts.read(result.artifacts.runReceipt), + ).resolves.toEqual(result.receipt.bytes) + }) + + it('authors paid model spans only from the closed router ledger', async () => { + const fixture = createCandidateExecutionFixture() + fixture.ports.models.settleGrant = async ({ preparationId }) => ({ + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [ + { + callId: 'call-paid-success', + generationId: 'generation-paid-success', + traceSpanId: 'generation-paid-success', + status: 'succeeded', + model: 'model-snapshot', + startedAtMs: 120, + endedAtMs: 180, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 2, + reasoningTokens: 1, + costUsdNanos: 10_000_000, + }, + ], + }) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async (request, context) => { + await expect( + context.traceStore.appendSpan({ + runId: request.trace.runId, + spanId: 'candidate-authored-model-span', + kind: 'llm', + name: 'untrusted model usage', + model: request.resolvedModel.model, + messages: [], + startedAt: 120, + }), + ).rejects.toThrow(/cannot author protected model spans/) + await terminalTrace(request, context.traceStore) + return { + executionId: request.executionId, + termination: { kind: 'exit', exitCode: 0 }, + } + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore, + ), + ) + if (!result.succeeded) throw new Error(result.reason) + expect(result.receipt.value).toMatchObject({ + usage: { modelCalls: 1, inputTokens: 10, outputTokens: 5, costUsd: 0.01 }, + modelSettlement: { + material: { + schemaVersion: 2, + calls: [ + { + generationId: 'generation-paid-success', + traceSpanId: 'generation-paid-success', + startedAtMs: 120, + endedAtMs: 180, + costUsdNanos: 10_000_000, + }, + ], + }, + }, + }) + expect(await traceStore.spans({ runId: prepared.trace.runId })).toEqual([ + expect.objectContaining({ + spanId: 'generation-paid-success', + kind: 'llm', + inputTokens: 10, + outputTokens: 5, + cachedTokens: 2, + reasoningTokens: 1, + costUsd: 0.01, + startedAt: 120, + endedAt: 180, + attributes: expect.objectContaining({ + 'tangle.protected_model.source': 'router-settlement', + }), + }), + ]) + }) + + it('rejects pre-launch staging mutation before activation or executor access', async () => { + const fixture = createCandidateExecutionFixture(true) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + if (!fixture.candidateRoot) throw new Error('active fixture root is missing') + writeFileSync(join(fixture.candidateRoot, 'run.js'), 'tampered-after-prepare\n') + let activations = 0 + let executions = 0 + let settlements = 0 + fixture.ports.models.activateGrant = async () => { + activations++ + return { env: { MODEL_GATEWAY_TOKEN: 'protected' } } + } + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settlements++ + return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + } + const result = await executePreparedAgentCandidate( + prepared, + options({ + execute: async () => { + executions++ + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/do not match the signed manifest/), + usage: { modelCalls: 0 }, + }) + expect({ activations, executions, settlements }).toEqual({ + activations: 0, + executions: 0, + settlements: 1, + }) + }) + + it('refuses activation when slow claim persistence consumed the execution window', async () => { + let now = Date.now() + vi.spyOn(Date, 'now').mockImplementation(() => now) + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const baseStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + let activations = 0 + fixture.ports.models.activateGrant = async () => { + activations++ + return { env: { MODEL_GATEWAY_TOKEN: 'protected' } } + } + const claimStore = { + tryClaim: async (claim: Parameters[0]) => { + const result = await baseStore.tryClaim(claim) + now += fixture.task.limits.timeoutMs + 1 + return result + }, + getAttempt: (attempt: Parameters[0]) => + baseStore.getAttempt(attempt), + markCandidateMayRun: (lease: Parameters[0]) => + baseStore.markCandidateMayRun(lease), + stageTerminal: ( + lease: Parameters[0], + result: Parameters[1], + ) => baseStore.stageTerminal(lease, result), + finish: ( + lease: Parameters[0], + terminalDigest: Parameters[1], + ) => baseStore.finish(lease, terminalDigest), + recoverExpired: ( + attempt: Parameters[0], + evidence: Parameters[1], + ) => baseStore.recoverExpired(attempt, evidence), + } + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async () => { + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + new InMemoryTraceStore(), + claimStore, + ), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/no longer covers/), + usage: { modelCalls: 0 }, + }) + expect(activations).toBe(0) + }) + + it('does not launch when phase persistence consumes the frozen execution window', async () => { + let now = 1_000_000 + vi.spyOn(Date, 'now').mockImplementation(() => now) + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 100 } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs: 25, resultTimeoutMs: 100 }, + ) + const baseStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claimStore = { + tryClaim: (claim: Parameters[0]) => baseStore.tryClaim(claim), + getAttempt: (attempt: Parameters[0]) => + baseStore.getAttempt(attempt), + markCandidateMayRun: async (lease: Parameters[0]) => { + const result = await baseStore.markCandidateMayRun(lease) + now += fixture.task.limits.timeoutMs + return result + }, + stageTerminal: ( + lease: Parameters[0], + terminal: Parameters[1], + ) => baseStore.stageTerminal(lease, terminal), + finish: ( + lease: Parameters[0], + terminalDigest: Parameters[1], + ) => baseStore.finish(lease, terminalDigest), + recoverExpired: ( + attempt: Parameters[0], + evidence: Parameters[1], + ) => baseStore.recoverExpired(attempt, evidence), + } + let executions = 0 + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async () => { + executions++ + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + new InMemoryTraceStore(), + claimStore, + ), + ) + + expect(executions).toBe(0) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/deadline elapsed while persisting its launch phase/), + usage: { modelCalls: 0 }, + }) + await expect( + baseStore.getAttempt({ executionId: prepared.executionId, attempt: 1 }), + ).resolves.toMatchObject({ terminal: { status: 'failed', failureClass: 'unknown' } }) + }) + + it('publishes after every post-run phase consumes nearly its full reserved interval', async () => { + let now = 1_000_000 + vi.spyOn(Date, 'now').mockImplementation(() => now) + const cleanupTimeoutMs = 100 + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 1_000 } + fixture.ports.models.settleGrant = async ({ preparationId }) => { + now += cleanupTimeoutMs - 1 + return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs }, + ) + const traceStore = new InMemoryTraceStore() + const baseStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claimStore = { + tryClaim: (claim: Parameters[0]) => baseStore.tryClaim(claim), + getAttempt: (attempt: Parameters[0]) => + baseStore.getAttempt(attempt), + markCandidateMayRun: (lease: Parameters[0]) => + baseStore.markCandidateMayRun(lease), + stageTerminal: async ( + lease: Parameters[0], + terminal: Parameters[1], + ) => { + now += 5_000 + return await baseStore.stageTerminal(lease, terminal) + }, + finish: async ( + lease: Parameters[0], + terminalDigest: Parameters[1], + ) => { + now += 4_999 + return await baseStore.finish(lease, terminalDigest) + }, + recoverExpired: ( + attempt: Parameters[0], + evidence: Parameters[1], + ) => baseStore.recoverExpired(attempt, evidence), + } + const outputs = createCandidateOutputFixture() + let advancedModelEvidence = false + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + if (input.purpose === 'model-settlement' && !advancedModelEvidence) { + advancedModelEvidence = true + now += cleanupTimeoutMs - 1 + } + return await outputs.outputArtifacts.put(input) + }, + } + const result = await executePreparedAgentCandidate(prepared, { + executor: { + execute: async (request, context) => { + now = context.deadlineAtMs - 1 + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => { + now += cleanupTimeoutMs - 1 + return { stopped: true, taskOutcome: unchangedTaskOutcomeCapture(fixture) } + }, + }, + grader: { + ...outputs.grader, + run: async (input) => { + now += cleanupTimeoutMs - 1 + return await outputs.grader.run(input) + }, + }, + outputArtifacts, + traceStore, + claimStore, + cleanupTimeoutMs, + }) + + expect(result.succeeded).toBe(true) + const attempt = await baseStore.getAttempt({ executionId: prepared.executionId, attempt: 1 }) + expect(attempt?.terminal?.status).toBe('succeeded') + expect(now).toBeLessThanOrEqual(attempt?.claim.leaseExpiresAtMs ?? 0) + }) + + it('allows executable grading to exceed cleanup time inside its frozen result budget', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 1_000 } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs: 10, resultTimeoutMs: 100 }, + ) + const traceStore = new InMemoryTraceStore() + const outputs = createCandidateOutputFixture() + vi.useFakeTimers({ now: Date.now() }) + let markGraderStarted!: () => void + const graderStarted = new Promise((resolve) => { + markGraderStarted = resolve + }) + const pending = executePreparedAgentCandidate(prepared, { + ...options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + }), + }, + traceStore, + ), + grader: { + ...outputs.grader, + run: async (input) => { + markGraderStarted() + await new Promise((resolve) => setTimeout(resolve, 50)) + return await outputs.grader.run(input) + }, + }, + outputArtifacts: outputs.outputArtifacts, + cleanupTimeoutMs: 10, + resultTimeoutMs: 100, + }) + + await graderStarted + await vi.advanceTimersByTimeAsync(50) + await expect(pending).resolves.toMatchObject({ succeeded: true }) + }) + + it('cancels over-budget grading without any output appearing after terminal failure', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 1_000 } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs: 25, resultTimeoutMs: 50 }, + ) + const traceStore = new InMemoryTraceStore() + const outputs = createCandidateOutputFixture() + const persistedPurposes: string[] = [] + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + input.signal?.throwIfAborted() + persistedPurposes.push(input.purpose) + return await outputs.outputArtifacts.put(input) + }, + } + vi.useFakeTimers({ now: Date.now() }) + let markGraderStarted!: () => void + const graderStarted = new Promise((resolve) => { + markGraderStarted = resolve + }) + const pending = executePreparedAgentCandidate(prepared, { + ...options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + }), + }, + traceStore, + ), + grader: { + ...outputs.grader, + run: async (input) => { + markGraderStarted() + await new Promise((resolve) => setTimeout(resolve, 100)) + return await outputs.grader.run(input) + }, + }, + outputArtifacts, + cleanupTimeoutMs: 25, + resultTimeoutMs: 50, + }) + + await graderStarted + await vi.advanceTimersByTimeAsync(50) + const result = await pending + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/result deadline/), + }) + const purposesAtTerminalFailure = [...persistedPurposes] + await vi.advanceTimersByTimeAsync(100) + await Promise.resolve() + expect(persistedPurposes).toEqual(purposesAtTerminalFailure) + expect(persistedPurposes).not.toContain('grader-evidence') + expect(persistedPurposes).not.toContain('benchmark-result') + expect(persistedPurposes).not.toContain('trace') + expect(persistedPurposes).not.toContain('run-receipt') + }) + + it('allows disposal to retry an unproven pre-claim cleanup', async () => { + const fixture = createCandidateExecutionFixture(true) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + if (!fixture.candidateRoot) throw new Error('active fixture root is missing') + writeFileSync(join(fixture.candidateRoot, 'run.js'), 'tampered-after-prepare\n') + let settlements = 0 + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settlements++ + if (settlements === 1) throw new Error('transient gateway failure') + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + } + } + const result = await executePreparedAgentCandidate( + prepared, + options({ + execute: async () => { + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/cleanup remains incomplete/), + }) + await expect(disposePreparedAgentCandidateExecution(prepared)).resolves.toEqual({ + disposed: true, + }) + expect(settlements).toBe(2) + }) + + it('executes detached original bytes when staging changes after its final check', async () => { + const fixture = createCandidateExecutionFixture(true) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + if (!fixture.candidateRoot) throw new Error('active fixture root is missing') + fixture.ports.models.activateGrant = async () => { + writeFileSync(join(fixture.candidateRoot as string, 'run.js'), 'concurrent-replacement\n') + return { env: { MODEL_GATEWAY_TOKEN: 'protected' } } + } + const traceStore = new InMemoryTraceStore() + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async (request) => { + expect( + Buffer.from(request.inputs.candidate?.files[0]?.bytes ?? []).toString('utf8'), + ).toBe('#!/usr/bin/env node\n') + const detached = request.inputs.candidate?.files[0]?.bytes + detached?.fill(0) + expect( + Buffer.from(request.inputs.candidate?.files[0]?.bytes ?? []).toString('utf8'), + ).toBe('#!/usr/bin/env node\n') + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore, + ), + ) + expect(result.succeeded).toBe(true) + }) + + it('withholds a receipt when process death has no captured task outcome', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const result = await executePreparedAgentCandidate(prepared, { + executor: { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + ...createCandidateOutputFixture(), + }) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/without a captured task outcome/), + usage: { modelCalls: 0, costUsd: 0 }, + }) + }) + + it('rejects candidate-authored score fields in the final capture before grading', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + let graderCalls = 0 + const outputs = createCandidateOutputFixture() + const result = await executePreparedAgentCandidate(prepared, { + executor: { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => + ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + score: 1, + }) as never, + }, + grader: { + ...outputs.grader, + run: async (input) => { + graderCalls++ + return await outputs.grader.run(input) + }, + }, + outputArtifacts: outputs.outputArtifacts, + traceStore, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + }) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/unknown field score/), + usage: { modelCalls: 0, costUsd: 0 }, + }) + expect(graderCalls).toBe(0) + }) + + it('rejects candidate-authored score fields in the execution capture before grading', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + let graderCalls = 0 + const outputs = createCandidateOutputFixture() + const result = await executePreparedAgentCandidate(prepared, { + executor: { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { + executionId: request.executionId, + termination: { kind: 'exit', exitCode: 0 }, + score: 1, + } as never + }, + stopAndCapture: async () => ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + }), + }, + grader: { + ...outputs.grader, + run: async (input) => { + graderCalls++ + return await outputs.grader.run(input) + }, + }, + outputArtifacts: outputs.outputArtifacts, + traceStore, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + }) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/unknown field score/), + usage: { modelCalls: 0, costUsd: 0 }, + }) + expect(graderCalls).toBe(0) + }) + + it('allows exactly one concurrent invocation of the same prepared value', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore() + let executions = 0 + let release!: () => void + const wait = new Promise((resolve) => { + release = resolve + }) + let started!: () => void + const didStart = new Promise((resolve) => { + started = resolve + }) + const executor: AgentCandidateExecutorPort = { + execute: async (request) => { + executions++ + started() + await wait + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + } + const first = executePreparedAgentCandidate(prepared, options(executor, traceStore, claimStore)) + await didStart + const second = await executePreparedAgentCandidate( + prepared, + options(executor, traceStore, claimStore), + ) + release() + const firstResult = await first + expect(firstResult.succeeded).toBe(true) + expect(second).toMatchObject({ succeeded: false, reason: expect.stringMatching(/already/) }) + expect(executions).toBe(1) + }) + + it('keeps independently prepared reservations isolated when one loses the durable claim', async () => { + const fixture = createCandidateExecutionFixture() + const openPreparations = new Set() + const closedPreparations = new Set() + let activePreparation: string | undefined + fixture.ports.models.reserveGrant = async ({ preparationId, expiresAtMs, limits }) => { + openPreparations.add(preparationId) + return { + preparationId, + digest: candidateSha('c'), + expiresAtMs, + enforcedLimits: limits, + network: + limits.maxModelCalls === 0 + ? { mode: 'disabled' as const } + : { mode: 'gateway-only' as const, domains: ['router.tangle.tools'] }, + } + } + fixture.ports.models.activateGrant = async ({ preparationId }) => { + if (!openPreparations.has(preparationId) || closedPreparations.has(preparationId)) { + throw new Error('activation used a closed preparation') + } + activePreparation = preparationId + return { env: { MODEL_GATEWAY_TOKEN: `protected-${preparationId}` } } + } + fixture.ports.models.settleGrant = async ({ preparationId }) => { + if (!openPreparations.has(preparationId)) throw new Error('unknown preparation') + closedPreparations.add(preparationId) + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + } + } + + const verified = await verifyAgentCandidateBundle(fixture.bundle, fixture.ports) + const firstPrepared = await prepareAgentCandidateExecution( + verified, + fixture.task, + fixture.ports, + ) + rmSync(fixture.task.stagingRoots.profileRoot, { recursive: true }) + mkdirSync(fixture.task.stagingRoots.profileRoot) + const secondPrepared = await prepareAgentCandidateExecution( + verified, + fixture.task, + fixture.ports, + ) + expect(firstPrepared.trace.runId).not.toBe(secondPrepared.trace.runId) + + const traceStore = new InMemoryTraceStore() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore() + let release!: () => void + const wait = new Promise((resolve) => { + release = resolve + }) + let started!: () => void + const didStart = new Promise((resolve) => { + started = resolve + }) + const executor: AgentCandidateExecutorPort = { + execute: async (request) => { + started() + await wait + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + } + + const first = executePreparedAgentCandidate( + firstPrepared, + options(executor, traceStore, claimStore), + ) + await didStart + const loser = await executePreparedAgentCandidate( + secondPrepared, + options(executor, traceStore, claimStore), + ) + expect(loser).toMatchObject({ succeeded: false, reason: expect.stringMatching(/claimed/) }) + expect(activePreparation).toBeDefined() + expect(closedPreparations.has(activePreparation as string)).toBe(false) + expect(closedPreparations.size).toBe(1) + + release() + expect((await first).succeeded).toBe(true) + expect(closedPreparations.size).toBe(2) + }) + + it('settles and records paid usage when the executor fails', async () => { + const fixture = createCandidateExecutionFixture() + const secret = 'sk-runtime-secret-123456789' + let closed = false + let settlementReads = 0 + const serveLateCall = () => { + if (closed) throw new Error('grant is closed') + } + fixture.ports.models.activateGrant = async () => ({ + env: { MODEL_GATEWAY_TOKEN: secret }, + }) + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settlementReads++ + closed = true + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [ + { + callId: 'call-paid-1', + generationId: 'generation-paid-1', + traceSpanId: 'generation-paid-1', + status: 'failed', + model: 'model-snapshot', + startedAtMs: 100, + endedAtMs: 150, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 0, + reasoningTokens: 0, + costUsdNanos: 10_000_000, + }, + ], + } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const result = await executePreparedAgentCandidate( + prepared, + options({ + execute: async () => Promise.reject(new Error(`container failed with ${secret}`)), + stopAndCapture: async () => ({ stopped: true }), + }), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.not.stringContaining(secret), + usage: { modelCalls: 1, inputTokens: 10, outputTokens: 5, costUsd: 0.01 }, + }) + expect(result.reason).toContain('[redacted:candidate-access]') + expect({ closed, settlementReads }).toEqual({ closed: true, settlementReads: 1 }) + expect(serveLateCall).toThrow(/closed/) + }) + + it('owns the deadline, aborts, waits for process death, and then settles', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 15 } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + let observedSignal: AbortSignal | undefined + let timeoutRequest: AgentCandidateExecutorRequest | undefined + let stoppedAfterAbort = false + let settledAfterStop = false + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settledAfterStop = stoppedAfterAbort + return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + } + vi.useFakeTimers({ now: Date.now() }) + const startedAt = Date.now() + let markExecutorStarted!: () => void + const executorStarted = new Promise((resolve) => { + markExecutorStarted = resolve + }) + const pending = executePreparedAgentCandidate( + prepared, + options( + { + execute: async (request, context) => { + timeoutRequest = request + observedSignal = context.signal + markExecutorStarted() + return await new Promise((_resolve, reject) => { + context.signal.addEventListener('abort', () => reject(context.signal.reason), { + once: true, + }) + }) + }, + stopAndCapture: async (_request, context) => { + expect(context.reason).toBe('timeout') + expect(context.signal).toBe(observedSignal) + expect(context.deadlineAtMs).toBe(startedAt + 15) + stoppedAfterAbort = observedSignal?.aborted === true + if (!timeoutRequest) throw new Error('timeout executor never received its request') + await terminalTrace(timeoutRequest, traceStore, 115) + return { stopped: true } + }, + }, + traceStore, + ), + ) + await executorStarted + await vi.advanceTimersByTimeAsync(15) + const result = await pending + expect(Date.now()).toBe(startedAt + 15) + expect({ stoppedAfterAbort, settledAfterStop }).toEqual({ + stoppedAfterAbort: true, + settledAfterStop: true, + }) + expect(result).toMatchObject({ + succeeded: true, + receipt: { value: { termination: { kind: 'timeout', timeoutMs: 15 } } }, + }) + }) + + it('cannot turn a stop acknowledgement after the frozen deadline into an exit success', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 15 } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + vi.useFakeTimers({ now: Date.now() }) + let markStopStarted!: () => void + const stopStarted = new Promise((resolve) => { + markStopStarted = resolve + }) + const pending = executePreparedAgentCandidate(prepared, { + ...options( + { + execute: async (request) => { + await terminalTrace(request, traceStore, 105) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => { + markStopStarted() + await new Promise((resolve) => setTimeout(resolve, 30)) + return { stopped: true } + }, + }, + traceStore, + ), + cleanupTimeoutMs: 100, + }) + await stopStarted + await vi.advanceTimersByTimeAsync(30) + const result = await pending + expect(result).toMatchObject({ + succeeded: true, + receipt: { value: { termination: { kind: 'timeout', timeoutMs: 15 } } }, + }) + }) + + it('bounds a hanging process stop and leaves the claim for explicit recovery', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore() + const startedAt = Date.now() + const result = await executePreparedAgentCandidate(prepared, { + ...options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => await new Promise(() => undefined), + }, + traceStore, + claimStore, + ), + cleanupTimeoutMs: 20, + }) + expect(Date.now() - startedAt).toBeLessThan(250) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/termination is not proven/), + usage: { modelCalls: 0 }, + }) + }) + + it('withholds a receipt but still revokes model access when process death is unproven', async () => { + const fixture = createCandidateExecutionFixture() + let settlements = 0 + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settlements++ + return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => { + throw new Error('container death not observed') + }, + }, + traceStore, + ), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/death not observed/), + usage: { modelCalls: 0 }, + }) + expect(settlements).toBe(1) + }) + + it('leaves unknown settlement unfinished so a retry cannot guess zero spend', async () => { + const claimStore = new InMemoryAgentCandidateExecutionClaimStore() + const first = createCandidateExecutionFixture() + first.task.attempt = { + number: 1, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + first.ports.models.settleGrant = async () => { + throw new Error('gateway settlement unavailable') + } + const firstPrepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(first.bundle, first.ports), + first.task, + first.ports, + ) + const firstResult = await executePreparedAgentCandidate( + firstPrepared, + options( + { + execute: async () => { + throw new Error('infrastructure failed') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + new InMemoryTraceStore(), + claimStore, + ), + ) + expect(firstResult).toMatchObject({ succeeded: false, usage: null }) + + rmSync(first.task.stagingRoots.profileRoot, { recursive: true }) + mkdirSync(first.task.stagingRoots.profileRoot) + first.task.attempt = { + number: 2, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + const retryPrepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(first.bundle, first.ports), + first.task, + first.ports, + ) + let retryExecutions = 0 + const retryResult = await executePreparedAgentCandidate( + retryPrepared, + options( + { + execute: async () => { + retryExecutions++ + throw new Error('retry must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + new InMemoryTraceStore(), + claimStore, + ), + ) + expect(retryResult).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/prior-attempt-running/), + }) + expect(retryExecutions).toBe(0) + }) + + it('admits a zero-call retry only for an explicit pre-model infrastructure failure', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.attempt = { + number: 1, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + const claimStore = new InMemoryAgentCandidateExecutionClaimStore() + let activations = 0 + fixture.ports.models.activateGrant = async () => { + activations++ + if (activations === 1) throw new Error('model gateway unavailable before launch') + return { env: { MODEL_GATEWAY_TOKEN: 'protected' } } + } + const firstPrepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const firstTraceStore = new InMemoryTraceStore() + const first = await executePreparedAgentCandidate( + firstPrepared, + options( + { + execute: async () => { + throw new Error('executor must not run before model activation') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + firstTraceStore, + claimStore, + ), + ) + expect(first).toMatchObject({ succeeded: false, usage: { modelCalls: 0 } }) + + rmSync(fixture.task.stagingRoots.profileRoot, { recursive: true }) + mkdirSync(fixture.task.stagingRoots.profileRoot) + fixture.task.attempt = { + number: 2, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + const retryPrepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const retry = await executePreparedAgentCandidate( + retryPrepared, + options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore, + claimStore, + ), + ) + expect(retry.succeeded).toBe(true) + }) + + it('closes isolated memory and persists a large after-state archive by reference', async () => { + const fixture = createCandidateExecutionFixture() + fixture.bundle = redigestCandidateBundle(fixture.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }) + const before = emptyCandidateSnapshot('before') + const after = emptyCandidateSnapshot('after') + const order: string[] = [] + fixture.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ + preparationId, + accessDigest: candidateSha('8'), + expiresAtMs, + evidence: before.manifest, + emptyStateDigest: before.digest, + beforeState: before, + }) + fixture.ports.memory.activate = async ({ effectiveNamespace }) => ({ + env: { TANGLE_MEMORY_NAMESPACE: effectiveNamespace }, + }) + fixture.ports.memory.close = async () => { + order.push('memory-close') + return { closed: true } + } + fixture.ports.models.settleGrant = async ({ preparationId }) => { + order.push('model-settle') + return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + if (prepared.memory.mode !== 'isolated') throw new Error('isolated memory was not prepared') + const traceStore = new InMemoryTraceStore() + const largeMemoryArchive = Buffer.alloc(4_000_001, 0x5a) + const executionOptions = options( + { + execute: async (request) => { + expect(request.launch.env.TANGLE_MEMORY_NAMESPACE).toBe( + prepared.memory.effectiveNamespace, + ) + await terminalTrace(request, traceStore) + return { + executionId: request.executionId, + termination: { kind: 'exit', exitCode: 0 }, + } + }, + stopAndCapture: async () => { + order.push('process-stop') + return { + stopped: true, + memoryAfter: { + afterState: after.material, + archive: largeMemoryArchive, + }, + } + }, + }, + traceStore, + ) + const result = await executePreparedAgentCandidate(prepared, executionOptions) + expect(result.succeeded).toBe(true) + expect(order).toEqual(['process-stop', 'memory-close', 'model-settle']) + if (!result.succeeded || result.receipt.value.memory.mode !== 'isolated') return + const archive = result.receipt.value.memory.afterState.archive + expect(archive).toMatchObject({ + byteLength: largeMemoryArchive.byteLength, + sha256: sha256Bytes(largeMemoryArchive), + locator: { kind: 's3' }, + }) + expect('content' in archive).toBe(false) + }) + + it('rejects a compressed protected value in isolated memory before persisting memory evidence', async () => { + const fixture = createCandidateExecutionFixture() + fixture.bundle = redigestCandidateBundle(fixture.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }) + const before = emptyCandidateSnapshot('secret-memory-before') + fixture.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ + preparationId, + accessDigest: candidateSha('8'), + expiresAtMs, + evidence: before.manifest, + emptyStateDigest: before.digest, + beforeState: before, + }) + fixture.ports.memory.activate = async () => ({ env: { TANGLE_MEMORY_ACCESS: 'memory-access' } }) + fixture.ports.memory.close = async () => ({ closed: true }) + const secret = 'sk-memory-secret-123456789' + fixture.ports.models.activateGrant = async () => ({ env: { MODEL_GATEWAY_TOKEN: secret } }) + const originalMaterialize = fixture.ports.workspaces.materialize + fixture.ports.workspaces.materialize = async (input) => { + if (input.role !== 'memory') return await originalMaterialize(input) + const output = join(input.destination, 'notes.txt') + writeFileSync(output, Buffer.from(secret, 'utf8')) + chmodSync(output, 0o644) + } + const memoryBytes = Buffer.from(secret, 'utf8') + const afterState = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: [ + { + path: 'notes.txt', + mode: 0o644 as const, + sha256: sha256Bytes(memoryBytes), + byteLength: memoryBytes.byteLength, + }, + ], + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const outputs = createCandidateOutputFixture() + const purposes: string[] = [] + const result = await executePreparedAgentCandidate(prepared, { + executor: { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + memoryAfter: { afterState, archive: gzipSync(memoryBytes) }, + }), + }, + grader: outputs.grader, + outputArtifacts: { + read: outputs.outputArtifacts.read, + put: async (input) => { + purposes.push(input.purpose) + return await outputs.outputArtifacts.put(input) + }, + }, + traceStore, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + }) + + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/protected value/), + }) + expect(purposes).not.toContain('memory-after-manifest') + expect(purposes).not.toContain('memory-after-archive') + }) + + it('redacts protected values from spans, events, artifacts, and receipt bytes', async () => { + const fixture = createCandidateExecutionFixture() + const secret = 'sk-redaction-proof-123456789' + fixture.ports.models.activateGrant = async () => ({ + env: { MODEL_GATEWAY_TOKEN: secret }, + }) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const executionOptions = options( + { + execute: async (request, context) => { + await terminalTrace(request, context.traceStore) + await context.traceStore.appendSpan({ + runId: request.trace.runId, + spanId: 'tool-secret', + kind: 'tool', + name: 'tool', + toolName: 'exec', + args: { + token: secret, + [secret]: Buffer.from(secret, 'utf8').toString('base64url'), + }, + result: secret, + startedAt: 110, + endedAt: 120, + status: 'ok', + }) + await context.traceStore.appendEvent({ + runId: request.trace.runId, + eventId: 'event-secret', + kind: 'log', + timestamp: 130, + payload: { message: secret }, + }) + await context.traceStore.appendArtifact({ + runId: request.trace.runId, + artifactId: 'artifact-secret', + contentType: 'text/plain', + sizeBytes: secret.length, + hash: '0'.repeat(64), + inlineContent: secret, + }) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore, + ) + const result = await executePreparedAgentCandidate(prepared, executionOptions) + if (!result.succeeded) throw new Error(result.reason) + const traceJson = Buffer.from( + await executionOptions.outputArtifacts.read(result.receipt.value.trace.artifact), + ).toString('utf8') + expect(traceJson).not.toContain(secret) + expect(traceJson).not.toContain(Buffer.from(secret, 'utf8').toString('base64url')) + expect(traceJson).toContain('[redacted:candidate-access]') + expect(Buffer.from(result.receipt.bytes).toString('utf8')).not.toContain(secret) + expect( + JSON.stringify({ + spans: await traceStore.spans({ runId: prepared.trace.runId }), + events: await traceStore.events({ runId: prepared.trace.runId }), + artifacts: await traceStore.artifacts(prepared.trace.runId), + }), + ).not.toContain(secret) + }) + + it('rejects protected environment collisions before executor access', async () => { + const fixture = createCandidateExecutionFixture() + fixture.bundle = candidateBundle({ + env: { PUBLIC_MODE: { kind: 'public', value: 'fixture' } }, + }) + fixture.ports.models.activateGrant = async () => ({ env: { PUBLIC_MODE: 'secret-value' } }) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let executions = 0 + const result = await executePreparedAgentCandidate( + prepared, + options({ + execute: async () => { + executions++ + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }), + ) + expect(result).toMatchObject({ succeeded: false, reason: expect.stringMatching(/collides/) }) + expect(executions).toBe(0) + }) +}) diff --git a/tests/candidate-execution-finalize.test.ts b/tests/candidate-execution-finalize.test.ts new file mode 100644 index 00000000..33707c6b --- /dev/null +++ b/tests/candidate-execution-finalize.test.ts @@ -0,0 +1,461 @@ +import { InMemoryTraceStore } from '@tangle-network/agent-eval' +import { afterEach, describe, expect, it } from 'vitest' + +import { finalizeAgentCandidateRun } from '../src/candidate-execution/finalize' +import { + type SealedAgentCandidateModelSettlement, + sealAgentCandidateModelSettlement, +} from '../src/candidate-execution/model-settlement' +import { + persistCandidateBenchmarkResult, + persistCandidateModelSettlement, + persistVerifiedCandidateTaskOutcome, +} from '../src/candidate-execution/outcome-evidence' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import { assertPreparedCandidateIntegrity } from '../src/candidate-execution/prepared-state' +import { + CANDIDATE_TRACE_TAGS, + type PreparedAgentCandidateExecution, +} from '../src/candidate-execution/types' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + candidateSha, + cleanupCandidateFixtures, + createCandidateExecutionFixture, + createCandidateOutputFixture, +} from './helpers/candidate-execution-fixture' + +afterEach(cleanupCandidateFixtures) + +async function prepared( + limits: { + timeoutMs: number + maxSteps: number + maxModelCalls: number + maxInputTokens: number + maxOutputTokens: number + maxCostUsd: number + } = { + timeoutMs: 1_000, + maxSteps: 2, + maxModelCalls: 1, + maxInputTokens: 20, + maxOutputTokens: 10, + maxCostUsd: 0.1, + }, +): Promise { + const fixture = createCandidateExecutionFixture() + fixture.task.limits = limits + return await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) +} + +async function finalizePrepared( + execution: PreparedAgentCandidateExecution, + capture: Parameters[1], + store: InMemoryTraceStore, + overrides: { + settlement?: SealedAgentCandidateModelSettlement + outputs?: ReturnType + } = {}, +) { + const state = assertPreparedCandidateIntegrity(execution) + const settlement = + overrides.settlement ?? + sealAgentCandidateModelSettlement( + { + preparationId: state.preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [ + { + callId: 'call-1', + generationId: 'llm-1', + traceSpanId: 'llm-1', + status: 'succeeded', + model: execution.resolvedModel.model, + startedAtMs: 120, + endedAtMs: 200, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 2, + reasoningTokens: 0, + costUsdNanos: 10_000_000, + }, + ], + }, + { + preparationId: state.preparationId, + grantDigest: candidateSha('c'), + model: execution.resolvedModel.model, + }, + ) + const outputs = overrides.outputs ?? createCandidateOutputFixture() + const finalCapture = { + stopped: true as const, + taskOutcome: { + resultTree: state.executionPlan.value.material.task.repository.baseTree, + afterState: state.executionPlan.value.material.task.workspace.material, + archive: Buffer.from('fixture unchanged task archive', 'utf8'), + gitDiff: Buffer.alloc(0), + }, + } + const [modelSettlement, taskOutcome] = await Promise.all([ + persistCandidateModelSettlement(state, settlement, outputs.outputArtifacts), + persistVerifiedCandidateTaskOutcome( + state, + finalCapture.taskOutcome, + outputs.outputArtifacts, + [], + ), + ]) + const benchmarkResult = await persistCandidateBenchmarkResult( + state, + capture.termination, + taskOutcome, + outputs.grader, + outputs.outputArtifacts, + [], + ) + return await finalizeAgentCandidateRun( + state, + capture, + store, + settlement, + { + finalCapture, + modelSettlement, + taskOutcome, + benchmarkResult, + outputArtifacts: outputs.outputArtifacts, + }, + [], + ) +} + +async function traceStore( + execution: PreparedAgentCandidateExecution, + overrides: { + tags?: Record + model?: string + omitUsage?: boolean + omitModelSpan?: boolean + toolSteps?: number + endedAt?: number + } = {}, +): Promise { + const store = new InMemoryTraceStore() + await store.appendRun({ + runId: execution.trace.runId, + scenarioId: 'task', + startedAt: 100, + endedAt: overrides.endedAt ?? 500, + status: 'completed', + tags: overrides.tags ?? execution.trace.tags, + }) + if (!overrides.omitModelSpan) { + await store.appendSpan({ + runId: execution.trace.runId, + spanId: 'llm-1', + kind: 'llm', + name: 'model call', + model: overrides.model ?? execution.resolvedModel.model, + messages: [], + startedAt: 120, + endedAt: 200, + status: 'ok', + attributes: { + 'tangle.protected_model.source': 'router-settlement', + 'tangle.router.call_id': 'call-1', + 'tangle.router.generation_id': 'llm-1', + }, + ...(overrides.omitUsage + ? {} + : { inputTokens: 10, outputTokens: 5, cachedTokens: 2, costUsd: 0.01 }), + }) + } + for (let index = 0; index < (overrides.toolSteps ?? 1); index++) { + await store.appendSpan({ + runId: execution.trace.runId, + spanId: `tool-${index}`, + kind: 'tool', + name: 'tool', + toolName: 'exec', + args: {}, + startedAt: 210 + index, + endedAt: 220 + index, + status: 'ok', + }) + } + return store +} + +describe('protected candidate run finalization', () => { + it('derives exact V2 evidence and durable trace bytes from tied agent-eval spans', async () => { + const execution = await prepared() + const store = await traceStore(execution) + const outputs = createCandidateOutputFixture() + const result = await finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + store, + { outputs }, + ) + expect(result.succeeded).toBe(true) + if (!result.succeeded) return + expect(result.receipt.value.usage).toEqual({ + costUsd: 0.01, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 2, + modelCalls: 1, + }) + expect(result.receipt.value.trace).toMatchObject({ eventCount: 3, modelCallCount: 1 }) + expect(result.receipt.bytes.byteLength).toBeGreaterThan(0) + const task = assertPreparedCandidateIntegrity(execution).executionPlan.value.material.task + expect(result.receipt.value).toMatchObject({ + schemaVersion: 2, + fixedUsage: { + costUsdNanos: 10_000_000, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 2, + modelCalls: 1, + }, + taskOutcome: { + artifact: result.artifacts.taskOutcome, + material: { + executionPlanDigest: execution.executionPlan.value.digest, + baseRepository: { + identity: task.repository.identity, + rootIdentity: task.repository.rootIdentity, + commit: task.repository.baseCommit, + tree: task.repository.baseTree, + }, + resultRepository: { + identity: task.repository.identity, + rootIdentity: task.repository.rootIdentity, + commit: expect.stringMatching(/^[0-9a-f]{40}$/), + tree: task.repository.baseTree, + }, + }, + }, + benchmarkResult: { + artifact: result.artifacts.benchmarkResult, + material: { + taskOutcomeDigest: result.receipt.value.taskOutcome.digest, + score: 1, + passed: true, + }, + }, + modelSettlement: { + artifact: result.artifacts.modelSettlement, + material: { + executionPlanDigest: execution.executionPlan.value.digest, + closed: true, + usage: { + costUsdNanos: 10_000_000, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 2, + modelCalls: 1, + }, + }, + }, + }) + await Promise.all([ + expect(outputs.outputArtifacts.read(result.artifacts.runReceipt)).resolves.toEqual( + result.receipt.bytes, + ), + expect( + outputs.outputArtifacts.read(result.receipt.value.taskOutcome.material.gitDiff.artifact), + ).resolves.toEqual(new Uint8Array()), + expect( + outputs.outputArtifacts.read(result.receipt.value.benchmarkResult.material.evidence), + ).resolves.toEqual(Uint8Array.from(Buffer.from('{"reward":1}\n', 'utf8'))), + expect( + outputs.outputArtifacts.read(result.receipt.value.benchmarkResult.material.grader.artifact), + ).resolves.toEqual(Uint8Array.from(Buffer.from('fixture executable grader', 'utf8'))), + ]) + }) + + it('returns invalid capture instead of minting zero usage when provider usage is missing', async () => { + const execution = await prepared() + const result = await finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { omitUsage: true }), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/model ledger/), + }) + }) + + it('rejects a dropped model span against the protected gateway ledger', async () => { + const execution = await prepared() + const result = await finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { omitModelSpan: true }), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/model ledger/), + }) + }) + + it('rejects trace-binding and resolved-model drift', async () => { + const execution = await prepared() + const badTags = { + ...execution.trace.tags, + [CANDIDATE_TRACE_TAGS.bundleDigest]: candidateSha('9'), + } + await expect( + finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { tags: badTags }), + ), + ).resolves.toMatchObject({ succeeded: false, reason: expect.stringMatching(/not bound/) }) + await expect( + finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { model: 'other-model' }), + ), + ).resolves.toMatchObject({ succeeded: false, reason: expect.stringMatching(/does not match/) }) + }) + + it('enforces tool-step and wall-time limits from protected spans', async () => { + const execution = await prepared() + await expect( + finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { toolSteps: 3 }), + ), + ).resolves.toMatchObject({ succeeded: false, reason: expect.stringMatching(/tool steps/) }) + await expect( + finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { endedAt: 1_101 }), + ), + ).resolves.toMatchObject({ succeeded: false, reason: expect.stringMatching(/wall time/) }) + }) + + it('preserves timeout usage but requires the exact frozen timeout', async () => { + const execution = await prepared() + const store = await traceStore(execution) + const valid = await finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'timeout', timeoutMs: 1_000 } }, + store, + ) + expect(valid.succeeded).toBe(true) + await expect( + finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'timeout', timeoutMs: 999 } }, + store, + ), + ).resolves.toMatchObject({ succeeded: false, reason: expect.stringMatching(/frozen/) }) + }) + + it('reconciles decimal costs in fixed-point units', async () => { + const execution = await prepared({ + timeoutMs: 1_000, + maxSteps: 2, + maxModelCalls: 2, + maxInputTokens: 20, + maxOutputTokens: 10, + maxCostUsd: 0.3, + }) + const store = new InMemoryTraceStore() + await store.appendRun({ + runId: execution.trace.runId, + scenarioId: 'decimal-cost', + startedAt: 100, + endedAt: 500, + status: 'completed', + tags: execution.trace.tags, + }) + for (const [index, costUsd] of [0.1, 0.2].entries()) { + await store.appendSpan({ + runId: execution.trace.runId, + spanId: `llm-${index + 1}`, + kind: 'llm', + name: 'model call', + model: execution.resolvedModel.model, + messages: [], + inputTokens: 5, + outputTokens: 2, + costUsd, + startedAt: 120 + index * 100, + endedAt: 180 + index * 100, + status: 'ok', + attributes: { + 'tangle.protected_model.source': 'router-settlement', + 'tangle.router.call_id': `call-${index + 1}`, + 'tangle.router.generation_id': `llm-${index + 1}`, + }, + }) + } + const settlement = sealAgentCandidateModelSettlement( + { + preparationId: assertPreparedCandidateIntegrity(execution).preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [ + { + callId: 'call-1', + generationId: 'llm-1', + traceSpanId: 'llm-1', + status: 'succeeded', + model: execution.resolvedModel.model, + startedAtMs: 120, + endedAtMs: 180, + inputTokens: 5, + outputTokens: 2, + cachedInputTokens: 0, + reasoningTokens: 0, + costUsdNanos: 100_000_000, + }, + { + callId: 'call-2', + generationId: 'llm-2', + traceSpanId: 'llm-2', + status: 'succeeded', + model: execution.resolvedModel.model, + startedAtMs: 220, + endedAtMs: 280, + inputTokens: 5, + outputTokens: 2, + cachedInputTokens: 0, + reasoningTokens: 0, + costUsdNanos: 200_000_000, + }, + ], + }, + { + preparationId: assertPreparedCandidateIntegrity(execution).preparationId, + grantDigest: candidateSha('c'), + model: execution.resolvedModel.model, + }, + ) + const result = await finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + store, + { settlement }, + ) + expect(result).toMatchObject({ + succeeded: true, + receipt: { value: { usage: { costUsd: 0.3, modelCalls: 2 } } }, + }) + }) +}) diff --git a/tests/candidate-execution-model-port.test.ts b/tests/candidate-execution-model-port.test.ts new file mode 100644 index 00000000..e2464ff5 --- /dev/null +++ b/tests/candidate-execution-model-port.test.ts @@ -0,0 +1,611 @@ +import { InMemoryTraceStore, type LlmSpan } from '@tangle-network/agent-eval' +import type { AgentCandidateResolvedModel, Sha256Digest } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { + appendAuthoritativeModelSettlementSpans, + assertTraceMatchesModelSettlement, + sealAgentCandidateModelSettlement, +} from '../src/candidate-execution/model-settlement' +import { + type AgentCandidateModelGrantActivateInput, + type AgentCandidateModelGrantClient, + type AgentCandidateModelGrantReservation, + type AgentCandidateModelGrantReserveInput, + type AgentCandidateModelGrantSettleInput, + createProtectedAgentCandidateModelPort, +} from '../src/candidate-execution/protected-model-port' +import type { + AgentCandidateModelPort, + AgentCandidateProtectedModelActivation, + AgentCandidateProtectedModelCall, + AgentCandidateProtectedModelSettlement, +} from '../src/candidate-execution/types' + +const GATEWAY_DOMAIN = 'router.tangle.tools' +const ACTIVATION_ENV_NAMES = ['MODEL_API_KEY', 'MODEL_BASE_URL'] as const +const EXPIRES_AT_MS = 2_000_000_000_000 + +function sha(character: string): Sha256Digest { + return `sha256:${character.repeat(64)}` as Sha256Digest +} + +const resolvedModel: AgentCandidateResolvedModel = { + requested: 'openai/gpt-5.2-codex', + provider: 'openai', + model: 'openai/gpt-5.2-codex-2026-07-01', + snapshot: 'gpt-5.2-codex-2026-07-01', + reasoningEffort: 'high', +} + +function reserveInput( + limits: AgentCandidateModelGrantReserveInput['limits'] = { + maxModelCalls: 2, + maxInputTokens: 30, + maxOutputTokens: 20, + maxCostUsd: 0.1, + }, +): AgentCandidateModelGrantReserveInput { + return { + executionId: 'execution-1', + preparationId: 'preparation-1', + expiresAtMs: EXPIRES_AT_MS, + attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, + bundleDigest: sha('a'), + resolved: resolvedModel, + limits, + } +} + +function reservation( + input: AgentCandidateModelGrantReserveInput, +): AgentCandidateModelGrantReservation { + return { + preparationId: input.preparationId, + digest: sha('b'), + expiresAtMs: input.expiresAtMs, + enforcedLimits: input.limits, + network: + input.limits.maxModelCalls === 0 + ? { mode: 'disabled' } + : { mode: 'gateway-only', domains: [GATEWAY_DOMAIN] }, + } +} + +function activateInput(): AgentCandidateModelGrantActivateInput { + return { + executionId: 'execution-1', + preparationId: 'preparation-1', + grantDigest: sha('b'), + resolved: resolvedModel, + deadlineAtMs: EXPIRES_AT_MS - 1_000, + } +} + +function settleInput( + reason: AgentCandidateModelGrantSettleInput['reason'] = 'completed', +): AgentCandidateModelGrantSettleInput { + return { + executionId: 'execution-1', + preparationId: 'preparation-1', + grantDigest: sha('b'), + resolved: resolvedModel, + reason, + } +} + +function modelCall( + index: number, + overrides: Partial = {}, +): AgentCandidateProtectedModelCall { + return { + callId: `call-${index}`, + generationId: `generation-${index}`, + traceSpanId: `generation-${index}`, + status: 'succeeded', + model: resolvedModel.model, + startedAtMs: 1_000 + index * 100, + endedAtMs: 1_050 + index * 100, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 0, + reasoningTokens: 0, + costUsdNanos: 10_000_000, + ...overrides, + } +} + +function settlement( + calls: readonly AgentCandidateProtectedModelCall[] = [], +): AgentCandidateProtectedModelSettlement { + return { + preparationId: 'preparation-1', + grantDigest: sha('b'), + closed: true, + calls, + } +} + +interface FakeClientOptions { + reserve?: ( + input: AgentCandidateModelGrantReserveInput, + ) => Promise + activate?: ( + input: AgentCandidateModelGrantActivateInput, + ) => Promise + settle?: ( + input: AgentCandidateModelGrantSettleInput, + ) => Promise +} + +function deferred(): { promise: Promise; resolve: (value: T) => void } { + let resolve!: (value: T) => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +function fakeClient(options: FakeClientOptions = {}): AgentCandidateModelGrantClient & { + reserveInputs: AgentCandidateModelGrantReserveInput[] + activateInputs: AgentCandidateModelGrantActivateInput[] + settleInputs: AgentCandidateModelGrantSettleInput[] +} { + const reserveInputs: AgentCandidateModelGrantReserveInput[] = [] + const activateInputs: AgentCandidateModelGrantActivateInput[] = [] + const settleInputs: AgentCandidateModelGrantSettleInput[] = [] + return { + reserveInputs, + activateInputs, + settleInputs, + reserve: async (input) => { + reserveInputs.push(input) + return options.reserve ? await options.reserve(input) : reservation(input) + }, + activate: async (input) => { + activateInputs.push(input) + return options.activate + ? await options.activate(input) + : { + env: { + MODEL_API_KEY: 'protected-candidate-token', + MODEL_BASE_URL: 'https://router.tangle.tools/v1', + }, + } + }, + settle: async (input) => { + settleInputs.push(input) + return options.settle ? await options.settle(input) : settlement() + }, + } +} + +function createPort(client: AgentCandidateModelGrantClient): AgentCandidateModelPort { + return createProtectedAgentCandidateModelPort({ + client, + gatewayDomain: GATEWAY_DOMAIN, + activationEnvNames: ACTIVATION_ENV_NAMES, + resolveModel: async ({ requested, reasoningEffort }) => ({ + ...resolvedModel, + requested, + reasoningEffort, + }), + }) +} + +async function reserve(port: AgentCandidateModelPort, input = reserveInput()) { + return await port.reserveGrant(input) +} + +describe('protected candidate model port', () => { + it('reserves without a live secret, activates exact protected env, and settles exact usage', async () => { + const client = fakeClient({ + settle: async () => + settlement([ + modelCall(1, { + inputTokens: 20, + outputTokens: 10, + cachedInputTokens: 4, + reasoningTokens: 3, + costUsdNanos: 100_000_000, + }), + ]), + }) + const port = createPort(client) + + await expect( + port.resolve({ + requested: resolvedModel.requested, + harness: 'opencode', + reasoningEffort: resolvedModel.reasoningEffort, + }), + ).resolves.toEqual(resolvedModel) + const reserved = await reserve(port) + expect(JSON.stringify(reserved)).not.toContain('protected-candidate-token') + expect(reserved).toEqual({ + preparationId: 'preparation-1', + digest: sha('b'), + expiresAtMs: EXPIRES_AT_MS, + enforcedLimits: reserveInput().limits, + network: { mode: 'gateway-only', domains: [GATEWAY_DOMAIN] }, + }) + expect(Object.isFrozen(reserved)).toBe(true) + + const activated = await port.activateGrant(activateInput()) + expect(activated).toEqual({ + env: { + MODEL_API_KEY: 'protected-candidate-token', + MODEL_BASE_URL: 'https://router.tangle.tools/v1', + }, + }) + expect(Object.isFrozen(activated.env)).toBe(true) + + const settled = await port.settleGrant(settleInput('completed')) + expect(settled.calls).toEqual([ + modelCall(1, { + inputTokens: 20, + outputTokens: 10, + cachedInputTokens: 4, + reasoningTokens: 3, + costUsdNanos: 100_000_000, + }), + ]) + expect(client.settleInputs).toEqual([settleInput('completed')]) + expect(Object.isFrozen(settled.calls)).toBe(true) + }) + + it('keeps zero-call reservation, activation, and settlement entirely credential-free', async () => { + const client = fakeClient({ activate: async () => ({ env: {} }) }) + const port = createPort(client) + const input = reserveInput({ + maxModelCalls: 0, + maxInputTokens: 0, + maxOutputTokens: 0, + maxCostUsd: 0, + }) + await expect(port.reserveGrant(input)).resolves.toMatchObject({ + network: { mode: 'disabled' }, + }) + await expect(port.activateGrant(activateInput())).resolves.toEqual({ env: {} }) + await expect(port.settleGrant(settleInput())).resolves.toEqual(settlement()) + expect(client.activateInputs).toHaveLength(1) + }) + + it('rejects any protected env or final model row for a zero-call reservation', async () => { + const client = fakeClient() + const port = createPort(client) + const input = reserveInput({ + maxModelCalls: 0, + maxInputTokens: 0, + maxOutputTokens: 0, + maxCostUsd: 0, + }) + await port.reserveGrant(input) + await expect(port.activateGrant(activateInput())).rejects.toThrow(/environment names/) + + const rowClient = fakeClient({ + activate: async () => ({ env: {} }), + settle: async () => settlement([modelCall(1)]), + }) + const rowPort = createPort(rowClient) + await rowPort.reserveGrant(input) + await rowPort.activateGrant(activateInput()) + await expect(rowPort.settleGrant(settleInput())).rejects.toThrow( + /model calls.*exceeds reserved/, + ) + }) + + it('rejects any secret or other extra field in the reservation response', async () => { + const client = fakeClient({ + reserve: async (input) => + ({ + ...reservation(input), + env: { MODEL_API_KEY: 'must-not-exist-before-activation' }, + }) as AgentCandidateModelGrantReservation, + }) + await expect(reserve(createPort(client))).rejects.toThrow(/unknown field env/) + }) + + it('requires reservation retries to preserve both request and returned evidence', async () => { + let digest = sha('b') + const client = fakeClient({ + reserve: async (input) => ({ ...reservation(input), digest }), + }) + const port = createPort(client) + await expect(reserve(port)).resolves.toMatchObject({ digest: sha('b') }) + await expect( + reserve(port, reserveInput({ ...reserveInput().limits, maxInputTokens: 29 })), + ).rejects.toThrow(/changed immutable input/) + expect(client.reserveInputs).toHaveLength(1) + + digest = sha('c') + await expect(reserve(port)).rejects.toThrow(/different evidence/) + }) + + it('linearizes concurrent reservation retries before accepting changed input', async () => { + const firstResponse = deferred() + const secondResponse = deferred() + const client = fakeClient({ + reserve: async (input) => + input.limits.maxInputTokens === 30 ? firstResponse.promise : secondResponse.promise, + }) + const port = createPort(client) + const firstInput = reserveInput() + const secondInput = reserveInput({ ...firstInput.limits, maxInputTokens: 29 }) + const first = reserve(port, firstInput) + const second = reserve(port, secondInput) + + firstResponse.resolve(reservation(firstInput)) + await expect(first).resolves.toMatchObject({ enforcedLimits: firstInput.limits }) + secondResponse.resolve(reservation(secondInput)) + await expect(second).rejects.toThrow(/changed immutable input/) + }) + + it('does not resurrect a reservation when a concurrent retry returns after settlement', async () => { + const firstResponse = deferred() + const secondResponse = deferred() + let call = 0 + const client = fakeClient({ + reserve: async () => (++call === 1 ? firstResponse.promise : secondResponse.promise), + }) + const port = createPort(client) + const input = reserveInput() + const first = reserve(port, input) + const second = reserve(port, input) + firstResponse.resolve(reservation(input)) + await first + await port.settleGrant(settleInput()) + + secondResponse.resolve(reservation(input)) + await expect(second).rejects.toThrow(/after the grant was settled/) + await expect(port.activateGrant(activateInput())).rejects.toThrow(/already settled/) + }) + + it.each([ + [ + 'expiry', + (input: AgentCandidateModelGrantReserveInput) => ({ + ...reservation(input), + expiresAtMs: input.expiresAtMs + 1, + }), + ], + [ + 'limits', + (input: AgentCandidateModelGrantReserveInput) => ({ + ...reservation(input), + enforcedLimits: { ...input.limits, maxInputTokens: input.limits.maxInputTokens + 1 }, + }), + ], + [ + 'domain', + (input: AgentCandidateModelGrantReserveInput) => ({ + ...reservation(input), + network: { mode: 'gateway-only' as const, domains: ['other.example.com'] }, + }), + ], + ])('rejects reservation %s drift', async (_name, mutate) => { + const client = fakeClient({ reserve: async (input) => mutate(input) }) + await expect(reserve(createPort(client))).rejects.toThrow(/changed|unexpected/) + }) + + it('requires the activation endpoint to return exactly the configured protected env', async () => { + const extra = fakeClient({ + activate: async () => ({ + env: { + MODEL_API_KEY: 'protected-candidate-token', + MODEL_BASE_URL: 'https://router.tangle.tools/v1', + DATABASE_URL: 'must-never-reach-candidate', + }, + }), + }) + const extraPort = createPort(extra) + await reserve(extraPort) + await expect(extraPort.activateGrant(activateInput())).rejects.toThrow(/environment names/) + + const missing = fakeClient({ + activate: async () => ({ env: { MODEL_API_KEY: 'protected-candidate-token' } }), + }) + const missingPort = createPort(missing) + await reserve(missingPort) + await expect(missingPort.activateGrant(activateInput())).rejects.toThrow(/environment names/) + }) + + it('rejects activation before reservation, after settlement, and with changed grant identity', async () => { + const client = fakeClient() + const port = createPort(client) + await expect(port.activateGrant(activateInput())).rejects.toThrow(/not reserved/) + + await reserve(port) + await expect(port.activateGrant({ ...activateInput(), grantDigest: sha('c') })).rejects.toThrow( + /immutable reservation/, + ) + await port.settleGrant(settleInput()) + await expect(port.activateGrant(activateInput())).rejects.toThrow(/already settled/) + expect(client.activateInputs).toHaveLength(0) + }) + + it('treats protected environment activation as single-use', async () => { + const client = fakeClient() + const port = createPort(client) + await reserve(port) + await expect(port.activateGrant(activateInput())).resolves.toMatchObject({ env: {} }) + await expect(port.activateGrant(activateInput())).rejects.toThrow(/single-use/) + expect(client.activateInputs).toHaveLength(1) + }) + + it('linearizes concurrent activation before the service returns a credential', async () => { + const response = deferred() + const client = fakeClient({ activate: async () => response.promise }) + const port = createPort(client) + await reserve(port) + const first = port.activateGrant(activateInput()) + await expect(port.activateGrant(activateInput())).rejects.toThrow(/single-use/) + response.resolve({ + env: { + MODEL_API_KEY: 'protected-candidate-token', + MODEL_BASE_URL: 'https://router.tangle.tools/v1', + }, + }) + await expect(first).resolves.toMatchObject({ env: {} }) + expect(client.activateInputs).toHaveLength(1) + }) + + it('does not return a credential when settlement wins an activation race', async () => { + const response = deferred() + const client = fakeClient({ activate: async () => response.promise }) + const port = createPort(client) + await reserve(port) + const activation = port.activateGrant(activateInput()) + await port.settleGrant(settleInput('failed')) + response.resolve({ + env: { + MODEL_API_KEY: 'protected-candidate-token', + MODEL_BASE_URL: 'https://router.tangle.tools/v1', + }, + }) + await expect(activation).rejects.toThrow(/settled while activation was in flight/) + }) + + it.each([ + [ + 'model calls', + [ + modelCall(1, { inputTokens: 0, outputTokens: 0, costUsdNanos: 0 }), + modelCall(2, { inputTokens: 0, outputTokens: 0, costUsdNanos: 0 }), + modelCall(3, { inputTokens: 0, outputTokens: 0, costUsdNanos: 0 }), + ], + ], + ['input tokens', [modelCall(1, { inputTokens: 31, outputTokens: 0, costUsdNanos: 0 })]], + ['output tokens', [modelCall(1, { inputTokens: 0, outputTokens: 21, costUsdNanos: 0 })]], + [ + 'cost USD nanos', + [modelCall(1, { inputTokens: 0, outputTokens: 0, costUsdNanos: 100_000_001 })], + ], + ])('rejects %s above the immutable reservation', async (label, calls) => { + const client = fakeClient({ settle: async () => settlement(calls) }) + const port = createPort(client) + await reserve(port) + await expect(port.settleGrant(settleInput())).rejects.toThrow( + new RegExp(`${label}.*exceeds reserved`), + ) + }) + + it.each([ + [ + 'duplicate call ids', + [modelCall(1), modelCall(2, { callId: 'call-1' })], + /duplicate call ids/, + ], + [ + 'duplicate trace span ids', + [modelCall(1), modelCall(2, { generationId: 'generation-1', traceSpanId: 'generation-1' })], + /duplicate trace span ids/, + ], + ])('rejects %s in the final ledger', async (_name, calls, pattern) => { + const client = fakeClient({ settle: async () => settlement(calls) }) + const port = createPort(client) + await reserve(port) + await expect(port.settleGrant(settleInput())).rejects.toThrow(pattern) + }) + + it.each([ + ['preparation', { ...settlement(), preparationId: 'other-preparation' }, /preparation/], + ['grant', { ...settlement(), grantDigest: sha('c') }, /grant digest/], + [ + 'router generation', + settlement([modelCall(1, { traceSpanId: 'caller-chosen-span' })]), + /router generationId/, + ], + ['model', settlement([modelCall(1, { model: 'openai/other-snapshot' })]), /unexpected model/], + ])('rejects a mismatched %s in the final ledger', async (_name, value, pattern) => { + const client = fakeClient({ settle: async () => value }) + const port = createPort(client) + await reserve(port) + await expect(port.settleGrant(settleInput())).rejects.toThrow(pattern) + }) + + it('rejects a partial or structurally extended final ledger', async () => { + const partial = fakeClient({ + settle: async () => ({ ...settlement(), closed: false }) as never, + }) + const partialPort = createPort(partial) + await reserve(partialPort) + await expect(partialPort.settleGrant(settleInput())).rejects.toThrow(/not closed/) + + const extended = fakeClient({ + settle: async () => + settlement([{ ...modelCall(1), providerRequest: 'untrusted-extra-field' } as never]), + }) + const extendedPort = createPort(extended) + await reserve(extendedPort) + await expect(extendedPort.settleGrant(settleInput())).rejects.toThrow(/unknown field/) + }) + + it('requires settlement retries to preserve both final rows and termination reason', async () => { + let calls = [modelCall(1)] + const client = fakeClient({ settle: async () => settlement(calls) }) + const port = createPort(client) + await reserve(port) + await expect(port.settleGrant(settleInput('failed'))).resolves.toEqual(settlement(calls)) + await expect(port.settleGrant(settleInput('failed'))).resolves.toEqual(settlement(calls)) + + calls = [modelCall(2)] + await expect(port.settleGrant(settleInput('failed'))).rejects.toThrow(/different final ledger/) + calls = [modelCall(1)] + await expect(port.settleGrant(settleInput('timeout'))).rejects.toThrow(/termination reason/) + expect(client.settleInputs).toHaveLength(3) + }) + + it('composes with the existing trace check to reject missing or extra model rows 1:1', async () => { + const client = fakeClient({ settle: async () => settlement([modelCall(1)]) }) + const port = createPort(client) + await reserve(port) + const settled = await port.settleGrant(settleInput()) + const sealed = sealAgentCandidateModelSettlement(settled, { + preparationId: 'preparation-1', + grantDigest: sha('b'), + model: resolvedModel.model, + }) + const span = (index: number): LlmSpan => ({ + spanId: `generation-${index}`, + runId: 'run-1', + kind: 'llm', + name: 'protected model call', + startedAt: 1_000 + index * 100, + endedAt: 1_050 + index * 100, + status: 'ok', + model: resolvedModel.model, + messages: [], + inputTokens: 10, + outputTokens: 5, + cachedTokens: 0, + reasoningTokens: 0, + costUsd: 0.01, + attributes: { + 'tangle.protected_model.source': 'router-settlement', + 'tangle.router.call_id': `call-${index}`, + 'tangle.router.generation_id': `generation-${index}`, + }, + }) + + expect(() => assertTraceMatchesModelSettlement([], sealed)).toThrow(/do not match/) + expect(() => assertTraceMatchesModelSettlement([span(1), span(2)], sealed)).toThrow( + /do not match/, + ) + expect(() => assertTraceMatchesModelSettlement([span(1)], sealed)).not.toThrow() + + const store = new InMemoryTraceStore() + await store.appendRun({ + runId: 'run-1', + scenarioId: 'candidate-model-port', + startedAt: 900, + endedAt: 1_200, + status: 'completed', + }) + await appendAuthoritativeModelSettlementSpans(store, 'run-1', sealed) + const protectedSpans = (await store.spans({ runId: 'run-1' })).filter( + (candidate): candidate is LlmSpan => candidate.kind === 'llm', + ) + expect(protectedSpans).toEqual([span(1)]) + expect(() => assertTraceMatchesModelSettlement(protectedSpans, sealed)).not.toThrow() + }) +}) diff --git a/tests/candidate-execution-outcome-evidence.test.ts b/tests/candidate-execution-outcome-evidence.test.ts new file mode 100644 index 00000000..05bdade4 --- /dev/null +++ b/tests/candidate-execution-outcome-evidence.test.ts @@ -0,0 +1,366 @@ +import { chmodSync, mkdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { gunzipSync, gzipSync } from 'node:zlib' + +import { afterEach, describe, expect, it } from 'vitest' +import { sha256Bytes } from '../src/candidate-execution/digest' +import { + persistCandidateBenchmarkResult, + persistVerifiedCandidateTaskOutcome, +} from '../src/candidate-execution/outcome-evidence' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import { assertPreparedCandidateIntegrity } from '../src/candidate-execution/prepared-state' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + cleanupCandidateFixtures, + createCandidateExecutionFixture, + createCandidateOutputFixture, + unchangedTaskOutcomeCapture, +} from './helpers/candidate-execution-fixture' + +afterEach(cleanupCandidateFixtures) + +describe('candidate outcome evidence', () => { + it('rejects protected values in the canonical task manifest before any output write', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + let puts = 0 + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + puts++ + return await outputs.outputArtifacts.put(input) + }, + } + + await expect( + persistVerifiedCandidateTaskOutcome( + state, + unchangedTaskOutcomeCapture(fixture), + outputArtifacts, + ['source.ts'], + ), + ).rejects.toThrow(/protected value/) + expect(puts).toBe(0) + }) + + it('validates task manifest material before any output write', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + let puts = 0 + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + puts++ + return await outputs.outputArtifacts.put(input) + }, + } + const capture = unchangedTaskOutcomeCapture(fixture) + const [file] = capture.afterState.files + if (!file) throw new Error('fixture task manifest is empty') + + await expect( + persistVerifiedCandidateTaskOutcome( + state, + { + ...capture, + afterState: { ...capture.afterState, files: [file, file] }, + }, + outputArtifacts, + [], + ), + ).rejects.toThrow(/workspace manifest paths must be unique/) + expect(puts).toBe(0) + }) + + it('rejects a compressed invalid archive before any candidate output is persisted', async () => { + const fixture = createCandidateExecutionFixture() + fixture.ports.workspaces.materialize = async ({ archive, destination }) => { + if (destination === fixture.task.stagingRoots.taskRoot) return + mkdirSync(destination, { recursive: true }) + const output = join(destination, 'source.ts') + writeFileSync(output, gunzipSync(archive)) + chmodSync(output, 0o644) + } + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + let puts = 0 + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + puts++ + return await outputs.outputArtifacts.put(input) + }, + } + const secret = 'sk-compressed-before-put-123456789' + + await expect( + persistVerifiedCandidateTaskOutcome( + state, + { + ...unchangedTaskOutcomeCapture(fixture), + archive: gzipSync(Buffer.from(secret, 'utf8')), + }, + outputArtifacts, + [secret], + ), + ).rejects.toThrow(/do not match the signed manifest/) + expect(puts).toBe(0) + }) + + it('rejects corrupted persisted archive bytes without publishing task outcome evidence', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const purposes: string[] = [] + const purposeByDigest = new Map() + const outputArtifacts = { + put: async (input: Parameters[0]) => { + purposes.push(input.purpose) + const ref = await outputs.outputArtifacts.put(input) + purposeByDigest.set(ref.sha256, input.purpose) + return ref + }, + read: async (ref: Parameters[0]) => { + const bytes = await outputs.outputArtifacts.read(ref) + if (purposeByDigest.get(ref.sha256) !== 'task-archive') return bytes + const corrupted = Uint8Array.from(bytes) + corrupted[0] = (corrupted[0] ?? 0) ^ 0xff + return corrupted + }, + } + + await expect( + persistVerifiedCandidateTaskOutcome( + state, + unchangedTaskOutcomeCapture(fixture), + outputArtifacts, + [], + ), + ).rejects.toThrow(/persisted candidate output digest/) + expect(purposes.sort()).toEqual(['task-archive', 'task-manifest']) + expect(purposes).not.toContain('task-patch') + expect(purposes).not.toContain('task-outcome') + }) + + it('rejects a false archive reference without publishing task outcome evidence', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const purposes: string[] = [] + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + purposes.push(input.purpose) + const ref = await outputs.outputArtifacts.put(input) + return input.purpose === 'task-archive' ? { ...ref, byteLength: ref.byteLength + 1 } : ref + }, + } + + await expect( + persistVerifiedCandidateTaskOutcome( + state, + unchangedTaskOutcomeCapture(fixture), + outputArtifacts, + [], + ), + ).rejects.toThrow(/does not identify the submitted bytes/) + expect(purposes.sort()).toEqual(['task-archive', 'task-manifest']) + expect(purposes).not.toContain('task-patch') + expect(purposes).not.toContain('task-outcome') + }) + + it('rejects protected access values in task archives and raw grader output', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const secret = 'sk-outcome-secret-123456789' + const capture = unchangedTaskOutcomeCapture(fixture) + await expect( + persistVerifiedCandidateTaskOutcome( + state, + { ...capture, archive: Buffer.from(secret, 'utf8') }, + outputs.outputArtifacts, + [secret], + ), + ).rejects.toThrow(/protected value/) + + const outcome = await persistVerifiedCandidateTaskOutcome( + state, + capture, + outputs.outputArtifacts, + [secret], + ) + await expect( + persistCandidateBenchmarkResult( + state, + { kind: 'exit', exitCode: 0 }, + outcome, + { + ...outputs.grader, + run: async ({ implementation, outcome }) => { + const evidence = Buffer.from(secret, 'utf8') + return { + evaluation: { score: 1, passed: true, raw: {} }, + evidence, + binding: { + implementationDigest: sha256Bytes(implementation.bytes), + taskOutcomeDigest: outcome.evidence.digest, + outputDigest: sha256Bytes(evidence), + }, + } + }, + }, + outputs.outputArtifacts, + [secret], + ), + ).rejects.toThrow(/protected value/) + }) + + it('retains raw executable-grade evidence but cannot pass a nonzero exit', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const outcome = await persistVerifiedCandidateTaskOutcome( + state, + unchangedTaskOutcomeCapture(fixture), + outputs.outputArtifacts, + [], + ) + const result = await persistCandidateBenchmarkResult( + state, + { kind: 'exit', exitCode: 1 }, + outcome, + outputs.grader, + outputs.outputArtifacts, + [], + ) + + expect(result.material).toMatchObject({ + score: 0, + passed: false, + dimensions: [{ name: 'tests', score: 0 }], + }) + expect(Buffer.from(await outputs.outputArtifacts.read(result.material.evidence))).toEqual( + Buffer.from('{"reward":1}\n', 'utf8'), + ) + expect(result.material.evidence.sha256).not.toBe(result.material.grader.artifact.sha256) + }) + + it('cannot persist a result when the runner binding does not match executed bytes', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const outcome = await persistVerifiedCandidateTaskOutcome( + state, + unchangedTaskOutcomeCapture(fixture), + outputs.outputArtifacts, + [], + ) + const persistedPurposes: string[] = [] + const observingArtifacts = { + ...outputs.outputArtifacts, + put: async (input: Parameters[0]) => { + persistedPurposes.push(input.purpose) + return await outputs.outputArtifacts.put(input) + }, + } + let admittedImplementation = Buffer.alloc(0) + + await expect( + persistCandidateBenchmarkResult( + state, + { kind: 'exit', exitCode: 0 }, + outcome, + { + ...outputs.grader, + run: async ({ implementation, outcome: gradedOutcome }) => { + admittedImplementation = Buffer.from(implementation.bytes) + const evidence = Buffer.from('{"reward":1}\n', 'utf8') + return { + evaluation: { score: 1, passed: true, raw: {} }, + evidence, + binding: { + implementationDigest: sha256Bytes(Buffer.from('different grader', 'utf8')), + taskOutcomeDigest: gradedOutcome.evidence.digest, + outputDigest: sha256Bytes(evidence), + }, + } + }, + }, + observingArtifacts, + [], + ), + ).rejects.toThrow(/executed implementation digest does not match/) + + expect(admittedImplementation).toEqual(Buffer.from('fixture executable grader', 'utf8')) + expect(persistedPurposes).toEqual([]) + }) + + it('cannot persist a result bound to another task outcome or raw output', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const outcome = await persistVerifiedCandidateTaskOutcome( + state, + unchangedTaskOutcomeCapture(fixture), + outputs.outputArtifacts, + [], + ) + const persistedPurposes: string[] = [] + const observingArtifacts = { + ...outputs.outputArtifacts, + put: async (input: Parameters[0]) => { + persistedPurposes.push(input.purpose) + return await outputs.outputArtifacts.put(input) + }, + } + + for (const mismatch of ['task', 'output'] as const) { + await expect( + persistCandidateBenchmarkResult( + state, + { kind: 'exit', exitCode: 0 }, + outcome, + { + ...outputs.grader, + run: async ({ implementation, outcome: gradedOutcome }) => { + const evidence = Buffer.from('{"reward":1}\n', 'utf8') + return { + evaluation: { score: 1, passed: true, raw: {} }, + evidence, + binding: { + implementationDigest: sha256Bytes(implementation.bytes), + taskOutcomeDigest: + mismatch === 'task' + ? sha256Bytes(Buffer.from('other task', 'utf8')) + : gradedOutcome.evidence.digest, + outputDigest: + mismatch === 'output' + ? sha256Bytes(Buffer.from('other output', 'utf8')) + : sha256Bytes(evidence), + }, + } + }, + }, + observingArtifacts, + [], + ), + ).rejects.toThrow(/does not match/) + } + + expect(persistedPurposes).toEqual([]) + }) +}) + +async function preparedState(fixture: ReturnType) { + return assertPreparedCandidateIntegrity( + await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ), + ) +} diff --git a/tests/candidate-execution-output-artifacts.test.ts b/tests/candidate-execution-output-artifacts.test.ts new file mode 100644 index 00000000..f138bcb4 --- /dev/null +++ b/tests/candidate-execution-output-artifacts.test.ts @@ -0,0 +1,75 @@ +import { createHash } from 'node:crypto' + +import { describe, expect, it } from 'vitest' + +import { persistCandidateOutputArtifact } from '../src/candidate-execution/output-artifacts' +import type { AgentCandidateOutputArtifactPort } from '../src/candidate-execution/types' +import { candidateSha } from './helpers/candidate-execution-fixture' + +describe('candidate output artifact persistence', () => { + it('does not call the durable store after result cancellation', async () => { + const controller = new AbortController() + controller.abort(new Error('result deadline')) + let puts = 0 + const port: AgentCandidateOutputArtifactPort = { + put: async () => { + puts++ + throw new Error('must not write') + }, + read: async () => new Uint8Array(), + } + + await expect( + persistCandidateOutputArtifact(port, { + executionId: 'execution-1', + purpose: 'trace', + bytes: Buffer.from('late'), + signal: controller.signal, + }), + ).rejects.toThrow(/result deadline/) + expect(puts).toBe(0) + }) + + it('reads back exact durable bytes before returning the locator', async () => { + const bytes = Buffer.from('durable outcome') + const stored = new Map() + const port: AgentCandidateOutputArtifactPort = { + put: async ({ bytes: value }) => { + const ref = { + locator: { kind: 's3' as const, bucket: 'candidate-results', key: 'outcome.bin' }, + sha256: `sha256:${createHash('sha256').update(value).digest('hex')}` as const, + byteLength: value.byteLength, + } + stored.set(ref.sha256, Uint8Array.from(value)) + return ref + }, + read: async (ref) => stored.get(ref.sha256) ?? new Uint8Array(), + } + + await expect( + persistCandidateOutputArtifact(port, { + executionId: 'execution-1', + purpose: 'task-patch', + bytes, + }), + ).resolves.toMatchObject({ byteLength: bytes.byteLength }) + }) + + it('rejects a locator that lies about the submitted content', async () => { + const port: AgentCandidateOutputArtifactPort = { + put: async () => ({ + locator: { kind: 's3', bucket: 'candidate-results', key: 'wrong.bin' }, + sha256: candidateSha('f'), + byteLength: 1, + }), + read: async () => Uint8Array.from([0]), + } + await expect( + persistCandidateOutputArtifact(port, { + executionId: 'execution-1', + purpose: 'trace', + bytes: Buffer.from('expected'), + }), + ).rejects.toThrow(/does not identify/) + }) +}) diff --git a/tests/candidate-execution-prepare.test.ts b/tests/candidate-execution-prepare.test.ts new file mode 100644 index 00000000..c1bf8239 --- /dev/null +++ b/tests/candidate-execution-prepare.test.ts @@ -0,0 +1,702 @@ +import { execFileSync } from 'node:child_process' +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { + AgentCandidateBundle, + AgentCandidateExecution, + AgentCandidateWorkspaceSnapshotEvidence, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it } from 'vitest' +import { MAX_CANDIDATE_TIMER_INTERVAL_MS } from '../src/candidate-execution/cleanup' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + embeddedCandidateArtifact, +} from '../src/candidate-execution/digest' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import type { + AgentCandidateExecutionPorts, + AgentCandidateTaskExecution, + ResolvedAgentCandidateContainer, +} from '../src/candidate-execution/types' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' + +const roots: string[] = [] +const sha = (character: string): Sha256Digest => `sha256:${character.repeat(64)}` + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +function temporaryRoot(prefix: string): string { + const root = mkdtempSync(join(tmpdir(), prefix)) + roots.push(root) + return root +} + +function git(root: string, args: string[]): string { + return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim() +} + +function taskRepository(): { root: string; commit: string; tree: string } { + const root = temporaryRoot('candidate-task-') + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'test@example.com']) + git(root, ['config', 'user.name', 'Test']) + git(root, ['config', 'core.hooksPath', '/dev/null']) + git(root, ['remote', 'add', 'origin', 'git@github.com:owner/repo.git']) + writeFileSync(join(root, 'source.ts'), 'export const value = 1\n') + chmodSync(join(root, 'source.ts'), 0o644) + git(root, ['add', 'source.ts']) + git(root, ['commit', '-m', 'base']) + return { + root, + commit: git(root, ['rev-parse', 'HEAD']), + tree: git(root, ['rev-parse', 'HEAD^{tree}']), + } +} + +function snapshot( + root: string, + files: Array<{ path: string; mode: 0o644 | 0o755 }>, +): AgentCandidateWorkspaceSnapshotEvidence { + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: files + .map((file) => { + const bytes = execFileSync('node', [ + '-e', + `process.stdout.write(require('fs').readFileSync(${JSON.stringify(join(root, file.path))}))`, + ]) + return { + path: file.path, + mode: file.mode, + sha256: embeddedCandidateArtifact(bytes).sha256, + byteLength: bytes.byteLength, + } + }) + .sort((a, b) => a.path.localeCompare(b.path)), + } + const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest, + archive: embeddedCandidateArtifact(Buffer.from(`archive:${manifest.sha256}`)), + } +} + +function bundle( + execution: Partial = {}, + active?: { commit: string; tree: string; workspace: AgentCandidateWorkspaceSnapshotEvidence }, +): AgentCandidateBundle { + const value = { + schemaVersion: 1 as const, + kind: 'agent-candidate-bundle' as const, + digestAlgorithm: 'rfc8785-sha256' as const, + profile: { + name: 'candidate', + prompt: { instructions: ['Inspect the repository, implement the fix, and run tests.'] }, + model: { default: 'provider/model', reasoningEffort: 'high' as const }, + harness: 'codex' as const, + resources: { failOnError: true as const }, + }, + code: active + ? { + kind: 'no-op' as const, + reason: 'proposer-no-change' as const, + repository: { kind: 'github' as const, owner: 'owner', repo: 'repo' }, + baseCommit: active.commit, + baseTree: active.tree, + } + : { kind: 'disabled' as const, reason: 'control' as const }, + execution: { + harness: 'codex' as const, + harnessVersion: '1.2.3', + launch: active + ? { + kind: 'candidate-entrypoint' as const, + entrypoint: 'run.js', + interpreter: 'node' as const, + } + : { kind: 'container-command' as const, executable: 'codex' }, + instructionDelivery: { kind: 'stdin-utf8' as const }, + cwd: { workspace: 'task' as const, path: '.' }, + environment: { kind: 'evaluator-task-container' as const }, + ...(active ? { workspace: active.workspace } : {}), + isolation: { + network: 'disabled' as const, + remoteIntegrations: 'disabled' as const, + candidateSecrets: 'disabled' as const, + }, + ...execution, + }, + memory: { mode: 'disabled' as const }, + lineage: active + ? { + source: 'optimizer' as const, + parentDigests: [sha('e')], + runIds: ['optimizer-run-1'], + benchmark: { name: 'development', version: '1', splitDigest: sha('f') }, + spend: { + proposal: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + evaluation: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + }, + } + : { source: 'human' as const }, + } + return { ...value, digest: canonicalCandidateDigest(value) } +} + +function redigestBundle( + source: AgentCandidateBundle, + overrides: Partial>, +): AgentCandidateBundle { + const { digest: _digest, ...withoutDigest } = source + const value = { ...withoutDigest, ...overrides } + return { ...value, digest: canonicalCandidateDigest(value) } +} + +function emptySnapshot(label: string): AgentCandidateWorkspaceSnapshotEvidence { + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: [], + } + const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest, + archive: embeddedCandidateArtifact(Buffer.from(`empty:${label}`)), + } +} + +function fixture(active = false): { + bundle: AgentCandidateBundle + task: AgentCandidateTaskExecution + ports: AgentCandidateExecutionPorts + candidateRoot?: string +} { + const repository = taskRepository() + const taskWorkspace = snapshot(repository.root, [{ path: 'source.ts', mode: 0o644 }]) + let candidateRoot: string | undefined + let candidateWorkspace: AgentCandidateWorkspaceSnapshotEvidence | undefined + if (active) { + candidateRoot = temporaryRoot('candidate-built-') + writeFileSync(join(candidateRoot, 'run.js'), '#!/usr/bin/env node\n') + chmodSync(join(candidateRoot, 'run.js'), 0o755) + candidateWorkspace = snapshot(candidateRoot, [{ path: 'run.js', mode: 0o755 }]) + } + const profileRoot = temporaryRoot('candidate-profile-') + const selectedContainer: ResolvedAgentCandidateContainer = { + source: 'evaluator-task-container', + image: 'ghcr.io/example/task', + indexDigest: sha('a'), + manifestDigest: sha('b'), + platform: { os: 'linux', architecture: 'amd64' }, + } + const ports: AgentCandidateExecutionPorts = { + artifacts: { + read: async () => { + throw new Error('all fixture artifacts are embedded') + }, + }, + repositories: { resolve: async () => repository.root }, + workspaces: { materialize: async () => undefined }, + containers: { resolve: async () => selectedContainer }, + models: { + resolve: async ({ requested, reasoningEffort }) => ({ + requested, + provider: 'provider', + model: 'model-snapshot', + snapshot: 'model-snapshot-2026-07-01', + reasoningEffort, + }), + reserveGrant: async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: sha('c'), + expiresAtMs, + enforcedLimits: limits, + network: + limits.maxModelCalls === 0 + ? { mode: 'disabled' as const } + : { mode: 'gateway-only' as const, domains: ['router.tangle.tools'] }, + }), + activateGrant: async () => ({ env: { MODEL_GATEWAY_TOKEN: 'protected' } }), + settleGrant: async ({ preparationId }) => ({ + preparationId, + grantDigest: sha('c'), + closed: true, + calls: [], + }), + }, + memory: { + reset: async () => { + throw new Error('disabled memory must not reset') + }, + activate: async () => { + throw new Error('disabled memory must not activate') + }, + close: async () => { + throw new Error('disabled memory must not close') + }, + }, + } + const task: AgentCandidateTaskExecution = { + executionId: 'execution-1', + benchmark: 'repository-disjoint-smoke', + benchmarkVersion: '1', + taskId: 'owner-repo-1', + splitDigest: sha('d'), + instruction: 'Fix the failing behavior without changing the public API.', + repository: { + identity: 'github.com/owner/repo', + rootIdentity: 'owner/repo', + baseCommit: repository.commit, + baseTree: repository.tree, + }, + attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, + model: { requested: 'provider/model', reasoningEffort: 'high' }, + grader: { + name: 'fixture-executable-grader', + version: '1.0.0', + artifact: { + locator: { kind: 's3', bucket: 'candidate-test-artifacts', key: 'grader/fixture' }, + sha256: sha('a'), + byteLength: 1, + }, + }, + executionRoots: { + taskRoot: '/workspace/task', + ...(active ? { candidateRoot: '/opt/candidate' } : {}), + }, + stagingRoots: { + taskRoot: repository.root, + ...(candidateRoot ? { candidateRoot } : {}), + profileRoot, + }, + workspace: taskWorkspace, + evaluatorTaskContainer: selectedContainer, + limits: { + timeoutMs: 60_000, + maxSteps: 100, + maxModelCalls: 50, + maxInputTokens: 100_000, + maxOutputTokens: 50_000, + maxCostUsd: 5, + }, + } + return { + bundle: bundle( + {}, + active && candidateWorkspace + ? { commit: repository.commit, tree: repository.tree, workspace: candidateWorkspace } + : undefined, + ), + task, + ports, + ...(candidateRoot ? { candidateRoot } : {}), + } +} + +describe('candidate execution preparation', () => { + it('binds exact instruction, repository, profile, model, image, roots, and limits', async () => { + const value = fixture() + value.bundle = bundle({ + instructionDelivery: { + kind: 'utf8-file', + env: 'TANGLE_CANDIDATE_TASK_PATH', + path: '/tangle/input/task.txt', + }, + }) + const verified = await verifyAgentCandidateBundle(value.bundle, value.ports) + const prepared = await prepareAgentCandidateExecution(verified, value.task, value.ports) + const plan = prepared.executionPlan.value.material + expect(plan.task.instruction).toEqual({ + encoding: 'utf8', + sha256: embeddedCandidateArtifact(Buffer.from(value.task.instruction)).sha256, + byteLength: Buffer.byteLength(value.task.instruction), + delivery: value.bundle.execution.instructionDelivery, + }) + expect(plan.task.repository).toEqual(value.task.repository) + expect(plan.profile).toEqual({ + planDigest: prepared.profilePlan.value.digest, + targetWorkspace: 'task', + mountPaths: ['AGENTS.md'], + }) + expect(plan.launch.env.TANGLE_CANDIDATE_TASK_PATH?.value).toBe('/tangle/input/task.txt') + expect(Buffer.from(prepared.instruction.bytes)).toEqual(Buffer.from(value.task.instruction)) + expect(Buffer.from(prepared.executionPlan.bytes)).toEqual( + Buffer.from(prepared.executionPlan.value.artifact.content, 'base64'), + ) + expect(prepared.materializationReceipt.bytes.byteLength).toBeGreaterThan(0) + expect(JSON.stringify(plan)).not.toContain(value.task.instruction) + expect(JSON.stringify(prepared)).not.toContain('MODEL_GATEWAY_TOKEN') + expect(JSON.stringify(prepared)).not.toContain('protected') + expect(plan.model.access.network).toEqual({ + mode: 'gateway-only', + domains: ['router.tangle.tools'], + }) + }) + + it('keeps argv task bytes out of fixed args and exposes deterministic delivery separately', async () => { + const value = fixture() + value.bundle = bundle({ instructionDelivery: { kind: 'argv-append' } }) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ) + expect(prepared.launch.args).not.toContain(value.task.instruction) + expect(prepared.instruction).toMatchObject({ delivery: { kind: 'argv-append' } }) + }) + + it('materializes and binds an active candidate entrypoint without task-root overlap', async () => { + const value = fixture(true) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ) + expect(prepared.launch).toMatchObject({ + executable: 'node', + args: ['/opt/candidate/run.js'], + cwd: '/workspace/task', + }) + expect(prepared.materializationReceipt.value.entrypoint).toMatchObject({ path: 'run.js' }) + expect(prepared.executionPlan.value.material.candidateWorkspace?.digest).toBe( + value.bundle.execution.workspace?.digest, + ) + }) + + it.each([ + { field: 'tools', profileValue: { shell: false } }, + { field: 'permissions', profileValue: { shell: 'deny' } }, + { field: 'modes', profileValue: { review: { model: 'provider/model' } } }, + { field: 'confidential', profileValue: { tee: 'tdx', sealed: true } }, + ] as const)('rejects non-empty $field before staging or reserving protected access', async ({ + field, + profileValue, + }) => { + const value = fixture() + value.bundle = redigestBundle(value.bundle, { + profile: { + ...value.bundle.profile, + [field]: profileValue, + } as AgentCandidateBundle['profile'], + }) + const verified = await verifyAgentCandidateBundle(value.bundle, value.ports) + let stagingCalls = 0 + let reservationCalls = 0 + value.ports.workspaces.materialize = async () => { + stagingCalls++ + } + value.ports.models.reserveGrant = async () => { + reservationCalls++ + throw new Error('candidate must fail before reserving protected access') + } + + await expect(prepareAgentCandidateExecution(verified, value.task, value.ports)).rejects.toThrow( + new RegExp(`non-empty AgentProfile fields: ${field}`), + ) + expect(stagingCalls).toBe(0) + expect(reservationCalls).toBe(0) + }) + + it('accepts explicit empty unsupported fields', async () => { + const value = fixture() + value.bundle = redigestBundle(value.bundle, { + profile: { + ...value.bundle.profile, + tools: {}, + permissions: {}, + modes: {}, + confidential: {}, + }, + }) + + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ) + expect(prepared.bundle.digest).toBe(value.bundle.digest) + }) + + it('rejects task Git drift, dirty profile staging, and unenforced model limits', async () => { + const gitDrift = fixture() + gitDrift.task.repository.baseTree = '0'.repeat(40) + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(gitDrift.bundle, gitDrift.ports), + gitDrift.task, + gitDrift.ports, + ), + ).rejects.toThrow(/base tree/) + + const profileDrift = fixture() + writeFileSync(join(profileDrift.task.stagingRoots.profileRoot, 'stale'), 'stale') + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(profileDrift.bundle, profileDrift.ports), + profileDrift.task, + profileDrift.ports, + ), + ).rejects.toThrow(/must be empty/) + + const unenforced = fixture() + let settledReservations = 0 + unenforced.ports.models.reserveGrant = async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: sha('c'), + expiresAtMs, + enforcedLimits: { ...limits, maxCostUsd: limits.maxCostUsd + 1 }, + network: { mode: 'gateway-only', domains: ['router.tangle.tools'] }, + }) + unenforced.ports.models.settleGrant = async ({ preparationId }) => { + settledReservations++ + return { preparationId, grantDigest: sha('c'), closed: true, calls: [] } + } + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(unenforced.bundle, unenforced.ports), + unenforced.task, + unenforced.ports, + ), + ).rejects.toThrow(/does not enforce/) + expect(settledReservations).toBe(1) + }) + + it('requires a frozen model gateway only when model calls are allowed', async () => { + const invalid = fixture() + let settledReservations = 0 + invalid.ports.models.reserveGrant = async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: sha('c'), + expiresAtMs, + enforcedLimits: limits, + network: { mode: 'disabled' }, + }) + invalid.ports.models.settleGrant = async ({ preparationId }) => { + settledReservations++ + return { preparationId, grantDigest: sha('c'), closed: true, calls: [] } + } + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(invalid.bundle, invalid.ports), + invalid.task, + invalid.ports, + ), + ).rejects.toThrow(/wrong network policy/) + expect(settledReservations).toBe(1) + + const zeroCall = fixture() + zeroCall.task.limits = { + ...zeroCall.task.limits, + maxModelCalls: 0, + maxInputTokens: 0, + maxOutputTokens: 0, + maxCostUsd: 0, + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(zeroCall.bundle, zeroCall.ports), + zeroCall.task, + zeroCall.ports, + ) + expect(prepared.executionPlan.value.material.model.access.network).toEqual({ + mode: 'disabled', + }) + }) + + it('bounds failed-preparation cleanup and rejects another preparation settlement', async () => { + const hanging = fixture() + hanging.ports.models.reserveGrant = async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: sha('c'), + expiresAtMs, + enforcedLimits: { ...limits, maxModelCalls: limits.maxModelCalls + 1 }, + network: { mode: 'gateway-only', domains: ['router.tangle.tools'] }, + }) + hanging.ports.models.settleGrant = async () => await new Promise(() => undefined) + const startedAt = Date.now() + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(hanging.bundle, hanging.ports), + hanging.task, + hanging.ports, + { cleanupTimeoutMs: 20 }, + ), + ).rejects.toThrow(/cleanup failed/) + expect(Date.now() - startedAt).toBeLessThan(250) + + const mismatched = fixture() + mismatched.ports.models.reserveGrant = async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: sha('c'), + expiresAtMs, + enforcedLimits: { ...limits, maxModelCalls: limits.maxModelCalls + 1 }, + network: { mode: 'gateway-only', domains: ['router.tangle.tools'] }, + }) + mismatched.ports.models.settleGrant = async () => ({ + preparationId: `candidate-preparation-v1.${'A'.repeat(43)}`, + grantDigest: sha('c'), + closed: true, + calls: [], + }) + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(mismatched.bundle, mismatched.ports), + mismatched.task, + mismatched.ports, + ), + ).rejects.toThrow(/cleanup failed/) + }) + + it('rejects a cost limit that cannot be represented by the protected integer ledger', async () => { + const value = fixture() + value.task.limits = { ...value.task.limits, maxCostUsd: 10 ** 20 } + let reservations = 0 + value.ports.models.reserveGrant = async () => { + reservations++ + throw new Error('must not reserve') + } + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ), + ).rejects.toThrow(/fixed-point range/) + expect(reservations).toBe(0) + }) + + it('rejects a wall-time limit that Node would clamp to an immediate timer', async () => { + const value = fixture() + value.task.limits = { + ...value.task.limits, + timeoutMs: MAX_CANDIDATE_TIMER_INTERVAL_MS + 1, + } + let reservations = 0 + value.ports.models.reserveGrant = async () => { + reservations++ + throw new Error('must not reserve') + } + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ), + ).rejects.toThrow(/limits are invalid/) + expect(reservations).toBe(0) + }) + + it('rejects unsigned hidden bytes in an active candidate workspace', async () => { + const value = fixture(true) + if (!value.candidateRoot) throw new Error('fixture missing candidate root') + mkdirSync(join(value.candidateRoot, '.sidecar')) + writeFileSync(join(value.candidateRoot, '.sidecar', 'hidden.js'), 'malicious') + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ), + ).rejects.toThrow(/do not match|unsupported mode/) + }) + + it('resets task memory into an execution-scoped namespace and carries verified knowledge', async () => { + const value = fixture() + value.task.executionId = '..' + const seedBytes = Buffer.from('seed-memory') + const knowledgeBytes = Buffer.from('{"documents":[]}') + const seed = { + locator: { kind: 's3' as const, bucket: 'test-artifacts', key: 'memory/seed.bin' }, + sha256: embeddedCandidateArtifact(seedBytes).sha256, + byteLength: seedBytes.byteLength, + } + const knowledge = { + locator: { kind: 's3' as const, bucket: 'test-artifacts', key: 'knowledge/manifest.json' }, + sha256: embeddedCandidateArtifact(knowledgeBytes).sha256, + byteLength: knowledgeBytes.byteLength, + } + value.bundle = redigestBundle(value.bundle, { + memory: { mode: 'isolated', scope: 'task', seed }, + knowledge: { snapshotId: 'knowledge-1', manifest: knowledge }, + }) + value.ports.artifacts.read = async (ref) => { + if (ref.sha256 === seed.sha256) return seedBytes + if (ref.sha256 === knowledge.sha256) return knowledgeBytes + throw new Error(`unexpected artifact ${ref.sha256}`) + } + let resetInput: Parameters[0] | undefined + value.ports.memory.reset = async (input) => { + resetInput = input + return { + preparationId: input.preparationId, + accessDigest: sha('8'), + expiresAtMs: input.expiresAtMs, + evidence: embeddedCandidateArtifact(Buffer.from('fresh-reset')), + emptyStateDigest: sha('7'), + beforeState: emptySnapshot('before'), + } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ) + expect(Buffer.from(resetInput?.seed ?? [])).toEqual(seedBytes) + expect(resetInput?.effectiveNamespace).not.toContain('..') + expect(resetInput?.effectiveNamespace.split('/')).toHaveLength(5) + expect(prepared.memory).toMatchObject({ + mode: 'isolated', + seedDigest: seed.sha256, + }) + expect(JSON.stringify(prepared)).not.toContain('TANGLE_MEMORY_NAMESPACE') + expect(prepared.knowledge).toMatchObject({ + snapshotId: 'knowledge-1', + manifestDigest: knowledge.sha256, + }) + expect(Buffer.from(prepared.knowledge?.manifest ?? [])).toEqual(knowledgeBytes) + }) + + it('closes memory immediately when reset evidence fails before preparation can retain it', async () => { + const value = fixture() + value.bundle = redigestBundle(value.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }) + const before = emptySnapshot('malformed-reset') + const closed: string[] = [] + value.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ + preparationId, + accessDigest: 'sha256:not-a-digest' as `sha256:${string}`, + expiresAtMs, + evidence: before.manifest, + emptyStateDigest: before.digest, + beforeState: before, + }) + value.ports.memory.close = async ({ preparationId, reason }) => { + closed.push(`${preparationId}:${reason}`) + return { closed: true } + } + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ), + ).rejects.toThrow(/not scoped/) + expect(closed).toHaveLength(1) + expect(closed[0]).toMatch(/candidate-preparation-v1\..+:preparation-failed/) + }) +}) diff --git a/tests/candidate-execution-recover.test.ts b/tests/candidate-execution-recover.test.ts new file mode 100644 index 00000000..4e3e380b --- /dev/null +++ b/tests/candidate-execution-recover.test.ts @@ -0,0 +1,342 @@ +import { createHash } from 'node:crypto' +import { mkdirSync, rmSync } from 'node:fs' + +import { InMemoryTraceStore } from '@tangle-network/agent-eval' +import { agentCandidateModelSettlementMaterialSchema } from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it } from 'vitest' + +import { + candidateExecutionClaim, + InMemoryAgentCandidateExecutionClaimStore, +} from '../src/candidate-execution/claim' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import { recoverExpiredAgentCandidateExecution } from '../src/candidate-execution/recover' +import type { AgentCandidateOutputArtifactPort } from '../src/candidate-execution/types' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + candidateSha, + cleanupCandidateFixtures, + createCandidateExecutionFixture, + emptyCandidateSnapshot, + redigestCandidateBundle, +} from './helpers/candidate-execution-fixture' + +afterEach(cleanupCandidateFixtures) + +describe('expired candidate recovery', () => { + it('uses persisted handles to prove cleanup and terminally fail a crashed attempt', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let now = Date.now() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claim = candidateExecutionClaim(prepared) + const acquired = await claimStore.tryClaim(claim) + expect(acquired.acquired).toBe(true) + now = claim.leaseExpiresAtMs + let stops = 0 + let settlements = 0 + const originalSettle = fixture.ports.models.settleGrant + fixture.ports.models.settleGrant = async (input) => { + settlements++ + return await originalSettle(input) + } + const outputArtifacts = outputArtifactPort() + + const result = await recoverExpiredAgentCandidateExecution({ + attempt: { executionId: claim.executionId, attempt: claim.attempt }, + claimStore, + traceStore: new InMemoryTraceStore(), + ports: fixture.ports, + outputArtifacts, + now: () => now, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async (request, context) => { + stops++ + expect(request).toEqual({ + executionId: claim.executionId, + executionPlanDigest: claim.executionPlanDigest, + }) + expect(context.signal.aborted).toBe(true) + expect(context.deadlineAtMs).toBe(claim.leaseExpiresAtMs) + return { stopped: true } + }, + }, + }) + expect(result).toMatchObject({ + finished: true, + terminal: { + status: 'failed', + failureClass: 'pre-model-infrastructure', + usage: { modelCalls: 0 }, + }, + }) + const recoveredSettlement = agentCandidateModelSettlementMaterialSchema.parse( + JSON.parse( + Buffer.from(await outputArtifacts.read(result.terminal.modelSettlement)).toString('utf8'), + ), + ) + expect(recoveredSettlement).toMatchObject({ + kind: 'agent-candidate-model-settlement-material', + executionPlanDigest: claim.executionPlanDigest, + preparationId: claim.cleanup.preparationId, + resolved: claim.cleanup.resolvedModel, + usage: { modelCalls: 0, costUsdNanos: 0 }, + }) + + const replay = await recoverExpiredAgentCandidateExecution({ + attempt: { executionId: claim.executionId, attempt: claim.attempt }, + claimStore, + traceStore: new InMemoryTraceStore(), + ports: fixture.ports, + outputArtifacts, + now: () => now, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async () => { + throw new Error('terminal recovery must not stop twice') + }, + }, + }) + expect(replay).toMatchObject({ finished: false, exactReplay: true }) + expect({ stops, settlements }).toEqual({ stops: 1, settlements: 1 }) + }) + + it('does no outward cleanup before expiry and records nothing when cleanup is unproven', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let now = Date.now() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claim = candidateExecutionClaim(prepared) + await claimStore.tryClaim(claim) + let stops = 0 + const outputArtifacts = outputArtifactPort() + const recover = () => + recoverExpiredAgentCandidateExecution({ + attempt: { executionId: claim.executionId, attempt: claim.attempt }, + claimStore, + traceStore: new InMemoryTraceStore(), + ports: fixture.ports, + outputArtifacts, + now: () => now, + cleanupTimeoutMs: 25, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async () => { + stops++ + throw new Error('process still live') + }, + }, + }) + + await expect(recover()).rejects.toThrow(/has not expired/) + expect(stops).toBe(0) + now = claim.leaseExpiresAtMs + await expect(recover()).rejects.toThrow(/cleanup could not be proven/) + expect(stops).toBe(1) + expect( + await claimStore.getAttempt({ executionId: claim.executionId, attempt: 1 }), + ).not.toHaveProperty('terminal') + }) + + it('unlocks attempt two only when a recovered pre-run crash used zero model calls', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.attempt = { + number: 1, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + const first = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let now = Date.now() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const firstClaim = candidateExecutionClaim(first) + await claimStore.tryClaim(firstClaim) + now = firstClaim.leaseExpiresAtMs + const recovered = await recoverExpiredAgentCandidateExecution({ + attempt: { executionId: firstClaim.executionId, attempt: 1 }, + claimStore, + traceStore: new InMemoryTraceStore(), + ports: fixture.ports, + outputArtifacts: outputArtifactPort(), + now: () => now, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + }) + expect(recovered).toMatchObject({ + finished: true, + terminal: { failureClass: 'pre-model-infrastructure', usage: { modelCalls: 0 } }, + }) + + rmSync(fixture.task.stagingRoots.profileRoot, { recursive: true, force: true }) + mkdirSync(fixture.task.stagingRoots.profileRoot) + fixture.task.attempt = { + number: 2, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + const second = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + await expect(claimStore.tryClaim(candidateExecutionClaim(second))).resolves.toMatchObject({ + acquired: true, + }) + }) + + it('repeats idempotent cleanup after a partial recovery failure', async () => { + const fixture = createCandidateExecutionFixture() + fixture.bundle = redigestCandidateBundle(fixture.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }) + const before = emptyCandidateSnapshot('recovery-before') + fixture.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ + preparationId, + accessDigest: candidateSha('8'), + expiresAtMs, + evidence: before.manifest, + emptyStateDigest: before.digest, + beforeState: before, + }) + let memoryCloses = 0 + fixture.ports.memory.close = async () => { + memoryCloses++ + if (memoryCloses === 1) throw new Error('transient memory service failure') + return { closed: true } + } + let settlements = 0 + const originalSettle = fixture.ports.models.settleGrant + fixture.ports.models.settleGrant = async (input) => { + settlements++ + return await originalSettle(input) + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let now = Date.now() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claim = candidateExecutionClaim(prepared) + await claimStore.tryClaim(claim) + now = claim.leaseExpiresAtMs + let stops = 0 + const outputArtifacts = outputArtifactPort() + const recover = () => + recoverExpiredAgentCandidateExecution({ + attempt: { executionId: claim.executionId, attempt: claim.attempt }, + claimStore, + traceStore: new InMemoryTraceStore(), + ports: fixture.ports, + outputArtifacts, + now: () => now, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async () => { + stops++ + return { stopped: true } + }, + }, + }) + + await expect(recover()).rejects.toThrow(/cleanup could not be proven/) + expect( + await claimStore.getAttempt({ executionId: claim.executionId, attempt: 1 }), + ).not.toHaveProperty('terminal') + await expect(recover()).resolves.toMatchObject({ finished: true }) + expect({ stops, settlements, memoryCloses }).toEqual({ + stops: 2, + settlements: 2, + memoryCloses: 2, + }) + }) + + it('rejects late trace writes during recovery instead of persisting unknown credentials', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let now = Date.now() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claim = candidateExecutionClaim(prepared) + await claimStore.tryClaim(claim) + now = claim.leaseExpiresAtMs + const traceStore = new InMemoryTraceStore() + + await expect( + recoverExpiredAgentCandidateExecution({ + attempt: { executionId: claim.executionId, attempt: claim.attempt }, + claimStore, + traceStore, + ports: fixture.ports, + outputArtifacts: outputArtifactPort(), + now: () => now, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async (_request, context) => { + await context.traceStore.appendEvent({ + runId: claim.cleanup.traceRunId, + eventId: 'late-secret', + kind: 'log', + timestamp: now, + payload: { token: 'expired-model-token' }, + }) + return { stopped: true } + }, + }, + }), + ).rejects.toThrow(/cleanup could not be proven/) + await expect(traceStore.events()).resolves.toEqual([]) + await expect( + claimStore.getAttempt({ executionId: claim.executionId, attempt: claim.attempt }), + ).resolves.not.toHaveProperty('terminal') + }) +}) + +function outputArtifactPort(): AgentCandidateOutputArtifactPort { + const stored = new Map() + return { + put: async ({ purpose, bytes }) => { + const sha256 = `sha256:${createHash('sha256').update(bytes).digest('hex')}` as const + stored.set(sha256, Uint8Array.from(bytes)) + return { + locator: { + kind: 's3', + bucket: 'candidate-recovery-test', + key: `${purpose}/${sha256}.json`, + }, + sha256, + byteLength: bytes.byteLength, + } + }, + read: async (ref) => Uint8Array.from(stored.get(ref.sha256) ?? []), + } +} diff --git a/tests/candidate-execution-redaction.test.ts b/tests/candidate-execution-redaction.test.ts new file mode 100644 index 00000000..7234ee20 --- /dev/null +++ b/tests/candidate-execution-redaction.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest' + +import { redactProtectedValue } from '../src/candidate-execution/protected-redaction' + +describe('candidate protected-value redaction', () => { + it('redacts overlapping credentials longest-first without retaining a suffix', () => { + const redacted = redactProtectedValue({ short: 'secret123', long: 'secret123456' }, [ + 'secret123', + 'secret123456', + ]) + expect(redacted.value).toEqual({ + short: '[redacted:candidate-access]', + long: '[redacted:candidate-access]', + }) + }) +}) diff --git a/tests/candidate-execution-task-outcome.test.ts b/tests/candidate-execution-task-outcome.test.ts new file mode 100644 index 00000000..4b907464 --- /dev/null +++ b/tests/candidate-execution-task-outcome.test.ts @@ -0,0 +1,115 @@ +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterEach, describe, expect, it } from 'vitest' + +import { verifyTaskOutcomePatch } from '../src/candidate-execution/git-materialize' +import { + cleanupCandidateFixtures, + createCandidateExecutionFixture, +} from './helpers/candidate-execution-fixture' + +const temporaryDirectories: string[] = [] + +afterEach(() => { + cleanupCandidateFixtures() + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function git(root: string, args: string[], input?: Uint8Array, env: Record = {}) { + return execFileSync('git', args, { + cwd: root, + encoding: 'utf8', + input, + env: { ...process.env, ...env }, + }).trim() +} + +function changedOutcome(root: string, baseTree: string) { + const patch = Buffer.from( + [ + 'diff --git a/source.ts b/source.ts', + '--- a/source.ts', + '+++ b/source.ts', + '@@ -1 +1 @@', + '-export const value = 1', + '+export const value = 2', + '', + ].join('\n'), + ) + const temporary = mkdtempSync(join(tmpdir(), 'candidate-outcome-test-')) + temporaryDirectories.push(temporary) + const environment = { GIT_INDEX_FILE: join(temporary, 'index') } + git(root, ['read-tree', baseTree], undefined, environment) + git(root, ['apply', '--cached', '--binary', '--whitespace=nowarn', '-'], patch, environment) + const resultTree = git(root, ['write-tree'], undefined, environment) + const resultBytes = Buffer.from('export const value = 2\n') + const afterState = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: [ + { + path: 'source.ts', + mode: 0o644 as const, + sha256: `sha256:${createHash('sha256').update(resultBytes).digest('hex')}` as const, + byteLength: resultBytes.byteLength, + }, + ], + } + return { patch, resultTree, afterState } +} + +describe('candidate task outcome verification', () => { + it('proves a captured binary patch and after-state against the signed base tree', async () => { + const fixture = createCandidateExecutionFixture() + const outcome = changedOutcome( + fixture.task.stagingRoots.taskRoot, + fixture.task.repository.baseTree, + ) + await expect( + verifyTaskOutcomePatch({ + repositoryRoot: fixture.task.stagingRoots.taskRoot, + baseCommit: fixture.task.repository.baseCommit, + baseTree: fixture.task.repository.baseTree, + ...outcome, + }), + ).resolves.toMatchObject({ + resultTree: outcome.resultTree, + resultCommit: expect.stringMatching(/^[a-f0-9]{40}$/), + }) + }) + + it('rejects a claimed result tree or after-state that does not match the patch', async () => { + const fixture = createCandidateExecutionFixture() + const outcome = changedOutcome( + fixture.task.stagingRoots.taskRoot, + fixture.task.repository.baseTree, + ) + await expect( + verifyTaskOutcomePatch({ + repositoryRoot: fixture.task.stagingRoots.taskRoot, + baseCommit: fixture.task.repository.baseCommit, + baseTree: fixture.task.repository.baseTree, + ...outcome, + resultTree: '0'.repeat(fixture.task.repository.baseTree.length), + }), + ).rejects.toThrow(/materialized tree/) + await expect( + verifyTaskOutcomePatch({ + repositoryRoot: fixture.task.stagingRoots.taskRoot, + baseCommit: fixture.task.repository.baseCommit, + baseTree: fixture.task.repository.baseTree, + ...outcome, + afterState: { + ...outcome.afterState, + files: [{ ...outcome.afterState.files[0]!, byteLength: 999 }], + }, + }), + ).rejects.toThrow(/after-state/) + }) +}) diff --git a/tests/candidate-execution-workspace-archive.test.ts b/tests/candidate-execution-workspace-archive.test.ts new file mode 100644 index 00000000..0a2fb4de --- /dev/null +++ b/tests/candidate-execution-workspace-archive.test.ts @@ -0,0 +1,496 @@ +import { execFileSync } from 'node:child_process' +import { + chmodSync, + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + truncateSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { runInNewContext } from 'node:vm' +import { type Headers, pack } from 'tar-stream' +import { afterEach, describe, expect, it } from 'vitest' + +import { + captureAgentCandidateWorkspace, + captureAgentCandidateWorkspaceFiles, + createAgentCandidateWorkspacePort, +} from '../src/candidate-execution' +import { embeddedCandidateArtifact, sha256Bytes } from '../src/candidate-execution/digest' +import { runCandidateGit } from '../src/candidate-execution/git-materialize' + +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +function temporaryRoot(prefix: string): string { + const root = mkdtempSync(join(tmpdir(), prefix)) + roots.push(root) + return root +} + +function git(root: string, args: string[]): string { + return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim() +} + +async function rawTar(header: Headers, bytes = Buffer.alloc(0)): Promise { + const archive = pack() + const output = (async () => { + const chunks: Buffer[] = [] + for await (const chunk of archive) chunks.push(chunk) + return Buffer.concat(chunks) + })() + await new Promise((resolveEntry, rejectEntry) => { + archive.entry(header, bytes, (error) => (error ? rejectEntry(error) : resolveEntry())) + }) + archive.finalize() + return output +} + +function repository(objectFormat: 'sha1' | 'sha256' = 'sha1'): string { + const root = temporaryRoot('candidate-workspace-source-') + git(root, [ + 'init', + '-b', + 'main', + ...(objectFormat === 'sha256' ? ['--object-format=sha256'] : []), + ]) + git(root, ['config', 'user.email', 'test@example.com']) + git(root, ['config', 'user.name', 'Test']) + git(root, ['config', 'core.hooksPath', '/dev/null']) + mkdirSync(join(root, 'bin')) + writeFileSync(join(root, '.gitignore'), 'ignored-cache\n', { mode: 0o644 }) + writeFileSync(join(root, 'README.md'), 'workspace\n', { mode: 0o644 }) + writeFileSync(join(root, 'bin', 'run'), Buffer.from([0, 1, 2, 255]), { mode: 0o755 }) + chmodSync(join(root, 'bin', 'run'), 0o755) + git(root, ['add', '.']) + git(root, ['commit', '-m', 'base']) + writeFileSync(join(root, 'ignored-cache'), 'not part of the task tree', { mode: 0o644 }) + return root +} + +describe('candidate workspace archive', () => { + it.each([ + 'sha1', + 'sha256', + ] as const)('captures and restores exact Git HEAD files for %s repositories', async (objectFormat) => { + const source = repository(objectFormat) + writeFileSync(join(source, 'README.md'), 'uncommitted change\n', { mode: 0o644 }) + const expectedHead = git(source, ['rev-parse', 'HEAD']) + const expectedTree = git(source, ['rev-parse', 'HEAD^{tree}']) + const captured = await captureAgentCandidateWorkspace(source, { + includeRepository: true, + }) + const destination = join(temporaryRoot('candidate-workspace-parent-'), 'restored') + + await createAgentCandidateWorkspacePort().materialize({ + role: 'task', + snapshot: captured.snapshot, + archive: captured.archive, + destination, + }) + + expect(readFileSync(join(destination, 'README.md'), 'utf8')).toBe('workspace\n') + expect([...readFileSync(join(destination, 'bin', 'run'))]).toEqual([0, 1, 2, 255]) + expect(git(destination, ['rev-parse', 'HEAD'])).toBe(expectedHead) + expect(git(destination, ['rev-parse', 'HEAD^{tree}'])).toBe(expectedTree) + expect(git(destination, ['status', '--porcelain=v1', '--untracked-files=all'])).toBe('') + expect(captured.snapshot.material.files.map((file) => file.path)).toEqual([ + '.gitignore', + 'README.md', + 'bin/run', + ]) + }) + + it('restores a non-repository workspace and rejects archive drift', async () => { + const source = temporaryRoot('candidate-workspace-files-') + writeFileSync(join(source, 'input.txt'), 'exact bytes', { mode: 0o644 }) + const captured = await captureAgentCandidateWorkspace(source) + const port = createAgentCandidateWorkspacePort() + const destination = join(temporaryRoot('candidate-workspace-parent-'), 'restored') + await port.materialize({ + role: 'candidate', + snapshot: captured.snapshot, + archive: captured.archive, + destination, + }) + expect(readFileSync(join(destination, 'input.txt'), 'utf8')).toBe('exact bytes') + + const tampered = Uint8Array.from(captured.archive) + const contentOffset = Buffer.from(tampered).indexOf('exact bytes') + if (contentOffset < 0) throw new Error('fixture tar does not contain the file payload') + tampered[contentOffset] = 'E'.charCodeAt(0) + await expect( + port.materialize({ + role: 'candidate', + snapshot: { + ...captured.snapshot, + archive: embeddedCandidateArtifact(tampered), + }, + archive: tampered, + destination: join(temporaryRoot('candidate-workspace-parent-'), 'tampered'), + }), + ).rejects.toThrow('manifest') + }) + + it('captures detached remote files into the same sorted immutable archive', async () => { + const mutable = Buffer.from('before') + const captured = await captureAgentCandidateWorkspaceFiles([ + { path: 'z.txt', mode: 0o644, bytes: mutable }, + { path: 'bin/run', mode: 0o755, bytes: Buffer.from([0, 255]) }, + ]) + mutable.fill(0) + const destination = join(temporaryRoot('candidate-workspace-parent-'), 'remote') + + await createAgentCandidateWorkspacePort().materialize({ + role: 'candidate', + snapshot: captured.snapshot, + archive: captured.archive, + destination, + }) + + expect(captured.snapshot.material.files.map((file) => file.path)).toEqual(['bin/run', 'z.txt']) + expect(Object.isFrozen(captured.snapshot)).toBe(true) + expect(Object.isFrozen(captured.snapshot.material)).toBe(true) + const changedArchive = Uint8Array.from(captured.archive) + changedArchive[0] = changedArchive[0] === 0 ? 1 : 0 + expect(captured.archive[0]).not.toBe(changedArchive[0]) + expect(readFileSync(join(destination, 'z.txt'), 'utf8')).toBe('before') + expect([...readFileSync(join(destination, 'bin', 'run'))]).toEqual([0, 255]) + await expect( + captureAgentCandidateWorkspaceFiles([ + { path: 'same', mode: 0o644, bytes: Buffer.alloc(0) }, + { path: 'same', mode: 0o644, bytes: Buffer.alloc(0) }, + ]), + ).rejects.toThrow('unique and sorted') + await expect( + captureAgentCandidateWorkspaceFiles([ + { path: 'parent', mode: 0o644, bytes: Buffer.alloc(0) }, + { path: 'parent/child', mode: 0o644, bytes: Buffer.alloc(0) }, + ]), + ).rejects.toThrow('unique and sorted') + await expect( + captureAgentCandidateWorkspaceFiles([ + { path: 'a', mode: 0o644, bytes: Buffer.alloc(0) }, + { path: 'a-', mode: 0o644, bytes: Buffer.alloc(0) }, + { path: 'a/b', mode: 0o644, bytes: Buffer.alloc(0) }, + ]), + ).rejects.toThrow('unique and sorted') + + const getterFile = { + path: 'getter', + mode: 0o644 as const, + get bytes(): Uint8Array { + return Buffer.alloc(0) + }, + } + await expect(captureAgentCandidateWorkspaceFiles([getterFile])).rejects.toThrow('fixed values') + + await expect( + captureAgentCandidateWorkspaceFiles([ + { + path: 'shared', + mode: 0o644, + bytes: new Uint8Array(new SharedArrayBuffer(1)), + }, + ]), + ).rejects.toThrow('shared bytes') + const crossRealmShared = runInNewContext( + 'new Uint8Array(new SharedArrayBuffer(1))', + ) as Uint8Array + await expect( + captureAgentCandidateWorkspaceFiles([ + { path: 'cross-realm-shared', mode: 0o644, bytes: crossRealmShared }, + ]), + ).rejects.toThrow('shared bytes') + const crossRealmBytes = runInNewContext('new Uint8Array([1, 2, 3])') as Uint8Array + const crossRealmCapture = await captureAgentCandidateWorkspaceFiles([ + { path: 'cross-realm', mode: 0o644, bytes: crossRealmBytes }, + ]) + expect(crossRealmCapture.snapshot.material.files[0]?.byteLength).toBe(3) + const overriddenMap = [{ path: 'map', mode: 0o644 as const, bytes: Buffer.from('checked') }] + Object.defineProperty(overriddenMap, 'map', { + value: () => [{ path: '../unchecked', mode: 0o644, bytes: Buffer.alloc(0) }], + }) + const overriddenMapCapture = await captureAgentCandidateWorkspaceFiles(overriddenMap) + expect(overriddenMapCapture.snapshot.material.files[0]?.path).toBe('map') + let lengthReads = 0 + const proxiedFiles = new Proxy( + [{ path: 'proxy', mode: 0o644 as const, bytes: Buffer.alloc(0) }], + { + get(target, property, receiver) { + if (property === 'length') { + lengthReads++ + return lengthReads === 1 ? 1 : 10_000 + } + return Reflect.get(target, property, receiver) + }, + }, + ) + const proxiedCapture = await captureAgentCandidateWorkspaceFiles(proxiedFiles, { + limits: { maxFiles: 1 }, + }) + expect(proxiedCapture.snapshot.material.files[0]?.path).toBe('proxy') + expect(lengthReads).toBe(1) + const iterable = Buffer.alloc(0) + iterable[Symbol.iterator] = function* () { + yield* Buffer.alloc(1_024) + } + const iterableCapture = await captureAgentCandidateWorkspaceFiles([ + { path: 'iterable', mode: 0o644, bytes: iterable }, + ]) + expect(iterableCapture.snapshot.material.files[0]?.byteLength).toBe(0) + await expect( + captureAgentCandidateWorkspaceFiles([ + { path: '\ud800', mode: 0o644, bytes: Buffer.alloc(0) }, + ]), + ).rejects.toThrow('unsafe path') + await expect( + captureAgentCandidateWorkspaceFiles( + [{ path: 'large', mode: 0o644, bytes: Buffer.alloc(1_024) }], + { limits: { maxArchiveBytes: 100 } }, + ), + ).rejects.toThrow('exceeds maxArchiveBytes') + }) + + it('stores large workspace artifacts by exact external reference', async () => { + const stored = new Map() + const outputArtifacts = { + put: async ({ bytes, purpose }: { bytes: Uint8Array; purpose: string }) => { + const detached = Uint8Array.from(bytes) + const sha256 = sha256Bytes(detached) + stored.set(sha256, detached) + return { + locator: { + kind: 's3' as const, + bucket: 'candidate-test-artifacts', + key: `${purpose}/${sha256}`, + }, + sha256, + byteLength: detached.byteLength, + } + }, + read: async (ref: { sha256: string }) => Uint8Array.from(stored.get(ref.sha256) ?? []), + } + const files = [{ path: 'large', mode: 0o644 as const, bytes: Buffer.alloc(1_024, 7) }] + + await expect( + captureAgentCandidateWorkspaceFiles(files, { + limits: { maxEmbeddedArtifactBytes: 100 }, + }), + ).rejects.toThrow('artifactPersistence is required') + const captured = await captureAgentCandidateWorkspaceFiles(files, { + limits: { maxEmbeddedArtifactBytes: 100 }, + artifactPersistence: { + executionId: 'workspace-capture-1', + outputArtifacts, + }, + }) + + expect('locator' in captured.snapshot.manifest).toBe(true) + expect('locator' in captured.snapshot.archive).toBe(true) + expect(Buffer.from(stored.get(captured.snapshot.archive.sha256) ?? [])).toEqual( + Buffer.from(captured.archive), + ) + const destination = join(temporaryRoot('candidate-workspace-parent-'), 'external') + await createAgentCandidateWorkspacePort().materialize({ + role: 'candidate', + snapshot: captured.snapshot, + archive: captured.archive, + destination, + }) + expect(readFileSync(join(destination, 'large'))).toEqual(Buffer.alloc(1_024, 7)) + + await expect( + captureAgentCandidateWorkspaceFiles(files, { + artifactPersistence: { + executionId: 'workspace-capture-2', + outputArtifacts: { + put: async ({ bytes }) => ({ + locator: { kind: 's3', bucket: 'candidate-test-artifacts', key: 'missing' }, + sha256: sha256Bytes(bytes), + byteLength: bytes.byteLength, + }), + read: async () => Uint8Array.from([0]), + }, + }, + }), + ).rejects.toThrow(/persisted candidate output/) + }) + + it('bounds archive plus decoded bytes before retaining an entry', async () => { + const captured = await captureAgentCandidateWorkspaceFiles([ + { path: 'payload', mode: 0o644, bytes: Buffer.alloc(1_024, 7) }, + ]) + const destination = join(temporaryRoot('candidate-workspace-parent-'), 'bounded') + + await expect( + createAgentCandidateWorkspacePort({ + limits: { maxArchiveBytes: captured.archive.byteLength + 1_023 }, + }).materialize({ + role: 'candidate', + snapshot: captured.snapshot, + archive: captured.archive, + destination, + }), + ).rejects.toThrow('exceeds maxArchiveBytes while decoding') + }) + + it('rejects unsafe, linked, and noncanonical tar entries', async () => { + const baseline = await captureAgentCandidateWorkspaceFiles([]) + const port = createAgentCandidateWorkspacePort() + const cases = [ + { + label: 'traversal', + archive: await rawTar({ name: 'workspace/../escape', mode: 0o644, type: 'file' }), + error: /unsafe path/, + }, + { + label: 'symlink', + archive: await rawTar({ + name: 'workspace/link', + mode: 0o755, + type: 'symlink', + linkname: '/etc/passwd', + }), + error: /invalid entry/, + }, + { + label: 'noncanonical', + archive: await rawTar( + { + name: 'workspace/input.txt', + mode: 0o644, + uid: 1, + gid: 1, + mtime: new Date(0), + type: 'file', + }, + Buffer.from('input'), + ), + error: /not canonical/, + }, + ] + + for (const attack of cases) { + await expect( + port.materialize({ + role: 'candidate', + snapshot: { ...baseline.snapshot, archive: embeddedCandidateArtifact(attack.archive) }, + archive: attack.archive, + destination: join(temporaryRoot('candidate-workspace-parent-'), attack.label), + }), + ).rejects.toThrow(attack.error) + } + }) + + it('round-trips long and non-ASCII paths through canonical tar headers', async () => { + const longPath = `${Array.from({ length: 6 }, (_, index) => `${index}-${'x'.repeat(48)}`).join('/')}/caf\u00e9.txt` + const captured = await captureAgentCandidateWorkspaceFiles([ + { path: longPath, mode: 0o644, bytes: Buffer.from('long path') }, + ]) + const destination = join(temporaryRoot('candidate-workspace-parent-'), 'long-path') + + await createAgentCandidateWorkspacePort().materialize({ + role: 'candidate', + snapshot: captured.snapshot, + archive: captured.archive, + destination, + }) + + expect(readFileSync(join(destination, longPath), 'utf8')).toBe('long path') + }) + + it('rejects symbolic and hard links instead of following them', async () => { + const symlinkRoot = temporaryRoot('candidate-workspace-symlink-') + writeFileSync(join(symlinkRoot, 'target'), 'secret', { mode: 0o644 }) + symlinkSync('target', join(symlinkRoot, 'link')) + await expect(captureAgentCandidateWorkspace(symlinkRoot)).rejects.toThrow('symlink') + + const hardlinkRoot = temporaryRoot('candidate-workspace-hardlink-') + writeFileSync(join(hardlinkRoot, 'target'), 'shared', { mode: 0o644 }) + linkSync(join(hardlinkRoot, 'target'), join(hardlinkRoot, 'link')) + await expect(captureAgentCandidateWorkspace(hardlinkRoot)).rejects.toThrow('hard-linked') + }) + + it('enforces archive limits before parsing or writing', async () => { + const source = temporaryRoot('candidate-workspace-limits-') + writeFileSync(join(source, 'input.txt'), 'bounded', { mode: 0o644 }) + const captured = await captureAgentCandidateWorkspace(source) + const port = createAgentCandidateWorkspacePort({ + limits: { maxArchiveBytes: captured.archive.byteLength - 1 }, + }) + await expect( + port.materialize({ + role: 'candidate', + snapshot: captured.snapshot, + archive: captured.archive, + destination: join(temporaryRoot('candidate-workspace-parent-'), 'limited'), + }), + ).rejects.toThrow('exceeds maxArchiveBytes') + + const sparse = join(source, 'sparse.bin') + writeFileSync(sparse, '', { mode: 0o644 }) + chmodSync(sparse, 0o644) + truncateSync(sparse, 1024 * 1024) + await expect( + captureAgentCandidateWorkspace(source, { limits: { maxFileBytes: 1_024 } }), + ).rejects.toThrow('exceeds maxFileBytes') + }) + + it('does not run ambient Git commands while capturing or restoring a repository', async () => { + const source = repository() + const configRoot = temporaryRoot('candidate-workspace-git-config-') + const marker = join(configRoot, 'fsmonitor-ran') + const fsmonitor = join(configRoot, 'fsmonitor.sh') + writeFileSync(fsmonitor, `#!/bin/sh\ntouch ${marker}\n`, { mode: 0o755 }) + chmodSync(fsmonitor, 0o755) + git(source, ['config', 'core.fsmonitor', fsmonitor]) + await runCandidateGit(source, ['status', '--porcelain=v1', '--untracked-files=all']) + const protectedCapture = await captureAgentCandidateWorkspace(source, { + includeRepository: true, + }) + await createAgentCandidateWorkspacePort().materialize({ + role: 'task', + snapshot: protectedCapture.snapshot, + archive: protectedCapture.archive, + destination: join(temporaryRoot('candidate-workspace-parent-'), 'restored'), + }) + expect(existsSync(marker)).toBe(false) + }) + + it('rejects repository object-store indirection before capture', async () => { + const source = repository() + const alternateObjects = join(temporaryRoot('candidate-workspace-alternate-'), 'objects') + mkdirSync(alternateObjects) + mkdirSync(join(source, '.git', 'objects', 'info'), { recursive: true }) + writeFileSync(join(source, '.git', 'objects', 'info', 'alternates'), `${alternateObjects}\n`) + + await expect( + captureAgentCandidateWorkspace(source, { includeRepository: true }), + ).rejects.toThrow('forbidden Git alternates') + }) + + it('rejects object-store indirection inherited by a linked worktree', async () => { + const source = repository() + const linked = temporaryRoot('candidate-workspace-linked-') + rmSync(linked, { recursive: true, force: true }) + git(source, ['worktree', 'add', '--detach', linked]) + const alternateObjects = join(temporaryRoot('candidate-workspace-alternate-'), 'objects') + mkdirSync(alternateObjects) + mkdirSync(join(source, '.git', 'objects', 'info'), { recursive: true }) + writeFileSync(join(source, '.git', 'objects', 'info', 'alternates'), `${alternateObjects}\n`) + + await expect( + captureAgentCandidateWorkspace(linked, { includeRepository: true }), + ).rejects.toThrow('forbidden Git alternates') + }) +}) diff --git a/tests/helpers/candidate-execution-fixture.ts b/tests/helpers/candidate-execution-fixture.ts new file mode 100644 index 00000000..8f663049 --- /dev/null +++ b/tests/helpers/candidate-execution-fixture.ts @@ -0,0 +1,402 @@ +import { execFileSync } from 'node:child_process' +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' + +import type { + AgentCandidateArtifactRef, + AgentCandidateBundle, + AgentCandidateExecution, + AgentCandidateWorkspaceSnapshotEvidence, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + embeddedCandidateArtifact, + sha256Bytes, +} from '../../src/candidate-execution/digest' +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateExecutionPorts, + AgentCandidateExecutorTaskOutcomeCapture, + AgentCandidateOutputArtifactPort, + AgentCandidateTaskExecution, + ResolvedAgentCandidateContainer, +} from '../../src/candidate-execution/types' + +const roots: string[] = [] + +export const candidateSha = (character: string): Sha256Digest => `sha256:${character.repeat(64)}` + +const fixtureGraderBytes = Buffer.from('fixture executable grader', 'utf8') + +export function cleanupCandidateFixtures(): void { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +} + +function temporaryRoot(prefix: string): string { + const root = mkdtempSync(join(tmpdir(), prefix)) + roots.push(root) + return root +} + +function git(root: string, args: string[]): string { + return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim() +} + +function taskRepository(): { root: string; commit: string; tree: string } { + const root = temporaryRoot('candidate-task-') + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'test@example.com']) + git(root, ['config', 'user.name', 'Test']) + git(root, ['config', 'core.hooksPath', '/dev/null']) + git(root, ['remote', 'add', 'origin', 'git@github.com:owner/repo.git']) + writeFileSync(join(root, 'source.ts'), 'export const value = 1\n') + chmodSync(join(root, 'source.ts'), 0o644) + git(root, ['add', 'source.ts']) + git(root, ['commit', '-m', 'base']) + return { + root, + commit: git(root, ['rev-parse', 'HEAD']), + tree: git(root, ['rev-parse', 'HEAD^{tree}']), + } +} + +function snapshot( + root: string, + files: Array<{ path: string; mode: 0o644 | 0o755 }>, +): AgentCandidateWorkspaceSnapshotEvidence { + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: files + .map((file) => { + const bytes = execFileSync('node', [ + '-e', + `process.stdout.write(require('fs').readFileSync(${JSON.stringify(join(root, file.path))}))`, + ]) + return { + path: file.path, + mode: file.mode, + sha256: embeddedCandidateArtifact(bytes).sha256, + byteLength: bytes.byteLength, + } + }) + .sort((a, b) => a.path.localeCompare(b.path)), + } + const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest, + archive: embeddedCandidateArtifact(Buffer.from(`archive:${manifest.sha256}`)), + } +} + +export function candidateBundle( + execution: Partial = {}, + active?: { + commit: string + tree: string + workspace: AgentCandidateWorkspaceSnapshotEvidence + }, +): AgentCandidateBundle { + const value = { + schemaVersion: 1 as const, + kind: 'agent-candidate-bundle' as const, + digestAlgorithm: 'rfc8785-sha256' as const, + profile: { + name: 'candidate', + prompt: { instructions: ['Inspect the repository, implement the fix, and run tests.'] }, + model: { default: 'provider/model', reasoningEffort: 'high' as const }, + harness: 'codex' as const, + resources: { failOnError: true as const }, + }, + code: active + ? { + kind: 'no-op' as const, + reason: 'proposer-no-change' as const, + repository: { kind: 'github' as const, owner: 'owner', repo: 'repo' }, + baseCommit: active.commit, + baseTree: active.tree, + } + : { kind: 'disabled' as const, reason: 'control' as const }, + execution: { + harness: 'codex' as const, + harnessVersion: '1.2.3', + launch: active + ? { + kind: 'candidate-entrypoint' as const, + entrypoint: 'run.js', + interpreter: 'node' as const, + } + : { kind: 'container-command' as const, executable: 'codex' }, + instructionDelivery: { kind: 'stdin-utf8' as const }, + cwd: { workspace: 'task' as const, path: '.' }, + environment: { kind: 'evaluator-task-container' as const }, + ...(active ? { workspace: active.workspace } : {}), + isolation: { + network: 'disabled' as const, + remoteIntegrations: 'disabled' as const, + candidateSecrets: 'disabled' as const, + }, + ...execution, + }, + memory: { mode: 'disabled' as const }, + lineage: active + ? { + source: 'optimizer' as const, + parentDigests: [candidateSha('e')], + runIds: ['optimizer-run-1'], + benchmark: { + name: 'development', + version: '1', + splitDigest: candidateSha('f'), + }, + spend: { + proposal: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + evaluation: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + }, + } + : { source: 'human' as const }, + } + return { ...value, digest: canonicalCandidateDigest(value) } +} + +export function redigestCandidateBundle( + source: AgentCandidateBundle, + overrides: Partial>, +): AgentCandidateBundle { + const { digest: _digest, ...withoutDigest } = source + const value = { ...withoutDigest, ...overrides } + return { ...value, digest: canonicalCandidateDigest(value) } +} + +export function emptyCandidateSnapshot(label: string): AgentCandidateWorkspaceSnapshotEvidence { + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: [], + } + const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest, + archive: embeddedCandidateArtifact(Buffer.from(`empty:${label}`)), + } +} + +export interface CandidateExecutionFixture { + bundle: AgentCandidateBundle + task: AgentCandidateTaskExecution + ports: AgentCandidateExecutionPorts + candidateRoot?: string +} + +export function unchangedTaskOutcomeCapture( + fixture: CandidateExecutionFixture, +): AgentCandidateExecutorTaskOutcomeCapture { + return { + resultTree: fixture.task.repository.baseTree, + afterState: fixture.task.workspace.material, + archive: Buffer.from(`task-archive:${fixture.task.workspace.digest}`, 'utf8'), + gitDiff: Buffer.alloc(0), + } +} + +export function createCandidateOutputFixture(): { + outputArtifacts: AgentCandidateOutputArtifactPort + grader: AgentCandidateBenchmarkGraderPort +} { + const stored = new Map() + const put = (bytes: Uint8Array, purpose: string): AgentCandidateArtifactRef => { + const detached = Uint8Array.from(bytes) + const sha256 = sha256Bytes(detached) + const ref: AgentCandidateArtifactRef = { + locator: { + kind: 's3', + bucket: 'candidate-test-artifacts', + key: `${purpose}/${sha256.slice('sha256:'.length)}`, + }, + sha256, + byteLength: detached.byteLength, + } + stored.set(sha256, detached) + return ref + } + const graderArtifact = put(fixtureGraderBytes, 'grader') + return { + outputArtifacts: { + put: async ({ bytes, purpose }) => put(bytes, purpose), + read: async (ref) => { + const bytes = stored.get(ref.sha256) + if (!bytes) throw new Error(`missing candidate test artifact ${ref.sha256}`) + return Uint8Array.from(bytes) + }, + }, + grader: { + name: 'fixture-executable-grader', + version: '1.0.0', + artifact: graderArtifact, + run: async ({ implementation, outcome }) => { + const evidence = Buffer.from('{"reward":1}\n', 'utf8') + return { + evaluation: { score: 1, passed: true, dimensions: { tests: 1 }, raw: {} }, + evidence, + binding: { + implementationDigest: sha256Bytes(implementation.bytes), + taskOutcomeDigest: outcome.evidence.digest, + outputDigest: sha256Bytes(evidence), + }, + } + }, + }, + } +} + +export function createCandidateExecutionFixture(active = false): CandidateExecutionFixture { + const repository = taskRepository() + const taskWorkspace = snapshot(repository.root, [{ path: 'source.ts', mode: 0o644 }]) + let candidateRoot: string | undefined + let candidateWorkspace: AgentCandidateWorkspaceSnapshotEvidence | undefined + if (active) { + candidateRoot = temporaryRoot('candidate-built-') + writeFileSync(join(candidateRoot, 'run.js'), '#!/usr/bin/env node\n') + chmodSync(join(candidateRoot, 'run.js'), 0o755) + candidateWorkspace = snapshot(candidateRoot, [{ path: 'run.js', mode: 0o755 }]) + } + const profileRoot = temporaryRoot('candidate-profile-') + const selectedContainer: ResolvedAgentCandidateContainer = { + source: 'evaluator-task-container', + image: 'ghcr.io/example/task', + indexDigest: candidateSha('a'), + manifestDigest: candidateSha('b'), + platform: { os: 'linux', architecture: 'amd64' }, + } + const ports: AgentCandidateExecutionPorts = { + artifacts: { + read: async () => { + throw new Error('all fixture artifacts are embedded') + }, + }, + repositories: { resolve: async () => repository.root }, + workspaces: { + materialize: async ({ role, snapshot: workspace, destination }) => { + if (role === 'memory') { + if (workspace.material.files.length !== 0) { + throw new Error('fixture memory materializer supports only an empty state') + } + return + } + const source = role === 'task' ? repository.root : candidateRoot + if (!source) throw new Error(`fixture ${role} workspace source is missing`) + if (source === destination) return + for (const file of workspace.material.files) { + const output = join(destination, file.path) + mkdirSync(dirname(output), { recursive: true }) + writeFileSync(output, readFileSync(join(source, file.path))) + chmodSync(output, file.mode) + } + }, + }, + containers: { resolve: async () => selectedContainer }, + models: { + resolve: async ({ requested, reasoningEffort }) => ({ + requested, + provider: 'provider', + model: 'model-snapshot', + snapshot: 'model-snapshot-2026-07-01', + reasoningEffort, + }), + reserveGrant: async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: candidateSha('c'), + expiresAtMs, + enforcedLimits: limits, + network: + limits.maxModelCalls === 0 + ? { mode: 'disabled' as const } + : { mode: 'gateway-only' as const, domains: ['router.tangle.tools'] }, + }), + activateGrant: async () => ({ env: { MODEL_GATEWAY_TOKEN: 'protected' } }), + settleGrant: async ({ preparationId }) => ({ + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + }), + }, + memory: { + reset: async () => { + throw new Error('disabled memory must not reset') + }, + activate: async () => { + throw new Error('disabled memory must not activate') + }, + close: async () => { + throw new Error('disabled memory must not close') + }, + }, + } + const task: AgentCandidateTaskExecution = { + executionId: 'execution-1', + benchmark: 'repository-disjoint-smoke', + benchmarkVersion: '1', + taskId: 'owner-repo-1', + splitDigest: candidateSha('d'), + instruction: 'Fix the failing behavior without changing the public API.', + repository: { + identity: 'github.com/owner/repo', + rootIdentity: 'owner/repo', + baseCommit: repository.commit, + baseTree: repository.tree, + }, + attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, + model: { requested: 'provider/model', reasoningEffort: 'high' }, + grader: { + name: 'fixture-executable-grader', + version: '1.0.0', + artifact: { + locator: { kind: 's3', bucket: 'candidate-test-artifacts', key: 'grader/fixture' }, + sha256: sha256Bytes(fixtureGraderBytes), + byteLength: fixtureGraderBytes.byteLength, + }, + }, + executionRoots: { + taskRoot: '/workspace/task', + ...(active ? { candidateRoot: '/opt/candidate' } : {}), + }, + stagingRoots: { + taskRoot: repository.root, + ...(candidateRoot ? { candidateRoot } : {}), + profileRoot, + }, + workspace: taskWorkspace, + evaluatorTaskContainer: selectedContainer, + limits: { + timeoutMs: 60_000, + maxSteps: 100, + maxModelCalls: 50, + maxInputTokens: 100_000, + maxOutputTokens: 50_000, + maxCostUsd: 5, + }, + } + return { + bundle: candidateBundle( + {}, + active && candidateWorkspace + ? { commit: repository.commit, tree: repository.tree, workspace: candidateWorkspace } + : undefined, + ), + task, + ports, + ...(candidateRoot ? { candidateRoot } : {}), + } +} diff --git a/tests/improve.test.ts b/tests/improve.test.ts index 0226afd4..18fe736c 100644 --- a/tests/improve.test.ts +++ b/tests/improve.test.ts @@ -27,9 +27,21 @@ const agent = async ( _scenario: DemoScenario, ctx: DispatchContext, ): Promise => { - ctx.cost.observe(0.0001, 'test') - ctx.cost.observeTokens({ input: 1, output: 1 }) - return String(surface) + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'test', + model: 'deterministic-test', + maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, + execute: async () => String(surface), + receipt: () => ({ + model: 'deterministic-test', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.0001, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value } /** Deterministic judge: the literal string `PROMOTED` scores 1.0, anything else @@ -110,7 +122,7 @@ describe('improve() — facade over selfImprove', () => { await expect( improve(profile, [], { surface: 'prompt', - generator: scriptedWinner, + gate: 'none', scenarios, judge, agent, @@ -125,7 +137,6 @@ describe('improve() — facade over selfImprove', () => { const out = await improve(profile, [], { surface: 'prompt', gate: 'none', - generator: scriptedWinner, scenarios, judge, agent, diff --git a/tests/improvement-cycle.test.ts b/tests/improvement-cycle.test.ts new file mode 100644 index 00000000..dae36255 --- /dev/null +++ b/tests/improvement-cycle.test.ts @@ -0,0 +1,573 @@ +import { type AnalystFinding, InMemoryTraceStore } from '@tangle-network/agent-eval' +import type { + DispatchContext, + JudgeConfig, + MutableSurface, + Scenario, + SurfaceProposer, +} from '@tangle-network/agent-eval/contract' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { AnalystRegistryLike } from '../src/analyst-loop/types' +import { buildAgentCandidateBundle } from '../src/candidate-execution/builder' +import { InMemoryAgentCandidateExecutionClaimStore } from '../src/candidate-execution/claim' +import { assertCandidateProfileBinding } from '../src/candidate-execution/profile' +import type { + AgentCandidateExecutorPort, + AgentCandidateExecutorRequest, +} from '../src/candidate-execution/types' +import { + createAgentImprovementProposal, + executeApprovedAgentCandidate, + proposeAgentImprovement, + reviewAgentImprovementProposal, + verifyAgentImprovementProposal, + verifyAgentImprovementReview, +} from '../src/intelligence/improvement-cycle' +import { + candidateSha, + cleanupCandidateFixtures, + createCandidateExecutionFixture, + createCandidateOutputFixture, + unchangedTaskOutcomeCapture, +} from './helpers/candidate-execution-fixture' + +interface DemoScenario extends Scenario { + kind: 'demo' +} + +const scenarios: DemoScenario[] = Array.from({ length: 12 }, (_, index) => ({ + id: `scenario-${index}`, + kind: 'demo' as const, +})) + +const finding: AnalystFinding = { + schema_version: '1.0.0', + finding_id: 'finding-1', + analyst_id: 'improvement', + produced_at: '2026-07-10T00:00:00.000Z', + severity: 'high', + area: 'prompt', + claim: 'The agent omits the required marker.', + evidence_refs: [{ kind: 'span', id: 'span-1' }], + recommended_action: 'Add the measured marker.', + confidence: 0.9, + subject: 'agent-profile:prompt.systemPrompt', +} + +const registry = (findings: AnalystFinding[]): AnalystRegistryLike => ({ + list: () => [{ id: 'improvement' }], + run: async (runId) => ({ + run_id: runId, + correlation_id: `correlation-${runId}`, + started_at: '2026-07-10T00:00:00.000Z', + ended_at: '2026-07-10T00:00:01.000Z', + findings, + per_analyst: [ + { + analyst_id: 'improvement', + status: 'ok', + findings_count: findings.length, + latency_ms: 1, + cost_usd: 0, + }, + ], + total_cost_usd: 0, + }), +}) + +const proposer: SurfaceProposer = { + kind: 'scripted', + propose: async () => [ + { surface: 'PROMOTED', label: 'measured winner', rationale: 'addresses finding-1' }, + ], +} + +const judge: JudgeConfig = { + name: 'literal-marker', + dimensions: [{ key: 'marker', description: 'Contains the required marker.' }], + score: ({ artifact }) => { + const composite = artifact.includes('PROMOTED') ? 1 : 0 + return { dimensions: { marker: composite }, composite, notes: '' } + }, +} + +async function agent( + surface: MutableSurface, + _scenario: DemoScenario, + context: DispatchContext, +): Promise { + const paid = await context.cost.runPaidCall({ + channel: 'agent', + actor: 'fixture', + model: 'fixture-model', + maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, + execute: async () => String(surface), + receipt: () => ({ + model: 'fixture-model', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.0001, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +function fixtureProfile() { + return { + name: 'candidate', + prompt: { + systemPrompt: 'BASELINE', + instructions: ['Inspect the repository, implement the fix, and run tests.'], + }, + model: { default: 'provider/model', reasoningEffort: 'high' as const }, + harness: 'codex' as const, + resources: { failOnError: true as const }, + } +} + +function alignedBundle( + bundle: Omit['bundle'], 'digest'>, + profile: { prompt?: { systemPrompt?: string } }, +) { + return { + ...bundle, + profile: { + ...bundle.profile, + prompt: { + ...bundle.profile.prompt, + systemPrompt: profile.prompt?.systemPrompt, + }, + }, + } +} + +function alignedSealedBundle( + bundle: Omit['bundle'], 'digest'>, + profile: { prompt?: { systemPrompt?: string } }, +) { + const aligned = alignedBundle(bundle, profile) + const { profileDiffIds: _profileDiffIds, ...lineage } = aligned.lineage + return buildAgentCandidateBundle({ + profile: { kind: 'candidate-profile', profile: aligned.profile }, + code: aligned.code, + execution: aligned.execution, + ...(aligned.knowledge ? { knowledge: aligned.knowledge } : {}), + memory: aligned.memory, + lineage, + }) +} + +afterEach(() => { + cleanupCandidateFixtures() + vi.restoreAllMocks() +}) + +describe('agent improvement lifecycle', () => { + it('refuses memory persistence before human approval', async () => { + const writeBack = vi.fn() + await expect( + proposeAgentImprovement({ + runId: 'analysis-run-memory-writeback', + profile: fixtureProfile(), + analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, + improvement: { + surface: 'memory', + memory: { + document: '# Durable memory\n', + writeBack, + }, + generator: proposer, + scenarios, + judge, + agent, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.5 }, + }, + }), + ).rejects.toThrow('cannot write memory before human approval') + expect(writeBack).not.toHaveBeenCalled() + }) + + it('disposes a retained code winner when candidate construction fails', async () => { + const { execFileSync } = await import('node:child_process') + const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') + const { tmpdir } = await import('node:os') + const { join } = await import('node:path') + const repoRoot = mkdtempSync(join(tmpdir(), 'improvement-cycle-code-cleanup-')) + const git = (...args: string[]) => + execFileSync('git', args, { cwd: repoRoot, encoding: 'utf8', stdio: 'pipe' }) + try { + git('init', '-q', '-b', 'main') + git('config', 'user.email', 'improvement-cycle@test.local') + git('config', 'user.name', 'improvement-cycle-test') + writeFileSync(join(repoRoot, 'module.txt'), 'BASELINE\n') + git('add', 'module.txt') + git('commit', '-qm', 'baseline') + + const buildCandidate = vi.fn(async () => { + throw new Error('candidate construction failed') + }) + await expect( + proposeAgentImprovement({ + runId: 'analysis-run-code-build-failure', + profile: fixtureProfile(), + analysis: { + registry: registry([finding]), + inputs: {}, + findingsStore: null, + log: () => {}, + }, + improvement: { + surface: 'code', + scenarios, + judge: { + name: 'code-marker', + dimensions: [{ key: 'marker', description: 'Code contains the promoted marker.' }], + score: ({ artifact }) => { + const composite = artifact.includes('PROMOTED') ? 1 : 0 + return { dimensions: { marker: composite }, composite, notes: '' } + }, + }, + agent: async (surface, _scenario, context) => { + const paid = await context.cost.runPaidCall({ + channel: 'agent', + actor: 'fixture', + model: 'fixture-model', + maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, + execute: async () => { + if (typeof surface === 'string') throw new Error('expected code surface') + return readFileSync(join(surface.worktreeRef, 'module.txt'), 'utf8') + }, + receipt: () => ({ + model: 'fixture-model', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.0001, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value + }, + code: { + repoRoot, + generator: { + kind: 'fixture', + async generate({ worktreePath }) { + writeFileSync(join(worktreePath, 'module.txt'), 'PROMOTED\n') + return { applied: true, summary: 'write promoted marker' } + }, + }, + }, + promotionGate: { + name: 'fixture-ship', + decide: async () => ({ + decision: 'ship', + reasons: ['candidate passes the fixture comparison'], + contributingGates: [], + }), + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }, + buildCandidate, + }), + ).rejects.toThrow('candidate construction failed') + + expect(buildCandidate).toHaveBeenCalledOnce() + expect(git('worktree', 'list', '--porcelain').match(/^worktree /gm)).toHaveLength(1) + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it('binds typed candidate hooks to their equivalent measured profile commands', () => { + expect(() => + assertCandidateProfileBinding( + { + hooks: { + beforeTool: [ + { + command: "node 'path with space.js'", + timeoutMs: 1_000, + env: { MODE: 'check' }, + }, + ], + }, + }, + { + hooks: { + beforeTool: [ + { + executable: 'node', + args: [{ kind: 'public', value: 'path with space.js' }], + timeoutMs: 1_000, + env: { MODE: { kind: 'public', value: 'check' } }, + }, + ], + }, + }, + ), + ).not.toThrow() + }) + + it('analyzes, measures, approves, executes, grades, and links one exact receipt', async () => { + const fixture = createCandidateExecutionFixture() + const { digest: _digest, ...bundleInput } = fixture.bundle + const profile = fixtureProfile() + const proposed = await proposeAgentImprovement({ + runId: 'analysis-run-1', + profile, + analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, + improvement: { + surface: 'prompt', + generator: proposer, + scenarios, + judge, + agent, + budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, + }, + buildCandidate: ({ improvement }) => alignedSealedBundle(bundleInput, improvement.profile), + now: () => new Date('2026-07-10T01:00:00.000Z'), + }) + + expect(proposed.proposal.evaluation.gateDecision).toBe('ship') + expect(proposed.proposal.evaluation.lift).toBeGreaterThan(0) + expect(proposed.proposal.findings).toEqual([finding]) + expect(proposed.proposal.candidateProfile.prompt?.systemPrompt).toBe('PROMOTED') + expect(proposed.proposal.candidateBundle?.digest).toMatch(/^sha256:[a-f0-9]{64}$/) + expect(verifyAgentImprovementProposal(proposed.proposal)).toEqual(proposed.proposal) + expect( + createAgentImprovementProposal({ + runId: 'analysis-run-1', + surface: 'prompt', + baselineProfile: profile, + candidateProfile: proposed.improvement.profile, + findings: proposed.analysis.analystResult.findings, + evaluation: proposed.improvement.raw, + candidateBundle: proposed.proposal.candidateBundle, + now: () => new Date('2026-07-10T01:00:00.000Z'), + }), + ).toEqual(proposed.proposal) + expect(() => + createAgentImprovementProposal({ + runId: 'analysis-run-unmeasured', + surface: 'prompt', + baselineProfile: profile, + candidateProfile: { + ...proposed.improvement.profile, + prompt: { systemPrompt: 'UNMEASURED' }, + }, + findings: proposed.analysis.analystResult.findings, + evaluation: proposed.improvement.raw, + }), + ).toThrow(/does not match the measured winner surface/) + expect(() => + createAgentImprovementProposal({ + runId: 'analysis-run-malformed-profile', + surface: 'agent-profile', + baselineProfile: profile, + candidateProfile: profile, + findings: proposed.analysis.analystResult.findings, + evaluation: { + ...proposed.improvement.raw, + winner: { + ...proposed.improvement.raw.winner, + surface: '{not-json', + }, + }, + }), + ).toThrow(/not valid JSON/) + + const review = reviewAgentImprovementProposal(proposed.proposal, { + decision: 'approve', + reviewedBy: 'operator@example.com', + reason: 'Measured winner passed the held-back scenarios.', + now: () => new Date('2026-07-10T02:00:00.000Z'), + }) + expect(verifyAgentImprovementReview(review)).toEqual(review) + + const traceStore = new InMemoryTraceStore() + let request: AgentCandidateExecutorRequest | undefined + const executor: AgentCandidateExecutorPort = { + execute: async (input) => { + request = input + await traceStore.appendRun({ + runId: input.trace.runId, + scenarioId: 'approved-candidate', + startedAt: 100, + endedAt: 200, + status: 'completed', + tags: { ...input.trace.tags }, + }) + return { executionId: input.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + }), + } + const outputs = createCandidateOutputFixture() + const executed = await executeApprovedAgentCandidate({ + proposal: proposed.proposal, + review, + authorizeReview: async (candidateReview) => candidateReview.digest === review.digest, + task: fixture.task, + ports: fixture.ports, + execution: { + executor, + traceStore, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + ...outputs, + }, + }) + + expect(request).toBeDefined() + expect(executed.finalization.succeeded).toBe(true) + expect(executed.evidence).toMatchObject({ + proposalDigest: proposed.proposal.digest, + reviewDigest: review.digest, + bundleDigest: proposed.proposal.candidateBundle?.digest, + executionId: fixture.task.executionId, + succeeded: true, + runReceiptDigest: expect.stringMatching(/^sha256:/), + }) + }) + + it('records rejection and refuses to execute it', async () => { + const fixture = createCandidateExecutionFixture() + const { digest: _digest, ...bundleInput } = fixture.bundle + const proposed = await proposeAgentImprovement({ + runId: 'analysis-run-2', + profile: fixtureProfile(), + analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, + improvement: { + surface: 'prompt', + generator: proposer, + scenarios, + judge, + agent, + budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, + }, + buildCandidate: ({ improvement }) => alignedBundle(bundleInput, improvement.profile), + }) + const review = reviewAgentImprovementProposal(proposed.proposal, { + decision: 'reject', + reviewedBy: 'operator@example.com', + reason: 'The change is not appropriate for this deployment.', + feedback: 'Keep the baseline behavior.', + }) + + await expect( + executeApprovedAgentCandidate({ + proposal: proposed.proposal, + review, + authorizeReview: async () => true, + task: fixture.task, + ports: fixture.ports, + execution: { + executor: { + execute: async () => { + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore: new InMemoryTraceStore(), + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + ...createCandidateOutputFixture(), + }, + }), + ).rejects.toThrow('not an approval') + }) + + it('rejects a sealed build candidate whose digest was tampered', async () => { + const fixture = createCandidateExecutionFixture() + const { digest: _digest, ...bundleInput } = fixture.bundle + + await expect( + proposeAgentImprovement({ + runId: 'analysis-run-tampered-bundle', + profile: fixtureProfile(), + analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, + improvement: { + surface: 'prompt', + generator: proposer, + scenarios, + judge, + agent, + budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, + }, + buildCandidate: ({ improvement }) => ({ + ...alignedSealedBundle(bundleInput, improvement.profile), + digest: candidateSha('0'), + }), + }), + ).rejects.toThrow('built candidate bundle digest is invalid') + }) + + it('refuses a structurally valid approval that its authority does not recognize', async () => { + const fixture = createCandidateExecutionFixture() + const { digest: _digest, ...bundleInput } = fixture.bundle + const proposed = await proposeAgentImprovement({ + runId: 'analysis-run-unauthorized', + profile: fixtureProfile(), + analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, + improvement: { + surface: 'prompt', + generator: proposer, + scenarios, + judge, + agent, + budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, + }, + buildCandidate: ({ improvement }) => alignedBundle(bundleInput, improvement.profile), + }) + const review = reviewAgentImprovementProposal(proposed.proposal, { + decision: 'approve', + reviewedBy: 'forged@example.com', + reason: 'Self-authored approval.', + }) + + await expect( + executeApprovedAgentCandidate({ + proposal: proposed.proposal, + review, + authorizeReview: async () => false, + task: fixture.task, + ports: fixture.ports, + execution: { + executor: { + execute: async () => { + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore: new InMemoryTraceStore(), + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + ...createCandidateOutputFixture(), + }, + }), + ).rejects.toThrow('not authorized') + }) + + it('rejects judge-derived findings before they can steer a proposal', async () => { + await expect( + proposeAgentImprovement({ + runId: 'analysis-run-3', + profile: fixtureProfile(), + analysis: { + registry: registry([{ ...finding, derived_from_judge: true }]), + inputs: {}, + findingsStore: null, + log: () => {}, + }, + improvement: { + surface: 'prompt', + generator: proposer, + scenarios, + judge, + agent, + budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, + }, + }), + ).rejects.toThrow(/judge/i) + }) +}) diff --git a/tests/improvement-driver.test.ts b/tests/improvement-driver.test.ts index 263b29be..62c85f62 100644 --- a/tests/improvement-driver.test.ts +++ b/tests/improvement-driver.test.ts @@ -1,13 +1,20 @@ import { execFileSync } from 'node:child_process' -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import type { AnalystFinding } from '@tangle-network/agent-eval' -import { gitWorktreeAdapter, type ProposeContext } from '@tangle-network/agent-eval/campaign' +import { type AnalystFinding, CostLedger } from '@tangle-network/agent-eval' +import { + gitWorktreeAdapter, + type ProposeContext, + verifyCodeSurface, + type Worktree, + type WorktreeAdapter, +} from '@tangle-network/agent-eval/campaign' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import type { SurfaceImprovementEdit } from '../src/agent/improvement-adapter' import type { ImprovementAdapter, ImprovementEditBatch } from '../src/analyst-loop/types' import { improvementDriver, reflectiveGenerator } from '../src/improvement' +import type { CandidateGenerator } from '../src/improvement/improvement-driver' function git(args: string[], cwd: string): string { return execFileSync('git', args, { cwd, encoding: 'utf8' }).trim() @@ -74,6 +81,34 @@ function reflectiveDriver(adapter: ImprovementAdapter) { } describe('improvementDriver — reflective generator', () => { + it('forwards the run-wide paid-call account and phase to every candidate author', async () => { + const observed: Array<{ + costLedger: CostLedger | undefined + costPhase: string | undefined + }> = [] + const costLedger = new CostLedger() + const generator: CandidateGenerator = { + kind: 'cost-forwarding-fixture', + async generate(args) { + observed.push({ costLedger: args.costLedger, costPhase: args.costPhase }) + return { applied: false, summary: '' } + }, + } + const driver = improvementDriver({ + generator, + worktree: gitWorktreeAdapter({ repoRoot }), + baseRef: 'main', + }) + const context = { + ...ctxWith(FINDINGS), + costLedger, + costPhase: 'search.proposal', + } as ProposeContext + + await expect(driver.propose(context)).resolves.toEqual([]) + expect(observed).toEqual([{ costLedger, costPhase: 'search.proposal' }]) + }) + it('applies drafted edits into one worktree and returns a CodeSurface', async () => { const adapter = stubAdapter({ edits: [editFixture(GOOD_PATCH)], skipped: 0, errors: [] }) const driver = reflectiveDriver(adapter) @@ -139,6 +174,136 @@ describe('improvementDriver — reflective generator', () => { expect(git(['worktree', 'list'], repoRoot).split('\n').length).toBe(1) }) + it('still proposes populationSize candidates on EMPTY findings when the generator opts in (proposesWithoutFindings)', async () => { + // The meta-harness contract: an agentic coder draws its signal from the repo + // + raw traces on disk, so it must run even when the distiller yielded no + // findings and there is no report (the first-generation case — its seed + // findings are empty and rawTraceDistiller has not run yet). Regression for + // the "surface:'code' generates ZERO candidates" bug. + let generateCalls = 0 + const seedDriver = improvementDriver({ + generator: { + kind: 'agentic-stub', + proposesWithoutFindings: true, + async generate({ worktreePath }) { + generateCalls++ + // Dirty the worktree so the driver finalizes it into a CodeSurface. + writeFileSync(join(worktreePath, 'prompt.md'), 'edited from raw traces\n') + return { applied: true, summary: 'from-seed edit' } + }, + }, + worktree: gitWorktreeAdapter({ repoRoot }), + baseRef: 'main', + }) + + const surfaces = await seedDriver.propose({ ...ctxWith([]), populationSize: 2 }) + + expect(generateCalls).toBe(2) + expect(surfaces).toHaveLength(2) + for (const surface of surfaces) { + if (typeof surface === 'string') throw new Error('expected CodeSurface') + expect(surface.kind).toBe('code') + expect(readFileSync(join(surface.worktreeRef, 'prompt.md'), 'utf8')).toBe( + 'edited from raw traces\n', + ) + } + }) + + it('forks isolated generation-two candidates from the promoted generation-one surface', async () => { + const worktree = gitWorktreeAdapter({ repoRoot }) + const baselineWorktree = await worktree.create({ baseRef: 'main', label: 'baseline' }) + const baseline = await worktree.finalize(baselineWorktree, 'baseline') + let call = 0 + const driver = improvementDriver({ + generator: { + kind: 'cumulative-stub', + proposesWithoutFindings: true, + async generate({ worktreePath }) { + expect(git(['status', '--porcelain'], worktreePath)).toBe('') + const current = readFileSync(join(worktreePath, 'prompt.md'), 'utf8') + if (call === 0) { + expect(current).toBe('lax rubric\n') + writeFileSync(join(worktreePath, 'prompt.md'), 'lax rubric\ngeneration one\n') + } else { + expect(current).toBe('lax rubric\ngeneration one\n') + writeFileSync( + join(worktreePath, 'prompt.md'), + `lax rubric\ngeneration one\ngeneration two candidate ${call}\n`, + ) + } + call++ + return { applied: true, summary: `generation ${call}` } + }, + }, + worktree, + baseRef: 'main', + }) + + const [generationOne] = await driver.propose({ + ...ctxWith([]), + currentSurface: baseline, + generation: 0, + }) + if (!generationOne || typeof generationOne === 'string') { + throw new Error('expected generation-one CodeSurface') + } + + const generationTwo = await driver.propose({ + ...ctxWith([]), + currentSurface: generationOne, + generation: 1, + populationSize: 2, + }) + expect(generationTwo).toHaveLength(2) + const codeSurfaces = generationTwo.map((surface) => { + if (typeof surface === 'string') throw new Error('expected generation-two CodeSurface') + return surface + }) + + for (const [index, surface] of codeSurfaces.entries()) { + const expected = `lax rubric\ngeneration one\ngeneration two candidate ${index + 1}\n` + expect(readFileSync(join(surface.worktreeRef, 'prompt.md'), 'utf8')).toBe(expected) + expect(surface.baseCommit).toBe(baseline.baseCommit) + expect(surface.baseTree).toBe(baseline.baseTree) + expect( + git(['merge-base', generationOne.candidateCommit, surface.candidateCommit], repoRoot), + ).toBe(generationOne.candidateCommit) + + const verified = verifyCodeSurface(surface) + const applyWorktree = await worktree.create({ + baseRef: surface.baseCommit, + label: `apply-check-${index}`, + }) + try { + execFileSync('git', ['apply', '--check', '--binary', '-'], { + cwd: applyWorktree.path, + input: Buffer.from(verified.patchBytes), + }) + execFileSync('git', ['apply', '--binary', '-'], { + cwd: applyWorktree.path, + input: Buffer.from(verified.patchBytes), + }) + expect(readFileSync(join(applyWorktree.path, 'prompt.md'), 'utf8')).toBe(expected) + } finally { + await worktree.discard(applyWorktree) + } + } + + expect(readFileSync(join(generationOne.worktreeRef, 'prompt.md'), 'utf8')).toBe( + 'lax rubric\ngeneration one\n', + ) + expect(git(['show', 'main:prompt.md'], repoRoot)).toBe('lax rubric') + + const [winner, loser] = codeSurfaces + if (!winner || !loser) throw new Error('expected two generation-two candidates') + await driver.cleanup([winner.worktreeRef]) + expect(existsSync(generationOne.worktreeRef)).toBe(false) + expect(existsSync(loser.worktreeRef)).toBe(false) + expect(existsSync(winner.worktreeRef)).toBe(true) + expect(verifyCodeSurface(winner).contentHash).toMatch(/^sha256:/) + expect(git(['show', 'main:prompt.md'], repoRoot)).toBe('lax rubric') + }) + it('rethrows and leaves NO orphaned worktree when the generator throws', async () => { // A generator whose generate() throws mid-candidate must not leak the // already-created worktree (the try/finally cleanup in propose()). @@ -159,4 +324,63 @@ describe('improvementDriver — reflective generator', () => { // worktree remains. expect(git(['worktree', 'list'], repoRoot).split('\n').length).toBe(1) }) + + it('retries candidate cleanup when generation and the first discard both fail', async () => { + const realWorktree = gitWorktreeAdapter({ repoRoot }) + let discardAttempts = 0 + const flakyWorktree: WorktreeAdapter = { + ...realWorktree, + async discard(worktree: Worktree) { + discardAttempts += 1 + if (discardAttempts === 1) throw new Error('transient discard failure') + await realWorktree.discard(worktree) + }, + } + const driver = improvementDriver({ + generator: { + kind: 'throws-with-flaky-cleanup', + async generate() { + throw new Error('generation failed') + }, + }, + worktree: flakyWorktree, + baseRef: 'main', + }) + + await expect(driver.propose(ctxWith(FINDINGS))).rejects.toThrow('generation failed') + expect(discardAttempts).toBe(2) + expect(git(['worktree', 'list'], repoRoot).split('\n')).toHaveLength(1) + }) + + it('attempts every candidate cleanup even when one discard fails', async () => { + let created = 0 + const discardAttempts: string[] = [] + const driver = improvementDriver({ + generator: { + kind: 'cleanup-stub', + proposesWithoutFindings: true, + async generate() { + return { applied: true, summary: 'candidate' } + }, + }, + worktree: { + async create({ baseRef }) { + const id = String(created++) + return { path: `/tmp/candidate-${id}`, branch: `candidate-${id}`, baseRef } + }, + async finalize(worktree) { + return { kind: 'code', worktreeRef: worktree.path, baseRef: worktree.baseRef } + }, + async discard(worktree) { + discardAttempts.push(worktree.path) + if (worktree.path.endsWith('-0')) throw new Error('first discard failed') + }, + }, + baseRef: 'main', + }) + + await driver.propose({ ...ctxWith([]), populationSize: 2 }) + await expect(driver.cleanup()).rejects.toThrow(/failed to discard candidate worktrees/) + expect(discardAttempts).toEqual(['/tmp/candidate-0', '/tmp/candidate-1']) + }) }) diff --git a/tests/loops/loop-dispatch.test.ts b/tests/loops/loop-dispatch.test.ts index 35c0eedc..1430d822 100644 --- a/tests/loops/loop-dispatch.test.ts +++ b/tests/loops/loop-dispatch.test.ts @@ -1,4 +1,5 @@ -import type { DispatchContext } from '@tangle-network/agent-eval/campaign' +import { CostLedger } from '@tangle-network/agent-eval' +import type { CampaignCostMeter, DispatchContext } from '@tangle-network/agent-eval/campaign' import type { CreateSandboxOptions, AgentProfile as SandboxAgentProfile, @@ -26,7 +27,7 @@ interface FakeScenario { kind: string } -const sandboxProfile: SandboxAgentProfile = { name: 'stub' } +const sandboxProfile: SandboxAgentProfile = { name: 'stub', model: { default: 'm' } } function spec(): AgentRunSpec { return { profile: sandboxProfile, name: 'agent', taskToPrompt: (t) => t.goal } @@ -60,15 +61,23 @@ function stubClient(events: SandboxEvent[]): { } /** Minimal campaign DispatchContext that records what the dispatch reports. */ -function fakeDispatchContext(): { +function fakeDispatchContext(costCeilingUsd?: number): { ctx: DispatchContext - observed: Array<{ usd: number; src: string }> - tokens: { input: number; output: number } + ledger: CostLedger spans: string[] } { - const observed: Array<{ usd: number; src: string }> = [] - const tokens = { input: 0, output: 0 } + const ledger = new CostLedger(costCeilingUsd === undefined ? {} : { costCeilingUsd }) const spans: string[] = [] + const cost: CampaignCostMeter = { + runPaidCall(input) { + return ledger.runPaidCall({ + ...input, + channel: input.channel ?? 'agent', + phase: 'test-cell', + tags: { cellId: 'cell-0' }, + }) + }, + } const ctx: DispatchContext = { cellId: 'cell-0', rep: 0, @@ -89,23 +98,9 @@ function fakeDispatchContext(): { return 'p' }, }, - cost: { - observe(usd: number, src: string) { - observed.push({ usd, src }) - }, - observeTokens(u: { input: number; output: number }) { - tokens.input += u.input - tokens.output += u.output - }, - current() { - return 0 - }, - tokens() { - return tokens - }, - }, + cost, } - return { ctx, observed, tokens, spans } + return { ctx, ledger, spans } } describe('loopDispatch', () => { @@ -130,8 +125,18 @@ describe('loopDispatch', () => { const artifact = await dispatch({ id: 'fixture-a', kind: 'eval-fixture' }, fake.ctx) expect(artifact).toEqual({ attempt: 3 }) - expect(fake.observed).toEqual([{ usd: 0.015, src: 'loop' }]) - expect(fake.tokens).toEqual({ input: 120, output: 40 }) + expect(fake.ledger.list()).toEqual([ + expect.objectContaining({ + channel: 'agent', + phase: 'test-cell', + actor: 'loop', + model: 'm', + inputTokens: 120, + outputTokens: 40, + actualCostUsd: 0.015, + costUsd: 0.015, + }), + ]) expect(fake.spans).toContain('loop.started') expect(fake.spans).toContain('loop.ended') }) @@ -154,14 +159,22 @@ describe('loopDispatch', () => { }) const fake = fakeDispatchContext() - const profile = { id: 'baseline', model: 'test-model@2025-01-01' } + const profile = { name: 'baseline', model: { default: 'test-model@2025-01-01' } } const artifact = await dispatch(profile, { id: 's1', kind: 'task' }, fake.ctx) // Returns the loop's winner output. expect(artifact).toEqual({ attempt: 2 }) // Usage reported to the campaign cost meter — the integrity guard's input. - expect(fake.observed).toEqual([{ usd: 0.02, src: 'loop' }]) - expect(fake.tokens).toEqual({ input: 150, output: 60 }) + expect(fake.ledger.list()).toEqual([ + expect.objectContaining({ + actor: 'loop', + model: 'test-model@2025-01-01', + inputTokens: 150, + outputTokens: 60, + actualCostUsd: 0.02, + costUsd: 0.02, + }), + ]) // Loop trace events forwarded into the campaign trace as spans. expect(fake.spans).toContain('loop.started') expect(fake.spans).toContain('loop.ended') @@ -189,10 +202,115 @@ describe('loopDispatch', () => { }), }) const fake = fakeDispatchContext() - await dispatch({ id: 'p', model: 'm@2025-01-01' }, { id: 's1', kind: 'task' }, fake.ctx) + await dispatch( + { name: 'p', model: { default: 'm@2025-01-01' } }, + { id: 's1', kind: 'task' }, + fake.ctx, + ) // The validator failed, but real LLM activity happened — tokens + cost MUST // still reach the cost meter, or the integrity guard would call it a stub. - expect(fake.tokens).toEqual({ input: 90, output: 20 }) - expect(fake.observed).toEqual([{ usd: 0.01, src: 'loop' }]) + expect(fake.ledger.list()).toEqual([ + expect.objectContaining({ + inputTokens: 90, + outputTokens: 20, + actualCostUsd: 0.01, + costUsd: 0.01, + }), + ]) + }) + + it('settles partial terminal usage when a sandbox stream fails after spending', async () => { + const dispatch = loopCampaignDispatch({ + sandboxClient: { + async create() { + return { + async *streamPrompt() { + yield { + type: 'llm_call', + data: { tokensIn: 33, tokensOut: 7, costUsd: 0.004, model: 'm' }, + } as SandboxEvent + throw new Error('sandbox stream failed') + }, + } as unknown as SandboxInstance + }, + }, + toLoopOptions: () => ({ + driver: refineDriver(), + agentRun: spec(), + output, + task: { goal: 'partial failure' }, + maxIterations: 1, + }), + }) + const fake = fakeDispatchContext() + + await expect(dispatch({ id: 'partial', kind: 'task' }, fake.ctx)).resolves.toBeUndefined() + expect(fake.ledger.list()).toEqual([ + expect.objectContaining({ + inputTokens: 33, + outputTokens: 7, + actualCostUsd: 0.004, + costUsd: 0.004, + }), + ]) + }) + + it('refuses a capped cell before creating a sandbox when no hard maximum is supplied', async () => { + let creates = 0 + const dispatch = loopCampaignDispatch({ + sandboxClient: { + async create() { + creates += 1 + throw new Error('must not dispatch') + }, + }, + toLoopOptions: () => ({ + driver: refineDriver(), + agentRun: spec(), + output, + task: { goal: 'bounded' }, + maxIterations: 1, + }), + }) + const fake = fakeDispatchContext(1) + + await expect(dispatch({ id: 'bounded', kind: 'task' }, fake.ctx)).rejects.toThrow( + /hard maximumCharge before execution/, + ) + expect(creates).toBe(0) + expect(fake.ledger.list()).toHaveLength(0) + }) + + it('admits a capped cell when the executor supplies an enforced maximum', async () => { + let creates = 0 + const dispatch = loopCampaignDispatch({ + sandboxClient: { + async create() { + creates += 1 + return stubClient([ + { type: 'llm_call', data: { tokensIn: 10, tokensOut: 5, costUsd: 0.01 } }, + { type: 'result', data: { attempt: 1 } }, + ]).create() + }, + }, + maximumCharge: { externallyEnforcedMaximumUsd: 0.02 }, + toLoopOptions: () => ({ + driver: refineDriver(), + agentRun: spec(), + output, + validator: passAlways, + task: { goal: 'bounded' }, + maxIterations: 1, + }), + }) + const fake = fakeDispatchContext(1) + + await expect(dispatch({ id: 'bounded', kind: 'task' }, fake.ctx)).resolves.toEqual({ + attempt: 1, + }) + expect(creates).toBe(1) + expect(fake.ledger.list()).toEqual([ + expect.objectContaining({ maximumCostUsd: 0.02, costUsd: 0.01 }), + ]) }) }) diff --git a/tests/loops/supervise.test.ts b/tests/loops/supervise.test.ts index acc3415b..014c3983 100644 --- a/tests/loops/supervise.test.ts +++ b/tests/loops/supervise.test.ts @@ -218,6 +218,21 @@ describe('conserved budget pool', () => { expect(pool.readout().usdLeft).toBe(0) }) + it('never interprets explicitly unknown dollar cost as $0 under a dollar limit', () => { + const pool = createBudgetPool({ maxIterations: 2, maxTokens: 1000, maxUsd: 1 }, () => 0) + const r = pool.reserve({ maxIterations: 1, maxTokens: 500, maxUsd: 1 } as Budget) + if (!r.ok) throw new Error('reserve should have succeeded') + expect(() => + pool.reconcile(r.ticket, { + iterations: 1, + tokens: { input: 40, output: 60 }, + usd: 0, + usdKnown: false, + ms: 0, + }), + ).toThrow(/unknown dollar cost/) + }) + it('spendFromUsageEvents folds tokens + usd on separate channels', () => { const spend = spendFromUsageEvents([ { kind: 'iteration' }, diff --git a/tests/loops/trajectory-recorder.test.ts b/tests/loops/trajectory-recorder.test.ts index 3ae3fcec..62e4b489 100644 --- a/tests/loops/trajectory-recorder.test.ts +++ b/tests/loops/trajectory-recorder.test.ts @@ -32,4 +32,16 @@ describe('analyzeTrace (settle-time agent-eval analyzers over a TraceSource)', ( expect(analysis.trajectory.toolCalls).toBe(3) expect(analysis.stuckLoop.findings).toHaveLength(0) }) + + it('does not mislabel overlapping repeated calls as a serial loop', async () => { + const { source, record } = createPushTraceSource({ runId: 'parallel' }) + record({ toolName: 'search', args: { q: 'x' }, startedAt: 0, endedAt: 100 }) + record({ toolName: 'search', args: { q: 'x' }, startedAt: 10, endedAt: 90 }) + record({ toolName: 'search', args: { q: 'x' }, startedAt: 20, endedAt: 80 }) + + const analysis = await analyzeTrace(source) + + expect(analysis.trajectory.toolCalls).toBe(3) + expect(analysis.stuckLoop.findings).toHaveLength(0) + }) }) diff --git a/tests/mcp-serve-verifier.test.ts b/tests/mcp-serve-verifier.test.ts index 2b042cec..ee9dbf5e 100644 --- a/tests/mcp-serve-verifier.test.ts +++ b/tests/mcp-serve-verifier.test.ts @@ -6,9 +6,21 @@ import { mcpServeVerifier } from '../src/improvement/mcp-serve-verifier' // A minimal stdio MCP server (newline-delimited JSON-RPC 2.0) used as the // "built server" the verifier boots. Variants exercise each outcome. -function serverScript(variant: 'ok' | 'no-tools' | 'crash' | 'hang'): string { +function serverScript(variant: 'ok' | 'no-tools' | 'crash' | 'hang' | 'closed-stdin'): string { if (variant === 'crash') return 'process.exit(1)\n' if (variant === 'hang') return 'setInterval(() => {}, 1000)\n' // never answers + if (variant === 'closed-stdin') { + return [ + 'import { closeSync } from "node:fs"', + 'process.stdin.on("error", () => {})', + 'process.stdin.once("data", (chunk) => {', + ' const request = JSON.parse(String(chunk).trim())', + ' closeSync(0)', + ' process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result: { protocolVersion: "2024-11-05", capabilities: {}, serverInfo: { name: "fake", version: "0" } } }) + "\\n")', + '})', + 'setInterval(() => {}, 1000)', + ].join('\n') + } const tools = variant === 'no-tools' ? '[]' @@ -31,7 +43,7 @@ beforeEach(() => { }) afterEach(() => rmSync(dir, { recursive: true, force: true })) -function write(variant: 'ok' | 'no-tools' | 'crash' | 'hang'): string { +function write(variant: 'ok' | 'no-tools' | 'crash' | 'hang' | 'closed-stdin'): string { const path = join(dir, `server-${variant}.mjs`) writeFileSync(path, serverScript(variant)) return path @@ -57,6 +69,17 @@ describe('mcpServeVerifier — boot-and-probe', () => { expect(res.feedback).toMatch(/exited|serving/) }) + it('fails when the server closes stdin during the handshake', async () => { + const verify = mcpServeVerifier({ + command: 'node', + args: [write('closed-stdin')], + timeoutMs: 800, + }) + const res = await verify(dir) + expect(res.ok).toBe(false) + expect(res.feedback).toContain('writing to MCP server stdin failed') + }) + it('fails (does not hang) a server that never answers, via timeout', async () => { const verify = mcpServeVerifier({ command: 'node', args: [write('hang')], timeoutMs: 800 }) const res = await verify(dir) diff --git a/tests/mcp/codex-diagnostics.test.ts b/tests/mcp/codex-diagnostics.test.ts new file mode 100644 index 00000000..66a07e3e --- /dev/null +++ b/tests/mcp/codex-diagnostics.test.ts @@ -0,0 +1,51 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + createCodexExecutionDiagnosticError, + readCodexAuthRedactionText, +} from '../../src/mcp/codex-diagnostics' + +describe('Codex failure diagnostics', () => { + it('redacts credentials from the message, structured fields, and stored cause', () => { + const secret = 'sk-cause-proof-1234567890' + const codexHome = '/tmp/private-codex-home' + const error = createCodexExecutionDiagnosticError({ + reason: `request failed: Bearer ${secret}`, + exitCode: 1, + killedBySignal: null, + timedOut: false, + durationMs: 12, + stdout: JSON.stringify({ type: 'error', access_token: secret }), + stderr: `auth path ${codexHome}; client_secret="${secret}"`, + codexHome, + redactionValues: [secret], + cause: new Error(`provider rejected api_key=${secret}`), + }) + + const cause = error.cause as Error + const serialized = JSON.stringify({ + message: error.message, + reason: error.reason, + diagnostic: error.diagnostic, + cause: cause.stack, + }) + expect(serialized).not.toContain(secret) + expect(serialized).not.toContain(codexHome) + expect(serialized).toContain('') + expect(serialized).toContain('') + }) + + it('refuses non-regular and oversized auth sources before reading them', async () => { + const root = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-auth-read-')) + const oversized = join(root, 'oversized.json') + writeFileSync(oversized, Buffer.alloc(1024 * 1024 + 1)) + try { + await expect(readCodexAuthRedactionText(root)).rejects.toThrow(/regular file/) + await expect(readCodexAuthRedactionText(oversized)).rejects.toThrow(/no larger than 1 MiB/) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/mcp/local-harness.test.ts b/tests/mcp/local-harness.test.ts index 691eb32c..46b6dbbb 100644 --- a/tests/mcp/local-harness.test.ts +++ b/tests/mcp/local-harness.test.ts @@ -1,8 +1,26 @@ import type { ChildProcess } from 'node:child_process' +import { createHash } from 'node:crypto' import { EventEmitter } from 'node:events' +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import type { AgentProfile } from '@tangle-network/sandbox' import { describe, expect, it, vi } from 'vitest' -import { harnessInvocation, runLocalHarness } from '../../src/mcp/local-harness' +import { + CodexExecutionDiagnosticError, + harnessInvocation, + parseCodexTokenUsage, + runLocalHarness, +} from '../../src/mcp/local-harness' function makeFakeChild(opts: { stdoutChunks?: string[] @@ -38,6 +56,156 @@ function makeFakeChild(opts: { return emitter } +function makeStaticElfFixture(directory: string): string { + const path = join(directory, 'codex-static-fixture') + const elf = Buffer.alloc(120) + Buffer.from([0x7f, 0x45, 0x4c, 0x46]).copy(elf) + elf[4] = 2 + elf[5] = 1 + elf.writeUInt16LE(3, 16) + elf.writeUInt16LE(process.arch === 'arm64' ? 183 : 62, 18) + elf.writeBigUInt64LE(64n, 32) + elf.writeUInt16LE(64, 52) + elf.writeUInt16LE(56, 54) + elf.writeUInt16LE(1, 56) + elf.writeUInt32LE(1, 64) + writeFileSync(path, elf) + chmodSync(path, 0o700) + return path +} + +async function captureReproducibleCodexFailure(opts: { + stdoutChunks?: string[] + stderrChunks?: string[] + exitCode?: number | null + signal?: NodeJS.Signals | null + delayCloseMs?: number + timeoutMs?: number + auth?: unknown + envSecret?: string +}): Promise { + const cwd = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-failure-')) + const authHome = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-auth-')) + writeFileSync(join(authHome, 'auth.json'), JSON.stringify(opts.auth ?? {})) + const staticCodex = makeStaticElfFixture(cwd) + const invocation = harnessInvocation( + 'codex', + { model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' } }, + 'task', + { codexReproducible: true }, + ) + try { + await runLocalHarness({ + harness: 'codex', + cwd, + taskPrompt: 'task', + invocation, + codexReproducible: true, + ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}), + env: { + CODEX_HOME: authHome, + ...(opts.envSecret ? { OPENAI_API_KEY: opts.envSecret } : {}), + }, + resolveCodexExecutable: async () => staticCodex, + // The injected child is the Codex CLI process boundary; filesystem/config logic stays real. + spawn: (_command, args) => { + if (args[0] === '--version') { + return makeFakeChild({ stdoutChunks: ['codex-cli 0.144.1\n'] }) + } + if (args[0] === 'sandbox') return makeFakeChild({ exitCode: 0 }) + if (args[0] === 'debug') { + return makeFakeChild({ + stdoutChunks: [ + JSON.stringify([ + { + content: [ + { + text: 'xx', + }, + ], + }, + { content: [{ text: args.at(-1) }] }, + ]), + ], + }) + } + return makeFakeChild({ + stdoutChunks: opts.stdoutChunks, + stderrChunks: opts.stderrChunks, + exitCode: opts.exitCode, + signal: opts.signal, + delayCloseMs: opts.delayCloseMs, + }) + }, + }) + } catch (error) { + if (error instanceof CodexExecutionDiagnosticError) return error + throw error + } finally { + rmSync(cwd, { recursive: true, force: true }) + rmSync(authHome, { recursive: true, force: true }) + } + throw new Error('expected reproducible Codex execution to fail') +} + +async function runCodexPromptEvidenceFixture( + requestedPrompt: string, + renderPrompt: (prompt: string) => string, +) { + const cwd = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-prompt-')) + const authHome = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-auth-')) + writeFileSync(join(authHome, 'auth.json'), '{}') + const staticCodex = makeStaticElfFixture(cwd) + const profile: AgentProfile = { + name: 'reproducible', + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + } + const invocation = harnessInvocation('codex', profile, requestedPrompt, { + codexReproducible: true, + }) + try { + return await runLocalHarness({ + harness: 'codex', + cwd, + taskPrompt: requestedPrompt, + invocation, + codexReproducible: true, + env: { CODEX_HOME: authHome }, + resolveCodexExecutable: async () => staticCodex, + spawn: (_command, args) => { + if (args[0] === '--version') { + return makeFakeChild({ stdoutChunks: ['codex-cli 0.144.1\n'] }) + } + if (args[0] === 'sandbox') return makeFakeChild({ exitCode: 0 }) + if (args[0] === 'debug') { + return makeFakeChild({ + stdoutChunks: [ + JSON.stringify([ + { + content: [ + { + text: 'xx', + }, + ], + }, + { content: [{ text: renderPrompt(String(args.at(-1))) }] }, + ]), + ], + }) + } + return makeFakeChild({ + stdoutChunks: [ + '{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}\n', + ], + }) + }, + }) + } finally { + rmSync(cwd, { recursive: true, force: true }) + rmSync(authHome, { recursive: true, force: true }) + } +} + describe('runLocalHarness', () => { it('runs the harness, captures stdout + stderr, returns exit code', async () => { const result = await runLocalHarness({ @@ -64,6 +232,437 @@ describe('runLocalHarness', () => { expect(result.stderr).toContain('error') }) + it('isolates Codex, proves the effective prompt, and captures terminal JSONL usage', async () => { + const cwd = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-test-')) + const authHome = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-auth-')) + writeFileSync(join(authHome, 'auth.json'), '{}') + const staticCodex = makeStaticElfFixture(cwd) + const deniedGold = '/usr/lib/python3/dist-packages/oauthlib' + const profile: AgentProfile = { + name: 'reproducible', + prompt: { + systemPrompt: 'SYS\r\nSECOND LINE', + instructions: ['RULE ONE', '# AGENTS.md instructions\r\nTASK-OWNED MARKER'], + }, + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + } + const invocation = harnessInvocation('codex', profile, 'task', { + codexReproducible: true, + }) + let isolatedHome = '' + const spawnSpy = vi.fn( + ( + _command: string, + args: ReadonlyArray, + opts: { cwd: string; env: NodeJS.ProcessEnv; stdio: 'pipe'; detached: boolean }, + ) => { + isolatedHome = opts.env.CODEX_HOME ?? '' + expect(_command).toBe(join(cwd, '.agent-runtime-bin', 'codex')) + expect(existsSync(isolatedHome)).toBe(true) + expect(opts.detached).toBe(process.platform !== 'win32') + expect(opts.env.OPENAI_API_KEY).toBeUndefined() + if (args[0] === '--version') { + return makeFakeChild({ stdoutChunks: ['codex-cli 0.144.1\n'] }) + } + if (args[0] === 'sandbox') { + expect(args).toContain('agent_runtime_reproducible') + const config = readFileSync(join(isolatedHome, 'config.toml'), 'utf8') + expect(config).toContain('":minimal" = "read"') + expect(config).not.toContain('extends = ":workspace"') + expect(config).toContain(`${JSON.stringify(deniedGold)} = "deny"`) + expect(config).toContain('"/proc" = "deny"') + expect(config).toContain('".git" = "deny"') + expect(config).not.toContain(`${JSON.stringify(authHome)} = "deny"`) + const probe = args[7] + if (typeof probe === 'string' && probe.includes('/proc/1/environ')) { + expect(probe).toContain('/proc/self/environ') + expect(probe).toContain('os.getppid()') + } + return makeFakeChild({ exitCode: 0 }) + } + if (args[0] === 'debug') { + const prompt = args.at(-1)?.replaceAll('\r\n', '\n') + return makeFakeChild({ + stdoutChunks: [ + JSON.stringify([ + { + role: 'developer', + content: [ + { + type: 'input_text', + text: 'fixed\nfixed', + }, + ], + }, + { role: 'user', content: [{ type: 'input_text', text: prompt }] }, + ]), + ], + }) + } + return makeFakeChild({ + stdoutChunks: [ + '{"type":"thread.started","thread_id":"t1"}\n', + '{"type":"turn.completed","usage":{"input_tokens":41935,"cached_input_tokens":19200,"output_tokens":273,"reasoning_output_tokens":191}}\n', + ], + stderrChunks: [`warning: ${opts.env.CODEX_HOME}/auth.json\n`], + }) + }, + ) + + const result = await runLocalHarness({ + harness: 'codex', + cwd, + taskPrompt: 'task', + invocation, + codexReproducible: true, + codexReadDeniedPaths: [deniedGold], + env: { CODEX_HOME: authHome, OPENAI_API_KEY: 'must-not-reach-child' }, + spawn: spawnSpy, + resolveCodexExecutable: async () => staticCodex, + }) + + expect(spawnSpy).toHaveBeenCalledTimes(6) + expect(isolatedHome).toMatch(/agent-runtime-codex-/) + expect(existsSync(isolatedHome)).toBe(false) + expect(result.stderr).toContain('/auth.json') + expect(result.stderr).not.toContain(isolatedHome) + expect(result.usage).toEqual({ + inputTokens: 41935, + cachedInputTokens: 19200, + outputTokens: 273, + reasoningOutputTokens: 191, + }) + expect(result.evidence?.cliVersion).toBe('codex-cli 0.144.1') + expect(result.evidence?.executableSha256).toMatch(/^[a-f0-9]{64}$/) + const composedPrompt = + 'SYS\r\nSECOND LINE\n\nRULE ONE\n\n# AGENTS.md instructions\r\nTASK-OWNED MARKER\n\ntask' + expect(invocation.args[1]).toBe(composedPrompt) + expect(result.evidence?.requestedPromptSha256).toBe( + createHash('sha256').update(composedPrompt).digest('hex'), + ) + expect(result.evidence?.effectivePromptSha256).toMatch(/^[a-f0-9]{64}$/) + expect(result.evidence?.effectivePromptSha256).not.toBe(result.evidence?.requestedPromptSha256) + expect(result.evidence?.nonPromptArgsSha256).toMatch(/^[a-f0-9]{64}$/) + expect(result.evidence?.controlledConfigSha256).toMatch(/^[a-f0-9]{64}$/) + expect(result.evidence?.readDeniedPathsSha256).toMatch(/^[a-f0-9]{64}$/) + expect(result.evidence?.readDeniedPathCount).toBe(1) + expect(result.evidence?.readDeniedPaths).toEqual([deniedGold]) + expect(result.evidence?.readDeniedPathsSha256).toBe( + createHash('sha256') + .update(JSON.stringify([deniedGold])) + .digest('hex'), + ) + expect(result.evidence?.policy).toEqual({ + sessionPersistence: 'ephemeral', + userConfig: false, + rules: false, + projectInstructions: false, + skillInstructions: false, + appInstructions: false, + toolSuggestions: false, + multiAgentInstructions: false, + sandbox: 'workspace-write', + permissionProfile: 'agent_runtime_reproducible', + approvalPolicy: 'never', + shellNetwork: false, + webSearch: false, + serviceTier: 'default', + shellEnvironment: 'core-filtered', + loginShell: false, + credentialsReadable: false, + hostHomeReadable: false, + procEnvironment: 'private-sanitized', + sensitiveEnvironmentNamesVisible: false, + parentRepoRead: false, + gitMetadata: false, + temporaryDirectory: 'workspace-private', + stagedExecutable: 'static-elf-read-only', + callerReadDeniedPaths: 'enforced', + containerSockets: false, + }) + expect(existsSync(cwd)).toBe(true) + expect(readdirSync(cwd).some((name) => name.startsWith('.agent-runtime-'))).toBe(false) + rmSync(cwd, { recursive: true, force: true }) + rmSync(authHome, { recursive: true, force: true }) + }) + + it.each([ + { + name: 'lone carriage return', + prompt: 'alpha\rbravo', + render: (prompt: string) => prompt.replaceAll('\r', '\n'), + }, + { + name: 'non-newline whitespace', + prompt: 'alpha bravo', + render: (prompt: string) => prompt.replaceAll(' ', ' '), + }, + ])('rejects a Codex prompt rewrite of $name', async ({ prompt, render }) => { + await expect(runCodexPromptEvidenceFixture(prompt, render)).rejects.toThrow( + /prompt evidence did not contain the exact task prompt/, + ) + }) + + it('rejects a reproducible Codex result without exactly one terminal usage event', async () => { + const cwd = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-test-')) + const authHome = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-auth-')) + writeFileSync(join(authHome, 'auth.json'), '{}') + const staticCodex = makeStaticElfFixture(cwd) + const profile: AgentProfile = { + name: 'reproducible', + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + } + const invocation = harnessInvocation('codex', profile, 'task', { + codexReproducible: true, + }) + await expect( + runLocalHarness({ + harness: 'codex', + cwd, + taskPrompt: 'task', + invocation, + codexReproducible: true, + env: { CODEX_HOME: authHome }, + resolveCodexExecutable: async () => staticCodex, + spawn: (_command, args) => { + if (args[0] === '--version') { + return makeFakeChild({ stdoutChunks: ['codex-cli 0.144.1\n'] }) + } + if (args[0] === 'sandbox') return makeFakeChild({ exitCode: 0 }) + if (args[0] === 'debug') { + return makeFakeChild({ + stdoutChunks: [ + JSON.stringify([ + { + content: [ + { + text: 'xx', + }, + ], + }, + { content: [{ text: args.at(-1) }] }, + ]), + ], + }) + } + return makeFakeChild({ stdoutChunks: ['{"type":"turn.started"}\n'] }) + }, + }), + ).rejects.toThrow(/exactly one Codex turn\.completed usage event/) + rmSync(cwd, { recursive: true, force: true }) + rmSync(authHome, { recursive: true, force: true }) + }) + + it('preserves auth error JSONL without leaking file or environment credentials', async () => { + const authSecret = 'sk-auth-proof-1234567890' + const envSecret = 'sk-env-proof-0987654321' + const unknownBearer = 'unknown-bearer-proof-123456' + const encodedAuthSecret = Buffer.from(authSecret, 'utf8').toString('base64') + const ansiSplitAuthSecret = `${authSecret.slice(0, 8)}\u001b[31m${authSecret.slice(8)}` + const error = await captureReproducibleCodexFailure({ + auth: { auth_mode: 'chatgpt', tokens: { access_token: authSecret } }, + envSecret, + exitCode: 1, + stdoutChunks: [`${JSON.stringify({ type: 'error', message: `401 Bearer ${authSecret}` })}\n`], + stderrChunks: [ + `OPENAI_API_KEY=${envSecret}\nAuthorization: Bearer ${unknownBearer}\nencoded=${encodedAuthSecret}\nansi=${ansiSplitAuthSecret}\n`, + ], + }) + + expect(error.code).toBe('CODEX_EXECUTION_DIAGNOSTIC') + expect(error.reason).toMatch(/exactly one Codex turn\.completed/) + expect(error.cause).toBeInstanceOf(Error) + expect((error.cause as Error).stack).toContain('parseCodexTokenUsage') + expect(error.diagnostic).toMatchObject({ + exitCode: 1, + killedBySignal: null, + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }) + expect(error.diagnostic.stdout).toContain('"type":"error"') + expect(error.diagnostic.stderr).toContain('') + const serialized = JSON.stringify({ + message: error.message, + reason: error.reason, + diagnostic: error.diagnostic, + }) + for (const secret of [authSecret, envSecret, unknownBearer, encodedAuthSecret]) { + expect(serialized).not.toContain(secret) + } + expect(serialized).not.toContain('\u001b') + }) + + it('preserves configuration stderr and exact exit metadata', async () => { + const configSecret = 'config-secret-proof-123456' + const error = await captureReproducibleCodexFailure({ + exitCode: 78, + stderrChunks: [`Error loading config.toml: api_key="${configSecret}"\n`], + }) + + expect(error.diagnostic.exitCode).toBe(78) + expect(error.diagnostic.killedBySignal).toBeNull() + expect(error.diagnostic.timedOut).toBe(false) + expect(error.diagnostic.durationMs).toBeGreaterThanOrEqual(0) + expect(error.diagnostic.stderr).toContain('Error loading config.toml') + expect(error.diagnostic.stderr).toContain('') + expect(error.message).not.toContain(configSecret) + }) + + it('bounds error JSONL while retaining both the opening event and terminal error', async () => { + const padding = 'x'.repeat(12_000) + const stdout = [ + JSON.stringify({ type: 'thread.started', padding }), + JSON.stringify({ type: 'error', message: 'provider request failed before turn completion' }), + '', + ].join('\n') + const error = await captureReproducibleCodexFailure({ + exitCode: 1, + stdoutChunks: [stdout], + }) + + expect(error.diagnostic.stdoutTruncated).toBe(true) + expect(error.diagnostic.stdout.length).toBeLessThanOrEqual(4096) + expect(error.diagnostic.stdout).toContain('thread.started') + expect(error.diagnostic.stdout).toContain('<... OUTPUT TRUNCATED ...>') + expect(error.diagnostic.stdout).toContain('provider request failed before turn completion') + }) + + it('preserves timeout and termination signal when usage is absent', async () => { + const error = await captureReproducibleCodexFailure({ + delayCloseMs: 100, + timeoutMs: 1, + }) + + expect(error.diagnostic.exitCode).toBeNull() + expect(error.diagnostic.killedBySignal).toBe('SIGTERM') + expect(error.diagnostic.timedOut).toBe(true) + }) + + it('rejects caller read-denial paths outside reproducible Codex mode', async () => { + const cwd = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-test-')) + const spawnSpy = vi.fn(() => makeFakeChild({ exitCode: 0 })) + await expect( + runLocalHarness({ + harness: 'codex', + cwd, + taskPrompt: 'task', + codexReadDeniedPaths: ['/usr/lib/example/gold.py'], + spawn: spawnSpy, + }), + ).rejects.toThrow(/requires codexReproducible/) + expect(spawnSpy).not.toHaveBeenCalled() + rmSync(cwd, { recursive: true, force: true }) + }) + + it('rejects relative caller read-denial paths before spawning Codex', async () => { + const cwd = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-test-')) + const invocation = harnessInvocation( + 'codex', + { model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' } }, + 'task', + { codexReproducible: true }, + ) + const spawnSpy = vi.fn(() => makeFakeChild({ exitCode: 0 })) + await expect( + runLocalHarness({ + harness: 'codex', + cwd, + taskPrompt: 'task', + invocation, + codexReproducible: true, + codexReadDeniedPaths: ['relative/gold.py'], + resolveCodexExecutable: async () => '/bin/true', + spawn: spawnSpy, + }), + ).rejects.toThrow(/must contain absolute paths/) + expect(spawnSpy).not.toHaveBeenCalled() + expect(readdirSync(cwd).some((name) => name.startsWith('.agent-runtime-'))).toBe(false) + rmSync(cwd, { recursive: true, force: true }) + }) + + it('requires file-based Codex auth so credentials never enter the process environment', async () => { + const cwd = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-test-')) + const staticCodex = makeStaticElfFixture(cwd) + const invocation = harnessInvocation( + 'codex', + { model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' } }, + 'task', + { codexReproducible: true }, + ) + const spawnSpy = vi.fn(() => makeFakeChild({ exitCode: 0 })) + await expect( + runLocalHarness({ + harness: 'codex', + cwd, + taskPrompt: 'task', + invocation, + codexReproducible: true, + env: { CODEX_HOME: join(cwd, 'missing-auth'), OPENAI_API_KEY: 'must-not-be-used' }, + resolveCodexExecutable: async () => staticCodex, + spawn: spawnSpy, + }), + ).rejects.toThrow(/requires readable auth\.json/) + expect(spawnSpy).not.toHaveBeenCalled() + expect(readdirSync(cwd).some((name) => name.startsWith('.agent-runtime-'))).toBe(false) + rmSync(cwd, { recursive: true, force: true }) + }) + + it('rejects a dynamically linked executable before any Codex probe or model call', async () => { + const cwd = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-test-')) + const invocation = harnessInvocation( + 'codex', + { model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' } }, + 'task', + { codexReproducible: true }, + ) + const spawnSpy = vi.fn(() => makeFakeChild({ exitCode: 0 })) + await expect( + runLocalHarness({ + harness: 'codex', + cwd, + taskPrompt: 'task', + invocation, + codexReproducible: true, + resolveCodexExecutable: async () => '/bin/true', + spawn: spawnSpy, + }), + ).rejects.toThrow(/statically linked Linux Codex ELF/) + expect(spawnSpy).not.toHaveBeenCalled() + expect(existsSync(join(cwd, '.agent-runtime-bin'))).toBe(false) + rmSync(cwd, { recursive: true, force: true }) + }) + + it('does not overwrite or remove a candidate-owned .agent-runtime-bin directory', async () => { + const cwd = mkdtempSync(join(tmpdir(), 'agent-runtime-codex-test-')) + const staticCodex = makeStaticElfFixture(cwd) + const stagedDirectory = join(cwd, '.agent-runtime-bin') + const marker = join(stagedDirectory, 'candidate-owned') + mkdirSync(stagedDirectory) + writeFileSync(marker, 'keep') + const invocation = harnessInvocation( + 'codex', + { model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' } }, + 'task', + { codexReproducible: true }, + ) + const spawnSpy = vi.fn(() => makeFakeChild({ exitCode: 0 })) + await expect( + runLocalHarness({ + harness: 'codex', + cwd, + taskPrompt: 'task', + invocation, + codexReproducible: true, + resolveCodexExecutable: async () => staticCodex, + spawn: spawnSpy, + }), + ).rejects.toThrow(/\.agent-runtime-bin must not already exist/) + expect(spawnSpy).not.toHaveBeenCalled() + expect(readFileSync(marker, 'utf8')).toBe('keep') + rmSync(cwd, { recursive: true, force: true }) + }) + it('throws on spawn error (binary not found)', async () => { await expect( runLocalHarness({ @@ -92,6 +691,39 @@ describe('runLocalHarness', () => { expect(result.killedBySignal).toBe('SIGTERM') }) + it('SIGKILLs the entire process group when it ignores SIGTERM', async () => { + vi.useFakeTimers() + const child = makeFakeChild({ + delayCloseMs: 10_000, + exitCode: 0, + }) + Object.defineProperty(child, 'pid', { value: 43210 }) + const processKill = vi.spyOn(process, 'kill').mockImplementation((pid, signal) => { + expect(pid).toBe(-43210) + if (signal === 'SIGKILL') child.emit('close', null, 'SIGKILL') + return true + }) + try { + const promise = runLocalHarness({ + harness: 'claude', + cwd: '/tmp/wt', + taskPrompt: 'ignore termination', + timeoutMs: 20, + spawn: () => child, + }) + await vi.advanceTimersByTimeAsync(300) + const result = await promise + expect(result.timedOut).toBe(true) + expect(result.killedBySignal).toBe('SIGKILL') + expect(processKill).toHaveBeenNthCalledWith(1, -43210, 'SIGTERM') + expect(processKill).toHaveBeenNthCalledWith(2, -43210, 'SIGKILL') + expect(child.kill).not.toHaveBeenCalled() + } finally { + processKill.mockRestore() + vi.useRealTimers() + } + }) + it('kills subprocess on AbortSignal', async () => { const ctl = new AbortController() const promise = runLocalHarness({ @@ -127,13 +759,27 @@ describe('runLocalHarness', () => { } const calls = spawnSpy.mock.calls expect(calls[0][0]).toBe('claude') - expect(calls[0][1]).toEqual(['--headless', '-p', 'go']) + expect(calls[0][1]).toEqual(['-p', 'go']) expect(calls[1][0]).toBe('codex') - expect(calls[1][1]).toEqual(['run', 'go']) + expect(calls[1][1]).toEqual(['exec', 'go']) expect(calls[2][0]).toBe('opencode') expect(calls[2][1]).toEqual(['run', 'go']) }) + it('adds Claude permission bypass only when the caller explicitly opts in', async () => { + const spawnSpy = vi.fn((_cmd: string, _args: ReadonlyArray) => + makeFakeChild({ exitCode: 0 }), + ) + await runLocalHarness({ + harness: 'claude', + cwd: '/tmp/isolated-worktree', + taskPrompt: 'go', + dangerouslySkipPermissions: true, + spawn: spawnSpy, + }) + expect(spawnSpy.mock.calls[0][1]).toEqual(['-p', 'go', '--dangerously-skip-permissions']) + }) + it('honors a pre-built invocation override (profile-aware args) without rebuilding from the prompt', async () => { const spawnSpy = vi.fn((_cmd: string, _args: ReadonlyArray) => makeFakeChild({ exitCode: 0 }), @@ -141,13 +787,13 @@ describe('runLocalHarness', () => { await runLocalHarness({ harness: 'claude', cwd: '/tmp/wt', - // The prompt-only fallback path would emit ['--headless','-p','go'] — the override wins. + // The prompt-only fallback path would emit ['-p','go'] — the override wins exactly. taskPrompt: 'go', - invocation: { command: 'claude', args: ['--headless', '-p', 'sys\n\ngo', '-m', 'deepseek'] }, + invocation: { command: 'claude', args: ['-p', 'sys\n\ngo', '-m', 'deepseek'] }, spawn: spawnSpy, }) expect(spawnSpy.mock.calls[0][0]).toBe('claude') - expect(spawnSpy.mock.calls[0][1]).toEqual(['--headless', '-p', 'sys\n\ngo', '-m', 'deepseek']) + expect(spawnSpy.mock.calls[0][1]).toEqual(['-p', 'sys\n\ngo', '-m', 'deepseek']) }) }) @@ -183,19 +829,171 @@ describe('harnessInvocation (the §1.5 profile-aware mapper)', () => { it('threads BOTH systemPrompt and model together', () => { const inv = harnessInvocation('claude', profileWith('SYS', 'kimi-k2.7'), 'task') expect(inv.command).toBe('claude') - expect(inv.args).toEqual(['--headless', '-p', 'SYS\n\ntask', '-m', 'kimi-k2.7']) + expect(inv.args).toEqual(['-p', 'SYS\n\ntask', '-m', 'kimi-k2.7']) }) - it('an empty/absent profile yields exactly the legacy prompt-only shape (byte-identical)', () => { - expect(harnessInvocation('claude', { name: 'x' }, 'go').args).toEqual([ - '--headless', - '-p', - 'go', + it('composes systemPrompt, every authored instruction, then the task in order', () => { + const inv = harnessInvocation( + 'codex', + { + name: 'authored', + prompt: { systemPrompt: 'SYS', instructions: ['FIRST', 'SECOND'] }, + }, + 'task', + ) + expect(inv.args).toEqual(['exec', 'SYS\n\nFIRST\n\nSECOND\n\ntask']) + }) + + it('maps the complete Codex profile onto the noninteractive exec command', () => { + const inv = harnessInvocation( + 'codex', + { + name: 'authored', + prompt: { systemPrompt: 'SYS' }, + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + }, + 'task', + ) + expect(inv.command).toBe('codex') + expect(inv.args).toEqual([ + 'exec', + 'SYS\n\ntask', + '-m', + 'gpt-5.4', + '-c', + 'model_reasoning_effort="xhigh"', + ]) + }) + + it('clamps the portable ultracode effort to Codex xhigh', () => { + const inv = harnessInvocation('codex', { model: { reasoningEffort: 'ultracode' } }, 'task') + expect(inv.args).toEqual(['exec', 'task', '-c', 'model_reasoning_effort="xhigh"']) + }) + + it('builds the exact reproducible Codex isolation argv', () => { + const inv = harnessInvocation( + 'codex', + { + name: 'authored', + prompt: { systemPrompt: 'SYS', instructions: ['INSTRUCTION'] }, + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + }, + 'task', + { codexReproducible: true }, + ) + expect(inv.args).toEqual([ + 'exec', + 'SYS\n\nINSTRUCTION\n\ntask', + '--ephemeral', + '--ignore-rules', + '--json', + '-c', + 'approval_policy="never"', + '-c', + 'web_search="disabled"', + '-c', + 'project_doc_max_bytes=0', + '-c', + 'skills.include_instructions=false', + '-c', + 'include_apps_instructions=false', + '--disable', + 'tool_suggest', + '-c', + 'features.multi_agent_v2.root_agent_usage_hint_text=""', + '-c', + 'features.multi_agent_v2.subagent_usage_hint_text=""', + '-c', + 'features.multi_agent_v2.multi_agent_mode_hint_text=""', + '--strict-config', + '-m', + 'gpt-5.4', + '-c', + 'model_reasoning_effort="xhigh"', ]) - expect(harnessInvocation('codex', { name: 'x' }, 'go').args).toEqual(['run', 'go']) + }) + + it('rejects a custom invocation that weakens reproducible Codex isolation', async () => { + const spawnSpy = vi.fn(() => makeFakeChild({ exitCode: 0 })) + await expect( + runLocalHarness({ + harness: 'codex', + cwd: '/tmp/wt', + taskPrompt: 'task', + invocation: { + command: 'codex', + args: [ + 'exec', + 'task', + '--ephemeral', + '--ignore-user-config', + '--ignore-rules', + '--json', + '-s', + 'workspace-write', + '-c', + 'web_search="cached"', + '-m', + 'gpt-5.4', + '-c', + 'model_reasoning_effort="xhigh"', + ], + }, + codexReproducible: true, + env: { CODEX_HOME: '/definitely/missing', OPENAI_API_KEY: 'test-only' }, + spawn: spawnSpy, + }), + ).rejects.toThrow(/does not match the required isolated argv/) + expect(spawnSpy).not.toHaveBeenCalled() + }) + + it('requires an explicit model and reasoning effort for reproducible Codex', () => { + expect(() => + harnessInvocation('codex', { name: 'missing-model' }, 'task', { + codexReproducible: true, + }), + ).toThrow(/requires profile\.model\.default/) + expect(() => + harnessInvocation('codex', { model: { default: 'gpt-5.4' } }, 'task', { + codexReproducible: true, + }), + ).toThrow(/requires profile\.model\.reasoningEffort/) + expect(() => + harnessInvocation( + 'claude', + { model: { default: 'claude-opus-4-1', reasoningEffort: 'high' } }, + 'task', + { codexReproducible: true }, + ), + ).toThrow(/requires the Codex harness/) + }) + + it('rejects an unknown Codex reasoning effort instead of emitting invalid config', () => { + expect(() => + harnessInvocation( + 'codex', + // @ts-expect-error testing runtime validation + { model: { reasoningEffort: 'unbounded' } }, + 'task', + ), + ).toThrow(/unsupported Codex reasoning effort unbounded/) + }) + + it('an empty/absent profile yields exactly the legacy prompt-only shape (byte-identical)', () => { + expect(harnessInvocation('claude', { name: 'x' }, 'go').args).toEqual(['-p', 'go']) + expect(harnessInvocation('codex', { name: 'x' }, 'go').args).toEqual(['exec', 'go']) expect(harnessInvocation('opencode', { name: 'x' }, 'go').args).toEqual(['run', 'go']) }) + it('adds Claude permission bypass only when an isolated worktree explicitly opts in', () => { + expect( + harnessInvocation('claude', { name: 'x' }, 'go', { + dangerouslySkipPermissions: true, + }).args, + ).toEqual(['-p', 'go', '--dangerously-skip-permissions']) + expect(harnessInvocation('claude', { name: 'x' }, 'go').args).toEqual(['-p', 'go']) + }) + it('throws on an unknown harness', () => { expect(() => // @ts-expect-error testing runtime validation @@ -204,6 +1002,17 @@ describe('harnessInvocation (the §1.5 profile-aware mapper)', () => { }) }) +describe('parseCodexTokenUsage', () => { + it('rejects malformed or internally inconsistent usage', () => { + expect(() => parseCodexTokenUsage('not json')).toThrow(/not valid JSON/) + expect(() => + parseCodexTokenUsage( + '{"type":"turn.completed","usage":{"input_tokens":2,"cached_input_tokens":3,"output_tokens":1,"reasoning_output_tokens":0}}', + ), + ).toThrow(/cached_input_tokens exceeds input_tokens/) + }) +}) + describe('runLocalHarness trace-context inheritance (in-process placement)', () => { it('spawned harness CLIs inherit TRACE_ID / PARENT_SPAN_ID from the MCP process env', async () => { const prevTrace = process.env.TRACE_ID diff --git a/tests/mcp/worktree-harness.test.ts b/tests/mcp/worktree-harness.test.ts new file mode 100644 index 00000000..df2337a6 --- /dev/null +++ b/tests/mcp/worktree-harness.test.ts @@ -0,0 +1,620 @@ +import { execFileSync, spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { describe, expect, it, vi } from 'vitest' +import type { RunLocalHarnessOptions } from '../../src/mcp/local-harness' +import { captureWorktreeDiff, type GitRunner } from '../../src/mcp/worktree' +import { runWorktreeHarness } from '../../src/mcp/worktree-harness' +import { createWorktreeCliExecutor } from '../../src/runtime/supervise/worktree-cli-executor' + +function git(repoRoot: string, args: string[]): string { + return execFileSync('git', args, { cwd: repoRoot, encoding: 'utf8' }).trim() +} + +function initializeRepository(files: Record): string { + const repoRoot = mkdtempSync(join(tmpdir(), 'agent-runtime-profile-worktree-')) + git(repoRoot, ['init', '-q', '--initial-branch=main']) + git(repoRoot, ['config', 'user.email', 'runtime-test@example.invalid']) + git(repoRoot, ['config', 'user.name', 'Runtime Test']) + for (const [path, content] of Object.entries(files)) { + const absolute = join(repoRoot, path) + mkdirSync(dirname(absolute), { recursive: true }) + writeFileSync(absolute, content) + } + git(repoRoot, ['add', '-A']) + git(repoRoot, ['commit', '-q', '-m', 'fixture']) + return repoRoot +} + +function successfulHarnessResult() { + return { + exitCode: 0, + stdout: 'done', + stderr: '', + killedBySignal: null, + durationMs: 1, + timedOut: false, + usage: { + inputTokens: 1, + cachedInputTokens: 0, + outputTokens: 1, + reasoningOutputTokens: 0, + }, + } as const +} + +function count(text: string, value: string): number { + return text.split(value).length - 1 +} + +describe('runWorktreeHarness profile materialization', () => { + it('fails instead of fabricating an empty patch when Git diff fails', async () => { + const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) + try { + await expect( + captureWorktreeDiff({ + worktree: { + path: repoRoot, + baseSha: git(repoRoot, ['rev-parse', 'HEAD']), + branch: 'unused', + }, + baseRef: 'missing-base-ref', + }), + ).rejects.toThrow(/git diff --cached missing-base-ref failed/u) + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it('delivers mounted inputs, captures the worker edit, and excludes tracked and untracked inputs', async () => { + const repoRoot = initializeRepository({ + 'src/value.ts': 'export const value = 1\n', + 'profile-tracked.txt': 'repository version\n', + }) + const runId = 'profile-real-path' + const newInputPath = '.agent-profile/[literal]*:context.txt' + const trackedInputPath = 'profile-tracked.txt' + const newInputMarker = 'PROFILE_NEW_INPUT_8f08e9f8' + const trackedInputMarker = 'PROFILE_TRACKED_INPUT_c521cd4d' + const systemMarker = 'SYSTEM_PROMPT_2ef237df' + const promptInstructionMarker = 'PROMPT_INSTRUCTION_6dcc8c4e' + const resourceInstructionMarker = 'RESOURCE_INSTRUCTION_75b98d68' + const taskMarker = 'TASK_PROMPT_d5ade1ac' + const profile: AgentProfile = { + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + prompt: { + systemPrompt: systemMarker, + instructions: [promptInstructionMarker], + }, + resources: { + files: [ + { + path: newInputPath, + resource: { kind: 'inline', name: 'new-context', content: newInputMarker }, + }, + { + path: trackedInputPath, + resource: { + kind: 'inline', + name: 'tracked-context', + content: trackedInputMarker, + }, + }, + ], + instructions: { + kind: 'inline', + name: 'resource-instructions', + content: resourceInstructionMarker, + }, + failOnError: true, + }, + } + + try { + const executor = createWorktreeCliExecutor({ + repoRoot, + profile, + harness: 'codex', + taskPrompt: taskMarker, + runId, + codexReproducible: true, + runHarness: async (options: RunLocalHarnessOptions) => { + expect(readFileSync(join(options.cwd, newInputPath), 'utf8')).toBe(newInputMarker) + expect(readFileSync(join(options.cwd, trackedInputPath), 'utf8')).toBe(trackedInputMarker) + const prompt = options.invocation?.args[1] ?? '' + expect(options.codexReproducible).toBe(true) + expect(options.invocation?.args).toContain('project_doc_max_bytes=0') + for (const marker of [ + systemMarker, + promptInstructionMarker, + resourceInstructionMarker, + taskMarker, + ]) { + expect(count(prompt, marker)).toBe(1) + } + + writeFileSync(join(options.cwd, 'src/value.ts'), 'export const value = 2\n') + writeFileSync(join(options.cwd, newInputPath), 'worker changed new profile input\n') + writeFileSync( + join(options.cwd, trackedInputPath), + 'worker changed tracked profile input\n', + ) + return successfulHarnessResult() + }, + }) + const execution = await executor.execute(undefined, new AbortController().signal) + + try { + expect(execution.out.patch).toContain('diff --git a/src/value.ts b/src/value.ts') + expect(execution.out.patch).toContain('+export const value = 2') + expect(execution.out.patch).not.toContain(newInputPath) + expect(execution.out.patch).not.toContain(trackedInputPath) + expect(execution.out.patch).not.toContain(newInputMarker) + expect(execution.out.patch).not.toContain(trackedInputMarker) + expect(execution.out.stats).toEqual({ filesChanged: 1, insertions: 1, deletions: 1 }) + expect(execution.out.profileMaterialization).toMatchObject({ + workspacePlanDigest: expect.stringMatching(/^sha256:[a-f0-9]{64}$/u), + writtenPaths: [newInputPath, trackedInputPath], + unsupported: [], + environmentNames: [], + flags: [], + resourceInstructions: { + delivery: 'invocation-prompt', + sha256: `sha256:${createHash('sha256') + .update(resourceInstructionMarker) + .digest('hex')}`, + byteLength: Buffer.byteLength(resourceInstructionMarker), + }, + }) + } finally { + await executor.teardown(0) + } + + expect(existsSync(join(repoRoot, '.agent-worktrees', runId))).toBe(false) + expect(git(repoRoot, ['branch', '--list', `delegate/${runId}`])).toBe('') + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it('materializes every Claude-supported structural axis and delivers instructions once', async () => { + const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) + const runId = 'supported-structural-axes' + const resourceInstructionMarker = 'DIRECT_RESOURCE_INSTRUCTION_343641ac' + const paths = { + file: 'profile-context.txt', + skill: '.claude/skills/reviewer/SKILL.md', + nativeAgent: '.claude/agents/native-reviewer.md', + command: '.claude/commands/audit.md', + subagent: '.claude/agents/helper.md', + mcp: '.mcp.json', + settings: '.claude/settings.json', + } as const + try { + const run = await runWorktreeHarness({ + repoRoot, + profile: { + harness: 'codex', + model: { + default: 'claude-model', + small: 'small-routing-hint', + provider: 'anthropic', + metadata: { tier: 'research' }, + }, + prompt: { systemPrompt: 'DIRECT_SYSTEM_a3563f03' }, + resources: { + files: [ + { + path: paths.file, + resource: { kind: 'inline', name: 'context', content: 'FILE_MARKER_5570e069' }, + }, + ], + skills: [ + { + kind: 'inline', + name: 'reviewer', + content: '# Reviewer\n\nSKILL_MARKER_edc681e1', + }, + ], + agents: [ + { + kind: 'inline', + name: 'native-reviewer.md', + content: 'NATIVE_AGENT_MARKER_b1b342a1', + }, + ], + commands: [ + { + kind: 'inline', + name: 'audit', + content: 'COMMAND_MARKER_b53220fa', + }, + ], + instructions: resourceInstructionMarker, + }, + mcp: { local: { command: 'node', args: ['server.mjs'] } }, + hooks: { PreToolUse: [{ command: 'node hook.mjs', matcher: 'Bash' }] }, + subagents: { helper: { description: 'Helper', prompt: 'SUBAGENT_MARKER_37bb713b' } }, + }, + harness: 'claude', + taskPrompt: 'DIRECT_TASK_72c5c757', + runId, + runHarness: async (options) => { + expect(options.harness).toBe('claude') + expect(options.invocation?.command).toBe('claude') + expect(options.invocation?.args).toContain('claude-model') + expect(options.invocation?.args).not.toContain('small-routing-hint') + expect(options.invocation?.args).not.toContain('anthropic') + expect(options.invocation?.args).not.toContain('research') + expect(readFileSync(join(options.cwd, paths.file), 'utf8')).toContain( + 'FILE_MARKER_5570e069', + ) + expect(readFileSync(join(options.cwd, paths.skill), 'utf8')).toContain( + 'SKILL_MARKER_edc681e1', + ) + expect(readFileSync(join(options.cwd, paths.nativeAgent), 'utf8')).toContain( + 'NATIVE_AGENT_MARKER_b1b342a1', + ) + expect(readFileSync(join(options.cwd, paths.command), 'utf8')).toContain( + 'COMMAND_MARKER_b53220fa', + ) + expect(readFileSync(join(options.cwd, paths.subagent), 'utf8')).toContain( + 'SUBAGENT_MARKER_37bb713b', + ) + expect(readFileSync(join(options.cwd, paths.mcp), 'utf8')).toContain('server.mjs') + const settings = JSON.parse( + readFileSync(join(options.cwd, paths.settings), 'utf8'), + ) as Record + expect(settings).toHaveProperty('enabledMcpjsonServers') + expect(settings).toHaveProperty('hooks') + const prompt = options.invocation?.args[1] ?? '' + expect(count(prompt, resourceInstructionMarker)).toBe(1) + expect(count(prompt, 'DIRECT_SYSTEM_a3563f03')).toBe(1) + expect(count(prompt, 'DIRECT_TASK_72c5c757')).toBe(1) + return successfulHarnessResult() + }, + }) + + try { + expect(run.result.patch).toBe('') + expect(run.result.stats).toEqual({ filesChanged: 0, insertions: 0, deletions: 0 }) + expect(run.result.profileMaterialization?.writtenPaths).toEqual( + Object.values(paths).sort((left, right) => left.localeCompare(right)), + ) + expect(run.result.profileMaterialization?.unsupported).toEqual([]) + } finally { + await run.cleanup() + await run.cleanup() + } + expect(existsSync(join(repoRoot, '.agent-worktrees', runId))).toBe(false) + expect(git(repoRoot, ['branch', '--list', `delegate/${runId}`])).toBe('') + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it('fails before worker launch and removes the real worktree for unsupported resources', async () => { + const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) + const runId = 'unsupported-profile' + const runHarness = vi.fn() + try { + await expect( + runWorktreeHarness({ + repoRoot, + profile: { + resources: { + failOnError: false, + files: [ + { + path: '.agent-profile/remote.txt', + resource: { kind: 'github', repository: 'owner/repo', path: 'remote.txt' }, + }, + ], + }, + }, + harness: 'codex', + taskPrompt: 'task', + runId, + runHarness, + }), + ).rejects.toThrow(/profile cannot be materialized.*files/u) + expect(runHarness).not.toHaveBeenCalled() + expect(existsSync(join(repoRoot, '.agent-worktrees', runId))).toBe(false) + expect(git(repoRoot, ['branch', '--list', `delegate/${runId}`])).toBe('') + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it('rejects unresolved resource instructions and removes the real worktree', async () => { + const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) + const runId = 'unresolved-resource-instructions' + const runHarness = vi.fn() + try { + await expect( + runWorktreeHarness({ + repoRoot, + profile: { + resources: { + instructions: { + kind: 'github', + repository: 'owner/repo', + path: 'INSTRUCTIONS.md', + }, + }, + }, + harness: 'codex', + taskPrompt: 'task', + runId, + runHarness, + }), + ).rejects.toThrow(/resources\.instructions.*pre-resolution/u) + expect(runHarness).not.toHaveBeenCalled() + expect(existsSync(join(repoRoot, '.agent-worktrees', runId))).toBe(false) + expect(git(repoRoot, ['branch', '--list', `delegate/${runId}`])).toBe('') + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it('rejects unsupported behavior axes before worker launch and removes the worktree', async () => { + const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) + const runId = 'unsupported-behavior-axes' + const runHarness = vi.fn() + try { + await expect( + runWorktreeHarness({ + repoRoot, + profile: { + tools: { shell: false }, + permissions: { shell: 'deny' }, + connections: [{ connectionId: 'connection-1', capabilities: ['read'] }], + confidential: { tee: 'any' }, + modes: { review: { prompt: 'Review only.' } }, + extensions: { codex: { feature: true } }, + }, + harness: 'codex', + taskPrompt: 'task', + runId, + runHarness, + }), + ).rejects.toThrow( + /unsupported worktree behavior: tools, permissions, connections, confidential, modes, extensions/u, + ) + expect(runHarness).not.toHaveBeenCalled() + expect(existsSync(join(repoRoot, '.agent-worktrees', runId))).toBe(false) + expect(git(repoRoot, ['branch', '--list', `delegate/${runId}`])).toBe('') + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it('preserves the run error when Git cleanup also fails', async () => { + const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) + const runId = 'run-and-cleanup-failures' + const worktreePath = join(repoRoot, '.agent-worktrees', runId) + const runHarness = vi.fn() + let interceptedRemoval = false + const runGit: GitRunner = (args, { cwd }) => { + if (args[0] === 'worktree' && args[1] === 'remove' && !interceptedRemoval) { + interceptedRemoval = true + return { stdout: '', stderr: 'simulated cleanup failure', exitCode: 128 } + } + const result = spawnSync('git', [...args], { cwd, encoding: 'utf8' }) + return { + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.status ?? -1, + } + } + + try { + let error: unknown + try { + await runWorktreeHarness({ + repoRoot, + profile: { tools: { shell: false } }, + harness: 'codex', + taskPrompt: 'task', + runId, + runHarness, + runGit, + }) + } catch (caught) { + error = caught + } + + expect(error).toBeInstanceOf(AggregateError) + const errors = (error as AggregateError).errors as Error[] + expect(errors[0]?.message).toContain('unsupported worktree behavior: tools') + expect(errors[1]).toBeInstanceOf(AggregateError) + const cleanupErrors = (errors[1] as AggregateError).errors as Error[] + expect(cleanupErrors[0]?.message).toContain('worktree remove') + expect(cleanupErrors[1]?.message).toContain('branch -D') + expect(runHarness).not.toHaveBeenCalled() + expect(interceptedRemoval).toBe(true) + expect(existsSync(worktreePath)).toBe(true) + } finally { + spawnSync('git', ['worktree', 'remove', '--force', worktreePath], { cwd: repoRoot }) + spawnSync('git', ['branch', '-D', `delegate/${runId}`], { cwd: repoRoot }) + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it('rejects nested controls the pinned materializer would silently drop', async () => { + const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) + const runHarness = vi.fn() + const cases: Array<{ + runId: string + harness: 'claude' | 'codex' | 'opencode' + profile: AgentProfile + dropped: string[] + }> = [ + { + runId: 'codex-nested-controls', + harness: 'codex', + profile: { + mcp: { + disabled: { + command: 'node', + enabled: false, + headers: { Authorization: 'redacted' }, + }, + }, + subagents: { + helper: { + prompt: 'Help.', + tools: { shell: false }, + permissions: { shell: 'deny' }, + maxSteps: 1, + }, + }, + }, + dropped: [ + 'mcp["disabled"].enabled', + 'mcp["disabled"].headers', + 'subagents["helper"].permissions', + 'subagents["helper"].maxSteps', + 'subagents["helper"].tools', + ], + }, + { + runId: 'claude-nested-controls', + harness: 'claude', + profile: { + model: { reasoningEffort: 'high' }, + hooks: { + PreToolUse: [{ command: 'node hook.mjs', env: { MODE: 'strict' }, blocking: false }], + }, + }, + dropped: [ + 'model.reasoningEffort', + 'hooks["PreToolUse"][0].env', + 'hooks["PreToolUse"][0].blocking', + ], + }, + { + runId: 'opencode-nested-controls', + harness: 'opencode', + profile: { mcp: { local: { command: 'node', cwd: 'required-directory' } } }, + dropped: ['mcp["local"].cwd'], + }, + ] + + try { + for (const testCase of cases) { + let error: unknown + try { + await runWorktreeHarness({ + repoRoot, + profile: testCase.profile, + harness: testCase.harness, + taskPrompt: 'task', + runId: testCase.runId, + runHarness, + }) + } catch (caught) { + error = caught + } + expect(error).toBeInstanceOf(Error) + for (const path of testCase.dropped) { + expect((error as Error).message).toContain(path) + } + expect(existsSync(join(repoRoot, '.agent-worktrees', testCase.runId))).toBe(false) + expect(git(repoRoot, ['branch', '--list', `delegate/${testCase.runId}`])).toBe('') + } + expect(runHarness).not.toHaveBeenCalled() + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it('rejects profile files targeting Git metadata before worker launch', async () => { + const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) + const runHarness = vi.fn() + try { + for (const [index, path] of [ + '.git', + 'nested/.git/config', + '.GIT.', + '.git:stream', + ].entries()) { + const runId = `reserved-git-metadata-${index}` + await expect( + runWorktreeHarness({ + repoRoot, + profile: { + resources: { + files: [ + { + path, + resource: { + kind: 'inline', + name: 'poison', + content: 'gitdir: attacker\n', + }, + }, + ], + }, + }, + harness: 'codex', + taskPrompt: 'task', + runId, + runHarness, + }), + ).rejects.toThrow(/profile file cannot target reserved Git metadata/u) + expect(existsSync(join(repoRoot, '.agent-worktrees', runId))).toBe(false) + expect(git(repoRoot, ['branch', '--list', `delegate/${runId}`])).toBe('') + } + expect(runHarness).not.toHaveBeenCalled() + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + + it('removes the real worktree when applying a profile file fails', async () => { + const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) + symlinkSync('src', join(repoRoot, 'profile-link')) + git(repoRoot, ['add', 'profile-link']) + git(repoRoot, ['commit', '-q', '-m', 'add profile symlink']) + const runId = 'profile-apply-failure' + const runHarness = vi.fn() + try { + await expect( + runWorktreeHarness({ + repoRoot, + profile: { + resources: { + files: [ + { + path: 'profile-link/input.txt', + resource: { kind: 'inline', name: 'input', content: 'marker' }, + }, + ], + }, + }, + harness: 'codex', + taskPrompt: 'task', + runId, + runHarness, + }), + ).rejects.toThrow(/symlink path/u) + expect(runHarness).not.toHaveBeenCalled() + expect(existsSync(join(repoRoot, '.agent-worktrees', runId))).toBe(false) + expect(git(repoRoot, ['branch', '--list', `delegate/${runId}`])).toBe('') + } finally { + rmSync(repoRoot, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/otel-export.test.ts b/tests/otel-export.test.ts index b5949953..fa502d8c 100644 --- a/tests/otel-export.test.ts +++ b/tests/otel-export.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { buildLoopOtelSpans, + buildRuntimeEventOtelSpans, createOtelExporter, exportEvalRuns, INTELLIGENCE_WIRE_VERSION, @@ -21,6 +22,80 @@ function attrMap(span: OtelSpan): Record { + it('preserves opted-in MCP tool payloads and LLM usage without truncation', () => { + const longQuery = 'q'.repeat(5000) + const spans = buildRuntimeEventOtelSpans( + [ + { + type: 'tool_call', + toolName: 'mcp__linear__linear_graphql', + toolCallId: 'call-1', + args: { query: longQuery }, + timestamp: '2026-07-10T00:00:00.000Z', + }, + { + type: 'llm_call', + model: 'anthropic/claude-sonnet-4.6', + tokensIn: 12, + tokensOut: 7, + costUsd: 0.004, + latencyMs: 350, + timestamp: '2026-07-10T00:00:01.000Z', + }, + ], + 'a'.repeat(32), + 'b'.repeat(16), + { includeControlPayloads: true }, + ) + + const tool = attrMap(spans[0]!) + expect(tool['tool.name']).toBe('mcp__linear__linear_graphql') + expect(tool['mcp.server']).toBe('linear') + expect(tool['mcp.tool.name']).toBe('linear_graphql') + expect(String(tool['tool.input'])).toContain(longQuery) + expect(String(tool['tangle.runtime.event'])).not.toContain('[truncated]') + + const llm = attrMap(spans[1]!) + expect(llm['gen_ai.request.model']).toBe('anthropic/claude-sonnet-4.6') + expect(llm['gen_ai.usage.input_tokens']).toBe(12) + expect(llm['gen_ai.usage.output_tokens']).toBe(7) + expect(llm['tangle.cost.usd']).toBe(0.004) + expect(BigInt(spans[1]!.endTimeUnixNano) - BigInt(spans[1]!.startTimeUnixNano)).toBe( + 350_000_000n, + ) + }) + + it('omits control payloads by default while retaining tool identity', () => { + const [span] = buildRuntimeEventOtelSpans( + [{ type: 'tool_call', toolName: 'search', args: { secret: 'value' } }], + 'a'.repeat(32), + ) + const attrs = attrMap(span!) + expect(attrs['tool.name']).toBe('search') + expect(attrs['tool.input']).toBeUndefined() + expect(String(attrs['tangle.runtime.event'])).not.toContain('value') + }) + + it('omits non-finite numeric attributes instead of emitting invalid OTLP JSON', () => { + const [span] = buildRuntimeEventOtelSpans( + [ + { + type: 'llm_call', + model: 'test', + costUsd: Number.NaN, + latencyMs: Number.POSITIVE_INFINITY, + }, + ], + 'a'.repeat(32), + ) + const attrs = attrMap(span!) + expect(attrs['tangle.cost.usd']).toBeUndefined() + expect(attrs['tangle.latency_ms']).toBeUndefined() + expect(JSON.stringify(span)).not.toContain('NaN') + }) +}) + describe('buildLoopOtelSpans — nested GenAI topology tree', () => { // One dynamic-loop run: round 0 fans out 2 branches (with rationale), then stops. const events = [ @@ -492,7 +567,7 @@ describe('otel-export', () => { describe('exportEvalRuns (Intelligence self-improvement provenance)', () => { afterEach(() => { delete process.env.TANGLE_API_KEY - delete process.env.INTELLIGENCE_BASE + delete process.env.TANGLE_INTELLIGENCE_URL vi.unstubAllGlobals() }) diff --git a/tests/profile-improvement-stack.test.ts b/tests/profile-improvement-stack.test.ts index e8af0150..7117df9c 100644 --- a/tests/profile-improvement-stack.test.ts +++ b/tests/profile-improvement-stack.test.ts @@ -70,9 +70,21 @@ async function promptAgent( _scenario: ProfileScenario, ctx: DispatchContext, ): Promise<{ prompt: string }> { - ctx.cost.observe(0.0001, 'profile-stack-test') - ctx.cost.observeTokens({ input: 1, output: 1 }) - return { prompt: String(surface) } + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'profile-stack-test', + model: 'deterministic-test', + maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, + execute: async () => ({ prompt: String(surface) }), + receipt: () => ({ + model: 'deterministic-test', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.0001, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value } interface LoopTask { diff --git a/tests/runtime/worktree-cli-executor.test.ts b/tests/runtime/worktree-cli-executor.test.ts index 481bee77..18185e61 100644 --- a/tests/runtime/worktree-cli-executor.test.ts +++ b/tests/runtime/worktree-cli-executor.test.ts @@ -1,4 +1,5 @@ import type { ChildProcess } from 'node:child_process' +import { createHash } from 'node:crypto' import { EventEmitter } from 'node:events' import { PassThrough, type Readable } from 'node:stream' import type { AgentProfile } from '@tangle-network/sandbox' @@ -93,6 +94,15 @@ const authoredProfile: AgentProfile = { model: { default: 'deepseek/deepseek-v4-flash' }, } +const reproducibleCodexProfile: AgentProfile = { + name: 'reproducible-codex', + prompt: { + systemPrompt: 'You are a careful refactorer.', + instructions: ['Never search for a public solution.'], + }, + model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, +} + function bridgeSseResponse(content: string): Readable { const payload = [ `data: ${JSON.stringify({ @@ -171,6 +181,7 @@ describe('createWorktreeCliExecutor', () => { ) // ... and the authored model reaches the harness `-m` selector. const args = seen?.invocation?.args ?? [] + expect(args).toContain('--dangerously-skip-permissions') const mIdx = args.indexOf('-m') expect(mIdx).toBeGreaterThanOrEqual(0) expect(args[mIdx + 1]).toBe('deepseek/deepseek-v4-flash') @@ -248,6 +259,193 @@ describe('createWorktreeCliExecutor', () => { expect(exec.budgetExempt).toBe(true) }) + it('rejects caller read-denial paths outside reproducible Codex mode', () => { + expect(() => + createWorktreeCliExecutor({ + repoRoot: '/workspace', + profile: authoredProfile, + harness: 'codex', + taskPrompt: 'x', + codexReadDeniedPaths: ['/usr/lib/example/gold.py'], + runGit: makeFakeGit(freshGitState()), + runHarness: vi.fn(), + }), + ).toThrow(/requires codexReproducible/) + }) + + it('meters reproducible Codex usage and surfaces isolation evidence', async () => { + const state = freshGitState() + let seen: RunLocalHarnessOptions | undefined + const composedPrompt = + 'You are a careful refactorer.\n\nNever search for a public solution.\n\nfix the bug' + const requestedPromptSha256 = createHash('sha256').update(composedPrompt).digest('hex') + const exec = createWorktreeCliExecutor({ + repoRoot: '/workspace', + profile: reproducibleCodexProfile, + harness: 'codex', + taskPrompt: 'fix the bug', + codexReproducible: true, + codexReadDeniedPaths: ['/usr/lib/example/gold.py'], + runGit: makeFakeGit(state), + runHarness: vi.fn(async (options) => { + seen = options + return { + exitCode: 0, + stdout: '{"type":"turn.completed"}', + stderr: '', + killedBySignal: null, + durationMs: 12, + timedOut: false, + usage: { + inputTokens: 41935, + cachedInputTokens: 19200, + outputTokens: 273, + reasoningOutputTokens: 191, + }, + evidence: { + cliVersion: 'codex-cli 0.144.1', + executableSha256: 'd'.repeat(64), + requestedPromptSha256, + effectivePromptSha256: 'a'.repeat(64), + nonPromptArgsSha256: 'b'.repeat(64), + controlledConfigSha256: 'c'.repeat(64), + readDeniedPaths: ['/usr/lib/example/gold.py'], + readDeniedPathsSha256: 'e'.repeat(64), + readDeniedPathCount: 1, + policy: { + sessionPersistence: 'ephemeral', + userConfig: false, + rules: false, + projectInstructions: false, + skillInstructions: false, + appInstructions: false, + toolSuggestions: false, + multiAgentInstructions: false, + sandbox: 'workspace-write', + permissionProfile: 'agent_runtime_reproducible', + approvalPolicy: 'never', + shellNetwork: false, + webSearch: false, + serviceTier: 'default', + shellEnvironment: 'core-filtered', + loginShell: false, + credentialsReadable: false, + hostHomeReadable: false, + procEnvironment: 'private-sanitized', + sensitiveEnvironmentNamesVisible: false, + parentRepoRead: false, + gitMetadata: false, + temporaryDirectory: 'workspace-private', + stagedExecutable: 'static-elf-read-only', + callerReadDeniedPaths: 'enforced', + containerSockets: false, + }, + }, + } + }), + }) + + expect(exec.budgetExempt).toBe(false) + const result = await exec.execute(undefined, new AbortController().signal) + expect(seen?.codexReproducible).toBe(true) + expect(seen?.codexReadDeniedPaths).toEqual(['/usr/lib/example/gold.py']) + expect(seen?.invocation?.args).toContain('--ephemeral') + expect(seen?.invocation?.args).toContain('--ignore-rules') + expect(seen?.invocation?.args).toContain('web_search="disabled"') + expect(seen?.invocation?.args[1]).toBe(composedPrompt) + expect(result.spent).toMatchObject({ + iterations: 1, + tokens: { input: 41935, output: 273 }, + usd: 0, + usdKnown: false, + }) + expect(result.out.harness).toMatchObject({ + cliVersion: 'codex-cli 0.144.1', + executableSha256: 'd'.repeat(64), + requestedPromptSha256, + effectivePromptSha256: 'a'.repeat(64), + nonPromptArgsSha256: 'b'.repeat(64), + controlledConfigSha256: 'c'.repeat(64), + readDeniedPaths: ['/usr/lib/example/gold.py'], + readDeniedPathsSha256: 'e'.repeat(64), + readDeniedPathCount: 1, + executionPolicy: { + sessionPersistence: 'ephemeral', + userConfig: false, + rules: false, + projectInstructions: false, + skillInstructions: false, + appInstructions: false, + toolSuggestions: false, + multiAgentInstructions: false, + sandbox: 'workspace-write', + permissionProfile: 'agent_runtime_reproducible', + approvalPolicy: 'never', + shellNetwork: false, + webSearch: false, + serviceTier: 'default', + shellEnvironment: 'core-filtered', + loginShell: false, + credentialsReadable: false, + hostHomeReadable: false, + procEnvironment: 'private-sanitized', + sensitiveEnvironmentNamesVisible: false, + parentRepoRead: false, + gitMetadata: false, + temporaryDirectory: 'workspace-private', + stagedExecutable: 'static-elf-read-only', + callerReadDeniedPaths: 'enforced', + containerSockets: false, + }, + }) + }) + + it('rejects contradictory reproducible Codex configuration', () => { + expect(() => + createWorktreeCliExecutor({ + repoRoot: '/workspace', + profile: reproducibleCodexProfile, + harness: 'claude', + taskPrompt: 'x', + codexReproducible: true, + }), + ).toThrow(/requires harness "codex"/) + expect(() => + createWorktreeCliExecutor({ + repoRoot: '/workspace', + profile: reproducibleCodexProfile, + harness: 'codex', + taskPrompt: 'x', + codexReproducible: true, + budgetExempt: true, + }), + ).toThrow(/cannot be budgetExempt/) + }) + + it('fails and removes the worktree when a metered run returns no usage', async () => { + const state = freshGitState() + const exec = createWorktreeCliExecutor({ + repoRoot: '/workspace', + profile: reproducibleCodexProfile, + harness: 'codex', + taskPrompt: 'x', + codexReproducible: true, + runGit: makeFakeGit(state), + runHarness: vi.fn(async () => ({ + exitCode: 0, + stdout: '', + stderr: '', + killedBySignal: null, + durationMs: 1, + timedOut: false, + })), + }) + await expect(exec.execute(undefined, new AbortController().signal)).rejects.toThrow( + /returned no token usage/, + ) + expect(state.worktreesRemoved).toEqual(state.worktreesCreated) + }) + it('budgetExempt: false opts the leaf into metering (explicit, not a buried hardcode)', () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', @@ -261,6 +459,22 @@ describe('createWorktreeCliExecutor', () => { expect(exec.budgetExempt).toBe(false) }) + it('threads reproducible Codex through the backend-as-data factory', () => { + const factory = createExecutor({ + backend: 'cli-worktree', + repoRoot: '/workspace', + harness: 'codex', + taskPrompt: 'x', + codexReproducible: true, + }) + const exec = factory( + { profile: reproducibleCodexProfile, harness: null }, + { signal: new AbortController().signal, seams: {} }, + ) + expect(exec.runtime).toBe('cli') + expect(exec.budgetExempt).toBe(false) + }) + it('resultArtifact() before execute() resolves throws (fail loud, no fabricated artifact)', () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', diff --git a/tsup.config.ts b/tsup.config.ts index 5d4f4f49..19747b40 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -12,6 +12,8 @@ export default defineConfig({ knowledge: 'src/knowledge/index.ts', profiles: 'src/profiles/index.ts', platform: 'src/platform/index.ts', + 'primeintellect/index': 'src/primeintellect/index.ts', + 'candidate-execution/index': 'src/candidate-execution/index.ts', 'mcp/index': 'src/mcp/index.ts', 'mcp/bin': 'src/mcp/bin.ts', 'loop-runner-bin': 'src/loop-runner-bin.ts', diff --git a/typedoc.json b/typedoc.json index 31f675d5..b4319cf1 100644 --- a/typedoc.json +++ b/typedoc.json @@ -11,6 +11,8 @@ "src/knowledge/index.ts", "src/profiles/index.ts", "src/platform/index.ts", + "src/primeintellect/index.ts", + "src/candidate-execution/index.ts", "src/mcp/index.ts" ], "entryPointStrategy": "resolve", From 3c2ed5d53f7bd3bd1ad7184c9dabc03ab4c54951 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 15 Jul 2026 14:08:46 -0600 Subject: [PATCH 3/5] fix(examples): satisfy strict index checks in the ablation-suite rigs typecheck:examples runs with noUncheckedIndexedAccess; the aisdk and multi-contract rigs indexed per-task vectors and task specs without narrowing, so pnpm typecheck failed at head. --- examples/ablation-suite/aisdk-env.ts | 5 ++++- examples/ablation-suite/multi-contract-env.ts | 10 +--------- examples/ablation-suite/run-aisdk.ts | 8 ++++---- examples/ablation-suite/run-multi-contract.ts | 14 +++++++------- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/examples/ablation-suite/aisdk-env.ts b/examples/ablation-suite/aisdk-env.ts index 3c98a5e9..2aeab5d0 100644 --- a/examples/ablation-suite/aisdk-env.ts +++ b/examples/ablation-suite/aisdk-env.ts @@ -88,7 +88,9 @@ const SDK_TASKS: SdkTask[] = [ function taskSpec(id: string): SdkTask { const key = id.split('#')[0] - return SDK_TASKS.find((t) => t.key === key) ?? SDK_TASKS[0] + const spec = SDK_TASKS.find((t) => t.key === key) ?? SDK_TASKS[0] + if (!spec) throw new Error(`aisdk-env: no task spec for '${id}'`) + return spec } let searchCallTotal = 0 @@ -269,6 +271,7 @@ export function aiSdkTasks(n: number): Promise { return Promise.resolve( Array.from({ length: n }, (_, i) => { const spec = SDK_TASKS[i % SDK_TASKS.length] + if (!spec) throw new Error('aisdk-env: SDK_TASKS is empty') return { id: `${spec.key}#${i}`, systemPrompt: spec.systemPrompt, diff --git a/examples/ablation-suite/multi-contract-env.ts b/examples/ablation-suite/multi-contract-env.ts index 29c9813f..a13a97a4 100644 --- a/examples/ablation-suite/multi-contract-env.ts +++ b/examples/ablation-suite/multi-contract-env.ts @@ -20,15 +20,7 @@ */ import { execFileSync } from 'node:child_process' -import { - existsSync, - mkdirSync, - mkdtempSync, - readdirSync, - readFileSync, - rmSync, - writeFileSync, -} from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' diff --git a/examples/ablation-suite/run-aisdk.ts b/examples/ablation-suite/run-aisdk.ts index 77e76f4a..db07c022 100644 --- a/examples/ablation-suite/run-aisdk.ts +++ b/examples/ablation-suite/run-aisdk.ts @@ -86,12 +86,12 @@ async function runArm( ) } const vIdx = valid.map((v, i) => (v ? i : -1)).filter((i) => i >= 0) - const resolve = vIdx.length ? vIdx.reduce((a, i) => a + perTask[i], 0) / vIdx.length : 0 + const resolve = vIdx.length ? vIdx.reduce((a, i) => a + (perTask[i] ?? 0), 0) / vIdx.length : 0 return { resolve, perTask, valid, fired, usd } } function pairedDeltaCI(a: number[], b: number[]): { delta: number; lo: number; hi: number } { - const d = a.map((x, i) => x - b[i]) + const d = a.map((x, i) => x - (b[i] ?? 0)) const mean = d.reduce((s, x) => s + x, 0) / d.length const v = d.reduce((s, x) => s + (x - mean) ** 2, 0) / Math.max(1, d.length - 1) const se = Math.sqrt(v / d.length) @@ -127,8 +127,8 @@ console.error( ) const both = tasks.map((_, i) => i).filter((i) => noSearch.valid[i] && search.valid[i]) -const nsT = both.map((i) => noSearch.perTask[i]) -const seT = both.map((i) => search.perTask[i]) +const nsT = both.map((i) => noSearch.perTask[i] ?? 0) +const seT = both.map((i) => search.perTask[i] ?? 0) const ci = pairedDeltaCI(seT, nsT) console.error(`\n=== VERDICT (pre-registered) ===`) console.error(`paired tasks valid in both arms: ${both.length}/${N}`) diff --git a/examples/ablation-suite/run-multi-contract.ts b/examples/ablation-suite/run-multi-contract.ts index 5454974f..cf636a8c 100644 --- a/examples/ablation-suite/run-multi-contract.ts +++ b/examples/ablation-suite/run-multi-contract.ts @@ -103,15 +103,15 @@ async function runArm( ) } const vIdx = valid.map((v, i) => (v ? i : -1)).filter((i) => i >= 0) - const vResolve = vIdx.length ? vIdx.reduce((a, i) => a + perTask[i], 0) / vIdx.length : 0 - const vScore = vIdx.length ? vIdx.reduce((a, i) => a + perScore[i], 0) / vIdx.length : 0 + const vResolve = vIdx.length ? vIdx.reduce((a, i) => a + (perTask[i] ?? 0), 0) / vIdx.length : 0 + const vScore = vIdx.length ? vIdx.reduce((a, i) => a + (perScore[i] ?? 0), 0) / vIdx.length : 0 return { resolve: vResolve, scoreMean: vScore, perTask, perScore, valid, fired, usd } } // paired bootstrap CI of (search − nosearch) on the aligned per-task vectors — no random seed // available in this env, so use a deterministic jackknife-style spread as a conservative CI proxy. function pairedDeltaCI(a: number[], b: number[]): { delta: number; lo: number; hi: number } { - const d = a.map((x, i) => x - b[i]) + const d = a.map((x, i) => x - (b[i] ?? 0)) const mean = d.reduce((s, x) => s + x, 0) / d.length const v = d.reduce((s, x) => s + (x - mean) ** 2, 0) / Math.max(1, d.length - 1) const se = Math.sqrt(v / d.length) @@ -153,10 +153,10 @@ console.error( // paired on tasks VALID in BOTH arms only (drop empty-run flukes so the pairing is honest) const both = tasks.map((_, i) => i).filter((i) => noSearch.valid[i] && search.valid[i]) -const nsT = both.map((i) => noSearch.perTask[i]) -const seT = both.map((i) => search.perTask[i]) -const nsS = both.map((i) => noSearch.perScore[i]) -const seS = both.map((i) => search.perScore[i]) +const nsT = both.map((i) => noSearch.perTask[i] ?? 0) +const seT = both.map((i) => search.perTask[i] ?? 0) +const nsS = both.map((i) => noSearch.perScore[i] ?? 0) +const seS = both.map((i) => search.perScore[i] ?? 0) const ci = pairedDeltaCI(seT, nsT) const sci = pairedDeltaCI(seS, nsS) console.error(`\n=== VERDICT (pre-registered) ===`) From 72bbfec3b8acde5002081b924718ca0af6972032 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 15 Jul 2026 14:08:56 -0600 Subject: [PATCH 4/5] docs(improvement): name the distiller evidence caps and pin them with a test - hoist the 1500/500 slice bounds to DISTILLED_NOTES_MAX_CHARS / DISTILLED_ERROR_MAX_CHARS so the caps and the doc comments that cite them cannot drift apart - pin the contract in improve.test.ts: a traceback-sized note survives intact, a runaway note clips at exactly 1500, a cell error clips at 500 - regenerate docs/api so docs:check passes at head (the committed docs still described the ~400-char caps) --- docs/api/index.md | 6 ++-- src/improvement/improve.test.ts | 57 +++++++++++++++++++++++++++++++++ src/improvement/improve.ts | 21 ++++++++---- 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index 1b1713b5..dd62d123 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -11290,7 +11290,7 @@ Per-generation findings producer passthrough (see selfImprove.analyzeGeneration) > `optional` **rawTraceContext?**: `boolean` -META-HARNESS mode: instead of the ~400-char distilled findings, feed the +META-HARNESS mode: instead of the ~1500-char distilled findings, feed the proposer RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's real run traces under `runDir` (per-cell `spans.jsonl` event logs + `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose @@ -13295,7 +13295,7 @@ Build the starting instruction for a coder agent tasked with implementing a new > **applyImprovementWinnerToProfile**(`profile`, `surface`, `winner`): `AgentProfile` -Defined in: [improvement/improve.ts:466](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L466) +Defined in: [improvement/improve.ts:478](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L478) Apply a promoted winner surface back into the profile field for `surface`. Returns a shallow copy; never mutates the input profile. @@ -13324,7 +13324,7 @@ Apply a promoted winner surface back into the profile field for `surface`. > **improve**\<`TScenario`, `TArtifact`\>(`profile`, `findings`, `opts`): `Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> -Defined in: [improvement/improve.ts:529](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L529) +Defined in: [improvement/improve.ts:541](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L541) Run the held-out-gated self-improvement loop on ONE profile surface. diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 7f983170..1f074138 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -585,6 +585,63 @@ describe('improve() — default proposer resolution (substrate export drift guar } }) + it('the distiller keeps traceback-sized notes intact and clips at the 1500/500 caps', async () => { + // A realistic executable-judge note (~1000 chars) must survive whole; a + // runaway note is clipped to exactly 1500; a cell error is clipped to 500. + const intactNote = `Traceback (most recent call last):\n${' assert tour == expected\n'.repeat(38)}` + expect(intactNote.length).toBeGreaterThan(900) + expect(intactNote.length).toBeLessThan(1500) + const runawayNote = 'n'.repeat(1600) + const longError = 'e'.repeat(800) + const cappingJudge: JudgeConfig<{ text: string }, Scenario> = { + name: 'capping-judge', + dimensions: [{ key: 'q', description: 'fixture quality' }], + score: ({ scenario }) => { + if (scenario.id === 'a') return { dimensions: { q: 0 }, composite: 0, notes: intactNote } + if (scenario.id === 'b') return { dimensions: { q: 0 }, composite: 0, notes: runawayNote } + return { dimensions: { q: 1 }, composite: 1, notes: 'ok' } + }, + } + const findingsSeen: unknown[][] = [] + const stubProposer = { + kind: 'stub-recorder', + async propose(ctx: { findings: unknown[]; populationSize: number }) { + findingsSeen.push(ctx.findings) + return [{ surface: `candidate-${findingsSeen.length}`, label: 'stub', rationale: 'stub' }] + }, + } + const failingAgent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext) => { + // The baseline must stay complete (an incomplete incumbent is refused), + // so the error lands on a generation-1 candidate cell instead. + if (scenario.id === 'c' && typeof surface === 'string' && surface.startsWith('candidate-')) { + throw new Error(longError) + } + return stubAgent(surface, scenario, ctx) + } + await improve(promptProfile(), [], { + surface: 'prompt', + scenarios, + judge: cappingJudge, + agent: failingAgent, + generator: stubProposer as never, + budget: { generations: 2, populationSize: 1, holdoutFraction: 0.25 }, + }) + expect(findingsSeen.length).toBeGreaterThanOrEqual(1) + const rows = findingsSeen.flat() as Array<{ + scenario: string + notes?: string + error?: string + }> + const intact = rows.find((row) => row.scenario === 'a') + expect(intact?.notes).toBe(intactNote) // below the cap ⇒ untouched + const clipped = rows.find((row) => row.scenario === 'b') + expect(clipped?.notes).toBe('n'.repeat(1500)) // at the cap ⇒ exactly 1500 + const errored = rows.find((row) => row.scenario === 'c') + expect(errored?.error).toBeDefined() + expect(errored?.error).toContain('e'.repeat(400)) + expect(errored?.error?.length).toBeLessThanOrEqual(500) + }) + it("surface 'code' + opts.code assembles the worktree pipeline and measures a candidate", async () => { const { execSync } = await import('node:child_process') const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 92243f90..f259905f 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -262,6 +262,16 @@ function baselineSurfaceFor( } } +/** Slice bound for distilled judge notes: wide enough that a real traceback / + * failing-assertion note survives intact — clipping mid-traceback would leave + * the proposer trace-blind. Referenced by the `rawTraceContext` docs. */ +const DISTILLED_NOTES_MAX_CHARS = 1500 + +/** Slice bound for a cell's error string — tighter than notes (errors are + * usually one line; the full text stays on the raw cell). Also bounds the + * error-derived `claim` fallback. */ +const DISTILLED_ERROR_MAX_CHARS = 500 + /** The default `analyzeGeneration`: distill each generation's failing cells into * findings for the next proposal round. Deliberately dependency-free — judge notes * and errors are already the domain's own diagnosis (executable gates put their @@ -296,21 +306,20 @@ function generationFailureDistiller( ? 0 : judgeScores.reduce((sum, j) => sum + (j.composite ?? 0), 0) / judgeScores.length if (!error && composite >= 0.999) continue - // 1500 chars keeps a real traceback / failing assertion intact — the old - // 400 clipped executable-judge notes down to a stub, leaving the proposer - // trace-blind. Errors stay tighter (they are usually one-line). const notes = judgeScores .map((j) => j.notes) .filter((n): n is string => typeof n === 'string' && n.length > 0) .join('; ') - .slice(0, 1500) - const claim = notes || (error ? `Scenario ${scenario} failed: ${error.slice(0, 500)}` : '') + .slice(0, DISTILLED_NOTES_MAX_CHARS) + const claim = + notes || + (error ? `Scenario ${scenario} failed: ${error.slice(0, DISTILLED_ERROR_MAX_CHARS)}` : '') failures.push({ scenario, composite: Number(composite.toFixed(3)), notes, ...(claim ? { claim } : {}), - ...(error ? { error: error.slice(0, 500) } : {}), + ...(error ? { error: error.slice(0, DISTILLED_ERROR_MAX_CHARS) } : {}), }) } } From 01eebf985d86e91959578a9d8e5353c941ad1355 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 15 Jul 2026 14:53:47 -0600 Subject: [PATCH 5/5] refactor(examples): move web-search ablation rigs to their own branch The aisdk/multi-contract search-ablation rigs are unrelated to this PR's titled change (failure-distiller evidence widths) and drew blocking review findings of their own (grader files readable from the worker workspace, agent-writable check.ts at score time, bash inheriting TANGLE_API_KEY in the no-search arm). They live on feat/web-search-ablation-rigs now, where those findings can be addressed as their own change; this PR carries exactly the evidence-width contribution. --- examples/ablation-suite/aisdk-env.ts | 282 ------------------ examples/ablation-suite/extract-contracts.mjs | 50 ---- .../multi-contract/contracts.all.json | 135 --------- .../fixtures/multi-contract/contracts.json | 72 ----- examples/ablation-suite/multi-contract-env.ts | 276 ----------------- examples/ablation-suite/run-aisdk.ts | 153 ---------- examples/ablation-suite/run-multi-contract.ts | 189 ------------ 7 files changed, 1157 deletions(-) delete mode 100644 examples/ablation-suite/aisdk-env.ts delete mode 100644 examples/ablation-suite/extract-contracts.mjs delete mode 100644 examples/ablation-suite/fixtures/multi-contract/contracts.all.json delete mode 100644 examples/ablation-suite/fixtures/multi-contract/contracts.json delete mode 100644 examples/ablation-suite/multi-contract-env.ts delete mode 100644 examples/ablation-suite/run-aisdk.ts delete mode 100644 examples/ablation-suite/run-multi-contract.ts diff --git a/examples/ablation-suite/aisdk-env.ts b/examples/ablation-suite/aisdk-env.ts deleted file mode 100644 index 2aeab5d0..00000000 --- a/examples/ablation-suite/aisdk-env.ts +++ /dev/null @@ -1,282 +0,0 @@ -/** - * aisdk-env — a long-horizon coding task on a genuinely POST-CUTOFF contract: build a working tool - * for the CURRENT Vercel `ai` SDK (v7). The coder's training knows v4, where the tool schema field - * was `parameters:`; v5+ renamed it to `inputSchema:`, so v4-shaped code no longer type-checks. The - * grader is the TypeScript compiler itself against the installed ai@7 — no rubric, the compiler is - * the judge. This is the first web-grounded task where the knowledge is genuinely absent from the - * coder AND search returns it AND a real compiler catches the difference. - * - * ABLATION DELTA: `search` toggles a `web_search` tool (you.com via the Tangle router). read/write/ - * run_tests (= tsc, returns the compiler errors as feedback) stay ON in both arms — additive - * isolation. The NO-SEARCH arm is the control: with compiler feedback but no web, does the coder - * recover the current contract on its own, or stay stuck on its v4 prior? - */ - -import { execFileSync } from 'node:child_process' -import { - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - rmSync, - symlinkSync, - writeFileSync, -} from 'node:fs' -import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' -import type { - AgenticSurface, - AgenticTask, - AgenticTool, - ArtifactHandle, - SurfaceScore, -} from '../../src/runtime/strategy.js' - -// Prebuilt template with node_modules { ai@7, zod, typescript } + the proven tsc invocation. Each -// workspace symlinks its node_modules and .bin from here so there is no per-task npm install. -const TEMPLATE = - process.env.AISDK_TEMPLATE ?? - '/tmp/claude-1000/-home-drew-code-blueprint-agent/8b62a3c4-97a5-46cd-92bf-5a4f3bf95f75/scratchpad/aisdk-probe' -const TSC = join(TEMPLATE, 'node_modules', '.bin', 'tsc') -const TSC_ARGS = [ - '--ignoreConfig', - '--pretty', - 'false', - '--noEmit', - '--strict', - '--skipLibCheck', - '--moduleResolution', - 'bundler', - '--module', - 'esnext', - '--target', - 'es2022', -] - -// Each task is a structurally-required rename in the current ai SDK: the coder's v4 memory writes a -// name the compiler now rejects, and the task cannot be completed without the current name — so a -// pass means the coder produced the current API, not a lucky stub. `starter` fails to compile (an -// unedited attempt is a fail); `check` compiles solution.ts with a real use of the export. -interface SdkTask { - key: string - systemPrompt: string - userPrompt: string - starter: string - check: string -} - -const SDK_TASKS: SdkTask[] = [ - { - key: 'tool', - systemPrompt: - 'You are a senior TypeScript engineer. Build code for the CURRENT version of the library, not the version you remember.', - userPrompt: - 'Edit `solution.ts` so it exports a tool named `weatherTool`, built with the CURRENT `ai` package via its `tool()` helper: a description, a zod schema for a `location: string` input, and an `execute` returning a string. The tool() schema field may have been renamed since your training — if run_tests fails, the current field name differs from what you wrote.', - starter: `import { tool } from 'ai'\nimport { z } from 'zod'\n// Build weatherTool with the CURRENT ai SDK tool() helper.\nexport const weatherTool = tool({})\n`, - check: `import { generateText } from 'ai'\nimport { weatherTool } from './solution'\nexport async function _check() {\n return generateText({ model: 'gpt' as unknown as never, tools: { weather: weatherTool }, prompt: 'x' })\n}\n`, - }, - { - key: 'stream', - systemPrompt: - 'You are a senior TypeScript engineer. Build code for the CURRENT version of the library, not the version you remember.', - userPrompt: - 'Edit `solution.ts` so it exports a function `handler(): Response` that calls `streamText` (model can be `"x" as any`, prompt `"hi"`) and returns its HTTP streaming Response for a chat UI. The method on the streamText result that returns this Response may have been renamed since your training — if run_tests fails, the current method name differs from what you wrote.', - starter: `import { streamText } from 'ai'\n// Return the streamText result's HTTP streaming Response.\nexport function handler(): Response {\n}\n`, - check: `import { handler } from './solution'\nexport const _r: Response = handler()\n`, - }, -] - -function taskSpec(id: string): SdkTask { - const key = id.split('#')[0] - const spec = SDK_TASKS.find((t) => t.key === key) ?? SDK_TASKS[0] - if (!spec) throw new Error(`aisdk-env: no task spec for '${id}'`) - return spec -} - -let searchCallTotal = 0 -export function resetSearchCalls(): void { - searchCallTotal = 0 -} -export function searchCalls(): number { - return searchCallTotal -} - -function ensureTemplate(): void { - if (!existsSync(join(TEMPLATE, 'node_modules', 'ai'))) { - throw new Error( - `AISDK_TEMPLATE has no node_modules/ai — install ai@7 zod typescript in ${TEMPLATE}`, - ) - } -} - -/** tsc the workspace (solution.ts + check.ts). Returns { pass, output } — output is the feedback. */ -function compile(dir: string): { pass: boolean; output: string } { - try { - execFileSync(TSC, [...TSC_ARGS, 'solution.ts', 'check.ts'], { - cwd: dir, - encoding: 'utf8', - timeout: 120_000, - stdio: ['ignore', 'pipe', 'pipe'], - }) - return { pass: true, output: 'tsc: 0 errors — compiles against the current ai SDK.' } - } catch (e) { - const out = - ((e as { stdout?: string }).stdout ?? '') + ((e as { stderr?: string }).stderr ?? '') - return { pass: false, output: out.trim().slice(0, 2_000) || 'tsc failed (no output)' } - } -} - -async function youSearch(query: string): Promise { - const key = process.env.TANGLE_API_KEY - if (!key) return 'web_search error: TANGLE_API_KEY unset' - try { - const res = await fetch('https://router.tangle.tools/v1/search', { - method: 'POST', - headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' }, - body: JSON.stringify({ query, provider: 'you' }), - }) - const j = (await res.json()) as { - data?: Array<{ title?: string; url?: string; snippet?: string }> - } - const rows = (j.data ?? []) - .slice(0, 6) - .map((r, i) => `${i + 1}. ${r.title ?? ''}\n ${r.url ?? ''}\n ${r.snippet ?? ''}`) - return rows.length ? rows.join('\n') : `web_search: no results for ${JSON.stringify(query)}` - } catch (err) { - return `web_search error: ${err instanceof Error ? err.message : String(err)}` - } -} - -export function makeAiSdkSurface(opts: { search: boolean }): AgenticSurface { - const name = `aisdk-v7${opts.search ? '+search' : ''}` - return { - name, - async open(task: AgenticTask): Promise { - ensureTemplate() - const spec = taskSpec(task.id) - const dir = mkdtempSync(join(tmpdir(), 'aisdk-')) - symlinkSync(join(TEMPLATE, 'node_modules'), join(dir, 'node_modules')) - writeFileSync(join(dir, 'solution.ts'), spec.starter) - writeFileSync(join(dir, 'check.ts'), spec.check) - return { id: `${name}:${task.id}`, surface: name, ctx: { dir } } - }, - async tools(): Promise { - const base: AgenticTool[] = [ - { - type: 'function', - function: { - name: 'write_file', - description: 'Write a file (path relative to project root, e.g. solution.ts).', - parameters: { - type: 'object', - properties: { path: { type: 'string' }, content: { type: 'string' } }, - required: ['path', 'content'], - }, - }, - }, - { - type: 'function', - function: { - name: 'read_file', - description: 'Read a file in the project.', - parameters: { - type: 'object', - properties: { path: { type: 'string' } }, - required: ['path'], - }, - }, - }, - ] - // NO_RUN_TESTS models the no-oracle regime (one-shot / non-executing agent): remove the - // compiler-feedback tool so the coder cannot trial-and-error to the current API. This is - // where web search should stop being redundant. - if (!process.env.NO_RUN_TESTS) - base.push({ - type: 'function', - function: { - name: 'run_tests', - description: - 'Type-check solution.ts against the installed ai SDK and get the compiler errors.', - parameters: { type: 'object', properties: {}, required: [] }, - }, - }) - if (opts.search) - base.push({ - type: 'function', - function: { - name: 'web_search', - description: - 'Search the live web (you.com) for the CURRENT ai SDK API you cannot recall (e.g. the tool() schema field name).', - parameters: { - type: 'object', - properties: { query: { type: 'string' } }, - required: ['query'], - }, - }, - }) - return base - }, - async call( - handle: ArtifactHandle, - toolName: string, - args: Record, - ): Promise { - const dir = (handle.ctx as { dir: string }).dir - if (toolName === 'web_search') { - searchCallTotal += 1 - return youSearch(String(args.query ?? '')) - } - if (toolName === 'write_file') { - const p = join(dir, String(args.path)) - mkdirSync(dirname(p), { recursive: true }) - writeFileSync(p, String(args.content ?? '')) - return `wrote ${args.path}` - } - if (toolName === 'read_file') { - const p = join(dir, String(args.path)) - return existsSync(p) - ? readFileSync(p, 'utf8').slice(0, 20_000) - : `no such file: ${args.path}` - } - if (toolName === 'run_tests') { - const { pass, output } = compile(dir) - return pass - ? output - : `tsc FAILED — fix the tool definition for the current SDK:\n${output}` - } - return `unknown tool: ${toolName}` - }, - async score(_task: AgenticTask, handle: ArtifactHandle): Promise { - const dir = (handle.ctx as { dir: string }).dir - return { passes: compile(dir).pass ? 1 : 0, total: 1, errored: 0 } - }, - async close(handle: ArtifactHandle): Promise { - const dir = (handle.ctx as { dir: string }).dir - if (process.env.AISDK_KEEP) { - console.error(` kept workspace: ${dir}`) - return - } - try { - rmSync(dir, { recursive: true, force: true }) - } catch { - /* best effort */ - } - }, - } -} - -// Cycle the verified tasks so n tasks spread evenly across the distinct renames (n=12 → 6 each), -// proving the search effect is not specific to one API change. -export function aiSdkTasks(n: number): Promise { - return Promise.resolve( - Array.from({ length: n }, (_, i) => { - const spec = SDK_TASKS[i % SDK_TASKS.length] - if (!spec) throw new Error('aisdk-env: SDK_TASKS is empty') - return { - id: `${spec.key}#${i}`, - systemPrompt: spec.systemPrompt, - userPrompt: spec.userPrompt, - } - }), - ) -} diff --git a/examples/ablation-suite/extract-contracts.mjs b/examples/ablation-suite/extract-contracts.mjs deleted file mode 100644 index 730eb644..00000000 --- a/examples/ablation-suite/extract-contracts.mjs +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Auto-discover every reusable (vendor, fn, testFile, graderB64) sub-contract in blueprint-agent's - * VB web-grounded API-integration leaves. Each grader is a self-contained mock+pytest that boots its - * own mock and tests ONE exported function of the worker's solution.py — compose N of them into one - * long-horizon integration (multi-contract-env). The fn is read from the decoded grader itself - * (hasattr(module, "")), the ground truth, not guessed. - */ -import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs' -import { dirname, join } from 'node:path' -import { fileURLToPath } from 'node:url' - -const here = dirname(fileURLToPath(import.meta.url)) -const VB = '/home/drew/code/blueprint-agent/scripts/experiments/scenarios/verticals/web-grounded' - -// The HTTP-client-shaped API-integration files (composable: solution.py exports base_url/api_key fns). -const FILES = readdirSync(VB).filter((f) => (f.startsWith('api-') || f.startsWith('pipedrive')) && f.endsWith('.ts')) - -function constVal(src, name) { - const m = src.match(new RegExp(name + "\\s*=\\s*((?:'[^']*'\\s*\\+?\\s*)+)")) - return m ? [...m[1].matchAll(/'([^']*)'/g)].map((x) => x[1]).join('') : null -} - -const all = [] -for (const file of FILES) { - const src = readFileSync(join(VB, file), 'utf8') - const vendor = (src.match(/partner:\s*'([^']+)'/) || src.match(/company:\s*'([^']+)'/) || [, file.replace(/\.ts$/, '')])[1] - for (const call of src.matchAll(/oracleCommand\('([^']+)'\s*,\s*([A-Z0-9_]+)\)/g)) { - const [, testFile, constName] = call - const b64 = constVal(src, constName) - if (!b64) { console.error('MISS b64', constName, file); continue } - let grader - try { grader = Buffer.from(b64, 'base64').toString('utf8') } catch { continue } - const fn = grader.match(/hasattr\(module,\s*"([a-z_][a-z0-9_]*)"\)/)?.[1] - ?? grader.match(/module\.([a-z_][a-z0-9_]*)\(/)?.[1] - if (!fn) { console.error('MISS fn in grader', testFile, file); continue } - // pull the signature from the grader's call or the seed - const sig = (src.match(new RegExp('def ' + fn + '\\([^\\n)]*\\)[^\\n]*')) || [, `def ${fn}(base_url, api_key, ...)`])[0] - all.push({ vendor: `${vendor}:${fn}`, fn, testFile, signature: (typeof sig === 'string' ? sig : `def ${fn}(base_url, api_key)`).trim(), graderB64: b64 }) - console.error('OK', vendor, fn, `(${testFile}, ${b64.length}b64)`) - } -} - -// dedup by fn (a solution.py exports each fn once) + keep GET-single-resource style first -const seen = new Set() -const contracts = all.filter((c) => (seen.has(c.fn) ? false : (seen.add(c.fn), true))) -const outDir = join(here, 'fixtures', 'multi-contract') -mkdirSync(outDir, { recursive: true }) -writeFileSync(join(outDir, 'contracts.all.json'), JSON.stringify(contracts, null, 2)) -console.error(`\nDISCOVERED ${contracts.length} unique-fn contracts -> contracts.all.json`) -console.error('vendors:', contracts.map((c) => c.fn).join(', ')) diff --git a/examples/ablation-suite/fixtures/multi-contract/contracts.all.json b/examples/ablation-suite/fixtures/multi-contract/contracts.all.json deleted file mode 100644 index 7ec1d8a7..00000000 --- a/examples/ablation-suite/fixtures/multi-contract/contracts.all.json +++ /dev/null @@ -1,135 +0,0 @@ -[ - { - "vendor": "posthog:get_flag_variant_payload", - "fn": "get_flag_variant_payload", - "testFile": "test_posthog_flags_variant_payload.py", - "signature": "def get_flag_variant_payload(base_url: str, project_api_key: str, distinct_id: str, flag_key: str) -> tuple\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcG9zdGhvZy1mbGFncy12Mi12YXJpYW50LXBheWxvYWQgKGhhcmQpIGxlYWYuCgpTYW1lIFBvc3RIb2cgQ1VSUkVOVCBmZWF0dXJlLWZsYWcgY29udHJhY3QgYXMgdGhlIG1lZGl1bSBsZWFmLCBidXQgZXhlcmNpc2VzIHRoZSByZXNwb25zZS1TSEFQRQpheGlzIGluIGZ1bGw6IHRoZSBhZ2VudCBtdXN0IHJlYWQgYSBtdWx0aXZhcmlhdGUgZmxhZydzIGFzc2lnbmVkIGB2YXJpYW50YCBBTkQgZGVjb2RlIGl0cwpKU09OLWVuY29kZWQgYHBheWxvYWRgLCBib3RoIG9mIHdoaWNoIG1vdmVkIGxvY2F0aW9uIGluIHRoZSB2MiBlbnZlbG9wZS4KCiAgLSBlbmRwb2ludCAgICAgUE9TVCAvZmxhZ3M/dj0yCiAgLSBhdXRoICAgICAgICAgcHJvamVjdCB0b2tlbiBhcyBhcGlfa2V5IGluIHRoZSBKU09OIGJvZHkKICAtIHYyIHNoYXBlICAgICBmbGFnc1trZXldWyJ2YXJpYW50Il0gICAgICAgICAgICAgICAgICAodjEvZGVjaWRlOiBmZWF0dXJlRmxhZ3Nba2V5XSBJUyB0aGUgdmFyaWFudCBzdHJpbmcpCiAgICAgICAgICAgICAgICAgZmxhZ3Nba2V5XVsibWV0YWRhdGEiXVsicGF5bG9hZCJdICAgICAgKHYxL2RlY2lkZTogYSBzZXBhcmF0ZSBmZWF0dXJlRmxhZ1BheWxvYWRzW2tleV0gbWFwKQogICAgICAgICAgICAgICAgIHBheWxvYWQgaXMgYSBKU09OLUVOQ09ERUQgU1RSSU5HIHRoYXQgdGhlIGNsaWVudCBtdXN0IGpzb24ubG9hZHMgaW50byBhIGRpY3QuCgpBIGNsaWVudCB3cml0dGVuIGZyb20gQ1VSUkVOVCBQb3N0SG9nIGRvY3MgcmVhZHMgZmxhZ3NbJ2NoZWNrb3V0LWV4cGVyaW1lbnQnXVsndmFyaWFudCddID09Cid2YXJpYW50LWInIGFuZCBkZWNvZGVzIG1ldGFkYXRhLnBheWxvYWQgLT4geyJkaXNjb3VudF9wY3QiOiAyMH0gYW5kIHBhc3Nlcy4gQSBzdGFsZS9mcm9tLW1lbW9yeQpjbGllbnQgUE9TVHMgL2RlY2lkZSBhbmQgcmVhZHMgZmVhdHVyZUZsYWdzIC8gZmVhdHVyZUZsYWdQYXlsb2FkcyAoYWJzZW50IGluIHRoZSB2MiBlbnZlbG9wZSkgYW5kCkZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL3Bvc3Rob2cuY29tL2RvY3MvYXBpL2ZsYWdzCiAgaHR0cHM6Ly9wb3N0aG9nLmNvbS9kb2NzL2ludGVncmF0ZS9mZWF0dXJlLWZsYWdzLWNvZGUKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9UT0tFTiA9ICJwaGNfbGl2ZV9wcm9qZWN0X3Rva2VuX2FiYzEyMyIKCkZMQUdTID0gewogICAgIm5ldy1kYXNoYm9hcmQiOiB7CiAgICAgICAgImtleSI6ICJuZXctZGFzaGJvYXJkIiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJjb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogMCwgImRlc2NyaXB0aW9uIjogIkNvbmRpdGlvbiBzZXQgMSBtYXRjaGVkIn0sCiAgICAgICAgIm1ldGFkYXRhIjogeyJpZCI6IDEsICJ2ZXJzaW9uIjogMywgInBheWxvYWQiOiBOb25lfSwKICAgIH0sCiAgICAibGVnYWN5LWJhbm5lciI6IHsKICAgICAgICAia2V5IjogImxlZ2FjeS1iYW5uZXIiLAogICAgICAgICJlbmFibGVkIjogRmFsc2UsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJub19jb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogTm9uZSwgImRlc2NyaXB0aW9uIjogIk5vIGNvbmRpdGlvbiBzZXQgbWF0Y2hlZCJ9LAogICAgICAgICJtZXRhZGF0YSI6IHsiaWQiOiAyLCAidmVyc2lvbiI6IDEsICJwYXlsb2FkIjogTm9uZX0sCiAgICB9LAogICAgImNoZWNrb3V0LWV4cGVyaW1lbnQiOiB7CiAgICAgICAgImtleSI6ICJjaGVja291dC1leHBlcmltZW50IiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInZhcmlhbnQiOiAidmFyaWFudC1iIiwKICAgICAgICAicmVhc29uIjogeyJjb2RlIjogImNvbmRpdGlvbl9tYXRjaCIsICJjb25kaXRpb25faW5kZXgiOiAxLCAiZGVzY3JpcHRpb24iOiAiQ29uZGl0aW9uIHNldCAyIG1hdGNoZWQifSwKICAgICAgICAibWV0YWRhdGEiOiB7ImlkIjogMywgInZlcnNpb24iOiA3LCAicGF5bG9hZCI6ICJ7XCJkaXNjb3VudF9wY3RcIjogMjB9In0sCiAgICB9LAp9CgoKY2xhc3MgX01vY2tQb3N0SG9nKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOgogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fUE9TVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIGxlbmd0aCA9IGludChzZWxmLmhlYWRlcnMuZ2V0KCJDb250ZW50LUxlbmd0aCIsIDApIG9yIDApCiAgICAgICAgcmF3ID0gc2VsZi5yZmlsZS5yZWFkKGxlbmd0aCkgaWYgbGVuZ3RoIGVsc2UgYiIiCiAgICAgICAgdHJ5OgogICAgICAgICAgICBib2R5ID0ganNvbi5sb2FkcyhyYXcuZGVjb2RlKCkpIGlmIHJhdyBlbHNlIHt9CiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAgICAgYm9keSA9IE5vbmUKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJ2IjogcXMuZ2V0KCJ2IiwgW05vbmVdKVswXSwKICAgICAgICAgICAgImJvZHlfa2V5cyI6IHNvcnRlZChib2R5LmtleXMoKSkgaWYgaXNpbnN0YW5jZShib2R5LCBkaWN0KSBlbHNlIE5vbmUsCiAgICAgICAgfSkKCiAgICAgICAgaWYgcGF0aCAhPSAiL2ZsYWdzIjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsidHlwZSI6ICJpbnZhbGlkX3JlcXVlc3QiLCAiY29kZSI6ICJub3RfZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXRhaWwiOiAidW5rbm93biBlbmRwb2ludCAoUG9zdEhvZyBldmFsdWF0ZXMgZmVhdHVyZSBmbGFncyBhdCBQT1NUIC9mbGFncz92PTIpIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBpZiBxcy5nZXQoInYiLCBbTm9uZV0pWzBdICE9ICIyIjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsidHlwZSI6ICJ2YWxpZGF0aW9uX2Vycm9yIiwgImNvZGUiOiAibWlzc2luZ192ZXJzaW9uIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGV0YWlsIjogInRoZSAvZmxhZ3MgZW5kcG9pbnQgcmVxdWlyZXMgP3Y9MiBmb3IgdGhlIGN1cnJlbnQgZmxhZ3MgcmVzcG9uc2UgZm9ybWF0In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBpZiBub3QgaXNpbnN0YW5jZShib2R5LCBkaWN0KToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsidHlwZSI6ICJ2YWxpZGF0aW9uX2Vycm9yIiwgImNvZGUiOiAibWFsZm9ybWVkX2JvZHkiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXRhaWwiOiAiUE9TVCBib2R5IG11c3QgYmUgSlNPTiBjb250YWluaW5nIGFwaV9rZXkgYW5kIGRpc3RpbmN0X2lkIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBpZiBib2R5LmdldCgiYXBpX2tleSIpICE9IEVYUEVDVEVEX1RPS0VOOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJ0eXBlIjogImF1dGhlbnRpY2F0aW9uX2Vycm9yIiwgImNvZGUiOiAiaW52YWxpZF9hcGlfa2V5IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGV0YWlsIjogInByb3ZpZGUgdGhlIHByb2plY3QgQVBJIGtleSBhcyBhcGlfa2V5IGluIHRoZSBKU09OIGJvZHkifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIG5vdCBib2R5LmdldCgiZGlzdGluY3RfaWQiKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsidHlwZSI6ICJ2YWxpZGF0aW9uX2Vycm9yIiwgImNvZGUiOiAibWlzc2luZ19kaXN0aW5jdF9pZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJkaXN0aW5jdF9pZCBpcyByZXF1aXJlZCB0byBldmFsdWF0ZSBmbGFncyBmb3IgYSB1c2VyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAiZmxhZ3MiOiBGTEFHUywKICAgICAgICAgICAgImVycm9yc1doaWxlQ29tcHV0aW5nRmxhZ3MiOiBGYWxzZSwKICAgICAgICAgICAgInJlcXVlc3RJZCI6ICI1NTBlODQwMC1lMjliLTQxZDQtYTcxNi00NDY2NTU0NDAwMDAiLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1Bvc3RIb2cpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpLnN0YXJ0KCkKICAgIHRyeToKICAgICAgICB5aWVsZCBmImh0dHA6Ly8xMjcuMC4wLjE6e3BvcnR9IgogICAgZmluYWxseToKICAgICAgICBzZXJ2ZXIuc2h1dGRvd24oKQogICAgICAgIHNlcnZlci5zZXJ2ZXJfY2xvc2UoKQoKCmRlZiBfbG9hZF9hZ2VudF9zb2x1dGlvbigpOgogICAgcm9vdCA9IG9zLmVudmlyb24uZ2V0KCJWQl9TT0xVVElPTl9ST09UIiwgb3MuZ2V0Y3dkKCkpCiAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4ocm9vdCwgInNvbHV0aW9uIiwgInNvbHV0aW9uLnB5IikKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIGZvciBkaXJwYXRoLCBfZGlycywgZmlsZXMgaW4gb3Mud2Fsayhyb290KToKICAgICAgICAgICAgaWYgInNvbHV0aW9uLnB5IiBpbiBmaWxlczoKICAgICAgICAgICAgICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihkaXJwYXRoLCAic29sdXRpb24ucHkiKQogICAgICAgICAgICAgICAgYnJlYWsKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIHB5dGVzdC5mYWlsKGYiYWdlbnQgc29sdXRpb24gbm90IGZvdW5kOiBleHBlY3RlZCBzb2x1dGlvbi9zb2x1dGlvbi5weSB1bmRlciB7cm9vdH0iKQogICAgc3BlYyA9IGltcG9ydGxpYi51dGlsLnNwZWNfZnJvbV9maWxlX2xvY2F0aW9uKCJhZ2VudF9zb2x1dGlvbiIsIGNhbmRpZGF0ZSkKICAgIG1vZHVsZSA9IGltcG9ydGxpYi51dGlsLm1vZHVsZV9mcm9tX3NwZWMoc3BlYykKICAgIHNwZWMubG9hZGVyLmV4ZWNfbW9kdWxlKG1vZHVsZSkKICAgIHJldHVybiBtb2R1bGUKCgpkZWYgdGVzdF9nZXRfZmxhZ192YXJpYW50X2FuZF9wYXlsb2FkX2Zyb21fdjJfbWV0YWRhdGEobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAiZ2V0X2ZsYWdfdmFyaWFudF9wYXlsb2FkIiksIFwKICAgICAgICAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgZ2V0X2ZsYWdfdmFyaWFudF9wYXlsb2FkKGJhc2VfdXJsLCBwcm9qZWN0X2FwaV9rZXksIGRpc3RpbmN0X2lkLCBmbGFnX2tleSkiCgogICAgcmVzdWx0ID0gbW9kdWxlLmdldF9mbGFnX3ZhcmlhbnRfcGF5bG9hZChtb2NrX3NlcnZlciwgRVhQRUNURURfVE9LRU4sICJ1c2VyLTEyMyIsICJjaGVja291dC1leHBlcmltZW50IikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsICh0dXBsZSwgbGlzdCkpIGFuZCBsZW4ocmVzdWx0KSA9PSAyLCAoCiAgICAgICAgZiJleHBlY3RlZCBhICh2YXJpYW50LCBwYXlsb2FkKSBwYWlyLCBnb3Qge3Jlc3VsdCFyfS4gU2VydmVyIHNhdzoge19Nb2NrUG9zdEhvZy5yZXF1ZXN0X2xvZ30iCiAgICApCiAgICB2YXJpYW50LCBwYXlsb2FkID0gcmVzdWx0WzBdLCByZXN1bHRbMV0KCiAgICBhc3NlcnQgdmFyaWFudCA9PSAidmFyaWFudC1iIiwgKAogICAgICAgIGYiZXhwZWN0ZWQgdmFyaWFudCAndmFyaWFudC1iJyBmcm9tIGZsYWdzWydjaGVja291dC1leHBlcmltZW50J11bJ3ZhcmlhbnQnXSwgZ290IHt2YXJpYW50IXJ9LiAiCiAgICAgICAgZiJBIHN0YWxlIGNsaWVudCByZWFkaW5nIHRoZSAvZGVjaWRlIGZlYXR1cmVGbGFncyBtYXAgKHdoZXJlIHRoZSB2YWx1ZSBpdHNlbGYgaXMgdGhlIHZhcmlhbnQgc3RyaW5nKSBvciAiCiAgICAgICAgZiJ0aGUgd3JvbmcgZW5kcG9pbnQgZmFpbHMgaGVyZS4gU2VydmVyIHNhdzoge19Nb2NrUG9zdEhvZy5yZXF1ZXN0X2xvZ30iCiAgICApCgogICAgcGFyc2VkID0ganNvbi5sb2FkcyhwYXlsb2FkKSBpZiBpc2luc3RhbmNlKHBheWxvYWQsIHN0cikgZWxzZSBwYXlsb2FkCiAgICBhc3NlcnQgcGFyc2VkID09IHsiZGlzY291bnRfcGN0IjogMjB9LCAoCiAgICAgICAgZiJleHBlY3RlZCBwYXlsb2FkIHt7J2Rpc2NvdW50X3BjdCc6IDIwfX0gZGVjb2RlZCBmcm9tIGZsYWdzWydjaGVja291dC1leHBlcmltZW50J11bJ21ldGFkYXRhJ11bJ3BheWxvYWQnXSAiCiAgICAgICAgZiIoYSBKU09OLWVuY29kZWQgc3RyaW5nKSwgZ290IHtwYXlsb2FkIXJ9LiBBIHN0YWxlIGNsaWVudCByZWFkcyBhIHRvcC1sZXZlbCBmZWF0dXJlRmxhZ1BheWxvYWRzIG1hcCAoYWJzZW50ICIKICAgICAgICBmImluIHRoZSB2MiBlbnZlbG9wZSkgYW5kIGZhaWxzLiBTZXJ2ZXIgc2F3OiB7X01vY2tQb3N0SG9nLnJlcXVlc3RfbG9nfSIKICAgICkK" - }, - { - "vendor": "posthog:is_feature_enabled", - "fn": "is_feature_enabled", - "testFile": "test_posthog_flags_is_enabled.py", - "signature": "def is_feature_enabled(base_url: str, project_api_key: str, distinct_id: str, flag_key: str) -> bool\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcG9zdGhvZy1mbGFncy12Mi1pcy1lbmFibGVkIChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBQb3N0SG9nJ3MgQ1VSUkVOVCBmZWF0dXJlLWZsYWcKZXZhbHVhdGlvbiBjb250cmFjdDoKICAtIGVuZHBvaW50ICAgICBQT1NUIC9mbGFncz92PTIgICAgICAgICAgICAodGhlIG9sZGVyIC9kZWNpZGUgZW5kcG9pbnQgaXMgbm90IHNlcnZlZCkKICAtIGF1dGggICAgICAgICBwcm9qZWN0IHRva2VuIGFzIGFwaV9rZXkgaW4gdGhlIEpTT04gYm9keQogIC0gcmVzcG9uc2UgICAgIHsiZmxhZ3MiOiB7PGtleT46IHsia2V5IiwiZW5hYmxlZCIsInZhcmlhbnQiPywicmVhc29uIiwibWV0YWRhdGEifX0sIC4uLn0KICAgICAgICAgICAgICAgICAodGhlIG9sZGVyIC9kZWNpZGUgcmV0dXJuZWQgYSBmbGF0IHsiZmVhdHVyZUZsYWdzIjogezxrZXk+OiB0cnVlfCJ2YXJpYW50In19IG1hcCkKCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIFBvc3RIb2cgZG9jcyBQT1NUcyAvZmxhZ3M/dj0yIGFuZCByZWFkcwpmbGFnc1trZXldWyJlbmFibGVkIl0sIHNvIGl0IHJlcG9ydHMgbmV3LWRhc2hib2FyZCBUcnVlIGFuZCBsZWdhY3ktYmFubmVyIEZhbHNlIGFuZCBwYXNzZXMuCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtY3V0b2ZmIG1lbW9yeSBQT1NUcyAvZGVjaWRlLz92PTMgYW5kIHJlYWRzIGZlYXR1cmVGbGFnc1trZXldOwppdCA0MDRzIG9uIHRoZSBlbmRwb2ludCAob3IgS2V5RXJyb3JzIG9uIHRoZSBtaXNzaW5nIGZlYXR1cmVGbGFncyBtYXApIGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9wb3N0aG9nLmNvbS9kb2NzL2FwaS9mbGFncwogIGh0dHBzOi8vcG9zdGhvZy5jb20vZG9jcy9pbnRlZ3JhdGUvZmVhdHVyZS1mbGFncy1jb2RlCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgcHJvamVjdCB0b2tlbiB0aGUgY2xpZW50IG11c3Qgc2VuZCBhcyBgYXBpX2tleWAgaW4gdGhlIEpTT04gYm9keSAodjIgY29udHJhY3QpLgpFWFBFQ1RFRF9UT0tFTiA9ICJwaGNfbGl2ZV9wcm9qZWN0X3Rva2VuX2FiYzEyMyIKCiMgVGhlIGhpZGRlbiBmbGFnIHN0YXRlLiBUaGUgYWdlbnQgY2Fubm90IGhhcmRjb2RlIHRoaXMg4oCUIGl0IGlzIG9ubHkgb2JzZXJ2YWJsZSBieSBhY3R1YWxseQojIFBPU1RpbmcgL2ZsYWdzP3Y9MiBhbmQgcGFyc2luZyB0aGUgdjIgZW52ZWxvcGUuCkZMQUdTID0gewogICAgIm5ldy1kYXNoYm9hcmQiOiB7CiAgICAgICAgImtleSI6ICJuZXctZGFzaGJvYXJkIiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJjb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogMCwgImRlc2NyaXB0aW9uIjogIkNvbmRpdGlvbiBzZXQgMSBtYXRjaGVkIn0sCiAgICAgICAgIm1ldGFkYXRhIjogeyJpZCI6IDEsICJ2ZXJzaW9uIjogMywgInBheWxvYWQiOiBOb25lfSwKICAgIH0sCiAgICAibGVnYWN5LWJhbm5lciI6IHsKICAgICAgICAia2V5IjogImxlZ2FjeS1iYW5uZXIiLAogICAgICAgICJlbmFibGVkIjogRmFsc2UsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJub19jb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogTm9uZSwgImRlc2NyaXB0aW9uIjogIk5vIGNvbmRpdGlvbiBzZXQgbWF0Y2hlZCJ9LAogICAgICAgICJtZXRhZGF0YSI6IHsiaWQiOiAyLCAidmVyc2lvbiI6IDEsICJwYXlsb2FkIjogTm9uZX0sCiAgICB9LAogICAgImNoZWNrb3V0LWV4cGVyaW1lbnQiOiB7CiAgICAgICAgImtleSI6ICJjaGVja291dC1leHBlcmltZW50IiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInZhcmlhbnQiOiAidmFyaWFudC1iIiwKICAgICAgICAicmVhc29uIjogeyJjb2RlIjogImNvbmRpdGlvbl9tYXRjaCIsICJjb25kaXRpb25faW5kZXgiOiAxLCAiZGVzY3JpcHRpb24iOiAiQ29uZGl0aW9uIHNldCAyIG1hdGNoZWQifSwKICAgICAgICAibWV0YWRhdGEiOiB7ImlkIjogMywgInZlcnNpb24iOiA3LCAicGF5bG9hZCI6ICJ7XCJkaXNjb3VudF9wY3RcIjogMjB9In0sCiAgICB9LAp9CgoKY2xhc3MgX01vY2tQb3N0SG9nKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgIyBDbGFzcy1sZXZlbCBsb2cgc28gdGhlIHRlc3QgY2FuIHJlcG9ydCB0aGUgcGF0aC92ZXJzaW9uL2JvZHkgdGhlIGNsaWVudCBhY3R1YWxseSB1c2VkLgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOiAgIyBzaWxlbmNlIHN0ZGVyciBhY2Nlc3MgbG9nCiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19QT1NUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgbGVuZ3RoID0gaW50KHNlbGYuaGVhZGVycy5nZXQoIkNvbnRlbnQtTGVuZ3RoIiwgMCkgb3IgMCkKICAgICAgICByYXcgPSBzZWxmLnJmaWxlLnJlYWQobGVuZ3RoKSBpZiBsZW5ndGggZWxzZSBiIiIKICAgICAgICB0cnk6CiAgICAgICAgICAgIGJvZHkgPSBqc29uLmxvYWRzKHJhdy5kZWNvZGUoKSkgaWYgcmF3IGVsc2Uge30KICAgICAgICBleGNlcHQgRXhjZXB0aW9uOgogICAgICAgICAgICBib2R5ID0gTm9uZQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgInYiOiBxcy5nZXQoInYiLCBbTm9uZV0pWzBdLAogICAgICAgICAgICAiYm9keV9rZXlzIjogc29ydGVkKGJvZHkua2V5cygpKSBpZiBpc2luc3RhbmNlKGJvZHksIGRpY3QpIGVsc2UgTm9uZSwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogY3VycmVudCBmbGFnIGV2YWx1YXRpb24gbGl2ZXMgYXQgL2ZsYWdzICh0aGUgb2xkIC9kZWNpZGUgaXMgZ29uZSBoZXJlKS4KICAgICAgICBpZiBwYXRoICE9ICIvZmxhZ3MiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdCIsICJjb2RlIjogIm5vdF9mb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJ1bmtub3duIGVuZHBvaW50IChQb3N0SG9nIGV2YWx1YXRlcyBmZWF0dXJlIGZsYWdzIGF0IFBPU1QgL2ZsYWdzP3Y9MikifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgdmVyc2lvbiBheGlzOiB0aGUgY3VycmVudCBmbGFncyByZXNwb25zZSBzaGFwZSByZXF1aXJlcyA/dj0yLgogICAgICAgIGlmIHFzLmdldCgidiIsIFtOb25lXSlbMF0gIT0gIjIiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJ0eXBlIjogInZhbGlkYXRpb25fZXJyb3IiLCAiY29kZSI6ICJtaXNzaW5nX3ZlcnNpb24iLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXRhaWwiOiAidGhlIC9mbGFncyBlbmRwb2ludCByZXF1aXJlcyA/dj0yIGZvciB0aGUgY3VycmVudCBmbGFncyByZXNwb25zZSBmb3JtYXQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIG5vdCBpc2luc3RhbmNlKGJvZHksIGRpY3QpOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJ0eXBlIjogInZhbGlkYXRpb25fZXJyb3IiLCAiY29kZSI6ICJtYWxmb3JtZWRfYm9keSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJQT1NUIGJvZHkgbXVzdCBiZSBKU09OIGNvbnRhaW5pbmcgYXBpX2tleSBhbmQgZGlzdGluY3RfaWQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiB0aGUgcHJvamVjdCB0b2tlbiB0cmF2ZWxzIGFzIGFwaV9rZXkgaW4gdGhlIEpTT04gYm9keS4KICAgICAgICBpZiBib2R5LmdldCgiYXBpX2tleSIpICE9IEVYUEVDVEVEX1RPS0VOOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJ0eXBlIjogImF1dGhlbnRpY2F0aW9uX2Vycm9yIiwgImNvZGUiOiAiaW52YWxpZF9hcGlfa2V5IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGV0YWlsIjogInByb3ZpZGUgdGhlIHByb2plY3QgQVBJIGtleSBhcyBhcGlfa2V5IGluIHRoZSBKU09OIGJvZHkifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIG5vdCBib2R5LmdldCgiZGlzdGluY3RfaWQiKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsidHlwZSI6ICJ2YWxpZGF0aW9uX2Vycm9yIiwgImNvZGUiOiAibWlzc2luZ19kaXN0aW5jdF9pZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJkaXN0aW5jdF9pZCBpcyByZXF1aXJlZCB0byBldmFsdWF0ZSBmbGFncyBmb3IgYSB1c2VyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAiZmxhZ3MiOiBGTEFHUywKICAgICAgICAgICAgImVycm9yc1doaWxlQ29tcHV0aW5nRmxhZ3MiOiBGYWxzZSwKICAgICAgICAgICAgInJlcXVlc3RJZCI6ICI1NTBlODQwMC1lMjliLTQxZDQtYTcxNi00NDY2NTU0NDAwMDAiLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1Bvc3RIb2cpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpLnN0YXJ0KCkKICAgIHRyeToKICAgICAgICB5aWVsZCBmImh0dHA6Ly8xMjcuMC4wLjE6e3BvcnR9IgogICAgZmluYWxseToKICAgICAgICBzZXJ2ZXIuc2h1dGRvd24oKQogICAgICAgIHNlcnZlci5zZXJ2ZXJfY2xvc2UoKQoKCmRlZiBfbG9hZF9hZ2VudF9zb2x1dGlvbigpOgogICAgcm9vdCA9IG9zLmVudmlyb24uZ2V0KCJWQl9TT0xVVElPTl9ST09UIiwgb3MuZ2V0Y3dkKCkpCiAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4ocm9vdCwgInNvbHV0aW9uIiwgInNvbHV0aW9uLnB5IikKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIGZvciBkaXJwYXRoLCBfZGlycywgZmlsZXMgaW4gb3Mud2Fsayhyb290KToKICAgICAgICAgICAgaWYgInNvbHV0aW9uLnB5IiBpbiBmaWxlczoKICAgICAgICAgICAgICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihkaXJwYXRoLCAic29sdXRpb24ucHkiKQogICAgICAgICAgICAgICAgYnJlYWsKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIHB5dGVzdC5mYWlsKGYiYWdlbnQgc29sdXRpb24gbm90IGZvdW5kOiBleHBlY3RlZCBzb2x1dGlvbi9zb2x1dGlvbi5weSB1bmRlciB7cm9vdH0iKQogICAgc3BlYyA9IGltcG9ydGxpYi51dGlsLnNwZWNfZnJvbV9maWxlX2xvY2F0aW9uKCJhZ2VudF9zb2x1dGlvbiIsIGNhbmRpZGF0ZSkKICAgIG1vZHVsZSA9IGltcG9ydGxpYi51dGlsLm1vZHVsZV9mcm9tX3NwZWMoc3BlYykKICAgIHNwZWMubG9hZGVyLmV4ZWNfbW9kdWxlKG1vZHVsZSkKICAgIHJldHVybiBtb2R1bGUKCgpkZWYgdGVzdF9pc19mZWF0dXJlX2VuYWJsZWRfcmVhZHNfdjJfZmxhZ3NfZW52ZWxvcGUobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAiaXNfZmVhdHVyZV9lbmFibGVkIiksIFwKICAgICAgICAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgaXNfZmVhdHVyZV9lbmFibGVkKGJhc2VfdXJsLCBwcm9qZWN0X2FwaV9rZXksIGRpc3RpbmN0X2lkLCBmbGFnX2tleSkiCgogICAgZW5hYmxlZCA9IG1vZHVsZS5pc19mZWF0dXJlX2VuYWJsZWQobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX1RPS0VOLCAidXNlci0xMjMiLCAibmV3LWRhc2hib2FyZCIpCiAgICBkaXNhYmxlZCA9IG1vZHVsZS5pc19mZWF0dXJlX2VuYWJsZWQobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX1RPS0VOLCAidXNlci0xMjMiLCAibGVnYWN5LWJhbm5lciIpCgogICAgYXNzZXJ0IGVuYWJsZWQgPT0gVHJ1ZSwgKCAgIyBub3FhOiBFNzEyIC0gcmVqZWN0IGRpY3QvTm9uZS90cnV0aHktb2JqZWN0LCBub3QganVzdCBmYWxzeQogICAgICAgIGYiZXhwZWN0ZWQgbmV3LWRhc2hib2FyZCBlbmFibGVkPVRydWUsIGdvdCB7ZW5hYmxlZCFyfS4gQSBjb3JyZWN0IGNsaWVudCBQT1NUcyAvZmxhZ3M/dj0yIGFuZCByZWFkcyAiCiAgICAgICAgZiJmbGFnc1snbmV3LWRhc2hib2FyZCddWydlbmFibGVkJ10uIFNlcnZlciBzYXc6IHtfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2d9IgogICAgKQogICAgYXNzZXJ0IGRpc2FibGVkID09IEZhbHNlLCAoICAjIG5vcWE6IEU3MTIgLSBhIHRydXRoeSBmbGFnIG9iamVjdCBvciBhIC9kZWNpZGUgZmVhdHVyZUZsYWdzIHJlYWQgZmFpbHMgaGVyZQogICAgICAgIGYiZXhwZWN0ZWQgbGVnYWN5LWJhbm5lciBlbmFibGVkPUZhbHNlLCBnb3Qge2Rpc2FibGVkIXJ9LiBBIHN0YWxlIGNsaWVudCAocmVhZGluZyB0aGUgL2RlY2lkZSBmZWF0dXJlRmxhZ3MgIgogICAgICAgIGYibWFwLCBvciB0cmVhdGluZyB0aGUgZmxhZyBPQkpFQ1QgYXMgdHJ1dGh5KSBnZXRzIHRoaXMgd3JvbmcuIFNlcnZlciBzYXc6IHtfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2d9IgogICAgKQo=" - }, - { - "vendor": "stripe:list_all_events", - "fn": "list_all_events", - "testFile": "test_stripe_list_events.py", - "signature": "def list_all_events(base_url: str, api_key: str, object_id: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgc3RyaXBlLXYyLWxpc3QtZXZlbnRzIChoYXJkKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUKdGltZSAoYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdAp0aGUgYWdlbnQncyBzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgU3RyaXBlJ3MgQ1VSUkVOVCAoQVBJIHYyKQpldmVudHMtbGlzdCBjb250cmFjdCBmb3IgdXNhZ2UtYmlsbGluZyBtZXRlciBldmVudHM6CiAgLSBlbmRwb2ludCBwYXRoICAgICAvdjIvY29yZS9ldmVudHMgICAgICAgICAgICAgICAgICAgICAgICh2MSB3YXMgL3YxL2V2ZW50cykKICAtIHZlcnNpb24gaGVhZGVyICAgIFN0cmlwZS1WZXJzaW9uOiA8WVlZWS1NTS1ERC5jb2RlbmFtZT4gICh2MSByZXF1aXJlZCBOTyB2ZXJzaW9uIGhlYWRlcikKICAtIGF1dGggICAgICAgICAgICAgIEF1dGhvcml6YXRpb246IEJlYXJlciA8a2V5PiAgICAgICAgICAgIChzYW1lIGFjcm9zcyB2MS92MikKICAtIHBhZ2luYXRpb24gICAgICAgIGZvbGxvdyB0aGUgZnVsbC1VUkwgYG5leHRfcGFnZV91cmxgIHVudGlsIGl0IGlzIG51bGwKICAgICAgICAgICAgICAgICAgICAgICh2MSB1c2VkID9zdGFydGluZ19hZnRlcj08bGFzdF9pZD4gKyByZXNwb25zZSBgaGFzX21vcmVgKQoKQSBjbGllbnQgd3JpdHRlbiBmcm9tIENVUlJFTlQgU3RyaXBlIHYyIGRvY3MgY29sbGVjdHMgYWxsIDcgZXZlbnRzIGFjcm9zcyAzIHBhZ2VzCmJ5IGZvbGxvd2luZyBuZXh0X3BhZ2VfdXJsIGFuZCBwYXNzZXMuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtY3V0b2ZmCm1lbW9yeSAodjEgL3YxL2V2ZW50cyBwYXRoLCBubyB2ZXJzaW9uIGhlYWRlciwgb3Igc3RhcnRpbmdfYWZ0ZXIgKyBoYXNfbW9yZQpwYWdpbmF0aW9uKSBoaXRzIDQwNCAvIDQwMCAvIHN0b3BzIGFmdGVyIHBhZ2UgMSBhbmQgRkFJTFMg4oCUIHRoZSB2MiBlbnZlbG9wZSBoYXMKbm8gYGhhc19tb3JlYCBhbmQgaWdub3JlcyBgc3RhcnRpbmdfYWZ0ZXJgLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGktdjItb3ZlcnZpZXcKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGkvdjIvY29yZS9ldmVudHMvbGlzdC5tZAogIGh0dHBzOi8vZG9jcy5zdHJpcGUuY29tL2FwaS9wYWdpbmF0aW9uICAgKHRoZSB2MSBzaGFwZSB0aGUgZnJvbS1tZW1vcnkgY2xpZW50IGVtaXRzKQoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9LRVkgPSAic2tfdGVzdF92Ml9rZXlfYWJjMTIzIgpPQkpFQ1RfSUQgPSAibXRyX3Rlc3RfNjFSQ2ppcWRUREM5MXpnaXA0MUlxUEN6UG54cSIKX1ZFUlNJT05fUkUgPSByZS5jb21waWxlKHIiXlxkezR9LVxkezJ9LVxkezJ9KFwuW2EtejAtOV0rKT8kIikKCiMgNyBiaWxsaW5nLW1ldGVyIGV2ZW50cywgbW9jayBwYWdlIHNpemUgMyAtPiAzIHBhZ2VzLiBPbmx5IG9ic2VydmFibGUgYnkKIyBwYWdpbmF0aW5nIHRoZSBtb2NrIHRvIHRoZSBlbmQ7IHRoZSBhZ2VudCBjYW5ub3QgaGFyZGNvZGUgdGhpcy4KRVZFTlRTID0gWwogICAgewogICAgICAgICJpZCI6IGYiZXZ0X3Rlc3RfbWV0ZXJfe2l9IiwKICAgICAgICAib2JqZWN0IjogInYyLmNvcmUuZXZlbnQiLAogICAgICAgICJ0eXBlIjogInYxLmJpbGxpbmcubWV0ZXIuZXJyb3JfcmVwb3J0X3RyaWdnZXJlZCIsCiAgICAgICAgImNyZWF0ZWQiOiBmIjIwMjYtMDYtezEwICsgaX1UMTc6NDY6MjIuMTM0WiIsCiAgICAgICAgImxpdmVtb2RlIjogRmFsc2UsCiAgICAgICAgInJlbGF0ZWRfb2JqZWN0IjogeyJpZCI6IE9CSkVDVF9JRCwgInR5cGUiOiAiYmlsbGluZy5tZXRlciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICJ1cmwiOiBmIi92MS9iaWxsaW5nL21ldGVycy97T0JKRUNUX0lEfSJ9LAogICAgfQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgOCkKXQpQQUdFX1NJWkUgPSAzCgoKY2xhc3MgX01vY2tTdHJpcGVWMihCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KICAgICMgQ2FwdHVyZWQgb24gdGhlIGZpeHR1cmUgc28gbmV4dF9wYWdlX3VybCBjYW4gYmUgYnVpbHQgYXMgYSBmdWxsIGFic29sdXRlIFVSTCwKICAgICMgbWF0Y2hpbmcgU3RyaXBlIHYyIChuZXh0X3BhZ2VfdXJsIGlzIGEgY29tcGxldGUgVVJMLCBub3QgYSBiYXJlIGN1cnNvcikuCiAgICBvcmlnaW4gPSAiIgoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOgogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgdmVyc2lvbiA9IHNlbGYuaGVhZGVycy5nZXQoIlN0cmlwZS1WZXJzaW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJzdHJpcGVfdmVyc2lvbiI6IHZlcnNpb24sCiAgICAgICAgICAgICJoYXNfYmVhcmVyIjogKHNlbGYuaGVhZGVycy5nZXQoIkF1dGhvcml6YXRpb24iKSBvciAiIikuc3RhcnRzd2l0aCgiQmVhcmVyICIpLAogICAgICAgICAgICAicGFnZSI6IHFzLmdldCgicGFnZSIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJzdGFydGluZ19hZnRlciI6IHFzLmdldCgic3RhcnRpbmdfYWZ0ZXIiLCBbTm9uZV0pWzBdLAogICAgICAgIH0pCgogICAgICAgICMgKDEpIGVuZHBvaW50LXZlcnNpb24gYXhpczogb25seSB0aGUgdjIgZXZlbnRzIHBhdGggZXhpc3RzLgogICAgICAgIGlmIHBhdGggIT0gIi92Mi9jb3JlL2V2ZW50cyI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiVW5yZWNvZ25pemVkIHJlcXVlc3QgVVJMLiBTdHJpcGUgQVBJIHYyIGxpc3RzIGV2ZW50cyBhdCAvdjIvY29yZS9ldmVudHMuIn19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMikgdmVyc2lvbiBheGlzOiB2MiByZXF1aXJlcyBhIHdlbGwtZm9ybWVkIFN0cmlwZS1WZXJzaW9uIGhlYWRlciBvbiBFVkVSWSByZXF1ZXN0LgogICAgICAgIGlmIG5vdCB2ZXJzaW9uIG9yIG5vdCBfVkVSU0lPTl9SRS5tYXRjaCh2ZXJzaW9uKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsiZXJyb3IiOiB7InR5cGUiOiAiaW52YWxpZF9yZXF1ZXN0X2Vycm9yIiwKICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJNaXNzaW5nIG9yIG1hbGZvcm1lZCBTdHJpcGUtVmVyc2lvbiBoZWFkZXIuIFRoZSB2MiBBUEkgcmVxdWlyZXMgU3RyaXBlLVZlcnNpb246IDxZWVlZLU1NLURELmNvZGVuYW1lPi4ifX0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgzKSBhdXRoIGF4aXM6IGJlYXJlciBzZWNyZXQga2V5IG9uIEVWRVJZIHJlcXVlc3QgKHNoYXJlZCBzY2hlbWUgd2l0aCB2MSkuCiAgICAgICAgaWYgc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIsICIiKSAhPSBmIkJlYXJlciB7RVhQRUNURURfS0VZfSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiSW52YWxpZCBBUEkga2V5LiBBdXRoZW50aWNhdGUgd2l0aCBBdXRob3JpemF0aW9uOiBCZWFyZXIgPHNlY3JldCBrZXk+LiJ9fSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDQpIHBhZ2luYXRpb24gYXhpczogY3Vyc29yIHJpZGVzIGFuIG9wYXF1ZSBgcGFnZWAgcGFyYW0sIHN1cmZhY2VkIE9OTFkgdmlhIHRoZQogICAgICAgICMgZnVsbC1VUkwgbmV4dF9wYWdlX3VybC4gYHN0YXJ0aW5nX2FmdGVyYCAodGhlIHYxIGN1cnNvcikgaXMgaWdub3JlZCDigJQgYSB2MSBjbGllbnQKICAgICAgICAjIHRoZXJlZm9yZSByZS1yZWFkcyBwYWdlIDEgZm9yZXZlciBvciwgcmVhZGluZyB0aGUgYWJzZW50IGBoYXNfbW9yZWAsIHN0b3BzIGFmdGVyCiAgICAgICAgIyBvbmUgcGFnZS4KICAgICAgICBwYWdlX3BhcmFtID0gcXMuZ2V0KCJwYWdlIiwgW05vbmVdKVswXQogICAgICAgIG9mZnNldCA9IGludChwYWdlX3BhcmFtKSBpZiBwYWdlX3BhcmFtIGlzIG5vdCBOb25lIGFuZCBwYWdlX3BhcmFtLmlzZGlnaXQoKSBlbHNlIDAKCiAgICAgICAgd2luZG93ID0gRVZFTlRTW29mZnNldDpvZmZzZXQgKyBQQUdFX1NJWkVdCiAgICAgICAgbmV4dF9vZmZzZXQgPSBvZmZzZXQgKyBQQUdFX1NJWkUKICAgICAgICBpZiBuZXh0X29mZnNldCA8IGxlbihFVkVOVFMpOgogICAgICAgICAgICBuZXh0X3VybCA9IGYie3R5cGUoc2VsZikub3JpZ2lufS92Mi9jb3JlL2V2ZW50cz9vYmplY3RfaWQ9e09CSkVDVF9JRH0mcGFnZT17bmV4dF9vZmZzZXR9IgogICAgICAgIGVsc2U6CiAgICAgICAgICAgIG5leHRfdXJsID0gTm9uZQogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7CiAgICAgICAgICAgICJkYXRhIjogd2luZG93LAogICAgICAgICAgICAibmV4dF9wYWdlX3VybCI6IG5leHRfdXJsLAogICAgICAgICAgICAicHJldmlvdXNfcGFnZV91cmwiOiBOb25lLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja1N0cmlwZVYyLnJlcXVlc3RfbG9nID0gW10KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tTdHJpcGVWMikKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIF9Nb2NrU3RyaXBlVjIub3JpZ2luID0gZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIF9Nb2NrU3RyaXBlVjIub3JpZ2luCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9ldmVudHNfYWNyb3NzX3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2V2ZW50cyIpLCBcCiAgICAgICAgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IGxpc3RfYWxsX2V2ZW50cyhiYXNlX3VybCwgYXBpX2tleSwgb2JqZWN0X2lkKSIKCiAgICByZXN1bHQgPSBtb2R1bGUubGlzdF9hbGxfZXZlbnRzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9LRVksIE9CSkVDVF9JRCkKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBldmVudHMsIGdvdCB7dHlwZShyZXN1bHQpLl9fbmFtZV9ffSIKICAgIGlkcyA9IHNvcnRlZChlWyJpZCJdIGZvciBlIGluIHJlc3VsdCBpZiBpc2luc3RhbmNlKGUsIGRpY3QpIGFuZCAiaWQiIGluIGUpCiAgICBleHBlY3RlZF9pZHMgPSBzb3J0ZWQoZVsiaWQiXSBmb3IgZSBpbiBFVkVOVFMpCiAgICBhc3NlcnQgaWRzID09IGV4cGVjdGVkX2lkcywgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIGV2ZW50IGlkcyB7aWRzfSwgZXhwZWN0ZWQge2V4cGVjdGVkX2lkc30uIEEgY29ycmVjdCB2MiBjbGllbnQgZm9sbG93cyAiCiAgICAgICAgZiJuZXh0X3BhZ2VfdXJsIChhIGZ1bGwgVVJMKSB0byB0aGUgZW5kOyBpdCBkb2VzIE5PVCB1c2UgdjEgc3RhcnRpbmdfYWZ0ZXIvaGFzX21vcmUuICIKICAgICAgICBmIlNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja1N0cmlwZVYyLnJlcXVlc3RfbG9nfSIKICAgICkK" - }, - { - "vendor": "stripe:get_recent_events", - "fn": "get_recent_events", - "testFile": "test_stripe_get_events.py", - "signature": "def get_recent_events(base_url: str, api_key: str, object_id: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgc3RyaXBlLXYyLWdldC1ldmVudHMgKG1lZGl1bSkgbGVhZi4KClNpbXBsZXIgc2xpY2Ugb2YgdGhlIFNBTUUgU3RyaXBlIEFQSSB2MiBjb250cmFjdCDigJQgYSBzaW5nbGUtcGFnZSBHRVQgb2YgdGhlCnVzYWdlLWJpbGxpbmcgbWV0ZXIgZXZlbnQgc3RyZWFtLCBubyBwYWdpbmF0aW9uLiBTdGlsbCBzZWFyY2gtbmVjZXNzYXJ5IG9uIHR3bwpheGVzOiB0aGUgdjIgZW5kcG9pbnQgbmFtZXNwYWNlIGFuZCB0aGUgTUFOREFUT1JZIFN0cmlwZS1WZXJzaW9uIGhlYWRlciAoYQpmcm9tLW1lbW9yeSB2MSBjbGllbnQgaGl0cyAvdjEvZXZlbnRzIHdpdGggbm8gdmVyc2lvbiBoZWFkZXIgYW5kIGZhaWxzKS4KCiAgLSBlbmRwb2ludCBwYXRoICAgICAvdjIvY29yZS9ldmVudHMgICAgICAgICAgICAgICh2MSB3YXMgL3YxL2V2ZW50cykKICAtIHZlcnNpb24gaGVhZGVyICAgIFN0cmlwZS1WZXJzaW9uOiA8WVlZWS1NTS1ERC5jb2RlbmFtZT4gICAodjEgcmVxdWlyZWQgTk8gdmVyc2lvbiBoZWFkZXIpCiAgLSBhdXRoICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGtleT4gIChzYW1lIGFjcm9zcyB2MS92MikKICAtIHJlc3BvbnNlIGFycmF5ICAgIGRhdGEgICAgICAgICAgICAgICAgICAgICAgICAgIChwYXJzZWQgZnJvbSB0aGUgSlNPTiBib2R5KQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGktdjItb3ZlcnZpZXcKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGkvdjIvY29yZS9ldmVudHMKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGkvdjIvY29yZS9ldmVudHMvbGlzdC5tZAoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgojIFRoZSBiZWFyZXIgc2VjcmV0IGtleSB0aGUgY2xpZW50IG11c3Qgc2VuZCAoc2FtZSBhdXRoIHNjaGVtZSBhcyB2MSkuCkVYUEVDVEVEX0tFWSA9ICJza190ZXN0X3YyX2tleV9hYmMxMjMiCiMgVGhlIG1ldGVyIHdob3NlIGV2ZW50cyB0aGUgY2xpZW50IGZldGNoZXMgKGEgdXNhZ2UtYmlsbGluZyBtZXRlciBpZCkuCk9CSkVDVF9JRCA9ICJtdHJfdGVzdF82MVJDamlxZFREQzkxemdpcDQxSXFQQ3pQbnhxIgojIHYyIG1hbmRhdGVzIGEgU3RyaXBlLVZlcnNpb24gaGVhZGVyIHNoYXBlZCBZWVlZLU1NLUREIHdpdGggYW4gb3B0aW9uYWwgY29kZW5hbWUKIyAoZS5nLiAyMDI2LTA2LTI0LmRhaGxpYSkuIFdlIGFjY2VwdCBhbnkgd2VsbC1mb3JtZWQgZGF0ZStvcHRpb25hbC1jb2RlbmFtZSBzbyBhCiMgY2xpZW50IHRoYXQgcmVhZHMgdGhlIGxpdmUgZG9jcyBhbmQgcGlja3MgYW55IGRvY3VtZW50ZWQgdmVyc2lvbiBwYXNzZXM7IHRoZQojIGRpc2NvdmVyYWJsZSBmYWN0IHVuZGVyIHRlc3QgaXMgdGhhdCB0aGUgaGVhZGVyIG11c3QgYmUgUFJFU0VOVCBhdCBhbGwuCl9WRVJTSU9OX1JFID0gcmUuY29tcGlsZShyIl5cZHs0fS1cZHsyfS1cZHsyfShcLlthLXowLTldKyk/JCIpCgojIFNpbmdsZSBwYWdlIG9mIGJpbGxpbmctbWV0ZXIgZXZlbnRzIGZvciBPQkpFQ1RfSUQuIE9ubHkgb2JzZXJ2YWJsZSBieSBjYWxsaW5nCiMgdGhlIG1vY2s7IHRoZSBhZ2VudCBjYW5ub3QgaGFyZGNvZGUgaXQuCkVWRU5UUyA9IFsKICAgIHsKICAgICAgICAiaWQiOiBmImV2dF90ZXN0X21ldGVyX3tpfSIsCiAgICAgICAgIm9iamVjdCI6ICJ2Mi5jb3JlLmV2ZW50IiwKICAgICAgICAidHlwZSI6ICJ2MS5iaWxsaW5nLm1ldGVyLmVycm9yX3JlcG9ydF90cmlnZ2VyZWQiLAogICAgICAgICJjcmVhdGVkIjogZiIyMDI2LTA2LTJ7aX1UMTc6NDY6MjIuMTM0WiIsCiAgICAgICAgImxpdmVtb2RlIjogRmFsc2UsCiAgICAgICAgInJlbGF0ZWRfb2JqZWN0IjogeyJpZCI6IE9CSkVDVF9JRCwgInR5cGUiOiAiYmlsbGluZy5tZXRlciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICJ1cmwiOiBmIi92MS9iaWxsaW5nL21ldGVycy97T0JKRUNUX0lEfSJ9LAogICAgfQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgNCkKXQoKCmNsYXNzIF9Nb2NrU3RyaXBlVjIoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICB2ZXJzaW9uID0gc2VsZi5oZWFkZXJzLmdldCgiU3RyaXBlLVZlcnNpb24iKQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgInN0cmlwZV92ZXJzaW9uIjogdmVyc2lvbiwKICAgICAgICAgICAgImhhc19iZWFyZXIiOiAoc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpIG9yICIiKS5zdGFydHN3aXRoKCJCZWFyZXIgIiksCiAgICAgICAgICAgICJzdGFydGluZ19hZnRlciI6IHFzLmdldCgic3RhcnRpbmdfYWZ0ZXIiLCBbTm9uZV0pWzBdLAogICAgICAgIH0pCgogICAgICAgICMgKDEpIGVuZHBvaW50LXZlcnNpb24gYXhpczogb25seSB0aGUgdjIgZXZlbnRzIHBhdGggZXhpc3RzLgogICAgICAgIGlmIHBhdGggIT0gIi92Mi9jb3JlL2V2ZW50cyI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiVW5yZWNvZ25pemVkIHJlcXVlc3QgVVJMLiBTdHJpcGUgQVBJIHYyIGxpc3RzIGV2ZW50cyBhdCAvdjIvY29yZS9ldmVudHMuIn19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMikgdmVyc2lvbiBheGlzOiB2MiByZXF1aXJlcyBhIHdlbGwtZm9ybWVkIFN0cmlwZS1WZXJzaW9uIGhlYWRlci4KICAgICAgICBpZiBub3QgdmVyc2lvbiBvciBub3QgX1ZFUlNJT05fUkUubWF0Y2godmVyc2lvbik6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAwLCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiTWlzc2luZyBvciBtYWxmb3JtZWQgU3RyaXBlLVZlcnNpb24gaGVhZGVyLiBUaGUgdjIgQVBJIHJlcXVpcmVzIFN0cmlwZS1WZXJzaW9uOiA8WVlZWS1NTS1ERC5jb2RlbmFtZT4uIn19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMykgYXV0aCBheGlzOiBiZWFyZXIgc2VjcmV0IGtleSAoc2hhcmVkIHdpdGggdjEpLgogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIiwgIiIpCiAgICAgICAgaWYgYXV0aCAhPSBmIkJlYXJlciB7RVhQRUNURURfS0VZfSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiSW52YWxpZCBBUEkga2V5LiBBdXRoZW50aWNhdGUgd2l0aCBBdXRob3JpemF0aW9uOiBCZWFyZXIgPHNlY3JldCBrZXk+LiJ9fSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7CiAgICAgICAgICAgICJkYXRhIjogRVZFTlRTLAogICAgICAgICAgICAibmV4dF9wYWdlX3VybCI6IE5vbmUsCiAgICAgICAgICAgICJwcmV2aW91c19wYWdlX3VybCI6IE5vbmUsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrU3RyaXBlVjIucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1N0cmlwZVYyKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X3JlY2VudF9ldmVudHMobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAiZ2V0X3JlY2VudF9ldmVudHMiKSwgXAogICAgICAgICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfcmVjZW50X2V2ZW50cyhiYXNlX3VybCwgYXBpX2tleSwgb2JqZWN0X2lkKSIKCiAgICBldmVudHMgPSBtb2R1bGUuZ2V0X3JlY2VudF9ldmVudHMobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX0tFWSwgT0JKRUNUX0lEKQoKICAgIGFzc2VydCBpc2luc3RhbmNlKGV2ZW50cywgbGlzdCksIGYiZXhwZWN0ZWQgYSBsaXN0IG9mIGV2ZW50cywgZ290IHt0eXBlKGV2ZW50cykuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGVbImlkIl0gZm9yIGUgaW4gZXZlbnRzIGlmIGlzaW5zdGFuY2UoZSwgZGljdCkgYW5kICJpZCIgaW4gZSkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChlWyJpZCJdIGZvciBlIGluIEVWRU5UUykKICAgIGFzc2VydCBpZHMgPT0gZXhwZWN0ZWRfaWRzLCAoCiAgICAgICAgZiJjbGllbnQgcmV0dXJuZWQgZXZlbnQgaWRzIHtpZHN9LCBleHBlY3RlZCB7ZXhwZWN0ZWRfaWRzfS4gQSBjb3JyZWN0IHYyIGNsaWVudCBHRVRzICIKICAgICAgICBmIi92Mi9jb3JlL2V2ZW50cyB3aXRoIGEgU3RyaXBlLVZlcnNpb24gaGVhZGVyIGFuZCByZWFkcyB0aGUgYGRhdGFgIGFycmF5LiAiCiAgICAgICAgZiJTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tTdHJpcGVWMi5yZXF1ZXN0X2xvZ30iCiAgICApCiAgICBhc3NlcnQgYWxsKGUuZ2V0KCJvYmplY3QiKSA9PSAidjIuY29yZS5ldmVudCIgZm9yIGUgaW4gZXZlbnRzKSwgKAogICAgICAgIGYiZXhwZWN0ZWQgdjIuY29yZS5ldmVudCBvYmplY3RzLiBTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tTdHJpcGVWMi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" - }, - { - "vendor": "novu:list_all_subscribers", - "fn": "list_all_subscribers", - "testFile": "test_novu_list_subscribers.py", - "signature": "def list_all_subscribers(base_url: str, api_key: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgbm92dS12Mi1saXN0LWFsbC1zdWJzY3JpYmVycyAoaGFyZCkgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpJdCBzdGFuZHMgdXAgYSBMT0NBTCBtb2NrIHRoYXQgZmFpdGhmdWxseSBpbXBsZW1lbnRzIE5vdnUncyBDVVJSRU5UIHN1YnNjcmliZXJzLWxpc3QKY29udHJhY3Qgb24gdGhyZWUgYXhlczoKICAtIGVuZHBvaW50IHBhdGggICAvdjIvc3Vic2NyaWJlcnMgICAgICAgICAgICAgICAgKGEgZnJvbS1tZW1vcnkgY2xpZW50IHVzZXMgL3YxL3N1YnNjcmliZXJzKQogIC0gYXV0aCAgICAgICAgICAgIEF1dGhvcml6YXRpb246IEFwaUtleSA8dG9rZW4+ICAgKE5vdnUgZG9lcyBOT1QgdXNlIEJlYXJlcjsgZG9jcyB3YXJuIGFnYWluc3QgaXQpCiAgLSBwYWdpbmF0aW9uICAgICAgY3Vyc29yOiBxdWVyeSBgYWZ0ZXJgICsgcmVzcG9uc2UgYG5leHRgIChvcGFxdWUgY3Vyc29yLCBudWxsID0gZW5kKSwKICAgICAgICAgICAgICAgICAgICBpbnNpZGUgZW52ZWxvcGUge2RhdGEsIG5leHQsIHByZXZpb3VzLCB0b3RhbENvdW50LCB0b3RhbENvdW50Q2FwcGVkfQogICAgICAgICAgICAgICAgICAgIChhIGZyb20tbWVtb3J5IGNsaWVudCB1c2VzIHBhZ2UvbGltaXQgb2Zmc2V0IHBhZ2luYXRpb24gYW5kIGEgYGhhc01vcmVgCiAgICAgICAgICAgICAgICAgICAgIGZsYWcgdGhlIHYyIGVudmVsb3BlIG5ldmVyIHNlbmRzIC0+IHN0b3BzIGFmdGVyIHRoZSBmaXJzdCBwYWdlKQoKQSBjbGllbnQgd3JpdHRlbiBmcm9tIENVUlJFTlQgTm92dSBkb2NzIGNvbGxlY3RzIGFsbCA3IHN1YnNjcmliZXJzIGFjcm9zcyAzIGN1cnNvciBwYWdlcyBhbmQKcGFzc2VzLiBBIGNsaWVudCB3cml0dGVuIGZyb20gc3RhbGUvcHJlLWN1dG9mZiBtZW1vcnkgKHYxIHBhdGgsIEJlYXJlciBhdXRoLCBvciBwYWdlLW51bWJlcgpvZmZzZXQgcGFnaW5hdGlvbikgaGl0cyA0MDQgLyA0MDEgLyBzdG9wcyBhZnRlciBwYWdlIDEgYW5kIEZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Mubm92dS5jby9hcGktcmVmZXJlbmNlL2F1dGhlbnRpY2F0aW9uCiAgaHR0cHM6Ly9kb2NzLm5vdnUuY28vYXBpLXJlZmVyZW5jZS9zdWJzY3JpYmVycy9zZWFyY2gtc3Vic2NyaWJlcnMKIiIiCgppbXBvcnQgYmFzZTY0CmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX1RPS0VOID0gIm52X3NlY3JldF9rZXlfYWJjMTIzIgoKIyBUaGUgaGlkZGVuIGRhdGFzZXQuIFRoZSBhZ2VudCBjYW5ub3QgaGFyZGNvZGUgdGhpcyDigJQgaXQgaXMgb25seSBvYnNlcnZhYmxlIGJ5IGFjdHVhbGx5CiMgY2FsbGluZyB0aGUgbW9jayBhbmQgZm9sbG93aW5nIHRoZSBjdXJzb3IgdG8gdGhlIGVuZC4gNyBzdWJzY3JpYmVycywgc2VydmVyIHBhZ2Ugc2l6ZSAzLgpTVUJTQ1JJQkVSUyA9IFsKICAgIHsiX2lkIjogZiJfaWRfe2l9IiwgInN1YnNjcmliZXJJZCI6IGYidXNlcl97aX0iLCAiZW1haWwiOiBmInVzZXJ7aX1AZXhhbXBsZS5jb20ifQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgOCkKXQpQQUdFX1NJWkUgPSAzCgoKZGVmIF9lbmNvZGVfY3Vyc29yKG9mZnNldDogaW50KSAtPiBzdHI6CiAgICAjIE9wYXF1ZSBjdXJzb3Igc3RyaW5nLCBtYXRjaGluZyBOb3Z1J3Mgb3BhcXVlLWN1cnNvciBjb250cmFjdC4KICAgIHJldHVybiBiYXNlNjQudXJsc2FmZV9iNjRlbmNvZGUoc3RyKG9mZnNldCkuZW5jb2RlKCkpLmRlY29kZSgpCgoKZGVmIF9kZWNvZGVfY3Vyc29yKGN1cnNvcjogc3RyKSAtPiBpbnQ6CiAgICByZXR1cm4gaW50KGJhc2U2NC51cmxzYWZlX2I2NGRlY29kZShjdXJzb3IuZW5jb2RlKCkpLmRlY29kZSgpKQoKCmNsYXNzIF9Nb2NrTm92dShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImFwaWtleV9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkFwaUtleSAiKSwKICAgICAgICAgICAgImJlYXJlcl9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkJlYXJlciAiKSwKICAgICAgICAgICAgImFmdGVyIjogcXMuZ2V0KCJhZnRlciIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJwYWdlIjogcXMuZ2V0KCJwYWdlIiwgW05vbmVdKVswXSwKICAgICAgICB9KQoKICAgICAgICAjICgxKSBlbmRwb2ludC12ZXJzaW9uIGF4aXM6IG9ubHkgdGhlIHYyIHBhdGggZXhpc3RzLgogICAgICAgIGlmIHBhdGggIT0gIi92Mi9zdWJzY3JpYmVycyI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN0YXR1c0NvZGUiOiA0MDQsICJtZXNzYWdlIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yIjogIk5vdnUgQVBJIHYyIGxpc3RzIHN1YnNjcmliZXJzIGF0IC92Mi9zdWJzY3JpYmVycyJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMikgYXV0aCBheGlzOiB2MiByZXF1aXJlcyBgQXV0aG9yaXphdGlvbjogQXBpS2V5IDx0b2tlbj5gOyBCZWFyZXIgaXMgbm90IGhvbm9yZWQuCiAgICAgICAgaWYgYXV0aCAhPSBmIkFwaUtleSB7RVhQRUNURURfVE9LRU59IjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsic3RhdHVzQ29kZSI6IDQwMSwgIm1lc3NhZ2UiOiAidW5hdXRob3JpemVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3IiOiAiTm92dSBhdXRoZW50aWNhdGVzIHZpYSBgQXV0aG9yaXphdGlvbjogQXBpS2V5IDxzZWNyZXQ+YCAobm90IEJlYXJlcikifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDMpIHBhZ2luYXRpb24gYXhpczogY3Vyc29yLWJhc2VkIHZpYSBgYWZ0ZXJgLiBwYWdlLW51bWJlciBvZmZzZXQgaXMgaWdub3JlZCAodjIgaGFzIG5vIGBwYWdlYCkuCiAgICAgICAgYWZ0ZXIgPSBxcy5nZXQoImFmdGVyIiwgW05vbmVdKVswXQogICAgICAgIGlmIGFmdGVyIGlzIE5vbmU6CiAgICAgICAgICAgIG9mZnNldCA9IDAKICAgICAgICBlbHNlOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBvZmZzZXQgPSBfZGVjb2RlX2N1cnNvcihhZnRlcikKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAgICAgICAgIHNlbGYuX2pzb24oNDAwLCB7InN0YXR1c0NvZGUiOiA0MDAsICJtZXNzYWdlIjogImludmFsaWQgY3Vyc29yIn0pCiAgICAgICAgICAgICAgICByZXR1cm4KCiAgICAgICAgcGFnZSA9IFNVQlNDUklCRVJTW29mZnNldDpvZmZzZXQgKyBQQUdFX1NJWkVdCiAgICAgICAgbmV4dF9vZmZzZXQgPSBvZmZzZXQgKyBQQUdFX1NJWkUKICAgICAgICBuZXh0X2N1cnNvciA9IF9lbmNvZGVfY3Vyc29yKG5leHRfb2Zmc2V0KSBpZiBuZXh0X29mZnNldCA8IGxlbihTVUJTQ1JJQkVSUykgZWxzZSBOb25lCiAgICAgICAgcHJldl9jdXJzb3IgPSBfZW5jb2RlX2N1cnNvcihtYXgob2Zmc2V0IC0gUEFHRV9TSVpFLCAwKSkgaWYgb2Zmc2V0ID4gMCBlbHNlIE5vbmUKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAiZGF0YSI6IHBhZ2UsCiAgICAgICAgICAgICJuZXh0IjogbmV4dF9jdXJzb3IsCiAgICAgICAgICAgICJwcmV2aW91cyI6IHByZXZfY3Vyc29yLAogICAgICAgICAgICAidG90YWxDb3VudCI6IGxlbihTVUJTQ1JJQkVSUyksCiAgICAgICAgICAgICJ0b3RhbENvdW50Q2FwcGVkIjogRmFsc2UsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrTm92dS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrTm92dSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9zdWJzY3JpYmVyc19hY3Jvc3NfY3Vyc29yX3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX3N1YnNjcmliZXJzIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBsaXN0X2FsbF9zdWJzY3JpYmVycyhiYXNlX3VybCwgYXBpX2tleSkiCgogICAgcmVzdWx0ID0gbW9kdWxlLmxpc3RfYWxsX3N1YnNjcmliZXJzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBzdWJzY3JpYmVycywgZ290IHt0eXBlKHJlc3VsdCkuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKHNbInN1YnNjcmliZXJJZCJdIGZvciBzIGluIHJlc3VsdCBpZiBpc2luc3RhbmNlKHMsIGRpY3QpIGFuZCAic3Vic2NyaWJlcklkIiBpbiBzKQogICAgZXhwZWN0ZWRfaWRzID0gc29ydGVkKHNbInN1YnNjcmliZXJJZCJdIGZvciBzIGluIFNVQlNDUklCRVJTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBzdWJzY3JpYmVyIGlkcyB7aWRzfSwgZXhwZWN0ZWQge2V4cGVjdGVkX2lkc30uICIKICAgICAgICBmIkEgY29ycmVjdCB2MiBjbGllbnQgZm9sbG93cyB0aGUgYG5leHRgIGN1cnNvciB0byB0aGUgZW5kLiAiCiAgICAgICAgZiJTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tOb3Z1LnJlcXVlc3RfbG9nfSIKICAgICkK" - }, - { - "vendor": "novu:get_subscriber", - "fn": "get_subscriber", - "testFile": "test_novu_get_subscriber.py", - "signature": "def get_subscriber(base_url: str, api_key: str, subscriber_id: str) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgbm92dS12Mi1nZXQtc3Vic2NyaWJlciAobWVkaXVtKSBsZWFmLgoKU2ltcGxlciBzbGljZSBvZiB0aGUgU0FNRSBOb3Z1IEFQSSBjb250cmFjdCDigJQgYSBzaW5nbGUtcmVzb3VyY2UgR0VULCBubyBwYWdpbmF0aW9uLgpTdGlsbCBzZWFyY2gtbmVjZXNzYXJ5IG9uIHR3byBheGVzOiB0aGUgdjIgZW5kcG9pbnQgcGF0aCBhbmQgdGhlIE5PTi1TVEFOREFSRCBBcGlLZXkKYXV0aCBzY2hlbWUgKGEgZnJvbS1tZW1vcnkgY2xpZW50IHVzZXMgQmVhcmVyIGF1dGggYW5kL29yIHRoZSBvbGRlciAvdjEgcGF0aCBhbmQgZmFpbHMpLgoKICAtIGVuZHBvaW50IHBhdGggICAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0gICAoYSBmcm9tLW1lbW9yeSBjbGllbnQgdXNlcyAvdjEvc3Vic2NyaWJlcnMve2lkfSkKICAtIGF1dGggICAgICAgICAgICBBdXRob3JpemF0aW9uOiBBcGlLZXkgPHRva2VuPiAgICAgKE5vdnUgZG9lcyBOT1QgdXNlIEJlYXJlcjsgdGhlIGRvY3MKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBleHBsaWNpdGx5IHdhcm4gYWdhaW5zdCB0aGUgQmVhcmVyIHByZWZpeCkKICAtIHJlc3BvbnNlIHNoYXBlICB0aGUgc3Vic2NyaWJlciBvYmplY3QgaXMgcmV0dXJuZWQgRElSRUNUTFkgKG5vIGBkYXRhYCB3cmFwcGVyKQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Mubm92dS5jby9hcGktcmVmZXJlbmNlL2F1dGhlbnRpY2F0aW9uCiAgaHR0cHM6Ly9kb2NzLm5vdnUuY28vYXBpLXJlZmVyZW5jZS9zdWJzY3JpYmVycy9yZXRyaWV2ZS1hLXN1YnNjcmliZXIKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgc2VjcmV0IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBBdXRob3JpemF0aW9uIGhlYWRlciwgcHJlZml4ZWQgd2l0aCBgQXBpS2V5YC4KRVhQRUNURURfVE9LRU4gPSAibnZfc2VjcmV0X2tleV9hYmMxMjMiClNVQlNDUklCRVJfSUQgPSAidXNlcl80MiIgICMgdGhlIGlkIHRoZSB0ZXN0IGZldGNoZXM7IHRoZSBtb2NrIHN5bnRoZXNpemVzIGFueSB2YWxpZCBpZAoKCmNsYXNzIF9Nb2NrTm92dShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImFwaWtleV9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkFwaUtleSAiKSwKICAgICAgICAgICAgImJlYXJlcl9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkJlYXJlciAiKSwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogdjIgZmV0Y2hlcyBvbmUgc3Vic2NyaWJlciBhdCAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi92Mi9zdWJzY3JpYmVycy8oW14vXSspIiwgcGF0aCkKICAgICAgICBpZiBub3QgbToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsic3RhdHVzQ29kZSI6IDQwNCwgIm1lc3NhZ2UiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3IiOiAiTm92dSBBUEkgdjIgcmV0cmlldmVzIGEgc3Vic2NyaWJlciBhdCAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiBOb3Z1IGF1dGhlbnRpY2F0ZXMgd2l0aCBgQXV0aG9yaXphdGlvbjogQXBpS2V5IDx0b2tlbj5gIChOT1QgQmVhcmVyKS4KICAgICAgICBpZiBhdXRoICE9IGYiQXBpS2V5IHtFWFBFQ1RFRF9UT0tFTn0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJzdGF0dXNDb2RlIjogNDAxLCAibWVzc2FnZSI6ICJ1bmF1dGhvcml6ZWQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6ICJOb3Z1IGF1dGhlbnRpY2F0ZXMgdmlhIGBBdXRob3JpemF0aW9uOiBBcGlLZXkgPHNlY3JldD5gIChub3QgQmVhcmVyKSJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc3Vic2NyaWJlcl9pZCA9IG0uZ3JvdXAoMSkKICAgICAgICAjIHJlc3BvbnNlLXNoYXBlIGF4aXM6IHRoZSBzdWJzY3JpYmVyIG9iamVjdCBpcyByZXR1cm5lZCBESVJFQ1RMWSAobm8gYGRhdGFgIHdyYXBwZXIpLgogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7CiAgICAgICAgICAgICJfaWQiOiBmIl9pZF97c3Vic2NyaWJlcl9pZH0iLAogICAgICAgICAgICAic3Vic2NyaWJlcklkIjogc3Vic2NyaWJlcl9pZCwKICAgICAgICAgICAgImZpcnN0TmFtZSI6ICJBZGEiLAogICAgICAgICAgICAibGFzdE5hbWUiOiAiTG92ZWxhY2UiLAogICAgICAgICAgICAiZW1haWwiOiBmIntzdWJzY3JpYmVyX2lkfUBleGFtcGxlLmNvbSIsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrTm92dS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrTm92dSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9zdWJzY3JpYmVyX2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9zdWJzY3JpYmVyIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfc3Vic2NyaWJlcihiYXNlX3VybCwgYXBpX2tleSwgc3Vic2NyaWJlcl9pZCkiCgogICAgc3Vic2NyaWJlciA9IG1vZHVsZS5nZXRfc3Vic2NyaWJlcihtb2NrX3NlcnZlciwgRVhQRUNURURfVE9LRU4sIFNVQlNDUklCRVJfSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2Uoc3Vic2NyaWJlciwgZGljdCksIGYiZXhwZWN0ZWQgYSBzdWJzY3JpYmVyIGRpY3QsIGdvdCB7dHlwZShzdWJzY3JpYmVyKS5fX25hbWVfX30iCiAgICBhc3NlcnQgc3Vic2NyaWJlci5nZXQoInN1YnNjcmliZXJJZCIpID09IFNVQlNDUklCRVJfSUQgYW5kIHN1YnNjcmliZXIuZ2V0KCJfaWQiKSA9PSBmIl9pZF97U1VCU0NSSUJFUl9JRH0iLCAoCiAgICAgICAgZiJjbGllbnQgcmV0dXJuZWQge3N1YnNjcmliZXIhcn07IGEgY29ycmVjdCB2MiBjbGllbnQgR0VUcyAvdjIvc3Vic2NyaWJlcnMve1NVQlNDUklCRVJfSUR9IHdpdGggdGhlICIKICAgICAgICBmImBBdXRob3JpemF0aW9uOiBBcGlLZXlgIGhlYWRlciBhbmQgcmV0dXJucyB0aGUgc3Vic2NyaWJlciBvYmplY3QuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja05vdnUucmVxdWVzdF9sb2d9IgogICAgKQo=" - }, - { - "vendor": "honeycomb:run_query", - "fn": "run_query", - "testFile": "test_honeycomb_run_query.py", - "signature": "def run_query(base_url: str, api_key: str, dataset_slug: str, query: dict) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgaG9uZXljb21iLXJ1bi1xdWVyeSAoaGFyZCkgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpJdCBzdGFuZHMgdXAgYSBMT0NBTCBtb2NrIHRoYXQgZmFpdGhmdWxseSBpbXBsZW1lbnRzIEhvbmV5Y29tYidzIENVUlJFTlQgKEFQSSB2MSkgUXVlcnkgRGF0YQpjb250cmFjdCBvbiBUSFJFRSBzZWFyY2gtbmVjZXNzYXJ5IGF4ZXM6CiAgLSBhdXRoICAgICAgICBYLUhvbmV5Y29tYi1UZWFtOiA8a2V5PiAgICAgICAgICAoY3VzdG9tIGhlYWRlciwgTk8gQXV0aG9yaXphdGlvbjogQmVhcmVyKQogIC0gZW5kcG9pbnQgICAgLzEvcXVlcmllcyArIC8xL3F1ZXJ5X3Jlc3VsdHMgICAgKHZlcnNpb24gcHJlZml4IGlzIHRoZSBsaXRlcmFsICIxIikKICAtIGZsb3cgICAgICAgIGFzeW5jIFRIUkVFLVNURVAgKyBwb2xsICAgICAgICAgIChjcmVhdGUgYSBRdWVyeSwgY3JlYXRlIGEgUXVlcnkgUmVzdWx0IHRoYXQKICAgICAgICAgICAgICAgIHJlZmVyZW5jZXMgdGhlIHF1ZXJ5IGlkLCB0aGVuIEdFVC1wb2xsIHRoZSBxdWVyeS1yZXN1bHQgZW5kcG9pbnQgdW50aWwKICAgICAgICAgICAgICAgIGNvbXBsZXRlID09IHRydWUgYW5kIHJlYWQgZGF0YS5yZXN1bHRzKQoKR2V0dGluZyByZXN1bHRzIGlzIE5PVCBhIHNpbmdsZSBzeW5jaHJvbm91cyBjYWxsLiBBIGZyb20tbWVtb3J5IGNsaWVudCB0aGF0IFBPU1RzIGEgcXVlcnkgYW5kCmV4cGVjdHMgcm93cyBiYWNrIGluIHRoZSByZXNwb25zZSwgb3IgcmVhZHMgcmVzdWx0cyBiZWZvcmUgY29tcGxldGUgPT0gdHJ1ZSwgb3IgdXNlcyBCZWFyZXIKYXV0aCwgRkFJTFMuIEFuIGVtcHR5IHNvbHV0aW9uIEZBSUxTLiBPbmx5IGEgY2xpZW50IGJ1aWx0IGZyb20gdGhlIGN1cnJlbnQgSG9uZXljb21iIFF1ZXJ5IERhdGEKZG9jcyBwYXNzZXMuIFRoZSByZXN1bHQgcm93cyBhcmUgb25seSBvYnNlcnZhYmxlIGJ5IHJ1bm5pbmcgdGhlIGZ1bGwgZmxvdywgc28gdGhlIGFuc3dlciBjYW5ub3QKYmUgaGFyZGNvZGVkLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3MuaG9uZXljb21iLmlvL2FwaS9hdXRoCiAgaHR0cHM6Ly9hcGktZG9jcy5ob25leWNvbWIuaW8vYXBpL3F1ZXJ5LWRhdGEKICBodHRwczovL2FwaS1kb2NzLmhvbmV5Y29tYi5pby9hcGkvcXVlcmllcy9jcmVhdGVxdWVyeQoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX0tFWSA9ICJoY2FpY19ob25leWNvbWJfY29uZmlnX2tleV9hYmMxMjMiCkRBVEFTRVRfU0xVRyA9ICJjaGVja291dC1zZXJ2aWNlIgoKIyBUaGUgaGlkZGVuIHJlc3VsdCByb3dzLiBIb25leWNvbWIgcmV0dXJucyBxdWVyeSByZXN1bHRzIGFzIGEgbGlzdCBvZiB7ImRhdGEiOiB7Li4ufX0gb2JqZWN0cy4KIyBPbmx5IG9ic2VydmFibGUgYnkgcnVubmluZyB0aGUgcXVlcnkgYW5kIHBvbGxpbmcgdG8gY29tcGxldGlvbjsgY2Fubm90IGJlIGhhcmRjb2RlZC4KRVhQRUNURURfUkVTVUxUUyA9IFsKICAgIHsiZGF0YSI6IHsic2VydmljZSI6ICJhcGkiLCAiQ09VTlQiOiA0MjEwfX0sCiAgICB7ImRhdGEiOiB7InNlcnZpY2UiOiAid2ViIiwgIkNPVU5UIjogMTg3NX19LAogICAgeyJkYXRhIjogeyJzZXJ2aWNlIjogIndvcmtlciIsICJDT1VOVCI6IDkzMn19LApdCiMgVGhlIHF1ZXJ5IHJlc3VsdCBiZWNvbWVzIGNvbXBsZXRlIG9ubHkgb24gdGhlIDJuZCBwb2xsLCBzbyBhIGNvcnJlY3QgY2xpZW50IE1VU1QgbG9vcC4KUE9MTFNfVU5USUxfQ09NUExFVEUgPSAyCgoKY2xhc3MgX01vY2tIb25leWNvbWIoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCiAgICBfcXVlcnlfaWRzID0gc2V0KCkKICAgIF9yZXN1bHRfcG9sbHMgPSB7fQogICAgX2NvdW50ZXIgPSB7InEiOiAwLCAiciI6IDB9CgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBfcmVhZF9qc29uX2JvZHkoc2VsZik6CiAgICAgICAgbGVuZ3RoID0gaW50KHNlbGYuaGVhZGVycy5nZXQoIkNvbnRlbnQtTGVuZ3RoIikgb3IgMCkKICAgICAgICBpZiBsZW5ndGggPD0gMDoKICAgICAgICAgICAgcmV0dXJuIE5vbmUKICAgICAgICB0cnk6CiAgICAgICAgICAgIHJldHVybiBqc29uLmxvYWRzKHNlbGYucmZpbGUucmVhZChsZW5ndGgpLmRlY29kZSgpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb246CiAgICAgICAgICAgIHJldHVybiBOb25lCgogICAgZGVmIF9hdXRoZWQoc2VsZik6CiAgICAgICAgcmV0dXJuIHNlbGYuaGVhZGVycy5nZXQoIlgtSG9uZXljb21iLVRlYW0iKSA9PSBFWFBFQ1RFRF9LRVkKCiAgICBkZWYgX2xvZyhzZWxmLCBtZXRob2QsIHBhdGgpOgogICAgICAgIGhlYWRlcnNfbG93ZXIgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAibWV0aG9kIjogbWV0aG9kLAogICAgICAgICAgICAicGF0aCI6IHBhdGgsCiAgICAgICAgICAgICJoYXNfdGVhbV9oZWFkZXIiOiAieC1ob25leWNvbWItdGVhbSIgaW4gaGVhZGVyc19sb3dlciwKICAgICAgICAgICAgImhhc19iZWFyZXIiOiAiYXV0aG9yaXphdGlvbiIgaW4gaGVhZGVyc19sb3dlciwKICAgICAgICB9KQoKICAgIGRlZiBkb19QT1NUKHNlbGYpOgogICAgICAgIHBhdGggPSB1cmxwYXJzZShzZWxmLnBhdGgpLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgYm9keSA9IHNlbGYuX3JlYWRfanNvbl9ib2R5KCkKICAgICAgICBzZWxmLl9sb2coIlBPU1QiLCBwYXRoKQoKICAgICAgICBpZiBub3Qgc2VsZi5fYXV0aGVkKCk6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiSG9uZXljb21iIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUhvbmV5Y29tYi1UZWFtIGhlYWRlciJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyBTdGVwIDEg4oCUIGNyZWF0ZSBhIFF1ZXJ5ICh2YWxpZGF0ZXMgdGhlIHNwZWMsIHJldHVybnMgYW4gaWQ7IGRvZXMgTk9UIHJ1biBpdCkuCiAgICAgICAgbV9xID0gcmUuZnVsbG1hdGNoKHIiLzEvcXVlcmllcy8oW14vXSspIiwgcGF0aCkKICAgICAgICBpZiBtX3E6CiAgICAgICAgICAgIGlmIG5vdCBpc2luc3RhbmNlKGJvZHksIGRpY3QpIG9yICJjYWxjdWxhdGlvbnMiIG5vdCBpbiBib2R5OgogICAgICAgICAgICAgICAgc2VsZi5fanNvbig0MjIsIHsiZXJyb3IiOiAidW5wcm9jZXNzYWJsZSIsICJlcnJvcl9pbmZvIjogInF1ZXJ5IHNwZWMgcmVxdWlyZXMgY2FsY3VsYXRpb25zIn0pCiAgICAgICAgICAgICAgICByZXR1cm4KICAgICAgICAgICAgdHlwZShzZWxmKS5fY291bnRlclsicSJdICs9IDEKICAgICAgICAgICAgcWlkID0gZiJxLXt0eXBlKHNlbGYpLl9jb3VudGVyWydxJ119IgogICAgICAgICAgICB0eXBlKHNlbGYpLl9xdWVyeV9pZHMuYWRkKHFpZCkKICAgICAgICAgICAgc2VsZi5fanNvbigyMDEsIHsiaWQiOiBxaWQsICJkYXRhc2V0IjogbV9xLmdyb3VwKDEpfSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgU3RlcCAyIOKAlCBjcmVhdGUgYSBRdWVyeSBSZXN1bHQgdGhhdCByZWZlcmVuY2VzIHRoZSBxdWVyeSBpZCAocnVucyBpdCBhc3luY2hyb25vdXNseSkuCiAgICAgICAgbV9yID0gcmUuZnVsbG1hdGNoKHIiLzEvcXVlcnlfcmVzdWx0cy8oW14vXSspIiwgcGF0aCkKICAgICAgICBpZiBtX3I6CiAgICAgICAgICAgIHFpZCA9IGJvZHkuZ2V0KCJxdWVyeV9pZCIpIGlmIGlzaW5zdGFuY2UoYm9keSwgZGljdCkgZWxzZSBOb25lCiAgICAgICAgICAgIGlmIHFpZCBub3QgaW4gdHlwZShzZWxmKS5fcXVlcnlfaWRzOgogICAgICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiZXJyb3IiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiY3JlYXRlIGEgUXVlcnkgZmlyc3QsIHRoZW4gcmVmZXJlbmNlIGl0cyBpZCBhcyBxdWVyeV9pZCJ9KQogICAgICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgICAgIHR5cGUoc2VsZikuX2NvdW50ZXJbInIiXSArPSAxCiAgICAgICAgICAgIHJpZCA9IGYici17dHlwZShzZWxmKS5fY291bnRlclsnciddfSIKICAgICAgICAgICAgdHlwZShzZWxmKS5fcmVzdWx0X3BvbGxzW3JpZF0gPSAwCiAgICAgICAgICAgIHNlbGYuX2pzb24oMjAxLCB7ImlkIjogcmlkLCAiY29tcGxldGUiOiBGYWxzZX0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJlcnJvciI6ICJub3QgZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiSG9uZXljb21iIHYxIHF1ZXJ5IGRhdGEgbGl2ZXMgYXQgLzEvcXVlcmllcyBhbmQgLzEvcXVlcnlfcmVzdWx0cyJ9KQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGF0aCA9IHVybHBhcnNlKHNlbGYucGF0aCkucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBzZWxmLl9sb2coIkdFVCIsIHBhdGgpCgogICAgICAgIGlmIG5vdCBzZWxmLl9hdXRoZWQoKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsiZXJyb3IiOiAidW5hdXRob3JpemVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3JfaW5mbyI6ICJIb25leWNvbWIgYXV0aGVudGljYXRlcyB2aWEgdGhlIFgtSG9uZXljb21iLVRlYW0gaGVhZGVyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjIFN0ZXAgMyDigJQgcG9sbCB0aGUgcXVlcnkgcmVzdWx0IHVudGlsIGNvbXBsZXRlID09IHRydWUsIHRoZW4gcmVhZCBkYXRhLnJlc3VsdHMuCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi8xL3F1ZXJ5X3Jlc3VsdHMvKFteL10rKS8oW14vXSspIiwgcGF0aCkKICAgICAgICBpZiBub3QgbToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiZXJyb3IiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3JfaW5mbyI6ICJwb2xsIGEgcXVlcnkgcmVzdWx0IGF0IC8xL3F1ZXJ5X3Jlc3VsdHMve2RhdGFzZXR9L3tyZXN1bHRfaWR9In0pCiAgICAgICAgICAgIHJldHVybgogICAgICAgIHJpZCA9IG0uZ3JvdXAoMikKICAgICAgICBpZiByaWQgbm90IGluIHR5cGUoc2VsZikuX3Jlc3VsdF9wb2xsczoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiZXJyb3IiOiAibm90IGZvdW5kIiwgImVycm9yX2luZm8iOiAidW5rbm93biBxdWVyeSByZXN1bHQgaWQifSkKICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgdHlwZShzZWxmKS5fcmVzdWx0X3BvbGxzW3JpZF0gKz0gMQogICAgICAgIGlmIHR5cGUoc2VsZikuX3Jlc3VsdF9wb2xsc1tyaWRdIDwgUE9MTFNfVU5USUxfQ09NUExFVEU6CiAgICAgICAgICAgICMgTm90IGRvbmUgeWV0OiBubyBkYXRhIGZpZWxkIHVudGlsIGNvbXBsZXRlIChtYXRjaGVzIHRoZSByZWFsIGFzeW5jIGNvbnRyYWN0KS4KICAgICAgICAgICAgc2VsZi5fanNvbigyMDAsIHsiaWQiOiByaWQsICJjb21wbGV0ZSI6IEZhbHNlfSkKICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgImlkIjogcmlkLAogICAgICAgICAgICAiY29tcGxldGUiOiBUcnVlLAogICAgICAgICAgICAiZGF0YSI6IHsic2VyaWVzIjogW10sICJyZXN1bHRzIjogRVhQRUNURURfUkVTVUxUU30sCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrSG9uZXljb21iLnJlcXVlc3RfbG9nID0gW10KICAgIF9Nb2NrSG9uZXljb21iLl9xdWVyeV9pZHMgPSBzZXQoKQogICAgX01vY2tIb25leWNvbWIuX3Jlc3VsdF9wb2xscyA9IHt9CiAgICBfTW9ja0hvbmV5Y29tYi5fY291bnRlciA9IHsicSI6IDAsICJyIjogMH0KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tIb25leWNvbWIpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpLnN0YXJ0KCkKICAgIHRyeToKICAgICAgICB5aWVsZCBmImh0dHA6Ly8xMjcuMC4wLjE6e3BvcnR9IgogICAgZmluYWxseToKICAgICAgICBzZXJ2ZXIuc2h1dGRvd24oKQogICAgICAgIHNlcnZlci5zZXJ2ZXJfY2xvc2UoKQoKCmRlZiBfbG9hZF9hZ2VudF9zb2x1dGlvbigpOgogICAgcm9vdCA9IG9zLmVudmlyb24uZ2V0KCJWQl9TT0xVVElPTl9ST09UIiwgb3MuZ2V0Y3dkKCkpCiAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4ocm9vdCwgInNvbHV0aW9uIiwgInNvbHV0aW9uLnB5IikKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIGZvciBkaXJwYXRoLCBfZGlycywgZmlsZXMgaW4gb3Mud2Fsayhyb290KToKICAgICAgICAgICAgaWYgInNvbHV0aW9uLnB5IiBpbiBmaWxlczoKICAgICAgICAgICAgICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihkaXJwYXRoLCAic29sdXRpb24ucHkiKQogICAgICAgICAgICAgICAgYnJlYWsKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIHB5dGVzdC5mYWlsKGYiYWdlbnQgc29sdXRpb24gbm90IGZvdW5kOiBleHBlY3RlZCBzb2x1dGlvbi9zb2x1dGlvbi5weSB1bmRlciB7cm9vdH0iKQogICAgc3BlYyA9IGltcG9ydGxpYi51dGlsLnNwZWNfZnJvbV9maWxlX2xvY2F0aW9uKCJhZ2VudF9zb2x1dGlvbiIsIGNhbmRpZGF0ZSkKICAgIG1vZHVsZSA9IGltcG9ydGxpYi51dGlsLm1vZHVsZV9mcm9tX3NwZWMoc3BlYykKICAgIHNwZWMubG9hZGVyLmV4ZWNfbW9kdWxlKG1vZHVsZSkKICAgIHJldHVybiBtb2R1bGUKCgpkZWYgdGVzdF9ydW5fcXVlcnlfYXN5bmNfZmxvdyhtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJydW5fcXVlcnkiKSwgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IHJ1bl9xdWVyeShiYXNlX3VybCwgYXBpX2tleSwgZGF0YXNldF9zbHVnLCBxdWVyeSkiCgogICAgcXVlcnkgPSB7ImNhbGN1bGF0aW9ucyI6IFt7Im9wIjogIkNPVU5UIn1dLCAiYnJlYWtkb3ducyI6IFsic2VydmljZSJdLCAidGltZV9yYW5nZSI6IDcyMDB9CiAgICByZXN1bHRzID0gbW9kdWxlLnJ1bl9xdWVyeShtb2NrX3NlcnZlciwgRVhQRUNURURfS0VZLCBEQVRBU0VUX1NMVUcsIHF1ZXJ5KQoKICAgIGFzc2VydCBpc2luc3RhbmNlKHJlc3VsdHMsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiByZXN1bHQgcm93cywgZ290IHt0eXBlKHJlc3VsdHMpLl9fbmFtZV9ffSIKICAgIGFzc2VydCByZXN1bHRzID09IEVYUEVDVEVEX1JFU1VMVFMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCB7cmVzdWx0cyFyfSwgZXhwZWN0ZWQge0VYUEVDVEVEX1JFU1VMVFMhcn0uIEEgY29ycmVjdCBjbGllbnQgcnVucyB0aGUgIgogICAgICAgIGYidGhyZWUtc3RlcCBhc3luYyBmbG93IChQT1NUIC8xL3F1ZXJpZXMgLT4gUE9TVCAvMS9xdWVyeV9yZXN1bHRzIC0+IHBvbGwgR0VUICIKICAgICAgICBmIi8xL3F1ZXJ5X3Jlc3VsdHMve3tkYXRhc2V0fX0ve3tpZH19IHVudGlsIGNvbXBsZXRlKSBhbmQgcmV0dXJucyBkYXRhLnJlc3VsdHMuICIKICAgICAgICBmIlNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0hvbmV5Y29tYi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" - }, - { - "vendor": "honeycomb:get_dataset", - "fn": "get_dataset", - "testFile": "test_honeycomb_get_dataset.py", - "signature": "def get_dataset(base_url: str, api_key: str, dataset_slug: str) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgaG9uZXljb21iLWdldC1kYXRhc2V0IChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBIb25leWNvbWIncyBDVVJSRU5UIChBUEkgdjEpIHNpbmdsZS0KZGF0YXNldCBjb250cmFjdCBvbiB0d28gc2VhcmNoLW5lY2Vzc2FyeSBheGVzOgogIC0gYXV0aCAgICAgIFgtSG9uZXljb21iLVRlYW06IDxrZXk+ICAgKGN1c3RvbSBoZWFkZXIsIE5PIEF1dGhvcml6YXRpb246IEJlYXJlciBzY2hlbWUpCiAgLSBlbmRwb2ludCAgLzEvZGF0YXNldHMve3NsdWd9ICAgICAgICAodGhlIEFQSSB2ZXJzaW9uIHByZWZpeCBpcyB0aGUgbGl0ZXJhbCAiMSIpCgpBIGZyb20tbWVtb3J5IGNsaWVudCB0aGF0IHJlYWNoZXMgZm9yIEF1dGhvcml6YXRpb246IEJlYXJlciAodGhlIG5lYXItdW5pdmVyc2FsIGRlZmF1bHQpIG9yCmd1ZXNzZXMgL3YxfC9hcGkvdjF8L2FwaS8xIGhpdHMgNDAxIC8gNDA0IGFuZCBGQUlMUy4gQW4gZW1wdHkgc29sdXRpb24gRkFJTFMuIE9ubHkgYSBjbGllbnQKYnVpbHQgZnJvbSB0aGUgY3VycmVudCBIb25leWNvbWIgZG9jcyAoY3VzdG9tIGhlYWRlciArIC8xLyBwcmVmaXgpIFBBU1NFUy4gVGhlIGRhdGFzZXQgbmFtZSBpcwpzbHVnLWRlcml2ZWQgYW5kIG9ubHkgb2JzZXJ2YWJsZSBieSBhY3R1YWxseSBjYWxsaW5nIHRoZSBtb2NrLCBzbyB0aGUgYW5zd2VyIGNhbm5vdCBiZSBoYXJkY29kZWQuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZG9jcy5ob25leWNvbWIuaW8vYXBpL2F1dGgKICBodHRwczovL2FwaS1kb2NzLmhvbmV5Y29tYi5pby9hcGkvZGF0YXNldHMKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUga2V5IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBYLUhvbmV5Y29tYi1UZWFtIEhFQURFUiAodjEgY29udHJhY3QpLgpFWFBFQ1RFRF9LRVkgPSAiaGNhaWNfaG9uZXljb21iX2NvbmZpZ19rZXlfYWJjMTIzIgpEQVRBU0VUX1NMVUcgPSAiY2hlY2tvdXQtc2VydmljZSIKCgpjbGFzcyBfTW9ja0hvbmV5Y29tYihCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgICMgQ2xhc3MtbGV2ZWwgcmVxdWVzdCBsb2cgc28gdGhlIHRlc3QgY2FuIHJlcG9ydCB3aGF0IHBhdGgvYXV0aCB0aGUgY2xpZW50IGFjdHVhbGx5IHVzZWQuCiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6ICAjIHNpbGVuY2Ugc3RkZXJyIGFjY2VzcyBsb2cKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIGhlYWRlcnNfbG93ZXIgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX3RlYW1faGVhZGVyIjogIngtaG9uZXljb21iLXRlYW0iIGluIGhlYWRlcnNfbG93ZXIsCiAgICAgICAgICAgICJoYXNfYmVhcmVyIjogImF1dGhvcml6YXRpb24iIGluIGhlYWRlcnNfbG93ZXIsCiAgICAgICAgICAgICJxdWVyeV9hcGlfa2V5IjogImFwaV9rZXkiIGluIHBhcnNlX3FzKHBhcnNlZC5xdWVyeSksCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB2MSBmZXRjaGVzIGEgZGF0YXNldCBhdCAvMS9kYXRhc2V0cy97c2x1Z30uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi8xL2RhdGFzZXRzLyhbXi9dKykiLCBwYXRoKQogICAgICAgIGlmIG5vdCBtOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJlcnJvciI6ICJub3QgZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvcl9pbmZvIjogIkhvbmV5Y29tYiBBUEkgdjEgZmV0Y2hlcyBhIGRhdGFzZXQgYXQgLzEvZGF0YXNldHMve3NsdWd9In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IHYxIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUhvbmV5Y29tYi1UZWFtIGhlYWRlcjsgQmVhcmVyIGlzIE5PVCBob25vcmVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIlgtSG9uZXljb21iLVRlYW0iKSAhPSBFWFBFQ1RFRF9LRVk6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiSG9uZXljb21iIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUhvbmV5Y29tYi1UZWFtIGhlYWRlciJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc2x1ZyA9IG0uZ3JvdXAoMSkKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAibmFtZSI6IGYiU2VydmljZSB7c2x1Z30iLAogICAgICAgICAgICAic2x1ZyI6IHNsdWcsCiAgICAgICAgICAgICJkZXNjcmlwdGlvbiI6IGYidGVsZW1ldHJ5IGZvciB7c2x1Z30iLAogICAgICAgICAgICAiZXhwYW5kX2pzb25fZGVwdGgiOiAyLAogICAgICAgICAgICAibGFzdF93cml0dGVuX2F0IjogIjIwMjYtMDctMDFUMDA6MDA6MDBaIiwKICAgICAgICAgICAgImNyZWF0ZWRfYXQiOiAiMjAyNS0wMS0wMVQwMDowMDowMFoiLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0hvbmV5Y29tYi5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrSG9uZXljb21iKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RhdGFzZXRfYnlfc2x1Zyhtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfZGF0YXNldCIpLCAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgZ2V0X2RhdGFzZXQoYmFzZV91cmwsIGFwaV9rZXksIGRhdGFzZXRfc2x1ZykiCgogICAgZGF0YXNldCA9IG1vZHVsZS5nZXRfZGF0YXNldChtb2NrX3NlcnZlciwgRVhQRUNURURfS0VZLCBEQVRBU0VUX1NMVUcpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoZGF0YXNldCwgZGljdCksIGYiZXhwZWN0ZWQgYSBkYXRhc2V0IGRpY3QsIGdvdCB7dHlwZShkYXRhc2V0KS5fX25hbWVfX30iCiAgICBhc3NlcnQgZGF0YXNldC5nZXQoInNsdWciKSA9PSBEQVRBU0VUX1NMVUcgYW5kIGRhdGFzZXQuZ2V0KCJuYW1lIikgPT0gZiJTZXJ2aWNlIHtEQVRBU0VUX1NMVUd9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkYXRhc2V0IXJ9OyBhIGNvcnJlY3QgdjEgY2xpZW50IEdFVHMgLzEvZGF0YXNldHMve0RBVEFTRVRfU0xVR30gd2l0aCB0aGUgIgogICAgICAgIGYiWC1Ib25leWNvbWItVGVhbSBoZWFkZXIuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0hvbmV5Y29tYi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" - }, - { - "vendor": "boldsign:list_all_documents", - "fn": "list_all_documents", - "testFile": "test_boldsign_client.py", - "signature": "def list_all_documents(base_url: str, api_key: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYm9sZHNpZ24tbGlzdC1hbGwtZG9jdW1lbnRzIChoYXJkKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUgdGltZQooYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdCB0aGUgYWdlbnQncwpzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgQm9sZFNpZ24ncyBDVVJSRU5UIERvY3VtZW50cy1saXN0CmNvbnRyYWN0OgogIC0gZW5kcG9pbnQgcGF0aCAgIEdFVCAvdjEvZG9jdW1lbnQvbGlzdD9wYWdlPXtufSZwYWdlU2l6ZT17bn0gICAoc2luZ3VsYXIgImRvY3VtZW50IiArIC9saXN0OwogICAgICAgICAgICAgICAgICAgIE5PVCB0aGUgUkVTVC1ndWVzcyAvdjEvZG9jdW1lbnRzKQogIC0gYXV0aCAgICAgICAgICAgIFgtQVBJLUtFWTogPGtleT4gcmVxdWVzdCBoZWFkZXIgICAgICAgICAgICAgICAoTk9UIEF1dGhvcml6YXRpb246IEJlYXJlciA8a2V5PikKICAtIHBhZ2luYXRpb24gICAgICAxLWluZGV4ZWQgYHBhZ2VgIHF1ZXJ5IHBhcmFtOyB0aGUgcmVzcG9uc2UgaXMKICAgICAgICAgICAgICAgICAgICB7ICJwYWdlRGV0YWlscyI6IHsgInBhZ2UiLCAicGFnZVNpemUiLCAidG90YWxSZWNvcmRzQ291bnQiLCAidG90YWxQYWdlcyIsIC4uLiB9LAogICAgICAgICAgICAgICAgICAgICAgInJlc3VsdCI6IFsgPGRvY3VtZW50PiwgLi4uIF0gfQogICAgICAgICAgICAgICAgICAgIFRoZSBjYWxsZXIgcmVhZHMgdGhlIGByZXN1bHRgIGFycmF5IChOT1QgYGRhdGFgL2Bkb2N1bWVudHNgKSBhbmQgZm9sbG93cwogICAgICAgICAgICAgICAgICAgIHBhZ2UvcGFnZURldGFpbHMudG90YWxQYWdlcyB0byB0aGUgZW5kLiBUaGUgc2VydmVyIGVuZm9yY2VzIGl0cyBvd24gcGFnZQogICAgICAgICAgICAgICAgICAgIHNpemUgKDMpLCBzbyBhIGNsaWVudCB0aGF0IHN0b3BzIGFmdGVyIHBhZ2UgMSBjb2xsZWN0cyBvbmx5IDMgb2YgNy4KCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIEJvbGRTaWduIGRvY3MgY29sbGVjdHMgYWxsIDcgZG9jdW1lbnRzIGFjcm9zcyAzIHBhZ2VzIGFuZCBwYXNzZXMuCkEgY2xpZW50IGZyb20gc3RhbGUvZ3Vlc3NlZCBtZW1vcnkgKEJlYXJlciBhdXRoLCAvdjEvZG9jdW1lbnRzIHBhdGgsIGEgYGRhdGFgIGVudmVsb3BlLCBjdXJzb3IKb3IgYG5leHRgLWxpbmsgcGFnaW5hdGlvbiwgb3IgZmlyc3QtcGFnZS1vbmx5KSBoaXRzIDQwMSAvIDQwNCAvIHN0b3BzIGVhcmx5IGFuZCBGQUlMUy4gQW4gZW1wdHkKc29sdXRpb24gRkFJTFMuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZGV2ZWxvcGVycy5ib2xkc2lnbi5jb20vYXV0aGVudGljYXRpb24vYXBpLWtleS8KICBodHRwczovL2RldmVsb3BlcnMuYm9sZHNpZ24uY29tL2RvY3VtZW50cy9saXN0LWRvY3VtZW50cy8KIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9LRVkgPSAiYnNfbGl2ZV9rZXlfYWJjMTIzIgoKIyBIaWRkZW4gZGF0YXNldDogNyBkb2N1bWVudHMuIE9ubHkgb2JzZXJ2YWJsZSBieSBjYWxsaW5nIHRoZSBtb2NrIGFuZCBwYWdpbmF0aW5nIHRvIHRoZSBlbmQsCiMgc28gdGhlIGFnZW50IGNhbm5vdCBoYXJkY29kZSB0aGUgYW5zd2VyLiBTZXJ2ZXIgcGFnZSBzaXplIGlzIDMgLT4gMyBwYWdlcy4KRE9DVU1FTlRTID0gWwogICAgeyJkb2N1bWVudElkIjogZiJkb2Mte2k6MDRkfSIsICJtZXNzYWdlVGl0bGUiOiBmIkNvbnRyYWN0IHtpfSIsICJzdGF0dXMiOiAiSW5Qcm9ncmVzcyJ9CiAgICBmb3IgaSBpbiByYW5nZSgxLCA4KQpdClBBR0VfU0laRSA9IDMKCgpjbGFzcyBfTW9ja0JvbGRTaWduKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOgogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgaGVhZGVyX2tleXMgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX2FwaV9rZXlfaGVhZGVyIjogIngtYXBpLWtleSIgaW4gaGVhZGVyX2tleXMsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uX2hlYWRlciI6IHNlbGYuaGVhZGVycy5nZXQoIkF1dGhvcml6YXRpb24iKSwKICAgICAgICAgICAgInBhZ2UiOiBxcy5nZXQoInBhZ2UiLCBbTm9uZV0pWzBdLAogICAgICAgIH0pCgogICAgICAgICMgKDEpIGVuZHBvaW50IGF4aXM6IEJvbGRTaWduIGxpc3RzIGRvY3VtZW50cyBhdCAvdjEvZG9jdW1lbnQvbGlzdC4KICAgICAgICBpZiBwYXRoICE9ICIvdjEvZG9jdW1lbnQvbGlzdCI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImVycm9yIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiQm9sZFNpZ24gbGlzdHMgZG9jdW1lbnRzIGF0IC92MS9kb2N1bWVudC9saXN0In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IEJvbGRTaWduIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUFQSS1LRVkgaGVhZGVyLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIlgtQVBJLUtFWSIpICE9IEVYUEVDVEVEX0tFWToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsiZXJyb3IiOiAidW5hdXRob3JpemVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJCb2xkU2lnbiBhdXRoZW50aWNhdGVzIHZpYSB0aGUgWC1BUEktS0VZIHJlcXVlc3QgaGVhZGVyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgzKSBwYWdpbmF0aW9uIGF4aXM6IDEtaW5kZXhlZCBwYWdlOyBzZXJ2ZXIgZW5mb3JjZXMgaXRzIG93biBwYWdlIHNpemUuCiAgICAgICAgcGFnZV9yYXcgPSBxcy5nZXQoInBhZ2UiLCBbIjEiXSlbMF0KICAgICAgICB0cnk6CiAgICAgICAgICAgIHBhZ2UgPSBpbnQocGFnZV9yYXcpCiAgICAgICAgZXhjZXB0IChUeXBlRXJyb3IsIFZhbHVlRXJyb3IpOgogICAgICAgICAgICBwYWdlID0gMQogICAgICAgIGlmIHBhZ2UgPCAxOgogICAgICAgICAgICBwYWdlID0gMQogICAgICAgIHN0YXJ0ID0gKHBhZ2UgLSAxKSAqIFBBR0VfU0laRQogICAgICAgIGNodW5rID0gRE9DVU1FTlRTW3N0YXJ0OnN0YXJ0ICsgUEFHRV9TSVpFXQogICAgICAgIHRvdGFsX3BhZ2VzID0gKGxlbihET0NVTUVOVFMpICsgUEFHRV9TSVpFIC0gMSkgLy8gUEFHRV9TSVpFCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgInBhZ2VEZXRhaWxzIjogewogICAgICAgICAgICAgICAgInBhZ2UiOiBwYWdlLAogICAgICAgICAgICAgICAgInBhZ2VTaXplIjogUEFHRV9TSVpFLAogICAgICAgICAgICAgICAgInRvdGFsUmVjb3Jkc0NvdW50IjogbGVuKERPQ1VNRU5UUyksCiAgICAgICAgICAgICAgICAidG90YWxQYWdlcyI6IHRvdGFsX3BhZ2VzLAogICAgICAgICAgICAgICAgInNvcnRlZENvbHVtbiI6ICJjcmVhdGVkRGF0ZSIsCiAgICAgICAgICAgICAgICAic29ydERpcmVjdGlvbiI6ICJkZXNjZW5kaW5nIiwKICAgICAgICAgICAgfSwKICAgICAgICAgICAgInJlc3VsdCI6IGNodW5rLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0JvbGRTaWduLnJlcXVlc3RfbG9nID0gW10KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tCb2xkU2lnbikKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9kb2N1bWVudHNfYWNyb3NzX3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2RvY3VtZW50cyIpLCAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgbGlzdF9hbGxfZG9jdW1lbnRzKGJhc2VfdXJsLCBhcGlfa2V5KSIKCiAgICByZXN1bHQgPSBtb2R1bGUubGlzdF9hbGxfZG9jdW1lbnRzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9LRVkpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UocmVzdWx0LCBsaXN0KSwgZiJleHBlY3RlZCBhIGxpc3Qgb2YgZG9jdW1lbnRzLCBnb3Qge3R5cGUocmVzdWx0KS5fX25hbWVfX30iCiAgICBpZHMgPSBzb3J0ZWQoZFsiZG9jdW1lbnRJZCJdIGZvciBkIGluIHJlc3VsdCBpZiBpc2luc3RhbmNlKGQsIGRpY3QpIGFuZCAiZG9jdW1lbnRJZCIgaW4gZCkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChkWyJkb2N1bWVudElkIl0gZm9yIGQgaW4gRE9DVU1FTlRTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBkb2N1bWVudCBpZHMge2lkc30sIGV4cGVjdGVkIHtleHBlY3RlZF9pZHN9LiBBIGNvcnJlY3QgY2xpZW50IHJlYWRzIHRoZSBgcmVzdWx0YCBhcnJheSAiCiAgICAgICAgZiJhbmQgZm9sbG93cyBwYWdlL3BhZ2VEZXRhaWxzLnRvdGFsUGFnZXMgcGFnaW5hdGlvbiB0byB0aGUgZW5kLiBTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tCb2xkU2lnbi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" - }, - { - "vendor": "boldsign:get_document", - "fn": "get_document", - "testFile": "test_boldsign_get_document.py", - "signature": "def get_document(base_url: str, api_key: str, document_id: str) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYm9sZHNpZ24tZ2V0LWRvY3VtZW50IChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBCb2xkU2lnbidzIENVUlJFTlQgc2luZ2xlLWRvY3VtZW50CmNvbnRyYWN0OgogIC0gZW5kcG9pbnQgcGF0aCAgIEdFVCAvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtpZH0gICAoc2luZ3VsYXIgImRvY3VtZW50IiwgYQogICAgICAgICAgICAgICAgICAgIC9wcm9wZXJ0aWVzIGFjdGlvbiArIHF1ZXJ5LXBhcmFtIGlkOyBOT1QgdGhlIFJFU1QtZ3Vlc3MgL3YxL2RvY3VtZW50cy97aWR9KQogIC0gYXV0aCAgICAgICAgICAgIFgtQVBJLUtFWTogPGtleT4gcmVxdWVzdCBoZWFkZXIgICAgICAgICAgICAgICAgKE5PVCBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGtleT4pCiAgLSByZXNwb25zZSBzaGFwZSAgdGhlIGRvY3VtZW50IG9iamVjdCBpcyByZXR1cm5lZCBESVJFQ1RMWSBhdCB0aGUgdG9wIGxldmVsIChubyBkYXRhL3Jlc3VsdCBlbnZlbG9wZSkKCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIEJvbGRTaWduIGRvY3MgcGFzc2VzLiBBIGNsaWVudCB3cml0dGVuIGZyb20gc3RhbGUvZ3Vlc3NlZAptZW1vcnkgKEJlYXJlciBhdXRoLCAvdjEvZG9jdW1lbnRzL3tpZH0gUkVTVCBwYXRoLCBvciBhIGRhdGEtZW52ZWxvcGUgcGFyc2UpIGhpdHMgNDAxIC8gNDA0Cm9yIHJlYWRzIHRoZSB3cm9uZyBzaGFwZSBhbmQgRkFJTFMuIEFuIGVtcHR5IHNvbHV0aW9uIEZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RldmVsb3BlcnMuYm9sZHNpZ24uY29tL2F1dGhlbnRpY2F0aW9uL2FwaS1rZXkvCiAgaHR0cHM6Ly9kZXZlbG9wZXJzLmJvbGRzaWduLmNvbS9kb2N1bWVudHMvZG9jdW1lbnQtZGV0YWlscy1hbmQtc3RhdHVzLwogIGh0dHBzOi8vZGV2ZWxvcGVycy5ib2xkc2lnbi5jb20vaG93LXRvLWd1aWRlcy9yZXRyaWV2ZS1lc2lnbmF0dXJlLWRvY3VtZW50LXByb3BlcnRpZXMvCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUga2V5IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBYLUFQSS1LRVkgSEVBREVSIChCb2xkU2lnbiBjb250cmFjdCkuIFBhc3NlZCB0byB0aGUKIyBjbGllbnQgYXQgY2FsbCB0aW1lLCBzbyB0aGUga25vd2xlZGdlIGdhcCBpcyB0aGUgaGVhZGVyIE5BTUUvc2NoZW1lLCBub3QgdGhlIHNlY3JldCB2YWx1ZS4KRVhQRUNURURfS0VZID0gImJzX2xpdmVfa2V5X2FiYzEyMyIKRE9DX0lEID0gImExYjJjM2Q0LTExMTEtMjIyMi0zMzMzLTAwMDAwMDAwMDA0MiIKCgpjbGFzcyBfTW9ja0JvbGRTaWduKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOiAgIyBzaWxlbmNlIHN0ZGVyciBhY2Nlc3MgbG9nCiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBoZWFkZXJfa2V5cyA9IHtrLmxvd2VyKCkgZm9yIGsgaW4gc2VsZi5oZWFkZXJzLmtleXMoKX0KICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJoYXNfYXBpX2tleV9oZWFkZXIiOiAieC1hcGkta2V5IiBpbiBoZWFkZXJfa2V5cywKICAgICAgICAgICAgImF1dGhvcml6YXRpb25faGVhZGVyIjogc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpLAogICAgICAgICAgICAiZG9jdW1lbnRJZCI6IHFzLmdldCgiZG9jdW1lbnRJZCIsIFtOb25lXSlbMF0sCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQgYXhpczogQm9sZFNpZ24gZmV0Y2hlcyBkb2N1bWVudCBwcm9wZXJ0aWVzIGF0IC92MS9kb2N1bWVudC9wcm9wZXJ0aWVzLgogICAgICAgIGlmIHBhdGggIT0gIi92MS9kb2N1bWVudC9wcm9wZXJ0aWVzIjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiZXJyb3IiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJCb2xkU2lnbiBmZXRjaGVzIGEgZG9jdW1lbnQncyBwcm9wZXJ0aWVzIGF0ICIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICIvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtpZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDIpIGF1dGggYXhpczogQm9sZFNpZ24gYXV0aGVudGljYXRlcyB2aWEgdGhlIFgtQVBJLUtFWSBoZWFkZXIgKHF1ZXJ5L0JlYXJlciBub3QgaG9ub3JlZCkuCiAgICAgICAgaWYgc2VsZi5oZWFkZXJzLmdldCgiWC1BUEktS0VZIikgIT0gRVhQRUNURURfS0VZOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJlcnJvciI6ICJ1bmF1dGhvcml6ZWQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkJvbGRTaWduIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUFQSS1LRVkgcmVxdWVzdCBoZWFkZXIifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGRvY3VtZW50X2lkID0gcXMuZ2V0KCJkb2N1bWVudElkIiwgW05vbmVdKVswXQogICAgICAgIGlmIG5vdCBkb2N1bWVudF9pZDoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsiZXJyb3IiOiAiZG9jdW1lbnRJZCBxdWVyeSBwYXJhbWV0ZXIgaXMgcmVxdWlyZWQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDMpIHJlc3BvbnNlIHNoYXBlOiB0aGUgZG9jdW1lbnQgb2JqZWN0IGlzIHJldHVybmVkIERJUkVDVExZIGF0IHRoZSB0b3AgbGV2ZWwuCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgImRvY3VtZW50SWQiOiBkb2N1bWVudF9pZCwKICAgICAgICAgICAgIm1lc3NhZ2VUaXRsZSI6IGYiQ29udHJhY3Qge2RvY3VtZW50X2lkfSIsCiAgICAgICAgICAgICJzdGF0dXMiOiAiSW5Qcm9ncmVzcyIsCiAgICAgICAgICAgICJzZW5kZXJEZXRhaWwiOiB7Im5hbWUiOiAiQWRhIFNlbmRlciIsICJlbWFpbEFkZHJlc3MiOiAiYWRhQGV4YW1wbGUuY29tIn0sCiAgICAgICAgICAgICJzaWduZXJEZXRhaWxzIjogW3sibmFtZSI6ICJCb3JpcyBTaWduZXIiLCAiZW1haWxBZGRyZXNzIjogImJvcmlzQGV4YW1wbGUuY29tIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzdGF0dXMiOiAiTm90Q29tcGxldGVkIn1dLAogICAgICAgICAgICAiY3JlYXRlZERhdGUiOiAxNzUxODQ2NDAwLAogICAgICAgICAgICAiZXhwaXJ5RGF0ZSI6IDE3NTQ0Mzg0MDAsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrQm9sZFNpZ24ucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0JvbGRTaWduKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RvY3VtZW50X2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9kb2N1bWVudCIpLCAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgZ2V0X2RvY3VtZW50KGJhc2VfdXJsLCBhcGlfa2V5LCBkb2N1bWVudF9pZCkiCgogICAgZG9jID0gbW9kdWxlLmdldF9kb2N1bWVudChtb2NrX3NlcnZlciwgRVhQRUNURURfS0VZLCBET0NfSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoZG9jLCBkaWN0KSwgZiJleHBlY3RlZCBhIGRvY3VtZW50IGRpY3QsIGdvdCB7dHlwZShkb2MpLl9fbmFtZV9ffSIKICAgIGFzc2VydCBkb2MuZ2V0KCJkb2N1bWVudElkIikgPT0gRE9DX0lEIGFuZCBkb2MuZ2V0KCJtZXNzYWdlVGl0bGUiKSA9PSBmIkNvbnRyYWN0IHtET0NfSUR9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkb2Mhcn07IGEgY29ycmVjdCBjbGllbnQgR0VUcyAvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtET0NfSUR9IHdpdGggdGhlICIKICAgICAgICBmIlgtQVBJLUtFWSBoZWFkZXIgYW5kIHJldHVybnMgdGhlIHRvcC1sZXZlbCBkb2N1bWVudCBvYmplY3QuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0JvbGRTaWduLnJlcXVlc3RfbG9nfSIKICAgICkK" - }, - { - "vendor": "finch:list_all_individuals", - "fn": "list_all_individuals", - "testFile": "test_finch_list_all.py", - "signature": "def list_all_individuals(base_url: str, access_token: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgZmluY2gtZGlyZWN0b3J5LXBhZ2luYXRlIChoYXJkKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUgdGltZQooYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdCB0aGUgYWdlbnQncwpzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgRmluY2gncyBDVVJSRU5UIGRpcmVjdG9yeSBjb250cmFjdCwKYWRkaW5nIHRoZSBPRkZTRVQtcGFnaW5hdGlvbiBheGlzIG9uIHRvcCBvZiB0aGUgbWVkaXVtIGxlYWY6CiAgLSBlbmRwb2ludCBwYXRoICAgICAgICAvZW1wbG95ZXIvZGlyZWN0b3J5CiAgLSBhdXRoICAgICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFjY2Vzc190b2tlbj4KICAtIHZlcnNpb24gaGVhZGVyICAgICAgIEZpbmNoLUFQSS1WZXJzaW9uOiA8WVlZWS1NTS1ERD4gICAoUkVRVUlSRUQ7IG1pc3NpbmcgLT4gNDAwKQogIC0gcGFnaW5hdGlvbiAgICAgICAgICAgb2Zmc2V0LWJhc2VkOiBgbGltaXRgICsgYG9mZnNldGAgcXVlcnkgcGFyYW1zOyB0aGUgcmVzcG9uc2UgY2FycmllcwogICAgICAgICAgICAgICAgICAgICAgICAgcGFnaW5nLmNvdW50ICh0b3RhbCkgYW5kIHBhZ2luZy5vZmZzZXQsIGFuZCB0aGVyZSBpcyBOTyBuZXh0L2hhc19tb3JlCiAgICAgICAgICAgICAgICAgICAgICAgICBjdXJzb3IgZmllbGQgLS0gdGhlIGNsaWVudCBtdXN0IGtlZXAgcGFnaW5nIHdoaWxlIG9mZnNldCtsZW4gPCBjb3VudAogIC0gcmVzcG9uc2UgZW52ZWxvcGUgICAgeyJwYWdpbmciOiB7ImNvdW50IiwgIm9mZnNldCJ9LCAiaW5kaXZpZHVhbHMiOiBbLi4uXX0KClRoZSBtb2NrIHJldHVybnMgYXQgbW9zdCBQQUdFX1NJWkUgaW5kaXZpZHVhbHMgcGVyIHJlc3BvbnNlIChzZXJ2ZXIgcGFnZSBzaXplKSwgc28gY29sbGVjdGluZwp0aGUgd2hvbGUgZGlyZWN0b3J5IFJFUVVJUkVTIG9mZnNldCBwYWdpbmF0aW9uIGRyaXZlbiBieSBwYWdpbmcuY291bnQuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbQpDVVJSRU5UIEZpbmNoIGRvY3MgY29sbGVjdHMgYWxsIDcgaW5kaXZpZHVhbHMgYWNyb3NzIDMgcGFnZXMuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtCmN1dG9mZiBtZW1vcnkgb21pdHMgdGhlIHZlcnNpb24gaGVhZGVyICg0MDApLCBndWVzc2VzIGEgL2VtcGxveWVlcyBwYXRoICg0MDQpLCByZWFkcyBhIGBkYXRhYAplbnZlbG9wZSwgb3IgYXNzdW1lcyBhIGBuZXh0YC9gaGFzX21vcmVgIGN1cnNvciBhbmQgc3RvcHMgYWZ0ZXIgcGFnZSAxICgzIG9mIDcpIGFuZCBGQUlMUy4KQW4gZW1wdHkgc29sdXRpb24gRkFJTFMuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZGV2ZWxvcGVyLnRyeWZpbmNoLmNvbS9hcGktcmVmZXJlbmNlL2RldmVsb3BtZW50LWd1aWRlcy9IZWFkZXJzCiAgaHR0cHM6Ly9kZXZlbG9wZXIudHJ5ZmluY2guY29tL2FwaS1yZWZlcmVuY2Uvb3JnYW5pemF0aW9uL2RpcmVjdG9yeQoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9UT0tFTiA9ICJmaW5jaF9hY2Nlc3NfdG9rZW5fYWJjMTIzIgoKIyBIaWRkZW4gZGF0YXNldDogNyBpbmRpdmlkdWFscywgb25seSBvYnNlcnZhYmxlIGJ5IGFjdHVhbGx5IGNhbGxpbmcgdGhlIG1vY2sgYW5kIHBhZ2luYXRpbmcuCklORElWSURVQUxTID0gWwogICAgewogICAgICAgICJpZCI6IGYiMjIyMjIyMjItMDAwMC00MDAwLTgwMDAtMDAwMDAwMDAwMDB7aX0iLAogICAgICAgICJmaXJzdF9uYW1lIjogZiJGaXJzdHtpfSIsCiAgICAgICAgIm1pZGRsZV9uYW1lIjogTm9uZSwKICAgICAgICAibGFzdF9uYW1lIjogZiJMYXN0e2l9IiwKICAgICAgICAiZGVwYXJ0bWVudCI6IHsibmFtZSI6ICJFbmdpbmVlcmluZyJ9LAogICAgICAgICJtYW5hZ2VyIjogTm9uZSwKICAgICAgICAiaXNfYWN0aXZlIjogVHJ1ZSwKICAgIH0KICAgIGZvciBpIGluIHJhbmdlKDEsIDgpCl0KUEFHRV9TSVpFID0gMyAgIyBzZXJ2ZXIgcGFnZSBzaXplOiBhdCBtb3N0IHRoaXMgbWFueSBpbmRpdmlkdWFscyBwZXIgcmVzcG9uc2UsIGZvcmNpbmcgcGFnaW5hdGlvbgoKCmNsYXNzIF9Nb2NrRmluY2goQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBoZWFkZXJfa2V5cyA9IHtrLmxvd2VyKCkgZm9yIGsgaW4gc2VsZi5oZWFkZXJzLmtleXMoKX0KICAgICAgICB2ZXJzaW9uID0gc2VsZi5oZWFkZXJzLmdldCgiRmluY2gtQVBJLVZlcnNpb24iKQogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJoYXNfdmVyc2lvbl9oZWFkZXIiOiAiZmluY2gtYXBpLXZlcnNpb24iIGluIGhlYWRlcl9rZXlzLAogICAgICAgICAgICAidmVyc2lvbiI6IHZlcnNpb24sCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImxpbWl0IjogcXMuZ2V0KCJsaW1pdCIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJvZmZzZXQiOiBxcy5nZXQoIm9mZnNldCIsIFtOb25lXSlbMF0sCiAgICAgICAgfSkKCiAgICAgICAgaWYgcGF0aCAhPSAiL2VtcGxveWVyL2RpcmVjdG9yeSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImNvZGUiOiA0MDQsICJuYW1lIjogIm5vdF9mb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiRmluY2ggbGlzdHMgdGhlIG9yZyBkaXJlY3RvcnkgYXQgR0VUIC9lbXBsb3llci9kaXJlY3RvcnkifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIGF1dGggIT0gZiJCZWFyZXIge0VYUEVDVEVEX1RPS0VOfSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImNvZGUiOiA0MDEsICJuYW1lIjogImludmFsaWRfYWNjZXNzX3Rva2VuIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJBdXRob3JpemF0aW9uIG11c3QgYmUgJ0JlYXJlciA8YWNjZXNzX3Rva2VuPicifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIHZlcnNpb24gaXMgTm9uZSBvciBub3QgcmUuZnVsbG1hdGNoKHIiXGR7NH0tXGR7Mn0tXGR7Mn0iLCB2ZXJzaW9uKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsiY29kZSI6IDQwMCwgIm5hbWUiOiAibWlzc2luZ192ZXJzaW9uX2hlYWRlciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiRmluY2ggcmVxdWlyZXMgdGhlIEZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciAoZS5nLiAyMDIwLTA5LTE3KSJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyBvZmZzZXQgcGFnaW5hdGlvbjogaG9ub3IgYG9mZnNldGAgKGRlZmF1bHQgMCk7IHNlcnZlciBjYXBzIHRoZSBwYWdlIGF0IFBBR0VfU0laRS4KICAgICAgICB0cnk6CiAgICAgICAgICAgIG9mZnNldCA9IGludChxcy5nZXQoIm9mZnNldCIsIFsiMCJdKVswXSkKICAgICAgICBleGNlcHQgKFR5cGVFcnJvciwgVmFsdWVFcnJvcik6CiAgICAgICAgICAgIG9mZnNldCA9IDAKICAgICAgICBpZiBvZmZzZXQgPCAwOgogICAgICAgICAgICBvZmZzZXQgPSAwCiAgICAgICAgdHJ5OgogICAgICAgICAgICByZXF1ZXN0ZWRfbGltaXQgPSBpbnQocXMuZ2V0KCJsaW1pdCIsIFtzdHIoUEFHRV9TSVpFKV0pWzBdKQogICAgICAgIGV4Y2VwdCAoVHlwZUVycm9yLCBWYWx1ZUVycm9yKToKICAgICAgICAgICAgcmVxdWVzdGVkX2xpbWl0ID0gUEFHRV9TSVpFCiAgICAgICAgcGFnZV9sZW4gPSBtYXgoMCwgbWluKFBBR0VfU0laRSwgcmVxdWVzdGVkX2xpbWl0KSkKCiAgICAgICAgcGFnZSA9IElORElWSURVQUxTW29mZnNldDpvZmZzZXQgKyBwYWdlX2xlbl0KICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAicGFnaW5nIjogeyJjb3VudCI6IGxlbihJTkRJVklEVUFMUyksICJvZmZzZXQiOiBvZmZzZXR9LAogICAgICAgICAgICAiaW5kaXZpZHVhbHMiOiBwYWdlLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0ZpbmNoLnJlcXVlc3RfbG9nID0gW10KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tGaW5jaCkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9pbmRpdmlkdWFsc19hY3Jvc3Nfb2Zmc2V0X3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2luZGl2aWR1YWxzIiksIFwKICAgICAgICAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgbGlzdF9hbGxfaW5kaXZpZHVhbHMoYmFzZV91cmwsIGFjY2Vzc190b2tlbikiCgogICAgcmVzdWx0ID0gbW9kdWxlLmxpc3RfYWxsX2luZGl2aWR1YWxzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBpbmRpdmlkdWFscywgZ290IHt0eXBlKHJlc3VsdCkuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGRbImlkIl0gZm9yIGQgaW4gcmVzdWx0IGlmIGlzaW5zdGFuY2UoZCwgZGljdCkgYW5kICJpZCIgaW4gZCkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChkWyJpZCJdIGZvciBkIGluIElORElWSURVQUxTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCB7bGVuKGlkcyl9IGluZGl2aWR1YWwgaWRzIHtpZHN9LCBleHBlY3RlZCB7bGVuKGV4cGVjdGVkX2lkcyl9IHtleHBlY3RlZF9pZHN9LiAiCiAgICAgICAgZiJBIGNvcnJlY3QgY2xpZW50IHBhZ2luYXRlcyBieSBgb2Zmc2V0YCB3aGlsZSBwYWdpbmcub2Zmc2V0K2xlbihpbmRpdmlkdWFscykgPCBwYWdpbmcuY291bnQgIgogICAgICAgIGYiKHRoZXJlIGlzIG5vIG5leHQvaGFzX21vcmUgY3Vyc29yKSwgc2VuZGluZyB0aGUgQmVhcmVyIHRva2VuIEFORCB0aGUgcmVxdWlyZWQgIgogICAgICAgIGYiRmluY2gtQVBJLVZlcnNpb24gaGVhZGVyIGVhY2ggcmVxdWVzdC4gU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrRmluY2gucmVxdWVzdF9sb2d9IgogICAgKQo=" - }, - { - "vendor": "finch:get_directory", - "fn": "get_directory", - "testFile": "test_finch_directory.py", - "signature": "def get_directory(base_url: str, access_token: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgZmluY2gtZGlyZWN0b3J5LXBhZ2UgKG1lZGl1bSkgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpJdCBzdGFuZHMgdXAgYSBMT0NBTCBtb2NrIHRoYXQgZmFpdGhmdWxseSBpbXBsZW1lbnRzIEZpbmNoJ3MgQ1VSUkVOVAooVW5pdmVyc2FsIEVtcGxveW1lbnQgQVBJKSBkaXJlY3RvcnkgY29udHJhY3QgZm9yIGEgc2luZ2xlIHJlc3BvbnNlOgogIC0gZW5kcG9pbnQgcGF0aCAgICAgICAgL2VtcGxveWVyL2RpcmVjdG9yeSAgICAgICAgKGJhc2UgaG9zdCBpcyBpbmplY3RlZDsgdGhlIHBhdGggaXMgZml4ZWQpCiAgLSBhdXRoICAgICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFjY2Vzc190b2tlbj4KICAtIHZlcnNpb24gaGVhZGVyICAgICAgIEZpbmNoLUFQSS1WZXJzaW9uOiA8WVlZWS1NTS1ERD4gICAoUkVRVUlSRUQ7IG1pc3NpbmcgLT4gNDAwKQogIC0gcmVzcG9uc2UgZW52ZWxvcGUgICAgeyJwYWdpbmciOiB7ImNvdW50IiwgIm9mZnNldCJ9LCAiaW5kaXZpZHVhbHMiOiBbLi4uXX0KCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIEZpbmNoIGRvY3Mgc2VuZHMgdGhlIEJlYXJlciB0b2tlbiBBTkQgdGhlIHJlcXVpcmVkCkZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciB0byAvZW1wbG95ZXIvZGlyZWN0b3J5IGFuZCByZWFkcyB0aGUgYGluZGl2aWR1YWxzYCBhcnJheS4KQSBjbGllbnQgd3JpdHRlbiBmcm9tIHN0YWxlL3ByZS1jdXRvZmYgbWVtb3J5IG9taXRzIHRoZSByZXF1aXJlZCB2ZXJzaW9uIGhlYWRlciAoNDAwKSwKZ3Vlc3NlcyBhIFJFU1QtaXNoIC9lbXBsb3llZXMgcGF0aCAoNDA0KSwgb3IgcmVhZHMgYSBgZGF0YWAvYHJlc3VsdHNgIGVudmVsb3BlLCBhbmQgRkFJTFMuCkFuIGVtcHR5IHNvbHV0aW9uIEZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RldmVsb3Blci50cnlmaW5jaC5jb20vYXBpLXJlZmVyZW5jZS9kZXZlbG9wbWVudC1ndWlkZXMvSGVhZGVycwogIGh0dHBzOi8vZGV2ZWxvcGVyLnRyeWZpbmNoLmNvbS9hcGktcmVmZXJlbmNlL29yZ2FuaXphdGlvbi9kaXJlY3RvcnkKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgdG9rZW4gdGhlIGNsaWVudCBtdXN0IHNlbmQgaW4gdGhlIEF1dGhvcml6YXRpb246IEJlYXJlciA8dG9rZW4+IGhlYWRlci4KRVhQRUNURURfVE9LRU4gPSAiZmluY2hfYWNjZXNzX3Rva2VuX2FiYzEyMyIKCiMgU21hbGwgZGlyZWN0b3J5IHJldHVybmVkIGluIGEgc2luZ2xlIHJlc3BvbnNlIChkZWZhdWx0cy10by1hbGw7IG5vIHBhZ2luYXRpb24gaW4gdGhpcyBsZWFmKS4KSU5ESVZJRFVBTFMgPSBbCiAgICB7CiAgICAgICAgImlkIjogZiIxMTExMTExMS0wMDAwLTQwMDAtODAwMC0wMDAwMDAwMDAwMHtpfSIsCiAgICAgICAgImZpcnN0X25hbWUiOiBmIkZpcnN0e2l9IiwKICAgICAgICAibWlkZGxlX25hbWUiOiBOb25lLAogICAgICAgICJsYXN0X25hbWUiOiBmIkxhc3R7aX0iLAogICAgICAgICJkZXBhcnRtZW50IjogeyJuYW1lIjogIkVuZ2luZWVyaW5nIn0sCiAgICAgICAgIm1hbmFnZXIiOiBOb25lLAogICAgICAgICJpc19hY3RpdmUiOiBUcnVlLAogICAgfQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgNCkKXQoKCmNsYXNzIF9Nb2NrRmluY2goQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICAjIENsYXNzLWxldmVsIHJlcXVlc3QgbG9nIHNvIHRoZSB0ZXN0IGNhbiByZXBvcnQgd2hhdCB0aGUgY2xpZW50IGFjdHVhbGx5IHNlbnQgb24gZmFpbHVyZS4KICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKTogICMgc2lsZW5jZSBzdGRlcnIgYWNjZXNzIGxvZwogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgaGVhZGVyX2tleXMgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdmVyc2lvbiA9IHNlbGYuaGVhZGVycy5nZXQoIkZpbmNoLUFQSS1WZXJzaW9uIikKICAgICAgICBhdXRoID0gc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpCiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX3ZlcnNpb25faGVhZGVyIjogImZpbmNoLWFwaS12ZXJzaW9uIiBpbiBoZWFkZXJfa2V5cywKICAgICAgICAgICAgInZlcnNpb24iOiB2ZXJzaW9uLAogICAgICAgICAgICAiYXV0aG9yaXphdGlvbiI6IGF1dGgsCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB0aGUgZGlyZWN0b3J5IGxpdmVzIGF0IC9lbXBsb3llci9kaXJlY3RvcnkgKGhvc3QgaXMgaW5qZWN0ZWQpLgogICAgICAgIGlmIHBhdGggIT0gIi9lbXBsb3llci9kaXJlY3RvcnkiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJjb2RlIjogNDA0LCAibmFtZSI6ICJub3RfZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkZpbmNoIGxpc3RzIHRoZSBvcmcgZGlyZWN0b3J5IGF0IEdFVCAvZW1wbG95ZXIvZGlyZWN0b3J5In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IEF1dGhvcml6YXRpb24gbXVzdCBiZSB0aGUgQmVhcmVyIGFjY2VzcyB0b2tlbi4KICAgICAgICBpZiBhdXRoICE9IGYiQmVhcmVyIHtFWFBFQ1RFRF9UT0tFTn0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJjb2RlIjogNDAxLCAibmFtZSI6ICJpbnZhbGlkX2FjY2Vzc190b2tlbiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiQXV0aG9yaXphdGlvbiBtdXN0IGJlICdCZWFyZXIgPGFjY2Vzc190b2tlbj4nIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgzKSB2ZXJzaW9uIGF4aXM6IEZpbmNoLUFQSS1WZXJzaW9uIGlzIGEgUkVRVUlSRUQgaGVhZGVyIChkYXRlLWZvcm1hdHRlZCkuCiAgICAgICAgaWYgdmVyc2lvbiBpcyBOb25lIG9yIG5vdCByZS5mdWxsbWF0Y2gociJcZHs0fS1cZHsyfS1cZHsyfSIsIHZlcnNpb24pOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJjb2RlIjogNDAwLCAibmFtZSI6ICJtaXNzaW5nX3ZlcnNpb25faGVhZGVyIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJGaW5jaCByZXF1aXJlcyB0aGUgRmluY2gtQVBJLVZlcnNpb24gaGVhZGVyIChlLmcuIDIwMjAtMDktMTcpIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAicGFnaW5nIjogeyJjb3VudCI6IGxlbihJTkRJVklEVUFMUyksICJvZmZzZXQiOiAwfSwKICAgICAgICAgICAgImluZGl2aWR1YWxzIjogSU5ESVZJRFVBTFMsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrRmluY2gucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0ZpbmNoKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RpcmVjdG9yeShtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfZGlyZWN0b3J5IiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfZGlyZWN0b3J5KGJhc2VfdXJsLCBhY2Nlc3NfdG9rZW4pIgoKICAgIHJlc3VsdCA9IG1vZHVsZS5nZXRfZGlyZWN0b3J5KG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBpbmRpdmlkdWFscywgZ290IHt0eXBlKHJlc3VsdCkuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGRbImlkIl0gZm9yIGQgaW4gcmVzdWx0IGlmIGlzaW5zdGFuY2UoZCwgZGljdCkgYW5kICJpZCIgaW4gZCkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChkWyJpZCJdIGZvciBkIGluIElORElWSURVQUxTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBpbmRpdmlkdWFsIGlkcyB7aWRzfSwgZXhwZWN0ZWQge2V4cGVjdGVkX2lkc30uICIKICAgICAgICBmIkEgY29ycmVjdCBjbGllbnQgR0VUcyAvZW1wbG95ZXIvZGlyZWN0b3J5IHdpdGggdGhlIEJlYXJlciB0b2tlbiBBTkQgdGhlIHJlcXVpcmVkICIKICAgICAgICBmIkZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciwgdGhlbiByZWFkcyB0aGUgYGluZGl2aWR1YWxzYCBhcnJheS4gIgogICAgICAgIGYiU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrRmluY2gucmVxdWVzdF9sb2d9IgogICAgKQo=" - }, - { - "vendor": "novu:sync_subscribers_to_deals", - "fn": "sync_subscribers_to_deals", - "testFile": "test_pipeline_sync_idempotent.py", - "signature": "def sync_subscribers_to_deals(novu_base_url: str, novu_api_key: str, pd_base_url: str, pd_api_token: str) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcGlwZWxpbmUtbm92dS10by1waXBlZHJpdmUtc3luYy1pZGVtcG90ZW50IChoYXJkKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUKdGltZSAoYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdAp0aGUgYWdlbnQncyBzb2x1dGlvbi9zb2x1dGlvbi5weS4KClNhbWUgVFdPIHZlbmRvciBtb2NrcyBhcyB0aGUgbWVkaXVtIGxlYWYgKE5vdnUgc291cmNlICsgUGlwZWRyaXZlIHNpbms7IHNlZSB0aGUKYXhlcyBiZWxvdyksIGJ1dCB0aGUgdGVzdCBjYWxscyB0aGUgcGlwZWxpbmUgVFdJQ0UgYWdhaW5zdCB0aGUgc2FtZSBQaXBlZHJpdmUKc3RvcmUuIEEgY29ycmVjdCBwaXBlbGluZSBpcyBJREVNUE9URU5UOiBpdCBkZXJpdmVzIGFscmVhZHktc3luY2VkIHN0YXRlIGZyb20KUGlwZWRyaXZlIElUU0VMRiDigJQgR0VUIC9hcGkvdjIvZGVhbHMsIGZvbGxvd2luZyBjdXJzb3IgcGFnaW5hdGlvbiB0byBjb21wbGV0aW9uCih0aGUgc3RvcmUgaG9sZHMgNyBkZWFscyA9IDMgcGFnZXMgYXQgc2VydmVyIHBhZ2Ugc2l6ZSAzLCBzbyBhIGNsaWVudCBzdGFsZSBvbgp0aGUgTElTVCBjb250cmFjdCBvbmx5IHNlZXMgcGFnZSAxIGFuZCByZS1jcmVhdGVzIHRoZSByZXN0KSDigJQgYW5kIHNraXBzCnN1YnNjcmliZXJzIHdob3NlIGVtYWlsIGFscmVhZHkgdGl0bGVzIGEgZGVhbC4KCiAgTk9WVSAoc291cmNlKTogICAgL3YyL3N1YnNjcmliZXJzICsgQXV0aG9yaXphdGlvbjogQXBpS2V5IDx0b2tlbj4gKyBjdXJzb3IKICAgICAgICAgICAgICAgICAgICBgYWZ0ZXJgL2BuZXh0YCAoe2RhdGEsIG5leHQsIHByZXZpb3VzLCB0b3RhbENvdW50LCB0b3RhbENvdW50Q2FwcGVkfSkKICBQSVBFRFJJVkUgKHNpbmspOiBQT1NUIC9hcGkvdjIvZGVhbHMgKEpTT04sIGB0aXRsZWAgcmVxdWlyZWQsIDIwMSkgYW5kCiAgICAgICAgICAgICAgICAgICAgR0VUIC9hcGkvdjIvZGVhbHMgKGN1cnNvciBgY3Vyc29yYCArIGFkZGl0aW9uYWxfZGF0YS5uZXh0X2N1cnNvciksCiAgICAgICAgICAgICAgICAgICAgYm90aCB4LWFwaS10b2tlbiBIRUFERVIgYXV0aCAobmV2ZXIgP2FwaV90b2tlbj0pCgpBc3NlcnRlZDogcnVuIDEgc3luY3MgYWxsIDcgKGNvcnJlY3QgdGl0bGVzLCBjb3JyZWN0IGF1dGgsIE5vdnUgY3Vyc29yIGZvbGxvd2VkKTsKcnVuIDIgcmV0dXJucyBzeW5jZWQgPT0gMCwgY3JlYXRlcyBOTyBuZXcgZGVhbCAoc3RvcmUgc3RpbGwgZXhhY3RseSA3KSwgYW5kCmFjdHVhbGx5IHJlLXJlYWQgdGhlIFBpcGVkcml2ZSBzdG9yZSAoPj0gMyBsaXN0IEdFVHMgaW4gcnVuIDIg4oCUIGFuIGluLXByb2Nlc3MKY2FjaGUgaW5zdGVhZCBvZiByZWFkaW5nIHRoZSBzaW5rIEZBSUxTKS4gQSBub24taWRlbXBvdGVudCBwaXBlbGluZSBkdXBsaWNhdGVzCjcgLT4gMTQgYW5kIEZBSUxTLiBTdGFsZSB2MS9CZWFyZXIvb2Zmc2V0IGNsaWVudHMgRkFJTCBhcyBpbiB0aGUgbWVkaXVtIGxlYWYuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZG9jcy5ub3Z1LmNvL2FwaS1yZWZlcmVuY2UvYXV0aGVudGljYXRpb24KICBodHRwczovL2RvY3Mubm92dS5jby9hcGktcmVmZXJlbmNlL3N1YnNjcmliZXJzL3NlYXJjaC1zdWJzY3JpYmVycwogIGh0dHBzOi8vcGlwZWRyaXZlLnJlYWRtZS5pby9kb2NzL2NvcmUtYXBpLWNvbmNlcHRzLWF1dGhlbnRpY2F0aW9uCiAgaHR0cHM6Ly9waXBlZHJpdmUucmVhZG1lLmlvL2RvY3MvY29yZS1hcGktY29uY2VwdHMtcGFnaW5hdGlvbgogIGh0dHBzOi8vcGlwZWRyaXZlLnJlYWRtZS5pby9kb2NzL3BpcGVkcml2ZS1hcGktdjItbWlncmF0aW9uLWd1aWRlCiIiIgoKaW1wb3J0IGJhc2U2NAppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgojIEdyYWRlci1zaWRlIHNlY3JldHMg4oCUIERJRkZFUkVOVCBmcm9tIHRoZSBsaXZlLXR3aW4gdG9rZW5zIHRoZSBhZ2VudCBvYnNlcnZlZC4KTk9WVV9UT0tFTiA9ICJudl9zeW5jX2dyYWRlcl9rZXlfMzE0MTU5IgpQRF9UT0tFTiA9ICJwZF9zeW5jX2dyYWRlcl90b2tlbl8yNzE4MjgiCgpTVUJTQ1JJQkVSUyA9IFsKICAgIHsiX2lkIjogZiJfaWRfe2l9IiwgInN1YnNjcmliZXJJZCI6IGYidXNlcl97aX0iLCAiZW1haWwiOiBmInVzZXJ7aX1AZ3JhZGVyY29ycC5leGFtcGxlIn0KICAgIGZvciBpIGluIHJhbmdlKDEsIDgpCl0KTk9WVV9QQUdFX1NJWkUgPSAzClBEX1BBR0VfU0laRSA9IDMKCgpkZWYgX2VuY29kZV9jdXJzb3Iob2Zmc2V0OiBpbnQpIC0+IHN0cjoKICAgIHJldHVybiBiYXNlNjQudXJsc2FmZV9iNjRlbmNvZGUoc3RyKG9mZnNldCkuZW5jb2RlKCkpLmRlY29kZSgpCgoKZGVmIF9kZWNvZGVfY3Vyc29yKGN1cnNvcjogc3RyKSAtPiBpbnQ6CiAgICByZXR1cm4gaW50KGJhc2U2NC51cmxzYWZlX2I2NGRlY29kZShjdXJzb3IuZW5jb2RlKCkpLmRlY29kZSgpKQoKCmNsYXNzIF9Nb2NrTm92dShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImFwaWtleV9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkFwaUtleSAiKSwKICAgICAgICAgICAgImJlYXJlcl9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkJlYXJlciAiKSwKICAgICAgICAgICAgImFmdGVyIjogcXMuZ2V0KCJhZnRlciIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJwYWdlIjogcXMuZ2V0KCJwYWdlIiwgW05vbmVdKVswXSwKICAgICAgICB9KQoKICAgICAgICBpZiBwYXRoICE9ICIvdjIvc3Vic2NyaWJlcnMiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJzdGF0dXNDb2RlIjogNDA0LCAibWVzc2FnZSI6ICJub3QgZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6ICJOb3Z1IEFQSSB2MiBsaXN0cyBzdWJzY3JpYmVycyBhdCAvdjIvc3Vic2NyaWJlcnMifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIGF1dGggIT0gZiJBcGlLZXkge05PVlVfVE9LRU59IjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsic3RhdHVzQ29kZSI6IDQwMSwgIm1lc3NhZ2UiOiAidW5hdXRob3JpemVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3IiOiAiTm92dSBhdXRoZW50aWNhdGVzIHZpYSBgQXV0aG9yaXphdGlvbjogQXBpS2V5IDxzZWNyZXQ+YCAobm90IEJlYXJlcikifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGFmdGVyID0gcXMuZ2V0KCJhZnRlciIsIFtOb25lXSlbMF0KICAgICAgICBpZiBhZnRlciBpcyBOb25lOgogICAgICAgICAgICBvZmZzZXQgPSAwCiAgICAgICAgZWxzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgb2Zmc2V0ID0gX2RlY29kZV9jdXJzb3IoYWZ0ZXIpCiAgICAgICAgICAgIGV4Y2VwdCBFeGNlcHRpb246CiAgICAgICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJzdGF0dXNDb2RlIjogNDAwLCAibWVzc2FnZSI6ICJpbnZhbGlkIGN1cnNvciJ9KQogICAgICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIHBhZ2UgPSBTVUJTQ1JJQkVSU1tvZmZzZXQ6b2Zmc2V0ICsgTk9WVV9QQUdFX1NJWkVdCiAgICAgICAgbmV4dF9vZmZzZXQgPSBvZmZzZXQgKyBOT1ZVX1BBR0VfU0laRQogICAgICAgIG5leHRfY3Vyc29yID0gX2VuY29kZV9jdXJzb3IobmV4dF9vZmZzZXQpIGlmIG5leHRfb2Zmc2V0IDwgbGVuKFNVQlNDUklCRVJTKSBlbHNlIE5vbmUKICAgICAgICBwcmV2X2N1cnNvciA9IF9lbmNvZGVfY3Vyc29yKG1heChvZmZzZXQgLSBOT1ZVX1BBR0VfU0laRSwgMCkpIGlmIG9mZnNldCA+IDAgZWxzZSBOb25lCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsiZGF0YSI6IHBhZ2UsICJuZXh0IjogbmV4dF9jdXJzb3IsICJwcmV2aW91cyI6IHByZXZfY3Vyc29yLAogICAgICAgICAgICAgICAgICAgICAgICAgInRvdGFsQ291bnQiOiBsZW4oU1VCU0NSSUJFUlMpLCAidG90YWxDb3VudENhcHBlZCI6IEZhbHNlfSkKCgpjbGFzcyBfTW9ja1BpcGVkcml2ZShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KICAgIGRlYWxzID0gW10KICAgIF9sb2NrID0gdGhyZWFkaW5nLkxvY2soKQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOgogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgX3JlY29yZChzZWxmLCBwYXJzZWQsICoqZXh0cmEpOgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIGVudHJ5ID0gewogICAgICAgICAgICAibWV0aG9kIjogc2VsZi5jb21tYW5kLAogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX2hlYWRlcl9hdXRoIjogIngtYXBpLXRva2VuIiBpbiB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9LAogICAgICAgICAgICAicXVlcnlfYXBpX3Rva2VuIjogImFwaV90b2tlbiIgaW4gcXMsCiAgICAgICAgfQogICAgICAgIGVudHJ5LnVwZGF0ZShleHRyYSkKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZChlbnRyeSkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgY3Vyc29yID0gcXMuZ2V0KCJjdXJzb3IiLCBbTm9uZV0pWzBdCiAgICAgICAgc2VsZi5fcmVjb3JkKHBhcnNlZCwgY3Vyc29yPWN1cnNvciwgc3RhcnQ9cXMuZ2V0KCJzdGFydCIsIFtOb25lXSlbMF0pCgogICAgICAgIGlmIHBhdGggIT0gIi9hcGkvdjIvZGVhbHMiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJzdWNjZXNzIjogRmFsc2UsICJlcnJvciI6ICJub3QgZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvcl9pbmZvIjogIlBpcGVkcml2ZSBBUEkgdjIgbGlzdHMgZGVhbHMgYXQgL2FwaS92Mi9kZWFscyJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgaWYgc2VsZi5oZWFkZXJzLmdldCgieC1hcGktdG9rZW4iKSAhPSBQRF9UT0tFTjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsic3VjY2VzcyI6IEZhbHNlLCAiZXJyb3IiOiAidW5hdXRob3JpemVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3JfaW5mbyI6ICJQaXBlZHJpdmUgQVBJIHYyIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSB4LWFwaS10b2tlbiBoZWFkZXIifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIG9mZnNldCA9IDAKICAgICAgICBpZiBjdXJzb3IgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIG9mZnNldCA9IF9kZWNvZGVfY3Vyc29yKGN1cnNvcikKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAgICAgICAgIHNlbGYuX2pzb24oNDAwLCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogImludmFsaWQgY3Vyc29yIn0pCiAgICAgICAgICAgICAgICByZXR1cm4KICAgICAgICB3aXRoIHR5cGUoc2VsZikuX2xvY2s6CiAgICAgICAgICAgIHNuYXBzaG90ID0gbGlzdCh0eXBlKHNlbGYpLmRlYWxzKQogICAgICAgIHBhZ2UgPSBzbmFwc2hvdFtvZmZzZXQ6b2Zmc2V0ICsgUERfUEFHRV9TSVpFXQogICAgICAgIG5leHRfb2Zmc2V0ID0gb2Zmc2V0ICsgUERfUEFHRV9TSVpFCiAgICAgICAgbmV4dF9jdXJzb3IgPSBfZW5jb2RlX2N1cnNvcihuZXh0X29mZnNldCkgaWYgbmV4dF9vZmZzZXQgPCBsZW4oc25hcHNob3QpIGVsc2UgTm9uZQogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7InN1Y2Nlc3MiOiBUcnVlLCAiZGF0YSI6IHBhZ2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAiYWRkaXRpb25hbF9kYXRhIjogeyJuZXh0X2N1cnNvciI6IG5leHRfY3Vyc29yfX0pCgogICAgZGVmIGRvX1BPU1Qoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKCiAgICAgICAgaWYgcGF0aCAhPSAiL2FwaS92Mi9kZWFscyI6CiAgICAgICAgICAgIHNlbGYuX3JlY29yZChwYXJzZWQsIHRpdGxlPU5vbmUpCiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBjcmVhdGVzIGEgZGVhbCBhdCBQT1NUIC9hcGkvdjIvZGVhbHMifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIngtYXBpLXRva2VuIikgIT0gUERfVE9LRU46CiAgICAgICAgICAgIHNlbGYuX3JlY29yZChwYXJzZWQsIHRpdGxlPU5vbmUpCiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBhdXRoZW50aWNhdGVzIHZpYSB0aGUgeC1hcGktdG9rZW4gaGVhZGVyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBsZW5ndGggPSBpbnQoc2VsZi5oZWFkZXJzLmdldCgiQ29udGVudC1MZW5ndGgiKSBvciAwKQogICAgICAgIHJhdyA9IHNlbGYucmZpbGUucmVhZChsZW5ndGgpIGlmIGxlbmd0aCA+IDAgZWxzZSBiIiIKICAgICAgICB0cnk6CiAgICAgICAgICAgIGJvZHkgPSBqc29uLmxvYWRzKHJhdy5kZWNvZGUoKSBvciAibnVsbCIpCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAgICAgYm9keSA9IE5vbmUKICAgICAgICB0aXRsZSA9IChib2R5IG9yIHt9KS5nZXQoInRpdGxlIikgaWYgaXNpbnN0YW5jZShib2R5LCBkaWN0KSBlbHNlIE5vbmUKICAgICAgICBzZWxmLl9yZWNvcmQocGFyc2VkLCB0aXRsZT10aXRsZSkKICAgICAgICBpZiBub3QgaXNpbnN0YW5jZSh0aXRsZSwgc3RyKSBvciBub3QgdGl0bGUuc3RyaXAoKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsic3VjY2VzcyI6IEZhbHNlLCAiZXJyb3IiOiAiYmFkIHJlcXVlc3QiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvcl9pbmZvIjogInRoZSBgdGl0bGVgIGZpZWxkIGlzIHJlcXVpcmVkIChKU09OIGJvZHkpIn0pCiAgICAgICAgICAgIHJldHVybgogICAgICAgIHdpdGggdHlwZShzZWxmKS5fbG9jazoKICAgICAgICAgICAgZGVhbCA9IHsiaWQiOiBsZW4odHlwZShzZWxmKS5kZWFscykgKyAxLCAidGl0bGUiOiB0aXRsZSwgInZhbHVlIjogMCwKICAgICAgICAgICAgICAgICAgICAiY3VycmVuY3kiOiAiVVNEIiwgInN0YXR1cyI6ICJvcGVuIn0KICAgICAgICAgICAgdHlwZShzZWxmKS5kZWFscy5hcHBlbmQoZGVhbCkKICAgICAgICBzZWxmLl9qc29uKDIwMSwgeyJzdWNjZXNzIjogVHJ1ZSwgImRhdGEiOiBkZWFsfSkKCgpkZWYgX3NlcnZlKGhhbmRsZXJfY2xzKToKICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgaGFuZGxlcl9jbHMpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpLnN0YXJ0KCkKICAgIHJldHVybiBzZXJ2ZXIsIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIHNlcnZpY2VzKCk6CiAgICBfTW9ja05vdnUucmVxdWVzdF9sb2cgPSBbXQogICAgX01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2cgPSBbXQogICAgX01vY2tQaXBlZHJpdmUuZGVhbHMgPSBbXQogICAgbm92dV9zZXJ2ZXIsIG5vdnVfdXJsID0gX3NlcnZlKF9Nb2NrTm92dSkKICAgIHBkX3NlcnZlciwgcGRfdXJsID0gX3NlcnZlKF9Nb2NrUGlwZWRyaXZlKQogICAgdHJ5OgogICAgICAgIHlpZWxkIG5vdnVfdXJsLCBwZF91cmwKICAgIGZpbmFsbHk6CiAgICAgICAgbm92dV9zZXJ2ZXIuc2h1dGRvd24oKQogICAgICAgIG5vdnVfc2VydmVyLnNlcnZlcl9jbG9zZSgpCiAgICAgICAgcGRfc2VydmVyLnNodXRkb3duKCkKICAgICAgICBwZF9zZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3Rfc3luY19pc19jb21wbGV0ZV9hbmRfaWRlbXBvdGVudChzZXJ2aWNlcyk6CiAgICBub3Z1X3VybCwgcGRfdXJsID0gc2VydmljZXMKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgInN5bmNfc3Vic2NyaWJlcnNfdG9fZGVhbHMiKSwgKAogICAgICAgICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBzeW5jX3N1YnNjcmliZXJzX3RvX2RlYWxzKG5vdnVfYmFzZV91cmwsIG5vdnVfYXBpX2tleSwgcGRfYmFzZV91cmwsIHBkX2FwaV90b2tlbikiCiAgICApCgogICAgIyBSdW4gMTogZW1wdHkgc2luayDigJQgZXZlcnkgc3Vic2NyaWJlciBiZWNvbWVzIGEgZGVhbC4KICAgIHJlc3VsdDEgPSBtb2R1bGUuc3luY19zdWJzY3JpYmVyc190b19kZWFscyhub3Z1X3VybCwgTk9WVV9UT0tFTiwgcGRfdXJsLCBQRF9UT0tFTikKICAgIGFzc2VydCBpc2luc3RhbmNlKHJlc3VsdDEsIGRpY3QpIGFuZCByZXN1bHQxLmdldCgic3luY2VkIikgPT0gbGVuKFNVQlNDUklCRVJTKSwgKAogICAgICAgIGYiZmlyc3QgcnVuOiBleHBlY3RlZCB7eydzeW5jZWQnOiB7bGVuKFNVQlNDUklCRVJTKX19fSwgZ290IHtyZXN1bHQxIXJ9LiAiCiAgICAgICAgZiJOb3Z1IHNhdzoge19Nb2NrTm92dS5yZXF1ZXN0X2xvZ307IFBpcGVkcml2ZSBzYXc6IHtfTW9ja1BpcGVkcml2ZS5yZXF1ZXN0X2xvZ30iCiAgICApCiAgICB0aXRsZXMgPSBzb3J0ZWQoZFsidGl0bGUiXSBmb3IgZCBpbiBfTW9ja1BpcGVkcml2ZS5kZWFscykKICAgIGV4cGVjdGVkX3RpdGxlcyA9IHNvcnRlZChzWyJlbWFpbCJdIGZvciBzIGluIFNVQlNDUklCRVJTKQogICAgYXNzZXJ0IHRpdGxlcyA9PSBleHBlY3RlZF90aXRsZXMsICgKICAgICAgICBmImZpcnN0IHJ1bjogUGlwZWRyaXZlIHN0b3JlIGhvbGRzIHRpdGxlcyB7dGl0bGVzfSwgZXhwZWN0ZWQge2V4cGVjdGVkX3RpdGxlc30uICIKICAgICAgICBmIk5vdnUgc2F3OiB7X01vY2tOb3Z1LnJlcXVlc3RfbG9nfTsgUGlwZWRyaXZlIHNhdzoge19Nb2NrUGlwZWRyaXZlLnJlcXVlc3RfbG9nfSIKICAgICkKICAgIG5vdnVfaGl0cyA9IFtyIGZvciByIGluIF9Nb2NrTm92dS5yZXF1ZXN0X2xvZyBpZiByWyJwYXRoIl0uc3RhcnRzd2l0aCgiL3YyL3N1YnNjcmliZXJzIildCiAgICBhc3NlcnQgbGVuKG5vdnVfaGl0cykgPj0gMyBhbmQgYWxsKHJbImFwaWtleV9zY2hlbWUiXSBmb3IgciBpbiBub3Z1X2hpdHMpLCAoCiAgICAgICAgZiJOb3Z1IGN1cnNvciBtdXN0IGJlIGZvbGxvd2VkIGFjcm9zcyAzIHBhZ2VzIHdpdGggQXBpS2V5IGF1dGg6IHtfTW9ja05vdnUucmVxdWVzdF9sb2d9IgogICAgKQoKICAgIGdldHNfYmVmb3JlID0gbGVuKFtyIGZvciByIGluIF9Nb2NrUGlwZWRyaXZlLnJlcXVlc3RfbG9nIGlmIHJbIm1ldGhvZCJdID09ICJHRVQiXSkKICAgIHBvc3RzX2JlZm9yZSA9IGxlbihbciBmb3IgciBpbiBfTW9ja1BpcGVkcml2ZS5yZXF1ZXN0X2xvZyBpZiByWyJtZXRob2QiXSA9PSAiUE9TVCJdKQoKICAgICMgUnVuIDI6IHNpbmsgYWxyZWFkeSBob2xkcyBldmVyeSBzdWJzY3JpYmVyIOKAlCBub3RoaW5nIG5ldyBtYXkgYmUgY3JlYXRlZC4KICAgIHJlc3VsdDIgPSBtb2R1bGUuc3luY19zdWJzY3JpYmVyc190b19kZWFscyhub3Z1X3VybCwgTk9WVV9UT0tFTiwgcGRfdXJsLCBQRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQyLCBkaWN0KSBhbmQgcmVzdWx0Mi5nZXQoInN5bmNlZCIpID09IDAsICgKICAgICAgICBmInNlY29uZCBydW4gbXVzdCBjcmVhdGUgbm90aGluZyBhbmQgcmV0dXJuIHt7J3N5bmNlZCc6IDB9fSwgZ290IHtyZXN1bHQyIXJ9LiAiCiAgICAgICAgZiJQaXBlZHJpdmUgc2F3OiB7X01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2d9IgogICAgKQogICAgdGl0bGVzX2FmdGVyID0gc29ydGVkKGRbInRpdGxlIl0gZm9yIGQgaW4gX01vY2tQaXBlZHJpdmUuZGVhbHMpCiAgICBhc3NlcnQgdGl0bGVzX2FmdGVyID09IGV4cGVjdGVkX3RpdGxlcywgKAogICAgICAgIGYic2Vjb25kIHJ1biBkdXBsaWNhdGVkIGRlYWxzOiBzdG9yZSBob2xkcyB7dGl0bGVzX2FmdGVyfSwgZXhwZWN0ZWQgc3RpbGwgZXhhY3RseSAiCiAgICAgICAgZiJ7ZXhwZWN0ZWRfdGl0bGVzfS4gRXhpc3Rpbmcgc3RhdGUgbXVzdCBiZSByZWFkIGZyb20gUGlwZWRyaXZlIChsaXN0IGRlYWxzLCBmb2xsb3dpbmcgIgogICAgICAgIGYiY3Vyc29yIHBhZ2luYXRpb24gdG8gY29tcGxldGlvbikgYW5kIGFscmVhZHktc3luY2VkIHRpdGxlcyBza2lwcGVkLiAiCiAgICAgICAgZiJQaXBlZHJpdmUgc2F3OiB7X01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2d9IgogICAgKQogICAgcG9zdHNfYWZ0ZXIgPSBsZW4oW3IgZm9yIHIgaW4gX01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2cgaWYgclsibWV0aG9kIl0gPT0gIlBPU1QiXSkKICAgIGFzc2VydCBwb3N0c19hZnRlciA9PSBwb3N0c19iZWZvcmUsICgKICAgICAgICBmInNlY29uZCBydW4gbXVzdCBQT1NUIG5vdGhpbmc7IHNhdyB7cG9zdHNfYWZ0ZXIgLSBwb3N0c19iZWZvcmV9IG5ldyBQT1NUczogIgogICAgICAgIGYie19Nb2NrUGlwZWRyaXZlLnJlcXVlc3RfbG9nfSIKICAgICkKICAgICMgVGhlIHNlY29uZCBydW4gbXVzdCBkZXJpdmUgc3RhdGUgZnJvbSB0aGUgU0lOSyBpdHNlbGY6IDcgc3RvcmVkIGRlYWxzIGF0CiAgICAjIHNlcnZlciBwYWdlIHNpemUgMyA9IDMgbGlzdCBHRVRzIHRvIGVudW1lcmF0ZS4gQW4gaW4tcHJvY2VzcyBjYWNoZSAobm8KICAgICMgcmUtcmVhZCkgb3IgYW4gb2Zmc2V0LXBhZ2luYXRlZCBsaXN0IChzdG9wcyBhZnRlciBwYWdlIDEpIGZhaWxzIGhlcmUuCiAgICBnZXRzX2FmdGVyID0gbGVuKFtyIGZvciByIGluIF9Nb2NrUGlwZWRyaXZlLnJlcXVlc3RfbG9nIGlmIHJbIm1ldGhvZCJdID09ICJHRVQiXSkKICAgIGFzc2VydCBnZXRzX2FmdGVyIC0gZ2V0c19iZWZvcmUgPj0gMywgKAogICAgICAgIGYic2Vjb25kIHJ1biBtdXN0IHJlLXJlYWQgdGhlIFBpcGVkcml2ZSBkZWFscyBsaXN0IHRvIGNvbXBsZXRpb24gIgogICAgICAgIGYiKD49IDMgY3Vyc29yIHBhZ2VzKSwgc2F3IHtnZXRzX2FmdGVyIC0gZ2V0c19iZWZvcmV9IEdFVHM6IHtfTW9ja1BpcGVkcml2ZS5yZXF1ZXN0X2xvZ30iCiAgICApCg==" - }, - { - "vendor": "cal-com:list_all_bookings", - "fn": "list_all_bookings", - "testFile": "test_calcom_list.py", - "signature": "def list_all_bookings(base_url: str, api_key: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgY2FsY29tLXYyLWxpc3QtYWxsLWJvb2tpbmdzIChoYXJkKSB3ZWItZ3JvdW5kZWQgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpJdCBzdGFuZHMgdXAgYSBMT0NBTCBtb2NrIHRoYXQgZmFpdGhmdWxseSBpbXBsZW1lbnRzIENhbC5jb20ncyBDVVJSRU5UIChBUEkgdjIpIGJvb2tpbmdzLWxpc3QKY29udHJhY3Q6CiAgLSBlbmRwb2ludCBwYXRoICAgICAgICAvdjIvYm9va2luZ3MgICAgICAgICAgICAgICAgICAgICAodjEgd2FzIC92MS9ib29raW5ncykKICAtIGF1dGggICAgICAgICAgICAgICAgIEF1dGhvcml6YXRpb246IEJlYXJlciA8YXBpX2tleT4gICAodjEgdXNlZCA/YXBpS2V5PTxrZXk+IHF1ZXJ5IHBhcmFtKQogIC0gYXBpLXZlcnNpb24gaGVhZGVyICAgY2FsLWFwaS12ZXJzaW9uOiAyMDI2LTA1LTAxICAgICAgIChQT1NULUNVVE9GRjsgYSBmcm9tLW1lbW9yeSBjbGllbnQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb21pdHMgaXQgb3Igc2VuZHMgYW4gb2xkZXIgZGF0ZSBhbmQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2lsZW50bHkgZ2V0cyB0aGUgbGVnYWN5IHJlc3BvbnNlIHNoYXBlKQogIC0gcmVzcG9uc2UgZW52ZWxvcGUgICAgeyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6Wy4uLl0sInBhZ2luYXRpb24iOnsibmV4dEN1cnNvciIsImhhc01vcmUifX0KICAgICAgICAgICAgICAgICAgICAgICAgIChsZWdhY3kgc2hhcGUgd2FzIGEgYmFyZSB7ImJvb2tpbmdzIjpbLi4uXX0gd2l0aCBubyBjdXJzb3IgcGFnaW5hdGlvbikKICAtIHBhZ2luYXRpb24gICAgICAgICAgIGN1cnNvci1iYXNlZDogcXVlcnkgYGN1cnNvcmAsIHJlc3BvbnNlIHBhZ2luYXRpb24ubmV4dEN1cnNvciAvIGhhc01vcmUKCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIENhbC5jb20gZG9jcyBjb2xsZWN0cyBhbGwgNyBib29raW5ncyBhY3Jvc3MgMyBjdXJzb3IgcGFnZXMgYW5kCnBhc3Nlcy4gQSBjbGllbnQgd3JpdHRlbiBmcm9tIHN0YWxlL3ByZS1jdXRvZmYgbWVtb3J5ICh2MSBwYXRoLCBxdWVyeS1wYXJhbSBhdXRoLCBtaXNzaW5nL29sZApjYWwtYXBpLXZlcnNpb24sIG9yIGJhcmUteyJib29raW5ncyJ9IHBhcnNpbmcpIGhpdHMgNDA0IC8gNDAxIC8gYSBsZWdhY3kgZW52ZWxvcGUgaXQgY2Fubm90CnBhcnNlIC8gc3RvcHMgYWZ0ZXIgcGFnZSAxIGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9jYWwuY29tL2RvY3MvYXBpLXJlZmVyZW5jZS92Mi9ib29raW5ncy9nZXQtYWxsLWJvb2tpbmdzCiAgaHR0cHM6Ly9jYWwuY29tL2RvY3MvYXBpLXJlZmVyZW5jZS92Mi9pbnRyb2R1Y3Rpb24KIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgojIFRoZSBBUEkga2V5IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBBdXRob3JpemF0aW9uOiBCZWFyZXIgSEVBREVSICh2MiBjb250cmFjdCkuCkVYUEVDVEVEX0FQSV9LRVkgPSAiY2FsX2xpdmVfa2V5X2FiYzEyMyIKCiMgVGhlIENVUlJFTlQgKHBvc3QtY3V0b2ZmKSBhcGktdmVyc2lvbiB0aGUgR0VUIC92Mi9ib29raW5ncyBlbmRwb2ludCByZXF1aXJlcy4gQSBjbGllbnQgdGhhdAojIG9taXRzIHRoaXMgaGVhZGVyIG9yIHNlbmRzIGFuIG9sZGVyIGRhdGUgaXMgZGVmYXVsdGVkIHRvIHRoZSBsZWdhY3kgcmVzcG9uc2Ugc2hhcGUuClJFUVVJUkVEX0FQSV9WRVJTSU9OID0gIjIwMjYtMDUtMDEiCgojIFRoZSBoaWRkZW4gZGF0YXNldC4gT25seSBvYnNlcnZhYmxlIGJ5IGFjdHVhbGx5IGNhbGxpbmcgdGhlIG1vY2sgYW5kIHBhZ2luYXRpbmcgdG8gdGhlIGVuZC4KIyA3IGJvb2tpbmdzLCBzZXJ2ZXIgcGFnZSBzaXplIDMgLT4gMyBjdXJzb3IgcGFnZXMuCkJPT0tJTkdTID0gWwogICAgeyJpZCI6IGksICJ1aWQiOiBmImJrX3tpfSIsICJ0aXRsZSI6IGYiTWVldGluZyB7aX0iLCAic3RhdHVzIjogImFjY2VwdGVkIn0KICAgIGZvciBpIGluIHJhbmdlKDEsIDgpCl0KUEFHRV9TSVpFID0gMwoKCmNsYXNzIF9Nb2NrQ2FsY29tKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgIyBDbGFzcy1sZXZlbCByZXF1ZXN0IGxvZyBzbyB0aGUgdGVzdCBjYW4gaW5zcGVjdCB3aGF0IHBhdGgvYXV0aC92ZXJzaW9uIHRoZSBjbGllbnQgdXNlZC4KICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKTogICMgc2lsZW5jZSBzdGRlcnIgYWNjZXNzIGxvZwogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgYXV0aCA9IHNlbGYuaGVhZGVycy5nZXQoIkF1dGhvcml6YXRpb24iKQogICAgICAgIGFwaV92ZXJzaW9uID0gc2VsZi5oZWFkZXJzLmdldCgiY2FsLWFwaS12ZXJzaW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgInF1ZXJ5X2FwaUtleSI6ICJhcGlLZXkiIGluIHFzLAogICAgICAgICAgICAiY2FsX2FwaV92ZXJzaW9uIjogYXBpX3ZlcnNpb24sCiAgICAgICAgICAgICJjdXJzb3IiOiBxcy5nZXQoImN1cnNvciIsIFtOb25lXSlbMF0sCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiBvbmx5IHRoZSB2MiBwYXRoIGV4aXN0cy4KICAgICAgICBpZiBwYXRoICE9ICIvdjIvYm9va2luZ3MiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJzdGF0dXMiOiAiZXJyb3IiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6IHsiY29kZSI6ICJOT1RfRk9VTkQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJDYWwuY29tIEFQSSB2MiBsaXN0cyBib29raW5ncyBhdCAvdjIvYm9va2luZ3MifX0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IHYyIHJlcXVpcmVzIEF1dGhvcml6YXRpb246IEJlYXJlciA8a2V5PjsgcXVlcnktcGFyYW0gYXBpS2V5IGlzIG5vdCBob25vcmVkLgogICAgICAgIGlmIGF1dGggIT0gZiJCZWFyZXIge0VYUEVDVEVEX0FQSV9LRVl9IjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsic3RhdHVzIjogImVycm9yIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3IiOiB7ImNvZGUiOiAiVU5BVVRIT1JJWkVEIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiQ2FsLmNvbSBBUEkgdjIgYXV0aGVudGljYXRlcyB2aWEgQXV0aG9yaXphdGlvbjogQmVhcmVyIDxhcGlfa2V5PiJ9fSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDMpIGFwaS12ZXJzaW9uIGF4aXM6IHdpdGhvdXQgdGhlIENVUlJFTlQgKHBvc3QtY3V0b2ZmKSBjYWwtYXBpLXZlcnNpb24gdGhlIGVuZHBvaW50CiAgICAgICAgIyBkZWZhdWx0cyB0byB0aGUgTEVHQUNZIHJlc3BvbnNlIHNoYXBlIChiYXJlIHsiYm9va2luZ3MiOlsuLi5dfSwgbm8gY3Vyc29yIHBhZ2luYXRpb24pLgogICAgICAgICMgSVNPIFlZWVktTU0tREQgZGF0ZXMgY29tcGFyZSBjb3JyZWN0bHkgYXMgc3RyaW5ncy4KICAgICAgICBpZiBhcGlfdmVyc2lvbiBpcyBOb25lIG9yIGFwaV92ZXJzaW9uIDwgUkVRVUlSRURfQVBJX1ZFUlNJT046CiAgICAgICAgICAgIHNlbGYuX2pzb24oMjAwLCB7ImJvb2tpbmdzIjogQk9PS0lOR1NbOlBBR0VfU0laRV19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoNCkgcGFnaW5hdGlvbiBheGlzOiBjdXJzb3ItYmFzZWQgYWdhaW5zdCB0aGUgY3VycmVudCBlbnZlbG9wZS4KICAgICAgICBjdXJzb3IgPSBxcy5nZXQoImN1cnNvciIsIFtOb25lXSlbMF0KICAgICAgICBvZmZzZXQgPSBpbnQoY3Vyc29yKSBpZiBjdXJzb3IgaXMgbm90IE5vbmUgYW5kIGN1cnNvci5pc2RpZ2l0KCkgZWxzZSAwCiAgICAgICAgcGFnZSA9IEJPT0tJTkdTW29mZnNldDpvZmZzZXQgKyBQQUdFX1NJWkVdCiAgICAgICAgbmV4dF9vZmZzZXQgPSBvZmZzZXQgKyBQQUdFX1NJWkUKICAgICAgICBoYXNfbW9yZSA9IG5leHRfb2Zmc2V0IDwgbGVuKEJPT0tJTkdTKQogICAgICAgIG5leHRfY3Vyc29yID0gc3RyKG5leHRfb2Zmc2V0KSBpZiBoYXNfbW9yZSBlbHNlIE5vbmUKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAic3RhdHVzIjogInN1Y2Nlc3MiLAogICAgICAgICAgICAiZGF0YSI6IHBhZ2UsCiAgICAgICAgICAgICJwYWdpbmF0aW9uIjogeyJuZXh0Q3Vyc29yIjogbmV4dF9jdXJzb3IsICJoYXNNb3JlIjogaGFzX21vcmV9LAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0NhbGNvbS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrQ2FsY29tKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfbGlzdHNfYWxsX2Jvb2tpbmdzX2Fjcm9zc19jdXJzb3JfcGFnZXMobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAibGlzdF9hbGxfYm9va2luZ3MiKSwgXAogICAgICAgICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBsaXN0X2FsbF9ib29raW5ncyhiYXNlX3VybCwgYXBpX2tleSkiCgogICAgcmVzdWx0ID0gbW9kdWxlLmxpc3RfYWxsX2Jvb2tpbmdzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9BUElfS0VZKQoKICAgIGFzc2VydCBpc2luc3RhbmNlKHJlc3VsdCwgbGlzdCksIGYiZXhwZWN0ZWQgYSBsaXN0IG9mIGJvb2tpbmdzLCBnb3Qge3R5cGUocmVzdWx0KS5fX25hbWVfX30iCiAgICBpZHMgPSBzb3J0ZWQoYlsiaWQiXSBmb3IgYiBpbiByZXN1bHQgaWYgaXNpbnN0YW5jZShiLCBkaWN0KSBhbmQgImlkIiBpbiBiKQogICAgZXhwZWN0ZWRfaWRzID0gW2JbImlkIl0gZm9yIGIgaW4gQk9PS0lOR1NdCiAgICBhc3NlcnQgaWRzID09IGV4cGVjdGVkX2lkcywgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIGJvb2tpbmcgaWRzIHtpZHN9LCBleHBlY3RlZCB7ZXhwZWN0ZWRfaWRzfS4gIgogICAgICAgIGYiQSBjb3JyZWN0IHYyIGNsaWVudCBzZW5kcyBBdXRob3JpemF0aW9uOiBCZWFyZXIsIGNhbC1hcGktdmVyc2lvbjoge1JFUVVJUkVEX0FQSV9WRVJTSU9OfSwgIgogICAgICAgIGYiYW5kIGZvbGxvd3MgcGFnaW5hdGlvbi5uZXh0Q3Vyc29yIHRvIHRoZSBlbmQuICIKICAgICAgICBmIlNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0NhbGNvbS5yZXF1ZXN0X2xvZ30iCiAgICApCg==" - }, - { - "vendor": "cal-com:get_booking", - "fn": "get_booking", - "testFile": "test_calcom_get.py", - "signature": "def get_booking(base_url: str, api_key: str, booking_uid: str) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgY2FsY29tLXYyLWdldC1ib29raW5nIChtZWRpdW0pIHdlYi1ncm91bmRlZCBsZWFmLgoKU2ltcGxlciBzbGljZSBvZiB0aGUgU0FNRSBDYWwuY29tIEFQSSB2MiBjb250cmFjdCAtLSBhIHNpbmdsZS1yZXNvdXJjZSBHRVQsIG5vIHBhZ2luYXRpb24uClN0aWxsIHNlYXJjaC1uZWNlc3Nhcnkgb24gdGhyZWUgYXhlczogdGhlIHYyIGVuZHBvaW50IHBhdGgsIHRoZSBBdXRob3JpemF0aW9uOiBCZWFyZXIgaGVhZGVyLAphbmQgdGhlIFBPU1QtQ1VUT0ZGIGNhbC1hcGktdmVyc2lvbiBoZWFkZXIgKDIwMjYtMDItMjUgZm9yIHRoZSBzaW5nbGUtYm9va2luZyBlbmRwb2ludCkuIEEKZnJvbS1tZW1vcnkgY2xpZW50IHVzZXMgL3YxL2Jvb2tpbmdzL3tpZH0/YXBpS2V5PS4uLiBvciBvbWl0cyB0aGUgdmVyc2lvbiBoZWFkZXIgYW5kIGZhaWxzLgoKICAtIGVuZHBvaW50IHBhdGggICAgICAgIC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0gICAgICAgICAodjEgd2FzIC92MS9ib29raW5ncy97aWR9KQogIC0gYXV0aCAgICAgICAgICAgICAgICAgQXV0aG9yaXphdGlvbjogQmVhcmVyIDxhcGlfa2V5PiAgICh2MSB1c2VkID9hcGlLZXk9PGtleT4gcXVlcnkgcGFyYW0pCiAgLSBhcGktdmVyc2lvbiBoZWFkZXIgICBjYWwtYXBpLXZlcnNpb246IDIwMjYtMDItMjUgICAgICAgKHBvc3QtY3V0b2ZmOyBvbWl0IGl0IGFuZCB0aGUgZW5kcG9pbnQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGVmYXVsdHMgdG8gdGhlIGxlZ2FjeSByZXNwb25zZSBzaGFwZSkKICAtIHJlc3BvbnNlIGVudmVsb3BlICAgIHsic3RhdHVzIjoic3VjY2VzcyIsImRhdGEiOnsuLi59fSAobGVnYWN5IHNoYXBlIHdhcyBhIGJhcmUgeyJib29raW5nIjp7Li4ufX0pCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vY2FsLmNvbS9kb2NzL2FwaS1yZWZlcmVuY2UvdjIvYm9va2luZ3MvZ2V0LWEtYm9va2luZwogIGh0dHBzOi8vY2FsLmNvbS9kb2NzL2FwaS1yZWZlcmVuY2UvdjIvaW50cm9kdWN0aW9uCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgcmUKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX0FQSV9LRVkgPSAiY2FsX2xpdmVfa2V5X2FiYzEyMyIKUkVRVUlSRURfQVBJX1ZFUlNJT04gPSAiMjAyNi0wMi0yNSIKQk9PS0lOR19VSUQgPSAiYmtfZXZ0XzlmM2EiICAjIHRoZSB1aWQgdGhlIHRlc3QgZmV0Y2hlczsgdGhlIG1vY2sgc3ludGhlc2l6ZXMgYSBib29raW5nIGZvciBhbnkgdWlkCgoKY2xhc3MgX01vY2tDYWxjb20oQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBhdXRoID0gc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpCiAgICAgICAgYXBpX3ZlcnNpb24gPSBzZWxmLmhlYWRlcnMuZ2V0KCJjYWwtYXBpLXZlcnNpb24iKQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImF1dGhvcml6YXRpb24iOiBhdXRoLAogICAgICAgICAgICAicXVlcnlfYXBpS2V5IjogImFwaUtleSIgaW4gcXMsCiAgICAgICAgICAgICJjYWxfYXBpX3ZlcnNpb24iOiBhcGlfdmVyc2lvbiwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogdjIgZmV0Y2hlcyBhIHNpbmdsZSBib29raW5nIGF0IC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi92Mi9ib29raW5ncy8oW0EtWmEtejAtOV9cLV0rKSIsIHBhdGgpCiAgICAgICAgaWYgbm90IG06CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN0YXR1cyI6ICJlcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yIjogeyJjb2RlIjogIk5PVF9GT1VORCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkNhbC5jb20gQVBJIHYyIGZldGNoZXMgYSBib29raW5nIGF0IC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0ifX0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjIGF1dGggYXhpczogdjIgcmVxdWlyZXMgQXV0aG9yaXphdGlvbjogQmVhcmVyIDxrZXk+OyBxdWVyeS1wYXJhbSBhcGlLZXkgaXMgbm90IGhvbm9yZWQuCiAgICAgICAgaWYgYXV0aCAhPSBmIkJlYXJlciB7RVhQRUNURURfQVBJX0tFWX0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJzdGF0dXMiOiAiZXJyb3IiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6IHsiY29kZSI6ICJVTkFVVEhPUklaRUQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJDYWwuY29tIEFQSSB2MiBhdXRoZW50aWNhdGVzIHZpYSBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFwaV9rZXk+In19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgYm9va2luZ191aWQgPSBtLmdyb3VwKDEpCiAgICAgICAgYm9va2luZyA9IHsiaWQiOiA3NzcsICJ1aWQiOiBib29raW5nX3VpZCwgInRpdGxlIjogZiJCb29raW5nIHtib29raW5nX3VpZH0iLCAic3RhdHVzIjogImFjY2VwdGVkIn0KCiAgICAgICAgIyBhcGktdmVyc2lvbiBheGlzOiB3aXRob3V0IHRoZSBDVVJSRU5UIChwb3N0LWN1dG9mZikgY2FsLWFwaS12ZXJzaW9uIHRoZSBlbmRwb2ludCBkZWZhdWx0cwogICAgICAgICMgdG8gdGhlIExFR0FDWSByZXNwb25zZSBzaGFwZSAoYmFyZSB7ImJvb2tpbmciOnsuLi59fSwgbm8gc3RhdHVzL2RhdGEgZW52ZWxvcGUpLgogICAgICAgIGlmIGFwaV92ZXJzaW9uIGlzIE5vbmUgb3IgYXBpX3ZlcnNpb24gPCBSRVFVSVJFRF9BUElfVkVSU0lPTjoKICAgICAgICAgICAgc2VsZi5fanNvbigyMDAsIHsiYm9va2luZyI6IGJvb2tpbmd9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsic3RhdHVzIjogInN1Y2Nlc3MiLCAiZGF0YSI6IGJvb2tpbmd9KQoKCkBweXRlc3QuZml4dHVyZSgpCmRlZiBtb2NrX3NlcnZlcigpOgogICAgX01vY2tDYWxjb20ucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0NhbGNvbSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9ib29raW5nX2J5X3VpZChtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfYm9va2luZyIpLCBcCiAgICAgICAgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IGdldF9ib29raW5nKGJhc2VfdXJsLCBhcGlfa2V5LCBib29raW5nX3VpZCkiCgogICAgYm9va2luZyA9IG1vZHVsZS5nZXRfYm9va2luZyhtb2NrX3NlcnZlciwgRVhQRUNURURfQVBJX0tFWSwgQk9PS0lOR19VSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoYm9va2luZywgZGljdCksIGYiZXhwZWN0ZWQgYSBib29raW5nIGRpY3QsIGdvdCB7dHlwZShib29raW5nKS5fX25hbWVfX30iCiAgICBhc3NlcnQgYm9va2luZy5nZXQoInVpZCIpID09IEJPT0tJTkdfVUlEIGFuZCBib29raW5nLmdldCgidGl0bGUiKSA9PSBmIkJvb2tpbmcge0JPT0tJTkdfVUlEfSIsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCB7Ym9va2luZyFyfTsgYSBjb3JyZWN0IHYyIGNsaWVudCBHRVRzIC92Mi9ib29raW5ncy97Qk9PS0lOR19VSUR9IHdpdGggIgogICAgICAgIGYiQXV0aG9yaXphdGlvbjogQmVhcmVyIGFuZCBjYWwtYXBpLXZlcnNpb246IHtSRVFVSVJFRF9BUElfVkVSU0lPTn0sIHRoZW4gcmVhZHMgZGF0YS4gIgogICAgICAgIGYiU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrQ2FsY29tLnJlcXVlc3RfbG9nfSIKICAgICkK" - }, - { - "vendor": "bunny:list_all_file_paths", - "fn": "list_all_file_paths", - "testFile": "test_bunny_list_all.py", - "signature": "def list_all_file_paths(base_url: str, storage_zone: str, access_key: str, path: str) -> list[str]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYnVubnktc3RvcmFnZS1saXN0LWFsbC1maWxlcyAoaGFyZCkgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpTYW1lIEJ1bm55Lm5ldCBFZGdlIFN0b3JhZ2UgY29udHJhY3QgYXMgdGhlIG1lZGl1bSBsZWFmLCBwbHVzIG9uZSBtb3JlIGF4aXM6IEJ1bm55J3MgbGlzdAplbmRwb2ludCBpcyBOT1QgcmVjdXJzaXZlLCBzbyBsaXN0aW5nIEVWRVJZIGZpbGUgcmVxdWlyZXMgZGVzY2VuZGluZyBpbnRvIGV2ZXJ5IGVudHJ5IHdob3NlCklzRGlyZWN0b3J5IGlzIHRydWUgYW5kIHJlLWxpc3RpbmcgaXQuIFRoZSBncmFkZSByZXF1aXJlczoKICAtIGF1dGggICAgICAgICAgICBBY2Nlc3NLZXk6IDxzdG9yYWdlX3pvbmVfcGFzc3dvcmQ+IEhFQURFUiAobm90IEJlYXJlcikKICAtIGVuZHBvaW50ICAgICAgICBHRVQge2Jhc2V9L3tzdG9yYWdlWm9uZU5hbWV9L3twYXRofS8gICAocGF0aC1zdHlsZSwgb25lIHJlcXVlc3QgcGVyIGRpcmVjdG9yeSkKICAtIHJlc3BvbnNlLXNoYXBlICBwYXJzZSBlYWNoIEpTT04tYXJyYXkgb2JqZWN0J3MgUGFzY2FsQ2FzZSBPYmplY3ROYW1lICsgSXNEaXJlY3RvcnkKICAtIHRyYXZlcnNhbCAgICAgICByZWN1cnNlIGludG8gSXNEaXJlY3Rvcnk9PXRydWUgZW50cmllcyBhbmQgYWNjdW11bGF0ZSB0aGUgZnVsbCByZWxhdGl2ZQogICAgICAgICAgICAgICAgICAgIHBhdGggb2YgZXZlcnkgSXNEaXJlY3Rvcnk9PWZhbHNlIGZpbGUKCkEgY29ycmVjdCBjdXJyZW50LWNvbnRyYWN0IGNsaWVudCB3YWxrcyB0aGUgdHJlZSBhbmQgcmV0dXJucyBhbGwgNSBmaWxlIHBhdGhzLiBBIGNsaWVudCB0aGF0CnVzZXMgQmVhcmVyIGF1dGggKDQwMSksIHBhcnNlcyBzbmFrZV9jYXNlIGZpZWxkIG5hbWVzIChlbXB0eSksIG9yIGRvZXMgbm90IHJlY3Vyc2UgKHJldHVybnMgb25seQp0aGUgc2luZ2xlIHRvcC1sZXZlbCBmaWxlKSBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9kb2NzLmJ1bm55Lm5ldC9yZWZlcmVuY2Uvc3RvcmFnZS1hcGkKICBodHRwczovL2RvY3MuYnVubnkubmV0L2RvY3MvZWRnZS1zdG9yYWdlLW92ZXJ2aWV3CiAgaHR0cHM6Ly9idW5ueWNkbnN0b3JhZ2UuZG9jcy5hcGlhcnkuaW8vCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9LRVkgPSAiYnVubnktc3otcHctOWYzYWMyMWUiCkVYUEVDVEVEX1pPTkUgPSAibWVkaWEtYXNzZXRzIgoKIyBUaGUgaGlkZGVuIGRpcmVjdG9yeSB0cmVlLCBrZXllZCBieSBkaXJlY3RvcnkgcGF0aCByZWxhdGl2ZSB0byB0aGUgem9uZSByb290ICgiIiA9IHJvb3QpLgojIEJ1bm55J3MgbGlzdCBlbmRwb2ludCByZXR1cm5zIG9ubHkgdGhlIGltbWVkaWF0ZSBjaGlsZHJlbiBvZiBvbmUgZGlyZWN0b3J5LCBzbyBhIGNsaWVudCBtdXN0CiMgaXNzdWUgb25lIHJlcXVlc3QgcGVyIGRpcmVjdG9yeSBhbmQgZm9sbG93IElzRGlyZWN0b3J5IHRvIHJlYWNoIGV2ZXJ5IGZpbGUuClRSRUUgPSB7CiAgICAiIjogWygiaW1hZ2VzIiwgVHJ1ZSksICgibm90ZXMudHh0IiwgRmFsc2UpLCAoImRhdGEiLCBUcnVlKV0sCiAgICAiaW1hZ2VzIjogWygibG9nby5wbmciLCBGYWxzZSksICgiaWNvbnMiLCBUcnVlKV0sCiAgICAiaW1hZ2VzL2ljb25zIjogWygiaG9tZS5zdmciLCBGYWxzZSksICgibWVudS5zdmciLCBGYWxzZSldLAogICAgImRhdGEiOiBbKCJyZWNvcmRzLmNzdiIsIEZhbHNlKV0sCn0KCiMgRXZlcnkgSXNEaXJlY3Rvcnk9PWZhbHNlIGxlYWYsIGFzIGEgZnVsbCByZWxhdGl2ZSBwYXRoIOKAlCB0aGUgZXhwZWN0ZWQgcmV0dXJuIG9mIGEgY29ycmVjdCBjbGllbnQuCkVYUEVDVEVEX0ZJTEVTID0gc29ydGVkKFsKICAgICJub3Rlcy50eHQiLAogICAgImltYWdlcy9sb2dvLnBuZyIsCiAgICAiaW1hZ2VzL2ljb25zL2hvbWUuc3ZnIiwKICAgICJpbWFnZXMvaWNvbnMvbWVudS5zdmciLAogICAgImRhdGEvcmVjb3Jkcy5jc3YiLApdKQoKCmRlZiBfbWFrZV9vYmplY3QoZGlyX3BhdGgsIG5hbWUsIGlzX2RpcmVjdG9yeSk6CiAgICBwYXJlbnQgPSAiLyIgKyBFWFBFQ1RFRF9aT05FICsgIi8iICsgKGRpcl9wYXRoICsgIi8iIGlmIGRpcl9wYXRoIGVsc2UgIiIpCiAgICByZXR1cm4gewogICAgICAgICJHdWlkIjogIjAwMDAwMDAwLTAwMDAtMDAwMC0wMDAwLTAwMDAwMDAwMDAwMCIsCiAgICAgICAgIlN0b3JhZ2Vab25lTmFtZSI6IEVYUEVDVEVEX1pPTkUsCiAgICAgICAgIlBhdGgiOiBwYXJlbnQsCiAgICAgICAgIk9iamVjdE5hbWUiOiBuYW1lLAogICAgICAgICJMZW5ndGgiOiAwIGlmIGlzX2RpcmVjdG9yeSBlbHNlIDIwNDgsCiAgICAgICAgIkxhc3RDaGFuZ2VkIjogIjIwMjYtMDctMDFUMDA6MDA6MDAuMDAwIiwKICAgICAgICAiU2VydmVySWQiOiAxLAogICAgICAgICJBcnJheU51bWJlciI6IDAsCiAgICAgICAgIklzRGlyZWN0b3J5IjogaXNfZGlyZWN0b3J5LAogICAgICAgICJVc2VySWQiOiAiMDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAwIiwKICAgICAgICAiQ29udGVudFR5cGUiOiAiIiBpZiBpc19kaXJlY3RvcnkgZWxzZSAiYXBwbGljYXRpb24vb2N0ZXQtc3RyZWFtIiwKICAgICAgICAiRGF0ZUNyZWF0ZWQiOiAiMjAyNi0wNy0wMVQwMDowMDowMC4wMDAiLAogICAgICAgICJTdG9yYWdlWm9uZUlkIjogMTIzNDU2LAogICAgICAgICJDaGVja3N1bSI6IE5vbmUgaWYgaXNfZGlyZWN0b3J5IGVsc2UgIjAiICogNjQsCiAgICAgICAgIlJlcGxpY2F0ZWRab25lcyI6ICIiLAogICAgfQoKCmNsYXNzIF9Nb2NrQnVubnlTdG9yYWdlKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOgogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBoZWFkZXJfa2V5cyA9IHtrLmxvd2VyKCkgZm9yIGsgaW4gc2VsZi5oZWFkZXJzLmtleXMoKX0KICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJoYXNfYWNjZXNza2V5X2hlYWRlciI6ICJhY2Nlc3NrZXkiIGluIGhlYWRlcl9rZXlzLAogICAgICAgICAgICAiaGFzX2F1dGhvcml6YXRpb25faGVhZGVyIjogImF1dGhvcml6YXRpb24iIGluIGhlYWRlcl9rZXlzLAogICAgICAgIH0pCgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIkFjY2Vzc0tleSIpICE9IEVYUEVDVEVEX0tFWToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsiSHR0cENvZGUiOiA0MDEsICJNZXNzYWdlIjogIlVuYXV0aG9yaXplZCJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgcGFydHMgPSBbcCBmb3IgcCBpbiBwYXJzZWQucGF0aC5zcGxpdCgiLyIpIGlmIHAgIT0gIiJdCiAgICAgICAgaWYgbm90IHBhcnRzIG9yIHBhcnRzWzBdICE9IEVYUEVDVEVEX1pPTkU6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7Ikh0dHBDb2RlIjogNDA0LCAiTWVzc2FnZSI6ICJOb3QgRm91bmQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGRpcl9wYXRoID0gIi8iLmpvaW4ocGFydHNbMTpdKQogICAgICAgIGlmIGRpcl9wYXRoIG5vdCBpbiBUUkVFOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJIdHRwQ29kZSI6IDQwNCwgIk1lc3NhZ2UiOiAiTm90IEZvdW5kIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBib2R5ID0gW19tYWtlX29iamVjdChkaXJfcGF0aCwgbmFtZSwgaXNfZGlyKSBmb3IgKG5hbWUsIGlzX2RpcikgaW4gVFJFRVtkaXJfcGF0aF1dCiAgICAgICAgc2VsZi5fanNvbigyMDAsIGJvZHkpCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0J1bm55U3RvcmFnZS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrQnVubnlTdG9yYWdlKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfbGlzdHNfYWxsX2ZpbGVzX3JlY3Vyc2l2ZWx5KG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2ZpbGVfcGF0aHMiKSwgKAogICAgICAgICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBsaXN0X2FsbF9maWxlX3BhdGhzKGJhc2VfdXJsLCBzdG9yYWdlX3pvbmUsIGFjY2Vzc19rZXksIHBhdGgpIgogICAgKQoKICAgIHJlc3VsdCA9IG1vZHVsZS5saXN0X2FsbF9maWxlX3BhdGhzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9aT05FLCBFWFBFQ1RFRF9LRVksICIiKQoKICAgIGFzc2VydCBpc2luc3RhbmNlKHJlc3VsdCwgbGlzdCksIGYiZXhwZWN0ZWQgYSBsaXN0IG9mIGZpbGUgcGF0aHMsIGdvdCB7dHlwZShyZXN1bHQpLl9fbmFtZV9ffSIKICAgIGdvdCA9IHNvcnRlZChzdHIoeCkuc3RyaXAoIi8iKSBmb3IgeCBpbiByZXN1bHQpCiAgICBhc3NlcnQgZ290ID09IEVYUEVDVEVEX0ZJTEVTLCAoCiAgICAgICAgZiJjbGllbnQgcmV0dXJuZWQgZmlsZSBwYXRocyB7Z290fSwgZXhwZWN0ZWQge0VYUEVDVEVEX0ZJTEVTfS4gIgogICAgICAgIGYiQSBjb3JyZWN0IGNsaWVudCBzZW5kcyB0aGUgQWNjZXNzS2V5IGhlYWRlciwgcmVhZHMgZWFjaCBlbnRyeSdzIFBhc2NhbENhc2UgT2JqZWN0TmFtZSArICIKICAgICAgICBmIklzRGlyZWN0b3J5LCBhbmQgcmVjdXJzZXMgaW50byBldmVyeSBkaXJlY3RvcnkgdG8gY29sbGVjdCBhbGwgZmlsZXMuICIKICAgICAgICBmIlNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0J1bm55U3RvcmFnZS5yZXF1ZXN0X2xvZ30iCiAgICApCg==" - }, - { - "vendor": "bunny:list_object_names", - "fn": "list_object_names", - "testFile": "test_bunny_list_directory.py", - "signature": "def list_object_names(base_url: str, storage_zone: str, access_key: str, path: str) -> list[str]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYnVubnktc3RvcmFnZS1saXN0LWRpcmVjdG9yeSAobWVkaXVtKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUgdGltZQooYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdCB0aGUgYWdlbnQncwpzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgQnVubnkubmV0J3MgRWRnZSBTdG9yYWdlCmRpcmVjdG9yeS1saXN0aW5nIGNvbnRyYWN0OgogIC0gYXV0aCAgICAgICAgICAgIEFjY2Vzc0tleTogPHN0b3JhZ2Vfem9uZV9wYXNzd29yZD4gSFRUUCBIRUFERVIKICAgICAgICAgICAgICAgICAgICAoTk9UIEF1dGhvcml6YXRpb24gLyBOT1QgQmVhcmVyOyBhIGZyb20tbWVtb3J5IGNsaWVudCBndWVzc2VzIEJlYXJlciBhbmQgNDAxcykKICAtIGVuZHBvaW50ICAgICAgICBHRVQge2Jhc2V9L3tzdG9yYWdlWm9uZU5hbWV9L3twYXRofS8gICAgICAgIChwYXRoLXN0eWxlLCB0cmFpbGluZyBzbGFzaCBsaXN0cyBhIGRpcikKICAtIHJlc3BvbnNlICAgICAgICBhIEpTT04gQVJSQVkgKG5vIGVudmVsb3BlKSBvZiBvYmplY3RzIHdob3NlIGtleXMgYXJlIFBhc2NhbENhc2U6CiAgICAgICAgICAgICAgICAgICAgT2JqZWN0TmFtZSwgSXNEaXJlY3RvcnksIExlbmd0aCwgUGF0aCwgU3RvcmFnZVpvbmVOYW1lLCBHdWlkLCBDaGVja3N1bSwKICAgICAgICAgICAgICAgICAgICBDb250ZW50VHlwZSwgTGFzdENoYW5nZWQsIERhdGVDcmVhdGVkLCBTdG9yYWdlWm9uZUlkLCBTZXJ2ZXJJZCwgVXNlcklkLAogICAgICAgICAgICAgICAgICAgIFJlcGxpY2F0ZWRab25lcwogICAgICAgICAgICAgICAgICAgIChhIGZyb20tbWVtb3J5IGNsaWVudCBndWVzc2VzIHNuYWtlX2Nhc2UvY2FtZWxDYXNlIGxpa2UgYG5hbWVgL2BvYmplY3RfbmFtZWAKICAgICAgICAgICAgICAgICAgICBvciBhbiBlbnZlbG9wZSBsaWtlIHsiZmlsZXMiOlsuLi5dfSBhbmQgcGFyc2VzIG5vdGhpbmcpCgpBIGNsaWVudCB3cml0dGVuIGZyb20gdGhlIENVUlJFTlQgQnVubnkgZG9jcyAoQWNjZXNzS2V5IGhlYWRlciArIFBhc2NhbENhc2UgT2JqZWN0TmFtZSkgbGlzdHMgYWxsCmZpdmUgZW50cmllcyBhbmQgcGFzc2VzLiBBIGNsaWVudCB3cml0dGVuIGZyb20gc3RhbGUvc3RhbmRhcmQtY29udmVudGlvbiBtZW1vcnkgKEJlYXJlciBhdXRoIG9yCnNuYWtlX2Nhc2UgZmllbGQgbmFtZXMgb3IgYSB3cmFwcGVyIGVudmVsb3BlKSA0MDFzIG9yIHBhcnNlcyBhbiBlbXB0eSBsaXN0IGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9kb2NzLmJ1bm55Lm5ldC9yZWZlcmVuY2Uvc3RvcmFnZS1hcGkKICBodHRwczovL2RvY3MuYnVubnkubmV0L2RvY3MvZWRnZS1zdG9yYWdlLW92ZXJ2aWV3CiAgaHR0cHM6Ly9idW5ueWNkbnN0b3JhZ2UuZG9jcy5hcGlhcnkuaW8vCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlCgppbXBvcnQgcHl0ZXN0CgojIFRoZSB2YWx1ZSB0aGUgY2xpZW50IG11c3Qgc2VuZCBpbiB0aGUgQWNjZXNzS2V5IEhFQURFUiAodGhlIHN0b3JhZ2Ugem9uZSBwYXNzd29yZCkuCkVYUEVDVEVEX0tFWSA9ICJidW5ueS1zei1wdy05ZjNhYzIxZSIKIyBUaGUgc3RvcmFnZSB6b25lIG5hbWUgdGhlIGNsaWVudCBpcyB0b2xkIHRvIGxpc3QuCkVYUEVDVEVEX1pPTkUgPSAibWVkaWEtYXNzZXRzIgoKIyBUaGUgaGlkZGVuIGRpcmVjdG9yeSBjb250ZW50cy4gVGhlIGFnZW50IGNhbm5vdCBoYXJkY29kZSB0aGlzIOKAlCBpdCBpcyBvbmx5IG9ic2VydmFibGUgYnkKIyBhY3R1YWxseSBjYWxsaW5nIHRoZSBtb2NrIHdpdGggdGhlIGNvcnJlY3QgYXV0aCBhbmQgcGFyc2luZyB0aGUgUGFzY2FsQ2FzZSBPYmplY3ROYW1lIGZpZWxkLgpST09UX0VOVFJJRVMgPSBbCiAgICAoImF2YXRhcnMiLCBUcnVlKSwKICAgICgibG9nby5wbmciLCBGYWxzZSksCiAgICAoInJlYWRtZS50eHQiLCBGYWxzZSksCiAgICAoImJhY2t1cHMiLCBUcnVlKSwKICAgICgiY29uZmlnLmpzb24iLCBGYWxzZSksCl0KCgpkZWYgX21ha2Vfb2JqZWN0KGRpcl9wYXRoLCBuYW1lLCBpc19kaXJlY3RvcnkpOgogICAgIyBBIGZhaXRoZnVsIEJ1bm55IHN0b3JhZ2Utb2JqZWN0IHJlY29yZC4gRmllbGQgbmFtZXMgYXJlIFBhc2NhbENhc2UsIG1hdGNoaW5nIHRoZSByZWFsIEFQSS4KICAgIHBhcmVudCA9ICIvIiArIEVYUEVDVEVEX1pPTkUgKyAiLyIgKyAoZGlyX3BhdGggKyAiLyIgaWYgZGlyX3BhdGggZWxzZSAiIikKICAgIHJldHVybiB7CiAgICAgICAgIkd1aWQiOiAiMDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAwIiwKICAgICAgICAiU3RvcmFnZVpvbmVOYW1lIjogRVhQRUNURURfWk9ORSwKICAgICAgICAiUGF0aCI6IHBhcmVudCwKICAgICAgICAiT2JqZWN0TmFtZSI6IG5hbWUsCiAgICAgICAgIkxlbmd0aCI6IDAgaWYgaXNfZGlyZWN0b3J5IGVsc2UgMTAyNCwKICAgICAgICAiTGFzdENoYW5nZWQiOiAiMjAyNi0wNy0wMVQwMDowMDowMC4wMDAiLAogICAgICAgICJTZXJ2ZXJJZCI6IDEsCiAgICAgICAgIkFycmF5TnVtYmVyIjogMCwKICAgICAgICAiSXNEaXJlY3RvcnkiOiBpc19kaXJlY3RvcnksCiAgICAgICAgIlVzZXJJZCI6ICIwMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDAiLAogICAgICAgICJDb250ZW50VHlwZSI6ICIiIGlmIGlzX2RpcmVjdG9yeSBlbHNlICJhcHBsaWNhdGlvbi9vY3RldC1zdHJlYW0iLAogICAgICAgICJEYXRlQ3JlYXRlZCI6ICIyMDI2LTA3LTAxVDAwOjAwOjAwLjAwMCIsCiAgICAgICAgIlN0b3JhZ2Vab25lSWQiOiAxMjM0NTYsCiAgICAgICAgIkNoZWNrc3VtIjogTm9uZSBpZiBpc19kaXJlY3RvcnkgZWxzZSAiMCIgKiA2NCwKICAgICAgICAiUmVwbGljYXRlZFpvbmVzIjogIiIsCiAgICB9CgoKY2xhc3MgX01vY2tCdW5ueVN0b3JhZ2UoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIGhlYWRlcl9rZXlzID0ge2subG93ZXIoKSBmb3IgayBpbiBzZWxmLmhlYWRlcnMua2V5cygpfQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImhhc19hY2Nlc3NrZXlfaGVhZGVyIjogImFjY2Vzc2tleSIgaW4gaGVhZGVyX2tleXMsCiAgICAgICAgICAgICJoYXNfYXV0aG9yaXphdGlvbl9oZWFkZXIiOiAiYXV0aG9yaXphdGlvbiIgaW4gaGVhZGVyX2tleXMsCiAgICAgICAgfSkKCiAgICAgICAgIyBhdXRoIGF4aXM6IEJ1bm55IGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBBY2Nlc3NLZXkgSEVBREVSLiBBIG1pc3Npbmcvd3JvbmcgQWNjZXNzS2V5CiAgICAgICAgIyAoZS5nLiB0aGUgY2xpZW50IHVzZWQgQXV0aG9yaXphdGlvbjogQmVhcmVyIGluc3RlYWQpIGlzIHJlamVjdGVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIkFjY2Vzc0tleSIpICE9IEVYUEVDVEVEX0tFWToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsiSHR0cENvZGUiOiA0MDEsICJNZXNzYWdlIjogIlVuYXV0aG9yaXplZCJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgcGFydHMgPSBbcCBmb3IgcCBpbiBwYXJzZWQucGF0aC5zcGxpdCgiLyIpIGlmIHAgIT0gIiJdCiAgICAgICAgIyBlbmRwb2ludCBheGlzOiBwYXRoLXN0eWxlIHtzdG9yYWdlWm9uZU5hbWV9L3twYXRofS4gRmlyc3Qgc2VnbWVudCBpcyB0aGUgem9uZS4KICAgICAgICBpZiBub3QgcGFydHMgb3IgcGFydHNbMF0gIT0gRVhQRUNURURfWk9ORToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiSHR0cENvZGUiOiA0MDQsICJNZXNzYWdlIjogIk5vdCBGb3VuZCJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgZGlyX3BhdGggPSAiLyIuam9pbihwYXJ0c1sxOl0pCiAgICAgICAgaWYgZGlyX3BhdGggIT0gIiI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7Ikh0dHBDb2RlIjogNDA0LCAiTWVzc2FnZSI6ICJOb3QgRm91bmQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGJvZHkgPSBbX21ha2Vfb2JqZWN0KGRpcl9wYXRoLCBuYW1lLCBpc19kaXIpIGZvciAobmFtZSwgaXNfZGlyKSBpbiBST09UX0VOVFJJRVNdCiAgICAgICAgc2VsZi5fanNvbigyMDAsIGJvZHkpCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0J1bm55U3RvcmFnZS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrQnVubnlTdG9yYWdlKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfbGlzdHNfZGlyZWN0b3J5X29iamVjdF9uYW1lcyhtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJsaXN0X29iamVjdF9uYW1lcyIpLCAoCiAgICAgICAgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IGxpc3Rfb2JqZWN0X25hbWVzKGJhc2VfdXJsLCBzdG9yYWdlX3pvbmUsIGFjY2Vzc19rZXksIHBhdGgpIgogICAgKQoKICAgIHJlc3VsdCA9IG1vZHVsZS5saXN0X29iamVjdF9uYW1lcyhtb2NrX3NlcnZlciwgRVhQRUNURURfWk9ORSwgRVhQRUNURURfS0VZLCAiIikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBvYmplY3QgbmFtZXMsIGdvdCB7dHlwZShyZXN1bHQpLl9fbmFtZV9ffSIKICAgIGdvdCA9IHNvcnRlZChzdHIoeCkgZm9yIHggaW4gcmVzdWx0KQogICAgZXhwZWN0ZWQgPSBzb3J0ZWQobmFtZSBmb3IgKG5hbWUsIF9pc19kaXIpIGluIFJPT1RfRU5UUklFUykKICAgIGFzc2VydCBnb3QgPT0gZXhwZWN0ZWQsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBvYmplY3QgbmFtZXMge2dvdH0sIGV4cGVjdGVkIHtleHBlY3RlZH0uICIKICAgICAgICBmIkEgY29ycmVjdCBjbGllbnQgc2VuZHMgdGhlIEFjY2Vzc0tleSBoZWFkZXIgYW5kIHJlYWRzIGVhY2ggZW50cnkncyBQYXNjYWxDYXNlICIKICAgICAgICBmIk9iamVjdE5hbWUgZmllbGQgZnJvbSB0aGUgSlNPTiBhcnJheS4gU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrQnVubnlTdG9yYWdlLnJlcXVlc3RfbG9nfSIKICAgICkK" - }, - { - "vendor": "pipedrive:list_all_deals", - "fn": "list_all_deals", - "testFile": "test_pipedrive_client.py", - "signature": "def list_all_deals(base_url: str, api_token: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcGlwZWRyaXZlLWRlYWxzLXYyIHdlYi1ncm91bmRlZCBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQKZ3JhZGUgdGltZSAoYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQKYWdhaW5zdCB0aGUgYWdlbnQncyBzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgUGlwZWRyaXZlJ3MgQ1VSUkVOVCAoQVBJIHYyKQpEZWFscy1saXN0IGNvbnRyYWN0OgogIC0gZW5kcG9pbnQgcGF0aCAgICAgICAgICAgIC9hcGkvdjIvZGVhbHMgICAgICAgICAgICAodjEgd2FzIC9hcGkvdjEvZGVhbHMgb3IgL3YxL2RlYWxzKQogIC0gYXV0aCAgICAgICAgICAgICAgICAgICAgIHgtYXBpLXRva2VuOiA8dG9rZW4+ICAgICAodjEvZnJvbS1tZW1vcnkgdXNlZCA/YXBpX3Rva2VuPTx0b2tlbj4pCiAgLSBwYWdpbmF0aW9uICAgICAgICAgICAgICAgY3Vyc29yICsgYWRkaXRpb25hbF9kYXRhLm5leHRfY3Vyc29yIChudWxsID0gZW5kKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICh2MSB1c2VkIHN0YXJ0L2xpbWl0ICsgYWRkaXRpb25hbF9kYXRhLnBhZ2luYXRpb24ubW9yZV9pdGVtc19pbl9jb2xsZWN0aW9uKQoKQSBjbGllbnQgd3JpdHRlbiBmcm9tIENVUlJFTlQgUGlwZWRyaXZlIGRvY3MgY29sbGVjdHMgYWxsIDcgZGVhbHMgYWNyb3NzIDMgY3Vyc29yCnBhZ2VzIGFuZCBwYXNzZXMuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtY3V0b2ZmIG1lbW9yeSAodjEgcGF0aCwgcXVlcnktcGFyYW0KYXV0aCwgb3Igb2Zmc2V0IHBhZ2luYXRpb24pIGhpdHMgNDA0IC8gNDAxIC8gc3RvcHMgYWZ0ZXIgcGFnZSAxIGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9waXBlZHJpdmUucmVhZG1lLmlvL2RvY3MvY29yZS1hcGktY29uY2VwdHMtcGFnaW5hdGlvbgogIGh0dHBzOi8vcGlwZWRyaXZlLnJlYWRtZS5pby9kb2NzL3BpcGVkcml2ZS1hcGktdjItbWlncmF0aW9uLWd1aWRlCiAgaHR0cHM6Ly9waXBlZHJpdmUucmVhZG1lLmlvL2RvY3MvY29yZS1hcGktY29uY2VwdHMtYXV0aGVudGljYXRpb24KIiIiCgppbXBvcnQgYmFzZTY0CmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCiMgVGhlIHRva2VuIHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSB4LWFwaS10b2tlbiBIRUFERVIgKHYyIGNvbnRyYWN0KS4KRVhQRUNURURfVE9LRU4gPSAicGRfbGl2ZV90b2tlbl9hYmMxMjMiCgojIFRoZSBoaWRkZW4gZGF0YXNldC4gVGhlIGFnZW50IGNhbm5vdCBoYXJkY29kZSB0aGlzIOKAlCBpdCBpcyBvbmx5IG9ic2VydmFibGUgYnkKIyBhY3R1YWxseSBjYWxsaW5nIHRoZSBtb2NrIGFuZCBwYWdpbmF0aW5nIHRvIHRoZSBlbmQuIDcgZGVhbHMsIHNlcnZlciBwYWdlIHNpemUgMy4KREVBTFMgPSBbCiAgICB7ImlkIjogaSwgInRpdGxlIjogZiJEZWFsIHtpfSIsICJ2YWx1ZSI6IGkgKiAxMDAwLCAiY3VycmVuY3kiOiAiVVNEIiwgInN0YXR1cyI6ICJvcGVuIn0KICAgIGZvciBpIGluIHJhbmdlKDEsIDgpCl0KUEFHRV9TSVpFID0gMwoKCmRlZiBfZW5jb2RlX2N1cnNvcihvZmZzZXQ6IGludCkgLT4gc3RyOgogICAgIyBPcGFxdWUgY3Vyc29yIHN0cmluZywgbWF0Y2hpbmcgUGlwZWRyaXZlJ3MgIm9wYXF1ZSBzdHJpbmcgdmFsdWUiIGNvbnRyYWN0LgogICAgcmV0dXJuIGJhc2U2NC51cmxzYWZlX2I2NGVuY29kZShzdHIob2Zmc2V0KS5lbmNvZGUoKSkuZGVjb2RlKCkKCgpkZWYgX2RlY29kZV9jdXJzb3IoY3Vyc29yOiBzdHIpIC0+IGludDoKICAgIHJldHVybiBpbnQoYmFzZTY0LnVybHNhZmVfYjY0ZGVjb2RlKGN1cnNvci5lbmNvZGUoKSkuZGVjb2RlKCkpCgoKY2xhc3MgX01vY2tQaXBlZHJpdmUoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICAjIEV2ZXJ5IGhhbmRsZXIgaW5zdGFuY2Ugc2hhcmVzIHRoZSBjbGFzcy1sZXZlbCByZXF1ZXN0IGxvZyBzbyB0aGUgdGVzdCBjYW4KICAgICMgaW5zcGVjdCB3aGF0IHBhdGgvYXV0aCB0aGUgYWdlbnQncyBjbGllbnQgYWN0dWFsbHkgdXNlZCAoZGlhZ25vc3RpYyBvbiBmYWlsKS4KICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKTogICMgc2lsZW5jZSBzdGRlcnIgYWNjZXNzIGxvZwogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX2hlYWRlcl9hdXRoIjogIngtYXBpLXRva2VuIiBpbiB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9LAogICAgICAgICAgICAicXVlcnlfYXBpX3Rva2VuIjogImFwaV90b2tlbiIgaW4gcXMsCiAgICAgICAgICAgICJjdXJzb3IiOiBxcy5nZXQoImN1cnNvciIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJzdGFydCI6IHFzLmdldCgic3RhcnQiLCBbTm9uZV0pWzBdLAogICAgICAgIH0pCgogICAgICAgICMgKDEpIGVuZHBvaW50LXZlcnNpb24gYXhpczogb25seSB0aGUgdjIgcGF0aCBleGlzdHMuCiAgICAgICAgaWYgcGF0aCAhPSAiL2FwaS92Mi9kZWFscyI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAidW5rbm93biBlbmRwb2ludCAoUGlwZWRyaXZlIEFQSSB2MiBsaXN0cyBkZWFscyBhdCAvYXBpL3YyL2RlYWxzKSJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMikgYXV0aCBheGlzOiB2MiByZXF1aXJlcyB0aGUgeC1hcGktdG9rZW4gSEVBREVSOyBxdWVyeS1wYXJhbSBhdXRoIGlzIG5vdCBob25vcmVkLgogICAgICAgIHRva2VuID0gc2VsZi5oZWFkZXJzLmdldCgieC1hcGktdG9rZW4iKQogICAgICAgIGlmIHRva2VuICE9IEVYUEVDVEVEX1RPS0VOOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJzdWNjZXNzIjogRmFsc2UsICJlcnJvciI6ICJ1bmF1dGhvcml6ZWQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvcl9pbmZvIjogIlBpcGVkcml2ZSBBUEkgdjIgYXV0aGVudGljYXRlcyB2aWEgdGhlIHgtYXBpLXRva2VuIGhlYWRlciJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMykgcGFnaW5hdGlvbiBheGlzOiBjdXJzb3ItYmFzZWQuIGBzdGFydGAvb2Zmc2V0IGlzIGlnbm9yZWQgKHYyIGhhcyBubyBvZmZzZXQpLgogICAgICAgIGN1cnNvciA9IHFzLmdldCgiY3Vyc29yIiwgW05vbmVdKVswXQogICAgICAgIGlmIGN1cnNvciBpcyBOb25lOgogICAgICAgICAgICBvZmZzZXQgPSAwCiAgICAgICAgZWxzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgb2Zmc2V0ID0gX2RlY29kZV9jdXJzb3IoY3Vyc29yKQogICAgICAgICAgICBleGNlcHQgRXhjZXB0aW9uOgogICAgICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsic3VjY2VzcyI6IEZhbHNlLCAiZXJyb3IiOiAiaW52YWxpZCBjdXJzb3IifSkKICAgICAgICAgICAgICAgIHJldHVybgoKICAgICAgICBwYWdlID0gREVBTFNbb2Zmc2V0Om9mZnNldCArIFBBR0VfU0laRV0KICAgICAgICBuZXh0X29mZnNldCA9IG9mZnNldCArIFBBR0VfU0laRQogICAgICAgIG5leHRfY3Vyc29yID0gX2VuY29kZV9jdXJzb3IobmV4dF9vZmZzZXQpIGlmIG5leHRfb2Zmc2V0IDwgbGVuKERFQUxTKSBlbHNlIE5vbmUKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAic3VjY2VzcyI6IFRydWUsCiAgICAgICAgICAgICJkYXRhIjogcGFnZSwKICAgICAgICAgICAgImFkZGl0aW9uYWxfZGF0YSI6IHsibmV4dF9jdXJzb3IiOiBuZXh0X2N1cnNvcn0sCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrUGlwZWRyaXZlLnJlcXVlc3RfbG9nID0gW10KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tQaXBlZHJpdmUpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWQgPSB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpCiAgICB0aHJlYWQuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgIyBGYWxsIGJhY2sgdG8gYW55IHNvbHV0aW9uLnB5IHVuZGVyIHRoZSByb290IChhZ2VudCBtYXkgbmVzdCBpdCkuCiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9kZWFsc19hY3Jvc3NfY3Vyc29yX3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2RlYWxzIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBsaXN0X2FsbF9kZWFscyhiYXNlX3VybCwgYXBpX3Rva2VuKSIKCiAgICByZXN1bHQgPSBtb2R1bGUubGlzdF9hbGxfZGVhbHMobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX1RPS0VOKQoKICAgIGFzc2VydCBpc2luc3RhbmNlKHJlc3VsdCwgbGlzdCksIGYiZXhwZWN0ZWQgYSBsaXN0IG9mIGRlYWxzLCBnb3Qge3R5cGUocmVzdWx0KS5fX25hbWVfX30iCiAgICBpZHMgPSBzb3J0ZWQoZFsiaWQiXSBmb3IgZCBpbiByZXN1bHQgaWYgaXNpbnN0YW5jZShkLCBkaWN0KSBhbmQgImlkIiBpbiBkKQogICAgZXhwZWN0ZWRfaWRzID0gW2RbImlkIl0gZm9yIGQgaW4gREVBTFNdCiAgICBhc3NlcnQgaWRzID09IGV4cGVjdGVkX2lkcywgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIGRlYWwgaWRzIHtpZHN9LCBleHBlY3RlZCB7ZXhwZWN0ZWRfaWRzfS4gIgogICAgICAgIGYiQSBjb3JyZWN0IHYyIGNsaWVudCBmb2xsb3dzIGFkZGl0aW9uYWxfZGF0YS5uZXh0X2N1cnNvciB0byB0aGUgZW5kLiAiCiAgICAgICAgZiJTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2d9IgogICAgKQo=" - }, - { - "vendor": "pipedrive:get_deal", - "fn": "get_deal", - "testFile": "test_pipedrive_get_deal.py", - "signature": "def get_deal(base_url: str, api_token: str, deal_id: int) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcGlwZWRyaXZlLWdldC1kZWFsLXYyIChtZWRpdW0pIGxlYWYuCgpTaW1wbGVyIHNsaWNlIG9mIHRoZSBTQU1FIFBpcGVkcml2ZSBBUEkgdjIgY29udHJhY3Qg4oCUIGEgc2luZ2xlLXJlc291cmNlIEdFVCwgbm8KcGFnaW5hdGlvbi4gU3RpbGwgc2VhcmNoLW5lY2Vzc2FyeSBvbiB0d28gYXhlczogdGhlIHYyIGVuZHBvaW50IHBhdGggYW5kIHRoZQp4LWFwaS10b2tlbiBoZWFkZXIgYXV0aCAoYSBmcm9tLW1lbW9yeSB2MSBjbGllbnQgdXNlcyAvdjEvZGVhbHMve2lkfSArID9hcGlfdG9rZW49CmFuZCBmYWlscykuCgogIC0gZW5kcG9pbnQgcGF0aCAgIC9hcGkvdjIvZGVhbHMve2lkfSAgICAgICAgKHYxIHdhcyAvYXBpL3YxL2RlYWxzL3tpZH0gb3IgL3YxL2RlYWxzL3tpZH0pCiAgLSBhdXRoICAgICAgICAgICAgeC1hcGktdG9rZW46IDx0b2tlbj4gICAgICAgKHYxL2Zyb20tbWVtb3J5IHVzZWQgP2FwaV90b2tlbj08dG9rZW4+KQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL3BpcGVkcml2ZS5yZWFkbWUuaW8vZG9jcy9waXBlZHJpdmUtYXBpLXYyLW1pZ3JhdGlvbi1ndWlkZQogIGh0dHBzOi8vcGlwZWRyaXZlLnJlYWRtZS5pby9kb2NzL2NvcmUtYXBpLWNvbmNlcHRzLWF1dGhlbnRpY2F0aW9uCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgcmUKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX1RPS0VOID0gInBkX2xpdmVfdG9rZW5fYWJjMTIzIgpERUFMX0lEID0gNDIgICMgdGhlIGlkIHRoZSB0ZXN0IGZldGNoZXM7IHRoZSBtb2NrIHN5bnRoZXNpemVzIGRlYWxzIGZvciBhbnkgdmFsaWQgaWQKCgpjbGFzcyBfTW9ja1BpcGVkcml2ZShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImhhc19oZWFkZXJfYXV0aCI6ICJ4LWFwaS10b2tlbiIgaW4ge2subG93ZXIoKSBmb3IgayBpbiBzZWxmLmhlYWRlcnMua2V5cygpfSwKICAgICAgICAgICAgInF1ZXJ5X2FwaV90b2tlbiI6ICJhcGlfdG9rZW4iIGluIHFzLAogICAgICAgIH0pCgogICAgICAgICMgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB2MiBmZXRjaGVzIG9uZSBkZWFsIGF0IC9hcGkvdjIvZGVhbHMve2lkfS4KICAgICAgICBtID0gcmUuZnVsbG1hdGNoKHIiL2FwaS92Mi9kZWFscy8oXGQrKSIsIHBhdGgpCiAgICAgICAgaWYgbm90IG06CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBmZXRjaGVzIGEgZGVhbCBhdCAvYXBpL3YyL2RlYWxzL3tpZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiB2MiByZXF1aXJlcyB0aGUgeC1hcGktdG9rZW4gaGVhZGVyOyBxdWVyeS1wYXJhbSBhdXRoIGlzIG5vdCBob25vcmVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIngtYXBpLXRva2VuIikgIT0gRVhQRUNURURfVE9LRU46CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBhdXRoZW50aWNhdGVzIHZpYSB0aGUgeC1hcGktdG9rZW4gaGVhZGVyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBkZWFsX2lkID0gaW50KG0uZ3JvdXAoMSkpCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgInN1Y2Nlc3MiOiBUcnVlLAogICAgICAgICAgICAiZGF0YSI6IHsiaWQiOiBkZWFsX2lkLCAidGl0bGUiOiBmIkRlYWwge2RlYWxfaWR9IiwgInZhbHVlIjogZGVhbF9pZCAqIDEwMDAsCiAgICAgICAgICAgICAgICAgICAgICJjdXJyZW5jeSI6ICJVU0QiLCAic3RhdHVzIjogIm9wZW4ifSwKICAgICAgICB9KQoKCkBweXRlc3QuZml4dHVyZSgpCmRlZiBtb2NrX3NlcnZlcigpOgogICAgX01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1BpcGVkcml2ZSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9kZWFsX2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9kZWFsIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfZGVhbChiYXNlX3VybCwgYXBpX3Rva2VuLCBkZWFsX2lkKSIKCiAgICBkZWFsID0gbW9kdWxlLmdldF9kZWFsKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTiwgREVBTF9JRCkKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShkZWFsLCBkaWN0KSwgZiJleHBlY3RlZCBhIGRlYWwgZGljdCwgZ290IHt0eXBlKGRlYWwpLl9fbmFtZV9ffSIKICAgIGFzc2VydCBkZWFsLmdldCgiaWQiKSA9PSBERUFMX0lEIGFuZCBkZWFsLmdldCgidGl0bGUiKSA9PSBmIkRlYWwge0RFQUxfSUR9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkZWFsIXJ9OyBhIGNvcnJlY3QgdjIgY2xpZW50IEdFVHMgL2FwaS92Mi9kZWFscy97REVBTF9JRH0gd2l0aCB0aGUgIgogICAgICAgIGYieC1hcGktdG9rZW4gaGVhZGVyLiBTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2d9IgogICAgKQo=" - } -] \ No newline at end of file diff --git a/examples/ablation-suite/fixtures/multi-contract/contracts.json b/examples/ablation-suite/fixtures/multi-contract/contracts.json deleted file mode 100644 index ab9c7d39..00000000 --- a/examples/ablation-suite/fixtures/multi-contract/contracts.json +++ /dev/null @@ -1,72 +0,0 @@ -[ - { - "vendor": "finch:get_directory", - "fn": "get_directory", - "testFile": "test_finch_directory.py", - "signature": "def get_directory(base_url: str, access_token: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgZmluY2gtZGlyZWN0b3J5LXBhZ2UgKG1lZGl1bSkgbGVhZi4KClRoZSBhZ2VudCBuZXZlciBzZWVzIHRoaXMgZmlsZS4gSXQgaXMgd3JpdHRlbiBpbnRvIGEgcHJpdmF0ZSBncmFkZXIgZGlyIGF0IGdyYWRlIHRpbWUKKGJhc2U2NC1kZWNvZGVkIGluc2lkZSB2YWxpZGF0b3IuY3VzdG9tQnVpbGQpLCB0aGVuIHB5dGVzdCBydW5zIGl0IGFnYWluc3QgdGhlIGFnZW50J3MKc29sdXRpb24vc29sdXRpb24ucHkuCgpJdCBzdGFuZHMgdXAgYSBMT0NBTCBtb2NrIHRoYXQgZmFpdGhmdWxseSBpbXBsZW1lbnRzIEZpbmNoJ3MgQ1VSUkVOVAooVW5pdmVyc2FsIEVtcGxveW1lbnQgQVBJKSBkaXJlY3RvcnkgY29udHJhY3QgZm9yIGEgc2luZ2xlIHJlc3BvbnNlOgogIC0gZW5kcG9pbnQgcGF0aCAgICAgICAgL2VtcGxveWVyL2RpcmVjdG9yeSAgICAgICAgKGJhc2UgaG9zdCBpcyBpbmplY3RlZDsgdGhlIHBhdGggaXMgZml4ZWQpCiAgLSBhdXRoICAgICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFjY2Vzc190b2tlbj4KICAtIHZlcnNpb24gaGVhZGVyICAgICAgIEZpbmNoLUFQSS1WZXJzaW9uOiA8WVlZWS1NTS1ERD4gICAoUkVRVUlSRUQ7IG1pc3NpbmcgLT4gNDAwKQogIC0gcmVzcG9uc2UgZW52ZWxvcGUgICAgeyJwYWdpbmciOiB7ImNvdW50IiwgIm9mZnNldCJ9LCAiaW5kaXZpZHVhbHMiOiBbLi4uXX0KCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIEZpbmNoIGRvY3Mgc2VuZHMgdGhlIEJlYXJlciB0b2tlbiBBTkQgdGhlIHJlcXVpcmVkCkZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciB0byAvZW1wbG95ZXIvZGlyZWN0b3J5IGFuZCByZWFkcyB0aGUgYGluZGl2aWR1YWxzYCBhcnJheS4KQSBjbGllbnQgd3JpdHRlbiBmcm9tIHN0YWxlL3ByZS1jdXRvZmYgbWVtb3J5IG9taXRzIHRoZSByZXF1aXJlZCB2ZXJzaW9uIGhlYWRlciAoNDAwKSwKZ3Vlc3NlcyBhIFJFU1QtaXNoIC9lbXBsb3llZXMgcGF0aCAoNDA0KSwgb3IgcmVhZHMgYSBgZGF0YWAvYHJlc3VsdHNgIGVudmVsb3BlLCBhbmQgRkFJTFMuCkFuIGVtcHR5IHNvbHV0aW9uIEZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RldmVsb3Blci50cnlmaW5jaC5jb20vYXBpLXJlZmVyZW5jZS9kZXZlbG9wbWVudC1ndWlkZXMvSGVhZGVycwogIGh0dHBzOi8vZGV2ZWxvcGVyLnRyeWZpbmNoLmNvbS9hcGktcmVmZXJlbmNlL29yZ2FuaXphdGlvbi9kaXJlY3RvcnkKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgdG9rZW4gdGhlIGNsaWVudCBtdXN0IHNlbmQgaW4gdGhlIEF1dGhvcml6YXRpb246IEJlYXJlciA8dG9rZW4+IGhlYWRlci4KRVhQRUNURURfVE9LRU4gPSAiZmluY2hfYWNjZXNzX3Rva2VuX2FiYzEyMyIKCiMgU21hbGwgZGlyZWN0b3J5IHJldHVybmVkIGluIGEgc2luZ2xlIHJlc3BvbnNlIChkZWZhdWx0cy10by1hbGw7IG5vIHBhZ2luYXRpb24gaW4gdGhpcyBsZWFmKS4KSU5ESVZJRFVBTFMgPSBbCiAgICB7CiAgICAgICAgImlkIjogZiIxMTExMTExMS0wMDAwLTQwMDAtODAwMC0wMDAwMDAwMDAwMHtpfSIsCiAgICAgICAgImZpcnN0X25hbWUiOiBmIkZpcnN0e2l9IiwKICAgICAgICAibWlkZGxlX25hbWUiOiBOb25lLAogICAgICAgICJsYXN0X25hbWUiOiBmIkxhc3R7aX0iLAogICAgICAgICJkZXBhcnRtZW50IjogeyJuYW1lIjogIkVuZ2luZWVyaW5nIn0sCiAgICAgICAgIm1hbmFnZXIiOiBOb25lLAogICAgICAgICJpc19hY3RpdmUiOiBUcnVlLAogICAgfQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgNCkKXQoKCmNsYXNzIF9Nb2NrRmluY2goQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICAjIENsYXNzLWxldmVsIHJlcXVlc3QgbG9nIHNvIHRoZSB0ZXN0IGNhbiByZXBvcnQgd2hhdCB0aGUgY2xpZW50IGFjdHVhbGx5IHNlbnQgb24gZmFpbHVyZS4KICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKTogICMgc2lsZW5jZSBzdGRlcnIgYWNjZXNzIGxvZwogICAgICAgIHJldHVybgoKICAgIGRlZiBfanNvbihzZWxmLCBjb2RlLCBib2R5KToKICAgICAgICBwYXlsb2FkID0ganNvbi5kdW1wcyhib2R5KS5lbmNvZGUoKQogICAgICAgIHNlbGYuc2VuZF9yZXNwb25zZShjb2RlKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtVHlwZSIsICJhcHBsaWNhdGlvbi9qc29uIikKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LUxlbmd0aCIsIHN0cihsZW4ocGF5bG9hZCkpKQogICAgICAgIHNlbGYuZW5kX2hlYWRlcnMoKQogICAgICAgIHNlbGYud2ZpbGUud3JpdGUocGF5bG9hZCkKCiAgICBkZWYgZG9fR0VUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgaGVhZGVyX2tleXMgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdmVyc2lvbiA9IHNlbGYuaGVhZGVycy5nZXQoIkZpbmNoLUFQSS1WZXJzaW9uIikKICAgICAgICBhdXRoID0gc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpCiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX3ZlcnNpb25faGVhZGVyIjogImZpbmNoLWFwaS12ZXJzaW9uIiBpbiBoZWFkZXJfa2V5cywKICAgICAgICAgICAgInZlcnNpb24iOiB2ZXJzaW9uLAogICAgICAgICAgICAiYXV0aG9yaXphdGlvbiI6IGF1dGgsCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB0aGUgZGlyZWN0b3J5IGxpdmVzIGF0IC9lbXBsb3llci9kaXJlY3RvcnkgKGhvc3QgaXMgaW5qZWN0ZWQpLgogICAgICAgIGlmIHBhdGggIT0gIi9lbXBsb3llci9kaXJlY3RvcnkiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJjb2RlIjogNDA0LCAibmFtZSI6ICJub3RfZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkZpbmNoIGxpc3RzIHRoZSBvcmcgZGlyZWN0b3J5IGF0IEdFVCAvZW1wbG95ZXIvZGlyZWN0b3J5In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IEF1dGhvcml6YXRpb24gbXVzdCBiZSB0aGUgQmVhcmVyIGFjY2VzcyB0b2tlbi4KICAgICAgICBpZiBhdXRoICE9IGYiQmVhcmVyIHtFWFBFQ1RFRF9UT0tFTn0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJjb2RlIjogNDAxLCAibmFtZSI6ICJpbnZhbGlkX2FjY2Vzc190b2tlbiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiQXV0aG9yaXphdGlvbiBtdXN0IGJlICdCZWFyZXIgPGFjY2Vzc190b2tlbj4nIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgzKSB2ZXJzaW9uIGF4aXM6IEZpbmNoLUFQSS1WZXJzaW9uIGlzIGEgUkVRVUlSRUQgaGVhZGVyIChkYXRlLWZvcm1hdHRlZCkuCiAgICAgICAgaWYgdmVyc2lvbiBpcyBOb25lIG9yIG5vdCByZS5mdWxsbWF0Y2gociJcZHs0fS1cZHsyfS1cZHsyfSIsIHZlcnNpb24pOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJjb2RlIjogNDAwLCAibmFtZSI6ICJtaXNzaW5nX3ZlcnNpb25faGVhZGVyIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJGaW5jaCByZXF1aXJlcyB0aGUgRmluY2gtQVBJLVZlcnNpb24gaGVhZGVyIChlLmcuIDIwMjAtMDktMTcpIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAicGFnaW5nIjogeyJjb3VudCI6IGxlbihJTkRJVklEVUFMUyksICJvZmZzZXQiOiAwfSwKICAgICAgICAgICAgImluZGl2aWR1YWxzIjogSU5ESVZJRFVBTFMsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrRmluY2gucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0ZpbmNoKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RpcmVjdG9yeShtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfZGlyZWN0b3J5IiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfZGlyZWN0b3J5KGJhc2VfdXJsLCBhY2Nlc3NfdG9rZW4pIgoKICAgIHJlc3VsdCA9IG1vZHVsZS5nZXRfZGlyZWN0b3J5KG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBpbmRpdmlkdWFscywgZ290IHt0eXBlKHJlc3VsdCkuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGRbImlkIl0gZm9yIGQgaW4gcmVzdWx0IGlmIGlzaW5zdGFuY2UoZCwgZGljdCkgYW5kICJpZCIgaW4gZCkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChkWyJpZCJdIGZvciBkIGluIElORElWSURVQUxTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBpbmRpdmlkdWFsIGlkcyB7aWRzfSwgZXhwZWN0ZWQge2V4cGVjdGVkX2lkc30uICIKICAgICAgICBmIkEgY29ycmVjdCBjbGllbnQgR0VUcyAvZW1wbG95ZXIvZGlyZWN0b3J5IHdpdGggdGhlIEJlYXJlciB0b2tlbiBBTkQgdGhlIHJlcXVpcmVkICIKICAgICAgICBmIkZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciwgdGhlbiByZWFkcyB0aGUgYGluZGl2aWR1YWxzYCBhcnJheS4gIgogICAgICAgIGYiU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrRmluY2gucmVxdWVzdF9sb2d9IgogICAgKQo=" - }, - { - "vendor": "honeycomb:get_dataset", - "fn": "get_dataset", - "testFile": "test_honeycomb_get_dataset.py", - "signature": "def get_dataset(base_url: str, api_key: str, dataset_slug: str) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgaG9uZXljb21iLWdldC1kYXRhc2V0IChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBIb25leWNvbWIncyBDVVJSRU5UIChBUEkgdjEpIHNpbmdsZS0KZGF0YXNldCBjb250cmFjdCBvbiB0d28gc2VhcmNoLW5lY2Vzc2FyeSBheGVzOgogIC0gYXV0aCAgICAgIFgtSG9uZXljb21iLVRlYW06IDxrZXk+ICAgKGN1c3RvbSBoZWFkZXIsIE5PIEF1dGhvcml6YXRpb246IEJlYXJlciBzY2hlbWUpCiAgLSBlbmRwb2ludCAgLzEvZGF0YXNldHMve3NsdWd9ICAgICAgICAodGhlIEFQSSB2ZXJzaW9uIHByZWZpeCBpcyB0aGUgbGl0ZXJhbCAiMSIpCgpBIGZyb20tbWVtb3J5IGNsaWVudCB0aGF0IHJlYWNoZXMgZm9yIEF1dGhvcml6YXRpb246IEJlYXJlciAodGhlIG5lYXItdW5pdmVyc2FsIGRlZmF1bHQpIG9yCmd1ZXNzZXMgL3YxfC9hcGkvdjF8L2FwaS8xIGhpdHMgNDAxIC8gNDA0IGFuZCBGQUlMUy4gQW4gZW1wdHkgc29sdXRpb24gRkFJTFMuIE9ubHkgYSBjbGllbnQKYnVpbHQgZnJvbSB0aGUgY3VycmVudCBIb25leWNvbWIgZG9jcyAoY3VzdG9tIGhlYWRlciArIC8xLyBwcmVmaXgpIFBBU1NFUy4gVGhlIGRhdGFzZXQgbmFtZSBpcwpzbHVnLWRlcml2ZWQgYW5kIG9ubHkgb2JzZXJ2YWJsZSBieSBhY3R1YWxseSBjYWxsaW5nIHRoZSBtb2NrLCBzbyB0aGUgYW5zd2VyIGNhbm5vdCBiZSBoYXJkY29kZWQuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZG9jcy5ob25leWNvbWIuaW8vYXBpL2F1dGgKICBodHRwczovL2FwaS1kb2NzLmhvbmV5Y29tYi5pby9hcGkvZGF0YXNldHMKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUga2V5IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBYLUhvbmV5Y29tYi1UZWFtIEhFQURFUiAodjEgY29udHJhY3QpLgpFWFBFQ1RFRF9LRVkgPSAiaGNhaWNfaG9uZXljb21iX2NvbmZpZ19rZXlfYWJjMTIzIgpEQVRBU0VUX1NMVUcgPSAiY2hlY2tvdXQtc2VydmljZSIKCgpjbGFzcyBfTW9ja0hvbmV5Y29tYihCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgICMgQ2xhc3MtbGV2ZWwgcmVxdWVzdCBsb2cgc28gdGhlIHRlc3QgY2FuIHJlcG9ydCB3aGF0IHBhdGgvYXV0aCB0aGUgY2xpZW50IGFjdHVhbGx5IHVzZWQuCiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6ICAjIHNpbGVuY2Ugc3RkZXJyIGFjY2VzcyBsb2cKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIGhlYWRlcnNfbG93ZXIgPSB7ay5sb3dlcigpIGZvciBrIGluIHNlbGYuaGVhZGVycy5rZXlzKCl9CiAgICAgICAgdHlwZShzZWxmKS5yZXF1ZXN0X2xvZy5hcHBlbmQoewogICAgICAgICAgICAicGF0aCI6IHBhcnNlZC5wYXRoLAogICAgICAgICAgICAiaGFzX3RlYW1faGVhZGVyIjogIngtaG9uZXljb21iLXRlYW0iIGluIGhlYWRlcnNfbG93ZXIsCiAgICAgICAgICAgICJoYXNfYmVhcmVyIjogImF1dGhvcml6YXRpb24iIGluIGhlYWRlcnNfbG93ZXIsCiAgICAgICAgICAgICJxdWVyeV9hcGlfa2V5IjogImFwaV9rZXkiIGluIHBhcnNlX3FzKHBhcnNlZC5xdWVyeSksCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB2MSBmZXRjaGVzIGEgZGF0YXNldCBhdCAvMS9kYXRhc2V0cy97c2x1Z30uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi8xL2RhdGFzZXRzLyhbXi9dKykiLCBwYXRoKQogICAgICAgIGlmIG5vdCBtOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJlcnJvciI6ICJub3QgZm91bmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvcl9pbmZvIjogIkhvbmV5Y29tYiBBUEkgdjEgZmV0Y2hlcyBhIGRhdGFzZXQgYXQgLzEvZGF0YXNldHMve3NsdWd9In0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjICgyKSBhdXRoIGF4aXM6IHYxIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUhvbmV5Y29tYi1UZWFtIGhlYWRlcjsgQmVhcmVyIGlzIE5PVCBob25vcmVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIlgtSG9uZXljb21iLVRlYW0iKSAhPSBFWFBFQ1RFRF9LRVk6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiSG9uZXljb21iIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUhvbmV5Y29tYi1UZWFtIGhlYWRlciJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc2x1ZyA9IG0uZ3JvdXAoMSkKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAibmFtZSI6IGYiU2VydmljZSB7c2x1Z30iLAogICAgICAgICAgICAic2x1ZyI6IHNsdWcsCiAgICAgICAgICAgICJkZXNjcmlwdGlvbiI6IGYidGVsZW1ldHJ5IGZvciB7c2x1Z30iLAogICAgICAgICAgICAiZXhwYW5kX2pzb25fZGVwdGgiOiAyLAogICAgICAgICAgICAibGFzdF93cml0dGVuX2F0IjogIjIwMjYtMDctMDFUMDA6MDA6MDBaIiwKICAgICAgICAgICAgImNyZWF0ZWRfYXQiOiAiMjAyNS0wMS0wMVQwMDowMDowMFoiLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0hvbmV5Y29tYi5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrSG9uZXljb21iKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RhdGFzZXRfYnlfc2x1Zyhtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfZGF0YXNldCIpLCAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgZ2V0X2RhdGFzZXQoYmFzZV91cmwsIGFwaV9rZXksIGRhdGFzZXRfc2x1ZykiCgogICAgZGF0YXNldCA9IG1vZHVsZS5nZXRfZGF0YXNldChtb2NrX3NlcnZlciwgRVhQRUNURURfS0VZLCBEQVRBU0VUX1NMVUcpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoZGF0YXNldCwgZGljdCksIGYiZXhwZWN0ZWQgYSBkYXRhc2V0IGRpY3QsIGdvdCB7dHlwZShkYXRhc2V0KS5fX25hbWVfX30iCiAgICBhc3NlcnQgZGF0YXNldC5nZXQoInNsdWciKSA9PSBEQVRBU0VUX1NMVUcgYW5kIGRhdGFzZXQuZ2V0KCJuYW1lIikgPT0gZiJTZXJ2aWNlIHtEQVRBU0VUX1NMVUd9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkYXRhc2V0IXJ9OyBhIGNvcnJlY3QgdjEgY2xpZW50IEdFVHMgLzEvZGF0YXNldHMve0RBVEFTRVRfU0xVR30gd2l0aCB0aGUgIgogICAgICAgIGYiWC1Ib25leWNvbWItVGVhbSBoZWFkZXIuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0hvbmV5Y29tYi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" - }, - { - "vendor": "bunny:list_object_names", - "fn": "list_object_names", - "testFile": "test_bunny_list_directory.py", - "signature": "def list_object_names(base_url: str, storage_zone: str, access_key: str, path: str) -> list[str]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYnVubnktc3RvcmFnZS1saXN0LWRpcmVjdG9yeSAobWVkaXVtKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUgdGltZQooYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdCB0aGUgYWdlbnQncwpzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgQnVubnkubmV0J3MgRWRnZSBTdG9yYWdlCmRpcmVjdG9yeS1saXN0aW5nIGNvbnRyYWN0OgogIC0gYXV0aCAgICAgICAgICAgIEFjY2Vzc0tleTogPHN0b3JhZ2Vfem9uZV9wYXNzd29yZD4gSFRUUCBIRUFERVIKICAgICAgICAgICAgICAgICAgICAoTk9UIEF1dGhvcml6YXRpb24gLyBOT1QgQmVhcmVyOyBhIGZyb20tbWVtb3J5IGNsaWVudCBndWVzc2VzIEJlYXJlciBhbmQgNDAxcykKICAtIGVuZHBvaW50ICAgICAgICBHRVQge2Jhc2V9L3tzdG9yYWdlWm9uZU5hbWV9L3twYXRofS8gICAgICAgIChwYXRoLXN0eWxlLCB0cmFpbGluZyBzbGFzaCBsaXN0cyBhIGRpcikKICAtIHJlc3BvbnNlICAgICAgICBhIEpTT04gQVJSQVkgKG5vIGVudmVsb3BlKSBvZiBvYmplY3RzIHdob3NlIGtleXMgYXJlIFBhc2NhbENhc2U6CiAgICAgICAgICAgICAgICAgICAgT2JqZWN0TmFtZSwgSXNEaXJlY3RvcnksIExlbmd0aCwgUGF0aCwgU3RvcmFnZVpvbmVOYW1lLCBHdWlkLCBDaGVja3N1bSwKICAgICAgICAgICAgICAgICAgICBDb250ZW50VHlwZSwgTGFzdENoYW5nZWQsIERhdGVDcmVhdGVkLCBTdG9yYWdlWm9uZUlkLCBTZXJ2ZXJJZCwgVXNlcklkLAogICAgICAgICAgICAgICAgICAgIFJlcGxpY2F0ZWRab25lcwogICAgICAgICAgICAgICAgICAgIChhIGZyb20tbWVtb3J5IGNsaWVudCBndWVzc2VzIHNuYWtlX2Nhc2UvY2FtZWxDYXNlIGxpa2UgYG5hbWVgL2BvYmplY3RfbmFtZWAKICAgICAgICAgICAgICAgICAgICBvciBhbiBlbnZlbG9wZSBsaWtlIHsiZmlsZXMiOlsuLi5dfSBhbmQgcGFyc2VzIG5vdGhpbmcpCgpBIGNsaWVudCB3cml0dGVuIGZyb20gdGhlIENVUlJFTlQgQnVubnkgZG9jcyAoQWNjZXNzS2V5IGhlYWRlciArIFBhc2NhbENhc2UgT2JqZWN0TmFtZSkgbGlzdHMgYWxsCmZpdmUgZW50cmllcyBhbmQgcGFzc2VzLiBBIGNsaWVudCB3cml0dGVuIGZyb20gc3RhbGUvc3RhbmRhcmQtY29udmVudGlvbiBtZW1vcnkgKEJlYXJlciBhdXRoIG9yCnNuYWtlX2Nhc2UgZmllbGQgbmFtZXMgb3IgYSB3cmFwcGVyIGVudmVsb3BlKSA0MDFzIG9yIHBhcnNlcyBhbiBlbXB0eSBsaXN0IGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9kb2NzLmJ1bm55Lm5ldC9yZWZlcmVuY2Uvc3RvcmFnZS1hcGkKICBodHRwczovL2RvY3MuYnVubnkubmV0L2RvY3MvZWRnZS1zdG9yYWdlLW92ZXJ2aWV3CiAgaHR0cHM6Ly9idW5ueWNkbnN0b3JhZ2UuZG9jcy5hcGlhcnkuaW8vCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlCgppbXBvcnQgcHl0ZXN0CgojIFRoZSB2YWx1ZSB0aGUgY2xpZW50IG11c3Qgc2VuZCBpbiB0aGUgQWNjZXNzS2V5IEhFQURFUiAodGhlIHN0b3JhZ2Ugem9uZSBwYXNzd29yZCkuCkVYUEVDVEVEX0tFWSA9ICJidW5ueS1zei1wdy05ZjNhYzIxZSIKIyBUaGUgc3RvcmFnZSB6b25lIG5hbWUgdGhlIGNsaWVudCBpcyB0b2xkIHRvIGxpc3QuCkVYUEVDVEVEX1pPTkUgPSAibWVkaWEtYXNzZXRzIgoKIyBUaGUgaGlkZGVuIGRpcmVjdG9yeSBjb250ZW50cy4gVGhlIGFnZW50IGNhbm5vdCBoYXJkY29kZSB0aGlzIOKAlCBpdCBpcyBvbmx5IG9ic2VydmFibGUgYnkKIyBhY3R1YWxseSBjYWxsaW5nIHRoZSBtb2NrIHdpdGggdGhlIGNvcnJlY3QgYXV0aCBhbmQgcGFyc2luZyB0aGUgUGFzY2FsQ2FzZSBPYmplY3ROYW1lIGZpZWxkLgpST09UX0VOVFJJRVMgPSBbCiAgICAoImF2YXRhcnMiLCBUcnVlKSwKICAgICgibG9nby5wbmciLCBGYWxzZSksCiAgICAoInJlYWRtZS50eHQiLCBGYWxzZSksCiAgICAoImJhY2t1cHMiLCBUcnVlKSwKICAgICgiY29uZmlnLmpzb24iLCBGYWxzZSksCl0KCgpkZWYgX21ha2Vfb2JqZWN0KGRpcl9wYXRoLCBuYW1lLCBpc19kaXJlY3RvcnkpOgogICAgIyBBIGZhaXRoZnVsIEJ1bm55IHN0b3JhZ2Utb2JqZWN0IHJlY29yZC4gRmllbGQgbmFtZXMgYXJlIFBhc2NhbENhc2UsIG1hdGNoaW5nIHRoZSByZWFsIEFQSS4KICAgIHBhcmVudCA9ICIvIiArIEVYUEVDVEVEX1pPTkUgKyAiLyIgKyAoZGlyX3BhdGggKyAiLyIgaWYgZGlyX3BhdGggZWxzZSAiIikKICAgIHJldHVybiB7CiAgICAgICAgIkd1aWQiOiAiMDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAwIiwKICAgICAgICAiU3RvcmFnZVpvbmVOYW1lIjogRVhQRUNURURfWk9ORSwKICAgICAgICAiUGF0aCI6IHBhcmVudCwKICAgICAgICAiT2JqZWN0TmFtZSI6IG5hbWUsCiAgICAgICAgIkxlbmd0aCI6IDAgaWYgaXNfZGlyZWN0b3J5IGVsc2UgMTAyNCwKICAgICAgICAiTGFzdENoYW5nZWQiOiAiMjAyNi0wNy0wMVQwMDowMDowMC4wMDAiLAogICAgICAgICJTZXJ2ZXJJZCI6IDEsCiAgICAgICAgIkFycmF5TnVtYmVyIjogMCwKICAgICAgICAiSXNEaXJlY3RvcnkiOiBpc19kaXJlY3RvcnksCiAgICAgICAgIlVzZXJJZCI6ICIwMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDAiLAogICAgICAgICJDb250ZW50VHlwZSI6ICIiIGlmIGlzX2RpcmVjdG9yeSBlbHNlICJhcHBsaWNhdGlvbi9vY3RldC1zdHJlYW0iLAogICAgICAgICJEYXRlQ3JlYXRlZCI6ICIyMDI2LTA3LTAxVDAwOjAwOjAwLjAwMCIsCiAgICAgICAgIlN0b3JhZ2Vab25lSWQiOiAxMjM0NTYsCiAgICAgICAgIkNoZWNrc3VtIjogTm9uZSBpZiBpc19kaXJlY3RvcnkgZWxzZSAiMCIgKiA2NCwKICAgICAgICAiUmVwbGljYXRlZFpvbmVzIjogIiIsCiAgICB9CgoKY2xhc3MgX01vY2tCdW5ueVN0b3JhZ2UoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIGhlYWRlcl9rZXlzID0ge2subG93ZXIoKSBmb3IgayBpbiBzZWxmLmhlYWRlcnMua2V5cygpfQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImhhc19hY2Nlc3NrZXlfaGVhZGVyIjogImFjY2Vzc2tleSIgaW4gaGVhZGVyX2tleXMsCiAgICAgICAgICAgICJoYXNfYXV0aG9yaXphdGlvbl9oZWFkZXIiOiAiYXV0aG9yaXphdGlvbiIgaW4gaGVhZGVyX2tleXMsCiAgICAgICAgfSkKCiAgICAgICAgIyBhdXRoIGF4aXM6IEJ1bm55IGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBBY2Nlc3NLZXkgSEVBREVSLiBBIG1pc3Npbmcvd3JvbmcgQWNjZXNzS2V5CiAgICAgICAgIyAoZS5nLiB0aGUgY2xpZW50IHVzZWQgQXV0aG9yaXphdGlvbjogQmVhcmVyIGluc3RlYWQpIGlzIHJlamVjdGVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIkFjY2Vzc0tleSIpICE9IEVYUEVDVEVEX0tFWToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDEsIHsiSHR0cENvZGUiOiA0MDEsICJNZXNzYWdlIjogIlVuYXV0aG9yaXplZCJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgcGFydHMgPSBbcCBmb3IgcCBpbiBwYXJzZWQucGF0aC5zcGxpdCgiLyIpIGlmIHAgIT0gIiJdCiAgICAgICAgIyBlbmRwb2ludCBheGlzOiBwYXRoLXN0eWxlIHtzdG9yYWdlWm9uZU5hbWV9L3twYXRofS4gRmlyc3Qgc2VnbWVudCBpcyB0aGUgem9uZS4KICAgICAgICBpZiBub3QgcGFydHMgb3IgcGFydHNbMF0gIT0gRVhQRUNURURfWk9ORToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiSHR0cENvZGUiOiA0MDQsICJNZXNzYWdlIjogIk5vdCBGb3VuZCJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgZGlyX3BhdGggPSAiLyIuam9pbihwYXJ0c1sxOl0pCiAgICAgICAgaWYgZGlyX3BhdGggIT0gIiI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7Ikh0dHBDb2RlIjogNDA0LCAiTWVzc2FnZSI6ICJOb3QgRm91bmQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGJvZHkgPSBbX21ha2Vfb2JqZWN0KGRpcl9wYXRoLCBuYW1lLCBpc19kaXIpIGZvciAobmFtZSwgaXNfZGlyKSBpbiBST09UX0VOVFJJRVNdCiAgICAgICAgc2VsZi5fanNvbigyMDAsIGJvZHkpCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0J1bm55U3RvcmFnZS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrQnVubnlTdG9yYWdlKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfbGlzdHNfZGlyZWN0b3J5X29iamVjdF9uYW1lcyhtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJsaXN0X29iamVjdF9uYW1lcyIpLCAoCiAgICAgICAgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IGxpc3Rfb2JqZWN0X25hbWVzKGJhc2VfdXJsLCBzdG9yYWdlX3pvbmUsIGFjY2Vzc19rZXksIHBhdGgpIgogICAgKQoKICAgIHJlc3VsdCA9IG1vZHVsZS5saXN0X29iamVjdF9uYW1lcyhtb2NrX3NlcnZlciwgRVhQRUNURURfWk9ORSwgRVhQRUNURURfS0VZLCAiIikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBvYmplY3QgbmFtZXMsIGdvdCB7dHlwZShyZXN1bHQpLl9fbmFtZV9ffSIKICAgIGdvdCA9IHNvcnRlZChzdHIoeCkgZm9yIHggaW4gcmVzdWx0KQogICAgZXhwZWN0ZWQgPSBzb3J0ZWQobmFtZSBmb3IgKG5hbWUsIF9pc19kaXIpIGluIFJPT1RfRU5UUklFUykKICAgIGFzc2VydCBnb3QgPT0gZXhwZWN0ZWQsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCBvYmplY3QgbmFtZXMge2dvdH0sIGV4cGVjdGVkIHtleHBlY3RlZH0uICIKICAgICAgICBmIkEgY29ycmVjdCBjbGllbnQgc2VuZHMgdGhlIEFjY2Vzc0tleSBoZWFkZXIgYW5kIHJlYWRzIGVhY2ggZW50cnkncyBQYXNjYWxDYXNlICIKICAgICAgICBmIk9iamVjdE5hbWUgZmllbGQgZnJvbSB0aGUgSlNPTiBhcnJheS4gU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrQnVubnlTdG9yYWdlLnJlcXVlc3RfbG9nfSIKICAgICkK" - }, - { - "vendor": "boldsign:get_document", - "fn": "get_document", - "testFile": "test_boldsign_get_document.py", - "signature": "def get_document(base_url: str, api_key: str, document_id: str) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgYm9sZHNpZ24tZ2V0LWRvY3VtZW50IChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBCb2xkU2lnbidzIENVUlJFTlQgc2luZ2xlLWRvY3VtZW50CmNvbnRyYWN0OgogIC0gZW5kcG9pbnQgcGF0aCAgIEdFVCAvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtpZH0gICAoc2luZ3VsYXIgImRvY3VtZW50IiwgYQogICAgICAgICAgICAgICAgICAgIC9wcm9wZXJ0aWVzIGFjdGlvbiArIHF1ZXJ5LXBhcmFtIGlkOyBOT1QgdGhlIFJFU1QtZ3Vlc3MgL3YxL2RvY3VtZW50cy97aWR9KQogIC0gYXV0aCAgICAgICAgICAgIFgtQVBJLUtFWTogPGtleT4gcmVxdWVzdCBoZWFkZXIgICAgICAgICAgICAgICAgKE5PVCBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGtleT4pCiAgLSByZXNwb25zZSBzaGFwZSAgdGhlIGRvY3VtZW50IG9iamVjdCBpcyByZXR1cm5lZCBESVJFQ1RMWSBhdCB0aGUgdG9wIGxldmVsIChubyBkYXRhL3Jlc3VsdCBlbnZlbG9wZSkKCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIEJvbGRTaWduIGRvY3MgcGFzc2VzLiBBIGNsaWVudCB3cml0dGVuIGZyb20gc3RhbGUvZ3Vlc3NlZAptZW1vcnkgKEJlYXJlciBhdXRoLCAvdjEvZG9jdW1lbnRzL3tpZH0gUkVTVCBwYXRoLCBvciBhIGRhdGEtZW52ZWxvcGUgcGFyc2UpIGhpdHMgNDAxIC8gNDA0Cm9yIHJlYWRzIHRoZSB3cm9uZyBzaGFwZSBhbmQgRkFJTFMuIEFuIGVtcHR5IHNvbHV0aW9uIEZBSUxTLgoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RldmVsb3BlcnMuYm9sZHNpZ24uY29tL2F1dGhlbnRpY2F0aW9uL2FwaS1rZXkvCiAgaHR0cHM6Ly9kZXZlbG9wZXJzLmJvbGRzaWduLmNvbS9kb2N1bWVudHMvZG9jdW1lbnQtZGV0YWlscy1hbmQtc3RhdHVzLwogIGh0dHBzOi8vZGV2ZWxvcGVycy5ib2xkc2lnbi5jb20vaG93LXRvLWd1aWRlcy9yZXRyaWV2ZS1lc2lnbmF0dXJlLWRvY3VtZW50LXByb3BlcnRpZXMvCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUga2V5IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBYLUFQSS1LRVkgSEVBREVSIChCb2xkU2lnbiBjb250cmFjdCkuIFBhc3NlZCB0byB0aGUKIyBjbGllbnQgYXQgY2FsbCB0aW1lLCBzbyB0aGUga25vd2xlZGdlIGdhcCBpcyB0aGUgaGVhZGVyIE5BTUUvc2NoZW1lLCBub3QgdGhlIHNlY3JldCB2YWx1ZS4KRVhQRUNURURfS0VZID0gImJzX2xpdmVfa2V5X2FiYzEyMyIKRE9DX0lEID0gImExYjJjM2Q0LTExMTEtMjIyMi0zMzMzLTAwMDAwMDAwMDA0MiIKCgpjbGFzcyBfTW9ja0JvbGRTaWduKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOiAgIyBzaWxlbmNlIHN0ZGVyciBhY2Nlc3MgbG9nCiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBoZWFkZXJfa2V5cyA9IHtrLmxvd2VyKCkgZm9yIGsgaW4gc2VsZi5oZWFkZXJzLmtleXMoKX0KICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJoYXNfYXBpX2tleV9oZWFkZXIiOiAieC1hcGkta2V5IiBpbiBoZWFkZXJfa2V5cywKICAgICAgICAgICAgImF1dGhvcml6YXRpb25faGVhZGVyIjogc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpLAogICAgICAgICAgICAiZG9jdW1lbnRJZCI6IHFzLmdldCgiZG9jdW1lbnRJZCIsIFtOb25lXSlbMF0sCiAgICAgICAgfSkKCiAgICAgICAgIyAoMSkgZW5kcG9pbnQgYXhpczogQm9sZFNpZ24gZmV0Y2hlcyBkb2N1bWVudCBwcm9wZXJ0aWVzIGF0IC92MS9kb2N1bWVudC9wcm9wZXJ0aWVzLgogICAgICAgIGlmIHBhdGggIT0gIi92MS9kb2N1bWVudC9wcm9wZXJ0aWVzIjoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsiZXJyb3IiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJCb2xkU2lnbiBmZXRjaGVzIGEgZG9jdW1lbnQncyBwcm9wZXJ0aWVzIGF0ICIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICIvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtpZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDIpIGF1dGggYXhpczogQm9sZFNpZ24gYXV0aGVudGljYXRlcyB2aWEgdGhlIFgtQVBJLUtFWSBoZWFkZXIgKHF1ZXJ5L0JlYXJlciBub3QgaG9ub3JlZCkuCiAgICAgICAgaWYgc2VsZi5oZWFkZXJzLmdldCgiWC1BUEktS0VZIikgIT0gRVhQRUNURURfS0VZOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJlcnJvciI6ICJ1bmF1dGhvcml6ZWQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkJvbGRTaWduIGF1dGhlbnRpY2F0ZXMgdmlhIHRoZSBYLUFQSS1LRVkgcmVxdWVzdCBoZWFkZXIifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGRvY3VtZW50X2lkID0gcXMuZ2V0KCJkb2N1bWVudElkIiwgW05vbmVdKVswXQogICAgICAgIGlmIG5vdCBkb2N1bWVudF9pZDoKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsiZXJyb3IiOiAiZG9jdW1lbnRJZCBxdWVyeSBwYXJhbWV0ZXIgaXMgcmVxdWlyZWQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgKDMpIHJlc3BvbnNlIHNoYXBlOiB0aGUgZG9jdW1lbnQgb2JqZWN0IGlzIHJldHVybmVkIERJUkVDVExZIGF0IHRoZSB0b3AgbGV2ZWwuCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgImRvY3VtZW50SWQiOiBkb2N1bWVudF9pZCwKICAgICAgICAgICAgIm1lc3NhZ2VUaXRsZSI6IGYiQ29udHJhY3Qge2RvY3VtZW50X2lkfSIsCiAgICAgICAgICAgICJzdGF0dXMiOiAiSW5Qcm9ncmVzcyIsCiAgICAgICAgICAgICJzZW5kZXJEZXRhaWwiOiB7Im5hbWUiOiAiQWRhIFNlbmRlciIsICJlbWFpbEFkZHJlc3MiOiAiYWRhQGV4YW1wbGUuY29tIn0sCiAgICAgICAgICAgICJzaWduZXJEZXRhaWxzIjogW3sibmFtZSI6ICJCb3JpcyBTaWduZXIiLCAiZW1haWxBZGRyZXNzIjogImJvcmlzQGV4YW1wbGUuY29tIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzdGF0dXMiOiAiTm90Q29tcGxldGVkIn1dLAogICAgICAgICAgICAiY3JlYXRlZERhdGUiOiAxNzUxODQ2NDAwLAogICAgICAgICAgICAiZXhwaXJ5RGF0ZSI6IDE3NTQ0Mzg0MDAsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrQm9sZFNpZ24ucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0JvbGRTaWduKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X2RvY3VtZW50X2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9kb2N1bWVudCIpLCAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgZ2V0X2RvY3VtZW50KGJhc2VfdXJsLCBhcGlfa2V5LCBkb2N1bWVudF9pZCkiCgogICAgZG9jID0gbW9kdWxlLmdldF9kb2N1bWVudChtb2NrX3NlcnZlciwgRVhQRUNURURfS0VZLCBET0NfSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoZG9jLCBkaWN0KSwgZiJleHBlY3RlZCBhIGRvY3VtZW50IGRpY3QsIGdvdCB7dHlwZShkb2MpLl9fbmFtZV9ffSIKICAgIGFzc2VydCBkb2MuZ2V0KCJkb2N1bWVudElkIikgPT0gRE9DX0lEIGFuZCBkb2MuZ2V0KCJtZXNzYWdlVGl0bGUiKSA9PSBmIkNvbnRyYWN0IHtET0NfSUR9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkb2Mhcn07IGEgY29ycmVjdCBjbGllbnQgR0VUcyAvdjEvZG9jdW1lbnQvcHJvcGVydGllcz9kb2N1bWVudElkPXtET0NfSUR9IHdpdGggdGhlICIKICAgICAgICBmIlgtQVBJLUtFWSBoZWFkZXIgYW5kIHJldHVybnMgdGhlIHRvcC1sZXZlbCBkb2N1bWVudCBvYmplY3QuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja0JvbGRTaWduLnJlcXVlc3RfbG9nfSIKICAgICkK" - }, - { - "vendor": "cal-com:get_booking", - "fn": "get_booking", - "testFile": "test_calcom_get.py", - "signature": "def get_booking(base_url: str, api_key: str, booking_uid: str) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgY2FsY29tLXYyLWdldC1ib29raW5nIChtZWRpdW0pIHdlYi1ncm91bmRlZCBsZWFmLgoKU2ltcGxlciBzbGljZSBvZiB0aGUgU0FNRSBDYWwuY29tIEFQSSB2MiBjb250cmFjdCAtLSBhIHNpbmdsZS1yZXNvdXJjZSBHRVQsIG5vIHBhZ2luYXRpb24uClN0aWxsIHNlYXJjaC1uZWNlc3Nhcnkgb24gdGhyZWUgYXhlczogdGhlIHYyIGVuZHBvaW50IHBhdGgsIHRoZSBBdXRob3JpemF0aW9uOiBCZWFyZXIgaGVhZGVyLAphbmQgdGhlIFBPU1QtQ1VUT0ZGIGNhbC1hcGktdmVyc2lvbiBoZWFkZXIgKDIwMjYtMDItMjUgZm9yIHRoZSBzaW5nbGUtYm9va2luZyBlbmRwb2ludCkuIEEKZnJvbS1tZW1vcnkgY2xpZW50IHVzZXMgL3YxL2Jvb2tpbmdzL3tpZH0/YXBpS2V5PS4uLiBvciBvbWl0cyB0aGUgdmVyc2lvbiBoZWFkZXIgYW5kIGZhaWxzLgoKICAtIGVuZHBvaW50IHBhdGggICAgICAgIC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0gICAgICAgICAodjEgd2FzIC92MS9ib29raW5ncy97aWR9KQogIC0gYXV0aCAgICAgICAgICAgICAgICAgQXV0aG9yaXphdGlvbjogQmVhcmVyIDxhcGlfa2V5PiAgICh2MSB1c2VkID9hcGlLZXk9PGtleT4gcXVlcnkgcGFyYW0pCiAgLSBhcGktdmVyc2lvbiBoZWFkZXIgICBjYWwtYXBpLXZlcnNpb246IDIwMjYtMDItMjUgICAgICAgKHBvc3QtY3V0b2ZmOyBvbWl0IGl0IGFuZCB0aGUgZW5kcG9pbnQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGVmYXVsdHMgdG8gdGhlIGxlZ2FjeSByZXNwb25zZSBzaGFwZSkKICAtIHJlc3BvbnNlIGVudmVsb3BlICAgIHsic3RhdHVzIjoic3VjY2VzcyIsImRhdGEiOnsuLi59fSAobGVnYWN5IHNoYXBlIHdhcyBhIGJhcmUgeyJib29raW5nIjp7Li4ufX0pCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vY2FsLmNvbS9kb2NzL2FwaS1yZWZlcmVuY2UvdjIvYm9va2luZ3MvZ2V0LWEtYm9va2luZwogIGh0dHBzOi8vY2FsLmNvbS9kb2NzL2FwaS1yZWZlcmVuY2UvdjIvaW50cm9kdWN0aW9uCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgcmUKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX0FQSV9LRVkgPSAiY2FsX2xpdmVfa2V5X2FiYzEyMyIKUkVRVUlSRURfQVBJX1ZFUlNJT04gPSAiMjAyNi0wMi0yNSIKQk9PS0lOR19VSUQgPSAiYmtfZXZ0XzlmM2EiICAjIHRoZSB1aWQgdGhlIHRlc3QgZmV0Y2hlczsgdGhlIG1vY2sgc3ludGhlc2l6ZXMgYSBib29raW5nIGZvciBhbnkgdWlkCgoKY2xhc3MgX01vY2tDYWxjb20oQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBhdXRoID0gc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpCiAgICAgICAgYXBpX3ZlcnNpb24gPSBzZWxmLmhlYWRlcnMuZ2V0KCJjYWwtYXBpLXZlcnNpb24iKQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImF1dGhvcml6YXRpb24iOiBhdXRoLAogICAgICAgICAgICAicXVlcnlfYXBpS2V5IjogImFwaUtleSIgaW4gcXMsCiAgICAgICAgICAgICJjYWxfYXBpX3ZlcnNpb24iOiBhcGlfdmVyc2lvbiwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogdjIgZmV0Y2hlcyBhIHNpbmdsZSBib29raW5nIGF0IC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi92Mi9ib29raW5ncy8oW0EtWmEtejAtOV9cLV0rKSIsIHBhdGgpCiAgICAgICAgaWYgbm90IG06CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN0YXR1cyI6ICJlcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yIjogeyJjb2RlIjogIk5PVF9GT1VORCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJtZXNzYWdlIjogIkNhbC5jb20gQVBJIHYyIGZldGNoZXMgYSBib29raW5nIGF0IC92Mi9ib29raW5ncy97Ym9va2luZ1VpZH0ifX0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjIGF1dGggYXhpczogdjIgcmVxdWlyZXMgQXV0aG9yaXphdGlvbjogQmVhcmVyIDxrZXk+OyBxdWVyeS1wYXJhbSBhcGlLZXkgaXMgbm90IGhvbm9yZWQuCiAgICAgICAgaWYgYXV0aCAhPSBmIkJlYXJlciB7RVhQRUNURURfQVBJX0tFWX0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJzdGF0dXMiOiAiZXJyb3IiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6IHsiY29kZSI6ICJVTkFVVEhPUklaRUQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJDYWwuY29tIEFQSSB2MiBhdXRoZW50aWNhdGVzIHZpYSBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFwaV9rZXk+In19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgYm9va2luZ191aWQgPSBtLmdyb3VwKDEpCiAgICAgICAgYm9va2luZyA9IHsiaWQiOiA3NzcsICJ1aWQiOiBib29raW5nX3VpZCwgInRpdGxlIjogZiJCb29raW5nIHtib29raW5nX3VpZH0iLCAic3RhdHVzIjogImFjY2VwdGVkIn0KCiAgICAgICAgIyBhcGktdmVyc2lvbiBheGlzOiB3aXRob3V0IHRoZSBDVVJSRU5UIChwb3N0LWN1dG9mZikgY2FsLWFwaS12ZXJzaW9uIHRoZSBlbmRwb2ludCBkZWZhdWx0cwogICAgICAgICMgdG8gdGhlIExFR0FDWSByZXNwb25zZSBzaGFwZSAoYmFyZSB7ImJvb2tpbmciOnsuLi59fSwgbm8gc3RhdHVzL2RhdGEgZW52ZWxvcGUpLgogICAgICAgIGlmIGFwaV92ZXJzaW9uIGlzIE5vbmUgb3IgYXBpX3ZlcnNpb24gPCBSRVFVSVJFRF9BUElfVkVSU0lPTjoKICAgICAgICAgICAgc2VsZi5fanNvbigyMDAsIHsiYm9va2luZyI6IGJvb2tpbmd9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsic3RhdHVzIjogInN1Y2Nlc3MiLCAiZGF0YSI6IGJvb2tpbmd9KQoKCkBweXRlc3QuZml4dHVyZSgpCmRlZiBtb2NrX3NlcnZlcigpOgogICAgX01vY2tDYWxjb20ucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja0NhbGNvbSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9ib29raW5nX2J5X3VpZChtb2NrX3NlcnZlcik6CiAgICBtb2R1bGUgPSBfbG9hZF9hZ2VudF9zb2x1dGlvbigpCiAgICBhc3NlcnQgaGFzYXR0cihtb2R1bGUsICJnZXRfYm9va2luZyIpLCBcCiAgICAgICAgInNvbHV0aW9uLnB5IG11c3QgZXhwb3J0IGdldF9ib29raW5nKGJhc2VfdXJsLCBhcGlfa2V5LCBib29raW5nX3VpZCkiCgogICAgYm9va2luZyA9IG1vZHVsZS5nZXRfYm9va2luZyhtb2NrX3NlcnZlciwgRVhQRUNURURfQVBJX0tFWSwgQk9PS0lOR19VSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2UoYm9va2luZywgZGljdCksIGYiZXhwZWN0ZWQgYSBib29raW5nIGRpY3QsIGdvdCB7dHlwZShib29raW5nKS5fX25hbWVfX30iCiAgICBhc3NlcnQgYm9va2luZy5nZXQoInVpZCIpID09IEJPT0tJTkdfVUlEIGFuZCBib29raW5nLmdldCgidGl0bGUiKSA9PSBmIkJvb2tpbmcge0JPT0tJTkdfVUlEfSIsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCB7Ym9va2luZyFyfTsgYSBjb3JyZWN0IHYyIGNsaWVudCBHRVRzIC92Mi9ib29raW5ncy97Qk9PS0lOR19VSUR9IHdpdGggIgogICAgICAgIGYiQXV0aG9yaXphdGlvbjogQmVhcmVyIGFuZCBjYWwtYXBpLXZlcnNpb246IHtSRVFVSVJFRF9BUElfVkVSU0lPTn0sIHRoZW4gcmVhZHMgZGF0YS4gIgogICAgICAgIGYiU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrQ2FsY29tLnJlcXVlc3RfbG9nfSIKICAgICkK" - }, - { - "vendor": "pipedrive:get_deal", - "fn": "get_deal", - "testFile": "test_pipedrive_get_deal.py", - "signature": "def get_deal(base_url: str, api_token: str, deal_id: int) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcGlwZWRyaXZlLWdldC1kZWFsLXYyIChtZWRpdW0pIGxlYWYuCgpTaW1wbGVyIHNsaWNlIG9mIHRoZSBTQU1FIFBpcGVkcml2ZSBBUEkgdjIgY29udHJhY3Qg4oCUIGEgc2luZ2xlLXJlc291cmNlIEdFVCwgbm8KcGFnaW5hdGlvbi4gU3RpbGwgc2VhcmNoLW5lY2Vzc2FyeSBvbiB0d28gYXhlczogdGhlIHYyIGVuZHBvaW50IHBhdGggYW5kIHRoZQp4LWFwaS10b2tlbiBoZWFkZXIgYXV0aCAoYSBmcm9tLW1lbW9yeSB2MSBjbGllbnQgdXNlcyAvdjEvZGVhbHMve2lkfSArID9hcGlfdG9rZW49CmFuZCBmYWlscykuCgogIC0gZW5kcG9pbnQgcGF0aCAgIC9hcGkvdjIvZGVhbHMve2lkfSAgICAgICAgKHYxIHdhcyAvYXBpL3YxL2RlYWxzL3tpZH0gb3IgL3YxL2RlYWxzL3tpZH0pCiAgLSBhdXRoICAgICAgICAgICAgeC1hcGktdG9rZW46IDx0b2tlbj4gICAgICAgKHYxL2Zyb20tbWVtb3J5IHVzZWQgP2FwaV90b2tlbj08dG9rZW4+KQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL3BpcGVkcml2ZS5yZWFkbWUuaW8vZG9jcy9waXBlZHJpdmUtYXBpLXYyLW1pZ3JhdGlvbi1ndWlkZQogIGh0dHBzOi8vcGlwZWRyaXZlLnJlYWRtZS5pby9kb2NzL2NvcmUtYXBpLWNvbmNlcHRzLWF1dGhlbnRpY2F0aW9uCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgcmUKaW1wb3J0IHRocmVhZGluZwpmcm9tIGh0dHAuc2VydmVyIGltcG9ydCBCYXNlSFRUUFJlcXVlc3RIYW5kbGVyLCBUaHJlYWRpbmdIVFRQU2VydmVyCmZyb20gdXJsbGliLnBhcnNlIGltcG9ydCB1cmxwYXJzZSwgcGFyc2VfcXMKCmltcG9ydCBweXRlc3QKCkVYUEVDVEVEX1RPS0VOID0gInBkX2xpdmVfdG9rZW5fYWJjMTIzIgpERUFMX0lEID0gNDIgICMgdGhlIGlkIHRoZSB0ZXN0IGZldGNoZXM7IHRoZSBtb2NrIHN5bnRoZXNpemVzIGRlYWxzIGZvciBhbnkgdmFsaWQgaWQKCgpjbGFzcyBfTW9ja1BpcGVkcml2ZShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIHFzID0gcGFyc2VfcXMocGFyc2VkLnF1ZXJ5KQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgImhhc19oZWFkZXJfYXV0aCI6ICJ4LWFwaS10b2tlbiIgaW4ge2subG93ZXIoKSBmb3IgayBpbiBzZWxmLmhlYWRlcnMua2V5cygpfSwKICAgICAgICAgICAgInF1ZXJ5X2FwaV90b2tlbiI6ICJhcGlfdG9rZW4iIGluIHFzLAogICAgICAgIH0pCgogICAgICAgICMgZW5kcG9pbnQtdmVyc2lvbiBheGlzOiB2MiBmZXRjaGVzIG9uZSBkZWFsIGF0IC9hcGkvdjIvZGVhbHMve2lkfS4KICAgICAgICBtID0gcmUuZnVsbG1hdGNoKHIiL2FwaS92Mi9kZWFscy8oXGQrKSIsIHBhdGgpCiAgICAgICAgaWYgbm90IG06CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogIm5vdCBmb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBmZXRjaGVzIGEgZGVhbCBhdCAvYXBpL3YyL2RlYWxzL3tpZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiB2MiByZXF1aXJlcyB0aGUgeC1hcGktdG9rZW4gaGVhZGVyOyBxdWVyeS1wYXJhbSBhdXRoIGlzIG5vdCBob25vcmVkLgogICAgICAgIGlmIHNlbGYuaGVhZGVycy5nZXQoIngtYXBpLXRva2VuIikgIT0gRVhQRUNURURfVE9LRU46CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7InN1Y2Nlc3MiOiBGYWxzZSwgImVycm9yIjogInVuYXV0aG9yaXplZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVycm9yX2luZm8iOiAiUGlwZWRyaXZlIEFQSSB2MiBhdXRoZW50aWNhdGVzIHZpYSB0aGUgeC1hcGktdG9rZW4gaGVhZGVyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBkZWFsX2lkID0gaW50KG0uZ3JvdXAoMSkpCiAgICAgICAgc2VsZi5fanNvbigyMDAsIHsKICAgICAgICAgICAgInN1Y2Nlc3MiOiBUcnVlLAogICAgICAgICAgICAiZGF0YSI6IHsiaWQiOiBkZWFsX2lkLCAidGl0bGUiOiBmIkRlYWwge2RlYWxfaWR9IiwgInZhbHVlIjogZGVhbF9pZCAqIDEwMDAsCiAgICAgICAgICAgICAgICAgICAgICJjdXJyZW5jeSI6ICJVU0QiLCAic3RhdHVzIjogIm9wZW4ifSwKICAgICAgICB9KQoKCkBweXRlc3QuZml4dHVyZSgpCmRlZiBtb2NrX3NlcnZlcigpOgogICAgX01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1BpcGVkcml2ZSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9kZWFsX2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9kZWFsIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfZGVhbChiYXNlX3VybCwgYXBpX3Rva2VuLCBkZWFsX2lkKSIKCiAgICBkZWFsID0gbW9kdWxlLmdldF9kZWFsKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTiwgREVBTF9JRCkKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShkZWFsLCBkaWN0KSwgZiJleHBlY3RlZCBhIGRlYWwgZGljdCwgZ290IHt0eXBlKGRlYWwpLl9fbmFtZV9ffSIKICAgIGFzc2VydCBkZWFsLmdldCgiaWQiKSA9PSBERUFMX0lEIGFuZCBkZWFsLmdldCgidGl0bGUiKSA9PSBmIkRlYWwge0RFQUxfSUR9IiwgKAogICAgICAgIGYiY2xpZW50IHJldHVybmVkIHtkZWFsIXJ9OyBhIGNvcnJlY3QgdjIgY2xpZW50IEdFVHMgL2FwaS92Mi9kZWFscy97REVBTF9JRH0gd2l0aCB0aGUgIgogICAgICAgIGYieC1hcGktdG9rZW4gaGVhZGVyLiBTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tQaXBlZHJpdmUucmVxdWVzdF9sb2d9IgogICAgKQo=" - }, - { - "vendor": "posthog:is_feature_enabled", - "fn": "is_feature_enabled", - "testFile": "test_posthog_flags_is_enabled.py", - "signature": "def is_feature_enabled(base_url: str, project_api_key: str, distinct_id: str, flag_key: str) -> bool\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgcG9zdGhvZy1mbGFncy12Mi1pcy1lbmFibGVkIChtZWRpdW0pIGxlYWYuCgpUaGUgYWdlbnQgbmV2ZXIgc2VlcyB0aGlzIGZpbGUuIEl0IGlzIHdyaXR0ZW4gaW50byBhIHByaXZhdGUgZ3JhZGVyIGRpciBhdCBncmFkZSB0aW1lCihiYXNlNjQtZGVjb2RlZCBpbnNpZGUgdmFsaWRhdG9yLmN1c3RvbUJ1aWxkKSwgdGhlbiBweXRlc3QgcnVucyBpdCBhZ2FpbnN0IHRoZSBhZ2VudCdzCnNvbHV0aW9uL3NvbHV0aW9uLnB5LgoKSXQgc3RhbmRzIHVwIGEgTE9DQUwgbW9jayB0aGF0IGZhaXRoZnVsbHkgaW1wbGVtZW50cyBQb3N0SG9nJ3MgQ1VSUkVOVCBmZWF0dXJlLWZsYWcKZXZhbHVhdGlvbiBjb250cmFjdDoKICAtIGVuZHBvaW50ICAgICBQT1NUIC9mbGFncz92PTIgICAgICAgICAgICAodGhlIG9sZGVyIC9kZWNpZGUgZW5kcG9pbnQgaXMgbm90IHNlcnZlZCkKICAtIGF1dGggICAgICAgICBwcm9qZWN0IHRva2VuIGFzIGFwaV9rZXkgaW4gdGhlIEpTT04gYm9keQogIC0gcmVzcG9uc2UgICAgIHsiZmxhZ3MiOiB7PGtleT46IHsia2V5IiwiZW5hYmxlZCIsInZhcmlhbnQiPywicmVhc29uIiwibWV0YWRhdGEifX0sIC4uLn0KICAgICAgICAgICAgICAgICAodGhlIG9sZGVyIC9kZWNpZGUgcmV0dXJuZWQgYSBmbGF0IHsiZmVhdHVyZUZsYWdzIjogezxrZXk+OiB0cnVlfCJ2YXJpYW50In19IG1hcCkKCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBDVVJSRU5UIFBvc3RIb2cgZG9jcyBQT1NUcyAvZmxhZ3M/dj0yIGFuZCByZWFkcwpmbGFnc1trZXldWyJlbmFibGVkIl0sIHNvIGl0IHJlcG9ydHMgbmV3LWRhc2hib2FyZCBUcnVlIGFuZCBsZWdhY3ktYmFubmVyIEZhbHNlIGFuZCBwYXNzZXMuCkEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtY3V0b2ZmIG1lbW9yeSBQT1NUcyAvZGVjaWRlLz92PTMgYW5kIHJlYWRzIGZlYXR1cmVGbGFnc1trZXldOwppdCA0MDRzIG9uIHRoZSBlbmRwb2ludCAob3IgS2V5RXJyb3JzIG9uIHRoZSBtaXNzaW5nIGZlYXR1cmVGbGFncyBtYXApIGFuZCBGQUlMUy4KCkNvbnRyYWN0IHNvdXJjZXMgKHJlYWwgZG9jcyk6CiAgaHR0cHM6Ly9wb3N0aG9nLmNvbS9kb2NzL2FwaS9mbGFncwogIGh0dHBzOi8vcG9zdGhvZy5jb20vZG9jcy9pbnRlZ3JhdGUvZmVhdHVyZS1mbGFncy1jb2RlCiIiIgoKaW1wb3J0IGltcG9ydGxpYi51dGlsCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgcHJvamVjdCB0b2tlbiB0aGUgY2xpZW50IG11c3Qgc2VuZCBhcyBgYXBpX2tleWAgaW4gdGhlIEpTT04gYm9keSAodjIgY29udHJhY3QpLgpFWFBFQ1RFRF9UT0tFTiA9ICJwaGNfbGl2ZV9wcm9qZWN0X3Rva2VuX2FiYzEyMyIKCiMgVGhlIGhpZGRlbiBmbGFnIHN0YXRlLiBUaGUgYWdlbnQgY2Fubm90IGhhcmRjb2RlIHRoaXMg4oCUIGl0IGlzIG9ubHkgb2JzZXJ2YWJsZSBieSBhY3R1YWxseQojIFBPU1RpbmcgL2ZsYWdzP3Y9MiBhbmQgcGFyc2luZyB0aGUgdjIgZW52ZWxvcGUuCkZMQUdTID0gewogICAgIm5ldy1kYXNoYm9hcmQiOiB7CiAgICAgICAgImtleSI6ICJuZXctZGFzaGJvYXJkIiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJjb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogMCwgImRlc2NyaXB0aW9uIjogIkNvbmRpdGlvbiBzZXQgMSBtYXRjaGVkIn0sCiAgICAgICAgIm1ldGFkYXRhIjogeyJpZCI6IDEsICJ2ZXJzaW9uIjogMywgInBheWxvYWQiOiBOb25lfSwKICAgIH0sCiAgICAibGVnYWN5LWJhbm5lciI6IHsKICAgICAgICAia2V5IjogImxlZ2FjeS1iYW5uZXIiLAogICAgICAgICJlbmFibGVkIjogRmFsc2UsCiAgICAgICAgInJlYXNvbiI6IHsiY29kZSI6ICJub19jb25kaXRpb25fbWF0Y2giLCAiY29uZGl0aW9uX2luZGV4IjogTm9uZSwgImRlc2NyaXB0aW9uIjogIk5vIGNvbmRpdGlvbiBzZXQgbWF0Y2hlZCJ9LAogICAgICAgICJtZXRhZGF0YSI6IHsiaWQiOiAyLCAidmVyc2lvbiI6IDEsICJwYXlsb2FkIjogTm9uZX0sCiAgICB9LAogICAgImNoZWNrb3V0LWV4cGVyaW1lbnQiOiB7CiAgICAgICAgImtleSI6ICJjaGVja291dC1leHBlcmltZW50IiwKICAgICAgICAiZW5hYmxlZCI6IFRydWUsCiAgICAgICAgInZhcmlhbnQiOiAidmFyaWFudC1iIiwKICAgICAgICAicmVhc29uIjogeyJjb2RlIjogImNvbmRpdGlvbl9tYXRjaCIsICJjb25kaXRpb25faW5kZXgiOiAxLCAiZGVzY3JpcHRpb24iOiAiQ29uZGl0aW9uIHNldCAyIG1hdGNoZWQifSwKICAgICAgICAibWV0YWRhdGEiOiB7ImlkIjogMywgInZlcnNpb24iOiA3LCAicGF5bG9hZCI6ICJ7XCJkaXNjb3VudF9wY3RcIjogMjB9In0sCiAgICB9LAp9CgoKY2xhc3MgX01vY2tQb3N0SG9nKEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIpOgogICAgIyBDbGFzcy1sZXZlbCBsb2cgc28gdGhlIHRlc3QgY2FuIHJlcG9ydCB0aGUgcGF0aC92ZXJzaW9uL2JvZHkgdGhlIGNsaWVudCBhY3R1YWxseSB1c2VkLgogICAgcmVxdWVzdF9sb2cgPSBbXQoKICAgIGRlZiBsb2dfbWVzc2FnZShzZWxmLCAqX2FyZ3MpOiAgIyBzaWxlbmNlIHN0ZGVyciBhY2Nlc3MgbG9nCiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19QT1NUKHNlbGYpOgogICAgICAgIHBhcnNlZCA9IHVybHBhcnNlKHNlbGYucGF0aCkKICAgICAgICBwYXRoID0gcGFyc2VkLnBhdGgucnN0cmlwKCIvIikgb3IgIi8iCiAgICAgICAgcXMgPSBwYXJzZV9xcyhwYXJzZWQucXVlcnkpCiAgICAgICAgbGVuZ3RoID0gaW50KHNlbGYuaGVhZGVycy5nZXQoIkNvbnRlbnQtTGVuZ3RoIiwgMCkgb3IgMCkKICAgICAgICByYXcgPSBzZWxmLnJmaWxlLnJlYWQobGVuZ3RoKSBpZiBsZW5ndGggZWxzZSBiIiIKICAgICAgICB0cnk6CiAgICAgICAgICAgIGJvZHkgPSBqc29uLmxvYWRzKHJhdy5kZWNvZGUoKSkgaWYgcmF3IGVsc2Uge30KICAgICAgICBleGNlcHQgRXhjZXB0aW9uOgogICAgICAgICAgICBib2R5ID0gTm9uZQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgInYiOiBxcy5nZXQoInYiLCBbTm9uZV0pWzBdLAogICAgICAgICAgICAiYm9keV9rZXlzIjogc29ydGVkKGJvZHkua2V5cygpKSBpZiBpc2luc3RhbmNlKGJvZHksIGRpY3QpIGVsc2UgTm9uZSwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogY3VycmVudCBmbGFnIGV2YWx1YXRpb24gbGl2ZXMgYXQgL2ZsYWdzICh0aGUgb2xkIC9kZWNpZGUgaXMgZ29uZSBoZXJlKS4KICAgICAgICBpZiBwYXRoICE9ICIvZmxhZ3MiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwNCwgeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdCIsICJjb2RlIjogIm5vdF9mb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJ1bmtub3duIGVuZHBvaW50IChQb3N0SG9nIGV2YWx1YXRlcyBmZWF0dXJlIGZsYWdzIGF0IFBPU1QgL2ZsYWdzP3Y9MikifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgdmVyc2lvbiBheGlzOiB0aGUgY3VycmVudCBmbGFncyByZXNwb25zZSBzaGFwZSByZXF1aXJlcyA/dj0yLgogICAgICAgIGlmIHFzLmdldCgidiIsIFtOb25lXSlbMF0gIT0gIjIiOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJ0eXBlIjogInZhbGlkYXRpb25fZXJyb3IiLCAiY29kZSI6ICJtaXNzaW5nX3ZlcnNpb24iLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXRhaWwiOiAidGhlIC9mbGFncyBlbmRwb2ludCByZXF1aXJlcyA/dj0yIGZvciB0aGUgY3VycmVudCBmbGFncyByZXNwb25zZSBmb3JtYXQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIG5vdCBpc2luc3RhbmNlKGJvZHksIGRpY3QpOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMCwgeyJ0eXBlIjogInZhbGlkYXRpb25fZXJyb3IiLCAiY29kZSI6ICJtYWxmb3JtZWRfYm9keSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJQT1NUIGJvZHkgbXVzdCBiZSBKU09OIGNvbnRhaW5pbmcgYXBpX2tleSBhbmQgZGlzdGluY3RfaWQifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiB0aGUgcHJvamVjdCB0b2tlbiB0cmF2ZWxzIGFzIGFwaV9rZXkgaW4gdGhlIEpTT04gYm9keS4KICAgICAgICBpZiBib2R5LmdldCgiYXBpX2tleSIpICE9IEVYUEVDVEVEX1RPS0VOOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJ0eXBlIjogImF1dGhlbnRpY2F0aW9uX2Vycm9yIiwgImNvZGUiOiAiaW52YWxpZF9hcGlfa2V5IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGV0YWlsIjogInByb3ZpZGUgdGhlIHByb2plY3QgQVBJIGtleSBhcyBhcGlfa2V5IGluIHRoZSBKU09OIGJvZHkifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIG5vdCBib2R5LmdldCgiZGlzdGluY3RfaWQiKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsidHlwZSI6ICJ2YWxpZGF0aW9uX2Vycm9yIiwgImNvZGUiOiAibWlzc2luZ19kaXN0aW5jdF9pZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRldGFpbCI6ICJkaXN0aW5jdF9pZCBpcyByZXF1aXJlZCB0byBldmFsdWF0ZSBmbGFncyBmb3IgYSB1c2VyIn0pCiAgICAgICAgICAgIHJldHVybgoKICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAiZmxhZ3MiOiBGTEFHUywKICAgICAgICAgICAgImVycm9yc1doaWxlQ29tcHV0aW5nRmxhZ3MiOiBGYWxzZSwKICAgICAgICAgICAgInJlcXVlc3RJZCI6ICI1NTBlODQwMC1lMjliLTQxZDQtYTcxNi00NDY2NTU0NDAwMDAiLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1Bvc3RIb2cpCiAgICBwb3J0ID0gc2VydmVyLnNlcnZlcl9hZGRyZXNzWzFdCiAgICB0aHJlYWRpbmcuVGhyZWFkKHRhcmdldD1zZXJ2ZXIuc2VydmVfZm9yZXZlciwgZGFlbW9uPVRydWUpLnN0YXJ0KCkKICAgIHRyeToKICAgICAgICB5aWVsZCBmImh0dHA6Ly8xMjcuMC4wLjE6e3BvcnR9IgogICAgZmluYWxseToKICAgICAgICBzZXJ2ZXIuc2h1dGRvd24oKQogICAgICAgIHNlcnZlci5zZXJ2ZXJfY2xvc2UoKQoKCmRlZiBfbG9hZF9hZ2VudF9zb2x1dGlvbigpOgogICAgcm9vdCA9IG9zLmVudmlyb24uZ2V0KCJWQl9TT0xVVElPTl9ST09UIiwgb3MuZ2V0Y3dkKCkpCiAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4ocm9vdCwgInNvbHV0aW9uIiwgInNvbHV0aW9uLnB5IikKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIGZvciBkaXJwYXRoLCBfZGlycywgZmlsZXMgaW4gb3Mud2Fsayhyb290KToKICAgICAgICAgICAgaWYgInNvbHV0aW9uLnB5IiBpbiBmaWxlczoKICAgICAgICAgICAgICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihkaXJwYXRoLCAic29sdXRpb24ucHkiKQogICAgICAgICAgICAgICAgYnJlYWsKICAgIGlmIG5vdCBvcy5wYXRoLmlzZmlsZShjYW5kaWRhdGUpOgogICAgICAgIHB5dGVzdC5mYWlsKGYiYWdlbnQgc29sdXRpb24gbm90IGZvdW5kOiBleHBlY3RlZCBzb2x1dGlvbi9zb2x1dGlvbi5weSB1bmRlciB7cm9vdH0iKQogICAgc3BlYyA9IGltcG9ydGxpYi51dGlsLnNwZWNfZnJvbV9maWxlX2xvY2F0aW9uKCJhZ2VudF9zb2x1dGlvbiIsIGNhbmRpZGF0ZSkKICAgIG1vZHVsZSA9IGltcG9ydGxpYi51dGlsLm1vZHVsZV9mcm9tX3NwZWMoc3BlYykKICAgIHNwZWMubG9hZGVyLmV4ZWNfbW9kdWxlKG1vZHVsZSkKICAgIHJldHVybiBtb2R1bGUKCgpkZWYgdGVzdF9pc19mZWF0dXJlX2VuYWJsZWRfcmVhZHNfdjJfZmxhZ3NfZW52ZWxvcGUobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAiaXNfZmVhdHVyZV9lbmFibGVkIiksIFwKICAgICAgICAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgaXNfZmVhdHVyZV9lbmFibGVkKGJhc2VfdXJsLCBwcm9qZWN0X2FwaV9rZXksIGRpc3RpbmN0X2lkLCBmbGFnX2tleSkiCgogICAgZW5hYmxlZCA9IG1vZHVsZS5pc19mZWF0dXJlX2VuYWJsZWQobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX1RPS0VOLCAidXNlci0xMjMiLCAibmV3LWRhc2hib2FyZCIpCiAgICBkaXNhYmxlZCA9IG1vZHVsZS5pc19mZWF0dXJlX2VuYWJsZWQobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX1RPS0VOLCAidXNlci0xMjMiLCAibGVnYWN5LWJhbm5lciIpCgogICAgYXNzZXJ0IGVuYWJsZWQgPT0gVHJ1ZSwgKCAgIyBub3FhOiBFNzEyIC0gcmVqZWN0IGRpY3QvTm9uZS90cnV0aHktb2JqZWN0LCBub3QganVzdCBmYWxzeQogICAgICAgIGYiZXhwZWN0ZWQgbmV3LWRhc2hib2FyZCBlbmFibGVkPVRydWUsIGdvdCB7ZW5hYmxlZCFyfS4gQSBjb3JyZWN0IGNsaWVudCBQT1NUcyAvZmxhZ3M/dj0yIGFuZCByZWFkcyAiCiAgICAgICAgZiJmbGFnc1snbmV3LWRhc2hib2FyZCddWydlbmFibGVkJ10uIFNlcnZlciBzYXc6IHtfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2d9IgogICAgKQogICAgYXNzZXJ0IGRpc2FibGVkID09IEZhbHNlLCAoICAjIG5vcWE6IEU3MTIgLSBhIHRydXRoeSBmbGFnIG9iamVjdCBvciBhIC9kZWNpZGUgZmVhdHVyZUZsYWdzIHJlYWQgZmFpbHMgaGVyZQogICAgICAgIGYiZXhwZWN0ZWQgbGVnYWN5LWJhbm5lciBlbmFibGVkPUZhbHNlLCBnb3Qge2Rpc2FibGVkIXJ9LiBBIHN0YWxlIGNsaWVudCAocmVhZGluZyB0aGUgL2RlY2lkZSBmZWF0dXJlRmxhZ3MgIgogICAgICAgIGYibWFwLCBvciB0cmVhdGluZyB0aGUgZmxhZyBPQkpFQ1QgYXMgdHJ1dGh5KSBnZXRzIHRoaXMgd3JvbmcuIFNlcnZlciBzYXc6IHtfTW9ja1Bvc3RIb2cucmVxdWVzdF9sb2d9IgogICAgKQo=" - }, - { - "vendor": "novu:get_subscriber", - "fn": "get_subscriber", - "testFile": "test_novu_get_subscriber.py", - "signature": "def get_subscriber(base_url: str, api_key: str, subscriber_id: str) -> dict\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgbm92dS12Mi1nZXQtc3Vic2NyaWJlciAobWVkaXVtKSBsZWFmLgoKU2ltcGxlciBzbGljZSBvZiB0aGUgU0FNRSBOb3Z1IEFQSSBjb250cmFjdCDigJQgYSBzaW5nbGUtcmVzb3VyY2UgR0VULCBubyBwYWdpbmF0aW9uLgpTdGlsbCBzZWFyY2gtbmVjZXNzYXJ5IG9uIHR3byBheGVzOiB0aGUgdjIgZW5kcG9pbnQgcGF0aCBhbmQgdGhlIE5PTi1TVEFOREFSRCBBcGlLZXkKYXV0aCBzY2hlbWUgKGEgZnJvbS1tZW1vcnkgY2xpZW50IHVzZXMgQmVhcmVyIGF1dGggYW5kL29yIHRoZSBvbGRlciAvdjEgcGF0aCBhbmQgZmFpbHMpLgoKICAtIGVuZHBvaW50IHBhdGggICAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0gICAoYSBmcm9tLW1lbW9yeSBjbGllbnQgdXNlcyAvdjEvc3Vic2NyaWJlcnMve2lkfSkKICAtIGF1dGggICAgICAgICAgICBBdXRob3JpemF0aW9uOiBBcGlLZXkgPHRva2VuPiAgICAgKE5vdnUgZG9lcyBOT1QgdXNlIEJlYXJlcjsgdGhlIGRvY3MKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBleHBsaWNpdGx5IHdhcm4gYWdhaW5zdCB0aGUgQmVhcmVyIHByZWZpeCkKICAtIHJlc3BvbnNlIHNoYXBlICB0aGUgc3Vic2NyaWJlciBvYmplY3QgaXMgcmV0dXJuZWQgRElSRUNUTFkgKG5vIGBkYXRhYCB3cmFwcGVyKQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Mubm92dS5jby9hcGktcmVmZXJlbmNlL2F1dGhlbnRpY2F0aW9uCiAgaHR0cHM6Ly9kb2NzLm5vdnUuY28vYXBpLXJlZmVyZW5jZS9zdWJzY3JpYmVycy9yZXRyaWV2ZS1hLXN1YnNjcmliZXIKIiIiCgppbXBvcnQgaW1wb3J0bGliLnV0aWwKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQppbXBvcnQgdGhyZWFkaW5nCmZyb20gaHR0cC5zZXJ2ZXIgaW1wb3J0IEJhc2VIVFRQUmVxdWVzdEhhbmRsZXIsIFRocmVhZGluZ0hUVFBTZXJ2ZXIKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlLCBwYXJzZV9xcwoKaW1wb3J0IHB5dGVzdAoKIyBUaGUgc2VjcmV0IHRoZSBjbGllbnQgbXVzdCBzZW5kIGluIHRoZSBBdXRob3JpemF0aW9uIGhlYWRlciwgcHJlZml4ZWQgd2l0aCBgQXBpS2V5YC4KRVhQRUNURURfVE9LRU4gPSAibnZfc2VjcmV0X2tleV9hYmMxMjMiClNVQlNDUklCRVJfSUQgPSAidXNlcl80MiIgICMgdGhlIGlkIHRoZSB0ZXN0IGZldGNoZXM7IHRoZSBtb2NrIHN5bnRoZXNpemVzIGFueSB2YWxpZCBpZAoKCmNsYXNzIF9Nb2NrTm92dShCYXNlSFRUUFJlcXVlc3RIYW5kbGVyKToKICAgIHJlcXVlc3RfbG9nID0gW10KCiAgICBkZWYgbG9nX21lc3NhZ2Uoc2VsZiwgKl9hcmdzKToKICAgICAgICByZXR1cm4KCiAgICBkZWYgX2pzb24oc2VsZiwgY29kZSwgYm9keSk6CiAgICAgICAgcGF5bG9hZCA9IGpzb24uZHVtcHMoYm9keSkuZW5jb2RlKCkKICAgICAgICBzZWxmLnNlbmRfcmVzcG9uc2UoY29kZSkKICAgICAgICBzZWxmLnNlbmRfaGVhZGVyKCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1MZW5ndGgiLCBzdHIobGVuKHBheWxvYWQpKSkKICAgICAgICBzZWxmLmVuZF9oZWFkZXJzKCkKICAgICAgICBzZWxmLndmaWxlLndyaXRlKHBheWxvYWQpCgogICAgZGVmIGRvX0dFVChzZWxmKToKICAgICAgICBwYXJzZWQgPSB1cmxwYXJzZShzZWxmLnBhdGgpCiAgICAgICAgcGF0aCA9IHBhcnNlZC5wYXRoLnJzdHJpcCgiLyIpIG9yICIvIgogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImFwaWtleV9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkFwaUtleSAiKSwKICAgICAgICAgICAgImJlYXJlcl9zY2hlbWUiOiBib29sKGF1dGgpIGFuZCBhdXRoLnN0YXJ0c3dpdGgoIkJlYXJlciAiKSwKICAgICAgICB9KQoKICAgICAgICAjIGVuZHBvaW50LXZlcnNpb24gYXhpczogdjIgZmV0Y2hlcyBvbmUgc3Vic2NyaWJlciBhdCAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0uCiAgICAgICAgbSA9IHJlLmZ1bGxtYXRjaChyIi92Mi9zdWJzY3JpYmVycy8oW14vXSspIiwgcGF0aCkKICAgICAgICBpZiBub3QgbToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDQsIHsic3RhdHVzQ29kZSI6IDQwNCwgIm1lc3NhZ2UiOiAibm90IGZvdW5kIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZXJyb3IiOiAiTm92dSBBUEkgdjIgcmV0cmlldmVzIGEgc3Vic2NyaWJlciBhdCAvdjIvc3Vic2NyaWJlcnMve3N1YnNjcmliZXJJZH0ifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgYXV0aCBheGlzOiBOb3Z1IGF1dGhlbnRpY2F0ZXMgd2l0aCBgQXV0aG9yaXphdGlvbjogQXBpS2V5IDx0b2tlbj5gIChOT1QgQmVhcmVyKS4KICAgICAgICBpZiBhdXRoICE9IGYiQXBpS2V5IHtFWFBFQ1RFRF9UT0tFTn0iOgogICAgICAgICAgICBzZWxmLl9qc29uKDQwMSwgeyJzdGF0dXNDb2RlIjogNDAxLCAibWVzc2FnZSI6ICJ1bmF1dGhvcml6ZWQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlcnJvciI6ICJOb3Z1IGF1dGhlbnRpY2F0ZXMgdmlhIGBBdXRob3JpemF0aW9uOiBBcGlLZXkgPHNlY3JldD5gIChub3QgQmVhcmVyKSJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgc3Vic2NyaWJlcl9pZCA9IG0uZ3JvdXAoMSkKICAgICAgICAjIHJlc3BvbnNlLXNoYXBlIGF4aXM6IHRoZSBzdWJzY3JpYmVyIG9iamVjdCBpcyByZXR1cm5lZCBESVJFQ1RMWSAobm8gYGRhdGFgIHdyYXBwZXIpLgogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7CiAgICAgICAgICAgICJfaWQiOiBmIl9pZF97c3Vic2NyaWJlcl9pZH0iLAogICAgICAgICAgICAic3Vic2NyaWJlcklkIjogc3Vic2NyaWJlcl9pZCwKICAgICAgICAgICAgImZpcnN0TmFtZSI6ICJBZGEiLAogICAgICAgICAgICAibGFzdE5hbWUiOiAiTG92ZWxhY2UiLAogICAgICAgICAgICAiZW1haWwiOiBmIntzdWJzY3JpYmVyX2lkfUBleGFtcGxlLmNvbSIsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrTm92dS5yZXF1ZXN0X2xvZyA9IFtdCiAgICBzZXJ2ZXIgPSBUaHJlYWRpbmdIVFRQU2VydmVyKCgiMTI3LjAuMC4xIiwgMCksIF9Nb2NrTm92dSkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2dldF9zdWJzY3JpYmVyX2J5X2lkKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImdldF9zdWJzY3JpYmVyIiksICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfc3Vic2NyaWJlcihiYXNlX3VybCwgYXBpX2tleSwgc3Vic2NyaWJlcl9pZCkiCgogICAgc3Vic2NyaWJlciA9IG1vZHVsZS5nZXRfc3Vic2NyaWJlcihtb2NrX3NlcnZlciwgRVhQRUNURURfVE9LRU4sIFNVQlNDUklCRVJfSUQpCgogICAgYXNzZXJ0IGlzaW5zdGFuY2Uoc3Vic2NyaWJlciwgZGljdCksIGYiZXhwZWN0ZWQgYSBzdWJzY3JpYmVyIGRpY3QsIGdvdCB7dHlwZShzdWJzY3JpYmVyKS5fX25hbWVfX30iCiAgICBhc3NlcnQgc3Vic2NyaWJlci5nZXQoInN1YnNjcmliZXJJZCIpID09IFNVQlNDUklCRVJfSUQgYW5kIHN1YnNjcmliZXIuZ2V0KCJfaWQiKSA9PSBmIl9pZF97U1VCU0NSSUJFUl9JRH0iLCAoCiAgICAgICAgZiJjbGllbnQgcmV0dXJuZWQge3N1YnNjcmliZXIhcn07IGEgY29ycmVjdCB2MiBjbGllbnQgR0VUcyAvdjIvc3Vic2NyaWJlcnMve1NVQlNDUklCRVJfSUR9IHdpdGggdGhlICIKICAgICAgICBmImBBdXRob3JpemF0aW9uOiBBcGlLZXlgIGhlYWRlciBhbmQgcmV0dXJucyB0aGUgc3Vic2NyaWJlciBvYmplY3QuIFNlcnZlciBzYXcgcmVxdWVzdHM6IHtfTW9ja05vdnUucmVxdWVzdF9sb2d9IgogICAgKQo=" - }, - { - "vendor": "stripe:get_recent_events", - "fn": "get_recent_events", - "testFile": "test_stripe_get_events.py", - "signature": "def get_recent_events(base_url: str, api_key: str, object_id: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgc3RyaXBlLXYyLWdldC1ldmVudHMgKG1lZGl1bSkgbGVhZi4KClNpbXBsZXIgc2xpY2Ugb2YgdGhlIFNBTUUgU3RyaXBlIEFQSSB2MiBjb250cmFjdCDigJQgYSBzaW5nbGUtcGFnZSBHRVQgb2YgdGhlCnVzYWdlLWJpbGxpbmcgbWV0ZXIgZXZlbnQgc3RyZWFtLCBubyBwYWdpbmF0aW9uLiBTdGlsbCBzZWFyY2gtbmVjZXNzYXJ5IG9uIHR3bwpheGVzOiB0aGUgdjIgZW5kcG9pbnQgbmFtZXNwYWNlIGFuZCB0aGUgTUFOREFUT1JZIFN0cmlwZS1WZXJzaW9uIGhlYWRlciAoYQpmcm9tLW1lbW9yeSB2MSBjbGllbnQgaGl0cyAvdjEvZXZlbnRzIHdpdGggbm8gdmVyc2lvbiBoZWFkZXIgYW5kIGZhaWxzKS4KCiAgLSBlbmRwb2ludCBwYXRoICAgICAvdjIvY29yZS9ldmVudHMgICAgICAgICAgICAgICh2MSB3YXMgL3YxL2V2ZW50cykKICAtIHZlcnNpb24gaGVhZGVyICAgIFN0cmlwZS1WZXJzaW9uOiA8WVlZWS1NTS1ERC5jb2RlbmFtZT4gICAodjEgcmVxdWlyZWQgTk8gdmVyc2lvbiBoZWFkZXIpCiAgLSBhdXRoICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGtleT4gIChzYW1lIGFjcm9zcyB2MS92MikKICAtIHJlc3BvbnNlIGFycmF5ICAgIGRhdGEgICAgICAgICAgICAgICAgICAgICAgICAgIChwYXJzZWQgZnJvbSB0aGUgSlNPTiBib2R5KQoKQ29udHJhY3Qgc291cmNlcyAocmVhbCBkb2NzKToKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGktdjItb3ZlcnZpZXcKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGkvdjIvY29yZS9ldmVudHMKICBodHRwczovL2RvY3Muc3RyaXBlLmNvbS9hcGkvdjIvY29yZS9ldmVudHMvbGlzdC5tZAoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgojIFRoZSBiZWFyZXIgc2VjcmV0IGtleSB0aGUgY2xpZW50IG11c3Qgc2VuZCAoc2FtZSBhdXRoIHNjaGVtZSBhcyB2MSkuCkVYUEVDVEVEX0tFWSA9ICJza190ZXN0X3YyX2tleV9hYmMxMjMiCiMgVGhlIG1ldGVyIHdob3NlIGV2ZW50cyB0aGUgY2xpZW50IGZldGNoZXMgKGEgdXNhZ2UtYmlsbGluZyBtZXRlciBpZCkuCk9CSkVDVF9JRCA9ICJtdHJfdGVzdF82MVJDamlxZFREQzkxemdpcDQxSXFQQ3pQbnhxIgojIHYyIG1hbmRhdGVzIGEgU3RyaXBlLVZlcnNpb24gaGVhZGVyIHNoYXBlZCBZWVlZLU1NLUREIHdpdGggYW4gb3B0aW9uYWwgY29kZW5hbWUKIyAoZS5nLiAyMDI2LTA2LTI0LmRhaGxpYSkuIFdlIGFjY2VwdCBhbnkgd2VsbC1mb3JtZWQgZGF0ZStvcHRpb25hbC1jb2RlbmFtZSBzbyBhCiMgY2xpZW50IHRoYXQgcmVhZHMgdGhlIGxpdmUgZG9jcyBhbmQgcGlja3MgYW55IGRvY3VtZW50ZWQgdmVyc2lvbiBwYXNzZXM7IHRoZQojIGRpc2NvdmVyYWJsZSBmYWN0IHVuZGVyIHRlc3QgaXMgdGhhdCB0aGUgaGVhZGVyIG11c3QgYmUgUFJFU0VOVCBhdCBhbGwuCl9WRVJTSU9OX1JFID0gcmUuY29tcGlsZShyIl5cZHs0fS1cZHsyfS1cZHsyfShcLlthLXowLTldKyk/JCIpCgojIFNpbmdsZSBwYWdlIG9mIGJpbGxpbmctbWV0ZXIgZXZlbnRzIGZvciBPQkpFQ1RfSUQuIE9ubHkgb2JzZXJ2YWJsZSBieSBjYWxsaW5nCiMgdGhlIG1vY2s7IHRoZSBhZ2VudCBjYW5ub3QgaGFyZGNvZGUgaXQuCkVWRU5UUyA9IFsKICAgIHsKICAgICAgICAiaWQiOiBmImV2dF90ZXN0X21ldGVyX3tpfSIsCiAgICAgICAgIm9iamVjdCI6ICJ2Mi5jb3JlLmV2ZW50IiwKICAgICAgICAidHlwZSI6ICJ2MS5iaWxsaW5nLm1ldGVyLmVycm9yX3JlcG9ydF90cmlnZ2VyZWQiLAogICAgICAgICJjcmVhdGVkIjogZiIyMDI2LTA2LTJ7aX1UMTc6NDY6MjIuMTM0WiIsCiAgICAgICAgImxpdmVtb2RlIjogRmFsc2UsCiAgICAgICAgInJlbGF0ZWRfb2JqZWN0IjogeyJpZCI6IE9CSkVDVF9JRCwgInR5cGUiOiAiYmlsbGluZy5tZXRlciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICJ1cmwiOiBmIi92MS9iaWxsaW5nL21ldGVycy97T0JKRUNUX0lEfSJ9LAogICAgfQogICAgZm9yIGkgaW4gcmFuZ2UoMSwgNCkKXQoKCmNsYXNzIF9Nb2NrU3RyaXBlVjIoQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICB2ZXJzaW9uID0gc2VsZi5oZWFkZXJzLmdldCgiU3RyaXBlLVZlcnNpb24iKQogICAgICAgIHR5cGUoc2VsZikucmVxdWVzdF9sb2cuYXBwZW5kKHsKICAgICAgICAgICAgInBhdGgiOiBwYXJzZWQucGF0aCwKICAgICAgICAgICAgInN0cmlwZV92ZXJzaW9uIjogdmVyc2lvbiwKICAgICAgICAgICAgImhhc19iZWFyZXIiOiAoc2VsZi5oZWFkZXJzLmdldCgiQXV0aG9yaXphdGlvbiIpIG9yICIiKS5zdGFydHN3aXRoKCJCZWFyZXIgIiksCiAgICAgICAgICAgICJzdGFydGluZ19hZnRlciI6IHFzLmdldCgic3RhcnRpbmdfYWZ0ZXIiLCBbTm9uZV0pWzBdLAogICAgICAgIH0pCgogICAgICAgICMgKDEpIGVuZHBvaW50LXZlcnNpb24gYXhpczogb25seSB0aGUgdjIgZXZlbnRzIHBhdGggZXhpc3RzLgogICAgICAgIGlmIHBhdGggIT0gIi92Mi9jb3JlL2V2ZW50cyI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiVW5yZWNvZ25pemVkIHJlcXVlc3QgVVJMLiBTdHJpcGUgQVBJIHYyIGxpc3RzIGV2ZW50cyBhdCAvdjIvY29yZS9ldmVudHMuIn19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMikgdmVyc2lvbiBheGlzOiB2MiByZXF1aXJlcyBhIHdlbGwtZm9ybWVkIFN0cmlwZS1WZXJzaW9uIGhlYWRlci4KICAgICAgICBpZiBub3QgdmVyc2lvbiBvciBub3QgX1ZFUlNJT05fUkUubWF0Y2godmVyc2lvbik6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAwLCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiTWlzc2luZyBvciBtYWxmb3JtZWQgU3RyaXBlLVZlcnNpb24gaGVhZGVyLiBUaGUgdjIgQVBJIHJlcXVpcmVzIFN0cmlwZS1WZXJzaW9uOiA8WVlZWS1NTS1ERC5jb2RlbmFtZT4uIn19KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyAoMykgYXV0aCBheGlzOiBiZWFyZXIgc2VjcmV0IGtleSAoc2hhcmVkIHdpdGggdjEpLgogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIiwgIiIpCiAgICAgICAgaWYgYXV0aCAhPSBmIkJlYXJlciB7RVhQRUNURURfS0VZfSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImVycm9yIjogeyJ0eXBlIjogImludmFsaWRfcmVxdWVzdF9lcnJvciIsCiAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiSW52YWxpZCBBUEkga2V5LiBBdXRoZW50aWNhdGUgd2l0aCBBdXRob3JpemF0aW9uOiBCZWFyZXIgPHNlY3JldCBrZXk+LiJ9fSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIHNlbGYuX2pzb24oMjAwLCB7CiAgICAgICAgICAgICJkYXRhIjogRVZFTlRTLAogICAgICAgICAgICAibmV4dF9wYWdlX3VybCI6IE5vbmUsCiAgICAgICAgICAgICJwcmV2aW91c19wYWdlX3VybCI6IE5vbmUsCiAgICAgICAgfSkKCgpAcHl0ZXN0LmZpeHR1cmUoKQpkZWYgbW9ja19zZXJ2ZXIoKToKICAgIF9Nb2NrU3RyaXBlVjIucmVxdWVzdF9sb2cgPSBbXQogICAgc2VydmVyID0gVGhyZWFkaW5nSFRUUFNlcnZlcigoIjEyNy4wLjAuMSIsIDApLCBfTW9ja1N0cmlwZVYyKQogICAgcG9ydCA9IHNlcnZlci5zZXJ2ZXJfYWRkcmVzc1sxXQogICAgdGhyZWFkaW5nLlRocmVhZCh0YXJnZXQ9c2VydmVyLnNlcnZlX2ZvcmV2ZXIsIGRhZW1vbj1UcnVlKS5zdGFydCgpCiAgICB0cnk6CiAgICAgICAgeWllbGQgZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fSIKICAgIGZpbmFsbHk6CiAgICAgICAgc2VydmVyLnNodXRkb3duKCkKICAgICAgICBzZXJ2ZXIuc2VydmVyX2Nsb3NlKCkKCgpkZWYgX2xvYWRfYWdlbnRfc29sdXRpb24oKToKICAgIHJvb3QgPSBvcy5lbnZpcm9uLmdldCgiVkJfU09MVVRJT05fUk9PVCIsIG9zLmdldGN3ZCgpKQogICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKHJvb3QsICJzb2x1dGlvbiIsICJzb2x1dGlvbi5weSIpCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBmb3IgZGlycGF0aCwgX2RpcnMsIGZpbGVzIGluIG9zLndhbGsocm9vdCk6CiAgICAgICAgICAgIGlmICJzb2x1dGlvbi5weSIgaW4gZmlsZXM6CiAgICAgICAgICAgICAgICBjYW5kaWRhdGUgPSBvcy5wYXRoLmpvaW4oZGlycGF0aCwgInNvbHV0aW9uLnB5IikKICAgICAgICAgICAgICAgIGJyZWFrCiAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoY2FuZGlkYXRlKToKICAgICAgICBweXRlc3QuZmFpbChmImFnZW50IHNvbHV0aW9uIG5vdCBmb3VuZDogZXhwZWN0ZWQgc29sdXRpb24vc29sdXRpb24ucHkgdW5kZXIge3Jvb3R9IikKICAgIHNwZWMgPSBpbXBvcnRsaWIudXRpbC5zcGVjX2Zyb21fZmlsZV9sb2NhdGlvbigiYWdlbnRfc29sdXRpb24iLCBjYW5kaWRhdGUpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIudXRpbC5tb2R1bGVfZnJvbV9zcGVjKHNwZWMpCiAgICBzcGVjLmxvYWRlci5leGVjX21vZHVsZShtb2R1bGUpCiAgICByZXR1cm4gbW9kdWxlCgoKZGVmIHRlc3RfZ2V0X3JlY2VudF9ldmVudHMobW9ja19zZXJ2ZXIpOgogICAgbW9kdWxlID0gX2xvYWRfYWdlbnRfc29sdXRpb24oKQogICAgYXNzZXJ0IGhhc2F0dHIobW9kdWxlLCAiZ2V0X3JlY2VudF9ldmVudHMiKSwgXAogICAgICAgICJzb2x1dGlvbi5weSBtdXN0IGV4cG9ydCBnZXRfcmVjZW50X2V2ZW50cyhiYXNlX3VybCwgYXBpX2tleSwgb2JqZWN0X2lkKSIKCiAgICBldmVudHMgPSBtb2R1bGUuZ2V0X3JlY2VudF9ldmVudHMobW9ja19zZXJ2ZXIsIEVYUEVDVEVEX0tFWSwgT0JKRUNUX0lEKQoKICAgIGFzc2VydCBpc2luc3RhbmNlKGV2ZW50cywgbGlzdCksIGYiZXhwZWN0ZWQgYSBsaXN0IG9mIGV2ZW50cywgZ290IHt0eXBlKGV2ZW50cykuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGVbImlkIl0gZm9yIGUgaW4gZXZlbnRzIGlmIGlzaW5zdGFuY2UoZSwgZGljdCkgYW5kICJpZCIgaW4gZSkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChlWyJpZCJdIGZvciBlIGluIEVWRU5UUykKICAgIGFzc2VydCBpZHMgPT0gZXhwZWN0ZWRfaWRzLCAoCiAgICAgICAgZiJjbGllbnQgcmV0dXJuZWQgZXZlbnQgaWRzIHtpZHN9LCBleHBlY3RlZCB7ZXhwZWN0ZWRfaWRzfS4gQSBjb3JyZWN0IHYyIGNsaWVudCBHRVRzICIKICAgICAgICBmIi92Mi9jb3JlL2V2ZW50cyB3aXRoIGEgU3RyaXBlLVZlcnNpb24gaGVhZGVyIGFuZCByZWFkcyB0aGUgYGRhdGFgIGFycmF5LiAiCiAgICAgICAgZiJTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tTdHJpcGVWMi5yZXF1ZXN0X2xvZ30iCiAgICApCiAgICBhc3NlcnQgYWxsKGUuZ2V0KCJvYmplY3QiKSA9PSAidjIuY29yZS5ldmVudCIgZm9yIGUgaW4gZXZlbnRzKSwgKAogICAgICAgIGYiZXhwZWN0ZWQgdjIuY29yZS5ldmVudCBvYmplY3RzLiBTZXJ2ZXIgc2F3IHJlcXVlc3RzOiB7X01vY2tTdHJpcGVWMi5yZXF1ZXN0X2xvZ30iCiAgICApCg==" - }, - { - "vendor": "finch:list_all_individuals", - "fn": "list_all_individuals", - "testFile": "test_finch_list_all.py", - "signature": "def list_all_individuals(base_url: str, access_token: str) -> list[dict]\\n\\n`", - "graderB64": "IiIiCkhJRERFTiBleGVjdXRpb24gb3JhY2xlIGZvciB0aGUgZmluY2gtZGlyZWN0b3J5LXBhZ2luYXRlIChoYXJkKSBsZWFmLgoKVGhlIGFnZW50IG5ldmVyIHNlZXMgdGhpcyBmaWxlLiBJdCBpcyB3cml0dGVuIGludG8gYSBwcml2YXRlIGdyYWRlciBkaXIgYXQgZ3JhZGUgdGltZQooYmFzZTY0LWRlY29kZWQgaW5zaWRlIHZhbGlkYXRvci5jdXN0b21CdWlsZCksIHRoZW4gcHl0ZXN0IHJ1bnMgaXQgYWdhaW5zdCB0aGUgYWdlbnQncwpzb2x1dGlvbi9zb2x1dGlvbi5weS4KCkl0IHN0YW5kcyB1cCBhIExPQ0FMIG1vY2sgdGhhdCBmYWl0aGZ1bGx5IGltcGxlbWVudHMgRmluY2gncyBDVVJSRU5UIGRpcmVjdG9yeSBjb250cmFjdCwKYWRkaW5nIHRoZSBPRkZTRVQtcGFnaW5hdGlvbiBheGlzIG9uIHRvcCBvZiB0aGUgbWVkaXVtIGxlYWY6CiAgLSBlbmRwb2ludCBwYXRoICAgICAgICAvZW1wbG95ZXIvZGlyZWN0b3J5CiAgLSBhdXRoICAgICAgICAgICAgICAgICBBdXRob3JpemF0aW9uOiBCZWFyZXIgPGFjY2Vzc190b2tlbj4KICAtIHZlcnNpb24gaGVhZGVyICAgICAgIEZpbmNoLUFQSS1WZXJzaW9uOiA8WVlZWS1NTS1ERD4gICAoUkVRVUlSRUQ7IG1pc3NpbmcgLT4gNDAwKQogIC0gcGFnaW5hdGlvbiAgICAgICAgICAgb2Zmc2V0LWJhc2VkOiBgbGltaXRgICsgYG9mZnNldGAgcXVlcnkgcGFyYW1zOyB0aGUgcmVzcG9uc2UgY2FycmllcwogICAgICAgICAgICAgICAgICAgICAgICAgcGFnaW5nLmNvdW50ICh0b3RhbCkgYW5kIHBhZ2luZy5vZmZzZXQsIGFuZCB0aGVyZSBpcyBOTyBuZXh0L2hhc19tb3JlCiAgICAgICAgICAgICAgICAgICAgICAgICBjdXJzb3IgZmllbGQgLS0gdGhlIGNsaWVudCBtdXN0IGtlZXAgcGFnaW5nIHdoaWxlIG9mZnNldCtsZW4gPCBjb3VudAogIC0gcmVzcG9uc2UgZW52ZWxvcGUgICAgeyJwYWdpbmciOiB7ImNvdW50IiwgIm9mZnNldCJ9LCAiaW5kaXZpZHVhbHMiOiBbLi4uXX0KClRoZSBtb2NrIHJldHVybnMgYXQgbW9zdCBQQUdFX1NJWkUgaW5kaXZpZHVhbHMgcGVyIHJlc3BvbnNlIChzZXJ2ZXIgcGFnZSBzaXplKSwgc28gY29sbGVjdGluZwp0aGUgd2hvbGUgZGlyZWN0b3J5IFJFUVVJUkVTIG9mZnNldCBwYWdpbmF0aW9uIGRyaXZlbiBieSBwYWdpbmcuY291bnQuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbQpDVVJSRU5UIEZpbmNoIGRvY3MgY29sbGVjdHMgYWxsIDcgaW5kaXZpZHVhbHMgYWNyb3NzIDMgcGFnZXMuIEEgY2xpZW50IHdyaXR0ZW4gZnJvbSBzdGFsZS9wcmUtCmN1dG9mZiBtZW1vcnkgb21pdHMgdGhlIHZlcnNpb24gaGVhZGVyICg0MDApLCBndWVzc2VzIGEgL2VtcGxveWVlcyBwYXRoICg0MDQpLCByZWFkcyBhIGBkYXRhYAplbnZlbG9wZSwgb3IgYXNzdW1lcyBhIGBuZXh0YC9gaGFzX21vcmVgIGN1cnNvciBhbmQgc3RvcHMgYWZ0ZXIgcGFnZSAxICgzIG9mIDcpIGFuZCBGQUlMUy4KQW4gZW1wdHkgc29sdXRpb24gRkFJTFMuCgpDb250cmFjdCBzb3VyY2VzIChyZWFsIGRvY3MpOgogIGh0dHBzOi8vZGV2ZWxvcGVyLnRyeWZpbmNoLmNvbS9hcGktcmVmZXJlbmNlL2RldmVsb3BtZW50LWd1aWRlcy9IZWFkZXJzCiAgaHR0cHM6Ly9kZXZlbG9wZXIudHJ5ZmluY2guY29tL2FwaS1yZWZlcmVuY2Uvb3JnYW5pemF0aW9uL2RpcmVjdG9yeQoiIiIKCmltcG9ydCBpbXBvcnRsaWIudXRpbAppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB0aHJlYWRpbmcKZnJvbSBodHRwLnNlcnZlciBpbXBvcnQgQmFzZUhUVFBSZXF1ZXN0SGFuZGxlciwgVGhyZWFkaW5nSFRUUFNlcnZlcgpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UsIHBhcnNlX3FzCgppbXBvcnQgcHl0ZXN0CgpFWFBFQ1RFRF9UT0tFTiA9ICJmaW5jaF9hY2Nlc3NfdG9rZW5fYWJjMTIzIgoKIyBIaWRkZW4gZGF0YXNldDogNyBpbmRpdmlkdWFscywgb25seSBvYnNlcnZhYmxlIGJ5IGFjdHVhbGx5IGNhbGxpbmcgdGhlIG1vY2sgYW5kIHBhZ2luYXRpbmcuCklORElWSURVQUxTID0gWwogICAgewogICAgICAgICJpZCI6IGYiMjIyMjIyMjItMDAwMC00MDAwLTgwMDAtMDAwMDAwMDAwMDB7aX0iLAogICAgICAgICJmaXJzdF9uYW1lIjogZiJGaXJzdHtpfSIsCiAgICAgICAgIm1pZGRsZV9uYW1lIjogTm9uZSwKICAgICAgICAibGFzdF9uYW1lIjogZiJMYXN0e2l9IiwKICAgICAgICAiZGVwYXJ0bWVudCI6IHsibmFtZSI6ICJFbmdpbmVlcmluZyJ9LAogICAgICAgICJtYW5hZ2VyIjogTm9uZSwKICAgICAgICAiaXNfYWN0aXZlIjogVHJ1ZSwKICAgIH0KICAgIGZvciBpIGluIHJhbmdlKDEsIDgpCl0KUEFHRV9TSVpFID0gMyAgIyBzZXJ2ZXIgcGFnZSBzaXplOiBhdCBtb3N0IHRoaXMgbWFueSBpbmRpdmlkdWFscyBwZXIgcmVzcG9uc2UsIGZvcmNpbmcgcGFnaW5hdGlvbgoKCmNsYXNzIF9Nb2NrRmluY2goQmFzZUhUVFBSZXF1ZXN0SGFuZGxlcik6CiAgICByZXF1ZXN0X2xvZyA9IFtdCgogICAgZGVmIGxvZ19tZXNzYWdlKHNlbGYsICpfYXJncyk6CiAgICAgICAgcmV0dXJuCgogICAgZGVmIF9qc29uKHNlbGYsIGNvZGUsIGJvZHkpOgogICAgICAgIHBheWxvYWQgPSBqc29uLmR1bXBzKGJvZHkpLmVuY29kZSgpCiAgICAgICAgc2VsZi5zZW5kX3Jlc3BvbnNlKGNvZGUpCiAgICAgICAgc2VsZi5zZW5kX2hlYWRlcigiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKQogICAgICAgIHNlbGYuc2VuZF9oZWFkZXIoIkNvbnRlbnQtTGVuZ3RoIiwgc3RyKGxlbihwYXlsb2FkKSkpCiAgICAgICAgc2VsZi5lbmRfaGVhZGVycygpCiAgICAgICAgc2VsZi53ZmlsZS53cml0ZShwYXlsb2FkKQoKICAgIGRlZiBkb19HRVQoc2VsZik6CiAgICAgICAgcGFyc2VkID0gdXJscGFyc2Uoc2VsZi5wYXRoKQogICAgICAgIHBhdGggPSBwYXJzZWQucGF0aC5yc3RyaXAoIi8iKSBvciAiLyIKICAgICAgICBxcyA9IHBhcnNlX3FzKHBhcnNlZC5xdWVyeSkKICAgICAgICBoZWFkZXJfa2V5cyA9IHtrLmxvd2VyKCkgZm9yIGsgaW4gc2VsZi5oZWFkZXJzLmtleXMoKX0KICAgICAgICB2ZXJzaW9uID0gc2VsZi5oZWFkZXJzLmdldCgiRmluY2gtQVBJLVZlcnNpb24iKQogICAgICAgIGF1dGggPSBzZWxmLmhlYWRlcnMuZ2V0KCJBdXRob3JpemF0aW9uIikKICAgICAgICB0eXBlKHNlbGYpLnJlcXVlc3RfbG9nLmFwcGVuZCh7CiAgICAgICAgICAgICJwYXRoIjogcGFyc2VkLnBhdGgsCiAgICAgICAgICAgICJoYXNfdmVyc2lvbl9oZWFkZXIiOiAiZmluY2gtYXBpLXZlcnNpb24iIGluIGhlYWRlcl9rZXlzLAogICAgICAgICAgICAidmVyc2lvbiI6IHZlcnNpb24sCiAgICAgICAgICAgICJhdXRob3JpemF0aW9uIjogYXV0aCwKICAgICAgICAgICAgImxpbWl0IjogcXMuZ2V0KCJsaW1pdCIsIFtOb25lXSlbMF0sCiAgICAgICAgICAgICJvZmZzZXQiOiBxcy5nZXQoIm9mZnNldCIsIFtOb25lXSlbMF0sCiAgICAgICAgfSkKCiAgICAgICAgaWYgcGF0aCAhPSAiL2VtcGxveWVyL2RpcmVjdG9yeSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDA0LCB7ImNvZGUiOiA0MDQsICJuYW1lIjogIm5vdF9mb3VuZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiRmluY2ggbGlzdHMgdGhlIG9yZyBkaXJlY3RvcnkgYXQgR0VUIC9lbXBsb3llci9kaXJlY3RvcnkifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIGF1dGggIT0gZiJCZWFyZXIge0VYUEVDVEVEX1RPS0VOfSI6CiAgICAgICAgICAgIHNlbGYuX2pzb24oNDAxLCB7ImNvZGUiOiA0MDEsICJuYW1lIjogImludmFsaWRfYWNjZXNzX3Rva2VuIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibWVzc2FnZSI6ICJBdXRob3JpemF0aW9uIG11c3QgYmUgJ0JlYXJlciA8YWNjZXNzX3Rva2VuPicifSkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIHZlcnNpb24gaXMgTm9uZSBvciBub3QgcmUuZnVsbG1hdGNoKHIiXGR7NH0tXGR7Mn0tXGR7Mn0iLCB2ZXJzaW9uKToKICAgICAgICAgICAgc2VsZi5fanNvbig0MDAsIHsiY29kZSI6IDQwMCwgIm5hbWUiOiAibWlzc2luZ192ZXJzaW9uX2hlYWRlciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm1lc3NhZ2UiOiAiRmluY2ggcmVxdWlyZXMgdGhlIEZpbmNoLUFQSS1WZXJzaW9uIGhlYWRlciAoZS5nLiAyMDIwLTA5LTE3KSJ9KQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyBvZmZzZXQgcGFnaW5hdGlvbjogaG9ub3IgYG9mZnNldGAgKGRlZmF1bHQgMCk7IHNlcnZlciBjYXBzIHRoZSBwYWdlIGF0IFBBR0VfU0laRS4KICAgICAgICB0cnk6CiAgICAgICAgICAgIG9mZnNldCA9IGludChxcy5nZXQoIm9mZnNldCIsIFsiMCJdKVswXSkKICAgICAgICBleGNlcHQgKFR5cGVFcnJvciwgVmFsdWVFcnJvcik6CiAgICAgICAgICAgIG9mZnNldCA9IDAKICAgICAgICBpZiBvZmZzZXQgPCAwOgogICAgICAgICAgICBvZmZzZXQgPSAwCiAgICAgICAgdHJ5OgogICAgICAgICAgICByZXF1ZXN0ZWRfbGltaXQgPSBpbnQocXMuZ2V0KCJsaW1pdCIsIFtzdHIoUEFHRV9TSVpFKV0pWzBdKQogICAgICAgIGV4Y2VwdCAoVHlwZUVycm9yLCBWYWx1ZUVycm9yKToKICAgICAgICAgICAgcmVxdWVzdGVkX2xpbWl0ID0gUEFHRV9TSVpFCiAgICAgICAgcGFnZV9sZW4gPSBtYXgoMCwgbWluKFBBR0VfU0laRSwgcmVxdWVzdGVkX2xpbWl0KSkKCiAgICAgICAgcGFnZSA9IElORElWSURVQUxTW29mZnNldDpvZmZzZXQgKyBwYWdlX2xlbl0KICAgICAgICBzZWxmLl9qc29uKDIwMCwgewogICAgICAgICAgICAicGFnaW5nIjogeyJjb3VudCI6IGxlbihJTkRJVklEVUFMUyksICJvZmZzZXQiOiBvZmZzZXR9LAogICAgICAgICAgICAiaW5kaXZpZHVhbHMiOiBwYWdlLAogICAgICAgIH0pCgoKQHB5dGVzdC5maXh0dXJlKCkKZGVmIG1vY2tfc2VydmVyKCk6CiAgICBfTW9ja0ZpbmNoLnJlcXVlc3RfbG9nID0gW10KICAgIHNlcnZlciA9IFRocmVhZGluZ0hUVFBTZXJ2ZXIoKCIxMjcuMC4wLjEiLCAwKSwgX01vY2tGaW5jaCkKICAgIHBvcnQgPSBzZXJ2ZXIuc2VydmVyX2FkZHJlc3NbMV0KICAgIHRocmVhZGluZy5UaHJlYWQodGFyZ2V0PXNlcnZlci5zZXJ2ZV9mb3JldmVyLCBkYWVtb249VHJ1ZSkuc3RhcnQoKQogICAgdHJ5OgogICAgICAgIHlpZWxkIGYiaHR0cDovLzEyNy4wLjAuMTp7cG9ydH0iCiAgICBmaW5hbGx5OgogICAgICAgIHNlcnZlci5zaHV0ZG93bigpCiAgICAgICAgc2VydmVyLnNlcnZlcl9jbG9zZSgpCgoKZGVmIF9sb2FkX2FnZW50X3NvbHV0aW9uKCk6CiAgICByb290ID0gb3MuZW52aXJvbi5nZXQoIlZCX1NPTFVUSU9OX1JPT1QiLCBvcy5nZXRjd2QoKSkKICAgIGNhbmRpZGF0ZSA9IG9zLnBhdGguam9pbihyb290LCAic29sdXRpb24iLCAic29sdXRpb24ucHkiKQogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgZm9yIGRpcnBhdGgsIF9kaXJzLCBmaWxlcyBpbiBvcy53YWxrKHJvb3QpOgogICAgICAgICAgICBpZiAic29sdXRpb24ucHkiIGluIGZpbGVzOgogICAgICAgICAgICAgICAgY2FuZGlkYXRlID0gb3MucGF0aC5qb2luKGRpcnBhdGgsICJzb2x1dGlvbi5weSIpCiAgICAgICAgICAgICAgICBicmVhawogICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGNhbmRpZGF0ZSk6CiAgICAgICAgcHl0ZXN0LmZhaWwoZiJhZ2VudCBzb2x1dGlvbiBub3QgZm91bmQ6IGV4cGVjdGVkIHNvbHV0aW9uL3NvbHV0aW9uLnB5IHVuZGVyIHtyb290fSIpCiAgICBzcGVjID0gaW1wb3J0bGliLnV0aWwuc3BlY19mcm9tX2ZpbGVfbG9jYXRpb24oImFnZW50X3NvbHV0aW9uIiwgY2FuZGlkYXRlKQogICAgbW9kdWxlID0gaW1wb3J0bGliLnV0aWwubW9kdWxlX2Zyb21fc3BlYyhzcGVjKQogICAgc3BlYy5sb2FkZXIuZXhlY19tb2R1bGUobW9kdWxlKQogICAgcmV0dXJuIG1vZHVsZQoKCmRlZiB0ZXN0X2xpc3RzX2FsbF9pbmRpdmlkdWFsc19hY3Jvc3Nfb2Zmc2V0X3BhZ2VzKG1vY2tfc2VydmVyKToKICAgIG1vZHVsZSA9IF9sb2FkX2FnZW50X3NvbHV0aW9uKCkKICAgIGFzc2VydCBoYXNhdHRyKG1vZHVsZSwgImxpc3RfYWxsX2luZGl2aWR1YWxzIiksIFwKICAgICAgICAic29sdXRpb24ucHkgbXVzdCBleHBvcnQgbGlzdF9hbGxfaW5kaXZpZHVhbHMoYmFzZV91cmwsIGFjY2Vzc190b2tlbikiCgogICAgcmVzdWx0ID0gbW9kdWxlLmxpc3RfYWxsX2luZGl2aWR1YWxzKG1vY2tfc2VydmVyLCBFWFBFQ1RFRF9UT0tFTikKCiAgICBhc3NlcnQgaXNpbnN0YW5jZShyZXN1bHQsIGxpc3QpLCBmImV4cGVjdGVkIGEgbGlzdCBvZiBpbmRpdmlkdWFscywgZ290IHt0eXBlKHJlc3VsdCkuX19uYW1lX199IgogICAgaWRzID0gc29ydGVkKGRbImlkIl0gZm9yIGQgaW4gcmVzdWx0IGlmIGlzaW5zdGFuY2UoZCwgZGljdCkgYW5kICJpZCIgaW4gZCkKICAgIGV4cGVjdGVkX2lkcyA9IHNvcnRlZChkWyJpZCJdIGZvciBkIGluIElORElWSURVQUxTKQogICAgYXNzZXJ0IGlkcyA9PSBleHBlY3RlZF9pZHMsICgKICAgICAgICBmImNsaWVudCByZXR1cm5lZCB7bGVuKGlkcyl9IGluZGl2aWR1YWwgaWRzIHtpZHN9LCBleHBlY3RlZCB7bGVuKGV4cGVjdGVkX2lkcyl9IHtleHBlY3RlZF9pZHN9LiAiCiAgICAgICAgZiJBIGNvcnJlY3QgY2xpZW50IHBhZ2luYXRlcyBieSBgb2Zmc2V0YCB3aGlsZSBwYWdpbmcub2Zmc2V0K2xlbihpbmRpdmlkdWFscykgPCBwYWdpbmcuY291bnQgIgogICAgICAgIGYiKHRoZXJlIGlzIG5vIG5leHQvaGFzX21vcmUgY3Vyc29yKSwgc2VuZGluZyB0aGUgQmVhcmVyIHRva2VuIEFORCB0aGUgcmVxdWlyZWQgIgogICAgICAgIGYiRmluY2gtQVBJLVZlcnNpb24gaGVhZGVyIGVhY2ggcmVxdWVzdC4gU2VydmVyIHNhdyByZXF1ZXN0czoge19Nb2NrRmluY2gucmVxdWVzdF9sb2d9IgogICAgKQo=" - } -] \ No newline at end of file diff --git a/examples/ablation-suite/multi-contract-env.ts b/examples/ablation-suite/multi-contract-env.ts deleted file mode 100644 index a13a97a4..00000000 --- a/examples/ablation-suite/multi-contract-env.ts +++ /dev/null @@ -1,276 +0,0 @@ -/** - * multi-contract-env — a LONG-HORIZON coding task: build ONE Python client module that correctly - * integrates N external services, each with its CURRENT, non-default contract (auth scheme, - * endpoint version, pagination), graded end-to-end. This is where web-search value COMPOUNDS: - * a small per-contract edge (search lifts p_no→p_yes on getting one contract right) becomes p^N - * end-to-end, detectable at low n. Compounding AMPLIFIES a real per-step edge; it can't invent one. - * - * ABLATION DELTA: the `search` config toggles a `web_search` tool (you.com via the Tangle router). - * Native read/write/run_tests/bash stay ON in BOTH arms — additive isolation (both can work; only - * one can also search). run_tests gives the worker per-vendor PASS/FAIL (realistic integration - * feedback: "novu FAIL" → go discover novu's current contract) WITHOUT revealing the hidden - * assertions — so it's a genuine multi-round task, not a leak. - * - * PRE-REGISTERED (encoded, not narrated past): calibrate no-search FIRST — end-to-end resolve MUST - * be <40% or the task is too easy (harden it). Success = search−nosearch resolve delta, paired - * bootstrap 95% CI excludes 0, n≥20, GLM-5.2. Kill = worker never calls web_search (arm invalid). - * - * Reuses the SAME self-contained mock+pytest graders VB web-grounded already ships (each boots its - * own mock, injects base_url, tests one exported function of solution.py). See extract-contracts.mjs. - */ - -import { execFileSync } from 'node:child_process' -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' -import { fileURLToPath } from 'node:url' -import type { - AgenticSurface, - AgenticTask, - AgenticTool, - ArtifactHandle, - SurfaceScore, -} from '../../src/runtime/strategy.js' - -const here = dirname(fileURLToPath(import.meta.url)) -interface Contract { - vendor: string - fn: string - testFile: string - signature: string - graderB64: string -} -const CONTRACTS: Contract[] = JSON.parse( - readFileSync(join(here, 'fixtures', 'multi-contract', 'contracts.json'), 'utf8'), -) - -const SOLUTION_REL = join('solution', 'solution.py') -// Global web_search firing counter (the strategy opens its OWN handle, so per-handle counting -// misses it). Reset per arm via resetSearchCalls(); read after each task for the firing gate. -let searchCallTotal = 0 -export function resetSearchCalls(): void { - searchCallTotal = 0 -} -export function searchCalls(): number { - return searchCallTotal -} -const dirsByHandle = new Map() - -/** Run one vendor's self-contained grader (boots its own mock) against the worker's solution.py. */ -function runGrader(dir: string, c: Contract): boolean { - const graderDir = join(dir, '.vb_grader') - mkdirSync(graderDir, { recursive: true }) - writeFileSync(join(graderDir, c.testFile), Buffer.from(c.graderB64, 'base64')) - try { - const out = execFileSync( - 'python3', - [ - '-m', - 'pytest', - '-q', - '--tb=no', - '--color=no', - '-p', - 'no:cacheprovider', - join('.vb_grader', c.testFile), - ], - { - cwd: dir, - encoding: 'utf8', - timeout: 180_000, - env: { ...process.env, VB_SOLUTION_ROOT: dir }, - stdio: ['ignore', 'pipe', 'pipe'], - }, - ) - return /(\d+) passed/.test(out) && !/(\d+) failed/.test(out) && !/(\d+) error/.test(out) - } catch (e) { - const out = (e as { stdout?: string }).stdout ?? '' - return /(\d+) passed/.test(out) && !/failed|error/.test(out) - } -} - -async function youSearch(query: string): Promise { - const key = process.env.TANGLE_API_KEY - if (!key) return 'web_search error: TANGLE_API_KEY unset' - try { - const res = await fetch('https://router.tangle.tools/v1/search', { - method: 'POST', - headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' }, - body: JSON.stringify({ query, provider: 'you' }), - }) - const j = (await res.json()) as { - data?: Array<{ title?: string; url?: string; snippet?: string }> - } - const rows = (j.data ?? []) - .slice(0, 6) - .map((r, i) => `${i + 1}. ${r.title ?? ''}\n ${r.url ?? ''}\n ${r.snippet ?? ''}`) - return rows.length ? rows.join('\n') : `web_search: no results for ${JSON.stringify(query)}` - } catch (err) { - return `web_search error: ${err instanceof Error ? err.message : String(err)}` - } -} - -export function makeMultiContractSurface(opts: { search: boolean }): AgenticSurface { - const name = `multi-contract-${CONTRACTS.length}${opts.search ? '+search' : ''}` - return { - name, - async open(task: AgenticTask): Promise { - const dir = mkdtempSync(join(tmpdir(), 'multi-contract-')) - mkdirSync(join(dir, 'solution'), { recursive: true }) - writeFileSync( - join(dir, SOLUTION_REL), - `# Implement the ${CONTRACTS.length} client functions below.\n`, - ) - const id = `${name}:${task.id}:${dirsByHandle.size}` - dirsByHandle.set(id, dir) - return { id, surface: name, ctx: { dir } } - }, - async tools(): Promise { - const base: AgenticTool[] = [ - { - type: 'function', - function: { - name: 'write_file', - description: 'Write a file (path relative to project root, e.g. solution/solution.py).', - parameters: { - type: 'object', - properties: { path: { type: 'string' }, content: { type: 'string' } }, - required: ['path', 'content'], - }, - }, - }, - { - type: 'function', - function: { - name: 'read_file', - description: 'Read a file in the project.', - parameters: { - type: 'object', - properties: { path: { type: 'string' } }, - required: ['path'], - }, - }, - }, - { - type: 'function', - function: { - name: 'bash', - description: 'Run a shell command in the project root (e.g. python -c, curl).', - parameters: { - type: 'object', - properties: { cmd: { type: 'string' } }, - required: ['cmd'], - }, - }, - }, - { - type: 'function', - function: { - name: 'run_tests', - description: - 'Run the hidden integration graders and get per-vendor PASS/FAIL for your solution/solution.py.', - parameters: { type: 'object', properties: {}, required: [] }, - }, - }, - ] - if (opts.search) - base.push({ - type: 'function', - function: { - name: 'web_search', - description: - 'Search the live web (you.com) for the CURRENT contract of an API. Use it to discover current auth schemes, endpoint versions, and pagination that you cannot recall.', - parameters: { - type: 'object', - properties: { query: { type: 'string' } }, - required: ['query'], - }, - }, - }) - return base - }, - async call( - handle: ArtifactHandle, - toolName: string, - args: Record, - ): Promise { - const dir = (handle.ctx as { dir: string }).dir - if (toolName === 'web_search') { - searchCallTotal += 1 - return youSearch(String(args.query ?? '')) - } - if (toolName === 'write_file') { - const p = join(dir, String(args.path)) - mkdirSync(dirname(p), { recursive: true }) - writeFileSync(p, String(args.content ?? '')) - return `wrote ${args.path}` - } - if (toolName === 'read_file') { - const p = join(dir, String(args.path)) - return existsSync(p) - ? readFileSync(p, 'utf8').slice(0, 20_000) - : `no such file: ${args.path}` - } - if (toolName === 'bash') { - try { - return execFileSync('bash', ['-lc', String(args.cmd)], { - cwd: dir, - encoding: 'utf8', - timeout: 60_000, - stdio: ['ignore', 'pipe', 'pipe'], - }).slice(0, 8_000) - } catch (e) { - return ( - ((e as { stdout?: string; stderr?: string }).stdout ?? '') + - ((e as { stderr?: string }).stderr ?? '') - ) - } - } - if (toolName === 'run_tests') { - const lines = CONTRACTS.map((c) => `${c.vendor}: ${runGrader(dir, c) ? 'PASS' : 'FAIL'}`) - const passed = lines.filter((l) => l.endsWith('PASS')).length - return `${passed}/${CONTRACTS.length} vendor contracts pass.\n${lines.join('\n')}\n(A FAIL means that vendor's current contract — auth/endpoint/version — is wrong. Discover the current contract and fix it.)` - } - return `unknown tool: ${toolName}` - }, - async score(_task: AgenticTask, handle: ArtifactHandle): Promise { - const dir = (handle.ctx as { dir: string }).dir - let passes = 0 - for (const c of CONTRACTS) if (runGrader(dir, c)) passes++ - return { passes, total: CONTRACTS.length, errored: 0 } - }, - async close(handle: ArtifactHandle): Promise { - const dir = (handle.ctx as { dir: string }).dir - try { - rmSync(dir, { recursive: true, force: true }) - } catch { - /* best effort */ - } - dirsByHandle.delete(handle.id) - }, - } -} - -/** The task: build one solution.py with all N current-contract clients. */ -export function multiContractTasks(offset: number, n: number): Promise { - const fnList = CONTRACTS.map((c) => ` - ${c.vendor}: \`${c.signature}\``).join('\n') - const userPrompt = [ - `Build a single Python module \`solution/solution.py\` that implements a client for ${CONTRACTS.length} different SaaS APIs. Export EXACTLY these functions:`, - fnList, - ``, - `Each function receives \`base_url\` (the API server root, injected at call time — append the CURRENT path to it, never hardcode the host) and \`api_key\`. Make REAL HTTP requests (use \`requests\` or \`urllib\`); no mocks, no hardcoded data.`, - ``, - `Each vendor's CURRENT contract is non-default and may have changed recently — the exact Authorization scheme (many are NOT \`Bearer\`), the endpoint path/version, and pagination. Do NOT rely on a from-memory integration; verify each vendor's current contract before wiring it.`, - ``, - `Call \`run_tests\` to see which vendors pass. Iterate until all ${CONTRACTS.length} pass. A vendor FAILS if its auth scheme, endpoint version, or response handling is wrong.`, - ].join('\n') - const systemPrompt = - 'You are a senior integrations engineer. Build a correct, real multi-API client. Use your tools; verify current contracts; iterate on run_tests until every vendor passes.' - return Promise.resolve( - Array.from({ length: n }, (_, i) => ({ - id: `multi-contract-${offset + i}`, - systemPrompt, - userPrompt, - })), - ) -} diff --git a/examples/ablation-suite/run-aisdk.ts b/examples/ablation-suite/run-aisdk.ts deleted file mode 100644 index db07c022..00000000 --- a/examples/ablation-suite/run-aisdk.ts +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Search-value A/B on the current-ai-SDK task (aisdk-env). Coder = glm-5.2 via the router, strategy - * = refine (multi-round, sees the tsc errors each round). ARM = the `search` knob (you.com web_search - * on/off; read/write/run_tests on in both — additive isolation). - * - * NO-SEARCH arm is the CONTROL: with compiler feedback but no web, does the coder recover the current - * tool() shape on its own? If no-search resolve is low and +search lifts it (paired CI excludes 0), - * search is the differentiator. Empty (0-token) router runs are retried, then dropped as INVALID. - * - * Env: WORKER_MODEL (glm-5.2), N (tasks/arm), BUDGET (refine rounds), ARMS=cal|full. - */ -import { refine, runAgentic } from '@tangle-network/agent-runtime/loops' -import { aiSdkTasks, makeAiSdkSurface, resetSearchCalls, searchCalls } from './aisdk-env.js' - -const ROUTER = process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1' -const KEY = process.env.TANGLE_API_KEY ?? '' -const MODEL = process.env.WORKER_MODEL ?? 'glm-5.2' -const N = Number(process.env.N ?? (process.env.ARMS === 'full' ? 12 : 6)) -const BUDGET = Number(process.env.BUDGET ?? 6) -const MODE = process.env.ARMS ?? 'cal' -if (!KEY) { - console.error('TANGLE_API_KEY unset') - process.exit(1) -} - -interface ArmOut { - resolve: number - perTask: number[] - valid: boolean[] - fired: number - usd: number -} - -async function runArm( - search: boolean, - tasks: Awaited>, -): Promise { - const surface = makeAiSdkSurface({ search }) - const perTask: number[] = [] - const valid: boolean[] = [] - let fired = 0 - let usd = 0 - for (const t of tasks) { - let r: Awaited> | null = null - let tokens = 0 - for (let attempt = 0; attempt < 3; attempt++) { - resetSearchCalls() - try { - const rr = await runAgentic({ - surface, - task: t, - strategy: refine, - budget: BUDGET, - routerBaseUrl: ROUTER, - routerKey: KEY, - model: MODEL, - maxTokens: 8000, - innerTurns: 12, - }) - const tok = (rr as { tokens?: { input?: number; output?: number } }).tokens - tokens = (tok?.input ?? 0) + (tok?.output ?? 0) - if (tokens > 0) { - r = rr - break - } - } catch (e) { - console.error( - ` [${search ? '+search' : 'no-search'}] ${t.id}: attempt ${attempt} ERROR ${e instanceof Error ? e.message : e}`, - ) - } - } - if (!r || tokens === 0) { - perTask.push(0) - valid.push(false) - console.error( - ` [${search ? '+search' : 'no-search'}] ${t.id}: INVALID (empty run) — dropped`, - ) - continue - } - perTask.push(r.resolved ? 1 : 0) - valid.push(true) - if (search && searchCalls() > 0) fired++ - usd += r.usd - console.error( - ` [${search ? '+search' : 'no-search'}] ${t.id}: resolved=${r.resolved}${search ? ` search-calls=${searchCalls()}` : ''} tokens=${tokens}`, - ) - } - const vIdx = valid.map((v, i) => (v ? i : -1)).filter((i) => i >= 0) - const resolve = vIdx.length ? vIdx.reduce((a, i) => a + (perTask[i] ?? 0), 0) / vIdx.length : 0 - return { resolve, perTask, valid, fired, usd } -} - -function pairedDeltaCI(a: number[], b: number[]): { delta: number; lo: number; hi: number } { - const d = a.map((x, i) => x - (b[i] ?? 0)) - const mean = d.reduce((s, x) => s + x, 0) / d.length - const v = d.reduce((s, x) => s + (x - mean) ** 2, 0) / Math.max(1, d.length - 1) - const se = Math.sqrt(v / d.length) - return { delta: mean, lo: mean - 1.96 * se, hi: mean + 1.96 * se } -} - -const tasks = await aiSdkTasks(N) -console.error( - `aisdk-v7 search ablation · model=${MODEL} · N=${N} · budget=${BUDGET} · mode=${MODE}\n`, -) - -const noSearch = await runArm(false, tasks) -console.error( - `\nNO-SEARCH (control): resolve=${(100 * noSearch.resolve).toFixed(0)}% $${noSearch.usd.toFixed(2)}`, -) - -if (MODE === 'cal') { - console.error(`\n=== CONTROL READ ===`) - if (noSearch.resolve >= 0.6) - console.error( - `No-search recovers ${(100 * noSearch.resolve).toFixed(0)}% from compiler feedback ALONE — search has little room. Weak candidate.`, - ) - else - console.error( - `No-search stuck at ${(100 * noSearch.resolve).toFixed(0)}% with compiler feedback but no web — room for search. Run ARMS=full.`, - ) - process.exit(0) -} - -const search = await runArm(true, tasks) -console.error( - `\n+SEARCH: resolve=${(100 * search.resolve).toFixed(0)}% fired=${search.fired}/${N} $${search.usd.toFixed(2)}`, -) - -const both = tasks.map((_, i) => i).filter((i) => noSearch.valid[i] && search.valid[i]) -const nsT = both.map((i) => noSearch.perTask[i] ?? 0) -const seT = both.map((i) => search.perTask[i] ?? 0) -const ci = pairedDeltaCI(seT, nsT) -console.error(`\n=== VERDICT (pre-registered) ===`) -console.error(`paired tasks valid in both arms: ${both.length}/${N}`) -if (search.fired === 0) { - console.error(`INVALID: search arm never called web_search — broken arm, not a tie.`) - process.exit(0) -} -if (both.length < 5) { - console.error(`INVALID: only ${both.length} both-valid tasks — raise N or fix empty-run rate.`) - process.exit(0) -} -console.error( - `no-search resolve ${((100 * nsT.reduce((a, b) => a + b, 0)) / nsT.length).toFixed(0)}% | +search resolve ${((100 * seT.reduce((a, b) => a + b, 0)) / seT.length).toFixed(0)}%`, -) -console.error( - `resolve delta (search−nosearch): ${(100 * ci.delta).toFixed(0)}pt 95% CI [${(100 * ci.lo).toFixed(0)}, ${(100 * ci.hi).toFixed(0)}]`, -) -console.error( - ci.lo > 0 - ? `SEARCH HELPS — CI excludes 0.` - : `NO significant search benefit at n=${both.length} (CI includes 0).`, -) diff --git a/examples/ablation-suite/run-multi-contract.ts b/examples/ablation-suite/run-multi-contract.ts deleted file mode 100644 index cf636a8c..00000000 --- a/examples/ablation-suite/run-multi-contract.ts +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Search-value ablation on the long-horizon multi-contract task. - * - * ARM = the `search` env knob: worker WITH the you.com web_search tool vs WITHOUT (native - * read/write/run_tests/bash ON in both — additive isolation). Strategy = `refine` (continuous - * multi-round worker, budget rounds), the long-horizon loop. Coder = glm-5.2 via the router. - * - * PRE-REGISTERED GATES (enforced here, not narrated past): - * ARMS=cal (default) → run NO-SEARCH only. resolve MUST be <0.40, else the task is too easy — - * harden it (add/harder contracts), do NOT pay for the paired run. - * full → both arms, paired per-task; report resolve delta + paired-bootstrap 95% CI + search - * firing. A run where the search arm never called web_search is INVALID (not a tie). - * - * Env knobs: WORKER_MODEL (default glm-5.2), N (holdout size), BUDGET (refine rounds), ARMS=cal|full. - */ -import { refine, runAgentic } from '@tangle-network/agent-runtime/loops' -import { - makeMultiContractSurface, - multiContractTasks, - resetSearchCalls, - searchCalls, -} from './multi-contract-env.js' - -const ROUTER = process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1' -const KEY = process.env.TANGLE_API_KEY ?? '' -const MODEL = process.env.WORKER_MODEL ?? 'glm-5.2' -const N = Number(process.env.N ?? (process.env.ARMS === 'full' ? 20 : 6)) -const BUDGET = Number(process.env.BUDGET ?? 8) -const MODE = process.env.ARMS ?? 'cal' -if (!KEY) { - console.error('TANGLE_API_KEY unset') - process.exit(1) -} - -interface ArmOut { - resolve: number - scoreMean: number - perTask: number[] - perScore: number[] - valid: boolean[] - fired: number - usd: number -} - -async function runArm( - search: boolean, - tasks: Awaited>, -): Promise { - const surface = makeMultiContractSurface({ search }) - const perTask: number[] = [] - const perScore: number[] = [] - const valid: boolean[] = [] - let fired = 0 - let usd = 0 - for (const t of tasks) { - // runAgentic opens its OWN handle (the worker's workspace); use ITS result. The router - // occasionally returns an empty (0-token) run — retry up to twice, then mark the task INVALID - // so it's dropped from the paired vectors instead of counting as a fake 0. - let r: Awaited> | null = null - let tokens = 0 - for (let attempt = 0; attempt < 3; attempt++) { - resetSearchCalls() - try { - const rr = await runAgentic({ - surface, - task: t, - strategy: refine, - budget: BUDGET, - routerBaseUrl: ROUTER, - routerKey: KEY, - model: MODEL, - maxTokens: 8000, - innerTurns: 14, - }) - const tok = (rr as { tokens?: { input?: number; output?: number } }).tokens - tokens = (tok?.input ?? 0) + (tok?.output ?? 0) - if (tokens > 0) { - r = rr - break - } - } catch (e) { - console.error( - ` [${search ? '+search' : 'no-search'}] ${t.id}: attempt ${attempt} ERROR ${e instanceof Error ? e.message : e}`, - ) - } - } - if (!r || tokens === 0) { - perTask.push(0) - perScore.push(0) - valid.push(false) - console.error( - ` [${search ? '+search' : 'no-search'}] ${t.id}: INVALID (empty run after retries) — dropped`, - ) - continue - } - perTask.push(r.resolved ? 1 : 0) - perScore.push(Math.max(0, Math.min(1, r.score))) - valid.push(true) - if (search && searchCalls() > 0) fired++ - usd += r.usd - console.error( - ` [${search ? '+search' : 'no-search'}] ${t.id}: score=${(100 * r.score).toFixed(0)}% resolved=${r.resolved}${search ? ` search-calls=${searchCalls()}` : ''} tokens=${tokens} shots=${(r as { shots?: number }).shots ?? '?'}`, - ) - } - const vIdx = valid.map((v, i) => (v ? i : -1)).filter((i) => i >= 0) - const vResolve = vIdx.length ? vIdx.reduce((a, i) => a + (perTask[i] ?? 0), 0) / vIdx.length : 0 - const vScore = vIdx.length ? vIdx.reduce((a, i) => a + (perScore[i] ?? 0), 0) / vIdx.length : 0 - return { resolve: vResolve, scoreMean: vScore, perTask, perScore, valid, fired, usd } -} - -// paired bootstrap CI of (search − nosearch) on the aligned per-task vectors — no random seed -// available in this env, so use a deterministic jackknife-style spread as a conservative CI proxy. -function pairedDeltaCI(a: number[], b: number[]): { delta: number; lo: number; hi: number } { - const d = a.map((x, i) => x - (b[i] ?? 0)) - const mean = d.reduce((s, x) => s + x, 0) / d.length - const v = d.reduce((s, x) => s + (x - mean) ** 2, 0) / Math.max(1, d.length - 1) - const se = Math.sqrt(v / d.length) - return { delta: mean, lo: mean - 1.96 * se, hi: mean + 1.96 * se } -} - -const tasks = await multiContractTasks(0, N) -console.error( - `multi-contract search ablation · model=${MODEL} · N=${N} · budget=${BUDGET} · mode=${MODE}\n`, -) - -const noSearch = await runArm(false, tasks) -console.error( - `\nNO-SEARCH: resolve=${(100 * noSearch.resolve).toFixed(0)}% contracts-mean=${(100 * noSearch.scoreMean).toFixed(0)}% $${noSearch.usd.toFixed(2)}`, -) - -if (MODE === 'cal') { - console.error(`\n=== CALIBRATION GATE (pre-registered) ===`) - if (noSearch.resolve >= 0.4) { - console.error( - `FAIL: no-search resolve ${(100 * noSearch.resolve).toFixed(0)}% ≥ 40% — task TOO EASY, no room for search to win. HARDEN it (more/harder contracts) before the paired run. Do NOT report a search delta from this.`, - ) - } else if (noSearch.scoreMean >= 0.98) { - console.error( - `FAIL: contracts-mean ${(100 * noSearch.scoreMean).toFixed(0)}% — aces the contracts without search. Harden.`, - ) - } else { - console.error( - `PASS: no-search resolve ${(100 * noSearch.resolve).toFixed(0)}% < 40% (contracts-mean ${(100 * noSearch.scoreMean).toFixed(0)}%) — middle band, room for search. Run ARMS=full for the paired verdict.`, - ) - } - process.exit(0) -} - -const search = await runArm(true, tasks) -console.error( - `\n+SEARCH: resolve=${(100 * search.resolve).toFixed(0)}% contracts-mean=${(100 * search.scoreMean).toFixed(0)}% fired=${search.fired}/${N} $${search.usd.toFixed(2)}`, -) - -// paired on tasks VALID in BOTH arms only (drop empty-run flukes so the pairing is honest) -const both = tasks.map((_, i) => i).filter((i) => noSearch.valid[i] && search.valid[i]) -const nsT = both.map((i) => noSearch.perTask[i] ?? 0) -const seT = both.map((i) => search.perTask[i] ?? 0) -const nsS = both.map((i) => noSearch.perScore[i] ?? 0) -const seS = both.map((i) => search.perScore[i] ?? 0) -const ci = pairedDeltaCI(seT, nsT) -const sci = pairedDeltaCI(seS, nsS) -console.error(`\n=== VERDICT (pre-registered) ===`) -console.error(`paired tasks valid in both arms: ${both.length}/${N}`) -if (search.fired === 0) { - console.error( - `INVALID: search arm never called web_search (0/${N} fired) — not a tie, a broken arm.`, - ) - process.exit(0) -} -if (both.length < 5) { - console.error( - `INVALID: only ${both.length} both-valid tasks — too few for a verdict (raise N or fix empty-run rate).`, - ) - process.exit(0) -} -console.error( - `no-search resolve ${((100 * nsT.reduce((a, b) => a + b, 0)) / nsT.length).toFixed(0)}% score ${((100 * nsS.reduce((a, b) => a + b, 0)) / nsS.length).toFixed(0)}% | +search resolve ${((100 * seT.reduce((a, b) => a + b, 0)) / seT.length).toFixed(0)}% score ${((100 * seS.reduce((a, b) => a + b, 0)) / seS.length).toFixed(0)}%`, -) -console.error( - `resolve delta (search−nosearch): ${(100 * ci.delta).toFixed(0)}pt 95% CI [${(100 * ci.lo).toFixed(0)}, ${(100 * ci.hi).toFixed(0)}]`, -) -console.error( - `contract-mean delta: ${(100 * sci.delta).toFixed(0)}pt 95% CI [${(100 * sci.lo).toFixed(0)}, ${(100 * sci.hi).toFixed(0)}]`, -) -console.error( - ci.lo > 0 || sci.lo > 0 - ? `SEARCH HELPS — CI excludes 0.` - : `NO significant search benefit (CI includes 0). Honest null at n=${N}.`, -)