diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b90e790e..48e1b5dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,9 +42,6 @@ jobs: agent-bench: runs-on: ubuntu-latest - defaults: - run: - working-directory: bench steps: - uses: actions/checkout@v4 @@ -54,16 +51,13 @@ jobs: with: node-version: 22 cache: pnpm - cache-dependency-path: bench/pnpm-lock.yaml + cache-dependency-path: pnpm-lock.yaml - - name: Install published dependencies + - name: Install package set run: pnpm install --frozen-lockfile - - name: Typecheck public entrypoint against installed packages - run: pnpm run typecheck:public - - - name: Test deterministic package paths - run: pnpm test + - name: Build current agent-runtime + run: pnpm run build - - name: Verify packed package in a clean consumer - run: pnpm run verify:package + - name: Verify agent-bench against current agent-runtime + run: pnpm run verify:bench diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 416bccf8..59061e70 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -40,6 +40,9 @@ jobs: - name: Verify packed package exports run: pnpm run verify:package + - name: Verify agent-bench against this release + run: pnpm run verify:bench + - name: Verify tag/version lock run: | NPM_VERSION=$(node -p "require('./package.json').version") @@ -98,9 +101,6 @@ jobs: # tag/version locking stays intact and the root package is never co-published. if: startsWith(github.ref, 'refs/tags/agent-bench-v') runs-on: ubuntu-latest - defaults: - run: - working-directory: bench steps: - uses: actions/checkout@v4 @@ -110,23 +110,17 @@ jobs: with: node-version: 22 cache: pnpm - cache-dependency-path: bench/pnpm-lock.yaml + cache-dependency-path: pnpm-lock.yaml - name: Install deps run: pnpm install --frozen-lockfile - - name: Typecheck public entrypoint against installed packages - run: pnpm run typecheck:public - - - name: Test deterministic package paths - run: pnpm test - - - name: Verify packed package in a clean consumer - run: pnpm run verify:package + - name: Verify packed agent-bench against published dependencies + run: pnpm run verify:bench:published - name: Verify tag/version lock run: | - NPM_VERSION=$(node -p "require('./package.json').version") + NPM_VERSION=$(node -p "require('./bench/package.json').version") TAG_VERSION="${GITHUB_REF#refs/tags/agent-bench-v}" if [ "$TAG_VERSION" != "$NPM_VERSION" ]; then echo "::error::Tag/version mismatch: tag=$TAG_VERSION package=$NPM_VERSION." @@ -141,9 +135,6 @@ jobs: permissions: contents: read id-token: write - defaults: - run: - working-directory: bench steps: - uses: actions/checkout@v4 @@ -153,7 +144,7 @@ jobs: with: node-version: 22 cache: pnpm - cache-dependency-path: bench/pnpm-lock.yaml + cache-dependency-path: pnpm-lock.yaml - run: pnpm install --frozen-lockfile @@ -162,10 +153,14 @@ jobs: # Requires the npmjs Trusted Publisher on @tangle-network/agent-bench: # org tangle-network, repo agent-runtime, workflow publish.yml. npm install -g npm@11 - NAME=$(node -p "require('./package.json').name") - VERSION=$(node -p "require('./package.json').version") + NAME=$(node -p "require('./bench/package.json').name") + VERSION=$(node -p "require('./bench/package.json').version") if npm view "$NAME@$VERSION" version >/dev/null 2>&1; then echo "$NAME@$VERSION already on registry; skipping publish" else - npm publish --provenance --access public + mkdir -p "$RUNNER_TEMP/agent-bench-package" + pnpm --dir bench pack --pack-destination "$RUNNER_TEMP/agent-bench-package" + package="$(find "$RUNNER_TEMP/agent-bench-package" -maxdepth 1 -name '*.tgz' -print -quit)" + test -n "$package" + npm publish "$package" --provenance --access public fi diff --git a/bench/package.json b/bench/package.json index c29a8aeb..e3f72697 100644 --- a/bench/package.json +++ b/bench/package.json @@ -24,23 +24,19 @@ "test": "node scripts/run-package-tests.mjs", "typecheck:public": "tsc -p tsconfig.public.json", "verify:package": "node scripts/verify-packed-consumer.mjs", + "verify:package:local-runtime": "node scripts/verify-packed-consumer.mjs --local-runtime", "verify:pier": "tsx scripts/verify-pier-pair.mts" }, "dependencies": { - "@tangle-network/agent-eval": "^0.117.1", - "@tangle-network/agent-interface": "^0.25.0", - "@tangle-network/agent-runtime": "0.94.9", - "@tangle-network/sandbox": "^0.10.3" + "@tangle-network/agent-eval": "0.122.1", + "@tangle-network/agent-interface": "0.30.0", + "@tangle-network/agent-runtime": "workspace:*", + "@tangle-network/sandbox": "^0.11.1" }, "devDependencies": { - "@types/node": "^25.0.0", - "tsx": "^4.19.0", - "typescript": "^6.0.3" - }, - "pnpm": { - "overrides": { - "@tangle-network/agent-eval": "0.117.1" - } + "@types/node": "^25.9.3", + "tsx": "^4.22.4", + "typescript": "^5.7.0" }, "files": [ "src", diff --git a/bench/pier_agents/candidate_contract.py b/bench/pier_agents/candidate_contract.py index 5d667163..bc773401 100644 --- a/bench/pier_agents/candidate_contract.py +++ b/bench/pier_agents/candidate_contract.py @@ -107,6 +107,17 @@ def _object(value: Any, label: str) -> dict[str, Any]: return value +def _contract_object( + value: Any, label: str, *, kind: str | None = None +) -> dict[str, Any]: + obj = _object(value, label) + if "schemaVersion" in obj or "version" in obj: + raise CandidateContractError(f"{label} contains obsolete version fields") + if kind is not None and obj.get("kind") != kind: + raise CandidateContractError(f"{label} has the wrong kind") + return obj + + def _string(value: Any, label: str) -> str: if not isinstance(value, str) or not value or "\0" in value: raise CandidateContractError(f"{label} must be a non-empty string without NUL") @@ -236,18 +247,14 @@ def _embedded_bytes(value: Any, label: str) -> bytes: def _workspace_files(value: Any, label: str) -> tuple[WorkspaceFile, ...]: - snapshot = _object(value, label) - if ( - snapshot.get("schemaVersion") != 1 - or snapshot.get("kind") != "agent-candidate-workspace-snapshot" - ): - raise CandidateContractError(f"{label} has the wrong snapshot kind") - material = _object(snapshot.get("material"), f"{label}.material") - if ( - material.get("schemaVersion") != 1 - or material.get("kind") != "agent-candidate-workspace-manifest" - ): - raise CandidateContractError(f"{label}.material has the wrong manifest kind") + snapshot = _contract_object( + value, label, kind="agent-candidate-workspace-snapshot" + ) + material = _contract_object( + snapshot.get("material"), + f"{label}.material", + kind="agent-candidate-workspace-manifest", + ) values = material.get("files") if not isinstance(values, list): raise CandidateContractError(f"{label}.material.files must be an array") @@ -255,7 +262,7 @@ def _workspace_files(value: Any, label: str) -> tuple[WorkspaceFile, ...]: for index, value in enumerate(values): obj = _object(value, f"{label}.material.files[{index}]") mode = _integer(obj.get("mode"), f"{label}.material.files[{index}].mode") - if mode not in {0o644, 0o755}: + if mode > 0o777: raise CandidateContractError( f"{label} contains unsupported file mode {mode:o}" ) @@ -287,7 +294,7 @@ def _profile_files(value: Any) -> tuple[ProfileFile, ...]: for index, value in enumerate(values): obj = _object(value, f"profilePlan.material.files[{index}]") mode = _integer(obj.get("mode"), f"profilePlan.material.files[{index}].mode") - if mode not in {0o644, 0o755}: + if mode > 0o777: raise CandidateContractError( f"profile plan contains unsupported mode {mode:o}" ) @@ -359,26 +366,23 @@ def load_prepared_candidate_contract( raise CandidateContractError( "materialization receipt bytes do not match the expected digest" ) - if plan.get("schemaVersion") != 1 or plan.get("kind") != _PLAN_KIND: - raise CandidateContractError("execution plan has the wrong schema or kind") - if receipt.get("schemaVersion") != 1 or receipt.get("kind") != _RECEIPT_KIND: - raise CandidateContractError( - "materialization receipt has the wrong schema or kind" - ) + plan = _contract_object(plan, "execution plan", kind=_PLAN_KIND) + receipt = _contract_object( + receipt, "materialization receipt", kind=_RECEIPT_KIND + ) if receipt.get("digestAlgorithm") != "rfc8785-sha256" or "digest" in receipt: raise CandidateContractError( "materialization receipt must be exact digest-free canonical material" ) - execution_evidence = _object(receipt.get("executionPlan"), "receipt.executionPlan") + execution_evidence = _contract_object( + receipt.get("executionPlan"), + "receipt.executionPlan", + kind=_EXECUTION_EVIDENCE_KIND, + ) plan_digest = _digest( execution_evidence.get("digest"), "receipt.executionPlan.digest" ) - if ( - execution_evidence.get("schemaVersion") != 1 - or execution_evidence.get("kind") != _EXECUTION_EVIDENCE_KIND - ): - raise CandidateContractError("receipt execution evidence has the wrong kind") if ( sha256_bytes(plan_raw) != plan_digest or execution_evidence.get("material") != plan @@ -402,11 +406,16 @@ def load_prepared_candidate_contract( execution_id = _string(plan.get("executionId"), "plan.executionId") task = _object(plan.get("task"), "plan.task") + outcome = _object(task.get("outcome"), "plan.task.outcome") + if outcome.get("kind") != "workspace": + raise CandidateContractError("Pier requires a workspace task outcome") repository = _object(task.get("repository"), "plan.task.repository") base_commit = _git_object( repository.get("baseCommit"), "plan.task.repository.baseCommit" ) - base_tree = _git_object(repository.get("baseTree"), "plan.task.repository.baseTree") + base_tree = _git_object( + repository.get("baseTree"), "plan.task.repository.baseTree" + ) if len(base_commit) != len(base_tree): raise CandidateContractError("task Git objects use different hash formats") instruction = _instruction(task.get("instruction")) @@ -471,12 +480,11 @@ def load_prepared_candidate_contract( if receipt.get("candidateWorkspace") != candidate_snapshot: raise CandidateContractError("receipt and plan candidate workspaces differ") - profile_evidence = _object(receipt.get("profilePlan"), "receipt.profilePlan") - if ( - profile_evidence.get("schemaVersion") != 1 - or profile_evidence.get("kind") != _PROFILE_PLAN_KIND - ): - raise CandidateContractError("receipt profile evidence has the wrong kind") + profile_evidence = _contract_object( + receipt.get("profilePlan"), + "receipt.profilePlan", + kind=_PROFILE_PLAN_KIND, + ) profile_digest = _digest( profile_evidence.get("digest"), "receipt.profilePlan.digest" ) @@ -489,7 +497,7 @@ def load_prepared_candidate_contract( profile_artifact_material = json.loads(profile_raw) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise CandidateContractError("profile artifact is not UTF-8 JSON") from exc - profile_material = _object( + profile_material = _contract_object( profile_evidence.get("material"), "receipt.profilePlan.material" ) if profile_artifact_material != profile_material: diff --git a/bench/pier_agents/tangle_candidate_test.py b/bench/pier_agents/tangle_candidate_test.py index 5d7de7bd..cc77be39 100644 --- a/bench/pier_agents/tangle_candidate_test.py +++ b/bench/pier_agents/tangle_candidate_test.py @@ -4,6 +4,7 @@ import importlib import json import os +import shlex import shutil import subprocess import sys @@ -162,7 +163,6 @@ def _embedded(raw): def _workspace(files): material = { - "schemaVersion": 1, "kind": "agent-candidate-workspace-manifest", "files": [ { @@ -176,7 +176,6 @@ def _workspace(files): } manifest = _canonical(material) return { - "schemaVersion": 1, "kind": "agent-candidate-workspace-snapshot", "digest": _sha(manifest), "material": material, @@ -242,18 +241,17 @@ def _fixture(root: Path): os.chmod(candidate_staging / "runner.py", 0o755) profile = b"fixture-profile\n" (profile_staging / "AGENTS.md").write_bytes(profile) - os.chmod(profile_staging / "AGENTS.md", 0o644) + os.chmod(profile_staging / "AGENTS.md", 0o600) instruction = "make the task ready".encode() task_snapshot = _workspace([("src/status.txt", 0o644, status)]) candidate_snapshot = _workspace([("runner.py", 0o755, runner)]) profile_material = { - "version": 1, "harness": "codex", "files": [ { "relPath": "AGENTS.md", - "mode": 0o644, + "mode": 0o600, "contentSha256": _sha(profile), } ], @@ -265,7 +263,6 @@ def _fixture(root: Path): profile_digest = _sha(profile_raw) bundle_digest = f"sha256:{'b' * 64}" plan = { - "schemaVersion": 1, "kind": "agent-candidate-execution-plan-material", "bundleDigest": bundle_digest, "executionId": "pier-fixture-execution", @@ -287,6 +284,7 @@ def _fixture(root: Path): "baseCommit": base_commit, "baseTree": base_tree, }, + "outcome": {"kind": "workspace"}, "workspace": task_snapshot, }, "workspaces": { @@ -345,21 +343,18 @@ def _fixture(root: Path): plan_raw = _canonical(plan) plan_digest = _sha(plan_raw) execution_evidence = { - "schemaVersion": 1, "kind": "agent-candidate-execution-plan", "digest": plan_digest, "material": plan, "artifact": _embedded(plan_raw), } profile_evidence = { - "schemaVersion": 1, "kind": "agent-profile-workspace-plan", "digest": profile_digest, "material": profile_material, "artifact": _embedded(profile_raw), } receipt = { - "schemaVersion": 1, "kind": "agent-candidate-materialization", "digestAlgorithm": "rfc8785-sha256", "bundleDigest": bundle_digest, @@ -403,7 +398,6 @@ def _rewrite_signed_plan(fixture): plan_digest = _sha(plan_raw) receipt = json.loads(fixture["receipt_path"].read_text()) receipt["executionPlan"] = { - "schemaVersion": 1, "kind": "agent-candidate-execution-plan", "digest": plan_digest, "material": fixture["plan"], @@ -781,6 +775,21 @@ def test_rejects_extra_utf8_file_delivery_fields(self): fixture["receipt_digest"], ) + def test_rejects_obsolete_contract_version_fields(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["schemaVersion"] = 1 + _rewrite_signed_plan(fixture) + + with self.assertRaisesRegex( + contract_module.CandidateContractError, "obsolete version fields" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + def test_accepts_only_frozen_public_model_gateway_domains(self): with tempfile.TemporaryDirectory() as directory: fixture = _fixture(Path(directory)) @@ -957,7 +966,7 @@ def test_enforces_exact_signed_timeout(self): with tempfile.TemporaryDirectory() as directory: fixture = _fixture(Path(directory)) fixture["plan"]["launch"]["env"] = { - "FIXTURE_SLEEP": {"kind": "public", "value": "0.25"} + "FIXTURE_SLEEP": {"kind": "public", "value": "2"} } fixture["plan"]["limits"]["timeoutMs"] = 10 _rewrite_signed_plan(fixture) @@ -968,9 +977,17 @@ def test_enforces_exact_signed_timeout(self): started = time.monotonic() with self.assertRaises(asyncio.TimeoutError): asyncio.run(agent.run("make the task ready", environment, context)) - self.assertLess(time.monotonic() - started, 0.2) + wall_elapsed = time.monotonic() - started + candidate_command = next( + command + for command in environment.commands + if "/run-" in command and "/timeout-" in command + ) + self.assertEqual(shlex.split(candidate_command)[4], "0.010") self.assertEqual(context.metadata["termination"]["kind"], "timeout") self.assertEqual(context.metadata["termination"]["timeoutMs"], 10) + self.assertLess(context.metadata["observedElapsedMs"], 250) + self.assertLess(wall_elapsed, 1) def test_profile_cleanup_does_not_follow_candidate_links(self): for attack in ("symlink", "hardlink"): diff --git a/bench/pnpm-lock.yaml b/bench/pnpm-lock.yaml deleted file mode 100644 index 94d8d185..00000000 --- a/bench/pnpm-lock.yaml +++ /dev/null @@ -1,958 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -overrides: - '@tangle-network/agent-eval': 0.117.1 - -importers: - - .: - dependencies: - '@tangle-network/agent-eval': - specifier: 0.117.1 - version: 0.117.1(typescript@6.0.3) - '@tangle-network/agent-interface': - specifier: ^0.25.0 - version: 0.25.0 - '@tangle-network/agent-runtime': - specifier: 0.94.9 - version: 0.94.9(@tangle-network/agent-eval@0.117.1(typescript@6.0.3))(@tangle-network/agent-interface@0.25.0)(@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) - '@tangle-network/sandbox': - specifier: ^0.10.3 - version: 0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) - devDependencies: - '@types/node': - specifier: ^25.0.0 - version: 25.9.5 - tsx: - specifier: ^4.19.0 - version: 4.22.4 - typescript: - specifier: ^6.0.3 - version: 6.0.3 - -packages: - - '@adraffy/ens-normalize@1.11.1': - resolution: {integrity: sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==} - - '@asteasolutions/zod-to-openapi@8.5.0': - resolution: {integrity: sha512-SABbKiObg5dLRiTFnqiW1WWwGcg1BJfmHtT2asIBnBHg6Smy/Ms2KHc650+JI4Hw7lSkdiNebEGXpwoxfben8Q==} - peerDependencies: - zod: ^4.0.0 - - '@ax-llm/ax@19.0.45': - resolution: {integrity: sha512-zC/OqUutTV0omOuAxBgeCihmxP3e1eUcbA78FCaMF3pPXZ3/FgQDZf0R04LH/lVIxm5BH4/qDdNNYcIVGwii8Q==} - hasBin: true - peerDependencies: - zod: ^3.24.0 || ^4.0.0 - peerDependenciesMeta: - zod: - optional: true - - '@esbuild/aix-ppc64@0.28.0': - resolution: {integrity: sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/android-arm64@0.28.0': - resolution: {integrity: sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm@0.28.0': - resolution: {integrity: sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-x64@0.28.0': - resolution: {integrity: sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/darwin-arm64@0.28.0': - resolution: {integrity: sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-x64@0.28.0': - resolution: {integrity: sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/freebsd-arm64@0.28.0': - resolution: {integrity: sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.28.0': - resolution: {integrity: sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/linux-arm64@0.28.0': - resolution: {integrity: sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm@0.28.0': - resolution: {integrity: sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-ia32@0.28.0': - resolution: {integrity: sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-loong64@0.28.0': - resolution: {integrity: sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-mips64el@0.28.0': - resolution: {integrity: sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-ppc64@0.28.0': - resolution: {integrity: sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-riscv64@0.28.0': - resolution: {integrity: sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-s390x@0.28.0': - resolution: {integrity: sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-x64@0.28.0': - resolution: {integrity: sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/netbsd-arm64@0.28.0': - resolution: {integrity: sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.28.0': - resolution: {integrity: sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/openbsd-arm64@0.28.0': - resolution: {integrity: sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.28.0': - resolution: {integrity: sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openharmony-arm64@0.28.0': - resolution: {integrity: sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/sunos-x64@0.28.0': - resolution: {integrity: sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/win32-arm64@0.28.0': - resolution: {integrity: sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-ia32@0.28.0': - resolution: {integrity: sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-x64@0.28.0': - resolution: {integrity: sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@hono/node-server@2.0.4': - resolution: {integrity: sha512-Ut3y0dMMPWy6bZ2kVfx25EOVbZlm15dhF4mOsezMlhpNHy+4MkU1qN9Y6lnruYi4wPmFzimGX2X7LF/FwHli4A==} - engines: {node: '>=20'} - peerDependencies: - hono: ^4 - - '@noble/ciphers@1.3.0': - resolution: {integrity: sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==} - engines: {node: ^14.21.3 || >=16} - - '@noble/curves@1.9.1': - resolution: {integrity: sha512-k11yZxZg+t+gWvBbIswW0yoJlu8cHOC7dhunwOzoWH/mXGBiYyR4YY6hAEK/3EUs4UpB8la1RfdRpeGsFHkWsA==} - engines: {node: ^14.21.3 || >=16} - - '@noble/curves@2.2.0': - resolution: {integrity: sha512-T/BoHgFXirb0ENSPBquzX0rcjXeM6Lo892a2jlYJkqk83LqZx0l1Of7DzlKJ6jkpvMrkHSnAcgb5JegL8SeIkQ==} - engines: {node: '>= 20.19.0'} - - '@noble/hashes@1.8.0': - resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} - engines: {node: ^14.21.3 || >=16} - - '@noble/hashes@2.2.0': - resolution: {integrity: sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg==} - engines: {node: '>= 20.19.0'} - - '@opentelemetry/api@1.9.1': - resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} - engines: {node: '>=8.0.0'} - - '@scure/base@1.2.6': - resolution: {integrity: sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==} - - '@scure/base@2.2.0': - resolution: {integrity: sha512-b8XEupJibegiXV+tDUseI8oLQc8ei3d/4Jkb2RpbHh3MfE054ov3uIz2dhFkB3FI8iwYkEh0gGCApkrYggkPNg==} - - '@scure/bip32@1.7.0': - resolution: {integrity: sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==} - - '@scure/bip32@2.2.0': - resolution: {integrity: sha512-zFr7t2F+a9+5tB7QbarF2HQNYrgjCNaoLAupZdKkrFMYMozJf5zqH2WJCQibMzm1qQ0QogrxVGO3qXfQDYMaQg==} - - '@scure/bip39@1.6.0': - resolution: {integrity: sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==} - - '@scure/bip39@2.2.0': - resolution: {integrity: sha512-T/Bj/YvYMNkIPq6EENO6/rcs2e7qTNuyoUXf0KBFDmp0ZDu0H2X4Lq6yC3i0c8PcWkov5EbW+yQZZbdMmk154A==} - - '@tangle-network/agent-core@0.3.4': - resolution: {integrity: sha512-Hvz3ABRouNtBmRvGqPxifAO2yuILneJMylWH5jW/jeS2F03RvqkGYuXyGXWWLqosYbb3hVAvSEe4Ykm2FMGEDQ==} - - '@tangle-network/agent-core@0.4.9': - resolution: {integrity: sha512-BFU4WV12Y6D28PX+o8LIVkIMD+cXwrL1uyV182l12Bn9zEu7VHt2oVMEEzbsN8L+X0xM9baEfMCHTFKFNJv7Aw==} - - '@tangle-network/agent-eval@0.117.1': - resolution: {integrity: sha512-mtBdGhGIC+nrPZbseLQo0Iako1LR66ZeIr2UjVBJcNj/lHYfib2jbhdWy3RmbQcHR0Cxdqqydn7pLJ5dSXVYtQ==} - engines: {node: '>=20'} - hasBin: true - - '@tangle-network/agent-interface@0.13.0': - resolution: {integrity: sha512-CeTPGRLoXqpt0h+BCyFgZPkfU1zyRpWmqfD+85i/uk+uvbqxkfI+JprfKVf3tBsQuCgJPSjPt5qjdW8n3h2BVg==} - - '@tangle-network/agent-interface@0.14.0': - resolution: {integrity: sha512-9CyGhIpl90E7v4MTm3b1ti3Bp7BfPigk2Nafgi21Lg0U+QxlNB656F2JmVpUuSbOo9aGZPtg5nXu5EBTlV5a1g==} - - '@tangle-network/agent-interface@0.21.0': - resolution: {integrity: sha512-jDxhVJgxymrvU1RLWxWKueuaWQIpBAfrW8BuVTB5m2Y4eFMLo1SawDBEMDLdZN4/Gf34xrFrsRk+PAj9brGKMQ==} - - '@tangle-network/agent-interface@0.22.0': - resolution: {integrity: sha512-7fsJhNdvTmOB1X9E2owl06jzyrqaN+jMkOPVKbK7dvNqQvf627PowCtL/edhUqEEv+K0FWtkwG3R3xqjX7VNZg==} - - '@tangle-network/agent-interface@0.24.0': - resolution: {integrity: sha512-iaHWNTYne49cBYXwb72NjGDw5bjT2KVJlfawXygbvkqnfUsQG57BS++Yjf/XYvwJJdDeEaGs+Xvy8SWYkvu0qA==} - - '@tangle-network/agent-interface@0.25.0': - resolution: {integrity: sha512-bMKjyjk8bk2651HleiTEUOTPgNE2bWxCEl9VBVUyub/UkP87Y09byXOxeDEsNGc4TIT19Y5Uo0b+k1kRWXkMVQ==} - - '@tangle-network/agent-knowledge@1.12.1': - resolution: {integrity: sha512-j5riyIvz5C+ZBELTtNVtmUbcKAMpgTHMNmDQ12MhOIPJv5Y7AeX2oCWuF6c8aPqg1hOrsLqgqXSr1zkV0ePsrA==} - engines: {node: '>=20'} - hasBin: true - - '@tangle-network/agent-profile-materialize@0.3.2': - resolution: {integrity: sha512-jCj1Hc/brPQ5eS7or9A+Klv0FAZcGtyFr7kx2cKfk+wCj0/4Di3k1pMjayrmRNgI/7Oc47gt6/t+qvdtZxBkAw==} - - '@tangle-network/agent-runtime@0.94.9': - resolution: {integrity: sha512-p0mKK4DbzmrVVQPhNTH+Si7mpIV5+/0avhGVNzJcq2YdcH0YWYW2+OUy86mhXkxv/vVNkdR70Sco91hZuoC6ig==} - engines: {node: '>=20'} - hasBin: true - peerDependencies: - '@tangle-network/agent-eval': 0.117.1 - '@tangle-network/agent-interface': '>=0.25.0 <0.26.0' - '@tangle-network/sandbox': '>=0.8.0 <1.0.0' - playwright: ^1.40.0 - peerDependenciesMeta: - '@tangle-network/sandbox': - optional: true - playwright: - optional: true - - '@tangle-network/sandbox@0.10.3': - resolution: {integrity: sha512-3nZnIaXc/vCH3Lb6SCZA3cYEJT4+ThGXLPkGo5z2kh434n3eJryk/PdnSXSwPoA6xuTQToJAPbGGIdMEHFU/Vw==} - peerDependencies: - '@mastra/core': ^1.36.0 - '@modelcontextprotocol/sdk': ^1.29.0 - ai: ^6.0.175 - openai: ^6.36.0 - viem: ^2.0.0 - peerDependenciesMeta: - '@mastra/core': - optional: true - '@modelcontextprotocol/sdk': - optional: true - ai: - optional: true - openai: - optional: true - viem: - optional: true - - '@tangle-network/sandbox@0.9.7': - resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} - peerDependencies: - '@mastra/core': ^1.36.0 - '@modelcontextprotocol/sdk': ^1.29.0 - ai: ^6.0.175 - openai: ^6.36.0 - viem: ^2.0.0 - peerDependenciesMeta: - '@mastra/core': - optional: true - '@modelcontextprotocol/sdk': - optional: true - ai: - optional: true - openai: - optional: true - viem: - optional: true - - '@tangle-network/tcloud-attestation@0.1.1': - resolution: {integrity: sha512-+TAF9s5t1jOWGyGHvKhIWe2FYmG7puVaxmmg0Et67ylAjGa7GqUAvISXGjG/6dzld7A170V0kQHK0WVdh2Wh0Q==} - engines: {node: '>=18'} - - '@tangle-network/tcloud@0.4.14': - resolution: {integrity: sha512-jWYt//cGdLBDOv0luLH6xAGS4gbuOt8uHIkaCWwDDpQ1zp0FUPATHIrA3RMuF0qtQq9Vq00IhLrmCnHdHBP+dg==} - engines: {node: '>=18'} - hasBin: true - - '@types/node@25.9.5': - resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} - - abitype@1.2.3: - resolution: {integrity: sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==} - peerDependencies: - typescript: '>=5.0.4' - zod: ^3.22.0 || ^4.0.0 - peerDependenciesMeta: - typescript: - optional: true - zod: - optional: true - - b4a@1.8.1: - resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==} - peerDependencies: - react-native-b4a: '*' - peerDependenciesMeta: - react-native-b4a: - optional: true - - bare-events@2.9.1: - resolution: {integrity: sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==} - peerDependencies: - bare-abort-controller: '*' - peerDependenciesMeta: - bare-abort-controller: - optional: true - - bare-fs@4.7.4: - resolution: {integrity: sha512-y1kC+ffIx/tPLdTE693uNjHfzTfr+ravR5tvWlMXe25nELbkqV400S71qHDwbkAQ1FVEZobB1NFRzFbCCcyBCQ==} - engines: {bare: '>=1.16.0'} - peerDependencies: - bare-buffer: '*' - peerDependenciesMeta: - bare-buffer: - optional: true - - bare-path@3.1.1: - resolution: {integrity: sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==} - - bare-stream@2.13.3: - resolution: {integrity: sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==} - peerDependencies: - bare-abort-controller: '*' - bare-buffer: '*' - bare-events: '*' - peerDependenciesMeta: - bare-abort-controller: - optional: true - bare-buffer: - optional: true - bare-events: - optional: true - - bare-url@2.4.5: - resolution: {integrity: sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==} - - commander@14.0.3: - resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} - engines: {node: '>=20'} - - dayjs@1.11.21: - resolution: {integrity: sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==} - - esbuild@0.28.0: - resolution: {integrity: sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==} - engines: {node: '>=18'} - hasBin: true - - eventemitter3@5.0.1: - resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} - - events-universal@1.0.1: - resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} - - fast-fifo@1.3.2: - resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} - - fsevents@2.3.3: - resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - - hono@4.12.30: - resolution: {integrity: sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==} - engines: {node: '>=16.9.0'} - - isows@1.0.7: - resolution: {integrity: sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==} - peerDependencies: - ws: '*' - - openapi3-ts@4.6.0: - resolution: {integrity: sha512-a4sfn6L2sIShhtzJqmjGrARvxAW/3F2BJDdyRVvNF9VhAsZSh5hSyI3a9TNvmzBxXmq66nY5LNT5bQcBxYAZZg==} - - ox@0.14.27: - resolution: {integrity: sha512-+xhLHo/f+f4BH121/1Pomm/1vgBBda1wYiFpTvjSo8o5OcEj76Pf1hGPJiepoYMTQoTm2SKdSBvWkFWk5l07PA==} - peerDependencies: - typescript: '>=5.4.0' - peerDependenciesMeta: - typescript: - optional: true - - streamx@2.28.0: - resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==} - - tar-stream@3.2.0: - resolution: {integrity: sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==} - - teex@1.0.1: - resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==} - - text-decoder@1.2.7: - resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} - - tsx@4.22.4: - resolution: {integrity: sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==} - engines: {node: '>=18.0.0'} - hasBin: true - - typescript@6.0.3: - resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} - engines: {node: '>=14.17'} - hasBin: true - - undici-types@7.24.6: - resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} - - viem@2.52.0: - resolution: {integrity: sha512-py2QPYe9e1f4DmPJCsXF7zHmyZ0PkJrBxdQZ5dvNXvzy3UzWkUn7dNfC0TMeNm6Qv1tKw3b6qXXExpx6L0oMbw==} - peerDependencies: - typescript: '>=5.0.4' - peerDependenciesMeta: - typescript: - optional: true - - ws@8.20.1: - resolution: {integrity: sha512-It4dO0K5v//JtTXuPkfEOaI3uUN87iYPnqo/ZzqCoG3g8uhA66QUMs/SrM0YK7/NAu+r4LMh/9dq2A7k+rHs+w==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - - yaml@2.9.0: - resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} - engines: {node: '>= 14.6'} - hasBin: true - - zod@4.4.3: - resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} - -snapshots: - - '@adraffy/ens-normalize@1.11.1': {} - - '@asteasolutions/zod-to-openapi@8.5.0(zod@4.4.3)': - dependencies: - openapi3-ts: 4.6.0 - zod: 4.4.3 - - '@ax-llm/ax@19.0.45(zod@4.4.3)': - dependencies: - '@opentelemetry/api': 1.9.1 - dayjs: 1.11.21 - optionalDependencies: - zod: 4.4.3 - - '@esbuild/aix-ppc64@0.28.0': - optional: true - - '@esbuild/android-arm64@0.28.0': - optional: true - - '@esbuild/android-arm@0.28.0': - optional: true - - '@esbuild/android-x64@0.28.0': - optional: true - - '@esbuild/darwin-arm64@0.28.0': - optional: true - - '@esbuild/darwin-x64@0.28.0': - optional: true - - '@esbuild/freebsd-arm64@0.28.0': - optional: true - - '@esbuild/freebsd-x64@0.28.0': - optional: true - - '@esbuild/linux-arm64@0.28.0': - optional: true - - '@esbuild/linux-arm@0.28.0': - optional: true - - '@esbuild/linux-ia32@0.28.0': - optional: true - - '@esbuild/linux-loong64@0.28.0': - optional: true - - '@esbuild/linux-mips64el@0.28.0': - optional: true - - '@esbuild/linux-ppc64@0.28.0': - optional: true - - '@esbuild/linux-riscv64@0.28.0': - optional: true - - '@esbuild/linux-s390x@0.28.0': - optional: true - - '@esbuild/linux-x64@0.28.0': - optional: true - - '@esbuild/netbsd-arm64@0.28.0': - optional: true - - '@esbuild/netbsd-x64@0.28.0': - optional: true - - '@esbuild/openbsd-arm64@0.28.0': - optional: true - - '@esbuild/openbsd-x64@0.28.0': - optional: true - - '@esbuild/openharmony-arm64@0.28.0': - optional: true - - '@esbuild/sunos-x64@0.28.0': - optional: true - - '@esbuild/win32-arm64@0.28.0': - optional: true - - '@esbuild/win32-ia32@0.28.0': - optional: true - - '@esbuild/win32-x64@0.28.0': - optional: true - - '@hono/node-server@2.0.4(hono@4.12.30)': - dependencies: - hono: 4.12.30 - - '@noble/ciphers@1.3.0': {} - - '@noble/curves@1.9.1': - dependencies: - '@noble/hashes': 1.8.0 - - '@noble/curves@2.2.0': - dependencies: - '@noble/hashes': 2.2.0 - - '@noble/hashes@1.8.0': {} - - '@noble/hashes@2.2.0': {} - - '@opentelemetry/api@1.9.1': {} - - '@scure/base@1.2.6': {} - - '@scure/base@2.2.0': {} - - '@scure/bip32@1.7.0': - dependencies: - '@noble/curves': 1.9.1 - '@noble/hashes': 1.8.0 - '@scure/base': 1.2.6 - - '@scure/bip32@2.2.0': - dependencies: - '@noble/curves': 2.2.0 - '@noble/hashes': 2.2.0 - '@scure/base': 2.2.0 - - '@scure/bip39@1.6.0': - dependencies: - '@noble/hashes': 1.8.0 - '@scure/base': 1.2.6 - - '@scure/bip39@2.2.0': - dependencies: - '@noble/hashes': 2.2.0 - '@scure/base': 2.2.0 - - '@tangle-network/agent-core@0.3.4': - dependencies: - '@tangle-network/agent-interface': 0.14.0 - zod: 4.4.3 - - '@tangle-network/agent-core@0.4.9': - dependencies: - '@tangle-network/agent-interface': 0.24.0 - zod: 4.4.3 - - '@tangle-network/agent-eval@0.117.1(typescript@6.0.3)': - dependencies: - '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) - '@ax-llm/ax': 19.0.45(zod@4.4.3) - '@hono/node-server': 2.0.4(hono@4.12.30) - '@tangle-network/agent-interface': 0.22.0 - '@tangle-network/tcloud': 0.4.14(typescript@6.0.3)(zod@4.4.3) - hono: 4.12.30 - zod: 4.4.3 - transitivePeerDependencies: - - '@mastra/core' - - '@modelcontextprotocol/sdk' - - ai - - bufferutil - - openai - - typescript - - utf-8-validate - - '@tangle-network/agent-interface@0.13.0': - dependencies: - zod: 4.4.3 - - '@tangle-network/agent-interface@0.14.0': - dependencies: - zod: 4.4.3 - - '@tangle-network/agent-interface@0.21.0': - dependencies: - zod: 4.4.3 - - '@tangle-network/agent-interface@0.22.0': - dependencies: - zod: 4.4.3 - - '@tangle-network/agent-interface@0.24.0': - dependencies: - zod: 4.4.3 - - '@tangle-network/agent-interface@0.25.0': - dependencies: - zod: 4.4.3 - - '@tangle-network/agent-knowledge@1.12.1(typescript@6.0.3)': - dependencies: - '@tangle-network/agent-eval': 0.117.1(typescript@6.0.3) - zod: 4.4.3 - transitivePeerDependencies: - - '@mastra/core' - - '@modelcontextprotocol/sdk' - - ai - - bufferutil - - openai - - typescript - - utf-8-validate - - '@tangle-network/agent-profile-materialize@0.3.2': - dependencies: - '@tangle-network/agent-interface': 0.25.0 - - '@tangle-network/agent-runtime@0.94.9(@tangle-network/agent-eval@0.117.1(typescript@6.0.3))(@tangle-network/agent-interface@0.25.0)(@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': - dependencies: - '@tangle-network/agent-eval': 0.117.1(typescript@6.0.3) - '@tangle-network/agent-interface': 0.25.0 - '@tangle-network/agent-knowledge': 1.12.1(typescript@6.0.3) - '@tangle-network/agent-profile-materialize': 0.3.2 - tar-stream: 3.2.0 - optionalDependencies: - '@tangle-network/sandbox': 0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) - transitivePeerDependencies: - - '@mastra/core' - - '@modelcontextprotocol/sdk' - - ai - - bare-abort-controller - - bare-buffer - - bufferutil - - openai - - react-native-b4a - - typescript - - utf-8-validate - - '@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))': - dependencies: - '@tangle-network/agent-core': 0.4.9 - '@tangle-network/agent-interface': 0.21.0 - optionalDependencies: - viem: 2.52.0(typescript@6.0.3)(zod@4.4.3) - - '@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))': - dependencies: - '@tangle-network/agent-core': 0.3.4 - '@tangle-network/agent-interface': 0.13.0 - optionalDependencies: - viem: 2.52.0(typescript@6.0.3)(zod@4.4.3) - - '@tangle-network/tcloud-attestation@0.1.1': {} - - '@tangle-network/tcloud@0.4.14(typescript@6.0.3)(zod@4.4.3)': - dependencies: - '@scure/bip32': 2.2.0 - '@scure/bip39': 2.2.0 - '@tangle-network/sandbox': 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) - '@tangle-network/tcloud-attestation': 0.1.1 - commander: 14.0.3 - viem: 2.52.0(typescript@6.0.3)(zod@4.4.3) - transitivePeerDependencies: - - '@mastra/core' - - '@modelcontextprotocol/sdk' - - ai - - bufferutil - - openai - - typescript - - utf-8-validate - - zod - - '@types/node@25.9.5': - dependencies: - undici-types: 7.24.6 - - abitype@1.2.3(typescript@6.0.3)(zod@4.4.3): - optionalDependencies: - typescript: 6.0.3 - zod: 4.4.3 - - b4a@1.8.1: {} - - bare-events@2.9.1: {} - - bare-fs@4.7.4: - dependencies: - bare-events: 2.9.1 - bare-path: 3.1.1 - bare-stream: 2.13.3(bare-events@2.9.1) - bare-url: 2.4.5 - fast-fifo: 1.3.2 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - - bare-path@3.1.1: {} - - bare-stream@2.13.3(bare-events@2.9.1): - dependencies: - b4a: 1.8.1 - streamx: 2.28.0 - teex: 1.0.1 - optionalDependencies: - bare-events: 2.9.1 - transitivePeerDependencies: - - react-native-b4a - - bare-url@2.4.5: - dependencies: - bare-path: 3.1.1 - - commander@14.0.3: {} - - dayjs@1.11.21: {} - - esbuild@0.28.0: - optionalDependencies: - '@esbuild/aix-ppc64': 0.28.0 - '@esbuild/android-arm': 0.28.0 - '@esbuild/android-arm64': 0.28.0 - '@esbuild/android-x64': 0.28.0 - '@esbuild/darwin-arm64': 0.28.0 - '@esbuild/darwin-x64': 0.28.0 - '@esbuild/freebsd-arm64': 0.28.0 - '@esbuild/freebsd-x64': 0.28.0 - '@esbuild/linux-arm': 0.28.0 - '@esbuild/linux-arm64': 0.28.0 - '@esbuild/linux-ia32': 0.28.0 - '@esbuild/linux-loong64': 0.28.0 - '@esbuild/linux-mips64el': 0.28.0 - '@esbuild/linux-ppc64': 0.28.0 - '@esbuild/linux-riscv64': 0.28.0 - '@esbuild/linux-s390x': 0.28.0 - '@esbuild/linux-x64': 0.28.0 - '@esbuild/netbsd-arm64': 0.28.0 - '@esbuild/netbsd-x64': 0.28.0 - '@esbuild/openbsd-arm64': 0.28.0 - '@esbuild/openbsd-x64': 0.28.0 - '@esbuild/openharmony-arm64': 0.28.0 - '@esbuild/sunos-x64': 0.28.0 - '@esbuild/win32-arm64': 0.28.0 - '@esbuild/win32-ia32': 0.28.0 - '@esbuild/win32-x64': 0.28.0 - - eventemitter3@5.0.1: {} - - events-universal@1.0.1: - dependencies: - bare-events: 2.9.1 - transitivePeerDependencies: - - bare-abort-controller - - fast-fifo@1.3.2: {} - - fsevents@2.3.3: - optional: true - - hono@4.12.30: {} - - isows@1.0.7(ws@8.20.1): - dependencies: - ws: 8.20.1 - - openapi3-ts@4.6.0: - dependencies: - yaml: 2.9.0 - - ox@0.14.27(typescript@6.0.3)(zod@4.4.3): - dependencies: - '@adraffy/ens-normalize': 1.11.1 - '@noble/ciphers': 1.3.0 - '@noble/curves': 1.9.1 - '@noble/hashes': 1.8.0 - '@scure/bip32': 1.7.0 - '@scure/bip39': 1.6.0 - abitype: 1.2.3(typescript@6.0.3)(zod@4.4.3) - eventemitter3: 5.0.1 - optionalDependencies: - typescript: 6.0.3 - transitivePeerDependencies: - - zod - - streamx@2.28.0: - dependencies: - events-universal: 1.0.1 - fast-fifo: 1.3.2 - text-decoder: 1.2.7 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - - tar-stream@3.2.0: - dependencies: - b4a: 1.8.1 - bare-fs: 4.7.4 - fast-fifo: 1.3.2 - streamx: 2.28.0 - transitivePeerDependencies: - - bare-abort-controller - - bare-buffer - - react-native-b4a - - teex@1.0.1: - dependencies: - streamx: 2.28.0 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - - text-decoder@1.2.7: - dependencies: - b4a: 1.8.1 - transitivePeerDependencies: - - react-native-b4a - - tsx@4.22.4: - dependencies: - esbuild: 0.28.0 - optionalDependencies: - fsevents: 2.3.3 - - typescript@6.0.3: {} - - undici-types@7.24.6: {} - - viem@2.52.0(typescript@6.0.3)(zod@4.4.3): - dependencies: - '@noble/curves': 1.9.1 - '@noble/hashes': 1.8.0 - '@scure/bip32': 1.7.0 - '@scure/bip39': 1.6.0 - abitype: 1.2.3(typescript@6.0.3)(zod@4.4.3) - isows: 1.0.7(ws@8.20.1) - ox: 0.14.27(typescript@6.0.3)(zod@4.4.3) - ws: 8.20.1 - optionalDependencies: - typescript: 6.0.3 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - zod - - ws@8.20.1: {} - - yaml@2.9.0: {} - - zod@4.4.3: {} diff --git a/bench/scripts/terminate-pier-trial.mts b/bench/scripts/terminate-pier-trial.mts index 4ea9d489..5c9b40a4 100644 --- a/bench/scripts/terminate-pier-trial.mts +++ b/bench/scripts/terminate-pier-trial.mts @@ -48,7 +48,7 @@ const controller = new FilePierCandidateTrialController({ ...(dockerConnection ? { dockerConnection } : {}), }) const executor = createPierCandidateRecoveryExecutor(controller) -const recovered = await executor.stopAndCapture( +const recovered = await executor.stop( { executionId, executionPlanDigest: executionPlanDigest as `sha256:${string}`, diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index 209261f2..02463758 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -1,5 +1,5 @@ import { execFile } from 'node:child_process' -import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' @@ -7,10 +7,14 @@ import { promisify } from 'node:util' const execFileAsync = promisify(execFile) const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const repoRoot = path.resolve(benchDir, '..') const scratch = await mkdtemp(path.join(tmpdir(), 'agent-bench-consumer-')) -const runtimePackage = process.env.AGENT_RUNTIME_PACKAGE - ? path.resolve(process.env.AGENT_RUNTIME_PACKAGE) - : undefined +const args = new Set(process.argv.slice(2)) +const useLocalRuntime = args.delete('--local-runtime') +if (args.size > 0) throw new Error(`unknown arguments: ${[...args].join(', ')}`) + +const TYPESCRIPT_5 = '5.9.3' +const TYPESCRIPT_6 = '6.0.3' async function run(command, args, cwd, env = process.env) { try { @@ -29,15 +33,38 @@ async function run(command, args, cwd, env = process.env) { } } +async function resolveRuntimePackage(packDir) { + if (process.env.AGENT_RUNTIME_PACKAGE) { + return path.resolve(process.env.AGENT_RUNTIME_PACKAGE) + } + if (!useLocalRuntime) return undefined + const manifest = JSON.parse(await readFile(path.join(repoRoot, 'package.json'), 'utf8')) + if (manifest.name !== '@tangle-network/agent-runtime') { + throw new Error('--local-runtime requires an agent-runtime source workspace') + } + await run('pnpm', ['pack', '--pack-destination', packDir], repoRoot) + const tarballs = (await readdir(packDir)).filter((name) => name.endsWith('.tgz')) + if (tarballs.length !== 1) { + throw new Error(`expected one packed agent-runtime tarball, found ${tarballs.length}`) + } + return path.join(packDir, tarballs[0]) +} + try { const packDir = path.join(scratch, 'pack') + const runtimePackDir = path.join(scratch, 'runtime-pack') const consumerDir = path.join(scratch, 'consumer') await mkdir(packDir) + await mkdir(runtimePackDir) await mkdir(consumerDir) - const packed = await run('npm', ['pack', '--json', '--pack-destination', packDir], benchDir) - const [{ filename }] = JSON.parse(packed.stdout) - const tarball = path.join(packDir, filename) + await run('pnpm', ['pack', '--pack-destination', packDir], benchDir) + const packedFiles = (await readdir(packDir)).filter((name) => name.endsWith('.tgz')) + if (packedFiles.length !== 1) { + throw new Error(`expected one packed agent-bench tarball, found ${packedFiles.length}`) + } + const tarball = path.join(packDir, packedFiles[0]) + const runtimePackage = await resolveRuntimePackage(runtimePackDir) const manifest = JSON.parse(await readFile(path.join(benchDir, 'package.json'), 'utf8')) const devDependencies = manifest.devDependencies if ( @@ -73,7 +100,7 @@ try { }, devDependencies: { '@types/node': devDependencies['@types/node'], - typescript: devDependencies.typescript, + typescript: TYPESCRIPT_5, tsx: devDependencies.tsx, }, }, @@ -128,6 +155,28 @@ for name in sorted(expected): consumerDir, ) await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir) + const typescript5 = await run('npm', ['exec', '--', 'tsc', '--version'], consumerDir) + if (typescript5.stdout.trim() !== `Version ${TYPESCRIPT_5}`) { + throw new Error(`expected TypeScript ${TYPESCRIPT_5}, received ${typescript5.stdout.trim()}`) + } + await run( + 'npm', + [ + 'install', + '--ignore-scripts', + '--no-audit', + '--no-fund', + '--no-save', + '--package-lock=false', + `typescript@${TYPESCRIPT_6}`, + ], + consumerDir, + ) + await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir) + const typescript6 = await run('npm', ['exec', '--', 'tsc', '--version'], consumerDir) + if (typescript6.stdout.trim() !== `Version ${TYPESCRIPT_6}`) { + throw new Error(`expected TypeScript ${TYPESCRIPT_6}, received ${typescript6.stdout.trim()}`) + } await run('npm', ['exec', '--', 'tsx', 'index.ts'], consumerDir) const installedPackage = path.join(consumerDir, 'node_modules', '@tangle-network', 'agent-bench') const prepared = await run( @@ -168,7 +217,7 @@ for name in sorted(expected): }) } console.log( - `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}; prepared ${prepareProof.executionPlanDigest}`, + `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}, TypeScript ${TYPESCRIPT_5} and ${TYPESCRIPT_6}; prepared ${prepareProof.executionPlanDigest}`, ) } finally { await rm(scratch, { recursive: true, force: true }) diff --git a/bench/scripts/verify-pier-agent.mts b/bench/scripts/verify-pier-agent.mts index cd5ae119..fff7ca27 100644 --- a/bench/scripts/verify-pier-agent.mts +++ b/bench/scripts/verify-pier-agent.mts @@ -16,6 +16,10 @@ import path from 'node:path' import { fileURLToPath } from 'node:url' import { canonicalJson, InMemoryTraceStore } from '@tangle-network/agent-eval' +import { + sealCandidateBenchmarkSuite, + sealCandidateBenchmarkTask, +} from '@tangle-network/agent-eval/contract' import type { AgentCandidateArtifactRef, AgentCandidateBundle, @@ -31,6 +35,7 @@ import { type AgentCandidateOutputArtifactPort, type AgentCandidateTaskExecution, type ResolvedAgentCandidateContainer, + sealAgentCandidateBundle, verifyAgentCandidateBundle, } from '@tangle-network/agent-runtime' @@ -83,6 +88,10 @@ function canonicalBytes(value: unknown): Buffer { return Buffer.from(canonicalJson(value), 'utf8') } +function sealCanonical(material: T): T & { digest: Sha256Digest } { + return { ...material, digest: sha256(canonicalBytes(material)) } +} + function embedded(bytes: Uint8Array) { return { encoding: 'base64' as const, @@ -113,13 +122,11 @@ function workspaceSnapshot( }) .sort((left, right) => left.path.localeCompare(right.path)) const material = { - schemaVersion: 1 as const, kind: 'agent-candidate-workspace-manifest' as const, files, - } + } satisfies AgentCandidateWorkspaceSnapshotEvidence['material'] const manifest = canonicalBytes(material) return { - schemaVersion: 1, kind: 'agent-candidate-workspace-snapshot', digest: sha256(manifest), material, @@ -309,7 +316,6 @@ try { `[environment]\ndocker_image = "${pinnedImage}"\nos = "linux"\n`, ), ) - mkdirSync(candidateRoot) mkdirSync(profileRoot) const instruction = readFileSync(path.join(taskDir, 'instruction.md'), 'utf8') @@ -330,7 +336,6 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= const taskWorkspace = workspaceSnapshot(taskRoot, ['src/status.txt']) const candidateWorkspace = workspaceSnapshot(candidateRoot, ['runner.py']) const bundleWithoutDigest = { - schemaVersion: 1 as const, kind: 'agent-candidate-bundle' as const, digestAlgorithm: 'rfc8785-sha256' as const, profile: { @@ -374,25 +379,8 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= }, }, memory: { mode: 'disabled' as const }, - lineage: { - source: 'optimizer' as const, - parentDigests: [`sha256:${'a'.repeat(64)}` as Sha256Digest], - runIds: [`pier-no-model-proposer-${proofArm}`], - benchmark: { - name: 'pier-runtime-proof', - version: '1', - splitDigest: `sha256:${'b'.repeat(64)}` as Sha256Digest, - }, - spend: { - proposal: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, - evaluation: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, - }, - }, } - const bundle = { - ...bundleWithoutDigest, - digest: sha256(canonicalBytes(bundleWithoutDigest)), - } as AgentCandidateBundle + const bundle: AgentCandidateBundle = sealAgentCandidateBundle(bundleWithoutDigest) const container: ResolvedAgentCandidateContainer = { source: 'evaluator-task-container', image: fixtureImage, @@ -408,12 +396,19 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= artifact: graderArtifact, }) const executionId = `pier-no-model-${proofArm}-${contextDigest}` - const task: AgentCandidateTaskExecution = { - executionId, - benchmark: 'pier-runtime-proof', - benchmarkVersion: '1', - taskId: `agent-bench/pier-candidate-no-model-${proofArm}`, - splitDigest: `sha256:${'b'.repeat(64)}`, + const benchmarkTask = sealCandidateBenchmarkTask({ + kind: 'agent-candidate-benchmark-task', + digestAlgorithm: 'rfc8785-sha256', + benchmark: { + name: 'pier-runtime-proof', + version: '1', + splitDigest: `sha256:${'b'.repeat(64)}`, + }, + scenario: { + id: `agent-bench/pier-candidate-no-model-${proofArm}`, + kind: 'coding', + scenarioDigest: sha256(Buffer.from(`pier-runtime-proof:${proofArm}`, 'utf8')), + }, instruction, repository: { identity: 'github.com/tangle-network/agent-bench-pier-fixture', @@ -421,15 +416,21 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= baseCommit: repositoryState.commit, baseTree: repositoryState.tree, }, - attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, - model: { requested: modelRequest, reasoningEffort: 'xhigh' }, + outcome: { kind: 'workspace' }, + attempt: { maxAttempts: 1, retryPolicy: 'none' }, + model: { + requested: modelRequest, + provider: 'openai', + model: 'gpt-5.4', + snapshot: 'gpt-5.4-no-model-proof', + reasoningEffort: 'xhigh', + }, grader: { name: grader.name, version: grader.version, + format: 'tangle-grader', artifact: grader.artifact, }, - executionRoots: { taskRoot: '/app', candidateRoot: '/opt/tangle-candidate' }, - stagingRoots: { taskRoot, candidateRoot, profileRoot }, workspace: taskWorkspace, evaluatorTaskContainer: container, limits: { @@ -440,6 +441,30 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= maxOutputTokens: 0, maxCostUsd: 0, }, + }) + const benchmark = sealCandidateBenchmarkSuite({ + tasks: [benchmarkTask], + reps: 1, + seeds: [42], + }) + const task: AgentCandidateTaskExecution = { + executionId, + runCell: sealCanonical({ + kind: 'agent-candidate-run-cell' as const, + experimentDigest: sha256(Buffer.from('pier-runtime-proof-experiment', 'utf8')), + arm: 'candidate' as const, + bundleDigest: bundle.digest, + suiteDigest: benchmark.suite.digest, + taskDigest: benchmarkTask.digest, + taskIndex: 0, + repetition: 0, + seed: 42, + attempt: 1, + }), + benchmarkSuite: benchmark.suite, + task: benchmarkTask, + executionRoots: { taskRoot: '/app', candidateRoot: '/opt/tangle-candidate' }, + stagingRoots: { taskRoot, candidateRoot, profileRoot }, } const workspaces = createAgentCandidateWorkspacePort() const ports: AgentCandidateExecutionPorts = { @@ -512,7 +537,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= prepared: true, disposed: true, executionPlanDigest: prepared.executionPlan.value.digest, - graderDigest: task.grader.artifact.sha256, + graderDigest: benchmarkTask.grader.artifact.sha256, })}\n`, ) } else { @@ -525,6 +550,17 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= const jobName = `tangle-runtime-candidate-no-model-${proofArm}` const controller = new FilePierCandidateTrialController({ directory: path.join(scratch, 'trial-control'), + readResult: async ({ jobsDirectory, jobName }) => { + const trialResult = findTrialResult(path.join(jobsDirectory, jobName)) + if (!trialResult) return undefined + return { + value: trialResult.value, + resultBytes: readFileSync(trialResult.path), + taskPatch: readFileSync( + path.join(path.dirname(trialResult.path), 'artifacts', 'model.patch'), + ), + } + }, launch: (staged, { request }) => { const evaluatorArgs = Object.keys(staged.evaluatorEnv).flatMap((name) => [ '--agent-env', @@ -590,7 +626,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= const endedAt = Date.now() await traceStore.appendRun({ runId: request.trace.runId, - scenarioId: task.taskId, + scenarioId: benchmarkTask.scenario.id, startedAt: endedAt - observedElapsedMs, endedAt, status: 'completed', diff --git a/bench/src/pier-agent.test.mts b/bench/src/pier-agent.test.mts index 61d3bae6..f664030b 100644 --- a/bench/src/pier-agent.test.mts +++ b/bench/src/pier-agent.test.mts @@ -23,10 +23,21 @@ const digest = (bytes: Uint8Array): `sha256:${string}` => function prepared(root: string): PreparedAgentCandidateExecution { const bundleDigest = `sha256:${'b'.repeat(64)}` as const const executionId = 'pier-test-execution' + const task = { + outcome: { + kind: 'workspace', + repository: { + identity: 'fixture/repository', + rootIdentity: 'fixture/repository', + baseCommit: '1'.repeat(40), + baseTree: '2'.repeat(40), + }, + }, + } const material = { schemaVersion: 1, kind: 'agent-candidate-execution-plan-material', - bundleDigest, + runCell: { bundleDigest }, executionId, limits: { timeoutMs: 60_000, @@ -51,6 +62,7 @@ function prepared(root: string): PreparedAgentCandidateExecution { const profileBytes = Buffer.from('fixture profile\n') return { bundle: { digest: bundleDigest }, + benchmark: { task }, executionId, roots: { execution: { taskRoot: '/app' }, @@ -78,6 +90,9 @@ function prepared(root: string): PreparedAgentCandidateExecution { bytes: Buffer.from('{}'), written: ['AGENTS.md'], }, + profileActivation: { + files: [{ path: 'AGENTS.md', mode: 0o644, content: profileBytes.toString('utf8') }], + }, materializationReceipt: { value: { ...receiptMaterial, digest: receiptDigest }, bytes: receiptBytes, @@ -133,12 +148,10 @@ function request(execution: PreparedAgentCandidateExecution): AgentCandidateExec }, files: [{ path: 'src/status.txt', mode: 0o644, bytes: taskBytes }], }, - profile: { - files: [{ path: 'AGENTS.md', mode: 0o644, bytes: profileBytes }], - }, }, roots: execution.roots.execution, profilePlan: execution.profilePlan, + profileActivation: execution.profileActivation, executionPlan: execution.executionPlan, materializationReceipt: execution.materializationReceipt, launch: { diff --git a/bench/src/pier-agent.ts b/bench/src/pier-agent.ts index efd48343..0d1c1495 100644 --- a/bench/src/pier-agent.ts +++ b/bench/src/pier-agent.ts @@ -16,7 +16,7 @@ import type { PreparedAgentCandidateExecution, } from '@tangle-network/agent-runtime' import { executePreparedAgentCandidate } from '@tangle-network/agent-runtime' -import type { TraceStore } from '@tangle-network/agent-eval' +import { canonicalJson, type TraceStore } from '@tangle-network/agent-eval' import { capturePierTaskOutcome } from './pier-task-outcome' @@ -81,6 +81,8 @@ export interface PierCandidateTrialController { terminateAndWait( identity: PierCandidateTrialIdentity, ): Promise + /** Read immutable official bytes after termination; undefined proves no result was emitted. */ + captureResult(identity: PierCandidateTrialIdentity): Promise } /** Evaluator-owned bytes captured from one completed official Pier trial. */ @@ -132,7 +134,7 @@ export function createPierCandidateRecoveryExecutor( execute: async () => { throw new Error('recovery-only Pier executor cannot start a candidate') }, - stopAndCapture: async (request) => { + stop: async (request) => { assertTerminationAcknowledged( await controller.terminateAndWait({ executionId: request.executionId, @@ -141,6 +143,21 @@ export function createPierCandidateRecoveryExecutor( ) return { stopped: true } }, + capture: async (request) => { + const result = await controller.captureResult(request) + if (!result) return {} + return { + evidence: Buffer.from( + canonicalJson({ + schemaVersion: 1, + kind: 'pier-candidate-recovery-capture', + executionPlanDigest: request.executionPlanDigest, + officialResult: Buffer.from(result.resultBytes).toString('base64'), + taskPatch: Buffer.from(result.taskPatch).toString('base64'), + }), + ), + } + }, } } @@ -294,9 +311,6 @@ export async function stagePreparedPierCandidateExecution( if (request.memory.mode !== 'disabled') { throw new Error('Pier transport does not yet implement protected isolated memory') } - if (request.knowledge !== undefined) { - throw new Error('Pier transport does not yet implement immutable knowledge mounts') - } const taskDirectory = join(directory, 'task') const candidateDirectory = request.inputs.candidate ? join(directory, 'candidate') : undefined const profileDirectory = join(directory, 'profile') @@ -316,7 +330,11 @@ export async function stagePreparedPierCandidateExecution( } await materializeExecutorFiles( profileDirectory, - request.inputs.profile.files, + request.profileActivation.files.map((file) => ({ + path: file.path, + mode: file.mode, + bytes: Buffer.from(file.content, 'utf8'), + })), request.profilePlan.value.material.files.map((file) => ({ path: file.relPath, mode: file.mode, @@ -413,7 +431,7 @@ export function protectedCaptureFromPierResult( const metadata = record(agentResult.metadata, 'Pier agent_result.metadata') const expected = { executionId: request.executionId, - bundleDigest: request.executionPlan.value.material.bundleDigest, + bundleDigest: request.executionPlan.value.material.runCell.bundleDigest, executionPlanDigest: request.executionPlan.value.digest, materializationReceiptDigest: request.materializationReceipt.digest, } @@ -565,39 +583,47 @@ export async function executePreparedPierCandidate( officialResults.set(request.executionId, result) return protectedCaptureFromPierResult(request, result.value) }, - stopAndCapture: async (request) => { - const identity = trialIdentity(request.executionId, request.executionPlanDigest) - const active = trials.get(identity) + stop: async (request) => { assertTerminationAcknowledged( await options.controller.terminateAndWait({ executionId: request.executionId, executionPlanDigest: request.executionPlanDigest, }), ) - if (!active) return { stopped: true } - let result = active.result - if (!result) { + return { stopped: true } + }, + capture: async (request) => { + const identity = trialIdentity(request.executionId, request.executionPlanDigest) + const active = trials.get(identity) + let result = active?.result + if (!result && active) { try { result = sealPierTrialResult(await active.handle.result) } catch { result = undefined } } - trials.delete(identity) - if (!result) return { stopped: true } + if (!result) { + const recovered = await options.controller.captureResult(request) + result = recovered ? sealPierTrialResult(recovered) : undefined + } + if (!result) return {} officialResults.set(request.executionId, result) - const repository = options.prepared.executionPlan.value.material.task.repository + const task = options.prepared.benchmark.task + const outcome = task.outcome + if (outcome.kind !== 'workspace') { + throw new Error('Pier candidate execution requires a workspace task outcome') + } + const repository = task.repository + if (!repository) throw new Error('Pier workspace task is missing repository identity') const taskOutcome = await capturePierTaskOutcome({ repositoryRoot: options.prepared.roots.staging.taskRoot, baseCommit: repository.baseCommit, baseTree: repository.baseTree, patch: result.taskPatch, - artifactPersistence: { - executionId: request.executionId, - outputArtifacts: options.outputArtifacts, - }, }) - return { stopped: true, taskOutcome } + trials.delete(identity) + return { taskOutcome } }, } const grader: AgentCandidateBenchmarkGraderPort = { diff --git a/bench/src/pier-task-outcome.test.mts b/bench/src/pier-task-outcome.test.mts index 2c6bcef6..933ff052 100644 --- a/bench/src/pier-task-outcome.test.mts +++ b/bench/src/pier-task-outcome.test.mts @@ -1,13 +1,11 @@ import assert from 'node:assert/strict' import { execFileSync } from 'node:child_process' -import { createHash } from 'node:crypto' import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import test from 'node:test' import { - type AgentCandidateOutputArtifactPort, captureAgentCandidateWorkspaceFiles, createAgentCandidateWorkspacePort, } from '@tangle-network/agent-runtime/candidate-execution' @@ -77,7 +75,7 @@ test('Pier task outcome preserves the signed base for an empty patch', async () } }) -test('Pier task outcome externalizes workspace evidence above the embedded limit', async () => { +test('Pier task outcome returns large workspace bytes for the runtime capture path', async () => { const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-')) try { const payload = Buffer.alloc(1024 * 1024 + 1, 7) @@ -89,45 +87,20 @@ test('Pier task outcome externalizes workspace evidence above the embedded limit git(root, ['commit', '-m', 'base']) const baseCommit = git(root, ['rev-parse', 'HEAD']) const baseTree = git(root, ['rev-parse', 'HEAD^{tree}']) - const stored = new Map() - const outputArtifacts: AgentCandidateOutputArtifactPort = { - put: async ({ bytes, purpose }) => { - const detached = Uint8Array.from(bytes) - const digest = sha256(detached) - stored.set(digest, detached) - return { - locator: { - kind: 's3', - bucket: 'pier-test-artifacts', - key: `${purpose}/${digest}`, - }, - sha256: digest, - byteLength: detached.byteLength, - } - }, - read: async (ref) => Uint8Array.from(stored.get(ref.sha256) ?? []), - } - const captured = await capturePierTaskOutcome({ repositoryRoot: root, baseCommit, baseTree, patch: Buffer.alloc(0), - artifactPersistence: { executionId: 'pier-large-1', outputArtifacts }, }) assert.ok(captured.archive.byteLength > 1024 * 1024) assert.equal(captured.afterState.files[0]?.byteLength, payload.byteLength) - assert.deepEqual(Buffer.from(stored.get(sha256(captured.archive)) ?? []), captured.archive) } finally { rmSync(root, { recursive: true, force: true }) } }) -function sha256(bytes: Uint8Array): `sha256:${string}` { - return `sha256:${createHash('sha256').update(bytes).digest('hex')}` -} - function git(root: string, args: string[]): string { return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', diff --git a/bench/src/pier-task-outcome.ts b/bench/src/pier-task-outcome.ts index 9ae8b4c1..f9f4d0e1 100644 --- a/bench/src/pier-task-outcome.ts +++ b/bench/src/pier-task-outcome.ts @@ -6,7 +6,6 @@ import { join, resolve } from 'node:path' import { type AgentCandidateExecutorTaskOutcomeCapture, - type AgentCandidateOutputArtifactPort, captureAgentCandidateWorkspaceFiles, } from '@tangle-network/agent-runtime/candidate-execution' @@ -22,11 +21,7 @@ export async function capturePierTaskOutcome(input: { readonly baseTree: string readonly patch: Uint8Array readonly signal?: AbortSignal - readonly artifactPersistence?: { - readonly executionId: string - readonly outputArtifacts: AgentCandidateOutputArtifactPort - } -}): Promise { +}): Promise> { input.signal?.throwIfAborted() const repositoryRoot = resolve(input.repositoryRoot) const head = (await git(repositoryRoot, ['rev-parse', 'HEAD'], undefined, {}, input.signal)).stdout @@ -124,16 +119,10 @@ export async function capturePierTaskOutcome(input: { // The executor outcome must still return exact bytes; the runtime verifies // and persists its final task references after this capture completes. const captured = await captureAgentCandidateWorkspaceFiles(files, { - ...(input.artifactPersistence - ? { - artifactPersistence: { - ...input.artifactPersistence, - ...(input.signal ? { signal: input.signal } : {}), - }, - } - : {}), + limits: { maxEmbeddedArtifactBytes: 512 * 1024 * 1024 }, }) return { + kind: 'workspace', resultTree, afterState: captured.snapshot.material, archive: captured.archive, diff --git a/bench/src/pier-trial-controller.ts b/bench/src/pier-trial-controller.ts index c818098a..dfbe621c 100644 --- a/bench/src/pier-trial-controller.ts +++ b/bench/src/pier-trial-controller.ts @@ -107,6 +107,11 @@ export interface FilePierCandidateTrialControllerOptions { readonly deadlineAtMs: number }, ) => PierCandidateProcessSpec + /** Restart-safe reader for immutable official bytes under the persisted Pier job identity. */ + readonly readResult?: (input: { + readonly jobsDirectory: string + readonly jobName: string + }) => Promise /** Omit only for the default local Docker socket with no environment variables. */ readonly dockerConnection?: PierDockerConnection readonly supervisorPath?: string @@ -496,6 +501,7 @@ function assertRealDirectory(path: string, label: string): void { export class FilePierCandidateTrialController implements PierCandidateTrialController { private readonly directory: string private readonly launch?: NonNullable + private readonly readResult?: NonNullable private readonly dockerConnection: PierDockerConnection private readonly supervisorPath: string private readonly pollIntervalMs: number @@ -503,6 +509,7 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr constructor(options: FilePierCandidateTrialControllerOptions) { this.directory = absolutePath(options.directory, 'Pier controller directory') this.launch = options.launch + this.readResult = options.readResult const configuredDocker = options.dockerConnection if (configuredDocker?.id === defaultDockerConnectionId) { throw new Error(`${defaultDockerConnectionId} is reserved for the implicit local connection`) @@ -717,6 +724,32 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr return { processExited: true, containersRemoved: true } } + async captureResult( + requested: PierCandidateTrialIdentity, + ): Promise { + const controlDirectory = this.controlDirectory(requested) + assertRealDirectory(controlDirectory, 'Pier trial control directory') + const persisted = parsePersistedIdentity(join(controlDirectory, identityFile)) + if ( + persisted.executionId !== requested.executionId || + persisted.executionPlanDigest !== requested.executionPlanDigest + ) { + throw new Error('persisted Pier trial identity differs from the capture request') + } + this.assertDockerConnection(persisted) + const terminal = parseTerminal(join(controlDirectory, terminalFile)) + if (!terminal.processExited || !terminal.containersRemoved) { + throw new Error('Pier result capture requires proven process and container death') + } + if (!this.readResult) { + throw new Error('Pier result capture has no restart-safe result reader') + } + return await this.readResult({ + jobsDirectory: persisted.jobsDirectory, + jobName: persisted.jobName, + }) + } + private controlDirectory(identity: PierCandidateTrialIdentity): string { return join(this.directory, trialKey(identity)) } diff --git a/docs/api/agent.md b/docs/api/agent.md index 8f3eff75..74871d21 100644 --- a/docs/api/agent.md +++ b/docs/api/agent.md @@ -10,7 +10,7 @@ ### AgentManifestError -Defined in: [agent/define-agent.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L309) +Defined in: [agent/define-agent.ts:284](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L284) Thrown when `defineAgent` finds a required surface missing on disk. @@ -24,7 +24,7 @@ Thrown when `defineAgent` finds a required surface missing on disk. > **new AgentManifestError**(`message`, `agentId`, `issues?`): [`AgentManifestError`](#agentmanifesterror) -Defined in: [agent/define-agent.ts:310](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L310) +Defined in: [agent/define-agent.ts:285](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L285) ###### Parameters @@ -54,13 +54,13 @@ readonly `unknown`[] = `[]` > `readonly` **agentId**: `string` -Defined in: [agent/define-agent.ts:312](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L312) +Defined in: [agent/define-agent.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L287) ##### issues > `readonly` **issues**: readonly `unknown`[] = `[]` -Defined in: [agent/define-agent.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L313) +Defined in: [agent/define-agent.ts:288](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L288) ## Interfaces @@ -187,26 +187,11 @@ Defined in: [agent/define-agent.ts:106](https://github.com/tangle-network/agent- Analyst LLM configuration. The substrate uses these for all four kinds (override per-kind via `analystKinds` if needed). -##### autoApply? - -> `optional` **autoApply?**: [`AutoApplyPolicy`](#autoapplypolicy) - -Defined in: [agent/define-agent.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L118) - -Auto-apply policy. Knowledge / improvement edits land only when -`enabled === true` AND the source finding's confidence meets the -threshold. `mode` controls how applies happen: `'write'` mutates -files in-place; `'open-pr'` writes to a branch and opens a PR. - -Default: knowledge auto-applies at confidence ≥0.85 in `'write'` -mode (wiki edits are git-reversible); improvement stays at -`enabled: false` until the agent author has measured precision. - ##### lifecycles? > `optional` **lifecycles?**: readonly [`SurfaceLifecycle`](#surfacelifecycle)[] -Defined in: [agent/define-agent.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L134) +Defined in: [agent/define-agent.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L122) Declarative per-surface artifact-lifecycle config the closed loop reads. @@ -225,7 +210,7 @@ absent map means "no lifecycle"; the manifest is otherwise unchanged. ### SurfaceLifecycle -Defined in: [agent/define-agent.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L142) +Defined in: [agent/define-agent.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L130) One profile surface's artifact-lifecycle wiring — the declarative config a `defineAgent` manifest carries and `runLifecycle` reads. It is config, not @@ -237,7 +222,7 @@ execution: it names the generator + gate; the loop runs them. > **surface**: [`ArtifactKind`](lifecycle.md#artifactkind) -Defined in: [agent/define-agent.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L144) +Defined in: [agent/define-agent.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L132) The profile surface this lifecycle grows. @@ -245,7 +230,7 @@ The profile surface this lifecycle grows. > **generator**: [`CandidateGenerator`](lifecycle.md#candidategenerator) -Defined in: [agent/define-agent.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L146) +Defined in: [agent/define-agent.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L134) Produces fresh candidate artifacts for `surface` from the agent's history. @@ -253,7 +238,7 @@ Produces fresh candidate artifacts for `surface` from the agent's history. > **gate**: [`PromotionGate`](lifecycle.md#promotiongate) -Defined in: [agent/define-agent.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L148) +Defined in: [agent/define-agent.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L136) The held-back exam that decides promotion of a measured candidate. @@ -261,7 +246,7 @@ The held-back exam that decides promotion of a measured candidate. > `optional` **composeK?**: `number` -Defined in: [agent/define-agent.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L151) +Defined in: [agent/define-agent.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L139) Top-`k` budget for `composeProfile` when folding this surface's promoted artifacts back in. Omit to fold in every active artifact. @@ -270,7 +255,7 @@ Top-`k` budget for `composeProfile` when folding this surface's promoted ### AgentRubric -Defined in: [agent/define-agent.ts:154](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L154) +Defined in: [agent/define-agent.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L142) `@tangle-network/agent-runtime/agent` — declarative agent manifest + substrate-default adapters. @@ -292,7 +277,7 @@ No per-vertical glue. No fabricated paths. No theater. > **dimensions**: readonly [`RubricDimension`](#rubricdimension)\<`TRunOutput`\>[] -Defined in: [agent/define-agent.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L156) +Defined in: [agent/define-agent.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L144) Dimensions composing the weighted score. Weights sum to 1.0 by convention. @@ -300,7 +285,7 @@ Dimensions composing the weighted score. Weights sum to 1.0 by convention. > `optional` **judges?**: readonly [`JudgeConfig`](#judgeconfig)\<`TRunOutput`\>[] -Defined in: [agent/define-agent.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L162) +Defined in: [agent/define-agent.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L150) Optional judges layered on top of deterministic dimensions. Each judge returns a score per dimension; the substrate averages judges @@ -310,7 +295,7 @@ judge returns a score per dimension; the substrate averages judges ### RubricDimension -Defined in: [agent/define-agent.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L165) +Defined in: [agent/define-agent.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L153) `@tangle-network/agent-runtime/agent` — declarative agent manifest + substrate-default adapters. @@ -332,7 +317,7 @@ No per-vertical glue. No fabricated paths. No theater. > **id**: `string` -Defined in: [agent/define-agent.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L167) +Defined in: [agent/define-agent.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L155) Unique identifier — appears in finding subjects (`rubric:`). @@ -340,7 +325,7 @@ Unique identifier — appears in finding subjects (`rubric:`). > **weight**: `number` -Defined in: [agent/define-agent.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L169) +Defined in: [agent/define-agent.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L157) 0..1 — weight in the composite. @@ -348,7 +333,7 @@ Defined in: [agent/define-agent.ts:169](https://github.com/tangle-network/agent- > **score**: (`input`) => `number` -Defined in: [agent/define-agent.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L175) +Defined in: [agent/define-agent.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L163) Deterministic scorer: given the persona + run output, returns a 0..1 score. The substrate sums weight × score across dimensions @@ -374,7 +359,7 @@ for the deterministic composite; judges supplement subjective dims. > `optional` **label?**: `string` -Defined in: [agent/define-agent.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L177) +Defined in: [agent/define-agent.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L165) Optional human-readable label for reports. @@ -382,7 +367,7 @@ Optional human-readable label for reports. ### JudgeConfig -Defined in: [agent/define-agent.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L180) +Defined in: [agent/define-agent.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L168) `@tangle-network/agent-runtime/agent` — declarative agent manifest + substrate-default adapters. @@ -404,7 +389,7 @@ No per-vertical glue. No fabricated paths. No theater. > **id**: `string` -Defined in: [agent/define-agent.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L182) +Defined in: [agent/define-agent.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L170) Judge identifier — appears in trace spans + manifest. @@ -412,7 +397,7 @@ Judge identifier — appears in trace spans + manifest. > **model**: `string` -Defined in: [agent/define-agent.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L184) +Defined in: [agent/define-agent.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L172) Model snapshot to invoke. Pin the snapshot (`claude-sonnet-4-6@2025-04-15`); the validator rejects bare aliases. @@ -420,7 +405,7 @@ Model snapshot to invoke. Pin the snapshot (`claude-sonnet-4-6@2025-04-15`); the > **dimensions**: readonly `string`[] -Defined in: [agent/define-agent.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L186) +Defined in: [agent/define-agent.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L174) Dimensions this judge scores. @@ -428,7 +413,7 @@ Dimensions this judge scores. > `optional` **anchors?**: readonly `object`[] -Defined in: [agent/define-agent.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L192) +Defined in: [agent/define-agent.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L180) Optional rubric anchors — text examples the judge sees as a few-shot prompt to calibrate. STRONGLY recommended for subjective @@ -438,7 +423,7 @@ dimensions; required by the calibration gate (Pearson ≥0.7). ### AgentRuntime -Defined in: [agent/define-agent.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L195) +Defined in: [agent/define-agent.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L183) `@tangle-network/agent-runtime/agent` — declarative agent manifest + substrate-default adapters. @@ -464,7 +449,7 @@ No per-vertical glue. No fabricated paths. No theater. > **act**: (`persona`, `ctx`) => [`AgentRunInvocation`](#agentruninvocation)\<`TRunOutput`\> -Defined in: [agent/define-agent.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L220) +Defined in: [agent/define-agent.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L208) Invoke the agent against one persona. Returns BOTH: - `events`: an `AsyncIterable` the chat-centric @@ -507,7 +492,7 @@ cancel. `ctx.signal` is the standard abort signal. ### AgentRunInvocation -Defined in: [agent/define-agent.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L223) +Defined in: [agent/define-agent.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L211) `@tangle-network/agent-runtime/agent` — declarative agent manifest + substrate-default adapters. @@ -529,7 +514,7 @@ No per-vertical glue. No fabricated paths. No theater. > **events**: `AsyncIterable`\<[`RuntimeStreamEvent`](index.md#runtimestreamevent)\> -Defined in: [agent/define-agent.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L225) +Defined in: [agent/define-agent.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L213) Live stream of typed runtime events. Consumed by chat UX directly. @@ -537,7 +522,7 @@ Live stream of typed runtime events. Consumed by chat UX directly. > **output**: `Promise`\<`TRunOutput`\> -Defined in: [agent/define-agent.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L227) +Defined in: [agent/define-agent.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L215) Final structured output the rubric scores. Resolves after `events` drains. @@ -545,7 +530,7 @@ Final structured output the rubric scores. Resolves after `events` drains. ### AgentRunContext -Defined in: [agent/define-agent.ts:267](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L267) +Defined in: [agent/define-agent.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L255) `@tangle-network/agent-runtime/agent` — declarative agent manifest + substrate-default adapters. @@ -561,7 +546,7 @@ No per-vertical glue. No fabricated paths. No theater. > **emitter**: `TraceEmitter` -Defined in: [agent/define-agent.ts:269](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L269) +Defined in: [agent/define-agent.ts:257](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L257) Substrate-managed trace emitter. @@ -569,7 +554,7 @@ Substrate-managed trace emitter. > **runId**: `string` -Defined in: [agent/define-agent.ts:271](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L271) +Defined in: [agent/define-agent.ts:259](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L259) Stable run id for this persona × variant cell. @@ -577,7 +562,7 @@ Stable run id for this persona × variant cell. > `optional` **variantId?**: `string` -Defined in: [agent/define-agent.ts:273](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L273) +Defined in: [agent/define-agent.ts:261](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L261) Variant the runtime is exercising (e.g. `'baseline'`, `'source-grounded'`). @@ -585,7 +570,7 @@ Variant the runtime is exercising (e.g. `'baseline'`, `'source-grounded'`). > `optional` **deadlineMs?**: `number` -Defined in: [agent/define-agent.ts:275](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L275) +Defined in: [agent/define-agent.ts:263](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L263) Wall-clock deadline (epoch ms). The runtime SHOULD honour for graceful cancel. @@ -593,7 +578,7 @@ Wall-clock deadline (epoch ms). The runtime SHOULD honour for graceful cancel. > `optional` **signal?**: `AbortSignal` -Defined in: [agent/define-agent.ts:277](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L277) +Defined in: [agent/define-agent.ts:265](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L265) Optional abort signal. @@ -601,7 +586,7 @@ Optional abort signal. ### AnalystConfig -Defined in: [agent/define-agent.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L280) +Defined in: [agent/define-agent.ts:268](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L268) `@tangle-network/agent-runtime/agent` — declarative agent manifest + substrate-default adapters. @@ -617,7 +602,7 @@ No per-vertical glue. No fabricated paths. No theater. > **model**: `string` -Defined in: [agent/define-agent.ts:282](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L282) +Defined in: [agent/define-agent.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L270) Model the analyst kinds use. Override per-kind via `analystKinds[i].cost.models`. @@ -625,7 +610,7 @@ Model the analyst kinds use. Override per-kind via `analystKinds[i].cost.models` > `optional` **budgetUsd?**: `number` -Defined in: [agent/define-agent.ts:284](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L284) +Defined in: [agent/define-agent.ts:272](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L272) Optional total budget across all kinds for one run. Substrate enforces via `BudgetGuard`. @@ -633,7 +618,7 @@ Optional total budget across all kinds for one run. Substrate enforces via `Budg > `optional` **backend?**: `object` -Defined in: [agent/define-agent.ts:286](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L286) +Defined in: [agent/define-agent.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L274) Backend hint for the AxAIService factory — same shape every kind uses. @@ -651,58 +636,6 @@ Backend hint for the AxAIService factory — same shape every kind uses. *** -### AutoApplyPolicy - -Defined in: [agent/define-agent.ts:293](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L293) - -`@tangle-network/agent-runtime/agent` — declarative agent manifest + -substrate-default adapters. - -Every vertical agent (tax / legal / gtm / creative / N future -verticals) ships ONE `defineAgent({...})` call + a thin invocation -of `runAnalystLoop` wired through the substrate-default adapters. -No per-vertical glue. No fabricated paths. No theater. - -#### Properties - -##### knowledge? - -> `optional` **knowledge?**: `object` - -Defined in: [agent/define-agent.ts:294](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L294) - -###### enabled - -> **enabled**: `boolean` - -###### confidenceThreshold? - -> `optional` **confidenceThreshold?**: `number` - -###### mode? - -> `optional` **mode?**: `"write"` \| `"open-pr"` - -##### improvement? - -> `optional` **improvement?**: `object` - -Defined in: [agent/define-agent.ts:299](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L299) - -###### enabled - -> **enabled**: `boolean` - -###### confidenceThreshold? - -> `optional` **confidenceThreshold?**: `number` - -###### mode? - -> `optional` **mode?**: `"write"` \| `"open-pr"` - -*** - ### SurfaceImprovementEdit Defined in: [agent/improvement-adapter.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/improvement-adapter.ts#L43) @@ -1745,7 +1678,7 @@ Materialization contract for a run path that injects prompt text plus inline res > **unimplementedAgentRun**\<`TRunOutput`\>(`reason?`): [`AgentRunInvocation`](#agentruninvocation)\<`TRunOutput`\> -Defined in: [agent/define-agent.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L239) +Defined in: [agent/define-agent.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L227) Stub for agents whose `runtime.act` is not yet wired to the substrate's eval path. Preserves the streaming contract (empty event stream + a @@ -1777,7 +1710,7 @@ the eval path consumes the manifest end-to-end. > **collectAgentRun**\<`TRunOutput`\>(`invocation`): `Promise`\<\{ `events`: readonly [`RuntimeStreamEvent`](index.md#runtimestreamevent)[]; `output`: `TRunOutput`; \}\> -Defined in: [agent/define-agent.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L258) +Defined in: [agent/define-agent.ts:246](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L246) Drain `act`'s `events` into an array AND await its `output`. Useful for eval / outcome-measurement code paths that don't care about live @@ -1810,7 +1743,7 @@ directly in the chat surface. > **defineAgent**\<`TPersona`, `TRunOutput`\>(`manifest`): [`AgentManifest`](#agentmanifest)\<`TPersona`, `TRunOutput`\> -Defined in: [agent/define-agent.ts:333](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L333) +Defined in: [agent/define-agent.ts:308](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/define-agent.ts#L308) Construct a validated agent manifest. Throws `AgentManifestError` if any required surface is missing on disk. diff --git a/docs/api/candidate-execution.md b/docs/api/candidate-execution.md index deaf0c9f..ae948607 100644 --- a/docs/api/candidate-execution.md +++ b/docs/api/candidate-execution.md @@ -140,12 +140,6 @@ Re-exports [AgentCandidateExecutionTerminalResult](index.md#agentcandidateexecut *** -### AgentCandidateExecutionUsage - -Re-exports [AgentCandidateExecutionUsage](index.md#agentcandidateexecutionusage) - -*** - ### AgentCandidateRetryRejection Re-exports [AgentCandidateRetryRejection](index.md#agentcandidateretryrejection) @@ -158,6 +152,12 @@ Re-exports [InMemoryAgentCandidateExecutionClaimStore](index.md#inmemoryagentcan *** +### AgentCandidatePreparationEvidence + +Re-exports [AgentCandidatePreparationEvidence](index.md#agentcandidatepreparationevidence) + +*** + ### FileAgentCandidateExecutionClaimStore Re-exports [FileAgentCandidateExecutionClaimStore](index.md#fileagentcandidateexecutionclaimstore) @@ -200,6 +200,24 @@ Re-exports [executePreparedAgentCandidate](index.md#executepreparedagentcandidat *** +### CANDIDATE\_KNOWLEDGE\_RETRIEVAL\_CONFIG\_ENV + +Re-exports [CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV](index.md#candidate_knowledge_retrieval_config_env) + +*** + +### CANDIDATE\_KNOWLEDGE\_ROOT\_ENV + +Re-exports [CANDIDATE_KNOWLEDGE_ROOT_ENV](index.md#candidate_knowledge_root_env) + +*** + +### candidateKnowledgeExecutionPaths + +Re-exports [candidateKnowledgeExecutionPaths](index.md#candidateknowledgeexecutionpaths) + +*** + ### persistCandidateOutputArtifact Re-exports [persistCandidateOutputArtifact](index.md#persistcandidateoutputartifact) @@ -296,12 +314,6 @@ Re-exports [AgentCandidateArtifactPort](index.md#agentcandidateartifactport) *** -### AgentCandidateBenchmarkGraderIdentity - -Re-exports [AgentCandidateBenchmarkGraderIdentity](index.md#agentcandidatebenchmarkgraderidentity) - -*** - ### AgentCandidateBenchmarkGraderPort Re-exports [AgentCandidateBenchmarkGraderPort](index.md#agentcandidatebenchmarkgraderport) @@ -416,12 +428,6 @@ Re-exports [AgentCandidateProtectedModelActivation](index.md#agentcandidateprote *** -### AgentCandidateProtectedModelCall - -Re-exports [AgentCandidateProtectedModelCall](index.md#agentcandidateprotectedmodelcall) - -*** - ### AgentCandidateProtectedModelReservation Re-exports [AgentCandidateProtectedModelReservation](index.md#agentcandidateprotectedmodelreservation) @@ -536,6 +542,12 @@ Re-exports [VerifiedAgentCandidateTaskOutcome](index.md#verifiedagentcandidateta *** +### AGENT\_CANDIDATE\_EXECUTION\_SUPPORT + +Re-exports [AGENT_CANDIDATE_EXECUTION_SUPPORT](index.md#agent_candidate_execution_support) + +*** + ### verifyAgentCandidateBundle Re-exports [verifyAgentCandidateBundle](index.md#verifyagentcandidatebundle) diff --git a/docs/api/index.md b/docs/api/index.md index e30dfab8..ba86e36a 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -10,7 +10,7 @@ ### FileAgentCandidateExecutionClaimStore -Defined in: [candidate-execution/claim-file-store.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L74) +Defined in: [candidate-execution/claim-file-store.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L68) Cross-process lifecycle implemented as fsynced, create-if-absent records. @@ -24,7 +24,7 @@ Cross-process lifecycle implemented as fsynced, create-if-absent records. > **new FileAgentCandidateExecutionClaimStore**(`options`): [`FileAgentCandidateExecutionClaimStore`](#fileagentcandidateexecutionclaimstore) -Defined in: [candidate-execution/claim-file-store.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L78) +Defined in: [candidate-execution/claim-file-store.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L72) ###### Parameters @@ -42,7 +42,7 @@ Defined in: [candidate-execution/claim-file-store.ts:78](https://github.com/tang > **tryClaim**(`requested`): `Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> -Defined in: [candidate-execution/claim-file-store.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L86) +Defined in: [candidate-execution/claim-file-store.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L80) ###### Parameters @@ -62,7 +62,7 @@ Defined in: [candidate-execution/claim-file-store.ts:86](https://github.com/tang > **getAttempt**(`requestedAttempt`): `Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> -Defined in: [candidate-execution/claim-file-store.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L115) +Defined in: [candidate-execution/claim-file-store.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L108) ###### Parameters @@ -82,7 +82,7 @@ Defined in: [candidate-execution/claim-file-store.ts:115](https://github.com/tan > **markCandidateMayRun**(`requestedLease`): `Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> -Defined in: [candidate-execution/claim-file-store.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L121) +Defined in: [candidate-execution/claim-file-store.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L114) Persist the point after which candidate code may have run. @@ -104,7 +104,7 @@ Persist the point after which candidate code may have run. > **stageTerminal**(`requestedLease`, `result`): `Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> -Defined in: [candidate-execution/claim-file-store.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L158) +Defined in: [candidate-execution/claim-file-store.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L150) Fsync the complete terminal record into the durable outbox. @@ -130,7 +130,7 @@ Fsync the complete terminal record into the durable outbox. > **finish**(`requestedLease`, `requestedTerminalDigest`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -Defined in: [candidate-execution/claim-file-store.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L191) +Defined in: [candidate-execution/claim-file-store.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L182) Publish exactly the staged terminal identified by `terminalDigest`. @@ -156,7 +156,7 @@ Publish exactly the staged terminal identified by `terminalDigest`. > **recoverExpired**(`requestedAttempt`, `evidence`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -Defined in: [candidate-execution/claim-file-store.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L228) +Defined in: [candidate-execution/claim-file-store.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L218) Write a failed terminal only after the lease expired and a trusted worker independently proved process death plus model and memory closure. @@ -183,7 +183,7 @@ independently proved process death plus model and memory closure. ### InMemoryAgentCandidateExecutionClaimStore -Defined in: [candidate-execution/claim.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L290) +Defined in: [candidate-execution/claim.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L280) Single-process lifecycle implementation. @@ -197,7 +197,7 @@ Single-process lifecycle implementation. > **new InMemoryAgentCandidateExecutionClaimStore**(`options?`): [`InMemoryAgentCandidateExecutionClaimStore`](#inmemoryagentcandidateexecutionclaimstore) -Defined in: [candidate-execution/claim.ts:296](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L296) +Defined in: [candidate-execution/claim.ts:286](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L286) ###### Parameters @@ -215,7 +215,7 @@ Defined in: [candidate-execution/claim.ts:296](https://github.com/tangle-network > **tryClaim**(`requested`): `Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> -Defined in: [candidate-execution/claim.ts:300](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L300) +Defined in: [candidate-execution/claim.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L290) ###### Parameters @@ -235,7 +235,7 @@ Defined in: [candidate-execution/claim.ts:300](https://github.com/tangle-network > **getAttempt**(`requestedAttempt`): `Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> -Defined in: [candidate-execution/claim.ts:323](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L323) +Defined in: [candidate-execution/claim.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L313) ###### Parameters @@ -255,7 +255,7 @@ Defined in: [candidate-execution/claim.ts:323](https://github.com/tangle-network > **markCandidateMayRun**(`requestedLease`): `Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> -Defined in: [candidate-execution/claim.ts:333](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L333) +Defined in: [candidate-execution/claim.ts:323](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L323) Persist the point after which candidate code may have run. @@ -277,7 +277,7 @@ Persist the point after which candidate code may have run. > **stageTerminal**(`requestedLease`, `result`): `Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> -Defined in: [candidate-execution/claim.ts:350](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L350) +Defined in: [candidate-execution/claim.ts:340](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L340) Fsync the complete terminal record into the durable outbox. @@ -303,7 +303,7 @@ Fsync the complete terminal record into the durable outbox. > **finish**(`requestedLease`, `requestedTerminalDigest`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -Defined in: [candidate-execution/claim.ts:365](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L365) +Defined in: [candidate-execution/claim.ts:355](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L355) Publish exactly the staged terminal identified by `terminalDigest`. @@ -329,7 +329,7 @@ Publish exactly the staged terminal identified by `terminalDigest`. > **recoverExpired**(`requestedAttempt`, `evidence`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -Defined in: [candidate-execution/claim.ts:383](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L383) +Defined in: [candidate-execution/claim.ts:373](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L373) Write a failed terminal only after the lease expired and a trusted worker independently proved process death plus model and memory closure. @@ -1245,7 +1245,7 @@ Defined in: [sessions.ts:129](https://github.com/tangle-network/agent-runtime/bl ### AgentCandidateCodeSurfaceSource -Defined in: [candidate-execution/builder.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L44) +Defined in: [candidate-execution/builder.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L43) The only accepted path from an agent-eval code candidate to executable bytes. @@ -1255,25 +1255,25 @@ The only accepted path from an agent-eval code candidate to executable bytes. > **kind**: `"code-surface"` -Defined in: [candidate-execution/builder.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L45) +Defined in: [candidate-execution/builder.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L44) ##### surface > **surface**: `CodeSurface` -Defined in: [candidate-execution/builder.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L46) +Defined in: [candidate-execution/builder.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L45) ##### repository > **repository**: `AgentCandidateGitHubRepository` -Defined in: [candidate-execution/builder.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L47) +Defined in: [candidate-execution/builder.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L46) ##### worktreeDir? > `optional` **worktreeDir?**: `string` -Defined in: [candidate-execution/builder.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L49) +Defined in: [candidate-execution/builder.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L48) Optional parent directory used to resolve a relative `surface.worktreeRef`. @@ -1281,7 +1281,7 @@ Optional parent directory used to resolve a relative `surface.worktreeRef`. ### BuildAgentCandidateBundleInput -Defined in: [candidate-execution/builder.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L59) +Defined in: [candidate-execution/builder.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L58) Complete measured surfaces and execution policy compiled into one candidate bundle. @@ -1291,45 +1291,57 @@ Complete measured surfaces and execution policy compiled into one candidate bund > **profile**: [`AgentCandidateProfileSource`](#agentcandidateprofilesource) -Defined in: [candidate-execution/builder.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L60) +Defined in: [candidate-execution/builder.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L59) ##### code > **code**: [`AgentCandidateCodeSource`](#agentcandidatecodesource) -Defined in: [candidate-execution/builder.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L61) +Defined in: [candidate-execution/builder.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L60) ##### execution > **execution**: `AgentCandidateExecution` -Defined in: [candidate-execution/builder.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L62) +Defined in: [candidate-execution/builder.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L61) ##### knowledge? > `optional` **knowledge?**: `AgentCandidateKnowledge` -Defined in: [candidate-execution/builder.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L63) +Defined in: [candidate-execution/builder.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L62) ##### memory > **memory**: `AgentCandidateMemoryPolicy` -Defined in: [candidate-execution/builder.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L64) +Defined in: [candidate-execution/builder.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L63) -##### lineage +*** + +### AgentCandidatePreparationEvidence + +Defined in: [candidate-execution/claim-file-formats.ts:11](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-formats.ts#L11) + +#### Properties -> **lineage**: `Omit`\<`AgentCandidateLineage`, `"profileDiffIds"`\> +##### executionPlan + +> `readonly` **executionPlan**: `AgentCandidateArtifactRef` + +Defined in: [candidate-execution/claim-file-formats.ts:12](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-formats.ts#L12) + +##### materializationReceipt -Defined in: [candidate-execution/builder.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L66) +> `readonly` **materializationReceipt**: `AgentCandidateArtifactRef` -`profileDiffIds` is derived from `profile`; callers cannot contradict it. +Defined in: [candidate-execution/claim-file-formats.ts:13](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-formats.ts#L13) *** ### FileAgentCandidateExecutionClaimStoreOptions -Defined in: [candidate-execution/claim-file-store.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L66) +Defined in: [candidate-execution/claim-file-store.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L60) #### Properties @@ -1337,7 +1349,7 @@ Defined in: [candidate-execution/claim-file-store.ts:66](https://github.com/tang > **directory**: `string` -Defined in: [candidate-execution/claim-file-store.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L68) +Defined in: [candidate-execution/claim-file-store.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L62) Evaluator-owned directory shared by every process allowed to execute candidates. @@ -1345,7 +1357,7 @@ Evaluator-owned directory shared by every process allowed to execute candidates. > `optional` **now?**: () => `number` -Defined in: [candidate-execution/claim-file-store.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L70) +Defined in: [candidate-execution/claim-file-store.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L64) Testable evaluator clock; defaults to `Date.now`. @@ -1357,7 +1369,7 @@ Testable evaluator clock; defaults to `Date.now`. ### AgentCandidateExecutionCleanupHandles -Defined in: [candidate-execution/claim.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L39) +Defined in: [candidate-execution/claim.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L38) Non-secret identities a trusted recovery worker needs to close an abandoned attempt. @@ -1367,37 +1379,37 @@ Non-secret identities a trusted recovery worker needs to close an abandoned atte > `readonly` **preparationId**: `string` -Defined in: [candidate-execution/claim.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L40) +Defined in: [candidate-execution/claim.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L39) ##### modelGrantDigest > `readonly` **modelGrantDigest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/claim.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L41) +Defined in: [candidate-execution/claim.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L40) ##### resolvedModel > `readonly` **resolvedModel**: `AgentCandidateResolvedModel` -Defined in: [candidate-execution/claim.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L42) +Defined in: [candidate-execution/claim.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L41) ##### traceRunId > `readonly` **traceRunId**: `string` -Defined in: [candidate-execution/claim.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L43) +Defined in: [candidate-execution/claim.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L42) ##### cleanupTimeoutMs > `readonly` **cleanupTimeoutMs**: `number` -Defined in: [candidate-execution/claim.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L44) +Defined in: [candidate-execution/claim.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L43) ##### memory? > `readonly` `optional` **memory?**: `object` -Defined in: [candidate-execution/claim.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L45) +Defined in: [candidate-execution/claim.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L44) ###### accessDigest @@ -1411,7 +1423,7 @@ Defined in: [candidate-execution/claim.ts:45](https://github.com/tangle-network/ ### AgentCandidateExecutionClaim -Defined in: [candidate-execution/claim.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L52) +Defined in: [candidate-execution/claim.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L51) Immutable signed identity stored for one execution attempt. @@ -1421,43 +1433,51 @@ Immutable signed identity stored for one execution attempt. > `readonly` **executionId**: `string` -Defined in: [candidate-execution/claim.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L53) +Defined in: [candidate-execution/claim.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L52) ##### attempt > `readonly` **attempt**: `number` -Defined in: [candidate-execution/claim.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L54) +Defined in: [candidate-execution/claim.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L53) ##### maxAttempts > `readonly` **maxAttempts**: `number` -Defined in: [candidate-execution/claim.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L55) +Defined in: [candidate-execution/claim.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L54) ##### retryPolicy > `readonly` **retryPolicy**: `"none"` \| `"pre-model-infrastructure-only"` -Defined in: [candidate-execution/claim.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L56) +Defined in: [candidate-execution/claim.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L55) ##### bundleDigest > `readonly` **bundleDigest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/claim.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L57) +Defined in: [candidate-execution/claim.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L56) ##### executionPlanDigest > `readonly` **executionPlanDigest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/claim.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L58) +Defined in: [candidate-execution/claim.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L57) + +##### preparationEvidence + +> `readonly` **preparationEvidence**: [`AgentCandidatePreparationEvidence`](#agentcandidatepreparationevidence) + +Defined in: [candidate-execution/claim.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L59) + +Durable canonical bytes needed to reconstruct the signed preparation. ##### retryLineageDigest > `readonly` **retryLineageDigest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/claim.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L60) +Defined in: [candidate-execution/claim.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L61) Frozen plan identity with only attempt number and per-attempt grant identity normalized. @@ -1465,7 +1485,7 @@ Frozen plan identity with only attempt number and per-attempt grant identity nor > `readonly` **leaseExpiresAtMs**: `number` -Defined in: [candidate-execution/claim.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L62) +Defined in: [candidate-execution/claim.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L63) The winning lease stops authorizing a new terminal write at this instant. @@ -1473,7 +1493,7 @@ The winning lease stops authorizing a new terminal write at this instant. > `readonly` **resultTimeoutMs**: `number` -Defined in: [candidate-execution/claim.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L64) +Defined in: [candidate-execution/claim.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L65) Frozen budget for task verification, executable grading, and receipt construction. @@ -1481,7 +1501,7 @@ Frozen budget for task verification, executable grading, and receipt constructio > `readonly` **cleanup**: [`AgentCandidateExecutionCleanupHandles`](#agentcandidateexecutioncleanuphandles) -Defined in: [candidate-execution/claim.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L66) +Defined in: [candidate-execution/claim.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L67) Non-secret handles retained so an expired attempt can be closed and reconciled. @@ -1489,7 +1509,7 @@ Non-secret handles retained so an expired attempt can be closed and reconciled. ### AgentCandidateExecutionLease -Defined in: [candidate-execution/claim.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L70) +Defined in: [candidate-execution/claim.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L71) Secret capability required to finish the acquired attempt. @@ -1499,77 +1519,31 @@ Secret capability required to finish the acquired attempt. > `readonly` **executionId**: `string` -Defined in: [candidate-execution/claim.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L71) +Defined in: [candidate-execution/claim.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L72) ##### attempt > `readonly` **attempt**: `number` -Defined in: [candidate-execution/claim.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L72) +Defined in: [candidate-execution/claim.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L73) ##### token > `readonly` **token**: `string` -Defined in: [candidate-execution/claim.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L73) +Defined in: [candidate-execution/claim.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L74) ##### expiresAtMs > `readonly` **expiresAtMs**: `number` -Defined in: [candidate-execution/claim.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L74) - -*** - -### AgentCandidateExecutionUsage - -Defined in: [candidate-execution/claim.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L85) - -Exact fixed-point usage proven by the closed evaluator model ledger. - -#### Properties - -##### costUsdNanos - -> `readonly` **costUsdNanos**: `number` - -Defined in: [candidate-execution/claim.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L86) - -##### inputTokens - -> `readonly` **inputTokens**: `number` - -Defined in: [candidate-execution/claim.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L87) - -##### outputTokens - -> `readonly` **outputTokens**: `number` - -Defined in: [candidate-execution/claim.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L88) - -##### cachedInputTokens - -> `readonly` **cachedInputTokens**: `number` - -Defined in: [candidate-execution/claim.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L89) - -##### reasoningTokens - -> `readonly` **reasoningTokens**: `number` - -Defined in: [candidate-execution/claim.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L90) - -##### modelCalls - -> `readonly` **modelCalls**: `number` - -Defined in: [candidate-execution/claim.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L91) +Defined in: [candidate-execution/claim.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L75) *** ### AgentCandidateExecutionRecoveryEvidence -Defined in: [candidate-execution/claim.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L128) +Defined in: [candidate-execution/claim.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L118) Trusted, independently observed closure facts for one expired winning lease. @@ -1579,31 +1553,31 @@ Trusted, independently observed closure facts for one expired winning lease. > `readonly` **failureClass**: [`AgentCandidateExecutionFailureClass`](#agentcandidateexecutionfailureclass) -Defined in: [candidate-execution/claim.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L129) +Defined in: [candidate-execution/claim.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L119) ##### usage -> `readonly` **usage**: [`AgentCandidateExecutionUsage`](#agentcandidateexecutionusage) +> `readonly` **usage**: `AgentCandidateFixedSpend` -Defined in: [candidate-execution/claim.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L130) +Defined in: [candidate-execution/claim.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L120) ##### modelSettlement > `readonly` **modelSettlement**: `AgentCandidateArtifactRef` -Defined in: [candidate-execution/claim.ts:131](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L131) +Defined in: [candidate-execution/claim.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L121) ##### failureEvidence? > `readonly` `optional` **failureEvidence?**: `AgentCandidateArtifactRef` -Defined in: [candidate-execution/claim.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L132) +Defined in: [candidate-execution/claim.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L122) ##### process > `readonly` **process**: `object` -Defined in: [candidate-execution/claim.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L133) +Defined in: [candidate-execution/claim.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L123) ###### stopped @@ -1617,7 +1591,7 @@ Defined in: [candidate-execution/claim.ts:133](https://github.com/tangle-network > `readonly` **model**: `object` -Defined in: [candidate-execution/claim.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L137) +Defined in: [candidate-execution/claim.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L127) ###### closed @@ -1635,7 +1609,7 @@ Defined in: [candidate-execution/claim.ts:137](https://github.com/tangle-network > `readonly` `optional` **memory?**: `object` -Defined in: [candidate-execution/claim.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L142) +Defined in: [candidate-execution/claim.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L132) ###### closed @@ -1657,7 +1631,7 @@ Defined in: [candidate-execution/claim.ts:142](https://github.com/tangle-network ### AgentCandidateExecutionAttemptRef -Defined in: [candidate-execution/claim.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L150) +Defined in: [candidate-execution/claim.ts:140](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L140) #### Properties @@ -1665,19 +1639,19 @@ Defined in: [candidate-execution/claim.ts:150](https://github.com/tangle-network > `readonly` **executionId**: `string` -Defined in: [candidate-execution/claim.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L151) +Defined in: [candidate-execution/claim.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L141) ##### attempt > `readonly` **attempt**: `number` -Defined in: [candidate-execution/claim.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L152) +Defined in: [candidate-execution/claim.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L142) *** ### AgentCandidateExecutionAttemptRecord -Defined in: [candidate-execution/claim.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L156) +Defined in: [candidate-execution/claim.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L146) Persisted state available to a fresh trusted recovery worker after a crash. @@ -1687,19 +1661,19 @@ Persisted state available to a fresh trusted recovery worker after a crash. > `readonly` **claim**: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) -Defined in: [candidate-execution/claim.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L157) +Defined in: [candidate-execution/claim.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L147) ##### phase > `readonly` **phase**: [`AgentCandidateExecutionPhase`](#agentcandidateexecutionphase) -Defined in: [candidate-execution/claim.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L158) +Defined in: [candidate-execution/claim.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L148) ##### staged? > `readonly` `optional` **staged?**: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord) -Defined in: [candidate-execution/claim.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L160) +Defined in: [candidate-execution/claim.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L150) Durable outbox content written before the terminal compare-and-set. @@ -1707,13 +1681,13 @@ Durable outbox content written before the terminal compare-and-set. > `readonly` `optional` **terminal?**: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord) -Defined in: [candidate-execution/claim.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L161) +Defined in: [candidate-execution/claim.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L151) *** ### AgentCandidateExecutionClaimStore -Defined in: [candidate-execution/claim.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L233) +Defined in: [candidate-execution/claim.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L223) Atomic one-shot store for candidate execution attempts. @@ -1729,7 +1703,7 @@ evidence rather than an ambiguous completed run. > **tryClaim**(`claim`): `Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> -Defined in: [candidate-execution/claim.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L234) +Defined in: [candidate-execution/claim.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L224) ###### Parameters @@ -1745,7 +1719,7 @@ Defined in: [candidate-execution/claim.ts:234](https://github.com/tangle-network > **getAttempt**(`attempt`): `Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> -Defined in: [candidate-execution/claim.ts:235](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L235) +Defined in: [candidate-execution/claim.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L225) ###### Parameters @@ -1761,7 +1735,7 @@ Defined in: [candidate-execution/claim.ts:235](https://github.com/tangle-network > **markCandidateMayRun**(`lease`): `Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> -Defined in: [candidate-execution/claim.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L239) +Defined in: [candidate-execution/claim.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L229) Persist the point after which candidate code may have run. @@ -1779,7 +1753,7 @@ Persist the point after which candidate code may have run. > **stageTerminal**(`lease`, `result`): `Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> -Defined in: [candidate-execution/claim.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L243) +Defined in: [candidate-execution/claim.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L233) Fsync the complete terminal record into the durable outbox. @@ -1801,7 +1775,7 @@ Fsync the complete terminal record into the durable outbox. > **finish**(`lease`, `terminalDigest`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -Defined in: [candidate-execution/claim.ts:248](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L248) +Defined in: [candidate-execution/claim.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L238) Publish exactly the staged terminal identified by `terminalDigest`. @@ -1823,7 +1797,7 @@ Publish exactly the staged terminal identified by `terminalDigest`. > **recoverExpired**(`attempt`, `evidence`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -Defined in: [candidate-execution/claim.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L256) +Defined in: [candidate-execution/claim.ts:246](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L246) Write a failed terminal only after the lease expired and a trusted worker independently proved process death plus model and memory closure. @@ -1860,7 +1834,7 @@ Defined in: [candidate-execution/dispose.ts:11](https://github.com/tangle-networ ### ExecutePreparedAgentCandidateOptions -Defined in: [candidate-execution/execute.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L61) +Defined in: [candidate-execution/execute.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L66) #### Properties @@ -1868,31 +1842,31 @@ Defined in: [candidate-execution/execute.ts:61](https://github.com/tangle-networ > **executor**: [`AgentCandidateExecutorPort`](#agentcandidateexecutorport) -Defined in: [candidate-execution/execute.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L62) +Defined in: [candidate-execution/execute.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L67) ##### grader > **grader**: [`AgentCandidateBenchmarkGraderPort`](#agentcandidatebenchmarkgraderport) -Defined in: [candidate-execution/execute.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L63) +Defined in: [candidate-execution/execute.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L68) ##### outputArtifacts > **outputArtifacts**: [`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) -Defined in: [candidate-execution/execute.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L64) +Defined in: [candidate-execution/execute.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L69) ##### traceStore > **traceStore**: `TraceStore` -Defined in: [candidate-execution/execute.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L65) +Defined in: [candidate-execution/execute.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L70) ##### claimStore > **claimStore**: [`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore) -Defined in: [candidate-execution/execute.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L67) +Defined in: [candidate-execution/execute.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L72) Long-lived evaluator-owned store shared by every process that can run this benchmark. @@ -1900,7 +1874,7 @@ Long-lived evaluator-owned store shared by every process that can run this bench > `optional` **cleanupTimeoutMs?**: `number` -Defined in: [candidate-execution/execute.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L69) +Defined in: [candidate-execution/execute.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L74) Maximum time to prove process death and revoke protected access after a run ends. @@ -1908,7 +1882,7 @@ Maximum time to prove process death and revoke protected access after a run ends > `optional` **resultTimeoutMs?**: `number` -Defined in: [candidate-execution/execute.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L71) +Defined in: [candidate-execution/execute.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L76) Maximum time for task verification, executable grading, and receipt construction. @@ -1916,7 +1890,7 @@ Maximum time for task verification, executable grading, and receipt construction ### PrepareAgentCandidateExecutionOptions -Defined in: [candidate-execution/prepare.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L92) +Defined in: [candidate-execution/prepare.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L85) #### Properties @@ -1924,13 +1898,13 @@ Defined in: [candidate-execution/prepare.ts:92](https://github.com/tangle-networ > `optional` **cleanupTimeoutMs?**: `number` -Defined in: [candidate-execution/prepare.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L93) +Defined in: [candidate-execution/prepare.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L86) ##### resultTimeoutMs? > `optional` **resultTimeoutMs?**: `number` -Defined in: [candidate-execution/prepare.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L95) +Defined in: [candidate-execution/prepare.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L88) Maximum time for task verification, executable grading, and receipt construction. @@ -2121,7 +2095,7 @@ Exact environment names the activation endpoint must return, no more or fewer. ### RecoverExpiredAgentCandidateOptions -Defined in: [candidate-execution/recover.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L23) +Defined in: [candidate-execution/recover.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L35) #### Properties @@ -2129,49 +2103,49 @@ Defined in: [candidate-execution/recover.ts:23](https://github.com/tangle-networ > **attempt**: [`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) -Defined in: [candidate-execution/recover.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L24) +Defined in: [candidate-execution/recover.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L36) ##### claimStore > **claimStore**: [`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore) -Defined in: [candidate-execution/recover.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L25) +Defined in: [candidate-execution/recover.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L37) ##### executor > **executor**: [`AgentCandidateExecutorPort`](#agentcandidateexecutorport) -Defined in: [candidate-execution/recover.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L26) +Defined in: [candidate-execution/recover.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L38) ##### traceStore > **traceStore**: `TraceStore` -Defined in: [candidate-execution/recover.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L27) +Defined in: [candidate-execution/recover.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L39) ##### ports > **ports**: `Pick`\<[`AgentCandidateExecutionPorts`](#agentcandidateexecutionports), `"models"` \| `"memory"`\> -Defined in: [candidate-execution/recover.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L28) +Defined in: [candidate-execution/recover.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L40) ##### outputArtifacts > **outputArtifacts**: [`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) -Defined in: [candidate-execution/recover.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L29) +Defined in: [candidate-execution/recover.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L41) ##### cleanupTimeoutMs? > `optional` **cleanupTimeoutMs?**: `number` -Defined in: [candidate-execution/recover.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L30) +Defined in: [candidate-execution/recover.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L42) ##### now? > `optional` **now?**: () => `number` -Defined in: [candidate-execution/recover.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L32) +Defined in: [candidate-execution/recover.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L44) Evaluator clock; must be the same clock used by the claim store. @@ -2183,7 +2157,7 @@ Evaluator clock; must be the same clock used by the claim store. ### AgentCandidateArtifactPort -Defined in: [candidate-execution/types.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L35) +Defined in: [candidate-execution/types.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L41) Reads one content-addressed object from the closed S3/IPFS locator set. @@ -2197,7 +2171,7 @@ Reads one content-addressed object from the closed S3/IPFS locator set. > **read**(`ref`): `Promise`\<`Uint8Array`\<`ArrayBufferLike`\>\> -Defined in: [candidate-execution/types.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L36) +Defined in: [candidate-execution/types.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L42) ###### Parameters @@ -2213,7 +2187,7 @@ Defined in: [candidate-execution/types.ts:36](https://github.com/tangle-network/ ### AgentCandidateOutputArtifactPort -Defined in: [candidate-execution/types.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L57) +Defined in: [candidate-execution/types.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L69) Durable content-addressed evidence store controlled only by the evaluator. @@ -2227,7 +2201,7 @@ Durable content-addressed evidence store controlled only by the evaluator. > **read**(`ref`): `Promise`\<`Uint8Array`\<`ArrayBufferLike`\>\> -Defined in: [candidate-execution/types.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L36) +Defined in: [candidate-execution/types.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L42) ###### Parameters @@ -2247,7 +2221,7 @@ Defined in: [candidate-execution/types.ts:36](https://github.com/tangle-network/ > **put**(`input`): `Promise`\<`AgentCandidateArtifactRef`\> -Defined in: [candidate-execution/types.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L59) +Defined in: [candidate-execution/types.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L71) Must be idempotent for identical bytes and return only a durable S3/IPFS locator. @@ -2281,7 +2255,7 @@ Abort must prevent durable publication when it happens before resolution. ### AgentCandidateRepositoryPort -Defined in: [candidate-execution/types.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L69) +Defined in: [candidate-execution/types.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L81) Resolves a declared GitHub repository to an already-present local Git object store. @@ -2291,7 +2265,7 @@ Resolves a declared GitHub repository to an already-present local Git object sto > **resolve**(`repository`): `Promise`\<`string`\> -Defined in: [candidate-execution/types.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L70) +Defined in: [candidate-execution/types.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L82) ###### Parameters @@ -2307,7 +2281,7 @@ Defined in: [candidate-execution/types.ts:70](https://github.com/tangle-network/ ### AgentCandidateVerificationPorts -Defined in: [candidate-execution/types.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L73) +Defined in: [candidate-execution/types.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L85) #### Extended by @@ -2319,19 +2293,19 @@ Defined in: [candidate-execution/types.ts:73](https://github.com/tangle-network/ > **artifacts**: [`AgentCandidateArtifactPort`](#agentcandidateartifactport) -Defined in: [candidate-execution/types.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L74) +Defined in: [candidate-execution/types.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L86) ##### repositories > **repositories**: [`AgentCandidateRepositoryPort`](#agentcandidaterepositoryport) -Defined in: [candidate-execution/types.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L75) +Defined in: [candidate-execution/types.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L87) *** ### AgentCandidateWorkspacePort -Defined in: [candidate-execution/types.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L85) +Defined in: [candidate-execution/types.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L97) Materializes an already-verified workspace archive. @@ -2345,7 +2319,7 @@ any archive encoding, or no-op when the exact workspace is already present. > **materialize**(`input`): `Promise`\<`void`\> -Defined in: [candidate-execution/types.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L86) +Defined in: [candidate-execution/types.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L98) ###### Parameters @@ -2375,7 +2349,7 @@ Defined in: [candidate-execution/types.ts:86](https://github.com/tangle-network/ ### ResolvedAgentCandidateContainer -Defined in: [candidate-execution/types.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L94) +Defined in: [candidate-execution/types.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L106) #### Properties @@ -2383,37 +2357,37 @@ Defined in: [candidate-execution/types.ts:94](https://github.com/tangle-network/ > **source**: `"pinned-container"` \| `"evaluator-task-container"` -Defined in: [candidate-execution/types.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L95) +Defined in: [candidate-execution/types.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L107) ##### image > **image**: `string` -Defined in: [candidate-execution/types.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L96) +Defined in: [candidate-execution/types.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L108) ##### indexDigest > **indexDigest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/types.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L97) +Defined in: [candidate-execution/types.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L109) ##### manifestDigest > **manifestDigest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/types.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L98) +Defined in: [candidate-execution/types.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L110) ##### platform > **platform**: `AgentCandidateOciPlatform` -Defined in: [candidate-execution/types.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L99) +Defined in: [candidate-execution/types.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L111) *** ### AgentCandidateContainerPort -Defined in: [candidate-execution/types.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L102) +Defined in: [candidate-execution/types.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L114) #### Methods @@ -2421,7 +2395,7 @@ Defined in: [candidate-execution/types.ts:102](https://github.com/tangle-network > **resolve**(`input`): `Promise`\<[`ResolvedAgentCandidateContainer`](#resolvedagentcandidatecontainer)\> -Defined in: [candidate-execution/types.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L103) +Defined in: [candidate-execution/types.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L115) ###### Parameters @@ -2443,7 +2417,7 @@ Defined in: [candidate-execution/types.ts:103](https://github.com/tangle-network ### AgentCandidateModelPort -Defined in: [candidate-execution/types.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L109) +Defined in: [candidate-execution/types.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L121) #### Methods @@ -2451,7 +2425,7 @@ Defined in: [candidate-execution/types.ts:109](https://github.com/tangle-network > **resolve**(`input`): `Promise`\<`AgentCandidateResolvedModel`\> -Defined in: [candidate-execution/types.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L110) +Defined in: [candidate-execution/types.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L122) ###### Parameters @@ -2477,7 +2451,7 @@ Defined in: [candidate-execution/types.ts:110](https://github.com/tangle-network > **reserveGrant**(`input`): `Promise`\<[`AgentCandidateProtectedModelReservation`](#agentcandidateprotectedmodelreservation)\> -Defined in: [candidate-execution/types.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L120) +Defined in: [candidate-execution/types.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L132) Reserve a stable access identity without creating a live credential. The reservation is scoped to `preparationId` and must automatically expire @@ -2523,7 +2497,7 @@ at `expiresAtMs`, even if this call returns ambiguously to the runtime. > **activateGrant**(`input`): `Promise`\<[`AgentCandidateProtectedModelActivation`](#agentcandidateprotectedmodelactivation)\> -Defined in: [candidate-execution/types.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L130) +Defined in: [candidate-execution/types.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L142) Create the live scoped credential only after the execution attempt is durably claimed. @@ -2559,7 +2533,7 @@ Create the live scoped credential only after the execution attempt is durably cl > **settleGrant**(`input`): `Promise`\<[`AgentCandidateProtectedModelSettlement`](#agentcandidateprotectedmodelsettlement)\> -Defined in: [candidate-execution/types.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L143) +Defined in: [candidate-execution/types.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L155) Atomically revoke the grant, drain in-flight calls, and return its immutable final ledger. This operation must be idempotent for the exact preparation and must also @@ -2596,35 +2570,9 @@ different preparation, even when both reservations report the same digest. *** -### AgentCandidateBenchmarkGraderIdentity - -Defined in: [candidate-execution/types.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L158) - -#### Properties - -##### name - -> **name**: `string` - -Defined in: [candidate-execution/types.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L159) - -##### version - -> **version**: `string` - -Defined in: [candidate-execution/types.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L160) - -##### artifact - -> **artifact**: `AgentCandidateArtifactRef` - -Defined in: [candidate-execution/types.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L161) - -*** - ### AgentCandidateProtectedModelReservation -Defined in: [candidate-execution/types.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L164) +Defined in: [candidate-execution/types.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L170) #### Properties @@ -2632,19 +2580,19 @@ Defined in: [candidate-execution/types.ts:164](https://github.com/tangle-network > **preparationId**: `string` -Defined in: [candidate-execution/types.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L165) +Defined in: [candidate-execution/types.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L171) ##### digest > **digest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/types.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L166) +Defined in: [candidate-execution/types.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L172) ##### expiresAtMs > **expiresAtMs**: `number` -Defined in: [candidate-execution/types.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L168) +Defined in: [candidate-execution/types.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L174) Evaluator service must expire and revoke this reservation at this epoch millisecond. @@ -2652,7 +2600,7 @@ Evaluator service must expire and revoke this reservation at this epoch millisec > **enforcedLimits**: [`AgentCandidateModelLimits`](#agentcandidatemodellimits) -Defined in: [candidate-execution/types.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L170) +Defined in: [candidate-execution/types.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L176) The gateway must stop calls before any one of these limits is exceeded. @@ -2660,7 +2608,7 @@ The gateway must stop calls before any one of these limits is exceeded. > **network**: `AgentCandidateModelAccessNetwork` -Defined in: [candidate-execution/types.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L172) +Defined in: [candidate-execution/types.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L178) Exact public endpoint exception; every other candidate destination stays blocked. @@ -2668,7 +2616,7 @@ Exact public endpoint exception; every other candidate destination stays blocked ### AgentCandidateProtectedModelActivation -Defined in: [candidate-execution/types.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L175) +Defined in: [candidate-execution/types.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L181) #### Properties @@ -2676,103 +2624,15 @@ Defined in: [candidate-execution/types.ts:175](https://github.com/tangle-network > **env**: `Readonly`\<`Record`\<`string`, `string`\>\> -Defined in: [candidate-execution/types.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L177) +Defined in: [candidate-execution/types.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L183) Injected only into the trusted executor after all pre-launch checks pass. *** -### AgentCandidateProtectedModelCall - -Defined in: [candidate-execution/types.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L181) - -One evaluator-gateway call in the final, revoked model-access ledger. - -#### Properties - -##### callId - -> **callId**: `string` - -Defined in: [candidate-execution/types.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L182) - -##### generationId - -> **generationId**: `string` - -Defined in: [candidate-execution/types.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L184) - -Router-generated public response identity. - -##### traceSpanId - -> **traceSpanId**: `string` - -Defined in: [candidate-execution/types.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L186) - -Exact protected agent-eval LLM span produced from the router ledger. - -##### status - -> **status**: `"failed"` \| `"succeeded"` - -Defined in: [candidate-execution/types.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L187) - -##### model - -> **model**: `string` - -Defined in: [candidate-execution/types.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L188) - -##### startedAtMs - -> **startedAtMs**: `number` - -Defined in: [candidate-execution/types.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L189) - -##### endedAtMs - -> **endedAtMs**: `number` - -Defined in: [candidate-execution/types.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L190) - -##### inputTokens - -> **inputTokens**: `number` - -Defined in: [candidate-execution/types.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L191) - -##### outputTokens - -> **outputTokens**: `number` - -Defined in: [candidate-execution/types.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L192) - -##### cachedInputTokens - -> **cachedInputTokens**: `number` - -Defined in: [candidate-execution/types.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L193) - -##### reasoningTokens - -> **reasoningTokens**: `number` - -Defined in: [candidate-execution/types.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L194) - -##### costUsdNanos - -> **costUsdNanos**: `number` - -Defined in: [candidate-execution/types.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L196) - -Integer billionths of one US dollar; avoids floating-point ledger drift. - -*** - ### AgentCandidateProtectedModelSettlement -Defined in: [candidate-execution/types.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L199) +Defined in: [candidate-execution/types.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L186) #### Properties @@ -2780,31 +2640,31 @@ Defined in: [candidate-execution/types.ts:199](https://github.com/tangle-network > **preparationId**: `string` -Defined in: [candidate-execution/types.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L200) +Defined in: [candidate-execution/types.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L187) ##### grantDigest > **grantDigest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/types.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L201) +Defined in: [candidate-execution/types.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L188) ##### closed > **closed**: `true` -Defined in: [candidate-execution/types.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L202) +Defined in: [candidate-execution/types.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L189) ##### calls -> **calls**: readonly [`AgentCandidateProtectedModelCall`](#agentcandidateprotectedmodelcall)[] +> **calls**: readonly `AgentCandidateModelSettlementCall`[] -Defined in: [candidate-execution/types.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L203) +Defined in: [candidate-execution/types.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L190) *** ### AgentCandidateMemoryResetResult -Defined in: [candidate-execution/types.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L206) +Defined in: [candidate-execution/types.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L193) #### Properties @@ -2812,43 +2672,43 @@ Defined in: [candidate-execution/types.ts:206](https://github.com/tangle-network > **preparationId**: `string` -Defined in: [candidate-execution/types.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L207) +Defined in: [candidate-execution/types.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L194) ##### accessDigest > **accessDigest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/types.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L208) +Defined in: [candidate-execution/types.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L195) ##### expiresAtMs > **expiresAtMs**: `number` -Defined in: [candidate-execution/types.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L209) +Defined in: [candidate-execution/types.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L196) ##### evidence > **evidence**: `AgentCandidateCapturedArtifact` -Defined in: [candidate-execution/types.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L210) +Defined in: [candidate-execution/types.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L197) ##### emptyStateDigest > **emptyStateDigest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/types.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L211) +Defined in: [candidate-execution/types.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L198) ##### beforeState > **beforeState**: `AgentCandidateWorkspaceSnapshotEvidence` -Defined in: [candidate-execution/types.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L212) +Defined in: [candidate-execution/types.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L199) *** ### AgentCandidateMemoryPort -Defined in: [candidate-execution/types.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L215) +Defined in: [candidate-execution/types.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L202) #### Methods @@ -2856,7 +2716,7 @@ Defined in: [candidate-execution/types.ts:215](https://github.com/tangle-network > **reset**(`input`): `Promise`\<[`AgentCandidateMemoryResetResult`](#agentcandidatememoryresetresult)\> -Defined in: [candidate-execution/types.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L221) +Defined in: [candidate-execution/types.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L208) Reset and reserve exact task memory without returning live access. The service must scope the reservation to `preparationId`, automatically @@ -2898,7 +2758,7 @@ revoke it at `expiresAtMs`, and never reuse it for another preparation. > **activate**(`input`): `Promise`\<\{ `env`: `Readonly`\<`Record`\<`string`, `string`\>\>; \}\> -Defined in: [candidate-execution/types.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L233) +Defined in: [candidate-execution/types.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L220) Create live scoped access only after the execution attempt is durably claimed. Activation must match the exact preparation/access pair and may not extend expiry. @@ -2935,7 +2795,7 @@ Activation must match the exact preparation/access pair and may not extend expir > **close**(`input`): `Promise`\<\{ `closed`: `true`; \}\> -Defined in: [candidate-execution/types.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L245) +Defined in: [candidate-execution/types.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L232) Revoke evaluator-owned access after process death or a failed preparation. Must be idempotent and concurrency-safe for the exact preparation/access @@ -2973,7 +2833,7 @@ pair and must never close a different preparation. ### AgentCandidateExecutionPorts -Defined in: [candidate-execution/types.ts:254](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L254) +Defined in: [candidate-execution/types.ts:241](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L241) #### Extends @@ -2985,7 +2845,7 @@ Defined in: [candidate-execution/types.ts:254](https://github.com/tangle-network > **artifacts**: [`AgentCandidateArtifactPort`](#agentcandidateartifactport) -Defined in: [candidate-execution/types.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L74) +Defined in: [candidate-execution/types.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L86) ###### Inherited from @@ -2995,7 +2855,7 @@ Defined in: [candidate-execution/types.ts:74](https://github.com/tangle-network/ > **repositories**: [`AgentCandidateRepositoryPort`](#agentcandidaterepositoryport) -Defined in: [candidate-execution/types.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L75) +Defined in: [candidate-execution/types.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L87) ###### Inherited from @@ -3005,31 +2865,33 @@ Defined in: [candidate-execution/types.ts:75](https://github.com/tangle-network/ > **workspaces**: [`AgentCandidateWorkspacePort`](#agentcandidateworkspaceport) -Defined in: [candidate-execution/types.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L255) +Defined in: [candidate-execution/types.ts:242](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L242) ##### containers > **containers**: [`AgentCandidateContainerPort`](#agentcandidatecontainerport) -Defined in: [candidate-execution/types.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L256) +Defined in: [candidate-execution/types.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L243) ##### models > **models**: [`AgentCandidateModelPort`](#agentcandidatemodelport) -Defined in: [candidate-execution/types.ts:257](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L257) +Defined in: [candidate-execution/types.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L244) ##### memory > **memory**: [`AgentCandidateMemoryPort`](#agentcandidatememoryport) -Defined in: [candidate-execution/types.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L258) +Defined in: [candidate-execution/types.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L245) *** ### AgentCandidateTaskExecution -Defined in: [candidate-execution/types.ts:261](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L261) +Defined in: [candidate-execution/types.ts:249](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L249) + +Runtime placement for one exact cell from a signed candidate experiment. #### Properties @@ -3037,93 +2899,31 @@ Defined in: [candidate-execution/types.ts:261](https://github.com/tangle-network > **executionId**: `string` -Defined in: [candidate-execution/types.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L262) - -##### benchmark - -> **benchmark**: `string` - -Defined in: [candidate-execution/types.ts:263](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L263) - -##### benchmarkVersion - -> **benchmarkVersion**: `string` - -Defined in: [candidate-execution/types.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L264) - -##### taskId - -> **taskId**: `string` - -Defined in: [candidate-execution/types.ts:265](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L265) - -##### splitDigest - -> **splitDigest**: `` `sha256:${string}` `` - -Defined in: [candidate-execution/types.ts:266](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L266) - -##### instruction - -> **instruction**: `string` - -Defined in: [candidate-execution/types.ts:268](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L268) - -Exact agent-visible task instruction. The runtime rejects malformed Unicode. +Defined in: [candidate-execution/types.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L250) -##### repository - -> **repository**: `object` - -Defined in: [candidate-execution/types.ts:269](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L269) - -###### identity +##### runCell -> **identity**: `string` +> **runCell**: `AgentCandidateRunCell` -###### rootIdentity +Defined in: [candidate-execution/types.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L251) -> **rootIdentity**: `string` +##### benchmarkSuite -###### baseCommit +> **benchmarkSuite**: `AgentCandidateBenchmarkSuite` -> **baseCommit**: `string` +Defined in: [candidate-execution/types.ts:252](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L252) -###### baseTree - -> **baseTree**: `string` - -##### attempt - -> **attempt**: `AgentCandidateAttemptPolicy` - -Defined in: [candidate-execution/types.ts:275](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L275) - -##### model - -> **model**: `object` - -Defined in: [candidate-execution/types.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L276) - -###### requested - -> **requested**: `string` - -###### reasoningEffort - -> **reasoningEffort**: `ReasoningEffort` - -##### grader +##### task -> **grader**: [`AgentCandidateBenchmarkGraderIdentity`](#agentcandidatebenchmarkgraderidentity) +> **task**: `AgentCandidateBenchmarkTask` -Defined in: [candidate-execution/types.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L280) +Defined in: [candidate-execution/types.ts:253](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L253) ##### executionRoots > **executionRoots**: `object` -Defined in: [candidate-execution/types.ts:282](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L282) +Defined in: [candidate-execution/types.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L255) Absolute paths inside the evaluator-owned execution environment. @@ -3139,7 +2939,7 @@ Absolute paths inside the evaluator-owned execution environment. > **stagingRoots**: `object` -Defined in: [candidate-execution/types.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L287) +Defined in: [candidate-execution/types.ts:260](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L260) Host-side staging roots. These are verified but never signed as container paths. @@ -3155,29 +2955,11 @@ Host-side staging roots. These are verified but never signed as container paths. > **profileRoot**: `string` -##### workspace - -> **workspace**: `AgentCandidateWorkspaceSnapshotEvidence` - -Defined in: [candidate-execution/types.ts:292](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L292) - -##### evaluatorTaskContainer? - -> `optional` **evaluatorTaskContainer?**: [`ResolvedAgentCandidateContainer`](#resolvedagentcandidatecontainer) - -Defined in: [candidate-execution/types.ts:293](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L293) - -##### limits - -> **limits**: `AgentCandidateExecutionLimits` - -Defined in: [candidate-execution/types.ts:294](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L294) - *** ### VerifiedAgentCandidate -Defined in: [candidate-execution/types.ts:297](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L297) +Defined in: [candidate-execution/types.ts:267](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L267) #### Properties @@ -3185,25 +2967,25 @@ Defined in: [candidate-execution/types.ts:297](https://github.com/tangle-network > `readonly` **bundle**: `AgentCandidateBundle` -Defined in: [candidate-execution/types.ts:298](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L298) +Defined in: [candidate-execution/types.ts:268](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L268) ##### materializedTree? > `readonly` `optional` **materializedTree?**: `string` -Defined in: [candidate-execution/types.ts:299](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L299) +Defined in: [candidate-execution/types.ts:269](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L269) ##### \[verifiedCandidateBrand\] > `readonly` **\[verifiedCandidateBrand\]**: `true` -Defined in: [candidate-execution/types.ts:300](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L300) +Defined in: [candidate-execution/types.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L270) *** ### CanonicalCandidateDocument -Defined in: [candidate-execution/types.ts:303](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L303) +Defined in: [candidate-execution/types.ts:273](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L273) #### Type Parameters @@ -3217,13 +2999,13 @@ Defined in: [candidate-execution/types.ts:303](https://github.com/tangle-network > `readonly` **value**: `T` -Defined in: [candidate-execution/types.ts:304](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L304) +Defined in: [candidate-execution/types.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L274) ##### bytes > `readonly` **bytes**: `Uint8Array` -Defined in: [candidate-execution/types.ts:306](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L306) +Defined in: [candidate-execution/types.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L276) Canonical UTF-8 bytes of `value` with its top-level digest omitted. @@ -3231,13 +3013,13 @@ Canonical UTF-8 bytes of `value` with its top-level digest omitted. > `readonly` **digest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/types.ts:307](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L307) +Defined in: [candidate-execution/types.ts:277](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L277) *** ### PreparedAgentCandidateLaunch -Defined in: [candidate-execution/types.ts:310](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L310) +Defined in: [candidate-execution/types.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L280) #### Properties @@ -3245,13 +3027,13 @@ Defined in: [candidate-execution/types.ts:310](https://github.com/tangle-network > **executable**: `string` -Defined in: [candidate-execution/types.ts:311](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L311) +Defined in: [candidate-execution/types.ts:281](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L281) ##### args > **args**: readonly `string`[] -Defined in: [candidate-execution/types.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L313) +Defined in: [candidate-execution/types.ts:283](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L283) Complete fixed argv, including profile materializer flags but excluding task delivery. @@ -3259,13 +3041,13 @@ Complete fixed argv, including profile materializer flags but excluding task del > **env**: `Readonly`\<`Record`\<`string`, `string`\>\> -Defined in: [candidate-execution/types.ts:314](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L314) +Defined in: [candidate-execution/types.ts:284](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L284) ##### flags > **flags**: readonly `string`[] -Defined in: [candidate-execution/types.ts:316](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L316) +Defined in: [candidate-execution/types.ts:286](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L286) Informational subset already present at the tail of `args`; executors must not append twice. @@ -3273,13 +3055,13 @@ Informational subset already present at the tail of `args`; executors must not a > **cwd**: `string` -Defined in: [candidate-execution/types.ts:317](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L317) +Defined in: [candidate-execution/types.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L287) *** ### PreparedAgentCandidateInstruction -Defined in: [candidate-execution/types.ts:320](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L320) +Defined in: [candidate-execution/types.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L290) #### Properties @@ -3287,19 +3069,19 @@ Defined in: [candidate-execution/types.ts:320](https://github.com/tangle-network > **bytes**: `Uint8Array` -Defined in: [candidate-execution/types.ts:321](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L321) +Defined in: [candidate-execution/types.ts:291](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L291) ##### delivery > **delivery**: `AgentCandidateInstructionDelivery` -Defined in: [candidate-execution/types.ts:322](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L322) +Defined in: [candidate-execution/types.ts:292](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L292) *** ### PreparedAgentCandidateKnowledge -Defined in: [candidate-execution/types.ts:326](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L326) +Defined in: [candidate-execution/types.ts:296](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L296) Exact file-backed knowledge admitted by the candidate bundle. @@ -3309,31 +3091,31 @@ Exact file-backed knowledge admitted by the candidate bundle. > `readonly` **candidate**: `AgentCandidateKnowledgeRef` -Defined in: [candidate-execution/types.ts:327](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L327) +Defined in: [candidate-execution/types.ts:297](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L297) ##### snapshot > `readonly` **snapshot**: `AgentCandidateWorkspaceSnapshotEvidence` -Defined in: [candidate-execution/types.ts:328](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L328) +Defined in: [candidate-execution/types.ts:298](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L298) ##### files > `readonly` **files**: readonly [`AgentCandidateExecutorWorkspaceFile`](#agentcandidateexecutorworkspacefile)[] -Defined in: [candidate-execution/types.ts:329](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L329) +Defined in: [candidate-execution/types.ts:299](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L299) ##### retrievalConfig? > `readonly` `optional` **retrievalConfig?**: `Uint8Array`\<`ArrayBufferLike`\> -Defined in: [candidate-execution/types.ts:330](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L330) +Defined in: [candidate-execution/types.ts:300](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L300) *** ### PreparedAgentCandidateTrace -Defined in: [candidate-execution/types.ts:333](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L333) +Defined in: [candidate-execution/types.ts:303](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L303) #### Properties @@ -3341,45 +3123,59 @@ Defined in: [candidate-execution/types.ts:333](https://github.com/tangle-network > **runId**: `string` -Defined in: [candidate-execution/types.ts:334](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L334) +Defined in: [candidate-execution/types.ts:304](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L304) ##### tags > **tags**: `Readonly`\<`Record`\<`string`, `string`\>\> -Defined in: [candidate-execution/types.ts:335](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L335) +Defined in: [candidate-execution/types.ts:305](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L305) ##### env > **env**: `Readonly`\<`Record`\<`string`, `string`\>\> -Defined in: [candidate-execution/types.ts:336](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L336) +Defined in: [candidate-execution/types.ts:306](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L306) *** -### PreparedAgentCandidateExecution +### PreparedAgentCandidateExecution + +Defined in: [candidate-execution/types.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L309) + +#### Properties + +##### bundle + +> `readonly` **bundle**: `AgentCandidateBundle` + +Defined in: [candidate-execution/types.ts:310](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L310) + +##### benchmark + +> `readonly` **benchmark**: `object` -Defined in: [candidate-execution/types.ts:339](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L339) +Defined in: [candidate-execution/types.ts:311](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L311) -#### Properties +###### suite -##### bundle +> `readonly` **suite**: `AgentCandidateBenchmarkSuite` -> `readonly` **bundle**: `AgentCandidateBundle` +###### task -Defined in: [candidate-execution/types.ts:340](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L340) +> `readonly` **task**: `AgentCandidateBenchmarkTask` ##### executionId > `readonly` **executionId**: `string` -Defined in: [candidate-execution/types.ts:341](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L341) +Defined in: [candidate-execution/types.ts:315](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L315) ##### roots > `readonly` **roots**: `object` -Defined in: [candidate-execution/types.ts:342](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L342) +Defined in: [candidate-execution/types.ts:316](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L316) ###### execution @@ -3413,7 +3209,7 @@ Defined in: [candidate-execution/types.ts:342](https://github.com/tangle-network > `readonly` **profilePlan**: `object` -Defined in: [candidate-execution/types.ts:353](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L353) +Defined in: [candidate-execution/types.ts:327](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L327) ###### value @@ -3427,11 +3223,17 @@ Defined in: [candidate-execution/types.ts:353](https://github.com/tangle-network > **written**: readonly `string`[] +##### profileActivation + +> `readonly` **profileActivation**: `AgentCandidateProfileActivation` + +Defined in: [candidate-execution/types.ts:332](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L332) + ##### executionPlan > `readonly` **executionPlan**: `object` -Defined in: [candidate-execution/types.ts:358](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L358) +Defined in: [candidate-execution/types.ts:333](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L333) ###### value @@ -3445,55 +3247,55 @@ Defined in: [candidate-execution/types.ts:358](https://github.com/tangle-network > `readonly` **materializationReceipt**: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateMaterializationReceipt`\> -Defined in: [candidate-execution/types.ts:362](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L362) +Defined in: [candidate-execution/types.ts:337](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L337) ##### launch > `readonly` **launch**: [`PreparedAgentCandidateLaunch`](#preparedagentcandidatelaunch) -Defined in: [candidate-execution/types.ts:363](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L363) +Defined in: [candidate-execution/types.ts:338](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L338) ##### instruction > `readonly` **instruction**: [`PreparedAgentCandidateInstruction`](#preparedagentcandidateinstruction) -Defined in: [candidate-execution/types.ts:364](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L364) +Defined in: [candidate-execution/types.ts:339](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L339) ##### resolvedModel > `readonly` **resolvedModel**: `AgentCandidateResolvedModel` -Defined in: [candidate-execution/types.ts:365](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L365) +Defined in: [candidate-execution/types.ts:340](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L340) ##### knowledge? > `readonly` `optional` **knowledge?**: [`PreparedAgentCandidateKnowledge`](#preparedagentcandidateknowledge) -Defined in: [candidate-execution/types.ts:366](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L366) +Defined in: [candidate-execution/types.ts:341](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L341) ##### trace > `readonly` **trace**: [`PreparedAgentCandidateTrace`](#preparedagentcandidatetrace) -Defined in: [candidate-execution/types.ts:367](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L367) +Defined in: [candidate-execution/types.ts:342](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L342) ##### memory > `readonly` **memory**: `AgentCandidateEffectiveMemory` -Defined in: [candidate-execution/types.ts:368](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L368) +Defined in: [candidate-execution/types.ts:343](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L343) ##### \[preparedCandidateBrand\] > `readonly` **\[preparedCandidateBrand\]**: `true` -Defined in: [candidate-execution/types.ts:369](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L369) +Defined in: [candidate-execution/types.ts:344](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L344) *** ### AgentCandidateProtectedRunCapture -Defined in: [candidate-execution/types.ts:372](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L372) +Defined in: [candidate-execution/types.ts:349](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L349) #### Properties @@ -3501,61 +3303,19 @@ Defined in: [candidate-execution/types.ts:372](https://github.com/tangle-network > **executionId**: `string` -Defined in: [candidate-execution/types.ts:373](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L373) +Defined in: [candidate-execution/types.ts:350](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L350) ##### termination > **termination**: `AgentCandidateTermination` -Defined in: [candidate-execution/types.ts:374](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L374) - -*** - -### AgentCandidateExecutorTaskOutcomeCapture - -Defined in: [candidate-execution/types.ts:378](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L378) - -Raw evaluator capture made only after the candidate process is dead. - -#### Properties - -##### resultTree - -> **resultTree**: `string` - -Defined in: [candidate-execution/types.ts:380](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L380) - -Claimed final tree. The runtime recomputes it independently from `gitDiff`. - -##### afterState - -> **afterState**: `AgentCandidateWorkspaceManifestMaterial` - -Defined in: [candidate-execution/types.ts:382](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L382) - -Complete evaluator-captured workspace description after candidate execution. - -##### archive - -> **archive**: `Uint8Array` - -Defined in: [candidate-execution/types.ts:384](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L384) - -Reproducible workspace archive corresponding to `afterState`. - -##### gitDiff - -> **gitDiff**: `Uint8Array` - -Defined in: [candidate-execution/types.ts:386](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L386) - -Exact binary patch from the signed task base to `afterState`. +Defined in: [candidate-execution/types.ts:351](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L351) *** ### AgentCandidateExecutorMemoryCapture -Defined in: [candidate-execution/types.ts:390](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L390) +Defined in: [candidate-execution/types.ts:374](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L374) Raw isolated-memory capture made only after access has been revoked. @@ -3565,83 +3325,51 @@ Raw isolated-memory capture made only after access has been revoked. > `readonly` **afterState**: `AgentCandidateWorkspaceManifestMaterial` -Defined in: [candidate-execution/types.ts:391](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L391) +Defined in: [candidate-execution/types.ts:375](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L375) ##### archive > `readonly` **archive**: `Uint8Array` -Defined in: [candidate-execution/types.ts:392](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L392) +Defined in: [candidate-execution/types.ts:376](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L376) *** ### AgentCandidateExecutorFinalCapture -Defined in: [candidate-execution/types.ts:396](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L396) +Defined in: [candidate-execution/types.ts:380](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L380) -Idempotent executor result after process death and trace drain. +Replayable evaluator result captured only after process death and trace drain. #### Properties -##### stopped - -> `readonly` **stopped**: `true` - -Defined in: [candidate-execution/types.ts:397](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L397) - ##### taskOutcome? > `readonly` `optional` **taskOutcome?**: [`AgentCandidateExecutorTaskOutcomeCapture`](#agentcandidateexecutortaskoutcomecapture) -Defined in: [candidate-execution/types.ts:398](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L398) +Defined in: [candidate-execution/types.ts:381](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L381) ##### memoryAfter? > `readonly` `optional` **memoryAfter?**: [`AgentCandidateExecutorMemoryCapture`](#agentcandidateexecutormemorycapture) -Defined in: [candidate-execution/types.ts:400](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L400) +Defined in: [candidate-execution/types.ts:383](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L383) Required only when the prepared candidate uses isolated task memory. -*** - -### VerifiedAgentCandidateTaskOutcome - -Defined in: [candidate-execution/types.ts:404](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L404) +##### evidence? -Branded task outcome that has survived independent patch and tree verification. +> `readonly` `optional` **evidence?**: `Uint8Array`\<`ArrayBufferLike`\> -#### Properties - -##### evidence - -> `readonly` **evidence**: `AgentCandidateTaskOutcomeEvidence` & `object` - -Defined in: [candidate-execution/types.ts:405](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L405) - -###### Type Declaration - -###### artifact - -> `readonly` **artifact**: `AgentCandidateArtifactRef` - -##### patch - -> `readonly` **patch**: `Uint8Array` - -Defined in: [candidate-execution/types.ts:408](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L408) +Defined in: [candidate-execution/types.ts:385](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L385) -##### \[verifiedTaskOutcomeBrand\] - -> `readonly` **\[verifiedTaskOutcomeBrand\]**: `true` - -Defined in: [candidate-execution/types.ts:409](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L409) +Executor-native bytes preserved when a fresh worker cannot reconstruct a verified outcome. *** ### AgentCandidateBenchmarkGraderPort -Defined in: [candidate-execution/types.ts:421](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L421) +Defined in: [candidate-execution/types.ts:422](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L422) Evaluator-owned executable grader, pinned by immutable implementation bytes. @@ -3657,19 +3385,19 @@ copying an expected digest from ambient configuration. > `readonly` **name**: `string` -Defined in: [candidate-execution/types.ts:422](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L422) +Defined in: [candidate-execution/types.ts:423](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L423) ##### version > `readonly` **version**: `string` -Defined in: [candidate-execution/types.ts:423](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L423) +Defined in: [candidate-execution/types.ts:424](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L424) ##### artifact > `readonly` **artifact**: `AgentCandidateArtifactRef` -Defined in: [candidate-execution/types.ts:424](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L424) +Defined in: [candidate-execution/types.ts:425](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L425) #### Methods @@ -3677,7 +3405,7 @@ Defined in: [candidate-execution/types.ts:424](https://github.com/tangle-network > **run**(`input`): `Promise`\<\{ `evaluation`: `BenchmarkEvaluation`; `evidence`: `Uint8Array`; `binding`: \{ `implementationDigest`: `` `sha256:${string}` ``; `taskOutcomeDigest`: `` `sha256:${string}` ``; `outputDigest`: `` `sha256:${string}` ``; \}; \}\> -Defined in: [candidate-execution/types.ts:425](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L425) +Defined in: [candidate-execution/types.ts:426](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L426) ###### Parameters @@ -3723,7 +3451,7 @@ Frozen result deadline; runners must stop work and side effects when aborted. ### AgentCandidateExecutorRequest -Defined in: [candidate-execution/types.ts:453](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L453) +Defined in: [candidate-execution/types.ts:454](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L454) One detached request passed to the trusted environment-specific executor. @@ -3733,13 +3461,27 @@ One detached request passed to the trusted environment-specific executor. > `readonly` **executionId**: `string` -Defined in: [candidate-execution/types.ts:454](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L454) +Defined in: [candidate-execution/types.ts:455](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L455) + +##### benchmark + +> `readonly` **benchmark**: `object` + +Defined in: [candidate-execution/types.ts:456](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L456) + +###### suite + +> `readonly` **suite**: `AgentCandidateBenchmarkSuite` + +###### task + +> `readonly` **task**: `AgentCandidateBenchmarkTask` ##### inputs > `readonly` **inputs**: `object` -Defined in: [candidate-execution/types.ts:456](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L456) +Defined in: [candidate-execution/types.ts:458](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L458) Immutable bytes from which the executor creates fresh isolated workspaces. @@ -3763,7 +3505,7 @@ Immutable bytes from which the executor creates fresh isolated workspaces. > `readonly` **roots**: `object` -Defined in: [candidate-execution/types.ts:463](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L463) +Defined in: [candidate-execution/types.ts:465](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L465) ###### taskRoot @@ -3777,7 +3519,7 @@ Defined in: [candidate-execution/types.ts:463](https://github.com/tangle-network > `readonly` **profilePlan**: `object` -Defined in: [candidate-execution/types.ts:464](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L464) +Defined in: [candidate-execution/types.ts:466](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L466) ###### value @@ -3791,11 +3533,17 @@ Defined in: [candidate-execution/types.ts:464](https://github.com/tangle-network > **written**: readonly `string`[] +##### profileActivation + +> `readonly` **profileActivation**: `AgentCandidateProfileActivation` + +Defined in: [candidate-execution/types.ts:467](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L467) + ##### executionPlan > `readonly` **executionPlan**: `object` -Defined in: [candidate-execution/types.ts:465](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L465) +Defined in: [candidate-execution/types.ts:468](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L468) ###### value @@ -3809,31 +3557,31 @@ Defined in: [candidate-execution/types.ts:465](https://github.com/tangle-network > `readonly` **materializationReceipt**: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateMaterializationReceipt`\> -Defined in: [candidate-execution/types.ts:466](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L466) +Defined in: [candidate-execution/types.ts:469](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L469) ##### launch > `readonly` **launch**: [`PreparedAgentCandidateLaunch`](#preparedagentcandidatelaunch) -Defined in: [candidate-execution/types.ts:467](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L467) +Defined in: [candidate-execution/types.ts:470](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L470) ##### instruction > `readonly` **instruction**: [`PreparedAgentCandidateInstruction`](#preparedagentcandidateinstruction) -Defined in: [candidate-execution/types.ts:468](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L468) +Defined in: [candidate-execution/types.ts:471](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L471) ##### resolvedModel > `readonly` **resolvedModel**: `AgentCandidateResolvedModel` -Defined in: [candidate-execution/types.ts:469](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L469) +Defined in: [candidate-execution/types.ts:472](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L472) ##### hardLimits > `readonly` **hardLimits**: `Pick`\<`AgentCandidateExecutionLimits`, `"timeoutMs"`\> -Defined in: [candidate-execution/types.ts:471](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L471) +Defined in: [candidate-execution/types.ts:474](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L474) Mechanically enforced by the runtime plus executor process-death acknowledgement. @@ -3841,7 +3589,7 @@ Mechanically enforced by the runtime plus executor process-death acknowledgement > `readonly` **observedLimits**: `Pick`\<`AgentCandidateExecutionLimits`, `"maxSteps"`\> -Defined in: [candidate-execution/types.ts:473](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L473) +Defined in: [candidate-execution/types.ts:476](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L476) Validity bound checked against protected traces; generic black-box executors cannot preempt it. @@ -3849,25 +3597,25 @@ Validity bound checked against protected traces; generic black-box executors can > `readonly` `optional` **knowledge?**: [`PreparedAgentCandidateKnowledge`](#preparedagentcandidateknowledge) -Defined in: [candidate-execution/types.ts:474](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L474) +Defined in: [candidate-execution/types.ts:477](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L477) ##### trace > `readonly` **trace**: [`PreparedAgentCandidateTrace`](#preparedagentcandidatetrace) -Defined in: [candidate-execution/types.ts:475](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L475) +Defined in: [candidate-execution/types.ts:478](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L478) ##### memory > `readonly` **memory**: `AgentCandidateEffectiveMemory` -Defined in: [candidate-execution/types.ts:476](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L476) +Defined in: [candidate-execution/types.ts:479](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L479) *** ### AgentCandidateExecutorPort -Defined in: [candidate-execution/types.ts:487](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L487) +Defined in: [candidate-execution/types.ts:490](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L490) Executes one prepared request inside an evaluator-owned isolation boundary. @@ -3882,7 +3630,7 @@ no candidate-authored usage or score fields. > **execute**(`request`, `context`): `Promise`\<[`AgentCandidateProtectedRunCapture`](#agentcandidateprotectedruncapture)\> -Defined in: [candidate-execution/types.ts:488](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L488) +Defined in: [candidate-execution/types.ts:491](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L491) ###### Parameters @@ -3912,17 +3660,13 @@ Absolute epoch-millisecond deadline owned by the runtime. `Promise`\<[`AgentCandidateProtectedRunCapture`](#agentcandidateprotectedruncapture)\> -##### stopAndCapture() +##### stop() -> **stopAndCapture**(`request`, `context`): `Promise`\<[`AgentCandidateExecutorFinalCapture`](#agentcandidateexecutorfinalcapture)\> +> **stop**(`request`, `context`): `Promise`\<\{ `stopped`: `true`; \}\> -Defined in: [candidate-execution/types.ts:505](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L505) +Defined in: [candidate-execution/types.ts:502](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L502) -Kill any process/container still associated with the request, drain trace -writes, and capture the final task workspace before teardown. -The runtime calls this on success, failure, and timeout before model settlement. -Implementations must be idempotent and concurrency-safe for this exact -execution/plan pair because a fresh worker may repeat crash recovery. +Kill the exact process/container and drain trace writes. Must be idempotent. ###### Parameters @@ -3954,13 +3698,67 @@ Absolute execution deadline; a later stop acknowledgement cannot produce success ###### Returns +`Promise`\<\{ `stopped`: `true`; \}\> + +##### capture() + +> **capture**(`request`, `context`): `Promise`\<[`AgentCandidateExecutorFinalCapture`](#agentcandidateexecutorfinalcapture)\> + +Defined in: [candidate-execution/types.ts:514](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L514) + +Capture immutable final evidence after stop. Must be replayable by a fresh worker. + +###### Parameters + +###### request + +[`AgentCandidateExecutorStopRequest`](#agentcandidateexecutorstoprequest) + +###### context + +###### traceStore + +`TraceStore` + +###### signal + +`AbortSignal` + +Aborted at the frozen execution deadline or evaluator cleanup deadline. + +###### Returns + `Promise`\<[`AgentCandidateExecutorFinalCapture`](#agentcandidateexecutorfinalcapture)\> +##### dispose()? + +> `optional` **dispose**(`request`, `context`): `Promise`\<\{ `disposed`: `true`; \}\> + +Defined in: [candidate-execution/types.ts:523](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L523) + +Remove evaluator-owned execution resources after final capture. Must be idempotent. + +###### Parameters + +###### request + +[`AgentCandidateExecutorStopRequest`](#agentcandidateexecutorstoprequest) + +###### context + +###### signal + +`AbortSignal` + +###### Returns + +`Promise`\<\{ `disposed`: `true`; \}\> + *** ### AgentCandidateExecutorStopRequest -Defined in: [candidate-execution/types.ts:519](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L519) +Defined in: [candidate-execution/types.ts:532](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L532) Opaque process identity used for termination without re-exposing launch credentials. @@ -3970,19 +3768,19 @@ Opaque process identity used for termination without re-exposing launch credenti > `readonly` **executionId**: `string` -Defined in: [candidate-execution/types.ts:520](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L520) +Defined in: [candidate-execution/types.ts:533](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L533) ##### executionPlanDigest > `readonly` **executionPlanDigest**: `` `sha256:${string}` `` -Defined in: [candidate-execution/types.ts:521](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L521) +Defined in: [candidate-execution/types.ts:534](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L534) *** ### AgentCandidateExecutorWorkspaceInput -Defined in: [candidate-execution/types.ts:524](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L524) +Defined in: [candidate-execution/types.ts:537](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L537) #### Properties @@ -3990,19 +3788,19 @@ Defined in: [candidate-execution/types.ts:524](https://github.com/tangle-network > `readonly` **snapshot**: `AgentCandidateWorkspaceSnapshotEvidence` -Defined in: [candidate-execution/types.ts:525](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L525) +Defined in: [candidate-execution/types.ts:538](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L538) ##### files > `readonly` **files**: readonly [`AgentCandidateExecutorWorkspaceFile`](#agentcandidateexecutorworkspacefile)[] -Defined in: [candidate-execution/types.ts:526](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L526) +Defined in: [candidate-execution/types.ts:539](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L539) *** ### AgentCandidateExecutorWorkspaceFile -Defined in: [candidate-execution/types.ts:529](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L529) +Defined in: [candidate-execution/types.ts:542](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L542) #### Properties @@ -4010,25 +3808,27 @@ Defined in: [candidate-execution/types.ts:529](https://github.com/tangle-network > `readonly` **path**: `string` -Defined in: [candidate-execution/types.ts:530](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L530) +Defined in: [candidate-execution/types.ts:543](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L543) ##### mode > `readonly` **mode**: `number` -Defined in: [candidate-execution/types.ts:531](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L531) +Defined in: [candidate-execution/types.ts:544](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L544) ##### bytes > `readonly` **bytes**: `Uint8Array` -Defined in: [candidate-execution/types.ts:532](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L532) +Defined in: [candidate-execution/types.ts:545](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L545) *** ### AgentCandidateExecutorProfileFile -Defined in: [candidate-execution/types.ts:535](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L535) +Defined in: [candidate-execution/types.ts:549](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L549) + +One exact profile file supplied to an evaluator-owned executor. #### Properties @@ -4036,19 +3836,19 @@ Defined in: [candidate-execution/types.ts:535](https://github.com/tangle-network > `readonly` **path**: `string` -Defined in: [candidate-execution/types.ts:536](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L536) +Defined in: [candidate-execution/types.ts:550](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L550) ##### mode > `readonly` **mode**: `number` -Defined in: [candidate-execution/types.ts:537](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L537) +Defined in: [candidate-execution/types.ts:551](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L551) ##### bytes > `readonly` **bytes**: `Uint8Array` -Defined in: [candidate-execution/types.ts:538](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L538) +Defined in: [candidate-execution/types.ts:552](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L552) *** @@ -4104,7 +3904,7 @@ Defined in: [candidate-execution/workspace-archive.ts:63](https://github.com/tan ### CaptureAgentCandidateWorkspaceOptions -Defined in: [candidate-execution/workspace-archive.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L102) +Defined in: [candidate-execution/workspace-archive.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L101) #### Properties @@ -4112,7 +3912,7 @@ Defined in: [candidate-execution/workspace-archive.ts:102](https://github.com/ta > `optional` **includeRepository?**: `boolean` -Defined in: [candidate-execution/workspace-archive.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L104) +Defined in: [candidate-execution/workspace-archive.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L103) Include Git HEAD so task preparation can prove its exact commit and tree. @@ -4120,13 +3920,13 @@ Include Git HEAD so task preparation can prove its exact commit and tree. > `optional` **limits?**: `Partial`\<[`AgentCandidateWorkspaceArchiveLimits`](#agentcandidateworkspacearchivelimits)\> -Defined in: [candidate-execution/workspace-archive.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L105) +Defined in: [candidate-execution/workspace-archive.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L104) ##### artifactPersistence? > `optional` **artifactPersistence?**: `object` -Defined in: [candidate-execution/workspace-archive.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L107) +Defined in: [candidate-execution/workspace-archive.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L106) Use the evaluator-owned artifact store when manifest or archive bytes should not be embedded. @@ -4146,7 +3946,7 @@ Use the evaluator-owned artifact store when manifest or archive bytes should not ### CreateAgentCandidateWorkspacePortOptions -Defined in: [candidate-execution/workspace-archive.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L114) +Defined in: [candidate-execution/workspace-archive.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L113) #### Properties @@ -4154,13 +3954,13 @@ Defined in: [candidate-execution/workspace-archive.ts:114](https://github.com/ta > `optional` **limits?**: `Partial`\<[`AgentCandidateWorkspaceArchiveLimits`](#agentcandidateworkspacearchivelimits)\> -Defined in: [candidate-execution/workspace-archive.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L115) +Defined in: [candidate-execution/workspace-archive.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L114) *** ### CapturedAgentCandidateWorkspace -Defined in: [candidate-execution/workspace-archive.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L118) +Defined in: [candidate-execution/workspace-archive.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L117) #### Properties @@ -4168,13 +3968,13 @@ Defined in: [candidate-execution/workspace-archive.ts:118](https://github.com/ta > `readonly` **snapshot**: `AgentCandidateWorkspaceSnapshotEvidence` -Defined in: [candidate-execution/workspace-archive.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L119) +Defined in: [candidate-execution/workspace-archive.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L118) ##### archive > `readonly` **archive**: `Uint8Array` -Defined in: [candidate-execution/workspace-archive.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L121) +Defined in: [candidate-execution/workspace-archive.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L120) Caller-owned bytes accepted by createAgentCandidateWorkspacePort. @@ -5743,29 +5543,23 @@ producer, which mutates an isolated git worktree via a pluggable #### Properties -##### schemaVersion - -> `readonly` **schemaVersion**: `1` - -Defined in: [improvement/agentic-generator.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L84) - ##### generation > `readonly` **generation**: `number` \| `null` -Defined in: [improvement/agentic-generator.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L85) +Defined in: [improvement/agentic-generator.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L84) ##### candidateIndex > `readonly` **candidateIndex**: `number` \| `null` -Defined in: [improvement/agentic-generator.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L86) +Defined in: [improvement/agentic-generator.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L85) ##### shot > `readonly` **shot**: `number` -Defined in: [improvement/agentic-generator.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L88) +Defined in: [improvement/agentic-generator.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L87) One-based shot number within this candidate. @@ -5773,103 +5567,103 @@ One-based shot number within this candidate. > `readonly` **maxShots**: `number` -Defined in: [improvement/agentic-generator.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L89) +Defined in: [improvement/agentic-generator.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L88) ##### harness > `readonly` **harness**: [`LocalHarness`](mcp.md#localharness) -Defined in: [improvement/agentic-generator.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L90) +Defined in: [improvement/agentic-generator.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L89) ##### model > `readonly` **model**: `string` \| `null` -Defined in: [improvement/agentic-generator.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L91) +Defined in: [improvement/agentic-generator.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L90) ##### reasoningEffort > `readonly` **reasoningEffort**: `ReasoningEffort` \| `null` -Defined in: [improvement/agentic-generator.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L92) +Defined in: [improvement/agentic-generator.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L91) ##### promptSha256 > `readonly` **promptSha256**: `` `sha256:${string}` `` -Defined in: [improvement/agentic-generator.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L93) +Defined in: [improvement/agentic-generator.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L92) ##### startedAt > `readonly` **startedAt**: `string` -Defined in: [improvement/agentic-generator.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L94) +Defined in: [improvement/agentic-generator.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L93) ##### completedAt > `readonly` **completedAt**: `string` -Defined in: [improvement/agentic-generator.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L95) +Defined in: [improvement/agentic-generator.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L94) ##### durationMs > `readonly` **durationMs**: `number` -Defined in: [improvement/agentic-generator.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L96) +Defined in: [improvement/agentic-generator.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L95) ##### exitCode > `readonly` **exitCode**: `number` \| `null` -Defined in: [improvement/agentic-generator.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L97) +Defined in: [improvement/agentic-generator.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L96) ##### timedOut > `readonly` **timedOut**: `boolean` -Defined in: [improvement/agentic-generator.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L98) +Defined in: [improvement/agentic-generator.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L97) ##### killedBySignal > `readonly` **killedBySignal**: `Signals` \| `null` -Defined in: [improvement/agentic-generator.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L99) +Defined in: [improvement/agentic-generator.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L98) ##### stdoutBytes > `readonly` **stdoutBytes**: `number` \| `null` -Defined in: [improvement/agentic-generator.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L100) +Defined in: [improvement/agentic-generator.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L99) ##### stdoutSha256 > `readonly` **stdoutSha256**: `` `sha256:${string}` `` \| `null` -Defined in: [improvement/agentic-generator.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L101) +Defined in: [improvement/agentic-generator.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L100) ##### stderrBytes > `readonly` **stderrBytes**: `number` \| `null` -Defined in: [improvement/agentic-generator.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L102) +Defined in: [improvement/agentic-generator.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L101) ##### stderrSha256 > `readonly` **stderrSha256**: `` `sha256:${string}` `` \| `null` -Defined in: [improvement/agentic-generator.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L103) +Defined in: [improvement/agentic-generator.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L102) ##### usage > `readonly` **usage**: [`CodexTokenUsage`](mcp.md#codextokenusage) \| `null` -Defined in: [improvement/agentic-generator.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L104) +Defined in: [improvement/agentic-generator.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L103) ##### profileWorkspacePlanDigest > `readonly` **profileWorkspacePlanDigest**: `string` \| `null` -Defined in: [improvement/agentic-generator.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L106) +Defined in: [improvement/agentic-generator.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L105) Digest of the exact profile-file workspace plan applied for this shot. @@ -5877,13 +5671,13 @@ Digest of the exact profile-file workspace plan applied for this shot. > `readonly` **profileWorkspaceFileCount**: `number` -Defined in: [improvement/agentic-generator.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L107) +Defined in: [improvement/agentic-generator.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L106) ##### costCallId > `readonly` **costCallId**: `string` \| `null` -Defined in: [improvement/agentic-generator.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L109) +Defined in: [improvement/agentic-generator.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L108) Shared run-ledger call id for this exact shot. @@ -5891,7 +5685,7 @@ Shared run-ledger call id for this exact shot. > `readonly` **costBasis**: `"unknown"` \| `"provider-reported"` \| `"estimated-pricing"` -Defined in: [improvement/agentic-generator.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L111) +Defined in: [improvement/agentic-generator.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L110) Whether dollars came from the provider, the pricing table, or are unknown. @@ -5899,13 +5693,13 @@ Whether dollars came from the provider, the pricing table, or are unknown. > `readonly` **costUsd**: `number` \| `null` -Defined in: [improvement/agentic-generator.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L112) +Defined in: [improvement/agentic-generator.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L111) ##### costUsdKnown > `readonly` **costUsdKnown**: `boolean` -Defined in: [improvement/agentic-generator.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L114) +Defined in: [improvement/agentic-generator.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L113) True only for a provider-reported amount, never for a pricing estimate. @@ -5913,19 +5707,19 @@ True only for a provider-reported amount, never for a pricing estimate. > `readonly` **evidence**: [`CodexExecutionEvidence`](mcp.md#codexexecutionevidence) \| `null` -Defined in: [improvement/agentic-generator.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L115) +Defined in: [improvement/agentic-generator.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L114) ##### error > `readonly` **error**: \{ `name`: `string`; `message`: `string`; \} \| `null` -Defined in: [improvement/agentic-generator.ts:116](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L116) +Defined in: [improvement/agentic-generator.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L115) *** ### AgenticGeneratorOptions -Defined in: [improvement/agentic-generator.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L159) +Defined in: [improvement/agentic-generator.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L158) `@tangle-network/agent-runtime` improvement — the CODE-surface proposer for agent-eval's improvement loop. @@ -5943,7 +5737,7 @@ producer, which mutates an isolated git worktree via a pluggable > `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) -Defined in: [improvement/agentic-generator.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L161) +Defined in: [improvement/agentic-generator.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L160) Local coding harness to run in the worktree. Default `claude`. @@ -5951,7 +5745,7 @@ Local coding harness to run in the worktree. Default `claude`. > `optional` **profile?**: `AgentProfile` -Defined in: [improvement/agentic-generator.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L164) +Defined in: [improvement/agentic-generator.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L163) Author profile rendered through the canonical harness mapper. Required for reproducible Codex so model and reasoning settings are explicit. @@ -5960,7 +5754,7 @@ Author profile rendered through the canonical harness mapper. Required > `optional` **codexReproducible?**: `boolean` -Defined in: [improvement/agentic-generator.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L167) +Defined in: [improvement/agentic-generator.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L166) Run Codex with isolated configuration, exact prompt evidence, and required terminal token usage. Requires `harness: 'codex'` and `profile`. @@ -5969,7 +5763,7 @@ Run Codex with isolated configuration, exact prompt evidence, and required > `optional` **codexReadDeniedPaths?**: readonly `string`[] \| ((`worktreePath`) => readonly `string`[]) -Defined in: [improvement/agentic-generator.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L170) +Defined in: [improvement/agentic-generator.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L169) Absolute paths reproducible Codex must not read. A function can derive candidate-specific paths after the driver creates its worktree. @@ -5978,7 +5772,7 @@ Absolute paths reproducible Codex must not read. A function can derive > `optional` **onShotCompleted?**: (`receipt`, `execution`) => `void` \| `Promise`\<`void`\> -Defined in: [improvement/agentic-generator.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L175) +Defined in: [improvement/agentic-generator.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L174) Awaited once for every attempted author shot, including process failures. The second argument preserves the exact harness result, including stdout @@ -6003,7 +5797,7 @@ Awaited once for every attempted author shot, including process failures. > `optional` **onShotDisposition?**: (`receipt`, `disposition`) => `void` \| `Promise`\<`void`\> -Defined in: [improvement/agentic-generator.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L181) +Defined in: [improvement/agentic-generator.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L180) Awaited after worktree inspection and before the shot is accepted, retried, or discarded. Throwing aborts the candidate. @@ -6026,7 +5820,7 @@ Awaited after worktree inspection and before the shot is accepted, > `optional` **maximumCharge?**: `MaximumCharge` -Defined in: [improvement/agentic-generator.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L189) +Defined in: [improvement/agentic-generator.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L188) Optional hard upper bound passed to the run-wide CostLedger before each author shot. This MUST be enforced by the provider or executor; a planning @@ -6037,7 +5831,7 @@ Optional hard upper bound passed to the run-wide CostLedger before each > `optional` **timeoutMs?**: `number` -Defined in: [improvement/agentic-generator.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L191) +Defined in: [improvement/agentic-generator.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L190) Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). @@ -6045,7 +5839,7 @@ Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). > `optional` **buildPrompt?**: (`args`) => `string` -Defined in: [improvement/agentic-generator.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L194) +Defined in: [improvement/agentic-generator.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L193) Build the harness task prompt from the report + findings. Override for domain phrasing; the default turns findings into a concrete coder task. @@ -6070,7 +5864,7 @@ Build the harness task prompt from the report + findings. Override for > `optional` **verify?**: [`Verifier`](#verifier) -Defined in: [improvement/agentic-generator.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L200) +Defined in: [improvement/agentic-generator.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L199) Verify the worktree after each dirtying shot. When set, a candidate that fails verification is NOT returned — the failure feeds the next shot @@ -6082,7 +5876,7 @@ Verify the worktree after each dirtying shot. When set, a candidate that > `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> -Defined in: [improvement/agentic-generator.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L202) +Defined in: [improvement/agentic-generator.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L201) Test seam — inject the harness runner (defaults to `runLocalHarness`). @@ -6118,7 +5912,7 @@ Does NOT throw when: > `optional` **isDirty?**: (`worktreePath`) => `boolean` -Defined in: [improvement/agentic-generator.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L204) +Defined in: [improvement/agentic-generator.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L203) Test seam — inject the worktree-dirty check (defaults to `git status`). @@ -6136,7 +5930,7 @@ Test seam — inject the worktree-dirty check (defaults to `git status`). ### ImproveSkillsOptions -Defined in: [improvement/improve.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L141) +Defined in: [improvement/improve.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L142) #### Properties @@ -6144,7 +5938,7 @@ Defined in: [improvement/improve.ts:141](https://github.com/tangle-network/agent > **document**: `string` -Defined in: [improvement/improve.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L143) +Defined in: [improvement/improve.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L144) The skill document's current text — the baseline `skillOptProposer` patches. @@ -6152,7 +5946,7 @@ The skill document's current text — the baseline `skillOptProposer` patches. > `optional` **writeBack?**: (`winnerDocument`) => `void` \| `Promise`\<`void`\> -Defined in: [improvement/improve.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L147) +Defined in: [improvement/improve.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L148) Persist the shipped winner document (write the file the profile ref points at). Called only on a ship verdict. When omitted, the winner is still returned in @@ -6172,7 +5966,7 @@ Persist the shipped winner document (write the file the profile ref points at). ### ImproveMemoryOptions -Defined in: [improvement/improve.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L150) +Defined in: [improvement/improve.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L151) #### Properties @@ -6180,7 +5974,7 @@ Defined in: [improvement/improve.ts:150](https://github.com/tangle-network/agent > **document**: `string` -Defined in: [improvement/improve.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L152) +Defined in: [improvement/improve.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L153) Current durable memory text used as the measured baseline. @@ -6188,7 +5982,7 @@ Current durable memory text used as the measured baseline. > `optional` **writeBack?**: (`winnerDocument`) => `void` \| `Promise`\<`void`\> -Defined in: [improvement/improve.ts:154](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L154) +Defined in: [improvement/improve.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L155) Persist the promoted memory document. Never called on hold or error. @@ -6206,7 +6000,7 @@ Persist the promoted memory document. Never called on hold or error. ### ImproveCodeOptions -Defined in: [improvement/improve.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L157) +Defined in: [improvement/improve.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L158) #### Properties @@ -6214,7 +6008,7 @@ Defined in: [improvement/improve.ts:157](https://github.com/tangle-network/agent > **repoRoot**: `string` -Defined in: [improvement/improve.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L159) +Defined in: [improvement/improve.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L160) Repo root candidate worktrees fork from. @@ -6222,7 +6016,7 @@ Repo root candidate worktrees fork from. > `optional` **baseRef?**: `string` -Defined in: [improvement/improve.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L161) +Defined in: [improvement/improve.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L162) Base ref candidates fork from. Default `main`. @@ -6230,7 +6024,7 @@ Base ref candidates fork from. Default `main`. > `optional` **worktreeDir?**: `string` -Defined in: [improvement/improve.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L163) +Defined in: [improvement/improve.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L164) Directory worktrees are created under. Default `/.worktrees`. @@ -6238,7 +6032,7 @@ Directory worktrees are created under. Default `/.worktrees`. > `optional` **worktree?**: `WorktreeAdapter` -Defined in: [improvement/improve.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L166) +Defined in: [improvement/improve.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L167) Git-compatible adapter override, primarily for tests. Candidate advancement still requires normal Git worktree and commit semantics. @@ -6247,7 +6041,7 @@ Git-compatible adapter override, primarily for tests. Candidate advancement > `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) -Defined in: [improvement/improve.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L168) +Defined in: [improvement/improve.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L169) Coding harness the agentic generator runs in each worktree. Default `claude`. @@ -6255,7 +6049,7 @@ Coding harness the agentic generator runs in each worktree. Default `claude`. > `optional` **verify?**: [`Verifier`](#verifier) -Defined in: [improvement/improve.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L171) +Defined in: [improvement/improve.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L172) Verify a candidate worktree before it becomes a measurable surface; failures feed the next shot (see `agenticGenerator.verify` / `commandVerifier`). @@ -6264,7 +6058,7 @@ Verify a candidate worktree before it becomes a measurable surface; failures > `optional` **timeoutMs?**: `number` -Defined in: [improvement/improve.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L173) +Defined in: [improvement/improve.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L174) Per-shot wall-clock timeout for the harness (ms). @@ -6272,7 +6066,7 @@ Per-shot wall-clock timeout for the harness (ms). > `optional` **generator?**: [`CandidateGenerator`](#candidategenerator) -Defined in: [improvement/improve.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L176) +Defined in: [improvement/improve.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L177) Byte-producer override — the test seam and the escape hatch for custom candidate production. When set, `harness`/`verify`/`timeoutMs` are unused. @@ -6281,7 +6075,7 @@ Byte-producer override — the test seam and the escape hatch for custom ### ImproveResult -Defined in: [improvement/improve.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L179) +Defined in: [improvement/improve.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L180) #### Type Parameters @@ -6299,7 +6093,7 @@ Defined in: [improvement/improve.ts:179](https://github.com/tangle-network/agent > **profile**: `AgentProfile` -Defined in: [improvement/improve.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L182) +Defined in: [improvement/improve.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L183) The profile after improvement: the winner surface applied back into the matching field when the gate shipped, else the input profile unchanged. @@ -6308,7 +6102,7 @@ The profile after improvement: the winner surface applied back into the > **shipped**: `boolean` -Defined in: [improvement/improve.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L184) +Defined in: [improvement/improve.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L185) True when `gateDecision === 'ship'`. @@ -6316,7 +6110,7 @@ True when `gateDecision === 'ship'`. > **lift**: `number` -Defined in: [improvement/improve.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L186) +Defined in: [improvement/improve.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L187) Held-out lift (`winner − baseline` composite). @@ -6324,7 +6118,7 @@ Held-out lift (`winner − baseline` composite). > **gateDecision**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` -Defined in: [improvement/improve.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L188) +Defined in: [improvement/improve.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L189) The five-valued gate verdict from `selfImprove`. @@ -6332,7 +6126,7 @@ The five-valued gate verdict from `selfImprove`. > **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> -Defined in: [improvement/improve.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L192) +Defined in: [improvement/improve.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L193) Full `selfImprove` result for advanced inspection. For code runs, `raw.winner.surface.worktreeRef` remains live after return whether the @@ -6344,7 +6138,7 @@ Full `selfImprove` result for advanced inspection. For code runs, > **dispose**(): `Promise`\<`void`\> -Defined in: [improvement/improve.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L195) +Defined in: [improvement/improve.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L196) Release resources owned by this result. Idempotent; currently disposes the returned code worktree and is a no-op for profile-only surfaces. @@ -6357,7 +6151,7 @@ Release resources owned by this result. Idempotent; currently disposes ### CandidateGenerator -Defined in: [improvement/improvement-driver.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L39) +Defined in: [improvement/improvement-driver.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L40) The byte-producing seam — the ONE thing that differs between the cheap reflective path and the full agentic path. A generator makes (uncommitted) @@ -6370,13 +6164,13 @@ The byte-producing seam — the ONE thing that differs between the cheap > **kind**: `string` -Defined in: [improvement/improvement-driver.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L40) +Defined in: [improvement/improvement-driver.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L41) ##### proposesWithoutFindings? > `optional` **proposesWithoutFindings?**: `boolean` -Defined in: [improvement/improvement-driver.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L51) +Defined in: [improvement/improvement-driver.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L52) Whether this generator can produce a candidate from an EMPTY findings set and no phase-2 report — i.e. it draws its change signal from the repo and @@ -6395,7 +6189,7 @@ Whether this generator can produce a candidate from an EMPTY findings set > **generate**(`args`): `Promise`\<\{ `applied`: `boolean`; `summary`: `string`; \}\> -Defined in: [improvement/improvement-driver.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L52) +Defined in: [improvement/improvement-driver.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L53) ###### Parameters @@ -6466,7 +6260,7 @@ Receipt attribution phase supplied alongside `costLedger`. ### ImprovementDriverOptions -Defined in: [improvement/improvement-driver.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L75) +Defined in: [improvement/improvement-driver.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L76) #### Properties @@ -6474,19 +6268,19 @@ Defined in: [improvement/improvement-driver.ts:75](https://github.com/tangle-net > **worktree**: `WorktreeAdapter` -Defined in: [improvement/improvement-driver.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L76) +Defined in: [improvement/improvement-driver.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L77) ##### generator > **generator**: [`CandidateGenerator`](#candidategenerator) -Defined in: [improvement/improvement-driver.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L77) +Defined in: [improvement/improvement-driver.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L78) ##### baseRef? > `optional` **baseRef?**: `string` -Defined in: [improvement/improvement-driver.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L80) +Defined in: [improvement/improvement-driver.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L81) Root ref for first-generation/direct callers. Default `main`. Later code generations retain the incumbent's original root. @@ -6495,7 +6289,7 @@ Root ref for first-generation/direct callers. Default `main`. ### ManagedImprovementDriver -Defined in: [improvement/improvement-driver.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L83) +Defined in: [improvement/improvement-driver.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L84) #### Extends @@ -6507,7 +6301,7 @@ Defined in: [improvement/improvement-driver.ts:83](https://github.com/tangle-net > **cleanup**(`retainWorktreeRefs?`): `Promise`\<`void`\> -Defined in: [improvement/improvement-driver.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L85) +Defined in: [improvement/improvement-driver.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L86) Remove every owned candidate except explicitly retained finalized winners. @@ -6545,59 +6339,33 @@ Defined in: [improvement/mcp-serve-verifier.ts:27](https://github.com/tangle-net ##### env? -> `optional` **env?**: `Record`\<`string`, `string`\> - -Defined in: [improvement/mcp-serve-verifier.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L29) - -Extra env for the server process (merged over `process.env`). - -##### timeoutMs? - -> `optional` **timeoutMs?**: `number` - -Defined in: [improvement/mcp-serve-verifier.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L31) - -Handshake timeout (ms). Default 30s. - -##### minTools? - -> `optional` **minTools?**: `number` - -Defined in: [improvement/mcp-serve-verifier.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L33) - -Minimum tools the server must expose to pass. Default 1. - -*** - -### AgentProfileDiffProposal - -Defined in: [improvement/profile-diff-proposer.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L20) +> `optional` **env?**: `Record`\<`string`, `string`\> -#### Properties +Defined in: [improvement/mcp-serve-verifier.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L29) -##### diff +Extra env for the server process (merged over `process.env`). -> **diff**: `AgentProfileDiff` +##### timeoutMs? -Defined in: [improvement/profile-diff-proposer.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L21) +> `optional` **timeoutMs?**: `number` -##### label? +Defined in: [improvement/mcp-serve-verifier.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L31) -> `optional` **label?**: `string` +Handshake timeout (ms). Default 30s. -Defined in: [improvement/profile-diff-proposer.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L22) +##### minTools? -##### rationale? +> `optional` **minTools?**: `number` -> `optional` **rationale?**: `string` +Defined in: [improvement/mcp-serve-verifier.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/mcp-serve-verifier.ts#L33) -Defined in: [improvement/profile-diff-proposer.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L23) +Minimum tools the server must expose to pass. Default 1. *** ### ProfileDiffProposerOptions -Defined in: [improvement/profile-diff-proposer.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L30) +Defined in: [improvement/profile-diff-proposer.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L24) #### Type Parameters @@ -6609,9 +6377,9 @@ Defined in: [improvement/profile-diff-proposer.ts:30](https://github.com/tangle- ##### proposeDiffs() -> **proposeDiffs**(`context`): readonly [`AgentProfileDiffProposal`](#agentprofilediffproposal)[] \| `Promise`\ +> **proposeDiffs**(`context`): readonly `AgentProfileDiff`[] \| `Promise`\ -Defined in: [improvement/profile-diff-proposer.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L31) +Defined in: [improvement/profile-diff-proposer.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L25) ###### Parameters @@ -6621,7 +6389,7 @@ Defined in: [improvement/profile-diff-proposer.ts:31](https://github.com/tangle- ###### Returns -readonly [`AgentProfileDiffProposal`](#agentprofilediffproposal)[] \| `Promise`\ +readonly `AgentProfileDiff`[] \| `Promise`\ *** @@ -6696,7 +6464,7 @@ Defined in: [improvement/reflective-generator.ts:22](https://github.com/tangle-n ### RunKnowledgeImprovementJobOptions -Defined in: [knowledge/improvement-job.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L21) +Defined in: [knowledge/improvement-job.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L46) #### Extends @@ -6708,61 +6476,61 @@ Defined in: [knowledge/improvement-job.ts:21](https://github.com/tangle-network/ > **budget**: [`Budget`](runtime.md#budget-12) -Defined in: [knowledge/improvement-job.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L23) +Defined in: [knowledge/improvement-job.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L48) ##### readinessCheck? > `optional` **readinessCheck?**: [`KnowledgeReadinessCheck`](#knowledgereadinesscheck) -Defined in: [knowledge/improvement-job.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L24) +Defined in: [knowledge/improvement-job.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L49) ##### backend? > `optional` **backend?**: [`ExecutorConfig`](runtime.md#executorconfig) -Defined in: [knowledge/improvement-job.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L25) +Defined in: [knowledge/improvement-job.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L50) ##### makeWorkerAgent? > `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) -Defined in: [knowledge/improvement-job.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L26) +Defined in: [knowledge/improvement-job.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L51) ##### harness? > `optional` **harness?**: `string` -Defined in: [knowledge/improvement-job.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L27) +Defined in: [knowledge/improvement-job.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L52) ##### supervisorModel? > `optional` **supervisorModel?**: `string` -Defined in: [knowledge/improvement-job.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L28) +Defined in: [knowledge/improvement-job.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L53) ##### supervisorSystemPrompt? > `optional` **supervisorSystemPrompt?**: `string` -Defined in: [knowledge/improvement-job.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L29) +Defined in: [knowledge/improvement-job.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L54) ##### superviseOptions? > `optional` **superviseOptions?**: `Partial`\<`Omit`\<[`SuperviseOptions`](runtime.md#superviseoptions), `"backend"` \| `"budget"` \| `"makeWorkerAgent"` \| `"deliverable"` \| `"allowedModels"`\>\> -Defined in: [knowledge/improvement-job.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L30) +Defined in: [knowledge/improvement-job.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L55) ##### allowedModels? > `optional` **allowedModels?**: readonly `string`[] -Defined in: [knowledge/improvement-job.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L36) +Defined in: [knowledge/improvement-job.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L61) ##### runSupervised? > `optional` **runSupervised?**: (`profile`, `task`, `opts`) => `Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> -Defined in: [knowledge/improvement-job.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L37) +Defined in: [knowledge/improvement-job.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L62) ###### Parameters @@ -6782,11 +6550,23 @@ Defined in: [knowledge/improvement-job.ts:37](https://github.com/tangle-network/ `Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> +##### candidateArtifacts? + +> `optional` **candidateArtifacts?**: [`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) + +Defined in: [knowledge/improvement-job.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L67) + +##### approval? + +> `optional` **approval?**: [`ApprovedKnowledgeImprovementCandidate`](#approvedknowledgeimprovementcandidate) + +Defined in: [knowledge/improvement-job.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L68) + ##### onMeasurement? > `optional` **onMeasurement?**: (`measurement`) => `void` \| `Promise`\<`void`\> -Defined in: [knowledge/improvement-job.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L42) +Defined in: [knowledge/improvement-job.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L69) ###### Parameters @@ -6800,9 +6580,59 @@ Defined in: [knowledge/improvement-job.ts:42](https://github.com/tangle-network/ *** +### ApprovedKnowledgeImprovementCandidate + +Defined in: [knowledge/improvement-job.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L72) + +#### Properties + +##### proposal + +> **proposal**: `AgentImprovementProposal` + +Defined in: [knowledge/improvement-job.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L73) + +##### review + +> **review**: `AgentImprovementReview` + +Defined in: [knowledge/improvement-job.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L74) + +##### activation + +> **activation**: `AgentImprovementActivation` + +Defined in: [knowledge/improvement-job.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L75) + +##### authorizeActivation + +> **authorizeActivation**: (`activation`, `proposal`, `review`) => `boolean` \| `Promise`\<`boolean`\> + +Defined in: [knowledge/improvement-job.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L76) + +###### Parameters + +###### activation + +`AgentImprovementActivation` + +###### proposal + +`AgentImprovementProposal` + +###### review + +`AgentImprovementReview` + +###### Returns + +`boolean` \| `Promise`\<`boolean`\> + +*** + ### KnowledgeImprovementJobMeasurement -Defined in: [knowledge/improvement-job.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L45) +Defined in: [knowledge/improvement-job.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L83) #### Properties @@ -6810,37 +6640,37 @@ Defined in: [knowledge/improvement-job.ts:45](https://github.com/tangle-network/ > **startedAt**: `string` -Defined in: [knowledge/improvement-job.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L46) +Defined in: [knowledge/improvement-job.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L84) ##### finishedAt > **finishedAt**: `string` -Defined in: [knowledge/improvement-job.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L47) +Defined in: [knowledge/improvement-job.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L85) ##### durationMs > **durationMs**: `number` -Defined in: [knowledge/improvement-job.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L48) +Defined in: [knowledge/improvement-job.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L86) ##### updateCalls > **updateCalls**: `number` -Defined in: [knowledge/improvement-job.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L49) +Defined in: [knowledge/improvement-job.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L87) ##### updateDurationMs > **updateDurationMs**: `number` -Defined in: [knowledge/improvement-job.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L50) +Defined in: [knowledge/improvement-job.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L88) ##### supervisedSpent > **supervisedSpent**: `object` -Defined in: [knowledge/improvement-job.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L51) +Defined in: [knowledge/improvement-job.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L89) ###### iterations @@ -6854,6 +6684,10 @@ Defined in: [knowledge/improvement-job.ts:51](https://github.com/tangle-network/ > **outputTokens**: `number` +###### usdKnown + +> **usdKnown**: `boolean` + ###### usd > **usd**: `number` @@ -6866,7 +6700,7 @@ Defined in: [knowledge/improvement-job.ts:51](https://github.com/tangle-network/ ### KnowledgeImprovementJobResult -Defined in: [knowledge/improvement-job.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L60) +Defined in: [knowledge/improvement-job.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L99) #### Properties @@ -6874,31 +6708,37 @@ Defined in: [knowledge/improvement-job.ts:60](https://github.com/tangle-network/ > **improvement**: `KnowledgeImprovementResult` -Defined in: [knowledge/improvement-job.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L61) +Defined in: [knowledge/improvement-job.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L100) + +##### candidateKnowledge? + +> `optional` **candidateKnowledge?**: `AgentCandidateKnowledge` + +Defined in: [knowledge/improvement-job.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L101) ##### measurement > **measurement**: [`KnowledgeImprovementJobMeasurement`](#knowledgeimprovementjobmeasurement) -Defined in: [knowledge/improvement-job.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L62) +Defined in: [knowledge/improvement-job.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L102) ##### promoted > **promoted**: `boolean` -Defined in: [knowledge/improvement-job.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L63) +Defined in: [knowledge/improvement-job.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L103) ##### blocked > **blocked**: `boolean` -Defined in: [knowledge/improvement-job.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L64) +Defined in: [knowledge/improvement-job.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L104) *** ### AgentKnowledgeReadinessCheckOptions -Defined in: [knowledge/improvement-job.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L67) +Defined in: [knowledge/improvement-job.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L107) #### Properties @@ -6906,43 +6746,43 @@ Defined in: [knowledge/improvement-job.ts:67](https://github.com/tangle-network/ > **goal**: `string` -Defined in: [knowledge/improvement-job.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L68) +Defined in: [knowledge/improvement-job.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L108) ##### readinessSpecs? > `optional` **readinessSpecs?**: readonly `KnowledgeReadinessSpec`[] -Defined in: [knowledge/improvement-job.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L69) +Defined in: [knowledge/improvement-job.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L109) ##### readinessTaskId? > `optional` **readinessTaskId?**: `string` -Defined in: [knowledge/improvement-job.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L70) +Defined in: [knowledge/improvement-job.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L110) ##### readiness? > `optional` **readiness?**: `Omit`\<`BuildEvalKnowledgeBundleOptions`, `"taskId"` \| `"index"` \| `"specs"`\> -Defined in: [knowledge/improvement-job.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L71) +Defined in: [knowledge/improvement-job.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L111) ##### strict? > `optional` **strict?**: `boolean` -Defined in: [knowledge/improvement-job.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L72) +Defined in: [knowledge/improvement-job.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L112) ##### kbQuality? > `optional` **kbQuality?**: `KnowledgeBaseQualityOptions` -Defined in: [knowledge/improvement-job.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L73) +Defined in: [knowledge/improvement-job.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L113) *** ### KnowledgeReadinessCheckInput -Defined in: [knowledge/supervised-update.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L22) +Defined in: [knowledge/supervised-update.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L23) #### Properties @@ -6950,37 +6790,37 @@ Defined in: [knowledge/supervised-update.ts:22](https://github.com/tangle-networ > **root**: `string` -Defined in: [knowledge/supervised-update.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L23) +Defined in: [knowledge/supervised-update.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L24) ##### goal > **goal**: `string` -Defined in: [knowledge/supervised-update.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L24) +Defined in: [knowledge/supervised-update.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L25) ##### readinessSpecs? > `optional` **readinessSpecs?**: readonly `unknown`[] -Defined in: [knowledge/supervised-update.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L25) +Defined in: [knowledge/supervised-update.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L26) ##### readinessTaskId? > `optional` **readinessTaskId?**: `string` -Defined in: [knowledge/supervised-update.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L26) +Defined in: [knowledge/supervised-update.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L27) ##### readiness? > `optional` **readiness?**: `unknown` -Defined in: [knowledge/supervised-update.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L27) +Defined in: [knowledge/supervised-update.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L28) *** ### SupervisedKnowledgeUpdateInput -Defined in: [knowledge/supervised-update.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L42) +Defined in: [knowledge/supervised-update.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L43) #### Properties @@ -6988,37 +6828,37 @@ Defined in: [knowledge/supervised-update.ts:42](https://github.com/tangle-networ > `optional` **goal?**: `string` -Defined in: [knowledge/supervised-update.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L43) +Defined in: [knowledge/supervised-update.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L44) ##### root? > `optional` **root?**: `string` -Defined in: [knowledge/supervised-update.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L44) +Defined in: [knowledge/supervised-update.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L45) ##### candidateRoot? > `optional` **candidateRoot?**: `string` -Defined in: [knowledge/supervised-update.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L45) +Defined in: [knowledge/supervised-update.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L46) ##### findings? > `optional` **findings?**: readonly `unknown`[] -Defined in: [knowledge/supervised-update.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L46) +Defined in: [knowledge/supervised-update.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L47) ##### metadata? > `optional` **metadata?**: `Record`\<`string`, `unknown`\> -Defined in: [knowledge/supervised-update.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L47) +Defined in: [knowledge/supervised-update.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L48) *** ### SupervisedKnowledgeUpdateResult -Defined in: [knowledge/supervised-update.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L50) +Defined in: [knowledge/supervised-update.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L51) #### Properties @@ -7026,31 +6866,31 @@ Defined in: [knowledge/supervised-update.ts:50](https://github.com/tangle-networ > **applied**: `boolean` -Defined in: [knowledge/supervised-update.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L51) +Defined in: [knowledge/supervised-update.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L52) ##### summary > **summary**: `string` -Defined in: [knowledge/supervised-update.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L52) +Defined in: [knowledge/supervised-update.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L53) ##### supervised > **supervised**: [`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\> -Defined in: [knowledge/supervised-update.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L53) +Defined in: [knowledge/supervised-update.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L54) ##### metadata -> **metadata**: `Record`\<`string`, `unknown`\> +> **metadata**: `NonNullable`\<`RagKnowledgeUpdateResult`\[`"metadata"`\]\> -Defined in: [knowledge/supervised-update.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L54) +Defined in: [knowledge/supervised-update.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L55) *** ### SupervisedKnowledgeUpdateOptions -Defined in: [knowledge/supervised-update.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L57) +Defined in: [knowledge/supervised-update.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L58) #### Properties @@ -7058,103 +6898,103 @@ Defined in: [knowledge/supervised-update.ts:57](https://github.com/tangle-networ > **root**: `string` -Defined in: [knowledge/supervised-update.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L58) +Defined in: [knowledge/supervised-update.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L59) ##### goal > **goal**: `string` -Defined in: [knowledge/supervised-update.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L59) +Defined in: [knowledge/supervised-update.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L60) ##### readiness > **readiness**: [`KnowledgeReadinessCheck`](#knowledgereadinesscheck) -Defined in: [knowledge/supervised-update.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L60) +Defined in: [knowledge/supervised-update.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L61) ##### readinessSpecs? > `optional` **readinessSpecs?**: readonly `unknown`[] -Defined in: [knowledge/supervised-update.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L61) +Defined in: [knowledge/supervised-update.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L62) ##### readinessTaskId? > `optional` **readinessTaskId?**: `string` -Defined in: [knowledge/supervised-update.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L62) +Defined in: [knowledge/supervised-update.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L63) ##### readinessOptions? > `optional` **readinessOptions?**: `unknown` -Defined in: [knowledge/supervised-update.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L63) +Defined in: [knowledge/supervised-update.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L64) ##### findings? > `optional` **findings?**: readonly `unknown`[] -Defined in: [knowledge/supervised-update.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L64) +Defined in: [knowledge/supervised-update.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L65) ##### metadata? > `optional` **metadata?**: `Record`\<`string`, `unknown`\> -Defined in: [knowledge/supervised-update.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L65) +Defined in: [knowledge/supervised-update.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L66) ##### budget > **budget**: [`Budget`](runtime.md#budget-12) -Defined in: [knowledge/supervised-update.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L66) +Defined in: [knowledge/supervised-update.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L67) ##### backend? > `optional` **backend?**: [`ExecutorConfig`](runtime.md#executorconfig) -Defined in: [knowledge/supervised-update.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L67) +Defined in: [knowledge/supervised-update.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L68) ##### makeWorkerAgent? > `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) -Defined in: [knowledge/supervised-update.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L68) +Defined in: [knowledge/supervised-update.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L69) ##### harness? > `optional` **harness?**: `string` -Defined in: [knowledge/supervised-update.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L69) +Defined in: [knowledge/supervised-update.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L70) ##### supervisorModel? > `optional` **supervisorModel?**: `string` -Defined in: [knowledge/supervised-update.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L70) +Defined in: [knowledge/supervised-update.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L71) ##### supervisorSystemPrompt? > `optional` **supervisorSystemPrompt?**: `string` -Defined in: [knowledge/supervised-update.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L71) +Defined in: [knowledge/supervised-update.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L72) ##### superviseOptions? > `optional` **superviseOptions?**: `Partial`\<`Omit`\<[`SuperviseOptions`](runtime.md#superviseoptions), `"backend"` \| `"budget"` \| `"makeWorkerAgent"` \| `"deliverable"` \| `"allowedModels"`\>\> -Defined in: [knowledge/supervised-update.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L72) +Defined in: [knowledge/supervised-update.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L73) ##### allowedModels? > `optional` **allowedModels?**: readonly `string`[] -Defined in: [knowledge/supervised-update.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L78) +Defined in: [knowledge/supervised-update.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L79) ##### runSupervised? > `optional` **runSupervised?**: (`profile`, `task`, `opts`) => `Promise`\<[`SupervisedResult`](runtime.md#supervisedresult)\<`unknown`\>\> -Defined in: [knowledge/supervised-update.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L79) +Defined in: [knowledge/supervised-update.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L80) ###### Parameters @@ -10719,7 +10559,7 @@ Defined in: [types.ts:571](https://github.com/tangle-network/agent-runtime/blob/ > **AgentCandidateProfileSource** = \{ `kind`: `"profile"`; `profile`: `AgentProfile`; \} \| \{ `kind`: `"profile-diffs"`; `base`: `AgentProfile`; `diffs`: readonly `AgentProfileDiff`[]; \} \| \{ `kind`: `"candidate-profile"`; `profile`: `AgentCandidateProfile`; \} -Defined in: [candidate-execution/builder.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L26) +Defined in: [candidate-execution/builder.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L25) A complete profile that can be frozen without losing behavior. @@ -10747,7 +10587,7 @@ A complete profile that can be frozen without losing behavior. > **diffs**: readonly `AgentProfileDiff`[] -Applied in order. Each exact diff is content-addressed into lineage. +Applied in order before the resulting profile is frozen into the bundle. *** @@ -10771,7 +10611,7 @@ Already converted to the closed, secret-free candidate profile contract. > **AgentCandidateCodeSource** = `AgentCandidateCodeDisabled` \| `AgentCandidateCodeNoOp` \| [`AgentCandidateCodeSurfaceSource`](#agentcandidatecodesurfacesource) -Defined in: [candidate-execution/builder.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L53) +Defined in: [candidate-execution/builder.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L52) Explicit control/no-op code or one finalized CodeSurface whose bytes must still verify. @@ -10791,7 +10631,7 @@ Exact candidate wire shape before the runtime computes its canonical digest. > **AgentCandidateExecutionFailureClass** = `"pre-model-infrastructure"` \| `"execution"` \| `"post-model-infrastructure"` \| `"unknown"` -Defined in: [candidate-execution/claim.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L78) +Defined in: [candidate-execution/claim.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L79) Only the first class is retryable, and only when the closed model ledger has zero calls. @@ -10799,9 +10639,9 @@ Only the first class is retryable, and only when the closed model ledger has zer ### AgentCandidateExecutionTerminalResult -> **AgentCandidateExecutionTerminalResult** = \{ `schemaVersion`: `1`; `status`: `"succeeded"`; `usage`: [`AgentCandidateExecutionUsage`](#agentcandidateexecutionusage); `modelSettlement`: `AgentCandidateArtifactRef`; `taskOutcome`: `AgentCandidateArtifactRef`; `benchmarkResult`: `AgentCandidateArtifactRef`; `runReceipt`: `AgentCandidateArtifactRef`; \} \| \{ `schemaVersion`: `1`; `status`: `"failed"`; `failureClass`: [`AgentCandidateExecutionFailureClass`](#agentcandidateexecutionfailureclass); `usage`: [`AgentCandidateExecutionUsage`](#agentcandidateexecutionusage); `modelSettlement`: `AgentCandidateArtifactRef`; `failureEvidence?`: `AgentCandidateArtifactRef`; \} +> **AgentCandidateExecutionTerminalResult** = \{ `status`: `"succeeded"`; `usage`: `AgentCandidateFixedSpend`; `modelSettlement`: `AgentCandidateArtifactRef`; `taskOutcome`: `AgentCandidateArtifactRef`; `benchmarkResult`: `AgentCandidateArtifactRef`; `runReceipt`: `AgentCandidateArtifactRef`; \} \| \{ `status`: `"failed"`; `failureClass`: [`AgentCandidateExecutionFailureClass`](#agentcandidateexecutionfailureclass); `usage`: `AgentCandidateFixedSpend`; `modelSettlement`: `AgentCandidateArtifactRef`; `failureEvidence?`: `AgentCandidateArtifactRef`; \} -Defined in: [candidate-execution/claim.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L95) +Defined in: [candidate-execution/claim.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L86) Evaluator-owned terminal facts staged durably before the terminal CAS. @@ -10811,7 +10651,7 @@ Evaluator-owned terminal facts staged durably before the terminal CAS. > **AgentCandidateExecutionTerminalRecord** = [`AgentCandidateExecutionTerminalResult`](#agentcandidateexecutionterminalresult) & `object` -Defined in: [candidate-execution/claim.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L115) +Defined in: [candidate-execution/claim.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L104) Durable terminal record for one acquired execution attempt. @@ -10833,6 +10673,10 @@ Durable terminal record for one acquired execution attempt. > `readonly` **executionPlanDigest**: `Sha256Digest` +##### preparationEvidence + +> `readonly` **preparationEvidence**: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim)\[`"preparationEvidence"`\] + ##### terminalDigest > `readonly` **terminalDigest**: `Sha256Digest` @@ -10845,7 +10689,7 @@ RFC 8785 SHA-256 of this record with `terminalDigest` omitted. > **AgentCandidateExecutionPhase** = `"claimed"` \| `"candidate-may-run"` -Defined in: [candidate-execution/claim.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L125) +Defined in: [candidate-execution/claim.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L115) Monotonic durable phase: the second value means candidate code could have started. @@ -10855,7 +10699,7 @@ Monotonic durable phase: the second value means candidate code could have starte > **AgentCandidateExecutionClaimResult** = \{ `acquired`: `true`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `lease`: [`AgentCandidateExecutionLease`](#agentcandidateexecutionlease); \} \| \{ `acquired`: `false`; `reason`: `"already-claimed"`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `exactReplay`: `boolean`; \} \| \{ `acquired`: `false`; `reason`: `"retry-not-eligible"`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `detail`: [`AgentCandidateRetryRejection`](#agentcandidateretryrejection); \} -Defined in: [candidate-execution/claim.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L165) +Defined in: [candidate-execution/claim.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L155) Result of atomically claiming one execution attempt. @@ -10903,7 +10747,7 @@ True only when every signed claim field matches the durable winner. > **AgentCandidateExecutionFinishResult** = \{ `finished`: `true`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); \} \| \{ `finished`: `false`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); `exactReplay`: `boolean`; \} -Defined in: [candidate-execution/claim.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L187) +Defined in: [candidate-execution/claim.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L177) Result of atomically recording an attempt's terminal facts. @@ -10939,7 +10783,7 @@ True when a repeated finish supplied the same terminal digest. > **AgentCandidateExecutionStageResult** = \{ `staged`: `true`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); \} \| \{ `staged`: `false`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); `exactReplay`: `boolean`; \} -Defined in: [candidate-execution/claim.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L200) +Defined in: [candidate-execution/claim.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L190) Result of durably staging the one immutable terminal outbox entry. @@ -10949,7 +10793,7 @@ Result of durably staging the one immutable terminal outbox entry. > **AgentCandidateExecutionPhaseResult** = \{ `marked`: `true`; `phase`: `"candidate-may-run"`; \} \| \{ `marked`: `false`; `phase`: `"candidate-may-run"`; \} -Defined in: [candidate-execution/claim.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L212) +Defined in: [candidate-execution/claim.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L202) Result of crossing the irreversible candidate-may-run boundary. @@ -10959,7 +10803,7 @@ Result of crossing the irreversible candidate-may-run boundary. > **AgentCandidateRetryRejection** = `"prior-attempt-missing"` \| `"prior-attempt-running"` \| `"prior-attempt-succeeded"` \| `"prior-attempt-spent-model-calls"` \| `"prior-attempt-not-pre-model-infrastructure"` \| `"retry-lineage-mismatch"` -Defined in: [candidate-execution/claim.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L216) +Defined in: [candidate-execution/claim.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L206) *** @@ -10999,9 +10843,9 @@ Secret-free response from the service's reservation endpoint. ### AgentCandidateOutputPurpose -> **AgentCandidateOutputPurpose** = `"candidate-workspace-manifest"` \| `"candidate-workspace-archive"` \| `"task-manifest"` \| `"task-archive"` \| `"task-patch"` \| `"task-outcome"` \| `"memory-after-manifest"` \| `"memory-after-archive"` \| `"grader-evidence"` \| `"benchmark-result"` \| `"model-settlement"` \| `"trace"` \| `"executor-capture"` \| `"run-receipt"` \| `"failure-evidence"` +> **AgentCandidateOutputPurpose** = `"execution-plan"` \| `"materialization-receipt"` \| `"candidate-workspace-manifest"` \| `"candidate-workspace-archive"` \| `"task-manifest"` \| `"task-archive"` \| `"task-patch"` \| `"task-output"` \| `"task-outcome"` \| `"memory-after-manifest"` \| `"memory-after-archive"` \| `"grader-evidence"` \| `"benchmark-result"` \| `"model-settlement"` \| `"trace"` \| `"executor-native-evidence"` \| `"executor-capture"` \| `"run-receipt"` \| `"knowledge-retrieval-config"` \| `"knowledge-evaluation"` \| `"failure-evidence"` -Defined in: [candidate-execution/types.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L39) +Defined in: [candidate-execution/types.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L45) *** @@ -11009,17 +10853,87 @@ Defined in: [candidate-execution/types.ts:39](https://github.com/tangle-network/ > **AgentCandidateModelLimits** = `Pick`\<`AgentCandidateExecutionLimits`, `"maxModelCalls"` \| `"maxInputTokens"` \| `"maxOutputTokens"` \| `"maxCostUsd"`\> -Defined in: [candidate-execution/types.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L153) +Defined in: [candidate-execution/types.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L165) Limits mechanically enforced by the evaluator-owned model gateway. *** +### AgentCandidateExecutorTaskOutcomeCapture + +> **AgentCandidateExecutorTaskOutcomeCapture** = \{ `kind`: `"workspace"`; `resultTree`: `string`; `afterState`: `AgentCandidateWorkspaceManifestMaterial`; `archive`: `Uint8Array`; `gitDiff`: `Uint8Array`; \} \| \{ `kind`: `"output"`; `bytes`: `Uint8Array`; \} + +Defined in: [candidate-execution/types.ts:355](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L355) + +Raw evaluator capture made only after the candidate process is dead. + +#### Union Members + +##### Type Literal + +\{ `kind`: `"workspace"`; `resultTree`: `string`; `afterState`: `AgentCandidateWorkspaceManifestMaterial`; `archive`: `Uint8Array`; `gitDiff`: `Uint8Array`; \} + +###### kind + +> `readonly` **kind**: `"workspace"` + +###### resultTree + +> `readonly` **resultTree**: `string` + +Claimed final tree. The runtime recomputes it independently from `gitDiff`. + +###### afterState + +> `readonly` **afterState**: `AgentCandidateWorkspaceManifestMaterial` + +Complete evaluator-captured workspace description after candidate execution. + +###### archive + +> `readonly` **archive**: `Uint8Array` + +Reproducible workspace archive corresponding to `afterState`. + +###### gitDiff + +> `readonly` **gitDiff**: `Uint8Array` + +Exact binary patch from the signed task base to `afterState`. + +*** + +##### Type Literal + +\{ `kind`: `"output"`; `bytes`: `Uint8Array`; \} + +###### kind + +> `readonly` **kind**: `"output"` + +###### bytes + +> `readonly` **bytes**: `Uint8Array` + +Exact evaluator-captured final output bytes. + +*** + +### VerifiedAgentCandidateTaskOutcome + +> **VerifiedAgentCandidateTaskOutcome** = \{ `kind`: `"workspace"`; `evidence`: `PersistedTaskOutcomeEvidence`\<`"workspace"`\>; `patch`: `Uint8Array`; `[verifiedTaskOutcomeBrand]`: `true`; \} \| \{ `kind`: `"output"`; `evidence`: `PersistedTaskOutcomeEvidence`\<`"output"`\>; `spec`: `AgentCandidateTaskOutputSpec`; `bytes`: `Uint8Array`; `[verifiedTaskOutcomeBrand]`: `true`; \} + +Defined in: [candidate-execution/types.ts:398](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L398) + +Branded task outcome that has survived independent evaluator verification. + +*** + ### AgentCandidateRunFinalization -> **AgentCandidateRunFinalization** = \{ `succeeded`: `true`; `receipt`: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateRunReceipt`\>; `artifacts`: \{ `executorCapture`: `AgentCandidateArtifactRef`; `modelSettlement`: `AgentCandidateArtifactRef`; `taskOutcome`: `AgentCandidateArtifactRef`; `benchmarkResult`: `AgentCandidateArtifactRef`; `runReceipt`: `AgentCandidateArtifactRef`; \}; \} \| \{ `succeeded`: `false`; `reason`: `string`; `partial`: \{ `executionId`: `string`; `bundleDigest`: `Sha256Digest`; `executionPlanDigest`: `Sha256Digest`; `materializationReceiptDigest`: `Sha256Digest`; `termination?`: `AgentCandidateTermination`; \}; `usage`: `AgentCandidateSpend` \| `null`; \} +> **AgentCandidateRunFinalization** = \{ `succeeded`: `true`; `receipt`: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateRunReceipt`\>; `artifacts`: \{ `executorCapture`: `AgentCandidateArtifactRef`; `modelSettlement`: `AgentCandidateArtifactRef`; `taskOutcome`: `AgentCandidateArtifactRef`; `benchmarkResult`: `AgentCandidateArtifactRef`; `runReceipt`: `AgentCandidateArtifactRef`; \}; \} \| \{ `succeeded`: `false`; `reason`: `string`; `partial`: \{ `executionId`: `string`; `bundleDigest`: `Sha256Digest`; `executionPlanDigest`: `Sha256Digest`; `materializationReceiptDigest`: `Sha256Digest`; `termination?`: `AgentCandidateTermination`; \}; `usage`: `AgentCandidateFixedSpend` \| `null`; \} -Defined in: [candidate-execution/types.ts:541](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L541) +Defined in: [candidate-execution/types.ts:555](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L555) #### Union Members @@ -11031,7 +10945,7 @@ Defined in: [candidate-execution/types.ts:541](https://github.com/tangle-network ##### Type Literal -\{ `succeeded`: `false`; `reason`: `string`; `partial`: \{ `executionId`: `string`; `bundleDigest`: `Sha256Digest`; `executionPlanDigest`: `Sha256Digest`; `materializationReceiptDigest`: `Sha256Digest`; `termination?`: `AgentCandidateTermination`; \}; `usage`: `AgentCandidateSpend` \| `null`; \} +\{ `succeeded`: `false`; `reason`: `string`; `partial`: \{ `executionId`: `string`; `bundleDigest`: `Sha256Digest`; `executionPlanDigest`: `Sha256Digest`; `materializationReceiptDigest`: `Sha256Digest`; `termination?`: `AgentCandidateTermination`; \}; `usage`: `AgentCandidateFixedSpend` \| `null`; \} ###### succeeded @@ -11067,7 +10981,7 @@ Defined in: [candidate-execution/types.ts:541](https://github.com/tangle-network ###### usage -> **usage**: `AgentCandidateSpend` \| `null` +> **usage**: `AgentCandidateFixedSpend` \| `null` Independent evaluator-gateway usage, even when execution or trace capture failed. @@ -11221,7 +11135,7 @@ Verifies the edited worktree. Sync or async; throws only on a setup fault > **AgenticGeneratorShotExecution** = `Readonly`\<`Omit`\<[`LocalHarnessResult`](mcp.md#localharnessresult), `"usage"` \| `"evidence"`\> & `object`\> -Defined in: [improvement/agentic-generator.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L123) +Defined in: [improvement/agentic-generator.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L122) Frozen exact harness result for an author shot: full streams, process state, token usage, and execution-policy evidence. @@ -11234,7 +11148,7 @@ Frozen exact harness result for an author shot: full streams, process state, > **AgenticGeneratorShotDisposition** = \{ `kind`: `"clean"`; `worktreePath`: `string`; \} \| \{ `kind`: `"rejected"`; `worktreePath`: `string`; `stage`: `"raw-trace-evidence"` \| `"verification"`; `feedback`: `string` \| `null`; \} \| \{ `kind`: `"accepted"`; `worktreePath`: `string`; `verified`: `boolean`; \} \| \{ `kind`: `"setup-error"`; `worktreePath`: `string`; `stage`: `"worktree-inspection"` \| `"raw-trace-evidence"` \| `"verification"`; `error`: \{ `name`: `string`; `message`: `string`; \}; \} -Defined in: [improvement/agentic-generator.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L136) +Defined in: [improvement/agentic-generator.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L135) Worktree decision emitted before a completed shot is retried, accepted, or discarded. The callback runs while `worktreePath` is still available, so @@ -11246,7 +11160,7 @@ Worktree decision emitted before a completed shot is retried, accepted, or > **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"subagents"` \| `"agent-profile"` \| `"memory"` \| `"code"` -Defined in: [improvement/improve.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L69) +Defined in: [improvement/improve.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L70) The executable agent lever `improve` optimizes. Profile fields remain portable AgentProfile coordinates; implementation and orchestration files @@ -11258,7 +11172,7 @@ The executable agent lever `improve` optimizes. Profile fields remain > **ImproveOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`SelfImproveOptions`\<`TScenario`, `TArtifact`\>, `"analyzeGeneration"` \| `"baselineSurface"` \| `"findings"` \| `"gate"` \| `"proposer"`\> & `object` -Defined in: [improvement/improve.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L80) +Defined in: [improvement/improve.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L81) #### Type Declaration @@ -11373,7 +11287,7 @@ Custom held-back-exam decision. The string `gate` above controls whether > **ProfileDiffProposerContext**\<`TFindings`\> = `ProposeContext`\<`TFindings`\> & `object` -Defined in: [improvement/profile-diff-proposer.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L26) +Defined in: [improvement/profile-diff-proposer.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L20) #### Type Declaration @@ -11393,7 +11307,7 @@ Defined in: [improvement/profile-diff-proposer.ts:26](https://github.com/tangle- > **KnowledgeReadinessCheckResult** = `boolean` \| \{ `ready`: `boolean`; `summary?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; \} -Defined in: [knowledge/supervised-update.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L30) +Defined in: [knowledge/supervised-update.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L31) *** @@ -11401,7 +11315,7 @@ Defined in: [knowledge/supervised-update.ts:30](https://github.com/tangle-networ > **KnowledgeReadinessCheck** = (`input`) => `Promise`\<[`KnowledgeReadinessCheckResult`](#knowledgereadinesscheckresult)\> \| [`KnowledgeReadinessCheckResult`](#knowledgereadinesscheckresult) -Defined in: [knowledge/supervised-update.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L38) +Defined in: [knowledge/supervised-update.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L39) #### Parameters @@ -11419,7 +11333,7 @@ Defined in: [knowledge/supervised-update.ts:38](https://github.com/tangle-networ > **SupervisedKnowledgeUpdater** = (`input`) => `Promise`\<[`SupervisedKnowledgeUpdateResult`](#supervisedknowledgeupdateresult)\> -Defined in: [knowledge/supervised-update.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L86) +Defined in: [knowledge/supervised-update.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L87) #### Parameters @@ -11948,11 +11862,31 @@ MUST map this to `RunRecord.error` rather than recording silent ## Variables +### CANDIDATE\_KNOWLEDGE\_ROOT\_ENV + +> `const` **CANDIDATE\_KNOWLEDGE\_ROOT\_ENV**: `"TANGLE_CANDIDATE_KNOWLEDGE_ROOT"` = `'TANGLE_CANDIDATE_KNOWLEDGE_ROOT'` + +Defined in: [candidate-execution/knowledge.ts:14](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/knowledge.ts#L14) + +Environment variable containing the materialized candidate knowledge root. + +*** + +### CANDIDATE\_KNOWLEDGE\_RETRIEVAL\_CONFIG\_ENV + +> `const` **CANDIDATE\_KNOWLEDGE\_RETRIEVAL\_CONFIG\_ENV**: `"TANGLE_CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG"` = `'TANGLE_CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG'` + +Defined in: [candidate-execution/knowledge.ts:16](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/knowledge.ts#L16) + +Environment variable containing the materialized retrieval configuration path. + +*** + ### CANDIDATE\_TRACE\_TAGS > `const` **CANDIDATE\_TRACE\_TAGS**: `object` -Defined in: [candidate-execution/types.ts:568](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L568) +Defined in: [candidate-execution/types.ts:582](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L582) Protected trace tags that bind a run to one prepared candidate execution. @@ -11980,7 +11914,7 @@ Protected trace tags that bind a run to one prepared candidate execution. > `const` **CANDIDATE\_TRACE\_ENV**: `object` -Defined in: [candidate-execution/types.ts:576](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L576) +Defined in: [candidate-execution/types.ts:590](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L590) Environment keys used to propagate immutable candidate trace identity. @@ -12008,6 +11942,16 @@ Environment keys used to propagate immutable candidate trace identity. *** +### AGENT\_CANDIDATE\_EXECUTION\_SUPPORT + +> `const` **AGENT\_CANDIDATE\_EXECUTION\_SUPPORT**: `Readonly`\<\{ `outcomes`: readonly \[`"workspace"`, `"output"`\]; `code`: readonly \[`"disabled"`, `"no-op"`, `"git-patch"`\]; `memory`: readonly \[`"disabled"`, `"isolated"`\]; `knowledge`: `true`; `profile`: `Readonly`\<\{ `mcpTransports`: readonly \[`"stdio"`\]; `remoteMcp`: `false`; `tools`: `false`; `permissions`: `false`; `modes`: `false`; `confidential`: `false`; \}\>; \}\> + +Defined in: [candidate-execution/verify.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/verify.ts#L38) + +Surfaces admitted by Runtime's verifier before an environment adapter is selected. + +*** + ### defaultIsRetryable > `const` **defaultIsRetryable**: [`RetryableErrorPredicate`](#retryableerrorpredicate) @@ -12093,7 +12037,7 @@ file must live below this root so cleanup cannot alter candidate-owned files. > `const` **RESEARCH\_SUPERVISOR\_SYSTEM\_PROMPT**: `string` -Defined in: [knowledge/supervised-update.ts:9](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L9) +Defined in: [knowledge/supervised-update.ts:10](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L10) Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. @@ -12342,13 +12286,13 @@ Maximum completion tokens, sent as OpenAI-compatible `max_tokens`. Omit for prov > **buildAgentCandidateBundle**(`input`): `AgentCandidateBundle` -Defined in: [candidate-execution/builder.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L76) +Defined in: [candidate-execution/builder.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/builder.ts#L73) Compile one measured profile/code candidate into the immutable execution contract. Code bytes are re-read and verified by agent-eval before they are embedded. The returned bundle is schema-validated, canonically digested, and deeply immutable; call `verifyAgentCandidateBundle` at the execution boundary -to re-read external knowledge, memory, repository, and workspace artifacts. +to re-read external memory, repository, and workspace artifacts. #### Parameters @@ -12384,7 +12328,7 @@ Validate and content-address a candidate bundle before it crosses an approval bo ### candidateExecutionClaim() -> **candidateExecutionClaim**(`prepared`): [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) +> **candidateExecutionClaim**(`prepared`, `preparationEvidence`): [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) Defined in: [candidate-execution/claim-plan.ts:10](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-plan.ts#L10) @@ -12396,6 +12340,16 @@ Extract the complete durable claim from a prepared execution. [`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution) +##### preparationEvidence + +###### executionPlan + +`AgentCandidateArtifactRef` + +###### materializationReceipt + +`AgentCandidateArtifactRef` + #### Returns [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) @@ -12430,7 +12384,7 @@ Revoke reservations held by a prepared candidate that will not be executed. > **executePreparedAgentCandidate**(`prepared`, `options`): `Promise`\<[`AgentCandidateRunFinalization`](#agentcandidaterunfinalization)\> -Defined in: [candidate-execution/execute.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L75) +Defined in: [candidate-execution/execute.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L80) Executes and finalizes one durably claimed candidate without exposing an unproven result. @@ -12450,6 +12404,38 @@ Executes and finalizes one durably claimed candidate without exposing an unprove *** +### candidateKnowledgeExecutionPaths() + +> **candidateKnowledgeExecutionPaths**(`taskRoot`, `hasRetrievalConfig`): `object` + +Defined in: [candidate-execution/knowledge.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/knowledge.ts#L20) + +Deterministic, signed locations used by every candidate executor. + +#### Parameters + +##### taskRoot + +`string` + +##### hasRetrievalConfig + +`boolean` + +#### Returns + +`object` + +##### root + +> **root**: `string` + +##### retrievalConfig? + +> `optional` **retrievalConfig?**: `string` + +*** + ### persistCandidateOutputArtifact() > **persistCandidateOutputArtifact**(`port`, `input`): `Promise`\<`AgentCandidateArtifactRef`\> @@ -12492,7 +12478,7 @@ Persist evaluator evidence, read it back, and bind the returned locator to the e > **prepareAgentCandidateExecution**(`candidate`, `task`, `ports`, `options?`): `Promise`\<[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution)\> -Defined in: [candidate-execution/prepare.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L99) +Defined in: [candidate-execution/prepare.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L92) Materializes a verified candidate into one immutable evaluator-owned execution plan. @@ -12524,7 +12510,7 @@ Materializes a verified candidate into one immutable evaluator-owned execution p > **parseExactAgentProfile**(`input`, `label`): `AgentProfile` -Defined in: [candidate-execution/profile.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/profile.ts#L96) +Defined in: [candidate-execution/profile.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/profile.ts#L190) Parse a complete profile without silently discarding unsupported fields. @@ -12548,7 +12534,7 @@ Parse a complete profile without silently discarding unsupported fields. > **parseExactAgentProfileDiff**(`input`, `label`): `AgentProfileDiff` -Defined in: [candidate-execution/profile.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/profile.ts#L103) +Defined in: [candidate-execution/profile.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/profile.ts#L202) Parse a profile diff without silently discarding unsupported fields. @@ -12572,7 +12558,7 @@ Parse a profile diff without silently discarding unsupported fields. > **applyExactAgentProfileDiff**(`baseInput`, `diffInput`, `label`): `AgentProfile` -Defined in: [candidate-execution/profile.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/profile.ts#L110) +Defined in: [candidate-execution/profile.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/profile.ts#L214) Apply one exact diff and reject any value that cannot be preserved canonically. @@ -12624,7 +12610,7 @@ it to cross into candidate execution or durable receipt finalization. > **recoverExpiredAgentCandidateExecution**(`options`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> -Defined in: [candidate-execution/recover.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L36) +Defined in: [candidate-execution/recover.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L48) Close an expired crashed attempt from persisted non-secret handles, then record failure. @@ -12644,7 +12630,7 @@ Close an expired crashed attempt from persisted non-secret handles, then record > **verifyAgentCandidateBundle**(`input`, `ports`): `Promise`\<[`VerifiedAgentCandidate`](#verifiedagentcandidate)\> -Defined in: [candidate-execution/verify.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/verify.ts#L37) +Defined in: [candidate-execution/verify.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/verify.ts#L54) Verifies every digest, resource, workspace, and Git object in a candidate bundle. @@ -12668,7 +12654,7 @@ Verifies every digest, resource, workspace, and Git object in a candidate bundle > **captureAgentCandidateWorkspace**(`rootInput`, `options?`): `Promise`\<[`CapturedAgentCandidateWorkspace`](#capturedagentcandidateworkspace)\> -Defined in: [candidate-execution/workspace-archive.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L125) +Defined in: [candidate-execution/workspace-archive.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L124) Capture one exact regular-file workspace for immutable candidate execution. @@ -12692,7 +12678,7 @@ Capture one exact regular-file workspace for immutable candidate execution. > **captureAgentCandidateWorkspaceFiles**(`input`, `options?`): `Promise`\<[`CapturedAgentCandidateWorkspace`](#capturedagentcandidateworkspace)\> -Defined in: [candidate-execution/workspace-archive.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L146) +Defined in: [candidate-execution/workspace-archive.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L145) Capture detached files returned by a remote executor into the standard archive. @@ -12716,7 +12702,7 @@ readonly [`AgentCandidateExecutorWorkspaceFile`](#agentcandidateexecutorworkspac > **createAgentCandidateWorkspacePort**(`options?`): [`AgentCandidateWorkspacePort`](#agentcandidateworkspaceport) -Defined in: [candidate-execution/workspace-archive.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L208) +Defined in: [candidate-execution/workspace-archive.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/workspace-archive.ts#L206) Create the standard bounded materializer for candidate execution ports. @@ -13221,7 +13207,7 @@ Wire integration: > **agenticGenerator**(`opts?`): [`CandidateGenerator`](#candidategenerator) -Defined in: [improvement/agentic-generator.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L208) +Defined in: [improvement/agentic-generator.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L207) Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. @@ -13241,7 +13227,7 @@ Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a rea > **commandVerifier**(`command`, `args?`, `timeoutMs?`): [`Verifier`](#verifier) -Defined in: [improvement/agentic-generator.ts:863](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L863) +Defined in: [improvement/agentic-generator.ts:861](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L861) A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other exit ⇒ failed with stdout+stderr as feedback. The common case — verify by @@ -13314,7 +13300,7 @@ Build the starting instruction for a coder agent tasked with implementing a new > **applyImprovementWinnerToProfile**(`profile`, `surface`, `winner`): `AgentProfile` -Defined in: [improvement/improve.ts:478](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L478) +Defined in: [improvement/improve.ts:456](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L456) Apply a promoted winner surface back into the profile field for `surface`. Returns a shallow copy; never mutates the input profile. @@ -13343,7 +13329,7 @@ Apply a promoted winner surface back into the profile field for `surface`. > **improve**\<`TScenario`, `TArtifact`\>(`profile`, `findings`, `opts`): `Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> -Defined in: [improvement/improve.ts:541](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L541) +Defined in: [improvement/improve.ts:519](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L519) Run the held-out-gated self-improvement loop on ONE profile surface. @@ -13395,7 +13381,7 @@ Optimize the system prompt, default holdout gate: > **improvementDriver**(`opts`): [`ManagedImprovementDriver`](#managedimprovementdriver) -Defined in: [improvement/improvement-driver.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L89) +Defined in: [improvement/improvement-driver.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L90) The one reflective/agentic improvement proposer (`SurfaceProposer`): owns the candidate worktree lifecycle and delegates HOW a change is produced to a pluggable `CandidateGenerator`. @@ -13435,7 +13421,7 @@ Build a `Verifier` that boots a generated MCP server over stdio and checks it ex > **profileDiffProposer**\<`TFindings`\>(`options`): `SurfaceProposer`\<`TFindings`\> -Defined in: [improvement/profile-diff-proposer.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L41) +Defined in: [improvement/profile-diff-proposer.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L35) Turn exact AgentProfileDiffs from any source into full profile candidates for the shared optimization loop. Research, catalogs, humans, and trace miners @@ -13525,7 +13511,7 @@ Cheap no-sandbox `CandidateGenerator` (the `shots=1` setting): draft surface edi > **createAgentKnowledgeReadinessCheck**(`options`): [`KnowledgeReadinessCheck`](#knowledgereadinesscheck) -Defined in: [knowledge/improvement-job.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L77) +Defined in: [knowledge/improvement-job.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L117) Build the default readiness check backed by `@tangle-network/agent-knowledge` validation and scoring. @@ -13545,9 +13531,9 @@ Build the default readiness check backed by `@tangle-network/agent-knowledge` va > **runKnowledgeImprovementJob**(`options`): `Promise`\<[`KnowledgeImprovementJobResult`](#knowledgeimprovementjobresult)\> -Defined in: [knowledge/improvement-job.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L108) +Defined in: [knowledge/improvement-job.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/improvement-job.ts#L148) -Run the full KB improvement job: candidate workspace, runtime supervisor update, readiness check, and promotion. +Produce a frozen KB candidate, and promote it only when an exact signed review is supplied. #### Parameters @@ -13565,7 +13551,7 @@ Run the full KB improvement job: candidate workspace, runtime supervisor update, > **knowledgeReadinessDeliverable**(`options`): [`DeliverableSpec`](runtime.md#deliverablespec)\<`unknown`\> -Defined in: [knowledge/supervised-update.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L91) +Defined in: [knowledge/supervised-update.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L92) Build the completion check a supervised KB update uses to stop only when the KB is ready. @@ -13585,7 +13571,7 @@ Build the completion check a supervised KB update uses to stop only when the KB > **createSupervisedKnowledgeUpdater**(`options`): [`SupervisedKnowledgeUpdater`](#supervisedknowledgeupdater) -Defined in: [knowledge/supervised-update.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L113) +Defined in: [knowledge/supervised-update.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L114) Create an `improveKnowledgeBase` update callback backed by runtime supervision. @@ -13605,7 +13591,7 @@ Create an `improveKnowledgeBase` update callback backed by runtime supervision. > **runSupervisedKnowledgeUpdate**(`options`): `Promise`\<[`SupervisedKnowledgeUpdateResult`](#supervisedknowledgeupdateresult)\> -Defined in: [knowledge/supervised-update.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L127) +Defined in: [knowledge/supervised-update.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L128) Run a runtime supervisor that updates one candidate knowledge base and stops on readiness. @@ -13625,7 +13611,7 @@ Run a runtime supervisor that updates one candidate knowledge base and stops on > **formatSupervisedKnowledgeTask**(`options`): `string` -Defined in: [knowledge/supervised-update.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L169) +Defined in: [knowledge/supervised-update.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/knowledge/supervised-update.ts#L170) Format the supervisor task with the KB root, readiness requirements, current findings, and metadata. diff --git a/docs/api/intelligence.md b/docs/api/intelligence.md index 5edf42a1..79e73852 100644 --- a/docs/api/intelligence.md +++ b/docs/api/intelligence.md @@ -65,6 +65,122 @@ Defined in: [intelligence/capability.ts:247](https://github.com/tangle-network/a Defined in: [intelligence/capability.ts:248](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/capability.ts#L248) +*** + +### AgentCandidateExperimentCellExecutionError + +Defined in: [intelligence/improvement-cycle.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L123) + +A failed baseline or candidate cell with its complete Runtime failure result. + +#### Extends + +- `Error` + +#### Constructors + +##### Constructor + +> **new AgentCandidateExperimentCellExecutionError**(`finalization`): [`AgentCandidateExperimentCellExecutionError`](#agentcandidateexperimentcellexecutionerror) + +Defined in: [intelligence/improvement-cycle.ts:126](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L126) + +###### Parameters + +###### finalization + +###### succeeded + +`false` + +###### reason + +`string` + +###### partial + +\{ `executionId`: `string`; `bundleDigest`: `` `sha256:${string}` ``; `executionPlanDigest`: `` `sha256:${string}` ``; `materializationReceiptDigest`: `` `sha256:${string}` ``; `termination?`: `AgentCandidateTermination`; \} + +###### partial.executionId + +`string` + +###### partial.bundleDigest + +`` `sha256:${string}` `` + +###### partial.executionPlanDigest + +`` `sha256:${string}` `` + +###### partial.materializationReceiptDigest + +`` `sha256:${string}` `` + +###### partial.termination? + +`AgentCandidateTermination` + +###### usage + +`AgentCandidateFixedSpend` \| `null` + +Independent evaluator-gateway usage, even when execution or trace capture failed. + +###### Returns + +[`AgentCandidateExperimentCellExecutionError`](#agentcandidateexperimentcellexecutionerror) + +###### Overrides + +`Error.constructor` + +#### Properties + +##### finalization + +> `readonly` **finalization**: `object` + +Defined in: [intelligence/improvement-cycle.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L124) + +###### succeeded + +> **succeeded**: `false` + +###### reason + +> **reason**: `string` + +###### partial + +> **partial**: `object` + +###### partial.executionId + +> **executionId**: `string` + +###### partial.bundleDigest + +> **bundleDigest**: `` `sha256:${string}` `` + +###### partial.executionPlanDigest + +> **executionPlanDigest**: `` `sha256:${string}` `` + +###### partial.materializationReceiptDigest + +> **materializationReceiptDigest**: `` `sha256:${string}` `` + +###### partial.termination? + +> `optional` **termination?**: `AgentCandidateTermination` + +###### usage + +> **usage**: `AgentCandidateFixedSpend` \| `null` + +Independent evaluator-gateway usage, even when execution or trace capture failed. + ## Interfaces ### CredentialRef @@ -1076,337 +1192,295 @@ Intelligence-class spend ceiling. `0` refuses every intelligence spawn; `null` u *** -### AgentImprovementProposal - -Defined in: [intelligence/improvement-cycle.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L62) - -#### Type Parameters - -##### TScenario +### AgentCandidateExperimentCellPlacement -`TScenario` *extends* `Scenario` = `Scenario` +Defined in: [intelligence/improvement-cycle.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L83) -##### TArtifact +#### Extended by -`TArtifact` = `unknown` +- [`ExecuteAgentCandidateExperimentCellOptions`](#executeagentcandidateexperimentcelloptions) #### Properties -##### schemaVersion - -> **schemaVersion**: `1` - -Defined in: [intelligence/improvement-cycle.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L66) - -##### kind - -> **kind**: `"agent-improvement-proposal"` - -Defined in: [intelligence/improvement-cycle.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L67) - -##### runId +##### executionId -> **runId**: `string` +> **executionId**: `string` -Defined in: [intelligence/improvement-cycle.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L68) +Defined in: [intelligence/improvement-cycle.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L84) -##### surface +##### attempt? -> **surface**: [`ImproveSurface`](index.md#improvesurface) +> `optional` **attempt?**: `number` -Defined in: [intelligence/improvement-cycle.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L69) +Defined in: [intelligence/improvement-cycle.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L85) -##### proposedAt +##### executionRoots -> **proposedAt**: `string` +> **executionRoots**: `object` -Defined in: [intelligence/improvement-cycle.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L70) +Defined in: [intelligence/improvement-cycle.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L86) -##### baselineProfile +###### taskRoot -> **baselineProfile**: `AgentProfile` +> **taskRoot**: `string` -Defined in: [intelligence/improvement-cycle.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L71) +###### candidateRoot? -##### baselineProfileHash +> `optional` **candidateRoot?**: `string` -> **baselineProfileHash**: `string` +##### stagingRoots -Defined in: [intelligence/improvement-cycle.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L72) +> **stagingRoots**: `object` -##### candidateProfile +Defined in: [intelligence/improvement-cycle.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L87) -> **candidateProfile**: `AgentProfile` +###### taskRoot -Defined in: [intelligence/improvement-cycle.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L73) +> **taskRoot**: `string` -##### candidateProfileHash +###### candidateRoot? -> **candidateProfileHash**: `string` +> `optional` **candidateRoot?**: `string` -Defined in: [intelligence/improvement-cycle.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L74) +###### profileRoot -##### findings +> **profileRoot**: `string` -> **findings**: `AnalystFinding`[] +##### ports -Defined in: [intelligence/improvement-cycle.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L75) +> **ports**: [`AgentCandidateExecutionPorts`](index.md#agentcandidateexecutionports) -##### evaluation +Defined in: [intelligence/improvement-cycle.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L88) -> **evaluation**: [`AgentImprovementEvaluation`](#agentimprovementevaluation)\<`TScenario`, `TArtifact`\> +##### preparation? -Defined in: [intelligence/improvement-cycle.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L76) +> `optional` **preparation?**: [`PrepareAgentCandidateExecutionOptions`](index.md#prepareagentcandidateexecutionoptions) -##### candidateBundle? +Defined in: [intelligence/improvement-cycle.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L89) -> `optional` **candidateBundle?**: `AgentCandidateBundle` +##### execution -Defined in: [intelligence/improvement-cycle.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L77) +> **execution**: [`ExecutePreparedAgentCandidateOptions`](index.md#executepreparedagentcandidateoptions) -##### digest +Defined in: [intelligence/improvement-cycle.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L90) -> **digest**: `` `sha256:${string}` `` +*** -Defined in: [intelligence/improvement-cycle.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L78) +### RunAgentCandidateExperimentOptions -*** +Defined in: [intelligence/improvement-cycle.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L93) -### AgentImprovementReview +#### Extends -Defined in: [intelligence/improvement-cycle.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L83) +- `Omit`\<`CompareCandidateExperimentOptions`, `"experiment"` \| `"measurements"`\> #### Properties -##### schemaVersion +##### experiment -> **schemaVersion**: `1` +> **experiment**: `AgentCandidateExperiment` -Defined in: [intelligence/improvement-cycle.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L84) +Defined in: [intelligence/improvement-cycle.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L95) -##### kind +##### placeCell -> **kind**: `"agent-improvement-review"` +> **placeCell**: (`input`) => [`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement) \| `Promise`\<[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement)\> -Defined in: [intelligence/improvement-cycle.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L85) +Defined in: [intelligence/improvement-cycle.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L96) -##### proposalDigest +###### Parameters -> **proposalDigest**: `` `sha256:${string}` `` +###### input -Defined in: [intelligence/improvement-cycle.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L86) +`CandidateExperimentExecutionInput` -##### candidateBundleDigest? +###### Returns -> `optional` **candidateBundleDigest?**: `` `sha256:${string}` `` +[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement) \| `Promise`\<[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement)\> -Defined in: [intelligence/improvement-cycle.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L87) +##### maxConcurrency? -##### decision +> `optional` **maxConcurrency?**: `number` -> **decision**: [`AgentImprovementReviewDecision`](#agentimprovementreviewdecision) +Defined in: [intelligence/improvement-cycle.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L99) -Defined in: [intelligence/improvement-cycle.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L88) +##### signal? -##### reviewedBy +> `optional` **signal?**: `AbortSignal` -> **reviewedBy**: `string` +Defined in: [intelligence/improvement-cycle.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L100) -Defined in: [intelligence/improvement-cycle.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L89) +*** -##### reviewedAt +### RunAgentCandidateExperimentResult -> **reviewedAt**: `string` +Defined in: [intelligence/improvement-cycle.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L103) -Defined in: [intelligence/improvement-cycle.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L90) +#### Properties -##### reason +##### experiment -> **reason**: `string` +> **experiment**: `AgentCandidateExperiment` -Defined in: [intelligence/improvement-cycle.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L91) +Defined in: [intelligence/improvement-cycle.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L104) -##### feedback? +##### measurements -> `optional` **feedback?**: `string` +> **measurements**: `AgentCandidateExperimentMeasurement`[] -Defined in: [intelligence/improvement-cycle.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L92) +Defined in: [intelligence/improvement-cycle.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L105) -##### digest +##### evaluation -> **digest**: `` `sha256:${string}` `` +> **evaluation**: `AgentImprovementMeasuredComparison` -Defined in: [intelligence/improvement-cycle.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L93) +Defined in: [intelligence/improvement-cycle.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L106) *** -### CandidateExecutionEvidence - -Defined in: [intelligence/improvement-cycle.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L96) - -#### Properties - -##### proposalDigest - -> **proposalDigest**: `` `sha256:${string}` `` - -Defined in: [intelligence/improvement-cycle.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L97) - -##### reviewDigest +### ExecuteAgentCandidateExperimentCellOptions -> **reviewDigest**: `` `sha256:${string}` `` - -Defined in: [intelligence/improvement-cycle.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L98) +Defined in: [intelligence/improvement-cycle.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L109) -##### bundleDigest +#### Extends -> **bundleDigest**: `` `sha256:${string}` `` +- `CandidateExperimentExecutionInput`.[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement) -Defined in: [intelligence/improvement-cycle.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L99) +#### Properties ##### executionId > **executionId**: `string` -Defined in: [intelligence/improvement-cycle.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L100) - -##### executionPlanDigest - -> **executionPlanDigest**: `` `sha256:${string}` `` - -Defined in: [intelligence/improvement-cycle.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L101) - -##### materializationReceiptDigest +Defined in: [intelligence/improvement-cycle.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L84) -> **materializationReceiptDigest**: `` `sha256:${string}` `` +###### Inherited from -Defined in: [intelligence/improvement-cycle.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L102) +[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement).[`executionId`](#executionid) -##### succeeded +##### attempt? -> **succeeded**: `boolean` +> `optional` **attempt?**: `number` -Defined in: [intelligence/improvement-cycle.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L103) +Defined in: [intelligence/improvement-cycle.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L85) -##### runReceiptDigest? +###### Inherited from -> `optional` **runReceiptDigest?**: `` `sha256:${string}` `` +[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement).[`attempt`](#attempt) -Defined in: [intelligence/improvement-cycle.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L104) +##### executionRoots -*** +> **executionRoots**: `object` -### ProposeAgentImprovementOptions +Defined in: [intelligence/improvement-cycle.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L86) -Defined in: [intelligence/improvement-cycle.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L107) +###### taskRoot -#### Type Parameters +> **taskRoot**: `string` -##### TScenario +###### candidateRoot? -`TScenario` *extends* `Scenario` +> `optional` **candidateRoot?**: `string` -##### TArtifact +###### Inherited from -`TArtifact` +[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement).[`executionRoots`](#executionroots) -#### Properties +##### stagingRoots -##### runId +> **stagingRoots**: `object` -> **runId**: `string` +Defined in: [intelligence/improvement-cycle.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L87) -Defined in: [intelligence/improvement-cycle.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L108) +###### taskRoot -##### profile +> **taskRoot**: `string` -> **profile**: `AgentProfile` +###### candidateRoot? -Defined in: [intelligence/improvement-cycle.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L109) +> `optional` **candidateRoot?**: `string` -##### analysis +###### profileRoot -> **analysis**: `Omit`\<[`RunAnalystLoopOpts`](analyst-loop.md#runanalystloopopts), `"runId"` \| `"improvementAdapter"` \| `"autoApply"`\> +> **profileRoot**: `string` -Defined in: [intelligence/improvement-cycle.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L110) +###### Inherited from -##### improvement +[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement).[`stagingRoots`](#stagingroots) -> **improvement**: [`ImproveOptions`](index.md#improveoptions)\<`TScenario`, `TArtifact`\> +##### ports -Defined in: [intelligence/improvement-cycle.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L111) +> **ports**: [`AgentCandidateExecutionPorts`](index.md#agentcandidateexecutionports) -##### buildCandidate? +Defined in: [intelligence/improvement-cycle.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L88) -> `optional` **buildCandidate?**: (`input`) => `AgentCandidateBundle` \| [`AgentCandidateBundleInput`](index.md#agentcandidatebundleinput) \| `Promise`\<`AgentCandidateBundle` \| [`AgentCandidateBundleInput`](index.md#agentcandidatebundleinput)\> +###### Inherited from -Defined in: [intelligence/improvement-cycle.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L117) +[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement).[`ports`](#ports-1) -Optional environment adapter that freezes an executable bundle after the -measured comparison recommends the candidate. Return the sealed output of -`buildAgentCandidateBundle` directly, or a low-level digest-free input. +##### preparation? -###### Parameters +> `optional` **preparation?**: [`PrepareAgentCandidateExecutionOptions`](index.md#prepareagentcandidateexecutionoptions) -###### input +Defined in: [intelligence/improvement-cycle.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L89) -###### analysis +###### Inherited from -[`RunAnalystLoopResult`](analyst-loop.md#runanalystloopresult) +[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement).[`preparation`](#preparation) -###### improvement +##### execution -[`ImproveResult`](index.md#improveresult)\<`TScenario`, `TArtifact`\> +> **execution**: [`ExecutePreparedAgentCandidateOptions`](index.md#executepreparedagentcandidateoptions) -###### Returns +Defined in: [intelligence/improvement-cycle.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L90) -`AgentCandidateBundle` \| [`AgentCandidateBundleInput`](index.md#agentcandidatebundleinput) \| `Promise`\<`AgentCandidateBundle` \| [`AgentCandidateBundleInput`](index.md#agentcandidatebundleinput)\> +###### Inherited from -##### now? +[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement).[`execution`](#execution) -> `optional` **now?**: () => `Date` +*** -Defined in: [intelligence/improvement-cycle.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L124) +### VerifyCandidateExecutionEvidenceOptions -###### Returns +Defined in: [intelligence/improvement-cycle.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L113) -`Date` +#### Properties -*** +##### experiment -### ProposeAgentImprovementResult +> **experiment**: `AgentCandidateExperiment` -Defined in: [intelligence/improvement-cycle.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L127) +Defined in: [intelligence/improvement-cycle.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L114) -#### Type Parameters +##### arm -##### TScenario +> **arm**: `"candidate"` \| `"baseline"` -`TScenario` *extends* `Scenario` +Defined in: [intelligence/improvement-cycle.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L115) -##### TArtifact +##### benchmarkCell -`TArtifact` +> **benchmarkCell**: `AgentCandidateBenchmarkCellRef` -#### Properties +Defined in: [intelligence/improvement-cycle.ts:116](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L116) -##### analysis +##### seed -> **analysis**: [`RunAnalystLoopResult`](analyst-loop.md#runanalystloopresult) +> **seed**: `number` -Defined in: [intelligence/improvement-cycle.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L128) +Defined in: [intelligence/improvement-cycle.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L117) -##### improvement +##### attempt? -> **improvement**: [`ImproveResult`](index.md#improveresult)\<`TScenario`, `TArtifact`\> +> `optional` **attempt?**: `number` -Defined in: [intelligence/improvement-cycle.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L129) +Defined in: [intelligence/improvement-cycle.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L118) -##### proposal +##### resolvedResources? -> **proposal**: [`AgentImprovementProposal`](#agentimprovementproposal)\<`TScenario`, `TArtifact`\> +> `optional` **resolvedResources?**: `ReadonlyMap`\<`` `sha256:${string}` ``, `string`\> -Defined in: [intelligence/improvement-cycle.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L130) +Defined in: [intelligence/improvement-cycle.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L119) *** @@ -1414,16 +1488,6 @@ Defined in: [intelligence/improvement-cycle.ts:130](https://github.com/tangle-ne Defined in: [intelligence/improvement-cycle.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L133) -#### Type Parameters - -##### TScenario - -`TScenario` *extends* `Scenario` - -##### TArtifact - -`TArtifact` - #### Properties ##### runId @@ -1432,49 +1496,25 @@ Defined in: [intelligence/improvement-cycle.ts:133](https://github.com/tangle-ne Defined in: [intelligence/improvement-cycle.ts:134](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L134) -##### surface +##### findings -> **surface**: [`ImproveSurface`](index.md#improvesurface) +> **findings**: readonly `AnalystFinding`[] Defined in: [intelligence/improvement-cycle.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L135) -##### baselineProfile +##### evaluation -> **baselineProfile**: `AgentProfile` +> **evaluation**: `AgentImprovementMeasuredComparison` Defined in: [intelligence/improvement-cycle.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L136) -##### candidateProfile +##### now? -> **candidateProfile**: `AgentProfile` +> `optional` **now?**: () => `Date` Defined in: [intelligence/improvement-cycle.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L137) -##### findings - -> **findings**: readonly `AnalystFinding`[] - -Defined in: [intelligence/improvement-cycle.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L138) - -##### evaluation - -> **evaluation**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> - -Defined in: [intelligence/improvement-cycle.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L139) - -##### candidateBundle? - -> `optional` **candidateBundle?**: `AgentCandidateBundle` \| [`AgentCandidateBundleInput`](index.md#agentcandidatebundleinput) - -Defined in: [intelligence/improvement-cycle.ts:140](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L140) - -##### now? - -> `optional` **now?**: () => `Date` - -Defined in: [intelligence/improvement-cycle.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L141) - -###### Returns +###### Returns `Date` @@ -1482,39 +1522,39 @@ Defined in: [intelligence/improvement-cycle.ts:141](https://github.com/tangle-ne ### ReviewAgentImprovementInput -Defined in: [intelligence/improvement-cycle.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L144) +Defined in: [intelligence/improvement-cycle.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L142) #### Properties ##### decision -> **decision**: [`AgentImprovementReviewDecision`](#agentimprovementreviewdecision) +> **decision**: `AgentImprovementReviewDecision` -Defined in: [intelligence/improvement-cycle.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L145) +Defined in: [intelligence/improvement-cycle.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L143) ##### reviewedBy > **reviewedBy**: `string` -Defined in: [intelligence/improvement-cycle.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L146) +Defined in: [intelligence/improvement-cycle.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L144) ##### reason > **reason**: `string` -Defined in: [intelligence/improvement-cycle.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L147) +Defined in: [intelligence/improvement-cycle.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L145) ##### feedback? > `optional` **feedback?**: `string` -Defined in: [intelligence/improvement-cycle.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L148) +Defined in: [intelligence/improvement-cycle.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L146) ##### now? > `optional` **now?**: () => `Date` -Defined in: [intelligence/improvement-cycle.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L149) +Defined in: [intelligence/improvement-cycle.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L147) ###### Returns @@ -1522,95 +1562,211 @@ Defined in: [intelligence/improvement-cycle.ts:149](https://github.com/tangle-ne *** -### ExecuteApprovedAgentCandidateOptions +### CreateAgentImprovementActivationOptions -Defined in: [intelligence/improvement-cycle.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L152) +Defined in: [intelligence/improvement-cycle.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L150) #### Properties -##### proposal +##### targets + +> **targets**: \[`AgentImprovementActivationTarget`, `...AgentImprovementActivationTarget[]`\] + +Defined in: [intelligence/improvement-cycle.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L151) -> **proposal**: [`AgentImprovementProposal`](#agentimprovementproposal) +##### fundingOwner + +> **fundingOwner**: `string` + +Defined in: [intelligence/improvement-cycle.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L152) + +##### authorizedBy + +> **authorizedBy**: `string` Defined in: [intelligence/improvement-cycle.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L153) -##### review +##### now? -> **review**: [`AgentImprovementReview`](#agentimprovementreview) +> `optional` **now?**: () => `Date` Defined in: [intelligence/improvement-cycle.ts:154](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L154) -##### authorizeReview +###### Returns -> **authorizeReview**: (`review`, `proposal`) => `boolean` \| `Promise`\<`boolean`\> +`Date` -Defined in: [intelligence/improvement-cycle.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L156) +*** -Product-owned authentication check for the persisted approval record. +### ProposeAgentImprovementOptions -###### Parameters +Defined in: [intelligence/improvement-cycle.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L157) -###### review +#### Type Parameters -[`AgentImprovementReview`](#agentimprovementreview) +##### TScenario -###### proposal +`TScenario` *extends* `Scenario` + +##### TArtifact -[`AgentImprovementProposal`](#agentimprovementproposal) +`TArtifact` -###### Returns +#### Properties -`boolean` \| `Promise`\<`boolean`\> +##### runId + +> **runId**: `string` -##### task +Defined in: [intelligence/improvement-cycle.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L158) -> **task**: [`AgentCandidateTaskExecution`](index.md#agentcandidatetaskexecution) +##### profile + +> **profile**: `AgentProfile` + +Defined in: [intelligence/improvement-cycle.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L159) + +##### analysis + +> **analysis**: `Omit`\<[`RunAnalystLoopOpts`](analyst-loop.md#runanalystloopopts), `"runId"` \| `"improvementAdapter"` \| `"autoApply"`\> Defined in: [intelligence/improvement-cycle.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L160) -##### ports +##### improvement -> **ports**: [`AgentCandidateExecutionPorts`](index.md#agentcandidateexecutionports) +> **improvement**: [`ImproveOptions`](index.md#improveoptions)\<`TScenario`, `TArtifact`\> Defined in: [intelligence/improvement-cycle.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L161) -##### preparation? +##### buildExperiment -> `optional` **preparation?**: [`PrepareAgentCandidateExecutionOptions`](index.md#prepareagentcandidateexecutionoptions) +> **buildExperiment**: (`input`) => `AgentCandidateExperiment` \| `Promise`\<`AgentCandidateExperiment`\> Defined in: [intelligence/improvement-cycle.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L162) -##### execution +###### Parameters -> **execution**: [`ExecutePreparedAgentCandidateOptions`](index.md#executepreparedagentcandidateoptions) +###### input -Defined in: [intelligence/improvement-cycle.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L163) +###### analysis -*** +[`RunAnalystLoopResult`](analyst-loop.md#runanalystloopresult) + +###### improvement -### ExecuteApprovedAgentCandidateResult +[`ImproveResult`](index.md#improveresult)\<`TScenario`, `TArtifact`\> + +###### Returns + +`AgentCandidateExperiment` \| `Promise`\<`AgentCandidateExperiment`\> + +##### placeCell + +> **placeCell**: (`input`) => [`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement) \| `Promise`\<[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement)\> Defined in: [intelligence/improvement-cycle.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L166) -#### Properties +###### Parameters -##### finalization +###### input + +`CandidateExperimentExecutionInput` + +###### Returns -> **finalization**: [`AgentCandidateRunFinalization`](index.md#agentcandidaterunfinalization) +[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement) \| `Promise`\<[`AgentCandidateExperimentCellPlacement`](#agentcandidateexperimentcellplacement)\> + +##### maxConcurrency? + +> `optional` **maxConcurrency?**: `number` Defined in: [intelligence/improvement-cycle.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L167) -##### evidence +##### signal? -> **evidence**: [`CandidateExecutionEvidence`](#candidateexecutionevidence) +> `optional` **signal?**: `AbortSignal` Defined in: [intelligence/improvement-cycle.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L168) +##### candidate? + +> `optional` **candidate?**: `object` + +Defined in: [intelligence/improvement-cycle.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L169) + +##### metadata? + +> `optional` **metadata?**: `object` + +Defined in: [intelligence/improvement-cycle.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L170) + +###### Index Signature + +\[`key`: `string`\]: `AgentCandidateJsonValue` + +##### now? + +> `optional` **now?**: () => `Date` + +Defined in: [intelligence/improvement-cycle.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L171) + +###### Returns + +`Date` + +*** + +### ProposeAgentImprovementResult + +Defined in: [intelligence/improvement-cycle.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L174) + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +#### Properties + +##### analysis + +> **analysis**: [`RunAnalystLoopResult`](analyst-loop.md#runanalystloopresult) + +Defined in: [intelligence/improvement-cycle.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L175) + +##### improvement + +> **improvement**: [`ImproveResult`](index.md#improveresult)\<`TScenario`, `TArtifact`\> + +Defined in: [intelligence/improvement-cycle.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L176) + +##### experiment + +> **experiment**: `AgentCandidateExperiment` + +Defined in: [intelligence/improvement-cycle.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L177) + +##### measurements + +> **measurements**: `AgentCandidateExperimentMeasurement`[] + +Defined in: [intelligence/improvement-cycle.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L178) + +##### proposal + +> **proposal**: `AgentImprovementProposal` + +Defined in: [intelligence/improvement-cycle.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L179) + *** ### UsageSplit -Defined in: [intelligence/index.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L142) +Defined in: [intelligence/index.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L165) The per-class cost split carried by every trace and outcome. `off` ⇒ `intelligenceUsd: 0` by construction — there is no intelligence spawn to @@ -1622,7 +1778,7 @@ bill. This is a classification on the trace, NOT a budget-pool split. > **inferenceUsd**: `number` -Defined in: [intelligence/index.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L144) +Defined in: [intelligence/index.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L167) Base-stream (model) spend in USD. @@ -1630,7 +1786,7 @@ Base-stream (model) spend in USD. > **intelligenceUsd**: `number` -Defined in: [intelligence/index.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L146) +Defined in: [intelligence/index.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L169) Intelligence-spawn spend in USD. Provably `0` at the OFF tier. @@ -1638,7 +1794,7 @@ Intelligence-spawn spend in USD. Provably `0` at the OFF tier. ### RunRecord -Defined in: [intelligence/index.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L156) +Defined in: [intelligence/index.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L179) The typed record `withIntelligence` sends per call — serialized through the shipped OTLP builders to the plane's `/v1/otlp` ingest. `input`/`output` are @@ -1652,43 +1808,43 @@ tree under the same `traceId`. > **runId**: `string` -Defined in: [intelligence/index.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L157) +Defined in: [intelligence/index.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L180) ##### traceId > **traceId**: `string` -Defined in: [intelligence/index.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L158) +Defined in: [intelligence/index.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L181) ##### project > **project**: `string` -Defined in: [intelligence/index.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L159) +Defined in: [intelligence/index.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L182) ##### target > **target**: `string` -Defined in: [intelligence/index.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L160) +Defined in: [intelligence/index.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L183) ##### input > **input**: `unknown` -Defined in: [intelligence/index.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L161) +Defined in: [intelligence/index.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L184) ##### output > **output**: `unknown` -Defined in: [intelligence/index.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L162) +Defined in: [intelligence/index.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L185) ##### outcome > **outcome**: `object` -Defined in: [intelligence/index.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L163) +Defined in: [intelligence/index.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L186) ###### success? @@ -1706,61 +1862,61 @@ Defined in: [intelligence/index.ts:163](https://github.com/tangle-network/agent- > `optional` **model?**: `string` -Defined in: [intelligence/index.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L168) +Defined in: [intelligence/index.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L191) ##### provider? > `optional` **provider?**: `string` -Defined in: [intelligence/index.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L169) +Defined in: [intelligence/index.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L192) ##### loopEvents? > `optional` **loopEvents?**: [`LoopTraceEvent`](runtime.md#looptraceevent)[] -Defined in: [intelligence/index.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L170) +Defined in: [intelligence/index.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L193) ##### runtimeEvents? > `optional` **runtimeEvents?**: [`RuntimeStreamEvent`](index.md#runtimestreamevent)[] -Defined in: [intelligence/index.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L171) +Defined in: [intelligence/index.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L194) ##### profile? > `optional` **profile?**: `AgentProfile` -Defined in: [intelligence/index.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L172) +Defined in: [intelligence/index.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L195) ##### sessionId? > `optional` **sessionId?**: `string` -Defined in: [intelligence/index.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L173) +Defined in: [intelligence/index.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L196) ##### harness? > `optional` **harness?**: `string` -Defined in: [intelligence/index.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L174) +Defined in: [intelligence/index.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L197) ##### repository? > `optional` **repository?**: `string` -Defined in: [intelligence/index.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L175) +Defined in: [intelligence/index.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L198) ##### commitSha? > `optional` **commitSha?**: `string` -Defined in: [intelligence/index.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L176) +Defined in: [intelligence/index.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L199) ##### timing? > `optional` **timing?**: `object` -Defined in: [intelligence/index.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L177) +Defined in: [intelligence/index.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L200) ###### startedAt @@ -1778,7 +1934,7 @@ Defined in: [intelligence/index.ts:177](https://github.com/tangle-network/agent- > `optional` **tokens?**: `object` -Defined in: [intelligence/index.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L178) +Defined in: [intelligence/index.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L201) ###### input @@ -1800,7 +1956,7 @@ Defined in: [intelligence/index.ts:178](https://github.com/tangle-network/agent- > `optional` **error?**: `object` -Defined in: [intelligence/index.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L184) +Defined in: [intelligence/index.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L207) ###### name @@ -1816,9 +1972,9 @@ Defined in: [intelligence/index.ts:184](https://github.com/tangle-network/agent- ##### candidateExecution? -> `optional` **candidateExecution?**: [`CandidateExecutionEvidence`](#candidateexecutionevidence) +> `optional` **candidateExecution?**: `CandidateExecutionEvidence` -Defined in: [intelligence/index.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L186) +Defined in: [intelligence/index.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L209) Exact proposal → review → execution → receipt linkage for candidate runs. @@ -1826,7 +1982,7 @@ Exact proposal → review → execution → receipt linkage for candidate runs. ### RunReport -Defined in: [intelligence/index.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L195) +Defined in: [intelligence/index.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L218) What an agent reports (via `applied.record`) to enrich the [RunRecord](#runrecord) sent for its call. All optional — an un-recorded run still sends input/output @@ -1839,79 +1995,79 @@ as pure inference (the base stream). > `optional` **success?**: `boolean` -Defined in: [intelligence/index.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L196) +Defined in: [intelligence/index.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L219) ##### score? > `optional` **score?**: `number` -Defined in: [intelligence/index.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L197) +Defined in: [intelligence/index.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L220) ##### usage? > `optional` **usage?**: `Partial`\<[`UsageSplit`](#usagesplit)\> -Defined in: [intelligence/index.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L198) +Defined in: [intelligence/index.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L221) ##### costUsd? > `optional` **costUsd?**: `number` -Defined in: [intelligence/index.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L199) +Defined in: [intelligence/index.ts:222](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L222) ##### model? > `optional` **model?**: `string` -Defined in: [intelligence/index.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L200) +Defined in: [intelligence/index.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L223) ##### provider? > `optional` **provider?**: `string` -Defined in: [intelligence/index.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L201) +Defined in: [intelligence/index.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L224) ##### loopEvents? > `optional` **loopEvents?**: [`LoopTraceEvent`](runtime.md#looptraceevent)[] -Defined in: [intelligence/index.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L202) +Defined in: [intelligence/index.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L225) ##### runtimeEvents? > `optional` **runtimeEvents?**: [`RuntimeStreamEvent`](index.md#runtimestreamevent)[] -Defined in: [intelligence/index.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L203) +Defined in: [intelligence/index.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L226) ##### profile? > `optional` **profile?**: `AgentProfile` -Defined in: [intelligence/index.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L204) +Defined in: [intelligence/index.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L227) ##### sessionId? > `optional` **sessionId?**: `string` -Defined in: [intelligence/index.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L205) +Defined in: [intelligence/index.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L228) ##### harness? > `optional` **harness?**: `string` -Defined in: [intelligence/index.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L206) +Defined in: [intelligence/index.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L229) ##### commitSha? > `optional` **commitSha?**: `string` -Defined in: [intelligence/index.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L207) +Defined in: [intelligence/index.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L230) ##### tokens? > `optional` **tokens?**: `object` -Defined in: [intelligence/index.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L208) +Defined in: [intelligence/index.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L231) ###### input @@ -1933,7 +2089,7 @@ Defined in: [intelligence/index.ts:208](https://github.com/tangle-network/agent- > `optional` **error?**: `object` -Defined in: [intelligence/index.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L209) +Defined in: [intelligence/index.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L232) ###### name @@ -1949,15 +2105,15 @@ Defined in: [intelligence/index.ts:209](https://github.com/tangle-network/agent- ##### candidateExecution? -> `optional` **candidateExecution?**: [`CandidateExecutionEvidence`](#candidateexecutionevidence) +> `optional` **candidateExecution?**: `CandidateExecutionEvidence` -Defined in: [intelligence/index.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L210) +Defined in: [intelligence/index.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L233) *** ### RepoConfig -Defined in: [intelligence/index.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L216) +Defined in: [intelligence/index.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L239) Repo coordinates a product may declare for the (later) Gated-PR mode. The Observe slice only records their PRESENCE for `doctor()`; it never touches @@ -1969,25 +2125,25 @@ Repo coordinates a product may declare for the (later) Gated-PR mode. The > **owner**: `string` -Defined in: [intelligence/index.ts:217](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L217) +Defined in: [intelligence/index.ts:240](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L240) ##### name > **name**: `string` -Defined in: [intelligence/index.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L218) +Defined in: [intelligence/index.ts:241](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L241) ##### baseBranch > **baseBranch**: `string` -Defined in: [intelligence/index.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L219) +Defined in: [intelligence/index.ts:242](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L242) *** ### IntelligenceConfig -Defined in: [intelligence/index.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L225) +Defined in: [intelligence/index.ts:248](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L248) Client configuration. `project` + `apiKey` are the Observe minimum; the rest tune effort, endpoint, redaction, and (for `doctor()` readiness) @@ -2003,7 +2159,7 @@ Client configuration. `project` + `apiKey` are the Observe minimum; the > **project**: `string` -Defined in: [intelligence/index.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L227) +Defined in: [intelligence/index.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L250) Stable project id — the tenant dimension every trace is tagged with. @@ -2011,7 +2167,7 @@ Stable project id — the tenant dimension every trace is tagged with. > `optional` **apiKey?**: `string` -Defined in: [intelligence/index.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L229) +Defined in: [intelligence/index.ts:252](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L252) Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. @@ -2019,7 +2175,7 @@ Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. > `optional` **effort?**: [`EffortTier`](#efforttier) \| \{ `tier`: [`EffortTier`](#efforttier); `overrides?`: `Partial`\<[`EffortSettings`](#effortsettings)\>; \} -Defined in: [intelligence/index.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L231) +Defined in: [intelligence/index.ts:254](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L254) Effort tier (default `'standard'`) plus optional per-field overrides. @@ -2027,7 +2183,7 @@ Effort tier (default `'standard'`) plus optional per-field overrides. > `optional` **baseUrl?**: `string` -Defined in: [intelligence/index.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L239) +Defined in: [intelligence/index.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L262) The ONE Tangle Intelligence base URL — both the send (OTLP `/v1/otlp`) and receive (`/v1/profiles/:target/composed`) paths derive from it. Reads @@ -2039,7 +2195,7 @@ key the ingest requires); absent a key, export is a no-op. > `optional` **redact?**: `false` \| [`Redactor`](#redactor) -Defined in: [intelligence/index.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L245) +Defined in: [intelligence/index.ts:268](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L268) Redaction hook run over every exported input/output. A function replaces the default scrubber; `false` opts out entirely (raw fidelity, caller has @@ -2049,7 +2205,7 @@ sanitized upstream); omitted ⇒ the built-in `defaultRedactor`. > `optional` **surfaces?**: `string`[] -Defined in: [intelligence/index.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L247) +Defined in: [intelligence/index.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L270) Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. @@ -2057,7 +2213,7 @@ Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. > `optional` **checks?**: `string`[] -Defined in: [intelligence/index.ts:249](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L249) +Defined in: [intelligence/index.ts:272](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L272) Verification checks a later PR mode would gate on. Recorded for `doctor()` only. @@ -2065,7 +2221,7 @@ Verification checks a later PR mode would gate on. Recorded for `doctor()` only. > `optional` **repo?**: [`RepoConfig`](#repoconfig) -Defined in: [intelligence/index.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L251) +Defined in: [intelligence/index.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L274) Repo access a later PR mode would need. Recorded for `doctor()` only. @@ -2073,7 +2229,7 @@ Repo access a later PR mode would need. Recorded for `doctor()` only. > `optional` **profile?**: `AgentProfile` -Defined in: [intelligence/index.ts:253](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L253) +Defined in: [intelligence/index.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L276) Full canonical profile used for this agent. Exported redacted with a stable hash. @@ -2081,7 +2237,7 @@ Full canonical profile used for this agent. Exported redacted with a stable hash > `optional` **commitSha?**: `string` -Defined in: [intelligence/index.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L255) +Defined in: [intelligence/index.ts:278](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L278) Commit that produced the running agent, when known. @@ -2089,7 +2245,7 @@ Commit that produced the running agent, when known. > `optional` **runtimeTelemetry?**: [`RuntimeTelemetryOptions`](index.md#runtimetelemetryoptions) -Defined in: [intelligence/index.ts:257](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L257) +Defined in: [intelligence/index.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L280) Runtime-event payload policy. Tool inputs/results remain off unless explicitly enabled. @@ -2097,7 +2253,7 @@ Runtime-event payload policy. Tool inputs/results remain off unless explicitly e > `optional` **payloadAttributes?**: `"metadata"` \| `"full"` -Defined in: [intelligence/index.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L264) +Defined in: [intelligence/index.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L287) Payloads are metadata-only by default: the run span carries a stable hash and UTF-8 byte count, but not the redacted content. Set `full` only when @@ -2108,7 +2264,7 @@ inputs, outputs, and profiles. ### TraceMeta -Defined in: [intelligence/index.ts:268](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L268) +Defined in: [intelligence/index.ts:291](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L291) Metadata describing one traced run. `runId`/`traceId` default to fresh ids. @@ -2118,7 +2274,7 @@ Metadata describing one traced run. `runId`/`traceId` default to fresh ids. > `optional` **input?**: `unknown` -Defined in: [intelligence/index.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L270) +Defined in: [intelligence/index.ts:293](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L293) The run's input — exported through the redactor. @@ -2126,7 +2282,7 @@ The run's input — exported through the redactor. > `optional` **runId?**: `string` -Defined in: [intelligence/index.ts:272](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L272) +Defined in: [intelligence/index.ts:295](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L295) Stable run id. Defaults to a fresh id. @@ -2134,7 +2290,7 @@ Stable run id. Defaults to a fresh id. > `optional` **traceId?**: `string` -Defined in: [intelligence/index.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L274) +Defined in: [intelligence/index.ts:297](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L297) 32-hex trace id. Defaults to a fresh id. @@ -2142,7 +2298,7 @@ Defined in: [intelligence/index.ts:274](https://github.com/tangle-network/agent- > `optional` **model?**: `string` -Defined in: [intelligence/index.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L276) +Defined in: [intelligence/index.ts:299](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L299) Model id, when known — stamped on the span. @@ -2150,7 +2306,7 @@ Model id, when known — stamped on the span. > `optional` **provider?**: `string` -Defined in: [intelligence/index.ts:278](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L278) +Defined in: [intelligence/index.ts:301](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L301) Provider name, when known — stamped on the span. @@ -2158,7 +2314,7 @@ Provider name, when known — stamped on the span. > `optional` **labels?**: `Record`\<`string`, `string` \| `number` \| `boolean`\> -Defined in: [intelligence/index.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L280) +Defined in: [intelligence/index.ts:303](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L303) Arbitrary extra labels (string/number/boolean) stamped on the span. @@ -2166,7 +2322,7 @@ Arbitrary extra labels (string/number/boolean) stamped on the span. ### TraceHandle -Defined in: [intelligence/index.ts:289](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L289) +Defined in: [intelligence/index.ts:312](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L312) The trace handle a `traceRun` body records into. `recordOutput` captures the agent's result (redacted on export); `recordOutcome` captures the scored @@ -2179,7 +2335,7 @@ an un-recorded run still exports a span with whatever was set. > **recordOutput**(`output`): `void` -Defined in: [intelligence/index.ts:291](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L291) +Defined in: [intelligence/index.ts:314](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L314) Capture the run's output. Exported through the redactor. @@ -2197,7 +2353,7 @@ Capture the run's output. Exported through the redactor. > **recordOutcome**(`outcome`): `void` -Defined in: [intelligence/index.ts:298](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L298) +Defined in: [intelligence/index.ts:321](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L321) Capture the run's outcome. `usage` defaults to inference-only (`intelligenceUsd: 0`) — the OFF baseline; an intelligence-enabled run @@ -2232,7 +2388,7 @@ treated as pure inference. ### RecordTraceMeta -Defined in: [intelligence/index.ts:307](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L307) +Defined in: [intelligence/index.ts:330](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L330) Metadata for [IntelligenceClient.recordTrace](#recordtrace). @@ -2242,7 +2398,7 @@ Metadata for [IntelligenceClient.recordTrace](#recordtrace). > `optional` **traceId?**: `string` -Defined in: [intelligence/index.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L309) +Defined in: [intelligence/index.ts:332](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L332) 32-hex trace id to anchor every span to. Defaults to a fresh id. @@ -2250,7 +2406,7 @@ Defined in: [intelligence/index.ts:309](https://github.com/tangle-network/agent- > `optional` **rootParentSpanId?**: `string` -Defined in: [intelligence/index.ts:312](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L312) +Defined in: [intelligence/index.ts:335](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L335) Span id of an enclosing span the loop root should parent under (e.g. a `traceRun` span). Omitted ⇒ the loop root is the trace root. @@ -2259,7 +2415,7 @@ Span id of an enclosing span the loop root should parent under (e.g. a ### TraceOutcome -Defined in: [intelligence/index.ts:317](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L317) +Defined in: [intelligence/index.ts:340](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L340) The resolved outcome of one traced run, surfaced on the export span and available to the caller for downstream billing assertions. @@ -2270,25 +2426,25 @@ The resolved outcome of one traced run, surfaced on the export span and > **runId**: `string` -Defined in: [intelligence/index.ts:318](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L318) +Defined in: [intelligence/index.ts:341](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L341) ##### traceId > **traceId**: `string` -Defined in: [intelligence/index.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L319) +Defined in: [intelligence/index.ts:342](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L342) ##### project > **project**: `string` -Defined in: [intelligence/index.ts:320](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L320) +Defined in: [intelligence/index.ts:343](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L343) ##### effort > **effort**: [`EffortSettings`](#effortsettings) -Defined in: [intelligence/index.ts:322](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L322) +Defined in: [intelligence/index.ts:345](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L345) The resolved effort settings this run executed under. @@ -2296,7 +2452,7 @@ The resolved effort settings this run executed under. > **intelligenceOff**: `boolean` -Defined in: [intelligence/index.ts:324](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L324) +Defined in: [intelligence/index.ts:347](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L347) True when this run ran as pure passthrough (the OFF floor). @@ -2304,19 +2460,19 @@ True when this run ran as pure passthrough (the OFF floor). > `optional` **success?**: `boolean` -Defined in: [intelligence/index.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L325) +Defined in: [intelligence/index.ts:348](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L348) ##### score? > `optional` **score?**: `number` -Defined in: [intelligence/index.ts:326](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L326) +Defined in: [intelligence/index.ts:349](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L349) ##### usage > **usage**: [`UsageSplit`](#usagesplit) -Defined in: [intelligence/index.ts:328](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L328) +Defined in: [intelligence/index.ts:351](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L351) Per-class billing split. `intelligenceUsd` is `0` at the OFF tier. @@ -2324,7 +2480,7 @@ Per-class billing split. `intelligenceUsd` is `0` at the OFF tier. ### IntelligenceClient -Defined in: [intelligence/index.ts:332](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L332) +Defined in: [intelligence/index.ts:355](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L355) The Observe-mode Intelligence client. @@ -2334,7 +2490,7 @@ The Observe-mode Intelligence client. > `readonly` **project**: `string` -Defined in: [intelligence/index.ts:334](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L334) +Defined in: [intelligence/index.ts:357](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L357) The resolved project id. @@ -2342,7 +2498,7 @@ The resolved project id. > `readonly` **effort**: [`EffortSettings`](#effortsettings) -Defined in: [intelligence/index.ts:336](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L336) +Defined in: [intelligence/index.ts:359](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L359) The resolved effort settings. @@ -2352,7 +2508,7 @@ The resolved effort settings. > **traceRun**\<`T`\>(`meta`, `fn`): `Promise`\<`T`\> -Defined in: [intelligence/index.ts:342](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L342) +Defined in: [intelligence/index.ts:365](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L365) Run `fn` under a trace, export one span best-effort, and return whatever `fn` returns. Telemetry-export failures are swallowed; an error THROWN by @@ -2382,7 +2538,7 @@ Run `fn` under a trace, export one span best-effort, and return whatever > **recordTrace**(`events`, `meta?`): `string` -Defined in: [intelligence/index.ts:352](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L352) +Defined in: [intelligence/index.ts:375](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L375) Export a run's full loop topology — the ordered `LoopTraceEvent` stream a `runLoop`/`Supervisor` run emits — as a nested OTLP span tree (loop → round → @@ -2410,7 +2566,7 @@ readonly [`LoopTraceEvent`](runtime.md#looptraceevent)[] > **exportRunRecord**(`record`): `string` -Defined in: [intelligence/index.ts:360](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L360) +Defined in: [intelligence/index.ts:383](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L383) Send one typed [RunRecord](#runrecord) — the run's flat span (input/output/outcome/ usage/model/provider, redacted) plus, when `loopEvents` are present, the @@ -2432,7 +2588,7 @@ Best-effort: export failures are swallowed. Returns the record's `traceId`. > **freshRunId**(): `string` -Defined in: [intelligence/index.ts:362](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L362) +Defined in: [intelligence/index.ts:385](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L385) Mint a fresh run id (`run-`). @@ -2444,7 +2600,7 @@ Mint a fresh run id (`run-`). > **freshTraceId**(): `string` -Defined in: [intelligence/index.ts:364](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L364) +Defined in: [intelligence/index.ts:387](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L387) Mint a fresh 32-hex trace id. @@ -2456,7 +2612,7 @@ Mint a fresh 32-hex trace id. > **doctor**(): [`DoctorReport`](#doctorreport) -Defined in: [intelligence/index.ts:370](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L370) +Defined in: [intelligence/index.ts:393](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L393) Network-free readiness report: which adoption modes are reachable given this config. Observe is always reachable; Recommend needs outcomes; PR @@ -2470,7 +2626,7 @@ needs checks + surfaces + repo. > **flush**(): `Promise`\<`void`\> -Defined in: [intelligence/index.ts:372](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L372) +Defined in: [intelligence/index.ts:395](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L395) Flush any pending export spans. Best-effort; resolves even if export fails. @@ -2482,7 +2638,7 @@ Flush any pending export spans. Best-effort; resolves even if export fails. ### ModeReadiness -Defined in: [intelligence/index.ts:376](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L376) +Defined in: [intelligence/index.ts:399](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L399) One mode's readiness verdict. @@ -2492,13 +2648,13 @@ One mode's readiness verdict. > **ready**: `boolean` -Defined in: [intelligence/index.ts:377](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L377) +Defined in: [intelligence/index.ts:400](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L400) ##### missing > **missing**: `string`[] -Defined in: [intelligence/index.ts:379](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L379) +Defined in: [intelligence/index.ts:402](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L402) Inputs this mode still needs, when not ready. Empty when ready. @@ -2506,7 +2662,7 @@ Inputs this mode still needs, when not ready. Empty when ready. ### DoctorReport -Defined in: [intelligence/index.ts:383](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L383) +Defined in: [intelligence/index.ts:406](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L406) The `doctor()` readiness report — Mode-readiness without any network call. @@ -2516,19 +2672,19 @@ The `doctor()` readiness report — Mode-readiness without any network call. > **project**: `string` -Defined in: [intelligence/index.ts:384](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L384) +Defined in: [intelligence/index.ts:407](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L407) ##### effort > **effort**: [`EffortSettings`](#effortsettings) -Defined in: [intelligence/index.ts:385](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L385) +Defined in: [intelligence/index.ts:408](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L408) ##### exportConfigured > **exportConfigured**: `boolean` -Defined in: [intelligence/index.ts:387](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L387) +Defined in: [intelligence/index.ts:410](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L410) True when an OTLP endpoint is configured (export will actually ship). @@ -2536,7 +2692,7 @@ True when an OTLP endpoint is configured (export will actually ship). > **modes**: `object` -Defined in: [intelligence/index.ts:388](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L388) +Defined in: [intelligence/index.ts:411](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L411) ###### observe @@ -2760,6 +2916,180 @@ carrying an un-admitted binding kind is a hard error, not a soft drop). *** +### CreateSandboxCandidateExperimentExecutorOptions + +Defined in: [intelligence/sandbox-approved-candidate.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L77) + +#### Properties + +##### client + +> **client**: `SandboxClientPort` + +Defined in: [intelligence/sandbox-approved-candidate.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L78) + +##### ports + +> **ports**: [`AgentCandidateExecutionPorts`](index.md#agentcandidateexecutionports) + +Defined in: [intelligence/sandbox-approved-candidate.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L79) + +##### grader + +> **grader**: [`AgentCandidateBenchmarkGraderPort`](index.md#agentcandidatebenchmarkgraderport) + +Defined in: [intelligence/sandbox-approved-candidate.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L80) + +##### outputArtifacts + +> **outputArtifacts**: [`AgentCandidateOutputArtifactPort`](index.md#agentcandidateoutputartifactport) + +Defined in: [intelligence/sandbox-approved-candidate.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L81) + +##### traceStore + +> **traceStore**: `TraceStore` + +Defined in: [intelligence/sandbox-approved-candidate.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L82) + +##### claimStore + +> **claimStore**: [`AgentCandidateExecutionClaimStore`](index.md#agentcandidateexecutionclaimstore) + +Defined in: [intelligence/sandbox-approved-candidate.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L83) + +##### sandbox? + +> `optional` **sandbox?**: `object` + +Defined in: [intelligence/sandbox-approved-candidate.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L84) + +###### teamId? + +> `optional` **teamId?**: `string` + +###### resources? + +> `optional` **resources?**: `SandboxResources` + +###### createTimeoutMs? + +> `optional` **createTimeoutMs?**: `number` + +###### evidenceRetentionSeconds? + +> `optional` **evidenceRetentionSeconds?**: `number` + +##### cleanupTimeoutMs? + +> `optional` **cleanupTimeoutMs?**: `number` + +Defined in: [intelligence/sandbox-approved-candidate.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L90) + +##### resultTimeoutMs? + +> `optional` **resultTimeoutMs?**: `number` + +Defined in: [intelligence/sandbox-approved-candidate.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L91) + +*** + +### SandboxCandidateExperimentExecution + +Defined in: [intelligence/sandbox-approved-candidate.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L94) + +#### Extends + +- `CandidateExperimentExecutionInput` + +#### Properties + +##### executionId + +> **executionId**: `string` + +Defined in: [intelligence/sandbox-approved-candidate.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L95) + +##### attempt? + +> `optional` **attempt?**: `number` + +Defined in: [intelligence/sandbox-approved-candidate.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L96) + +##### executionRoots + +> **executionRoots**: `object` + +Defined in: [intelligence/sandbox-approved-candidate.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L97) + +###### taskRoot + +> **taskRoot**: `string` + +###### candidateRoot? + +> `optional` **candidateRoot?**: `string` + +##### stagingRoots + +> **stagingRoots**: `object` + +Defined in: [intelligence/sandbox-approved-candidate.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L98) + +###### taskRoot + +> **taskRoot**: `string` + +###### candidateRoot? + +> `optional` **candidateRoot?**: `string` + +###### profileRoot + +> **profileRoot**: `string` + +##### preparation? + +> `optional` **preparation?**: [`PrepareAgentCandidateExecutionOptions`](index.md#prepareagentcandidateexecutionoptions) + +Defined in: [intelligence/sandbox-approved-candidate.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L99) + +*** + +### SandboxCandidateExperimentExecutor + +Defined in: [intelligence/sandbox-approved-candidate.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L102) + +#### Properties + +##### executor + +> `readonly` **executor**: [`AgentCandidateExecutorPort`](index.md#agentcandidateexecutorport) + +Defined in: [intelligence/sandbox-approved-candidate.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L104) + +The same port is usable by Runtime's expired-claim recovery path. + +#### Methods + +##### execute() + +> **execute**(`input`): `Promise`\<`CandidateExecutionEvidence`\> + +Defined in: [intelligence/sandbox-approved-candidate.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L105) + +###### Parameters + +###### input + +[`SandboxCandidateExperimentExecution`](#sandboxcandidateexperimentexecution) + +###### Returns + +`Promise`\<`CandidateExecutionEvidence`\> + +*** + ### AppliedIntelligence Defined in: [intelligence/with-intelligence.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/with-intelligence.ts#L53) @@ -2884,7 +3214,7 @@ Defined in: [intelligence/with-intelligence.ts:83](https://github.com/tangle-net > **project**: `string` -Defined in: [intelligence/index.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L227) +Defined in: [intelligence/index.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L250) Stable project id — the tenant dimension every trace is tagged with. @@ -2896,7 +3226,7 @@ Stable project id — the tenant dimension every trace is tagged with. > `optional` **apiKey?**: `string` -Defined in: [intelligence/index.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L229) +Defined in: [intelligence/index.ts:252](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L252) Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. @@ -2908,7 +3238,7 @@ Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. > `optional` **effort?**: [`EffortTier`](#efforttier) \| \{ `tier`: [`EffortTier`](#efforttier); `overrides?`: `Partial`\<[`EffortSettings`](#effortsettings)\>; \} -Defined in: [intelligence/index.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L231) +Defined in: [intelligence/index.ts:254](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L254) Effort tier (default `'standard'`) plus optional per-field overrides. @@ -2920,7 +3250,7 @@ Effort tier (default `'standard'`) plus optional per-field overrides. > `optional` **baseUrl?**: `string` -Defined in: [intelligence/index.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L239) +Defined in: [intelligence/index.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L262) The ONE Tangle Intelligence base URL — both the send (OTLP `/v1/otlp`) and receive (`/v1/profiles/:target/composed`) paths derive from it. Reads @@ -2936,7 +3266,7 @@ key the ingest requires); absent a key, export is a no-op. > `optional` **redact?**: `false` \| [`Redactor`](#redactor) -Defined in: [intelligence/index.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L245) +Defined in: [intelligence/index.ts:268](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L268) Redaction hook run over every exported input/output. A function replaces the default scrubber; `false` opts out entirely (raw fidelity, caller has @@ -2950,7 +3280,7 @@ sanitized upstream); omitted ⇒ the built-in `defaultRedactor`. > `optional` **surfaces?**: `string`[] -Defined in: [intelligence/index.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L247) +Defined in: [intelligence/index.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L270) Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. @@ -2962,7 +3292,7 @@ Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. > `optional` **checks?**: `string`[] -Defined in: [intelligence/index.ts:249](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L249) +Defined in: [intelligence/index.ts:272](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L272) Verification checks a later PR mode would gate on. Recorded for `doctor()` only. @@ -2974,7 +3304,7 @@ Verification checks a later PR mode would gate on. Recorded for `doctor()` only. > `optional` **repo?**: [`RepoConfig`](#repoconfig) -Defined in: [intelligence/index.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L251) +Defined in: [intelligence/index.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L274) Repo access a later PR mode would need. Recorded for `doctor()` only. @@ -2986,7 +3316,7 @@ Repo access a later PR mode would need. Recorded for `doctor()` only. > `optional` **profile?**: `AgentProfile` -Defined in: [intelligence/index.ts:253](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L253) +Defined in: [intelligence/index.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L276) Full canonical profile used for this agent. Exported redacted with a stable hash. @@ -2998,7 +3328,7 @@ Full canonical profile used for this agent. Exported redacted with a stable hash > `optional` **commitSha?**: `string` -Defined in: [intelligence/index.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L255) +Defined in: [intelligence/index.ts:278](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L278) Commit that produced the running agent, when known. @@ -3010,7 +3340,7 @@ Commit that produced the running agent, when known. > `optional` **runtimeTelemetry?**: [`RuntimeTelemetryOptions`](index.md#runtimetelemetryoptions) -Defined in: [intelligence/index.ts:257](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L257) +Defined in: [intelligence/index.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L280) Runtime-event payload policy. Tool inputs/results remain off unless explicitly enabled. @@ -3022,7 +3352,7 @@ Runtime-event payload policy. Tool inputs/results remain off unless explicitly e > `optional` **payloadAttributes?**: `"metadata"` \| `"full"` -Defined in: [intelligence/index.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L264) +Defined in: [intelligence/index.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L287) Payloads are metadata-only by default: the run span carries a stable hash and UTF-8 byte count, but not the redacted content. Set `full` only when @@ -3222,37 +3552,11 @@ Per-field overrides applied on top of a tier preset. Any subset of the *** -### AgentImprovementEvaluation - -> **AgentImprovementEvaluation**\<`TScenario`, `TArtifact`\> = `Pick`\<`SelfImproveResult`\<`TScenario`, `TArtifact`\>, `"baseline"` \| `"winner"` \| `"lift"` \| `"diff"` \| `"provenance"` \| `"gateDecision"` \| `"generationsExplored"` \| `"durationMs"` \| `"totalCostUsd"` \| `"insight"` \| `"power"`\> - -Defined in: [intelligence/improvement-cycle.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L47) - -#### Type Parameters - -##### TScenario - -`TScenario` *extends* `Scenario` - -##### TArtifact - -`TArtifact` - -*** - -### AgentImprovementReviewDecision - -> **AgentImprovementReviewDecision** = `"approve"` \| `"reject"` \| `"request-changes"` - -Defined in: [intelligence/improvement-cycle.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L81) - -*** - ### UsageClass > **UsageClass** = `"inference"` \| `"intelligence"` -Defined in: [intelligence/index.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L135) +Defined in: [intelligence/index.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L158) Usage class for billing. Base-stream tokens bill `'inference'`; every intelligence spawn (analyst, corpus, loop) bills `'intelligence'`. The @@ -3375,8 +3679,42 @@ The default tier when a client declares no effort. `'standard'` turns intelligence on with sensible knobs; opt down to `'off'`/`'eco'` or up to `'thorough'`/`'max'`. +*** + +### sandboxCandidateExperimentExecutionSupport + +> `const` **sandboxCandidateExperimentExecutionSupport**: `Readonly`\<\{ `outcomes`: readonly \[`"output"`\]; `outputMediaTypes`: readonly \[`"text/*"`, `"application/json"`, `"*+json"`\]; `code`: readonly \[`"disabled"`\]; `memory`: readonly \[`"disabled"`\]; `knowledge`: `true`; `profile`: `Readonly`\<\{ `mcpTransports`: readonly \[`"stdio"`\]; `remoteMcp`: `false`; `tools`: `false`; `permissions`: `false`; `modes`: `false`; `confidential`: `false`; \}\>; `isolation`: `Readonly`\<\{ `freshSandbox`: `true`; `exactProcess`: `true`; `egress`: readonly \[`"blocked"`, `"strict"`\]; \}\>; \}\> + +Defined in: [intelligence/sandbox-approved-candidate.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L63) + +Declares the exact candidate surfaces the sandbox executor can run. + ## Functions +### parseAgentCandidateProfileActivation() + +> **parseAgentCandidateProfileActivation**(`input`, `expectedProfilePlanDigest?`): `AgentCandidateProfileActivation` + +Defined in: [candidate-execution/profile.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/profile.ts#L93) + +Parse and check every native file hash plus both canonical document digests. + +#### Parameters + +##### input + +`unknown` + +##### expectedProfilePlanDigest? + +`` `sha256:${string}` `` + +#### Returns + +`AgentCandidateProfileActivation` + +*** + ### manifestFromProfile() > **manifestFromProfile**(`profile`): [`CapabilityManifest`](#capabilitymanifest) @@ -3606,45 +3944,73 @@ compile to `withAnalyst: true`, the tier's `fanout`, and `withLoops: true`. *** -### proposeAgentImprovement() +### runAgentCandidateExperiment() -> **proposeAgentImprovement**\<`TScenario`, `TArtifact`\>(`options`): `Promise`\<[`ProposeAgentImprovementResult`](#proposeagentimprovementresult)\<`TScenario`, `TArtifact`\>\> +> **runAgentCandidateExperiment**(`options`): `Promise`\<[`RunAgentCandidateExperimentResult`](#runagentcandidateexperimentresult)\> -Defined in: [intelligence/improvement-cycle.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L196) +Defined in: [intelligence/improvement-cycle.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L187) -Analyze one run and produce one measured, review-only improvement proposal. +Execute both arms of one immutable experiment and derive its paired result. -#### Type Parameters +#### Parameters -##### TScenario +##### options -`TScenario` *extends* `Scenario` +[`RunAgentCandidateExperimentOptions`](#runagentcandidateexperimentoptions) -##### TArtifact +#### Returns -`TArtifact` +`Promise`\<[`RunAgentCandidateExperimentResult`](#runagentcandidateexperimentresult)\> + +*** + +### executeAgentCandidateExperimentCell() + +> **executeAgentCandidateExperimentCell**(`options`): `Promise`\<`CandidateExecutionEvidence`\> + +Defined in: [intelligence/improvement-cycle.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L218) + +Execute one exact arm, task, repetition, seed, and attempt through Runtime. #### Parameters ##### options -[`ProposeAgentImprovementOptions`](#proposeagentimprovementoptions)\<`TScenario`, `TArtifact`\> +[`ExecuteAgentCandidateExperimentCellOptions`](#executeagentcandidateexperimentcelloptions) #### Returns -`Promise`\<[`ProposeAgentImprovementResult`](#proposeagentimprovementresult)\<`TScenario`, `TArtifact`\>\> +`Promise`\<`CandidateExecutionEvidence`\> *** -### createAgentImprovementProposal() +### createAgentImprovementMeasuredComparison() + +> **createAgentImprovementMeasuredComparison**(`options`): `AgentImprovementMeasuredComparison` -> **createAgentImprovementProposal**\<`TScenario`, `TArtifact`\>(`options`): [`AgentImprovementProposal`](#agentimprovementproposal)\<`TScenario`, `TArtifact`\> +Defined in: [intelligence/improvement-cycle.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L274) -Defined in: [intelligence/improvement-cycle.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L244) +Delegate all statistics and promotion checks to agent-eval's receipt-based comparison. -Freeze an already-measured improvement into the one reviewable proposal -contract. Products that run analysis or evaluation in separate workers use -this constructor instead of rerunning either phase or rebuilding digests. +#### Parameters + +##### options + +`CompareCandidateExperimentOptions` + +#### Returns + +`AgentImprovementMeasuredComparison` + +*** + +### proposeAgentImprovement() + +> **proposeAgentImprovement**\<`TScenario`, `TArtifact`\>(`options`): `Promise`\<[`ProposeAgentImprovementResult`](#proposeagentimprovementresult)\<`TScenario`, `TArtifact`\>\> + +Defined in: [intelligence/improvement-cycle.ts:281](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L281) + +Analyze, search, then remeasure the resulting exact candidate before proposing it. #### Type Parameters @@ -3660,27 +4026,47 @@ this constructor instead of rerunning either phase or rebuilding digests. ##### options -[`CreateAgentImprovementProposalOptions`](#createagentimprovementproposaloptions)\<`TScenario`, `TArtifact`\> +[`ProposeAgentImprovementOptions`](#proposeagentimprovementoptions)\<`TScenario`, `TArtifact`\> + +#### Returns + +`Promise`\<[`ProposeAgentImprovementResult`](#proposeagentimprovementresult)\<`TScenario`, `TArtifact`\>\> + +*** + +### createAgentImprovementProposal() + +> **createAgentImprovementProposal**(`options`): `AgentImprovementProposal` + +Defined in: [intelligence/improvement-cycle.ts:342](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L342) + +Create the reviewable record only from a complete, recomputable experiment result. + +#### Parameters + +##### options + +[`CreateAgentImprovementProposalOptions`](#createagentimprovementproposaloptions) #### Returns -[`AgentImprovementProposal`](#agentimprovementproposal)\<`TScenario`, `TArtifact`\> +`AgentImprovementProposal` *** ### reviewAgentImprovementProposal() -> **reviewAgentImprovementProposal**(`inputProposal`, `input`): [`AgentImprovementReview`](#agentimprovementreview) +> **reviewAgentImprovementProposal**(`inputProposal`, `input`): `AgentImprovementReview` -Defined in: [intelligence/improvement-cycle.ts:294](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L294) +Defined in: [intelligence/improvement-cycle.ts:373](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L373) -Persist an approve/reject/change-request decision bound to one exact proposal. +Persist a human or tenant-policy decision bound to one exact proposal. #### Parameters ##### inputProposal -[`AgentImprovementProposal`](#agentimprovementproposal) +`AgentImprovementProposal` ##### input @@ -3688,37 +4074,45 @@ Persist an approve/reject/change-request decision bound to one exact proposal. #### Returns -[`AgentImprovementReview`](#agentimprovementreview) +`AgentImprovementReview` *** -### executeApprovedAgentCandidate() +### createAgentImprovementActivation() -> **executeApprovedAgentCandidate**(`options`): `Promise`\<[`ExecuteApprovedAgentCandidateResult`](#executeapprovedagentcandidateresult)\> +> **createAgentImprovementActivation**(`inputProposal`, `inputReview`, `options`): `AgentImprovementActivation` -Defined in: [intelligence/improvement-cycle.ts:324](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L324) +Defined in: [intelligence/improvement-cycle.ts:397](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L397) -Verify, materialize, run, grade, and receipt only the exact approved bundle. +Authorize product-owned writes only after the exact candidate was measured and approved. #### Parameters +##### inputProposal + +`AgentImprovementProposal` + +##### inputReview + +`AgentImprovementReview` + ##### options -[`ExecuteApprovedAgentCandidateOptions`](#executeapprovedagentcandidateoptions) +[`CreateAgentImprovementActivationOptions`](#createagentimprovementactivationoptions) #### Returns -`Promise`\<[`ExecuteApprovedAgentCandidateResult`](#executeapprovedagentcandidateresult)\> +`AgentImprovementActivation` *** ### verifyAgentImprovementProposal() -> **verifyAgentImprovementProposal**(`input`): [`AgentImprovementProposal`](#agentimprovementproposal) +> **verifyAgentImprovementProposal**(`input`): `AgentImprovementProposal` -Defined in: [intelligence/improvement-cycle.ts:366](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L366) +Defined in: [intelligence/improvement-cycle.ts:428](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L428) -Validate a proposal's schema, profile, sealed bundle, and canonical digest. +Validate a proposal and recompute every binding to its measured experiment. #### Parameters @@ -3728,17 +4122,67 @@ Validate a proposal's schema, profile, sealed bundle, and canonical digest. #### Returns -[`AgentImprovementProposal`](#agentimprovementproposal) +`AgentImprovementProposal` *** ### verifyAgentImprovementReview() -> **verifyAgentImprovementReview**(`input`): [`AgentImprovementReview`](#agentimprovementreview) +> **verifyAgentImprovementReview**(`input`): `AgentImprovementReview` + +Defined in: [intelligence/improvement-cycle.ts:452](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L452) + +Validate the canonical identity and wire shape of an improvement review. + +#### Parameters + +##### input + +`unknown` + +#### Returns + +`AgentImprovementReview` + +*** + +### verifyAgentImprovementActivation() + +> **verifyAgentImprovementActivation**(`input`): `AgentImprovementActivation` + +Defined in: [intelligence/improvement-cycle.ts:460](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L460) + +Validate activation authority against the exact proposal, review, experiment, and base state. + +#### Parameters + +##### input + +###### proposal + +`unknown` + +###### review + +`unknown` + +###### activation -Defined in: [intelligence/improvement-cycle.ts:483](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L483) +`unknown` + +#### Returns -Validate a review's decision fields and canonical digest. +`AgentImprovementActivation` + +*** + +### verifyCandidateExecutionEvidence() + +> **verifyCandidateExecutionEvidence**(`input`, `options`): `CandidateExecutionEvidence` + +Defined in: [intelligence/improvement-cycle.ts:487](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L487) + +Recheck one Runtime receipt against its exact signed experiment cell. #### Parameters @@ -3746,9 +4190,13 @@ Validate a review's decision fields and canonical digest. `unknown` +##### options + +[`VerifyCandidateExecutionEvidenceOptions`](#verifycandidateexecutionevidenceoptions) + #### Returns -[`AgentImprovementReview`](#agentimprovementreview) +`CandidateExecutionEvidence` *** @@ -3756,7 +4204,7 @@ Validate a review's decision fields and canonical digest. > **createIntelligenceClient**(`config`): [`IntelligenceClient`](#intelligenceclient) -Defined in: [intelligence/index.ts:454](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L454) +Defined in: [intelligence/index.ts:477](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L477) Create an Observe-mode Intelligence client. Resolves effort, the base URL, and the redactor up front; the exporter is built lazily and is `undefined` when no @@ -3890,6 +4338,26 @@ Lower a plane `CertifiedProfile` straight into a `ResolvedSurface` via *** +### createSandboxCandidateExperimentExecutor() + +> **createSandboxCandidateExperimentExecutor**(`options`): [`SandboxCandidateExperimentExecutor`](#sandboxcandidateexperimentexecutor) + +Defined in: [intelligence/sandbox-approved-candidate.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/sandbox-approved-candidate.ts#L109) + +Execute one signed experiment cell inside a fresh Tangle sandbox. + +#### Parameters + +##### options + +[`CreateSandboxCandidateExperimentExecutorOptions`](#createsandboxcandidateexperimentexecutoroptions) + +#### Returns + +[`SandboxCandidateExperimentExecutor`](#sandboxcandidateexperimentexecutor) + +*** + ### withIntelligence() > **withIntelligence**\<`I`, `O`\>(`agent`, `config`): [`IntelligenceWrapped`](#intelligencewrapped)\<`I`, `O`\> diff --git a/docs/api/knowledge.md b/docs/api/knowledge.md index 2deb753d..cfef26aa 100644 --- a/docs/api/knowledge.md +++ b/docs/api/knowledge.md @@ -14,6 +14,12 @@ Re-exports [AgentKnowledgeReadinessCheckOptions](index.md#agentknowledgereadines *** +### ApprovedKnowledgeImprovementCandidate + +Re-exports [ApprovedKnowledgeImprovementCandidate](index.md#approvedknowledgeimprovementcandidate) + +*** + ### createAgentKnowledgeReadinessCheck Re-exports [createAgentKnowledgeReadinessCheck](index.md#createagentknowledgereadinesscheck) diff --git a/docs/api/mcp.md b/docs/api/mcp.md index 94756b9a..21bbec37 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -432,7 +432,7 @@ readonly `string`[] ### InMemoryFeedbackStore -Defined in: [mcp/feedback-store.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L42) +Defined in: [mcp/feedback-store.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L41) **`Experimental`** @@ -460,7 +460,7 @@ In-memory `FeedbackStore` — suitable for single-process use and tests. > **put**(`event`): `Promise`\<`void`\> -Defined in: [mcp/feedback-store.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L45) +Defined in: [mcp/feedback-store.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L44) **`Experimental`** @@ -484,7 +484,7 @@ Append a new event. Never dedupes — every rating is its own event. > **list**(`filter?`): `Promise`\<[`FeedbackEvent`](#feedbackevent)[]\> -Defined in: [mcp/feedback-store.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L49) +Defined in: [mcp/feedback-store.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L48) **`Experimental`** @@ -2081,7 +2081,7 @@ machineId so workers don't compete with the orchestrator on the same VM. ### FeedbackEvent -Defined in: [mcp/feedback-store.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L21) +Defined in: [mcp/feedback-store.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L20) **`Experimental`** @@ -2091,7 +2091,7 @@ Defined in: [mcp/feedback-store.ts:21](https://github.com/tangle-network/agent-r > **id**: `string` -Defined in: [mcp/feedback-store.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L22) +Defined in: [mcp/feedback-store.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L21) **`Experimental`** @@ -2099,7 +2099,7 @@ Defined in: [mcp/feedback-store.ts:22](https://github.com/tangle-network/agent-r > **refersTo**: [`FeedbackRefersTo`](#feedbackrefersto) -Defined in: [mcp/feedback-store.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L23) +Defined in: [mcp/feedback-store.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L22) **`Experimental`** @@ -2107,7 +2107,7 @@ Defined in: [mcp/feedback-store.ts:23](https://github.com/tangle-network/agent-r > **rating**: [`FeedbackRating`](#feedbackrating) -Defined in: [mcp/feedback-store.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L24) +Defined in: [mcp/feedback-store.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L23) **`Experimental`** @@ -2115,7 +2115,7 @@ Defined in: [mcp/feedback-store.ts:24](https://github.com/tangle-network/agent-r > **by**: `"agent"` \| `"user"` \| `"downstream-judge"` -Defined in: [mcp/feedback-store.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L25) +Defined in: [mcp/feedback-store.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L24) **`Experimental`** @@ -2123,7 +2123,7 @@ Defined in: [mcp/feedback-store.ts:25](https://github.com/tangle-network/agent-r > **capturedAt**: `string` -Defined in: [mcp/feedback-store.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L26) +Defined in: [mcp/feedback-store.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L25) **`Experimental`** @@ -2131,7 +2131,7 @@ Defined in: [mcp/feedback-store.ts:26](https://github.com/tangle-network/agent-r > `optional` **namespace?**: `string` -Defined in: [mcp/feedback-store.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L27) +Defined in: [mcp/feedback-store.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L26) **`Experimental`** @@ -2139,7 +2139,7 @@ Defined in: [mcp/feedback-store.ts:27](https://github.com/tangle-network/agent-r ### FeedbackStore -Defined in: [mcp/feedback-store.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L31) +Defined in: [mcp/feedback-store.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L30) **`Experimental`** @@ -2149,7 +2149,7 @@ Defined in: [mcp/feedback-store.ts:31](https://github.com/tangle-network/agent-r > **put**(`event`): `Promise`\<`void`\> -Defined in: [mcp/feedback-store.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L33) +Defined in: [mcp/feedback-store.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L32) **`Experimental`** @@ -2169,7 +2169,7 @@ Append a new event. Never dedupes — every rating is its own event. > **list**(`filter?`): `Promise`\<[`FeedbackEvent`](#feedbackevent)[]\> -Defined in: [mcp/feedback-store.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L38) +Defined in: [mcp/feedback-store.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L37) **`Experimental`** @@ -5503,9 +5503,9 @@ Defined in: [mcp/types.ts:235](https://github.com/tangle-network/agent-runtime/b **`Experimental`** -Loose shape of a research output over the wire — the substrate cannot -import the `ResearchOutput` type from agent-knowledge without inducing -a dependency cycle, so the MCP layer treats it structurally. +Provider-neutral research output carried over the MCP boundary. The MCP +layer accepts this structural shape instead of coupling its wire contract to +one research implementation. #### Indexable @@ -7580,7 +7580,7 @@ cross-sandbox copy step. > **eventToSnapshot**(`event`): [`DelegationFeedbackSnapshot`](#delegationfeedbacksnapshot) -Defined in: [mcp/feedback-store.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L67) +Defined in: [mcp/feedback-store.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/feedback-store.ts#L66) **`Experimental`** diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 4fed3f6c..ab3795a1 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.94.13` and `@tangle-network/agent-eval@0.120.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.95.0` and `@tangle-network/agent-eval@0.122.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 342 exports. +Import from `@tangle-network/agent-runtime` — 345 exports. | Symbol | Kind | Summary | |---|---|---| @@ -30,6 +30,7 @@ Import from `@tangle-network/agent-runtime` — 342 exports. | `buildLoopSpanNodes` | function | Sink-neutral core behind {@link buildLoopOtelSpans}: reconstruct the | | `buildRuntimeEventOtelSpans` | function | Convert normalized runtime events into lossless, redacted child spans. | | `candidateExecutionClaim` | function | Extract the complete durable claim from a prepared execution. | +| `candidateKnowledgeExecutionPaths` | function | Deterministic, signed locations used by every candidate executor. | | `captureAgentCandidateWorkspace` | function | Capture one exact regular-file workspace for immutable candidate execution. | | `captureAgentCandidateWorkspaceFiles` | function | Capture detached files returned by a remote executor into the standard archive. | | `cleanModelId` | function | Trim a candidate model id; `undefined` for non-strings and blanks. | @@ -91,7 +92,7 @@ Import from `@tangle-network/agent-runtime` — 342 exports. | `runConversation` | function | Conversation orchestrator. Drives N participants in turn through their own | | `runConversationStream` | function | Streaming conversation orchestrator: drives N participants in turn through their own backends, enforcing `maxTurns` / `maxCreditsCents` / `haltOn`, yielding per-event stream markers. | | `runDelegatedLoop` | function | Dispatch a configured loop by mode. Fails loud (throws `ConfigError`) when no | -| `runKnowledgeImprovementJob` | function | Run the full KB improvement job: candidate workspace, runtime supervisor update, readiness check, and promotion. | +| `runKnowledgeImprovementJob` | function | Produce a frozen KB candidate, and promote it only when an exact signed review is supplied. | | `runLoopRunnerCli` | function | Pure CLI core (no process / argv / IO) so it's unit-testable: validate the | | `runPersonaConversation` | function | Run one worker profile against one persona as a multi-round conversation. | | `runPersonaDispatch` | function | Wrap {@link runPersonaConversation} as a `ProfileDispatchFn` for | @@ -112,7 +113,10 @@ Import from `@tangle-network/agent-runtime` — 342 exports. | `validateChatModelId` | function | Validate a caller-supplied chat-model id. Rejects non-strings, malformed | | `verifyAgentCandidateBundle` | function | Verifies every digest, resource, workspace, and Git object in a candidate bundle. | | `worktreeLoopRunner` | function | `code` mode on the GENERIC recursive path: author one `AgentProfile` per harness, run them as a | +| `AGENT_CANDIDATE_EXECUTION_SUPPORT` | const | Surfaces admitted by Runtime's verifier before an environment adapter is selected. | | `AGENTIC_PROFILE_RESOURCE_ROOT` | const | Dedicated ephemeral root for generic author-profile files. Every declared | +| `CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV` | const | Environment variable containing the materialized retrieval configuration path. | +| `CANDIDATE_KNOWLEDGE_ROOT_ENV` | const | Environment variable containing the materialized candidate knowledge root. | | `CANDIDATE_TRACE_ENV` | const | Environment keys used to propagate immutable candidate trace identity. | | `CANDIDATE_TRACE_TAGS` | const | Protected trace tags that bind a run to one prepared candidate execution. | | `DEFAULT_MAX_DEPTH` | const | Hard cap on chained gateway hops; refused beyond this. Default keeps recursion bounded. | @@ -140,6 +144,7 @@ Import from `@tangle-network/agent-runtime` — 342 exports. | `SqlConversationJournal` | class | SQL-backed ConversationJournal. Two tables — runs (one row per runId, holds | | `ValidationError` | class | Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). | | `AgentCandidateArtifactPort` | interface | Reads one content-addressed object from the closed S3/IPFS locator set. | +| `AgentCandidateBenchmarkGraderIdentity` | interface | Immutable grader identity admitted for one benchmark task. | | `AgentCandidateBenchmarkGraderPort` | interface | Evaluator-owned executable grader, pinned by immutable implementation bytes. | | `AgentCandidateCodeSurfaceSource` | interface | The only accepted path from an agent-eval code candidate to executable bytes. | | `AgentCandidateExecutionAttemptRecord` | interface | Persisted state available to a fresh trusted recovery worker after a crash. | @@ -148,17 +153,16 @@ Import from `@tangle-network/agent-runtime` — 342 exports. | `AgentCandidateExecutionCleanupHandles` | interface | Non-secret identities a trusted recovery worker needs to close an abandoned attempt. | | `AgentCandidateExecutionLease` | interface | Secret capability required to finish the acquired attempt. | | `AgentCandidateExecutionRecoveryEvidence` | interface | Trusted, independently observed closure facts for one expired winning lease. | -| `AgentCandidateExecutionUsage` | interface | Exact fixed-point usage proven by the closed evaluator model ledger. | -| `AgentCandidateExecutorFinalCapture` | interface | Idempotent executor result after process death and trace drain. | +| `AgentCandidateExecutorFinalCapture` | interface | Replayable evaluator result captured only after process death and trace drain. | | `AgentCandidateExecutorMemoryCapture` | interface | Raw isolated-memory capture made only after access has been revoked. | | `AgentCandidateExecutorPort` | interface | Executes one prepared request inside an evaluator-owned isolation boundary. | +| `AgentCandidateExecutorProfileFile` | interface | One exact profile file supplied to an evaluator-owned executor. | | `AgentCandidateExecutorRequest` | interface | One detached request passed to the trusted environment-specific executor. | | `AgentCandidateExecutorStopRequest` | interface | Opaque process identity used for termination without re-exposing launch credentials. | -| `AgentCandidateExecutorTaskOutcomeCapture` | interface | Raw evaluator capture made only after the candidate process is dead. | | `AgentCandidateModelGrantClient` | interface | Narrow transport contract for a service that owns scoped model credentials | | `AgentCandidateOutputArtifactPort` | interface | Durable content-addressed evidence store controlled only by the evaluator. | -| `AgentCandidateProtectedModelCall` | interface | One evaluator-gateway call in the final, revoked model-access ledger. | | `AgentCandidateRepositoryPort` | interface | Resolves a declared GitHub repository to an already-present local Git object store. | +| `AgentCandidateTaskExecution` | interface | Runtime placement for one exact cell from a signed candidate experiment. | | `AgentCandidateWorkspacePort` | interface | Materializes an already-verified workspace archive. | | `BackendErrorDetail` | interface | Typed transport / backend failure detail. Carried on `backend_error` and | | `BuildAgentCandidateBundleInput` | interface | Complete measured surfaces and execution policy compiled into one candidate bundle. | @@ -183,7 +187,6 @@ Import from `@tangle-network/agent-runtime` — 342 exports. | `SqlAdapter` | interface | Minimal SQL driver shape. Implementations forward to whichever client the | | `ToolLoopAssistantToolCall` | interface | One OpenAI-shaped tool-call entry carried on an assistant message. | | `ToolLoopCall` | interface | Bounded turn-level tool-dispatch loop. | -| `VerifiedAgentCandidateTaskOutcome` | interface | Branded task outcome that has survived independent patch and tree verification. | | `VerifyResult` | interface | Outcome of verifying a candidate worktree. `feedback` (compiler errors, | | `AgentBackendKind` | type | The transport a chat backend runs on. | | `AgentCandidateBundleInput` | type | Exact candidate wire shape before the runtime computes its canonical digest. | @@ -196,6 +199,7 @@ Import from `@tangle-network/agent-runtime` — 342 exports. | `AgentCandidateExecutionStageResult` | type | Result of durably staging the one immutable terminal outbox entry. | | `AgentCandidateExecutionTerminalRecord` | type | Durable terminal record for one acquired execution attempt. | | `AgentCandidateExecutionTerminalResult` | type | Evaluator-owned terminal facts staged durably before the terminal CAS. | +| `AgentCandidateExecutorTaskOutcomeCapture` | type | Raw evaluator capture made only after the candidate process is dead. | | `AgentCandidateModelGrantReservation` | type | Secret-free response from the service's reservation endpoint. | | `AgentCandidateModelLimits` | type | Limits mechanically enforced by the evaluator-owned model gateway. | | `AgentCandidateProfileSource` | type | A complete profile that can be frozen without losing behavior. | @@ -213,13 +217,14 @@ Import from `@tangle-network/agent-runtime` — 342 exports. | `ToolCallOutcome` | type | Outcome of one tool dispatch — structurally compatible with a hub/integration | | `ToolLoopMessage` | type | A message in the running conversation the loop sends to `streamTurn`. | | `ToolLoopStopReason` | type | Why the loop stopped. `completed` = model finished naturally; `stuck-loop` = | +| `VerifiedAgentCandidateTaskOutcome` | type | Branded task outcome that has survived independent evaluator verification. | | `Verifier` | type | Verifies the edited worktree. Sync or async; throws only on a setup fault | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateBenchmarkGraderIdentity`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorProfileFile`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateTaskExecution`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentProfileDiffProposal`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveMemoryOptions`, `ImprovementDriverOptions`, `ImproveResult`, `ImproveSkillsOptions`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `ManagedImprovementDriver`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `ProfileDiffProposerOptions`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveOptions`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `ProfileDiffProposerContext`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `ApprovedKnowledgeImprovementCandidate`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveMemoryOptions`, `ImprovementDriverOptions`, `ImproveResult`, `ImproveSkillsOptions`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `ManagedImprovementDriver`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `ProfileDiffProposerOptions`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveOptions`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `ProfileDiffProposerContext`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. ### Vertical agent — manifest + improvement adapter -Import from `@tangle-network/agent-runtime/agent` — 48 exports. +Import from `@tangle-network/agent-runtime/agent` — 47 exports. | Symbol | Kind | Summary | |---|---|---| @@ -257,7 +262,7 @@ Import from `@tangle-network/agent-runtime/agent` — 48 exports. | `ValidateProfileMaterializationOptions` | interface | Input for checking a candidate diff against a run path. | | `AgentProfileMaterializationAxis` | type | AgentProfile axis name, with `custom:` reserved for caller-owned extensions. | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentRubric`, `AgentRunContext`, `AgentRunInvocation`, `AgentRuntime`, `AnalystConfig`, `AutoApplyPolicy`, `CreateSandboxActOptions`, `CreateSurfaceImprovementAdapterOpts`, `DraftPatchInput`, `DraftPatchOutput`, `JudgeConfig`, `OutcomeMeasurementOpts`, `ResolvedSurface`, `RubricDimension`, `KnownAgentProfileMaterializationAxis`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentRubric`, `AgentRunContext`, `AgentRunInvocation`, `AgentRuntime`, `AnalystConfig`, `CreateSandboxActOptions`, `CreateSurfaceImprovementAdapterOpts`, `DraftPatchInput`, `DraftPatchOutput`, `JudgeConfig`, `OutcomeMeasurementOpts`, `ResolvedSurface`, `RubricDimension`, `KnownAgentProfileMaterializationAxis`. ### Multi-turn conversations @@ -302,7 +307,7 @@ Import from `@tangle-network/agent-runtime/conversation` — 53 exports. ### Intelligence SDK — Observe + provable-OFF billing -Import from `@tangle-network/agent-runtime/intelligence` — 85 exports. +Import from `@tangle-network/agent-runtime/intelligence` — 102 exports. | Symbol | Kind | Summary | |---|---|---| @@ -310,26 +315,39 @@ Import from `@tangle-network/agent-runtime/intelligence` — 85 exports. | `composeCertifiedProfile` | function | Compose a certified profile into a uniform `ResolvedSurface`. Additive over | | `composeCertifiedProfileFromWire` | function | Lower a plane `CertifiedProfile` straight into a `ResolvedSurface` via | | `composeCertifiedPrompt` | function | Fold the certified prompt surface (and any certified prompt-folding artifacts: | -| `createAgentImprovementProposal` | function | Freeze an already-measured improvement into the one reviewable proposal | +| `createAgentImprovementActivation` | function | Authorize product-owned writes only after the exact candidate was measured and approved. | +| `createAgentImprovementMeasuredComparison` | function | Delegate all statistics and promotion checks to agent-eval's receipt-based comparison. | +| `createAgentImprovementProposal` | function | Create the reviewable record only from a complete, recomputable experiment result. | | `createCertifiedPromptSource` | function | Create the cached certified-prompt source — the ONE module-scope-cache + | | `createIntelligenceClient` | function | Create an Observe-mode Intelligence client. Resolves effort, the base URL, and | +| `createSandboxCandidateExperimentExecutor` | function | Execute one signed experiment cell inside a fresh Tangle sandbox. | | `defaultRedactor` | function | The built-in redactor. Walks objects and arrays; replaces values under | -| `executeApprovedAgentCandidate` | function | Verify, materialize, run, grade, and receipt only the exact approved bundle. | +| `executeAgentCandidateExperimentCell` | function | Execute one exact arm, task, repetition, seed, and attempt through Runtime. | | `isIntelligenceOff` | function | True when these settings admit NO intelligence spawn — the passthrough | | `manifestFromProfile` | function | Lower the EXISTING plane wire (`CertifiedProfile`) into a `CapabilityManifest`. | | `normalizeCertifiedProfile` | function | Deserialize the composed-endpoint response into a `CertifiedProfile`. The | -| `proposeAgentImprovement` | function | Analyze one run and produce one measured, review-only improvement proposal. | +| `parseAgentCandidateProfileActivation` | function | Parse and check every native file hash plus both canonical document digests. | +| `proposeAgentImprovement` | function | Analyze, search, then remeasure the resulting exact candidate before proposing it. | | `pullCertified` | function | Pull the certified composed profile for a target. Fail-closed: a network | | `resolveEffort` | function | Compile a named tier (plus optional per-field overrides) into the flat | | `resolveIntelligenceBaseUrl` | function | Resolve the ONE Intelligence base URL — the single knob both the send and | | `resolveRedactor` | function | Resolve the redactor a client uses. A caller-supplied hook replaces the | -| `reviewAgentImprovementProposal` | function | Persist an approve/reject/change-request decision bound to one exact proposal. | -| `verifyAgentImprovementProposal` | function | Validate a proposal's schema, profile, sealed bundle, and canonical digest. | -| `verifyAgentImprovementReview` | function | Validate a review's decision fields and canonical digest. | +| `reviewAgentImprovementProposal` | function | Persist a human or tenant-policy decision bound to one exact proposal. | +| `runAgentCandidateExperiment` | function | Execute both arms of one immutable experiment and derive its paired result. | +| `verifyAgentImprovementActivation` | function | Validate activation authority against the exact proposal, review, experiment, and base state. | +| `verifyAgentImprovementProposal` | function | Validate a proposal and recompute every binding to its measured experiment. | +| `verifyAgentImprovementReview` | function | Validate the canonical identity and wire shape of an improvement review. | +| `verifyCandidateExecutionEvidence` | function | Recheck one Runtime receipt against its exact signed experiment cell. | | `withIntelligence` | function | Wrap an agent so it (a) RECEIVES the tenant's certified profile — the prompt | | `defaultEffortTier` | const | The default tier when a client declares no effort. `'standard'` turns | +| `sandboxCandidateExperimentExecutionSupport` | const | Declares the exact candidate surfaces the sandbox executor can run. | +| `AgentCandidateExperimentCellExecutionError` | class | A failed baseline or candidate cell with its complete Runtime failure result. | | `CapabilityNotAdmittedError` | class | A binding kind whose resolver case is typed but not yet admitted (rag-index, | +| `AgentCandidateProfileActivation` | interface | Exact native profile files and the canonical plan that activated them. | +| `AgentImprovementMeasuredComparison` | interface | Portable paired held-out comparison produced by an evaluation package. | +| `AgentImprovementReview` | interface | Human or tenant-policy decision bound to one exact proposal. | | `AppliedIntelligence` | interface | What the hook hands the agent each run. Additive over the prompt-only | +| `CandidateExecutionEvidence` | interface | Complete execution of one exact experiment attempt. | | `CapabilityManifest` | interface | The strict generalization of `CertifiedProfile`. `promptSurface` is kept | | `CertifiedArtifact` | interface | A promoted, certified artifact (one entry in the composed profile). | | `CertifiedCapability` | interface | One certified unit of agent power. | @@ -380,7 +398,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 85 exports. | `Redactor` | type | A redactor maps an arbitrary trace value to a safe-to-export value. Pure; | | `UsageClass` | type | Usage class for billing. Base-stream tokens bill `'inference'`; every | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentImprovementProposal`, `AgentImprovementReview`, `CandidateExecutionEvidence`, `CreateAgentImprovementProposalOptions`, `ExecuteApprovedAgentCandidateOptions`, `ExecuteApprovedAgentCandidateResult`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `AgentImprovementEvaluation`, `AgentImprovementReviewDecision`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateExperimentCellPlacement`, `AgentImprovementProposal`, `CreateAgentImprovementActivationOptions`, `CreateAgentImprovementProposalOptions`, `CreateSandboxCandidateExperimentExecutorOptions`, `ExecuteAgentCandidateExperimentCellOptions`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `RunAgentCandidateExperimentOptions`, `RunAgentCandidateExperimentResult`, `SandboxCandidateExperimentExecution`, `SandboxCandidateExperimentExecutor`, `VerifyCandidateExecutionEvidenceOptions`, `AgentImprovementReviewDecision`. ### Recursive atom + loop kernel (alias of ./runtime) @@ -839,7 +857,7 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 59 exports. ### Knowledge orchestration — supervised KB updates -Import from `@tangle-network/agent-runtime/knowledge` — 18 exports. +Import from `@tangle-network/agent-runtime/knowledge` — 19 exports. | Symbol | Kind | Summary | |---|---|---| @@ -847,11 +865,11 @@ Import from `@tangle-network/agent-runtime/knowledge` — 18 exports. | `createSupervisedKnowledgeUpdater` | function | Create an `improveKnowledgeBase` update callback backed by runtime supervision. | | `formatSupervisedKnowledgeTask` | function | Format the supervisor task with the KB root, readiness requirements, current findings, and metadata. | | `knowledgeReadinessDeliverable` | function | Build the completion check a supervised KB update uses to stop only when the KB is ready. | -| `runKnowledgeImprovementJob` | function | Run the full KB improvement job: candidate workspace, runtime supervisor update, readiness check, and promotion. | +| `runKnowledgeImprovementJob` | function | Produce a frozen KB candidate, and promote it only when an exact signed review is supplied. | | `runSupervisedKnowledgeUpdate` | function | Run a runtime supervisor that updates one candidate knowledge base and stops on readiness. | | `RESEARCH_SUPERVISOR_SYSTEM_PROMPT` | const | Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentKnowledgeReadinessCheckOptions`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `RunKnowledgeImprovementJobOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `SupervisedKnowledgeUpdater`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentKnowledgeReadinessCheckOptions`, `ApprovedKnowledgeImprovementCandidate`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `RunKnowledgeImprovementJobOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `SupervisedKnowledgeUpdater`. ### Built-in agent profiles @@ -936,13 +954,14 @@ Import from `@tangle-network/agent-runtime/primeintellect` — 27 exports. ### Candidate execution — immutable prepare, run, grade, and receipt -Import from `@tangle-network/agent-runtime/candidate-execution` — 96 exports. +Import from `@tangle-network/agent-runtime/candidate-execution` — 99 exports. | Symbol | Kind | Summary | |---|---|---| | `applyExactAgentProfileDiff` | function | Apply one exact diff and reject any value that cannot be preserved canonically. | | `buildAgentCandidateBundle` | function | Compile one measured profile/code candidate into the immutable execution | | `candidateExecutionClaim` | function | Extract the complete durable claim from a prepared execution. | +| `candidateKnowledgeExecutionPaths` | function | Deterministic, signed locations used by every candidate executor. | | `captureAgentCandidateWorkspace` | function | Capture one exact regular-file workspace for immutable candidate execution. | | `captureAgentCandidateWorkspaceFiles` | function | Capture detached files returned by a remote executor into the standard archive. | | `createAgentCandidateWorkspacePort` | function | Create the standard bounded materializer for candidate execution ports. | @@ -956,11 +975,15 @@ Import from `@tangle-network/agent-runtime/candidate-execution` — 96 exports. | `recoverExpiredAgentCandidateExecution` | function | Close an expired crashed attempt from persisted non-secret handles, then record failure. | | `sealAgentCandidateBundle` | function | Validate and content-address a candidate bundle before it crosses an approval boundary. | | `verifyAgentCandidateBundle` | function | Verifies every digest, resource, workspace, and Git object in a candidate bundle. | +| `AGENT_CANDIDATE_EXECUTION_SUPPORT` | const | Surfaces admitted by Runtime's verifier before an environment adapter is selected. | +| `CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV` | const | Environment variable containing the materialized retrieval configuration path. | +| `CANDIDATE_KNOWLEDGE_ROOT_ENV` | const | Environment variable containing the materialized candidate knowledge root. | | `CANDIDATE_TRACE_ENV` | const | Environment keys used to propagate immutable candidate trace identity. | | `CANDIDATE_TRACE_TAGS` | const | Protected trace tags that bind a run to one prepared candidate execution. | | `FileAgentCandidateExecutionClaimStore` | class | Cross-process lifecycle implemented as fsynced, create-if-absent records. | | `InMemoryAgentCandidateExecutionClaimStore` | class | Single-process lifecycle implementation. | | `AgentCandidateArtifactPort` | interface | Reads one content-addressed object from the closed S3/IPFS locator set. | +| `AgentCandidateBenchmarkGraderIdentity` | interface | Immutable grader identity admitted for one benchmark task. | | `AgentCandidateBenchmarkGraderPort` | interface | Evaluator-owned executable grader, pinned by immutable implementation bytes. | | `AgentCandidateCodeSurfaceSource` | interface | The only accepted path from an agent-eval code candidate to executable bytes. | | `AgentCandidateExecutionAttemptRecord` | interface | Persisted state available to a fresh trusted recovery worker after a crash. | @@ -969,21 +992,19 @@ Import from `@tangle-network/agent-runtime/candidate-execution` — 96 exports. | `AgentCandidateExecutionCleanupHandles` | interface | Non-secret identities a trusted recovery worker needs to close an abandoned attempt. | | `AgentCandidateExecutionLease` | interface | Secret capability required to finish the acquired attempt. | | `AgentCandidateExecutionRecoveryEvidence` | interface | Trusted, independently observed closure facts for one expired winning lease. | -| `AgentCandidateExecutionUsage` | interface | Exact fixed-point usage proven by the closed evaluator model ledger. | -| `AgentCandidateExecutorFinalCapture` | interface | Idempotent executor result after process death and trace drain. | +| `AgentCandidateExecutorFinalCapture` | interface | Replayable evaluator result captured only after process death and trace drain. | | `AgentCandidateExecutorMemoryCapture` | interface | Raw isolated-memory capture made only after access has been revoked. | | `AgentCandidateExecutorPort` | interface | Executes one prepared request inside an evaluator-owned isolation boundary. | +| `AgentCandidateExecutorProfileFile` | interface | One exact profile file supplied to an evaluator-owned executor. | | `AgentCandidateExecutorRequest` | interface | One detached request passed to the trusted environment-specific executor. | | `AgentCandidateExecutorStopRequest` | interface | Opaque process identity used for termination without re-exposing launch credentials. | -| `AgentCandidateExecutorTaskOutcomeCapture` | interface | Raw evaluator capture made only after the candidate process is dead. | | `AgentCandidateModelGrantClient` | interface | Narrow transport contract for a service that owns scoped model credentials | | `AgentCandidateOutputArtifactPort` | interface | Durable content-addressed evidence store controlled only by the evaluator. | -| `AgentCandidateProtectedModelCall` | interface | One evaluator-gateway call in the final, revoked model-access ledger. | | `AgentCandidateRepositoryPort` | interface | Resolves a declared GitHub repository to an already-present local Git object store. | +| `AgentCandidateTaskExecution` | interface | Runtime placement for one exact cell from a signed candidate experiment. | | `AgentCandidateWorkspacePort` | interface | Materializes an already-verified workspace archive. | | `BuildAgentCandidateBundleInput` | interface | Complete measured surfaces and execution policy compiled into one candidate bundle. | | `PreparedAgentCandidateKnowledge` | interface | Exact file-backed knowledge admitted by the candidate bundle. | -| `VerifiedAgentCandidateTaskOutcome` | interface | Branded task outcome that has survived independent patch and tree verification. | | `AgentCandidateBundleInput` | type | Exact candidate wire shape before the runtime computes its canonical digest. | | `AgentCandidateCodeSource` | type | Explicit control/no-op code or one finalized CodeSurface whose bytes must still verify. | | `AgentCandidateExecutionClaimResult` | type | Result of atomically claiming one execution attempt. | @@ -994,11 +1015,13 @@ Import from `@tangle-network/agent-runtime/candidate-execution` — 96 exports. | `AgentCandidateExecutionStageResult` | type | Result of durably staging the one immutable terminal outbox entry. | | `AgentCandidateExecutionTerminalRecord` | type | Durable terminal record for one acquired execution attempt. | | `AgentCandidateExecutionTerminalResult` | type | Evaluator-owned terminal facts staged durably before the terminal CAS. | +| `AgentCandidateExecutorTaskOutcomeCapture` | type | Raw evaluator capture made only after the candidate process is dead. | | `AgentCandidateModelGrantReservation` | type | Secret-free response from the service's reservation endpoint. | | `AgentCandidateModelLimits` | type | Limits mechanically enforced by the evaluator-owned model gateway. | | `AgentCandidateProfileSource` | type | A complete profile that can be frozen without losing behavior. | +| `VerifiedAgentCandidateTaskOutcome` | type | Branded task outcome that has survived independent evaluator verification. | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateBenchmarkGraderIdentity`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorProfileFile`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateTaskExecution`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `CreateAgentCandidateWorkspacePortOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `DisposePreparedAgentCandidateOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResolvedAgentCandidateContainer`, `VerifiedAgentCandidate`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `CreateAgentCandidateWorkspacePortOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `DisposePreparedAgentCandidateOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResolvedAgentCandidateContainer`, `VerifiedAgentCandidate`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`. ### MCP servers — delegate / coordination / detached-session @@ -1093,7 +1116,7 @@ Import from `@tangle-network/agent-runtime/mcp` — 176 exports. | `DetachedSessionRefParts` | interface | Decoded `DelegationRecord.detachedSessionRef`. `sandboxId` is absent between | | `DriveTurnCapableBox` | interface | The box surface detached turns need. `SandboxInstance` | | `FleetHandle` | interface | Minimal `SandboxFleet` surface the fleet executor calls. Declared | -| `ResearchOutputShape` | interface | Loose shape of a research output over the wire — the substrate cannot | +| `ResearchOutputShape` | interface | Provider-neutral research output carried over the MCP boundary. The MCP | | `SettledWorker` | interface | A worker the driver has drained via `await_event`. | | `TraceContext` | interface | Trace context propagation for MCP subprocess. | | `UiAuditorDelegationOutput` | interface | Wire-shape of a completed UI-audit delegation. The `findings` array | @@ -1230,7 +1253,7 @@ Import from `@tangle-network/agent-eval` — 51 exports. ### CAMPAIGN — profile matrix, gates, improvement loop -Import from `@tangle-network/agent-eval/campaign` — 391 exports. +Import from `@tangle-network/agent-eval/campaign` — 392 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1252,6 +1275,7 @@ Import from `@tangle-network/agent-eval/campaign` — 391 exports. | `campaignScenarioIdentity` | function | Redacted but independently verifiable identity of one complete scenario. | | `campaignSplitDigest` | function | Canonical identity of the exact scenario payloads and replicate count. | | `campaignSplitDigestFromIdentities` | function | Canonical split identity reconstructed from redacted scenario identities. | +| `canonicalDigest` | function | _(no summary — add a TSDoc line at the declaration)_ | | `classifyUngroundedLiterals` | function | Scan revised artifact text for single-quoted single-word literals (the | | `codeSurfaceIdentityMaterial` | function | Canonical, location-independent identity of a finalized code candidate. | | `compareProposers` | function | Run a head-to-head lift benchmark across surface proposers on a shared holdout, returning per-proposer lift CIs and pairwise "who wins" verdicts. | diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 3fb6acc5..f17b9f53 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -2,7 +2,7 @@ -> **Version 0.94.13.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.120.1 <0.121.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.26.1 <0.27.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. +> **Version 0.95.0.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.122.1 <0.123.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.11.1 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.30.0 <0.31.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. > > **`./loops` is the runtime barrel** — `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. > @@ -110,10 +110,11 @@ A general "loop" primitive is the single most common modelling error in this rep | Have a **supervisor spawn + live-drive workers in a backend you choose** and observe or steer them while the coordinator is alive | the **coordination MCP** via `createCoordinationTools` / `serveCoordinationMcp` over a live `Scope`; each worker's leaf is `createExecutor({ backend })` | `detachedSessionDelegate`, which is own-sandbox-session only and one-shot. Supervised-tree restart recovery is not implemented. | | Stand up a vertical agent in the eval loop | `defineAgent(manifest)` + `createSurfaceImprovementAdapter` — `/agent` | a per-vertical manifest parser, surface-validator, or bespoke `ImprovementAdapter` | | Observe + deliver Intelligence on a live agent (send RunRecords + receive certified profile/diffs) | `withIntelligence(agent, { project, target })` — `/intelligence` (proposals surfaced, never auto-applied; `effort: 'off'` proves inference-only billing) | a custom trace-wrapper, a second receive path, or hand-rolled effort/tier config | -| Turn trace evidence into one measured, review-only agent proposal | `proposeAgentImprovement({ analysis, profile, improvement })` — `/intelligence` | manually joining analysts, optimizers, evidence, uncertainty, and candidate identity | +| Turn trace evidence into one measured, review-only agent proposal | `proposeAgentImprovement({ analysis, profile, improvement, buildExperiment, placeCell })` in `/intelligence` | manually joining analysis, optimization, exact candidate execution, uncertainty, and candidate identity | | Freeze a measured profile/diff plus a content-addressed code surface into one executable candidate | `buildAgentCandidateBundle(...)`, then `verifyAgentCandidateBundle(...)` at execution — `/candidate-execution` | a product callback that converts profile fields, reproduces Git diff flags, hashes bytes, or assembles `AgentCandidateBundle` by hand | | Record approve/reject/change-request feedback against one exact proposal | `reviewAgentImprovementProposal(proposal, review)` — `/intelligence` | a mutable status row that is not bound to candidate bytes | -| Execute and grade only an authenticated approved candidate | `executeApprovedAgentCandidate(options)` — `/intelligence`; low-level ports live at `/candidate-execution` | product-local claim/retry/isolation/receipt orchestration | +| Run and grade the exact signed baseline-versus-candidate matrix before review | `runAgentCandidateExperiment({ experiment, placeCell })` in `/intelligence`; use `createSandboxCandidateExperimentExecutor(...)` for fresh Tangle sandboxes | product-local pairing, retry, isolation, receipt, and comparison code | +| Authorize product writes only for the exact measured and approved candidate | `createAgentImprovementActivation(proposal, review, options)` and `verifyAgentImprovementActivation(...)` in `/intelligence` | a mutable approval flag that does not bind the experiment, candidate bytes, targets, or current base state | | Capture and restore exact task, candidate, or memory workspace bytes | `captureAgentCandidateWorkspace(...)` + `createAgentCandidateWorkspacePort(...)` — `/candidate-execution` | a product-specific archive format, ambient `git checkout`, or a materializer that skips byte/path/mode verification | | Fold **certified prompt additions into a system prompt you assemble yourself** (product chat routes) | `createCertifiedPromptSource({ target })` → `source.compose(base)` — `/intelligence` (cached, coalesced, fail-closed; `withIntelligence` rides the same source) | a module-scope cache + refresh-window + keep-last-known loop around `pullCertified` in product wiring | | Improve a KB with runtime agents, candidate workspaces, readiness checks, and measured supervised spend | `runKnowledgeImprovementJob(options)` from `/knowledge` | hand-wiring `improveKnowledgeBase` + a supervised updater + a readiness callback in every product | diff --git a/examples/coding-benchmark/dispatch.ts b/examples/coding-benchmark/dispatch.ts index c6db033d..d07135bf 100644 --- a/examples/coding-benchmark/dispatch.ts +++ b/examples/coding-benchmark/dispatch.ts @@ -40,13 +40,7 @@ import { sumSandboxUsage, } from '@tangle-network/agent-runtime/loops' import type { SandboxEvent } from '@tangle-network/sandbox' -import { - type CheckBox, - gradeOnHiddenCriteria, - layerOutput, - type RunArtifact, - runChecks, -} from './eval' +import { gradeOnHiddenCriteria, layerOutput, type RunArtifact, runChecks } from './eval' import { harnessOf, type ToolPreset, withTools } from './profiles' import { type CodingScenario, checkCmds, routeCodingFields } from './scenarios' @@ -162,7 +156,7 @@ export function codingDispatch( // these) steer the next round — the firewall keeps the held-out suite + rubric // out of the loop. `run.box` is a `SandboxInstance`; `CheckBox` is the minimal // `exec`(+optional `fs.write`) subset the checks use — a structural narrowing. - checks = await runChecks(run.box as CheckBox, scenario, cmds) + checks = await runChecks(run.box, scenario, cmds) if (checks.allPass) break // stop on worker-observable green only } @@ -174,7 +168,7 @@ export function codingDispatch( // composite). A solution that hardcoded the visible examples fails the held-out inputs // it never saw — execution truth behind a substrate-enforced firewall, not a regex. const heldout = await gradeOnHiddenCriteria( - run.box as CheckBox, + run.box, scenario, cmds.heldout, { fields: routedFields, agentContext: agentContextParts.join('\n') }, diff --git a/examples/coding-benchmark/eval.ts b/examples/coding-benchmark/eval.ts index 65c7866a..d5aa8e0c 100644 --- a/examples/coding-benchmark/eval.ts +++ b/examples/coding-benchmark/eval.ts @@ -94,7 +94,7 @@ export const heldoutPassRateOf = (artifact: RunArtifact): number => artifact.hel * seeding never interpolates a path into a command string. */ export interface CheckBox { exec(command: string): Promise<{ exitCode: number; stdout: string; stderr: string }> - fs?: { write(path: string, content: string): Promise } + fs?: { write(path: string, content: string): Promise } } /** Seed a test file into the box. Prefers the structured `fs.write` seam so the path/ diff --git a/package.json b/package.json index 4a723def..28a1cec8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.94.13", + "version": "0.95.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -100,13 +100,15 @@ "scripts": { "build": "tsup", "dev": "tsup --watch", - "prepare": "(git rev-parse --git-dir > /dev/null 2>&1 && git config core.hooksPath .githooks) || true; tsup", + "prepublishOnly": "pnpm run build", "test": "vitest run", "test:watch": "vitest", "lint": "biome check src tests examples", "lint:fix": "biome check --write src tests examples", "typecheck": "tsc --noEmit && pnpm run typecheck:examples", "typecheck:examples": "tsc --noEmit -p tsconfig.examples.json", + "verify:bench": "pnpm --filter @tangle-network/agent-bench run typecheck:public && pnpm --filter @tangle-network/agent-bench test && pnpm --filter @tangle-network/agent-bench run verify:package:local-runtime", + "verify:bench:published": "pnpm --filter @tangle-network/agent-bench run typecheck:public && pnpm --filter @tangle-network/agent-bench test && pnpm --filter @tangle-network/agent-bench run verify:package", "verify:package": "node scripts/verify-package-exports.mjs", "verify:primeintellect": "pnpm build && node scripts/verify-primeintellect-v1.mjs", "verify:primeintellect:live": "pnpm build && node scripts/verify-primeintellect-live.mjs", @@ -116,9 +118,9 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.15", - "@tangle-network/agent-eval": "0.120.1", - "@tangle-network/agent-interface": "0.26.1", - "@tangle-network/sandbox": "^0.9.7", + "@tangle-network/agent-eval": "0.122.1", + "@tangle-network/agent-interface": "0.30.0", + "@tangle-network/sandbox": "^0.11.1", "@types/node": "^25.9.3", "@types/tar-stream": "3.1.4", "playwright": "^1.61.0", @@ -134,6 +136,7 @@ "minimumReleaseAgeExclude": [ "@tangle-network/agent-eval", "@tangle-network/agent-interface", + "@tangle-network/agent-knowledge", "@tangle-network/agent-profile-materialize", "@tangle-network/sandbox" ], @@ -147,9 +150,9 @@ "license": "MIT", "packageManager": "pnpm@10.28.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.120.1 <0.121.0", - "@tangle-network/agent-interface": ">=0.26.1 <0.27.0", - "@tangle-network/sandbox": ">=0.8.0 <1.0.0", + "@tangle-network/agent-eval": ">=0.122.1 <0.123.0", + "@tangle-network/agent-interface": ">=0.30.0 <0.31.0", + "@tangle-network/sandbox": ">=0.11.1 <1.0.0", "playwright": "^1.40.0" }, "peerDependenciesMeta": { @@ -161,8 +164,8 @@ } }, "dependencies": { - "@tangle-network/agent-knowledge": "^1.12.1", - "@tangle-network/agent-profile-materialize": "0.3.2", + "@tangle-network/agent-knowledge": "^3.0.1", + "@tangle-network/agent-profile-materialize": "0.5.1", "tar-stream": "3.2.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d1de035a..b9e4027f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,11 +9,11 @@ importers: .: dependencies: '@tangle-network/agent-knowledge': - specifier: ^1.12.1 - version: 1.12.1(typescript@5.9.3) + specifier: ^3.0.1 + version: 3.0.1(typescript@5.9.3) '@tangle-network/agent-profile-materialize': - specifier: 0.3.2 - version: 0.3.2 + specifier: 0.5.1 + version: 0.5.1 tar-stream: specifier: 3.2.0 version: 3.2.0 @@ -22,14 +22,14 @@ importers: specifier: ^2.4.15 version: 2.4.15 '@tangle-network/agent-eval': - specifier: 0.120.1 - version: 0.120.1(typescript@5.9.3) + specifier: 0.122.1 + version: 0.122.1(typescript@5.9.3) '@tangle-network/agent-interface': - specifier: 0.26.1 - version: 0.26.1 + specifier: 0.30.0 + version: 0.30.0 '@tangle-network/sandbox': - specifier: ^0.9.7 - version: 0.9.7(viem@2.54.6(typescript@5.9.3)(zod@4.4.3)) + specifier: ^0.11.1 + version: 0.11.1(viem@2.54.6(typescript@5.9.3)(zod@4.4.3)) '@types/node': specifier: ^25.9.3 version: 25.9.3 @@ -58,6 +58,31 @@ importers: specifier: ^3.0.0 version: 3.2.4(@types/node@25.9.3)(tsx@4.22.4)(yaml@2.9.0) + bench: + dependencies: + '@tangle-network/agent-eval': + specifier: 0.122.1 + version: 0.122.1(typescript@5.9.3) + '@tangle-network/agent-interface': + specifier: 0.30.0 + version: 0.30.0 + '@tangle-network/agent-runtime': + specifier: workspace:* + version: link:.. + '@tangle-network/sandbox': + specifier: ^0.11.1 + version: 0.11.1(viem@2.54.6(typescript@5.9.3)(zod@4.4.3)) + devDependencies: + '@types/node': + specifier: ^25.9.3 + version: 25.9.3 + tsx: + specifier: ^4.22.4 + version: 4.22.4 + typescript: + specifier: ^5.7.0 + version: 5.9.3 + packages: '@adraffy/ens-normalize@1.11.1': @@ -68,15 +93,6 @@ packages: peerDependencies: zod: ^4.0.0 - '@ax-llm/ax@19.0.45': - resolution: {integrity: sha512-zC/OqUutTV0omOuAxBgeCihmxP3e1eUcbA78FCaMF3pPXZ3/FgQDZf0R04LH/lVIxm5BH4/qDdNNYcIVGwii8Q==} - hasBin: true - peerDependencies: - zod: ^3.24.0 || ^4.0.0 - peerDependenciesMeta: - zod: - optional: true - '@ax-llm/ax@23.0.1': resolution: {integrity: sha512-9glLBupY1XthY/WQGaUa3fK5BYYKTqc07Dxf4ZQmt0PDvX+s0XGyiua1jNnJGUlUeROyKN8k5IuVM84VXgS6Hw==} hasBin: true @@ -658,16 +674,11 @@ packages: '@tangle-network/agent-core@0.3.8': resolution: {integrity: sha512-bZfVpdiFjXbcQwSxSQABdtXEmUuapWChCcXrHkP6fAnwqEy9hWEfeZLlMZUy+1XaOfpTaMLUOEAwYhXfN018wQ==} - '@tangle-network/agent-core@0.4.15': - resolution: {integrity: sha512-sSYFh1N9jqRlDX/s2Dudv70ZK/Pg6naHLTAgzujPOgc7QvWLTwaKUxTcJqlYPA5/1JRH8vSKL+m2gQcy3J751g==} + '@tangle-network/agent-core@0.4.11': + resolution: {integrity: sha512-5B1IjrJ8xDR7w8Hv/MSk2ixul6NEJQ5Ftzo+z6l7ipeFpc0yaf1+Kkml4HDUEmGW/rqdrNpBf9/MpT7i/SY0PA==} - '@tangle-network/agent-eval@0.115.1': - resolution: {integrity: sha512-Yd487u8v0c8/+On9kaYGASS88oLhOJ5p0zwRMGdbWRSWC1UODbIgk1Fqdyq6EI60v+lZ+yDg/7LHkIrNi4kjPA==} - engines: {node: '>=20'} - hasBin: true - - '@tangle-network/agent-eval@0.120.1': - resolution: {integrity: sha512-XprtRhV6ncw5TPPJSJhSzfE2CoKALjUmKgSYiGtrl9bwvNN+uAPCraKv0tG9s77Ze3qKSePZaOu47gWFC4Pv3g==} + '@tangle-network/agent-eval@0.122.1': + resolution: {integrity: sha512-nBjoslpjtxDxBTeBKgxgSHb3CJH+nKmXX8u5X8fepB9OllmaqhnF4B/DoNp95b/KaezVnd4i2hT2Zip/WS2Kxg==} engines: {node: '>=20'} hasBin: true @@ -677,25 +688,39 @@ packages: '@tangle-network/agent-interface@0.17.1': resolution: {integrity: sha512-B7dRJTo0HSUtgBCB1VMwkTFYkLUaRr/4BcRglrQuGhGUwOzKv1RYyMejOVh5M3a5AagY9N79f7GYbjcA3UmnIA==} - '@tangle-network/agent-interface@0.22.0': - resolution: {integrity: sha512-7fsJhNdvTmOB1X9E2owl06jzyrqaN+jMkOPVKbK7dvNqQvf627PowCtL/edhUqEEv+K0FWtkwG3R3xqjX7VNZg==} + '@tangle-network/agent-interface@0.26.0': + resolution: {integrity: sha512-/z4HavFr/9AbaHxi/13bFP9iSUt8oitZbw4wS9g9KAt2TeN2ec5/A2C106udWkKIETZLnu465TfU+K4KDVNkKw==} - '@tangle-network/agent-interface@0.25.0': - resolution: {integrity: sha512-bMKjyjk8bk2651HleiTEUOTPgNE2bWxCEl9VBVUyub/UkP87Y09byXOxeDEsNGc4TIT19Y5Uo0b+k1kRWXkMVQ==} + '@tangle-network/agent-interface@0.30.0': + resolution: {integrity: sha512-GmwPwamrzOFPn+Qx7W0nt9QRUF9VUiZrMNLmF6QpL23R+7TD21HllPblVOYB5MVqA728FpDLXit3HfqY8mStwg==} - '@tangle-network/agent-interface@0.26.1': - resolution: {integrity: sha512-NnQIWtudBHt1P6UXeLMAXUUUxjbYlMLuO3x7yGUGuGWkGcJ0EK0Gpv6aI81g+uqpsV1aHQ4gYfDTCVpdNICUeA==} - - '@tangle-network/agent-interface@0.27.2': - resolution: {integrity: sha512-GzJJ3AO6S22Sf4hd5lGoQK7ft2hb6IvbcIiZ91vUfwlPNWO4oWK7eFVhKYE2YqHi/Tad2X+uE+Mvyhdfx/jH1w==} - - '@tangle-network/agent-knowledge@1.12.1': - resolution: {integrity: sha512-j5riyIvz5C+ZBELTtNVtmUbcKAMpgTHMNmDQ12MhOIPJv5Y7AeX2oCWuF6c8aPqg1hOrsLqgqXSr1zkV0ePsrA==} - engines: {node: '>=20'} + '@tangle-network/agent-knowledge@3.0.1': + resolution: {integrity: sha512-bzlllxFx+ZNwdx+Zr1v/ED2qElttrTTqFbs/3FapaHp+p1t5Jhr2seoyLMhiSUJqPusm8d8VvSctlL4EbpOaoA==} + engines: {node: '>=20.19.0'} hasBin: true - '@tangle-network/agent-profile-materialize@0.3.2': - resolution: {integrity: sha512-jCj1Hc/brPQ5eS7or9A+Klv0FAZcGtyFr7kx2cKfk+wCj0/4Di3k1pMjayrmRNgI/7Oc47gt6/t+qvdtZxBkAw==} + '@tangle-network/agent-profile-materialize@0.5.1': + resolution: {integrity: sha512-gU4J6mglvy9bpBAnErOruO1goF1cSzGGd06Oo5CFSj7RjR1SXWDFMYQtVLSd/O68ldCVAMVIMq5b4rwboMBbcw==} + + '@tangle-network/sandbox@0.11.1': + resolution: {integrity: sha512-9I5G1UHNhwTNFToe+Y0iQO1Joemd+7ba3LXIjfYUj+apRww6Eqq+1UGV2a11XaWUTlEqyhAc37F5OHBUj/xxCg==} + peerDependencies: + '@mastra/core': ^1.36.0 + '@modelcontextprotocol/sdk': ^1.29.0 + ai: ^6.0.175 + openai: ^6.36.0 + viem: ^2.0.0 + peerDependenciesMeta: + '@mastra/core': + optional: true + '@modelcontextprotocol/sdk': + optional: true + ai: + optional: true + openai: + optional: true + viem: + optional: true '@tangle-network/sandbox@0.9.7': resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} @@ -892,9 +917,6 @@ packages: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} - dayjs@1.11.21: - resolution: {integrity: sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==} - debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -963,6 +985,9 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + hono@4.12.30: resolution: {integrity: sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==} engines: {node: '>=16.9.0'} @@ -1095,6 +1120,9 @@ packages: resolution: {integrity: sha512-qif0+jGGZoLWdHey3UFHHWP0H7Gbmsk8T5VEqyYFbWqPr1XqvLGBbk/sl8V5exGmcYJklJOhOQq1pV9IcsiFag==} engines: {node: ^10 || ^12 || >=14} + proper-lockfile@4.1.2: + resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + punycode.js@2.3.1: resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==} engines: {node: '>=6'} @@ -1107,6 +1135,10 @@ packages: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} + retry@0.12.0: + resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} + engines: {node: '>= 4'} + rollup@4.60.2: resolution: {integrity: sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -1115,6 +1147,9 @@ packages: siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -1351,13 +1386,6 @@ snapshots: openapi3-ts: 4.6.0 zod: 4.4.3 - '@ax-llm/ax@19.0.45(zod@4.4.3)': - dependencies: - '@opentelemetry/api': 1.9.1 - dayjs: 1.11.21 - optionalDependencies: - zod: 4.4.3 - '@ax-llm/ax@23.0.1(zod@4.4.3)': dependencies: '@opentelemetry/api': 1.9.1 @@ -1723,36 +1751,18 @@ snapshots: '@tangle-network/agent-interface': 0.17.1 zod: 4.4.3 - '@tangle-network/agent-core@0.4.15': + '@tangle-network/agent-core@0.4.11': dependencies: - '@tangle-network/agent-interface': 0.27.2 + '@tangle-network/agent-interface': 0.26.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.115.1(typescript@5.9.3)': - dependencies: - '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) - '@ax-llm/ax': 19.0.45(zod@4.4.3) - '@hono/node-server': 2.0.8(hono@4.12.30) - '@tangle-network/agent-interface': 0.22.0 - '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) - hono: 4.12.30 - zod: 4.4.3 - transitivePeerDependencies: - - '@mastra/core' - - '@modelcontextprotocol/sdk' - - ai - - bufferutil - - openai - - typescript - - utf-8-validate - - '@tangle-network/agent-eval@0.120.1(typescript@5.9.3)': + '@tangle-network/agent-eval@0.122.1(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 23.0.1(zod@4.4.3) '@hono/node-server': 2.0.8(hono@4.12.30) - '@tangle-network/agent-core': 0.4.15 - '@tangle-network/agent-interface': 0.26.1 + '@tangle-network/agent-core': 0.4.11 + '@tangle-network/agent-interface': 0.30.0 '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) hono: 4.12.30 zod: 4.4.3 @@ -1773,27 +1783,21 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.22.0': - dependencies: - zod: 4.4.3 - - '@tangle-network/agent-interface@0.25.0': - dependencies: - zod: 4.4.3 - - '@tangle-network/agent-interface@0.26.1': + '@tangle-network/agent-interface@0.26.0': dependencies: '@noble/hashes': 1.8.0 zod: 4.4.3 - '@tangle-network/agent-interface@0.27.2': + '@tangle-network/agent-interface@0.30.0': dependencies: '@noble/hashes': 1.8.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@1.12.1(typescript@5.9.3)': + '@tangle-network/agent-knowledge@3.0.1(typescript@5.9.3)': dependencies: - '@tangle-network/agent-eval': 0.115.1(typescript@5.9.3) + '@tangle-network/agent-eval': 0.122.1(typescript@5.9.3) + '@tangle-network/agent-interface': 0.30.0 + proper-lockfile: 4.1.2 zod: 4.4.3 transitivePeerDependencies: - '@mastra/core' @@ -1804,9 +1808,16 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-profile-materialize@0.3.2': + '@tangle-network/agent-profile-materialize@0.5.1': dependencies: - '@tangle-network/agent-interface': 0.25.0 + '@tangle-network/agent-interface': 0.30.0 + + '@tangle-network/sandbox@0.11.1(viem@2.54.6(typescript@5.9.3)(zod@4.4.3))': + dependencies: + '@tangle-network/agent-core': 0.4.11 + '@tangle-network/agent-interface': 0.30.0 + optionalDependencies: + viem: 2.54.6(typescript@5.9.3)(zod@4.4.3) '@tangle-network/sandbox@0.9.7(viem@2.54.6(typescript@5.9.3)(zod@4.4.3))': dependencies: @@ -1979,8 +1990,6 @@ snapshots: consola@3.4.2: {} - dayjs@1.11.21: {} - debug@4.4.3: dependencies: ms: 2.1.3 @@ -2081,6 +2090,8 @@ snapshots: fsevents@2.3.3: optional: true + graceful-fs@4.2.11: {} + hono@4.12.30: {} isows@1.0.7(ws@8.21.0): @@ -2200,12 +2211,20 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + proper-lockfile@4.1.2: + dependencies: + graceful-fs: 4.2.11 + retry: 0.12.0 + signal-exit: 3.0.7 + punycode.js@2.3.1: {} readdirp@4.1.2: {} resolve-from@5.0.0: {} + retry@0.12.0: {} + rollup@4.60.2: dependencies: '@types/estree': 1.0.8 @@ -2239,6 +2258,8 @@ snapshots: siginfo@2.0.0: {} + signal-exit@3.0.7: {} + source-map-js@1.2.1: {} source-map@0.7.6: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 00000000..49250f39 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +packages: + - bench diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index b761fa21..91efd964 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -5,7 +5,7 @@ import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') -const tempRoot = mkdtempSync(join(tmpdir(), 'agent-runtime-package-exports-')) +const tempRoot = mkdtempSync(join(tmpdir(), 'agent-runtime-package-')) try { const packDir = join(tempRoot, 'pack') @@ -14,18 +14,6 @@ try { mkdirSync(packDir, { recursive: true }) mkdirSync(unpackDir, { recursive: true }) mkdirSync(appDir, { recursive: true }) - writeFileSync( - join(appDir, 'package.json'), - `${JSON.stringify( - { - name: 'agent-runtime-package-verification', - private: true, - type: 'module', - }, - null, - 2, - )}\n`, - ) run('pnpm', ['pack', '--pack-destination', packDir], repoRoot) const tarballs = run('find', [packDir, '-maxdepth', '1', '-name', '*.tgz', '-print'], repoRoot) @@ -42,23 +30,33 @@ try { if (packageJson.peerDependenciesMeta?.['@tangle-network/agent-eval']?.optional) { throw new Error('@tangle-network/agent-eval must stay required: root and ./loops import it at runtime') } - const requiredExports = { - '.': ['import', 'types'], - './agent': ['import', 'types'], - './conversation': ['import', 'types'], - './candidate-execution': ['import', 'types'], - './intelligence': ['import', 'types'], - './loops': ['import', 'types'], - './environment-provider': ['import', 'types'], - './primeintellect': ['import', 'types'], - './profiles': ['import', 'types'], - './mcp': ['import', 'types'], + const packageExports = packageJson.exports + if (!packageExports || typeof packageExports !== 'object') { + throw new Error('packed package has no exports map') } - - for (const [subpath, fields] of Object.entries(requiredExports)) { - const exportTarget = packageJson.exports?.[subpath] - if (!exportTarget) throw new Error(`missing package export ${subpath}`) - for (const field of fields) { + const requiredSubpaths = [ + '.', + './agent', + './conversation', + './intelligence', + './loops', + './environment-provider', + './analyst-loop', + './lifecycle', + './knowledge', + './profiles', + './platform', + './primeintellect', + './candidate-execution', + './mcp', + ] + for (const subpath of requiredSubpaths) { + if (!(subpath in packageExports)) { + throw new Error(`packed package removed public export ${subpath}`) + } + } + for (const [subpath, exportTarget] of Object.entries(packageExports)) { + for (const field of ['import', 'types']) { const relativeTarget = exportTarget[field] if (typeof relativeTarget !== 'string') { throw new Error(`missing ${field} target for package export ${subpath}`) @@ -67,20 +65,135 @@ try { } } - // Install into an empty app so dependency resolution uses only published package metadata. - run( - 'npm', - [ - 'install', - '--ignore-scripts', - '--no-package-lock', - '--no-save', - '--no-audit', - '--no-fund', - tarballs[0], - ], - appDir, + const repoPackageJson = JSON.parse(readFileSync(join(repoRoot, 'package.json'), 'utf8')) + const knowledgePackageDir = join( + repoRoot, + 'node_modules', + '@tangle-network', + 'agent-knowledge', + ) + const knowledgePackageJson = JSON.parse( + readFileSync(join(knowledgePackageDir, 'package.json'), 'utf8'), + ) + if ( + knowledgePackageJson.name !== '@tangle-network/agent-knowledge' || + typeof knowledgePackageJson.version !== 'string' || + knowledgePackageJson.version.length === 0 + ) { + throw new Error('packed consumer requires an installed @tangle-network/agent-knowledge release') + } + const peerPackages = [ + '@tangle-network/agent-eval', + '@tangle-network/agent-interface', + '@tangle-network/sandbox', + 'playwright', + ] + const peerDependencies = Object.fromEntries( + peerPackages.map((name) => { + const version = repoPackageJson.devDependencies?.[name] + if (typeof version !== 'string' || version.length === 0) { + throw new Error(`packed consumer requires a ${name} development dependency`) + } + return [name, version] + }), + ) + writeFileSync( + join(appDir, 'package.json'), + `${JSON.stringify( + { + private: true, + type: 'module', + dependencies: { + '@tangle-network/agent-runtime': `file:${tarballs[0]}`, + ...peerDependencies, + }, + devDependencies: { + '@types/node': repoPackageJson.devDependencies['@types/node'], + typescript: repoPackageJson.devDependencies.typescript, + }, + }, + null, + 2, + )}\n`, + ) + writeFileSync( + join(appDir, 'pnpm-workspace.yaml'), + `overrides:\n '@tangle-network/agent-knowledge': ${knowledgePackageJson.version}\n`, + ) + writeFileSync( + join(appDir, 'tsconfig.json'), + `${JSON.stringify( + { + compilerOptions: { + target: 'ES2022', + module: 'NodeNext', + moduleResolution: 'NodeNext', + strict: true, + noEmit: true, + skipLibCheck: false, + }, + include: ['consumer.ts'], + }, + null, + 2, + )}\n`, + ) + writeFileSync( + join(appDir, 'consumer.ts'), + ` + import type { + AgentCandidateProfileActivation, + CandidateExecutionEvidence, + } from '@tangle-network/agent-interface' + import { SandboxClient } from '@tangle-network/sandbox' + import { + createSandboxCandidateExperimentExecutor, + parseAgentCandidateProfileActivation, + sandboxCandidateExperimentExecutionSupport, + verifyCandidateExecutionEvidence, + type CreateSandboxCandidateExperimentExecutorOptions, + type SandboxCandidateExperimentExecution, + type VerifyCandidateExecutionEvidenceOptions, + } from '@tangle-network/agent-runtime/intelligence' + + const client = new SandboxClient({ + apiKey: 'sk_sandbox_compile_only', + baseUrl: 'https://sandbox.example.com', + trustLocalCliAuth: false, + }) + declare const ports: CreateSandboxCandidateExperimentExecutorOptions['ports'] + declare const grader: CreateSandboxCandidateExperimentExecutorOptions['grader'] + declare const outputArtifacts: CreateSandboxCandidateExperimentExecutorOptions['outputArtifacts'] + declare const traceStore: CreateSandboxCandidateExperimentExecutorOptions['traceStore'] + declare const claimStore: CreateSandboxCandidateExperimentExecutorOptions['claimStore'] + declare const executionInput: SandboxCandidateExperimentExecution + declare const verification: VerifyCandidateExecutionEvidenceOptions + declare const storedEvidence: unknown + + const executor = createSandboxCandidateExperimentExecutor({ + client, + ports, + grader, + outputArtifacts, + traceStore, + claimStore, + }) + const execution: Promise = executor.execute(executionInput) + const evidence = verifyCandidateExecutionEvidence(storedEvidence, verification) + const activation: AgentCandidateProfileActivation = + parseAgentCandidateProfileActivation( + evidence.materializationReceipt.profileActivation, + evidence.materializationReceipt.profileActivation.profilePlan.digest, + ) + const outcome: 'output' = sandboxCandidateExperimentExecutionSupport.outcomes[0] + void execution + void activation + void outcome + `, ) + run('pnpm', ['install', '--config.auto-install-peers=false'], appDir) + run('pnpm', ['exec', 'tsc', '-p', 'tsconfig.json'], appDir) + run( process.execPath, [ @@ -91,14 +204,16 @@ try { const packageJson = JSON.parse( readFileSync('node_modules/@tangle-network/agent-runtime/package.json', 'utf8'), ) - const subpaths = Object.keys(packageJson.exports).map((subpath) => - subpath === '.' ? packageJson.name : packageJson.name + subpath.slice(1), - ) - for (const subpath of subpaths) await import(subpath) + for (const subpath of Object.keys(packageJson.exports)) { + const specifier = + subpath === '.' ? packageJson.name : packageJson.name + subpath.slice(1) + await import(specifier) + } `, ], appDir, ) + run( process.execPath, [ @@ -139,6 +254,12 @@ try { 'composeCertifiedProfile', 'manifestFromProfile', 'CapabilityNotAdmittedError', + 'createSandboxCandidateExperimentExecutor', + 'executeAgentCandidateExperimentCell', + 'parseAgentCandidateProfileActivation', + 'runAgentCandidateExperiment', + 'sandboxCandidateExperimentExecutionSupport', + 'verifyCandidateExecutionEvidence', ] for (const name of expectedIntelligence) { if (!(name in intelligence)) throw new Error('missing intelligence export ' + name) @@ -166,7 +287,7 @@ try { kind: 'profile', profile: { name: 'packed-consumer', harness: 'codex' }, }, - code: { kind: 'disabled', reason: 'control' }, + code: { kind: 'disabled' }, execution: { harness: 'codex', harnessVersion: '1.0.0', @@ -181,7 +302,6 @@ try { }, }, memory: { mode: 'disabled' }, - lineage: { source: 'human' }, }) const verified = await candidates.verifyAgentCandidateBundle(bundle, { artifacts: { read: async () => { throw new Error('unexpected artifact read') } }, @@ -213,6 +333,31 @@ try { ], appDir, ) + + const installedPackageDir = join( + appDir, + 'node_modules', + '@tangle-network', + 'agent-runtime', + ) + const repackDir = join(tempRoot, 'repack') + mkdirSync(repackDir, { recursive: true }) + run( + 'npm', + ['pack', '--ignore-scripts=false', '--pack-destination', repackDir], + installedPackageDir, + ) + const repackedTarballs = run( + 'find', + [repackDir, '-maxdepth', '1', '-name', '*.tgz', '-print'], + repoRoot, + ) + .trim() + .split('\n') + .filter(Boolean) + if (repackedTarballs.length !== 1) { + throw new Error(`expected exactly one repacked runtime tarball, found ${repackedTarballs.length}`) + } } finally { rmSync(tempRoot, { recursive: true, force: true }) } diff --git a/src/agent/define-agent.ts b/src/agent/define-agent.ts index 22ec4604..3c7e8ac1 100644 --- a/src/agent/define-agent.ts +++ b/src/agent/define-agent.ts @@ -105,18 +105,6 @@ export interface AgentManifest { */ analyst: AnalystConfig - /** - * Auto-apply policy. Knowledge / improvement edits land only when - * `enabled === true` AND the source finding's confidence meets the - * threshold. `mode` controls how applies happen: `'write'` mutates - * files in-place; `'open-pr'` writes to a branch and opens a PR. - * - * Default: knowledge auto-applies at confidence ≥0.85 in `'write'` - * mode (wiki edits are git-reversible); improvement stays at - * `enabled: false` until the agent author has measured precision. - */ - autoApply?: AutoApplyPolicy - /** * Declarative per-surface artifact-lifecycle config the closed loop reads. * @@ -290,19 +278,6 @@ export interface AnalystConfig { } } -export interface AutoApplyPolicy { - knowledge?: { - enabled: boolean - confidenceThreshold?: number - mode?: 'write' | 'open-pr' - } - improvement?: { - enabled: boolean - confidenceThreshold?: number - mode?: 'write' | 'open-pr' - } -} - // ── factory + validation ───────────────────────────────────────────── /** Thrown when `defineAgent` finds a required surface missing on disk. */ diff --git a/src/agent/improvement-adapter.ts b/src/agent/improvement-adapter.ts index 00285dff..ef7efd5e 100644 --- a/src/agent/improvement-adapter.ts +++ b/src/agent/improvement-adapter.ts @@ -131,6 +131,9 @@ export function createSurfaceImprovementAdapter( opts: CreateSurfaceImprovementAdapterOpts, ): ImprovementAdapter { const mode = opts.mode ?? 'none' + if (mode === 'open-pr' && !opts.ghRepo) { + throw new Error('createSurfaceImprovementAdapter: mode=open-pr requires `ghRepo`') + } const allowCreate = opts.allowCreateForKinds ?? DEFAULT_CREATE_KINDS return { @@ -230,14 +233,7 @@ export function createSurfaceImprovementAdapter( if (mode === 'none') { warnings.push( - 'createSurfaceImprovementAdapter: mode=none; no edits applied — adjust manifest.autoApply.improvement.mode', - ) - return { applied, warnings } - } - - if (mode === 'open-pr' && !opts.ghRepo) { - warnings.push( - 'createSurfaceImprovementAdapter: mode=open-pr requires `ghRepo`; falling back to no-op', + 'createSurfaceImprovementAdapter: mode=none; no edits applied — configure mode=write or mode=open-pr', ) return { applied, warnings } } diff --git a/src/agent/index.ts b/src/agent/index.ts index 0b8c7b08..3d801670 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -15,7 +15,6 @@ export type { AgentRunInvocation, AgentRuntime, AnalystConfig, - AutoApplyPolicy, JudgeConfig, RubricDimension, SurfaceLifecycle, diff --git a/src/candidate-execution/artifacts.ts b/src/candidate-execution/artifacts.ts index 4cbe4446..60183d6f 100644 --- a/src/candidate-execution/artifacts.ts +++ b/src/candidate-execution/artifacts.ts @@ -117,7 +117,6 @@ export function candidateWorkspaceManifest( files: ReadonlyArray<{ path: string; mode: number; bytes: Uint8Array }>, ): AgentCandidateWorkspaceManifestMaterial { return { - schemaVersion: 2, kind: 'agent-candidate-workspace-manifest', files: files .map((file) => ({ diff --git a/src/candidate-execution/benchmark-grader.ts b/src/candidate-execution/benchmark-grader.ts index 26981081..6357c74b 100644 --- a/src/candidate-execution/benchmark-grader.ts +++ b/src/candidate-execution/benchmark-grader.ts @@ -1,11 +1,12 @@ import type { BenchmarkEvaluation } from '@tangle-network/agent-eval' import type { - AgentCandidateArtifactRef, + AgentCandidateBenchmarkGraderIdentity, + AgentCandidateFixedSpend, AgentCandidateTermination, } from '@tangle-network/agent-interface' import { readVerifiedArtifact } from './artifacts' -import { immutableCandidateValue, sha256Bytes } from './digest' +import { canonicalCandidateDigest, immutableCandidateValue, sha256Bytes } from './digest' import type { AgentCandidateBenchmarkGraderPort, AgentCandidateOutputArtifactPort, @@ -13,10 +14,14 @@ import type { } from './types' export interface BoundAgentCandidateBenchmarkRun { - readonly grader: { - readonly name: string - readonly version: string - readonly artifact: AgentCandidateArtifactRef + readonly grader: AgentCandidateBenchmarkGraderIdentity + readonly grading: { + readonly usage: AgentCandidateFixedSpend + readonly timing: { + readonly startedAtMs: number + readonly endedAtMs: number + readonly durationMs: number + } } readonly evaluation: BenchmarkEvaluation readonly evidence: Uint8Array @@ -30,12 +35,13 @@ export async function runBoundCandidateBenchmarkGrader(input: { executionId: string termination: AgentCandidateTermination outcome: VerifiedAgentCandidateTaskOutcome + expectedGrader: AgentCandidateBenchmarkGraderIdentity grader: AgentCandidateBenchmarkGraderPort artifacts: AgentCandidateOutputArtifactPort signal?: AbortSignal }): Promise { input.signal?.throwIfAborted() - const descriptor = snapshotGraderDescriptor(input.grader) + const descriptor = snapshotGraderDescriptor(input.grader, input.expectedGrader) const implementationBytes = await readVerifiedArtifact(descriptor.artifact, input.artifacts) if (implementationBytes.byteLength === 0) { throw new Error('candidate benchmark grader implementation cannot be empty') @@ -43,6 +49,7 @@ export async function runBoundCandidateBenchmarkGrader(input: { const expectedImplementationDigest = sha256Bytes(implementationBytes) const termination = immutableCandidateValue(input.termination) const run = input.grader.run + const startedAtMs = Date.now() const result = await run( Object.freeze({ executionId: input.executionId, @@ -52,6 +59,7 @@ export async function runBoundCandidateBenchmarkGrader(input: { signal: input.signal ?? new AbortController().signal, }), ) + const endedAtMs = Math.max(startedAtMs, Date.now()) input.signal?.throwIfAborted() assertExactRunnerResult(result) @@ -73,6 +81,14 @@ export async function runBoundCandidateBenchmarkGrader(input: { return Object.freeze({ grader: descriptor, + grading: immutableCandidateValue({ + usage: emptyFixedSpend(), + timing: { + startedAtMs, + endedAtMs, + durationMs: endedAtMs - startedAtMs, + }, + }), evaluation: result.evaluation, evidence, }) @@ -80,15 +96,32 @@ export async function runBoundCandidateBenchmarkGrader(input: { function snapshotGraderDescriptor( grader: AgentCandidateBenchmarkGraderPort, + expected: AgentCandidateBenchmarkGraderIdentity, ): BoundAgentCandidateBenchmarkRun['grader'] { if (!grader.name || !grader.version) { throw new Error('candidate benchmark grader name and version must be non-empty') } - return immutableCandidateValue({ + const actual = { name: grader.name, version: grader.version, + format: 'tangle-grader' as const, artifact: grader.artifact, - }) + } + if (canonicalCandidateDigest(actual) !== canonicalCandidateDigest(expected)) { + throw new Error('candidate benchmark grader does not match the signed task grader') + } + return immutableCandidateValue(expected) +} + +function emptyFixedSpend(): AgentCandidateFixedSpend { + return { + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + modelCalls: 0, + costUsdNanos: 0, + } } function detachedImplementation(bytes: Uint8Array): { diff --git a/src/candidate-execution/builder.ts b/src/candidate-execution/builder.ts index 6ff0bd60..640ba883 100644 --- a/src/candidate-execution/builder.ts +++ b/src/candidate-execution/builder.ts @@ -6,14 +6,13 @@ import type { AgentCandidateExecution, AgentCandidateGitHubRepository, AgentCandidateKnowledge, - AgentCandidateLineage, AgentCandidateMemoryPolicy, AgentCandidateProfile, AgentProfile, AgentProfileDiff, } from '@tangle-network/agent-interface' import { type AgentCandidateBundleInput, sealAgentCandidateBundle } from './bundle' -import { canonicalCandidateDigest, embeddedCandidateArtifact } from './digest' +import { embeddedCandidateArtifact } from './digest' import { applyExactAgentProfileDiff, freezeGenericAgentCandidateProfile, @@ -31,7 +30,7 @@ export type AgentCandidateProfileSource = | { kind: 'profile-diffs' base: AgentProfile - /** Applied in order. Each exact diff is content-addressed into lineage. */ + /** Applied in order before the resulting profile is frozen into the bundle. */ diffs: readonly AgentProfileDiff[] } | { @@ -62,8 +61,6 @@ export interface BuildAgentCandidateBundleInput { execution: AgentCandidateExecution knowledge?: AgentCandidateKnowledge memory: AgentCandidateMemoryPolicy - /** `profileDiffIds` is derived from `profile`; callers cannot contradict it. */ - lineage: Omit } /** @@ -71,46 +68,30 @@ export interface BuildAgentCandidateBundleInput { * contract. Code bytes are re-read and verified by agent-eval before they are * embedded. The returned bundle is schema-validated, canonically digested, and * deeply immutable; call `verifyAgentCandidateBundle` at the execution boundary - * to re-read external knowledge, memory, repository, and workspace artifacts. + * to re-read external memory, repository, and workspace artifacts. */ export function buildAgentCandidateBundle( input: BuildAgentCandidateBundleInput, ): ReturnType { - if (Object.hasOwn(input.lineage, 'profileDiffIds')) { - throw new Error('profileDiffIds is derived from the profile source and cannot be supplied') - } - const compiledProfile = compileCandidateProfile(input.profile) - const profileDiffIds = compiledProfile.profileDiffIds const bundle: AgentCandidateBundleInput = { - schemaVersion: 2, kind: 'agent-candidate-bundle', digestAlgorithm: 'rfc8785-sha256', - profile: compiledProfile.profile, + profile: compileCandidateProfile(input.profile), code: compileCandidateCode(input.code), execution: input.execution, - ...(input.knowledge ? { knowledge: input.knowledge } : {}), + ...(input.knowledge !== undefined ? { knowledge: input.knowledge } : {}), memory: input.memory, - lineage: { - ...input.lineage, - ...(profileDiffIds.length > 0 ? { profileDiffIds } : {}), - }, } return sealAgentCandidateBundle(bundle) } -function compileCandidateProfile(source: AgentCandidateProfileSource): { - profile: AgentCandidateProfile - profileDiffIds: string[] -} { +function compileCandidateProfile(source: AgentCandidateProfileSource): AgentCandidateProfile { if (source.kind === 'candidate-profile') { - return { - profile: parseExactCandidateProfile(source.profile), - profileDiffIds: [], - } + return parseExactCandidateProfile(source.profile) } if (source.kind === 'profile') { - return { profile: freezeGenericAgentCandidateProfile(source.profile), profileDiffIds: [] } + return freezeGenericAgentCandidateProfile(source.profile) } if (source.kind !== 'profile-diffs') { @@ -122,13 +103,11 @@ function compileCandidateProfile(source: AgentCandidateProfileSource): { throw new Error('profile-diffs source requires at least one AgentProfileDiff') } let profile = parseExactAgentProfile(source.base, 'base profile') - const profileDiffIds: string[] = [] for (const [index, inputDiff] of source.diffs.entries()) { const diff = parseExactAgentProfileDiff(inputDiff, `profile diff ${index}`) profile = applyExactAgentProfileDiff(profile, diff, `profile diff ${index}`) - profileDiffIds.push(canonicalCandidateDigest(diff)) } - return { profile: freezeGenericAgentCandidateProfile(profile), profileDiffIds } + return freezeGenericAgentCandidateProfile(profile) } function compileCandidateCode(source: AgentCandidateCodeSource): AgentCandidateBundleInput['code'] { diff --git a/src/candidate-execution/claim-file-formats.ts b/src/candidate-execution/claim-file-formats.ts index 54b99463..098f0b02 100644 --- a/src/candidate-execution/claim-file-formats.ts +++ b/src/candidate-execution/claim-file-formats.ts @@ -1,26 +1,54 @@ -import type { Sha256Digest } from '@tangle-network/agent-interface' +import { + type AgentCandidateArtifactRef, + agentCandidateArtifactRefSchema, + type Sha256Digest, +} from '@tangle-network/agent-interface' import type { AgentCandidateExecutionClaim, AgentCandidateExecutionTerminalRecord } from './claim' +import { immutableCandidateValue } from './digest' +import { assertExactObjectKeys } from './exact-object' -export const CLAIM_FORMAT_VERSION = 7 -export const PENDING_FORMAT_VERSION = 1 -export const TERMINAL_FORMAT_VERSION = 3 -export const PHASE_FORMAT_VERSION = 1 +export interface AgentCandidatePreparationEvidence { + readonly executionPlan: AgentCandidateArtifactRef + readonly materializationReceipt: AgentCandidateArtifactRef +} + +export function sealCandidatePreparationEvidence( + value: unknown, + executionPlanDigest: Sha256Digest, +): AgentCandidatePreparationEvidence { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('candidate execution preparation evidence must be an object') + } + const record = value as Record + assertExactObjectKeys( + record, + ['executionPlan', 'materializationReceipt'], + 'candidate execution preparation evidence', + ) + const executionPlan = immutableCandidateValue( + agentCandidateArtifactRefSchema.parse(record.executionPlan), + ) + const materializationReceipt = immutableCandidateValue( + agentCandidateArtifactRefSchema.parse(record.materializationReceipt), + ) + if (executionPlan.sha256 !== executionPlanDigest) { + throw new Error('candidate execution plan artifact digest does not match its claim') + } + return Object.freeze({ executionPlan, materializationReceipt }) +} export interface PersistedAgentCandidateExecutionClaim extends AgentCandidateExecutionClaim { - version: typeof CLAIM_FORMAT_VERSION phase: 'claimed' leaseDigest: Sha256Digest } export interface PersistedAgentCandidateExecutionPending { - version: typeof PENDING_FORMAT_VERSION kind: 'candidate-execution-pending-terminal' terminal: AgentCandidateExecutionTerminalRecord } export interface PersistedAgentCandidateExecutionPhase { - version: typeof PHASE_FORMAT_VERSION kind: 'candidate-execution-phase' executionId: string attempt: number @@ -29,6 +57,5 @@ export interface PersistedAgentCandidateExecutionPhase { } export interface PersistedAgentCandidateExecutionTerminal { - version: typeof TERMINAL_FORMAT_VERSION terminal: AgentCandidateExecutionTerminalRecord } diff --git a/src/candidate-execution/claim-file-store.ts b/src/candidate-execution/claim-file-store.ts index d4d9aeba..15fe4900 100644 --- a/src/candidate-execution/claim-file-store.ts +++ b/src/candidate-execution/claim-file-store.ts @@ -21,12 +21,6 @@ import { type AgentCandidateRetryRejection, candidateClaimFileInternals, } from './claim' -import { - CLAIM_FORMAT_VERSION, - PENDING_FORMAT_VERSION, - PHASE_FORMAT_VERSION, - TERMINAL_FORMAT_VERSION, -} from './claim-file-formats' import { assertRecoveryMatchesStaged, assertTerminalAllowedInPhase, @@ -100,7 +94,6 @@ export class FileAgentCandidateExecutionClaimStore implements AgentCandidateExec const lease = newLease(claim) const acquired = await writeRecordIfAbsent(this.directory, claimPath, { - version: CLAIM_FORMAT_VERSION, ...claim, phase: 'claimed', leaseDigest: leaseDigest(lease), @@ -138,7 +131,6 @@ export class FileAgentCandidateExecutionClaimStore implements AgentCandidateExec this.directory, this.transitionPath(stored.claim, 1), { - version: PHASE_FORMAT_VERSION, kind: 'candidate-execution-phase', executionId: stored.claim.executionId, attempt: stored.claim.attempt, @@ -174,7 +166,6 @@ export class FileAgentCandidateExecutionClaimStore implements AgentCandidateExec this.directory, this.transitionPath(stored.claim, transition), { - version: PENDING_FORMAT_VERSION, kind: 'candidate-execution-pending-terminal', terminal, }, @@ -212,7 +203,6 @@ export class FileAgentCandidateExecutionClaimStore implements AgentCandidateExec this.directory, terminalPath, { - version: TERMINAL_FORMAT_VERSION, terminal: staged, }, this.ownerPublication(stored.claim), @@ -245,7 +235,6 @@ export class FileAgentCandidateExecutionClaimStore implements AgentCandidateExec this.directory, this.transitionPath(record.claim, transition), { - version: PENDING_FORMAT_VERSION, kind: 'candidate-execution-pending-terminal', terminal: recovered, }, @@ -266,7 +255,6 @@ export class FileAgentCandidateExecutionClaimStore implements AgentCandidateExec const terminalPath = this.terminalPath(attempt) const didFinish = await writeRecordIfAbsent(this.directory, terminalPath, { - version: TERMINAL_FORMAT_VERSION, terminal: staged, }) if (didFinish) return Object.freeze({ finished: true, terminal: staged }) diff --git a/src/candidate-execution/claim-plan.ts b/src/candidate-execution/claim-plan.ts index 8d8333a3..a2a30425 100644 --- a/src/candidate-execution/claim-plan.ts +++ b/src/candidate-execution/claim-plan.ts @@ -1,4 +1,4 @@ -import type { Sha256Digest } from '@tangle-network/agent-interface' +import type { AgentCandidateArtifactRef, Sha256Digest } from '@tangle-network/agent-interface' import { type AgentCandidateExecutionClaim, candidateClaimFileInternals } from './claim' import { canonicalCandidateDigest } from './digest' @@ -9,10 +9,18 @@ import type { PreparedAgentCandidateExecution } from './types' /** Extract the complete durable claim from a prepared execution. */ export function candidateExecutionClaim( prepared: PreparedAgentCandidateExecution, + preparationEvidence: { + executionPlan: AgentCandidateArtifactRef + materializationReceipt: AgentCandidateArtifactRef + }, ): AgentCandidateExecutionClaim { const state = assertPreparedCandidateIntegrity(prepared) const material = prepared.executionPlan.value.material - const attempt = material.attempt + const attempt = { + number: material.runCell.attempt, + maxAttempts: state.benchmarkTask.attempt.maxAttempts, + retryPolicy: state.benchmarkTask.attempt.retryPolicy, + } as const const nowMs = Date.now() if (!Number.isSafeInteger(nowMs) || nowMs < 0) { throw new Error('candidate execution claim-store clock returned an invalid timestamp') @@ -32,6 +40,21 @@ export function candidateExecutionClaim( 'candidate preparation expires before its full execution and cleanup owner window', ) } + if ( + preparationEvidence.executionPlan.sha256 !== prepared.executionPlan.value.digest || + preparationEvidence.executionPlan.byteLength !== state.executionPlan.bytes.byteLength + ) { + throw new Error('persisted execution plan does not match its canonical preparation bytes') + } + if ( + preparationEvidence.materializationReceipt.sha256 !== state.materializationReceipt.digest || + preparationEvidence.materializationReceipt.byteLength !== + state.materializationReceipt.bytes.byteLength + ) { + throw new Error( + 'persisted materialization receipt does not match its canonical preparation bytes', + ) + } return candidateClaimFileInternals.sealClaim({ executionId: prepared.executionId, attempt: attempt.number, @@ -39,6 +62,7 @@ export function candidateExecutionClaim( retryPolicy: attempt.retryPolicy, bundleDigest: prepared.bundle.digest, executionPlanDigest: prepared.executionPlan.value.digest, + preparationEvidence, retryLineageDigest: retryLineageDigest(prepared, state.resultTimeoutMs), leaseExpiresAtMs, resultTimeoutMs: state.resultTimeoutMs, @@ -69,7 +93,11 @@ function retryLineageDigest( resultTimeoutMs, executionPlan: { ...material, - attempt: { ...material.attempt, number: 0 }, + runCell: { + ...material.runCell, + attempt: 0, + digest: `sha256:${'0'.repeat(64)}`, + }, model: { ...material.model, access: { diff --git a/src/candidate-execution/claim-terminal.ts b/src/candidate-execution/claim-terminal.ts index d71b1a9c..1a30735d 100644 --- a/src/candidate-execution/claim-terminal.ts +++ b/src/candidate-execution/claim-terminal.ts @@ -1,4 +1,8 @@ -import type { AgentCandidateArtifactRef, Sha256Digest } from '@tangle-network/agent-interface' +import type { + AgentCandidateArtifactRef, + AgentCandidateFixedSpend, + Sha256Digest, +} from '@tangle-network/agent-interface' import { agentCandidateArtifactRefSchema } from '@tangle-network/agent-interface' import type { @@ -10,8 +14,8 @@ import type { AgentCandidateExecutionStageResult, AgentCandidateExecutionTerminalRecord, AgentCandidateExecutionTerminalResult, - AgentCandidateExecutionUsage, } from './claim' +import { sealCandidatePreparationEvidence } from './claim-file-formats' import { canonicalCandidateDigest, immutableCandidateValue } from './digest' import { assertExactObjectKeys as assertExactKeys } from './exact-object' @@ -27,6 +31,7 @@ export function terminalRecord( attempt: claim.attempt, bundleDigest: claim.bundleDigest, executionPlanDigest: claim.executionPlanDigest, + preparationEvidence: claim.preparationEvidence, ...terminal, } return immutableCandidateValue({ @@ -42,7 +47,6 @@ export function recoveredTerminalRecord( ): AgentCandidateExecutionTerminalRecord { const recovered = sealRecoveryEvidence(evidence, claim) return terminalRecord(claim, { - schemaVersion: 1, status: 'failed', failureClass: recovered.failureClass === 'pre-model-infrastructure' && phase !== 'claimed' @@ -140,8 +144,8 @@ export function sealTerminalRecordValue( 'attempt', 'bundleDigest', 'executionPlanDigest', + 'preparationEvidence', 'terminalDigest', - 'schemaVersion', 'status', 'usage', 'modelSettlement', @@ -154,8 +158,8 @@ export function sealTerminalRecordValue( 'attempt', 'bundleDigest', 'executionPlanDigest', + 'preparationEvidence', 'terminalDigest', - 'schemaVersion', 'status', 'failureClass', 'usage', @@ -164,15 +168,20 @@ export function sealTerminalRecordValue( ], label, ) + const executionPlanDigest = requireString( + value.executionPlanDigest, + label, + 'executionPlanDigest', + ) as Sha256Digest const identity = { executionId: requireString(value.executionId, label, 'executionId'), attempt: requireNumber(value.attempt, label, 'attempt'), bundleDigest: requireString(value.bundleDigest, label, 'bundleDigest') as Sha256Digest, - executionPlanDigest: requireString( - value.executionPlanDigest, - label, - 'executionPlanDigest', - ) as Sha256Digest, + executionPlanDigest, + preparationEvidence: sealCandidatePreparationEvidence( + value.preparationEvidence, + executionPlanDigest, + ), } assertExecutionId(identity.executionId) if (!Number.isSafeInteger(identity.attempt) || identity.attempt < 1) { @@ -180,30 +189,23 @@ export function sealTerminalRecordValue( } assertSha256Digest(identity.bundleDigest, 'bundleDigest') assertSha256Digest(identity.executionPlanDigest, 'executionPlanDigest') + if (identity.preparationEvidence.executionPlan.sha256 !== identity.executionPlanDigest) { + throw new Error(`${label} execution plan artifact does not match executionPlanDigest`) + } const result = sealTerminalResult( status === 'succeeded' ? { - schemaVersion: requireNumber(value.schemaVersion, label, 'schemaVersion') as 1, status, - usage: requireObject( - value.usage, - label, - 'usage', - ) as unknown as AgentCandidateExecutionUsage, + usage: requireObject(value.usage, label, 'usage') as unknown as AgentCandidateFixedSpend, modelSettlement: requireArtifactRef(value.modelSettlement, label, 'modelSettlement'), taskOutcome: requireArtifactRef(value.taskOutcome, label, 'taskOutcome'), benchmarkResult: requireArtifactRef(value.benchmarkResult, label, 'benchmarkResult'), runReceipt: requireArtifactRef(value.runReceipt, label, 'runReceipt'), } : { - schemaVersion: requireNumber(value.schemaVersion, label, 'schemaVersion') as 1, status, failureClass: requireFailureClass(value.failureClass, label), - usage: requireObject( - value.usage, - label, - 'usage', - ) as unknown as AgentCandidateExecutionUsage, + usage: requireObject(value.usage, label, 'usage') as unknown as AgentCandidateFixedSpend, modelSettlement: requireArtifactRef(value.modelSettlement, label, 'modelSettlement'), ...(value.failureEvidence ? { @@ -241,7 +243,9 @@ export function assertTerminalMatchesClaim( terminal.executionId !== claim.executionId || terminal.attempt !== claim.attempt || terminal.bundleDigest !== claim.bundleDigest || - terminal.executionPlanDigest !== claim.executionPlanDigest + terminal.executionPlanDigest !== claim.executionPlanDigest || + canonicalCandidateDigest(terminal.preparationEvidence) !== + canonicalCandidateDigest(claim.preparationEvidence) ) { throw new Error(`candidate execution terminal record at ${path} does not match its claim`) } @@ -351,17 +355,8 @@ function sealTerminalResult( assertExactKeys( result, result.status === 'succeeded' - ? [ - 'schemaVersion', - 'status', - 'usage', - 'modelSettlement', - 'taskOutcome', - 'benchmarkResult', - 'runReceipt', - ] + ? ['status', 'usage', 'modelSettlement', 'taskOutcome', 'benchmarkResult', 'runReceipt'] : [ - 'schemaVersion', 'status', 'failureClass', 'usage', @@ -370,14 +365,10 @@ function sealTerminalResult( ], 'candidate execution terminal result', ) - if (result.schemaVersion !== 1) { - throw new Error('candidate execution terminal schemaVersion must be 1') - } const usage = sealUsage(result.usage) const modelSettlement = sealArtifactRef(result.modelSettlement, 'modelSettlement') if (result.status === 'succeeded') { return Object.freeze({ - schemaVersion: 1, status: 'succeeded', usage, modelSettlement, @@ -391,7 +382,6 @@ function sealTerminalResult( throw new Error('pre-model infrastructure failure cannot contain model calls') } return Object.freeze({ - schemaVersion: 1, status: 'failed', failureClass: result.failureClass, usage, @@ -402,7 +392,7 @@ function sealTerminalResult( }) } -function sealUsage(usage: AgentCandidateExecutionUsage): AgentCandidateExecutionUsage { +function sealUsage(usage: AgentCandidateFixedSpend): AgentCandidateFixedSpend { assertExactKeys( usage, [ diff --git a/src/candidate-execution/claim.ts b/src/candidate-execution/claim.ts index e31ac79d..84cecfce 100644 --- a/src/candidate-execution/claim.ts +++ b/src/candidate-execution/claim.ts @@ -5,19 +5,18 @@ import { readFile } from 'node:fs/promises' import { type AgentCandidateArtifactRef, type AgentCandidateAttemptPolicy, + type AgentCandidateFixedSpend, type AgentCandidateResolvedModel, agentCandidateResolvedModelSchema, type Sha256Digest, } from '@tangle-network/agent-interface' import { - CLAIM_FORMAT_VERSION, - PENDING_FORMAT_VERSION, + type AgentCandidatePreparationEvidence, type PersistedAgentCandidateExecutionClaim, type PersistedAgentCandidateExecutionPending, type PersistedAgentCandidateExecutionPhase, type PersistedAgentCandidateExecutionTerminal, - PHASE_FORMAT_VERSION, - TERMINAL_FORMAT_VERSION, + sealCandidatePreparationEvidence, } from './claim-file-formats' import { assertRecoveryMatchesStaged, @@ -56,6 +55,8 @@ export interface AgentCandidateExecutionClaim { readonly retryPolicy: AgentCandidateAttemptPolicy['retryPolicy'] readonly bundleDigest: Sha256Digest readonly executionPlanDigest: Sha256Digest + /** Durable canonical bytes needed to reconstruct the signed preparation. */ + readonly preparationEvidence: AgentCandidatePreparationEvidence /** Frozen plan identity with only attempt number and per-attempt grant identity normalized. */ readonly retryLineageDigest: Sha256Digest /** The winning lease stops authorizing a new terminal write at this instant. */ @@ -81,32 +82,20 @@ export type AgentCandidateExecutionFailureClass = | 'post-model-infrastructure' | 'unknown' -/** Exact fixed-point usage proven by the closed evaluator model ledger. */ -export interface AgentCandidateExecutionUsage { - readonly costUsdNanos: number - readonly inputTokens: number - readonly outputTokens: number - readonly cachedInputTokens: number - readonly reasoningTokens: number - readonly modelCalls: number -} - /** Evaluator-owned terminal facts staged durably before the terminal CAS. */ export type AgentCandidateExecutionTerminalResult = | { - readonly schemaVersion: 1 readonly status: 'succeeded' - readonly usage: AgentCandidateExecutionUsage + readonly usage: AgentCandidateFixedSpend readonly modelSettlement: AgentCandidateArtifactRef readonly taskOutcome: AgentCandidateArtifactRef readonly benchmarkResult: AgentCandidateArtifactRef readonly runReceipt: AgentCandidateArtifactRef } | { - readonly schemaVersion: 1 readonly status: 'failed' readonly failureClass: AgentCandidateExecutionFailureClass - readonly usage: AgentCandidateExecutionUsage + readonly usage: AgentCandidateFixedSpend readonly modelSettlement: AgentCandidateArtifactRef readonly failureEvidence?: AgentCandidateArtifactRef } @@ -117,6 +106,7 @@ export type AgentCandidateExecutionTerminalRecord = AgentCandidateExecutionTermi readonly attempt: number readonly bundleDigest: Sha256Digest readonly executionPlanDigest: Sha256Digest + readonly preparationEvidence: AgentCandidateExecutionClaim['preparationEvidence'] /** RFC 8785 SHA-256 of this record with `terminalDigest` omitted. */ readonly terminalDigest: Sha256Digest } @@ -127,7 +117,7 @@ export type AgentCandidateExecutionPhase = 'claimed' | 'candidate-may-run' /** Trusted, independently observed closure facts for one expired winning lease. */ export interface AgentCandidateExecutionRecoveryEvidence { readonly failureClass: AgentCandidateExecutionFailureClass - readonly usage: AgentCandidateExecutionUsage + readonly usage: AgentCandidateFixedSpend readonly modelSettlement: AgentCandidateArtifactRef readonly failureEvidence?: AgentCandidateArtifactRef readonly process: { @@ -409,8 +399,8 @@ export class InMemoryAgentCandidateExecutionClaimStore } const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/ -const LEASE_TOKEN_PATTERN = /^candidate-execution-lease-v1\.[A-Za-z0-9_-]{43}$/ -const PREPARATION_ID_PATTERN = /^candidate-preparation-v1\.[A-Za-z0-9_-]{43}$/ +const LEASE_TOKEN_PATTERN = /^candidate-execution-lease\.[A-Za-z0-9_-]{43}$/ +const PREPARATION_ID_PATTERN = /^candidate-preparation\.[A-Za-z0-9_-]{43}$/ function sealClaim(claim: AgentCandidateExecutionClaim): AgentCandidateExecutionClaim { assertExactKeys( @@ -422,6 +412,7 @@ function sealClaim(claim: AgentCandidateExecutionClaim): AgentCandidateExecution 'retryPolicy', 'bundleDigest', 'executionPlanDigest', + 'preparationEvidence', 'retryLineageDigest', 'leaseExpiresAtMs', 'resultTimeoutMs', @@ -447,6 +438,10 @@ function sealClaim(claim: AgentCandidateExecutionClaim): AgentCandidateExecution } assertSha256Digest(claim.bundleDigest, 'bundleDigest') assertSha256Digest(claim.executionPlanDigest, 'executionPlanDigest') + const preparationEvidence = sealCandidatePreparationEvidence( + claim.preparationEvidence, + claim.executionPlanDigest, + ) assertSha256Digest(claim.retryLineageDigest, 'retryLineageDigest') assertPositiveTimestamp(claim.leaseExpiresAtMs, 'leaseExpiresAtMs') candidateResultTimeout(claim.resultTimeoutMs, claim.resultTimeoutMs) @@ -458,6 +453,7 @@ function sealClaim(claim: AgentCandidateExecutionClaim): AgentCandidateExecution retryPolicy: claim.retryPolicy, bundleDigest: claim.bundleDigest, executionPlanDigest: claim.executionPlanDigest, + preparationEvidence, retryLineageDigest: claim.retryLineageDigest, leaseExpiresAtMs: claim.leaseExpiresAtMs, resultTimeoutMs: claim.resultTimeoutMs, @@ -554,7 +550,7 @@ function newLease(claim: AgentCandidateExecutionClaim): AgentCandidateExecutionL return Object.freeze({ executionId: claim.executionId, attempt: claim.attempt, - token: `candidate-execution-lease-v1.${randomBytes(32).toString('base64url')}`, + token: `candidate-execution-lease.${randomBytes(32).toString('base64url')}`, expiresAtMs: claim.leaseExpiresAtMs, }) } @@ -646,19 +642,16 @@ function rejectedRetry( async function readClaim(path: string): Promise { const parsed = await readJsonObject(path, 'claim') const record = parsed as Partial - if (record.version !== CLAIM_FORMAT_VERSION) { - throw new Error(`candidate execution claim at ${path} has unsupported version`) - } assertExactKeys( parsed, [ - 'version', 'executionId', 'attempt', 'maxAttempts', 'retryPolicy', 'bundleDigest', 'executionPlanDigest', + 'preparationEvidence', 'retryLineageDigest', 'leaseExpiresAtMs', 'resultTimeoutMs', @@ -679,6 +672,11 @@ async function readClaim(path: string): Promise { path, 'executionPlanDigest', ) as Sha256Digest, + preparationEvidence: requireObject( + record.preparationEvidence, + path, + 'preparationEvidence', + ) as unknown as AgentCandidateExecutionClaim['preparationEvidence'], retryLineageDigest: requireString( record.retryLineageDigest, path, @@ -716,10 +714,7 @@ async function readClaimIfPresent(path: string): Promise { const parsed = await readJsonObject(path, 'terminal record') const record = parsed as Partial - if (record.version !== TERMINAL_FORMAT_VERSION) { - throw new Error(`candidate execution terminal record at ${path} has unsupported version`) - } - assertExactKeys(parsed, ['version', 'terminal'], `candidate execution terminal record at ${path}`) + assertExactKeys(parsed, ['terminal'], `candidate execution terminal record at ${path}`) return sealTerminalRecordValue( requireObject(record.terminal, path, 'terminal'), `candidate execution terminal record at ${path}`, @@ -754,12 +749,9 @@ async function readTransitionIfPresent( } if (parsed.kind === 'candidate-execution-phase') { const record = parsed as unknown as PersistedAgentCandidateExecutionPhase - if (record.version !== PHASE_FORMAT_VERSION) { - throw new Error(`candidate execution phase record at ${path} has unsupported version`) - } assertExactKeys( parsed, - ['version', 'kind', 'executionId', 'attempt', 'executionPlanDigest', 'phase'], + ['kind', 'executionId', 'attempt', 'executionPlanDigest', 'phase'], `candidate execution phase record at ${path}`, ) if ( @@ -774,14 +766,7 @@ async function readTransitionIfPresent( } if (parsed.kind === 'candidate-execution-pending-terminal') { const record = parsed as unknown as PersistedAgentCandidateExecutionPending - if (record.version !== PENDING_FORMAT_VERSION) { - throw new Error(`candidate execution pending record at ${path} has unsupported version`) - } - assertExactKeys( - parsed, - ['version', 'kind', 'terminal'], - `candidate execution pending record at ${path}`, - ) + assertExactKeys(parsed, ['kind', 'terminal'], `candidate execution pending record at ${path}`) const terminal = sealTerminalRecordValue( requireObject(record.terminal, path, 'terminal'), `candidate execution pending record at ${path}`, diff --git a/src/candidate-execution/cleanup.ts b/src/candidate-execution/cleanup.ts index daea4b9e..4403de42 100644 --- a/src/candidate-execution/cleanup.ts +++ b/src/candidate-execution/cleanup.ts @@ -34,31 +34,13 @@ export function candidateResultTimeout( return effective } -/** Bound an evaluator cleanup call while keeping late rejection observed. */ +/** Bound an evaluator cleanup call and cancel the underlying port at expiry. */ export async function withinCandidateCleanupDeadline( - operation: () => Promise, + operation: (signal: AbortSignal) => Promise, deadlineAtMs: number, label: string, ): Promise { - const remainingMs = deadlineAtMs - Date.now() - if (remainingMs <= 0) throw new CandidateCleanupTimeoutError(label) - - const pending = Promise.resolve().then(operation) - void pending.catch(() => undefined) - let timer: ReturnType | undefined - try { - const result = await Promise.race([ - pending, - new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new CandidateCleanupTimeoutError(label)), remainingMs) - }), - ]) - // Exact-boundary completion is ambiguous under event-loop delay. - if (Date.now() >= deadlineAtMs) throw new CandidateCleanupTimeoutError(label) - return result - } finally { - if (timer) clearTimeout(timer) - } + return withinCandidateDeadline(operation, deadlineAtMs, new CandidateCleanupTimeoutError(label)) } /** @@ -69,12 +51,19 @@ export async function withinCandidateResultDeadline( operation: (signal: AbortSignal) => Promise, deadlineAtMs: number, label: string, +): Promise { + return withinCandidateDeadline(operation, deadlineAtMs, new CandidateResultTimeoutError(label)) +} + +async function withinCandidateDeadline( + operation: (signal: AbortSignal) => Promise, + deadlineAtMs: number, + timeoutError: Error, ): Promise { const remainingMs = deadlineAtMs - Date.now() - if (remainingMs <= 0) throw new CandidateResultTimeoutError(label) + if (remainingMs <= 0) throw timeoutError const controller = new AbortController() - const timeoutError = new CandidateResultTimeoutError(label) const pending = Promise.resolve().then(() => operation(controller.signal)) void pending.catch(() => undefined) let timer: ReturnType | undefined diff --git a/src/candidate-execution/digest.ts b/src/candidate-execution/digest.ts index 47fe0c7b..a1871626 100644 --- a/src/candidate-execution/digest.ts +++ b/src/candidate-execution/digest.ts @@ -43,6 +43,16 @@ export function canonicalCandidateDocument( }) } +export function verifyCanonicalCandidateDocument( + value: T, + label: string, +): T { + if (canonicalCandidateDigest(omitTopLevelDigest(value)) !== value.digest) { + throw new Error(`${label} digest does not match`) + } + return immutableCandidateValue(value) +} + export function embeddedCandidateArtifact(bytes: Uint8Array): AgentCandidateEmbeddedArtifact { return { encoding: 'base64', diff --git a/src/candidate-execution/execute.ts b/src/candidate-execution/execute.ts index e9e8339e..f9935288 100644 --- a/src/candidate-execution/execute.ts +++ b/src/candidate-execution/execute.ts @@ -1,7 +1,11 @@ import type { TraceStore } from '@tangle-network/agent-eval' -import type { AgentCandidateTermination } from '@tangle-network/agent-interface' +import type { + AgentCandidateTaskOutcomeSpec, + AgentCandidateTermination, +} from '@tangle-network/agent-interface' import type { + AgentCandidateExecutionClaim, AgentCandidateExecutionClaimStore, AgentCandidateExecutionFailureClass, AgentCandidateExecutionLease, @@ -18,7 +22,9 @@ import { import { canonicalCandidateBytes } from './digest' import { candidatePostRunWindowMs, candidateTerminalWindowMs } from './execution-window' import { + type SealedAgentCandidateExecutorFinalCapture, sealAgentCandidateExecutorFinalCapture, + sealAgentCandidateExecutorStopAcknowledgement, sealAgentCandidateProtectedRunCapture, } from './executor-capture' import { failedAgentCandidateRun, finalizeAgentCandidateRun } from './finalize' @@ -31,7 +37,7 @@ import { type PersistedAgentCandidateModelSettlement, persistCandidateBenchmarkResult, persistCandidateModelSettlement, - persistVerifiedCandidateTaskOutcome, + persistVerifiedAgentCandidateExecutorCapture, } from './outcome-evidence' import { persistCandidateOutputArtifact } from './output-artifacts' import { @@ -48,7 +54,6 @@ import { redactProtectedReason } from './protected-redaction' import { ProtectedAgentCandidateTraceStore } from './protected-trace-store' import type { AgentCandidateBenchmarkGraderPort, - AgentCandidateExecutorFinalCapture, AgentCandidateExecutorPort, AgentCandidateExecutorRequest, AgentCandidateOutputArtifactPort, @@ -105,9 +110,30 @@ export async function executePreparedAgentCandidate( return await failBeforeActivation(prepared, state, error, 'failed', cleanupTimeoutMs) } + let preparationEvidence: AgentCandidateExecutionClaim['preparationEvidence'] + try { + const [executionPlan, materializationReceipt] = await Promise.all([ + persistCandidateOutputArtifact(options.outputArtifacts, { + executionId: state.executionId, + purpose: 'execution-plan', + bytes: state.executionPlan.bytes, + }), + persistCandidateOutputArtifact(options.outputArtifacts, { + executionId: state.executionId, + purpose: 'materialization-receipt', + bytes: state.materializationReceipt.bytes, + }), + ]) + preparationEvidence = Object.freeze({ executionPlan, materializationReceipt }) + } catch (error) { + return await failBeforeActivation(prepared, state, error, 'failed', cleanupTimeoutMs) + } + let acquired: Awaited> try { - acquired = await options.claimStore.tryClaim(candidateExecutionClaim(prepared)) + acquired = await options.claimStore.tryClaim( + candidateExecutionClaim(prepared, preparationEvidence), + ) } catch (error) { return await failBeforeActivation(prepared, state, error, 'failed', cleanupTimeoutMs) } @@ -277,6 +303,7 @@ export async function executePreparedAgentCandidate( const execution = await runAndStopExecutor( options.executor, request, + state.benchmarkTask.outcome, protectedTraceStore, deadlineAtMs, cleanupTimeoutMs, @@ -381,13 +408,14 @@ export async function executePreparedAgentCandidate( state.trace.runId, settlementResult.settlement as SealedAgentCandidateModelSettlement, ) - const taskOutcome = await persistVerifiedCandidateTaskOutcome( - state, - execution.finalCapture.taskOutcome!, - options.outputArtifacts, - protectedValues, - signal, - ) + const { persistedCapture, taskOutcome } = + await persistVerifiedAgentCandidateExecutorCapture( + state, + execution.finalCapture, + options.outputArtifacts, + protectedValues, + signal, + ) const benchmarkResult = await persistCandidateBenchmarkResult( state, capture.termination, @@ -404,6 +432,7 @@ export async function executePreparedAgentCandidate( settlementResult.settlement as SealedAgentCandidateModelSettlement, { finalCapture: execution.finalCapture, + persistedCapture, modelSettlement, taskOutcome, benchmarkResult, @@ -431,19 +460,17 @@ export async function executePreparedAgentCandidate( try { terminal = result.succeeded ? { - schemaVersion: 1, status: 'succeeded', - usage: settlementResult.settlement.fixedUsage, + usage: settlementResult.settlement.usage, modelSettlement: result.artifacts.modelSettlement, taskOutcome: result.artifacts.taskOutcome, benchmarkResult: result.artifacts.benchmarkResult, runReceipt: result.artifacts.runReceipt, } : { - schemaVersion: 1, status: 'failed', failureClass, - usage: settlementResult.settlement.fixedUsage, + usage: settlementResult.settlement.usage, modelSettlement: modelSettlement.artifact, failureEvidence: await withinCandidateCleanupDeadline( () => @@ -499,14 +526,14 @@ type ExecutorOutcome = kind: 'capture' capture: AgentCandidateProtectedRunCapture termination: AgentCandidateTermination - finalCapture: AgentCandidateExecutorFinalCapture + finalCapture: SealedAgentCandidateExecutorFinalCapture processStopped: true error?: undefined } | { kind: 'timeout' termination: AgentCandidateTermination & { kind: 'timeout' } - finalCapture: AgentCandidateExecutorFinalCapture + finalCapture: SealedAgentCandidateExecutorFinalCapture processStopped: true error?: undefined } @@ -514,13 +541,14 @@ type ExecutorOutcome = kind: 'error' error: unknown termination?: AgentCandidateTermination - finalCapture?: AgentCandidateExecutorFinalCapture + finalCapture?: SealedAgentCandidateExecutorFinalCapture processStopped: boolean } async function runAndStopExecutor( executor: AgentCandidateExecutorPort, request: AgentCandidateExecutorRequest, + expectedOutcome: AgentCandidateTaskOutcomeSpec, traceStore: TraceStore, deadlineAtMs: number, cleanupTimeoutMs: number, @@ -585,11 +613,11 @@ async function runAndStopExecutor( if (!capture || timedOut) controller.abort(executionError) const stopReason = timedOut ? 'timeout' : capture ? 'completed' : 'failed' - let stopped: unknown + const cleanupDeadlineAtMs = Date.now() + cleanupTimeoutMs try { - stopped = await withinCandidateCleanupDeadline( - () => - executor.stopAndCapture( + const stopped = await withinCandidateCleanupDeadline( + (cleanupSignal) => + executor.stop( { executionId: request.executionId, executionPlanDigest: request.executionPlan.value.digest, @@ -597,40 +625,68 @@ async function runAndStopExecutor( { traceStore, reason: stopReason, - signal: controller.signal, - deadlineAtMs, + signal: cleanupSignal, + deadlineAtMs: cleanupDeadlineAtMs, }, ), - Date.now() + cleanupTimeoutMs, + cleanupDeadlineAtMs, 'candidate process termination', ) - if ( - !stopped || - typeof stopped !== 'object' || - (stopped as { stopped?: unknown }).stopped !== true - ) { - throw new Error('candidate executor did not acknowledge exact process termination') - } + sealAgentCandidateExecutorStopAcknowledgement(stopped) } catch (stopError) { + let disposalError: unknown + try { + await disposeExecutor(executor, request, cleanupDeadlineAtMs) + } catch (error) { + disposalError = error + } if (timer) clearTimeout(timer) controller.abort(stopError) return { kind: 'error', - error: new Error(joinErrors(executionError, stopError)), + error: new Error(joinErrors(executionError, stopError, disposalError)), ...(timedOut ? { termination: { kind: 'timeout', timeoutMs } } : {}), processStopped: false, } } - let finalCapture: AgentCandidateExecutorFinalCapture + let finalCapture: SealedAgentCandidateExecutorFinalCapture | undefined + let finalCaptureError: unknown try { - finalCapture = sealAgentCandidateExecutorFinalCapture(stopped) + finalCapture = sealAgentCandidateExecutorFinalCapture( + await withinCandidateCleanupDeadline( + (cleanupSignal) => + executor.capture( + { + executionId: request.executionId, + executionPlanDigest: request.executionPlan.value.digest, + }, + { + traceStore, + signal: cleanupSignal, + }, + ), + cleanupDeadlineAtMs, + 'candidate final evidence capture', + ), + expectedOutcome, + ) } catch (captureError) { + finalCaptureError = captureError + } + + let disposalError: unknown + try { + await disposeExecutor(executor, request, cleanupDeadlineAtMs) + } catch (disposeError) { + disposalError = disposeError + } + if (finalCaptureError || disposalError || !finalCapture) { if (timer) clearTimeout(timer) - controller.abort(captureError) + controller.abort(finalCaptureError ?? disposalError) return { kind: 'error', - error: new Error(joinErrors(executionError, captureError)), + error: new Error(joinErrors(executionError, finalCaptureError, disposalError)), ...(timedOut ? { termination: { kind: 'timeout', timeoutMs } } : {}), processStopped: true, } @@ -669,6 +725,35 @@ async function runAndStopExecutor( } } +async function disposeExecutor( + executor: AgentCandidateExecutorPort, + request: AgentCandidateExecutorRequest, + cleanupDeadlineAtMs: number, +): Promise { + const dispose = executor.dispose + if (!dispose) return + const disposed = await withinCandidateCleanupDeadline( + (cleanupSignal) => + dispose.call( + executor, + { + executionId: request.executionId, + executionPlanDigest: request.executionPlan.value.digest, + }, + { signal: cleanupSignal }, + ), + cleanupDeadlineAtMs, + 'candidate execution resource disposal', + ) + if ( + !disposed || + disposed.disposed !== true || + Object.keys(disposed).some((key) => key !== 'disposed') + ) { + throw new Error('candidate executor did not acknowledge resource disposal') + } +} + async function failBeforeActivation( prepared: PreparedAgentCandidateExecution, state: PreparedCandidateState, @@ -744,10 +829,9 @@ async function failClaimedExecution( claimStore, lease, { - schemaVersion: 1, status: 'failed', failureClass, - usage: settled.settlement.fixedUsage, + usage: settled.settlement.usage, modelSettlement: modelSettlement.artifact, failureEvidence, }, @@ -876,7 +960,6 @@ async function persistFailureEvidence( outputArtifacts: AgentCandidateOutputArtifactPort, ) { const bytes = canonicalCandidateBytes({ - schemaVersion: 1, kind: 'agent-candidate-execution-failure', executionId: state.executionId, bundleDigest: state.bundle.digest, diff --git a/src/candidate-execution/executor-capture-evidence.ts b/src/candidate-execution/executor-capture-evidence.ts new file mode 100644 index 00000000..26c8dd91 --- /dev/null +++ b/src/candidate-execution/executor-capture-evidence.ts @@ -0,0 +1,167 @@ +import type { + AgentCandidateArtifactRef, + AgentCandidateTaskOutcomeSpec, + AgentCandidateTaskOutputSpec, + AgentCandidateWorkspaceManifestMaterial, + Sha256Digest, +} from '@tangle-network/agent-interface' + +import { canonicalCandidateBytes } from './digest' +import type { SealedAgentCandidateExecutorFinalCapture } from './executor-capture' +import { persistCandidateOutputArtifact } from './output-artifacts' +import type { AgentCandidateOutputArtifactPort } from './types' + +export type PersistedAgentCandidateTaskCapture = + | { + readonly kind: 'workspace' + readonly resultTree: string + readonly afterState: AgentCandidateWorkspaceManifestMaterial + readonly manifest: AgentCandidateArtifactRef + readonly archive: AgentCandidateArtifactRef + readonly gitDiff: AgentCandidateArtifactRef + } + | { + readonly kind: 'output' + readonly spec: AgentCandidateTaskOutputSpec + readonly artifact: AgentCandidateArtifactRef + } + +export interface PersistedAgentCandidateExecutorCapture { + readonly evidence: AgentCandidateArtifactRef + readonly taskOutcome?: PersistedAgentCandidateTaskCapture + readonly memoryAfter?: { + readonly afterState: AgentCandidateWorkspaceManifestMaterial + readonly manifest: AgentCandidateArtifactRef + readonly archive: AgentCandidateArtifactRef + } +} + +/** Persist structurally valid post-stop bytes without claiming they passed outcome verification. */ +export async function persistAgentCandidateExecutorCapture( + identity: { executionId: string; executionPlanDigest: Sha256Digest }, + expected: AgentCandidateTaskOutcomeSpec, + capture: SealedAgentCandidateExecutorFinalCapture, + outputArtifacts: AgentCandidateOutputArtifactPort, + signal?: AbortSignal, +): Promise { + const executorEvidence = capture.evidence + ? await persistCandidateOutputArtifact(outputArtifacts, { + executionId: identity.executionId, + purpose: 'executor-native-evidence', + bytes: capture.evidence, + signal, + }) + : undefined + const taskOutcome = capture.taskOutcome + ? await persistTaskCapture( + identity.executionId, + expected, + capture.taskOutcome, + outputArtifacts, + signal, + ) + : undefined + const memoryAfter = capture.memoryAfter + ? await persistWorkspaceCapture( + identity.executionId, + 'memory-after-manifest', + 'memory-after-archive', + capture.memoryAfter.afterState, + capture.memoryAfter.archive, + outputArtifacts, + signal, + ) + : undefined + const bytes = canonicalCandidateBytes({ + kind: 'agent-candidate-executor-capture', + executionPlanDigest: identity.executionPlanDigest, + ...(executorEvidence ? { executorEvidence } : {}), + ...(taskOutcome ? { taskOutcome } : {}), + ...(memoryAfter ? { memoryAfter } : {}), + }) + const evidence = await persistCandidateOutputArtifact(outputArtifacts, { + executionId: identity.executionId, + purpose: 'executor-capture', + bytes, + signal, + }) + return Object.freeze({ + evidence, + ...(taskOutcome ? { taskOutcome: Object.freeze(taskOutcome) } : {}), + ...(memoryAfter ? { memoryAfter: Object.freeze(memoryAfter) } : {}), + }) +} + +async function persistTaskCapture( + executionId: string, + expected: AgentCandidateTaskOutcomeSpec, + capture: NonNullable, + outputArtifacts: AgentCandidateOutputArtifactPort, + signal?: AbortSignal, +): Promise { + if (capture.kind === 'output') { + if (expected.kind !== 'output') + throw new Error('captured output does not match the signed task') + const artifact = await persistCandidateOutputArtifact(outputArtifacts, { + executionId, + purpose: 'task-output', + bytes: capture.bytes, + signal, + }) + return { + kind: 'output', + spec: { mediaType: expected.mediaType, maxBytes: expected.maxBytes }, + artifact, + } + } + if (expected.kind !== 'workspace') { + throw new Error('captured workspace does not match the signed task') + } + const workspace = await persistWorkspaceCapture( + executionId, + 'task-manifest', + 'task-archive', + capture.afterState, + capture.archive, + outputArtifacts, + signal, + ) + const gitDiff = await persistCandidateOutputArtifact(outputArtifacts, { + executionId, + purpose: 'task-patch', + bytes: capture.gitDiff, + signal, + }) + return { kind: 'workspace', resultTree: capture.resultTree, ...workspace, gitDiff } +} + +async function persistWorkspaceCapture( + executionId: string, + manifestPurpose: 'task-manifest' | 'memory-after-manifest', + archivePurpose: 'task-archive' | 'memory-after-archive', + afterState: AgentCandidateWorkspaceManifestMaterial, + archiveBytes: Uint8Array, + outputArtifacts: AgentCandidateOutputArtifactPort, + signal?: AbortSignal, +): Promise<{ + afterState: AgentCandidateWorkspaceManifestMaterial + manifest: AgentCandidateArtifactRef + archive: AgentCandidateArtifactRef +}> { + const manifestBytes = canonicalCandidateBytes(afterState) + const [manifest, archive] = await Promise.all([ + persistCandidateOutputArtifact(outputArtifacts, { + executionId, + purpose: manifestPurpose, + bytes: manifestBytes, + signal, + }), + persistCandidateOutputArtifact(outputArtifacts, { + executionId, + purpose: archivePurpose, + bytes: archiveBytes, + signal, + }), + ]) + return { afterState, manifest, archive } +} diff --git a/src/candidate-execution/executor-capture.ts b/src/candidate-execution/executor-capture.ts index df455e12..1ee9e2d9 100644 --- a/src/candidate-execution/executor-capture.ts +++ b/src/candidate-execution/executor-capture.ts @@ -1,15 +1,17 @@ import { - type AgentCandidateTaskOutcomeEvidence, - type AgentCandidateTermination, - type AgentCandidateWorkspaceSnapshotEvidence, + type AgentCandidateTaskOutcomeSpec, agentCandidateTerminationSchema, agentCandidateWorkspaceManifestMaterialSchema, - type Sha256Digest, } from '@tangle-network/agent-interface' -import { canonicalCandidateBytes } from './digest' import { assertExactObjectKeys as assertExactKeys } from './exact-object' import type { AgentCandidateExecutorFinalCapture, AgentCandidateProtectedRunCapture } from './types' +const sealedFinalCaptureBrand: unique symbol = Symbol('sealedAgentCandidateExecutorFinalCapture') + +export type SealedAgentCandidateExecutorFinalCapture = AgentCandidateExecutorFinalCapture & { + readonly [sealedFinalCaptureBrand]: true +} + /** Validate and detach the only candidate-authored fields accepted from execution. */ export function sealAgentCandidateProtectedRunCapture( value: unknown, @@ -28,40 +30,38 @@ export function sealAgentCandidateProtectedRunCapture( /** Validate, detach, and freeze evaluator-owned evidence captured after process death. */ export function sealAgentCandidateExecutorFinalCapture( value: unknown, -): AgentCandidateExecutorFinalCapture { + expectedOutcome: AgentCandidateTaskOutcomeSpec, +): SealedAgentCandidateExecutorFinalCapture { const capture = requireRecord(value, 'candidate final capture') - assertExactKeys(capture, ['stopped'], 'candidate final capture', ['taskOutcome', 'memoryAfter']) - if (capture.stopped !== true) { - throw new Error('candidate final capture does not prove process death') - } + assertExactKeys(capture, [], 'candidate final capture', [ + 'taskOutcome', + 'memoryAfter', + 'evidence', + ]) - const taskOutcome = capture.taskOutcome ? sealTaskOutcomeCapture(capture.taskOutcome) : undefined + const taskOutcome = capture.taskOutcome + ? sealTaskOutcomeCapture(capture.taskOutcome, expectedOutcome) + : undefined const memoryAfter = capture.memoryAfter ? sealMemoryCapture(capture.memoryAfter) : undefined + if (capture.evidence !== undefined && !(capture.evidence instanceof Uint8Array)) { + throw new Error('candidate executor evidence must be a byte array') + } + const evidence = capture.evidence ? Uint8Array.from(capture.evidence) : undefined return Object.freeze({ - stopped: true, ...(taskOutcome ? { taskOutcome } : {}), ...(memoryAfter ? { memoryAfter: Object.freeze(memoryAfter) } : {}), + ...(evidence ? { evidence } : {}), + [sealedFinalCaptureBrand]: true as const, }) } -/** Encode the complete evaluator-owned process result as one durable artifact. */ -export function encodeAgentCandidateExecutorCapture(input: { - executionId: string - executionPlanDigest: Sha256Digest - termination: AgentCandidateTermination - taskOutcome: AgentCandidateTaskOutcomeEvidence - memoryAfter?: AgentCandidateWorkspaceSnapshotEvidence -}): Uint8Array { - return canonicalCandidateBytes({ - schemaVersion: 1, - kind: 'agent-candidate-executor-capture', - executionId: input.executionId, - executionPlanDigest: input.executionPlanDigest, - termination: input.termination, - stopped: true, - taskOutcome: input.taskOutcome, - ...(input.memoryAfter ? { memoryAfter: input.memoryAfter } : {}), - }) +/** Prove exact process death before any final evidence capture. */ +export function sealAgentCandidateExecutorStopAcknowledgement(value: unknown): void { + const capture = requireRecord(value, 'candidate stop acknowledgement') + assertExactKeys(capture, ['stopped'], 'candidate stop acknowledgement') + if (capture.stopped !== true) { + throw new Error('candidate stop acknowledgement does not prove process death') + } } function sealMemoryCapture( @@ -82,13 +82,33 @@ function sealMemoryCapture( function sealTaskOutcomeCapture( value: unknown, + expected: AgentCandidateTaskOutcomeSpec, ): NonNullable { const capture = requireRecord(value, 'candidate task capture') + if (capture.kind === 'output') { + assertExactKeys(capture, ['kind', 'bytes'], 'candidate task output capture') + if (expected.kind !== 'output') { + throw new Error('candidate task output capture does not match the signed outcome kind') + } + if (!(capture.bytes instanceof Uint8Array)) { + throw new Error('candidate task output capture must contain a byte array') + } + if (capture.bytes.byteLength > expected.maxBytes) { + throw new Error(`candidate task output exceeds the frozen ${expected.maxBytes}-byte maximum`) + } + return Object.freeze({ kind: 'output', bytes: Uint8Array.from(capture.bytes) }) + } assertExactKeys( capture, - ['resultTree', 'afterState', 'archive', 'gitDiff'], + ['kind', 'resultTree', 'afterState', 'archive', 'gitDiff'], 'candidate task capture', ) + if (capture.kind !== 'workspace') { + throw new Error('candidate task capture has an invalid outcome kind') + } + if (expected.kind !== 'workspace') { + throw new Error('candidate workspace capture does not match the signed outcome kind') + } if ( typeof capture.resultTree !== 'string' || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(capture.resultTree) @@ -100,6 +120,7 @@ function sealTaskOutcomeCapture( } const afterState = agentCandidateWorkspaceManifestMaterialSchema.parse(capture.afterState) return Object.freeze({ + kind: 'workspace', resultTree: capture.resultTree, afterState: Object.freeze(afterState), archive: Uint8Array.from(capture.archive), diff --git a/src/candidate-execution/finalize.ts b/src/candidate-execution/finalize.ts index 02571a02..f517dcaa 100644 --- a/src/candidate-execution/finalize.ts +++ b/src/candidate-execution/finalize.ts @@ -1,26 +1,21 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' - import type { Span, TraceStore } from '@tangle-network/agent-eval' import { isLlmSpan, REDACTION_VERSION } from '@tangle-network/agent-eval' import type { AgentCandidateBenchmarkResultEvidence, + AgentCandidateFixedSpend, AgentCandidateMemoryReceipt, AgentCandidateModelSettlementEvidence, AgentCandidateRunReceipt, - AgentCandidateSpend, AgentCandidateTermination, } from '@tangle-network/agent-interface' -import { agentCandidateRunReceiptSchema } from '@tangle-network/agent-interface' - -import { readMaterializedWorkspaceFiles } from './artifacts' -import { canonicalCandidateBytes, canonicalCandidateDocument } from './digest' import { - encodeAgentCandidateExecutorCapture, - sealAgentCandidateExecutorFinalCapture, - sealAgentCandidateProtectedRunCapture, -} from './executor-capture' + agentCandidateRunReceiptSchema, + agentCandidateWorkspaceSnapshotEvidenceSchema, +} from '@tangle-network/agent-interface' + +import { canonicalCandidateBytes, canonicalCandidateDocument, sha256Bytes } from './digest' +import type { SealedAgentCandidateExecutorFinalCapture } from './executor-capture' +import type { PersistedAgentCandidateExecutorCapture } from './executor-capture-evidence' import { assertTraceMatchesModelSettlement, type SealedAgentCandidateModelSettlement, @@ -35,20 +30,16 @@ import { redactProtectedValue, } from './protected-redaction' import { - type AgentCandidateExecutorFinalCapture, type AgentCandidateOutputArtifactPort, type AgentCandidateProtectedRunCapture, type AgentCandidateRunFinalization, CANDIDATE_TRACE_TAGS, type VerifiedAgentCandidateTaskOutcome, } from './types' -import { - persistCandidateWorkspaceSnapshot, - provisionalCandidateWorkspaceSnapshot, -} from './workspace-snapshot' interface CandidateFinalizationEvidence { - finalCapture: AgentCandidateExecutorFinalCapture + finalCapture: SealedAgentCandidateExecutorFinalCapture + persistedCapture: PersistedAgentCandidateExecutorCapture modelSettlement: AgentCandidateModelSettlementEvidence & { artifact: import('@tangle-network/agent-interface').AgentCandidateArtifactRef } @@ -73,11 +64,10 @@ export async function finalizeAgentCandidateRun( let termination: AgentCandidateTermination | undefined try { signal?.throwIfAborted() - const protectedCapture = sealAgentCandidateProtectedRunCapture(capture) - if (protectedCapture.executionId !== state.executionId) { + if (capture.executionId !== state.executionId) { throw new Error('protected capture execution id does not match the prepared execution') } - termination = protectedCapture.termination + termination = capture.termination if ( termination.kind === 'timeout' && termination.timeoutMs !== state.executionPlan.value.material.limits.timeoutMs @@ -114,32 +104,14 @@ export async function finalizeAgentCandidateRun( const modelSpans = orderedSpans.filter(isLlmSpan) assertTraceMatchesModelSettlement(modelSpans, settlement) enforceLimits(state, run.startedAt, run.endedAt, orderedSpans, settlement) - const finalCapture = sealAgentCandidateExecutorFinalCapture(evidence.finalCapture) const memory = await memoryReceipt( state, - finalCapture, - evidence.outputArtifacts, - protectedValues, + evidence.finalCapture, + evidence.persistedCapture.memoryAfter, signal, ) - const executorCaptureBytes = encodeAgentCandidateExecutorCapture({ - executionId: protectedCapture.executionId, - executionPlanDigest: state.executionPlan.value.digest, - termination: protectedCapture.termination, - taskOutcome: evidence.taskOutcome.evidence, - ...(memory.mode === 'isolated' ? { memoryAfter: memory.afterState } : {}), - }) - assertNoProtectedBytes(executorCaptureBytes, protectedValues) - const executorCapture = await persistCandidateOutputArtifact(evidence.outputArtifacts, { - executionId: state.executionId, - purpose: 'executor-capture', - bytes: executorCaptureBytes, - signal, - }) - const redacted = redactProtectedValue( { - schemaVersion: 1, run: { ...run, redactionVersion: REDACTION_VERSION }, spans: orderedSpans, events: orderedEvents, @@ -171,7 +143,6 @@ export async function finalizeAgentCandidateRun( signal, }) const trace = { - schemaVersion: 1 as const, artifact: traceArtifact, eventCount: 1 + @@ -182,16 +153,21 @@ export async function finalizeAgentCandidateRun( modelCallCount: modelSpans.length, } const document = canonicalCandidateDocument({ - schemaVersion: 3, kind: 'agent-candidate-run', digestAlgorithm: 'rfc8785-sha256', bundleDigest: state.bundle.digest, + runCellDigest: state.executionPlan.value.material.runCell.digest, materializationReceiptDigest: state.materializationReceipt.digest, executionPlanDigest: state.executionPlan.value.digest, + timing: { + startedAtMs: run.startedAt, + endedAtMs: run.endedAt, + durationMs: run.endedAt - run.startedAt, + }, memory, trace, termination, - executorCapture, + executorCapture: evidence.persistedCapture.evidence, modelSettlement: evidence.modelSettlement, taskOutcome: evidence.taskOutcome.evidence, benchmarkResult: evidence.benchmarkResult, @@ -207,8 +183,8 @@ export async function finalizeAgentCandidateRun( succeeded: true, receipt: document, artifacts: { - executorCapture, modelSettlement: evidence.modelSettlement.artifact, + executorCapture: evidence.persistedCapture.evidence, taskOutcome: evidence.taskOutcome.evidence.artifact, benchmarkResult: evidence.benchmarkResult.artifact, runReceipt, @@ -267,52 +243,36 @@ function enforceLimits( for (const [actual, limit, label] of checks) { if (actual > limit) throw new Error(`protected ${label} ${actual} exceeds ${limit}`) } - if (settlement.costUsdNanos > usdToNanos(limits.maxCostUsd, 'frozen maxCostUsd')) { - throw new Error(`protected cost USD ${usage.costUsd} exceeds ${limits.maxCostUsd}`) + if (usage.costUsdNanos > usdToNanos(limits.maxCostUsd, 'frozen maxCostUsd')) { + throw new Error( + `protected cost USD ${usage.costUsdNanos / 1_000_000_000} exceeds ${limits.maxCostUsd}`, + ) } } async function memoryReceipt( state: PreparedCandidateState, - capture: AgentCandidateExecutorFinalCapture, - outputArtifacts: AgentCandidateOutputArtifactPort, - protectedValues: readonly string[], + capture: SealedAgentCandidateExecutorFinalCapture, + persisted: PersistedAgentCandidateExecutorCapture['memoryAfter'], signal?: AbortSignal, ): Promise { signal?.throwIfAborted() if (state.memory.mode === 'disabled') { - if (capture.memoryAfter !== undefined) { + if (capture.memoryAfter !== undefined || persisted !== undefined) { throw new Error('disabled memory cannot return an after-state') } return { mode: 'disabled' } } if (!capture.memoryAfter) throw new Error('isolated memory is missing its protected after-state') - const archive = Uint8Array.from(capture.memoryAfter.archive) - if (archive.byteLength === 0) throw new Error('isolated memory archive cannot be empty') - const provisional = provisionalCandidateWorkspaceSnapshot(capture.memoryAfter.afterState, archive) - const afterState = provisional.material - assertNoProtectedBytes(provisional.manifestBytes, protectedValues) - assertNoProtectedBytes(archive, protectedValues) - const root = await mkdtemp(join(tmpdir(), 'agent-candidate-memory-after-')) - try { - await state.ports.workspaces.materialize({ - role: 'memory', - snapshot: provisional.snapshot, - archive: Uint8Array.from(archive), - destination: root, - }) - const files = await readMaterializedWorkspaceFiles(root, afterState) - for (const file of files) assertNoProtectedBytes(file.bytes, protectedValues) - signal?.throwIfAborted() - } finally { - await rm(root, { recursive: true, force: true }) - } - const snapshot = await persistCandidateWorkspaceSnapshot(outputArtifacts, { - executionId: state.executionId, + if (!persisted) throw new Error('isolated memory is missing persisted capture evidence') + const afterState = capture.memoryAfter.afterState + const manifestBytes = canonicalCandidateBytes(afterState) + const snapshot = agentCandidateWorkspaceSnapshotEvidenceSchema.parse({ + kind: 'agent-candidate-workspace-snapshot', + digest: sha256Bytes(manifestBytes), material: afterState, - archive, - purpose: 'memory-after', - signal, + manifest: persisted.manifest, + archive: persisted.archive, }) return { mode: 'isolated', @@ -328,7 +288,7 @@ export function failedAgentCandidateRun( state: PreparedCandidateState, reason: string, termination?: AgentCandidateTermination, - usage: AgentCandidateSpend | null = null, + usage: AgentCandidateFixedSpend | null = null, ): AgentCandidateRunFinalization & { succeeded: false } { return { succeeded: false, diff --git a/src/candidate-execution/index.ts b/src/candidate-execution/index.ts index 6f3e8701..a75b7185 100644 --- a/src/candidate-execution/index.ts +++ b/src/candidate-execution/index.ts @@ -25,10 +25,10 @@ export { type AgentCandidateExecutionStageResult, type AgentCandidateExecutionTerminalRecord, type AgentCandidateExecutionTerminalResult, - type AgentCandidateExecutionUsage, type AgentCandidateRetryRejection, InMemoryAgentCandidateExecutionClaimStore, } from './claim' +export type { AgentCandidatePreparationEvidence } from './claim-file-formats' export { FileAgentCandidateExecutionClaimStore, type FileAgentCandidateExecutionClaimStoreOptions, @@ -42,6 +42,11 @@ export { type ExecutePreparedAgentCandidateOptions, executePreparedAgentCandidate, } from './execute' +export { + CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV, + CANDIDATE_KNOWLEDGE_ROOT_ENV, + candidateKnowledgeExecutionPaths, +} from './knowledge' export { persistCandidateOutputArtifact } from './output-artifacts' export { type PrepareAgentCandidateExecutionOptions, @@ -87,7 +92,6 @@ export { type AgentCandidateOutputArtifactPort, type AgentCandidateOutputPurpose, type AgentCandidateProtectedModelActivation, - type AgentCandidateProtectedModelCall, type AgentCandidateProtectedModelReservation, type AgentCandidateProtectedModelSettlement, type AgentCandidateProtectedRunCapture, @@ -108,7 +112,7 @@ export { type VerifiedAgentCandidate, type VerifiedAgentCandidateTaskOutcome, } from './types' -export { verifyAgentCandidateBundle } from './verify' +export { AGENT_CANDIDATE_EXECUTION_SUPPORT, verifyAgentCandidateBundle } from './verify' export { type AgentCandidateWorkspaceArchiveLimits, type CaptureAgentCandidateWorkspaceOptions, diff --git a/src/candidate-execution/knowledge.ts b/src/candidate-execution/knowledge.ts index 90ca254a..83ab3220 100644 --- a/src/candidate-execution/knowledge.ts +++ b/src/candidate-execution/knowledge.ts @@ -1,6 +1,6 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { join, posix } from 'node:path' import { readMaterializedWorkspaceFiles } from './artifacts' import type { @@ -10,6 +10,29 @@ import type { } from './types' import { verifiedArtifactBytes } from './verify' +/** Environment variable containing the materialized candidate knowledge root. */ +export const CANDIDATE_KNOWLEDGE_ROOT_ENV = 'TANGLE_CANDIDATE_KNOWLEDGE_ROOT' +/** Environment variable containing the materialized retrieval configuration path. */ +export const CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV = + 'TANGLE_CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG' + +/** Deterministic, signed locations used by every candidate executor. */ +export function candidateKnowledgeExecutionPaths( + taskRoot: string, + hasRetrievalConfig: boolean, +): { root: string; retrievalConfig?: string } { + if (!posix.isAbsolute(taskRoot) || posix.normalize(taskRoot) !== taskRoot) { + throw new Error('candidate knowledge requires a normalized absolute task root') + } + const parent = posix.join(taskRoot, '.tangle') + return { + root: posix.join(parent, 'knowledge'), + ...(hasRetrievalConfig + ? { retrievalConfig: posix.join(parent, 'knowledge-retrieval-config.json') } + : {}), + } +} + /** Verify and detach the exact file-backed knowledge admitted by the bundle. */ export async function prepareAgentCandidateKnowledge( candidate: VerifiedAgentCandidate, diff --git a/src/candidate-execution/model-settlement.ts b/src/candidate-execution/model-settlement.ts index e188f4d2..5ffba8ce 100644 --- a/src/candidate-execution/model-settlement.ts +++ b/src/candidate-execution/model-settlement.ts @@ -1,19 +1,16 @@ import { isLlmSpan, type LlmSpan, type TraceStore } from '@tangle-network/agent-eval' -import type { AgentCandidateSpend } from '@tangle-network/agent-interface' -import type { AgentCandidateExecutionUsage } from './claim' -import { assertExactObjectKeys } from './exact-object' import type { - AgentCandidateProtectedModelCall, - AgentCandidateProtectedModelSettlement, -} from './types' + AgentCandidateFixedSpend, + AgentCandidateModelSettlementCall, +} from '@tangle-network/agent-interface' +import { assertExactObjectKeys } from './exact-object' +import type { AgentCandidateProtectedModelSettlement } from './types' const USD_NANOS = 1_000_000_000 export interface SealedAgentCandidateModelSettlement { readonly value: AgentCandidateProtectedModelSettlement - readonly usage: AgentCandidateSpend - readonly fixedUsage: AgentCandidateExecutionUsage - readonly costUsdNanos: number + readonly usage: AgentCandidateFixedSpend } /** Validate and detach the evaluator gateway's terminal, revoked call ledger. */ @@ -43,7 +40,6 @@ export function sealAgentCandidateModelSettlement( let outputTokens = 0 let cachedInputTokens = 0 let reasoningTokens = 0 - let hasCachedInput = false let costUsdNanos = 0 const calls = settlement.calls.map((source, index) => { assertExactObjectKeys( @@ -96,7 +92,6 @@ export function sealAgentCandidateModelSettlement( source.cachedInputTokens, 'cached input token total', ) - hasCachedInput = true assertCount(source.reasoningTokens, `model settlement call ${index} reasoningTokens`) reasoningTokens = safeAdd(reasoningTokens, source.reasoningTokens, 'reasoning token total') assertCount(source.costUsdNanos, `model settlement call ${index} costUsdNanos`) @@ -107,13 +102,6 @@ export function sealAgentCandidateModelSettlement( }) const usage = Object.freeze({ - costUsd: costUsdNanos / USD_NANOS, - inputTokens, - outputTokens, - ...(hasCachedInput ? { cachedInputTokens } : {}), - modelCalls: calls.length, - }) - const fixedUsage = Object.freeze({ costUsdNanos, inputTokens, outputTokens, @@ -129,8 +117,6 @@ export function sealAgentCandidateModelSettlement( calls: Object.freeze(calls), }), usage, - fixedUsage, - costUsdNanos, }) } @@ -213,7 +199,7 @@ export function usdToNanos(value: number, label: string): number { return nanos } -function assertTraceCall(span: LlmSpan, call: AgentCandidateProtectedModelCall): void { +function assertTraceCall(span: LlmSpan, call: AgentCandidateModelSettlementCall): void { if (span.model !== call.model) { throw new Error(`protected trace span ${span.spanId} model does not match model ledger`) } diff --git a/src/candidate-execution/outcome-evidence.ts b/src/candidate-execution/outcome-evidence.ts index e2ac338d..d9066f88 100644 --- a/src/candidate-execution/outcome-evidence.ts +++ b/src/candidate-execution/outcome-evidence.ts @@ -9,17 +9,33 @@ import { type AgentCandidateModelSettlementEvidence, type AgentCandidateResolvedModel, type AgentCandidateTaskOutcomeEvidence, + type AgentCandidateTaskOutcomeMaterial, + type AgentCandidateTaskOutcomeSpec, + type AgentCandidateTaskOutputSpec, + type AgentCandidateTaskRepository, type AgentCandidateTermination, type AgentCandidateWorkspaceSnapshotEvidence, agentCandidateBenchmarkResultEvidenceSchema, agentCandidateModelSettlementEvidenceSchema, agentCandidateTaskOutcomeEvidenceSchema, + agentCandidateWorkspaceSnapshotEvidenceSchema, type Sha256Digest, } from '@tangle-network/agent-interface' import { readMaterializedWorkspaceFiles } from './artifacts' import { runBoundCandidateBenchmarkGrader } from './benchmark-grader' -import { canonicalCandidateBytes, immutableCandidateValue, sha256Bytes } from './digest' +import { + canonicalCandidateBytes, + embeddedCandidateArtifact, + immutableCandidateValue, + sha256Bytes, +} from './digest' +import type { SealedAgentCandidateExecutorFinalCapture } from './executor-capture' +import { + type PersistedAgentCandidateExecutorCapture, + type PersistedAgentCandidateTaskCapture, + persistAgentCandidateExecutorCapture, +} from './executor-capture-evidence' import { verifyTaskOutcomePatch } from './git-materialize' import type { SealedAgentCandidateModelSettlement } from './model-settlement' import { persistCandidateOutputArtifact } from './output-artifacts' @@ -32,11 +48,6 @@ import type { VerifiedAgentCandidateTaskOutcome, } from './types' import { verifiedTaskOutcomeBrand } from './types' -import { - persistCandidateWorkspaceSnapshot, - provisionalCandidateWorkspaceSnapshot, -} from './workspace-snapshot' -import { workspaceTaskRepository } from './workspace-task' export type PersistedAgentCandidateModelSettlement = AgentCandidateModelSettlementEvidence & { artifact: AgentCandidateArtifactRef @@ -46,7 +57,7 @@ export type PersistedAgentCandidateBenchmarkResult = AgentCandidateBenchmarkResu artifact: AgentCandidateArtifactRef } -/** Persist the closed evaluator model ledger as canonical V2 receipt evidence. */ +/** Persist the closed evaluator model ledger as canonical receipt evidence. */ export async function persistCandidateModelSettlement( state: PreparedCandidateState, settlement: SealedAgentCandidateModelSettlement, @@ -74,7 +85,6 @@ export async function persistCandidateModelSettlementEvidence( outputArtifacts: AgentCandidateOutputArtifactPort, ): Promise { const material = { - schemaVersion: 2 as const, kind: 'agent-candidate-model-settlement-material' as const, executionPlanDigest: identity.executionPlanDigest, preparationId: settlement.value.preparationId, @@ -95,7 +105,7 @@ export async function persistCandidateModelSettlementEvidence( reasoningTokens: call.reasoningTokens ?? 0, costUsdNanos: call.costUsdNanos, })), - usage: settlement.fixedUsage, + usage: settlement.usage, } const bytes = canonicalCandidateBytes(material) const digest = sha256Bytes(bytes) @@ -106,7 +116,6 @@ export async function persistCandidateModelSettlementEvidence( }) return immutableCandidateValue( agentCandidateModelSettlementEvidenceSchema.parse({ - schemaVersion: 2, kind: 'agent-candidate-model-settlement', digest, material, @@ -115,23 +124,94 @@ export async function persistCandidateModelSettlementEvidence( ) as PersistedAgentCandidateModelSettlement } -/** Recompute the result tree from the patch, then persist its exact task evidence. */ -export async function persistVerifiedCandidateTaskOutcome( +/** Verify once, persist the sealed capture once, and bind the resulting task evidence. */ +export async function persistVerifiedAgentCandidateExecutorCapture( state: PreparedCandidateState, - capture: AgentCandidateExecutorTaskOutcomeCapture, + capture: SealedAgentCandidateExecutorFinalCapture, outputArtifacts: AgentCandidateOutputArtifactPort, protectedValues: readonly string[], signal?: AbortSignal, -): Promise { +): Promise<{ + persistedCapture: PersistedAgentCandidateExecutorCapture + taskOutcome: VerifiedAgentCandidateTaskOutcome +}> { signal?.throwIfAborted() + const taskCapture = capture.taskOutcome + if (!taskCapture) throw new Error('candidate final capture is missing the task outcome') + if (capture.evidence) assertNoProtectedBytes(capture.evidence, protectedValues) + const expected = state.benchmarkTask.outcome + if (taskCapture.kind !== expected.kind) { + throw new Error( + `candidate captured ${taskCapture.kind} outcome does not match expected ${expected.kind} outcome`, + ) + } + const verified = + taskCapture.kind === 'output' && expected.kind === 'output' + ? verifyOutputTaskCapture(taskCapture.bytes, expected, protectedValues) + : taskCapture.kind === 'workspace' && expected.kind === 'workspace' + ? await verifyWorkspaceTaskCapture(state, taskCapture, protectedValues, signal) + : undefined + if (!verified) throw new Error('candidate task outcome kind could not be narrowed') + await verifyMemoryCapture(state, capture, protectedValues, signal) + + const persistedCapture = await persistAgentCandidateExecutorCapture( + { + executionId: state.executionId, + executionPlanDigest: state.executionPlan.value.digest, + }, + expected, + capture, + outputArtifacts, + signal, + ) + const persisted = persistedCapture.taskOutcome + if (!persisted || persisted.kind !== verified.kind) { + throw new Error('persisted task capture kind changed') + } + const taskOutcome = + verified.kind === 'output' && persisted.kind === 'output' + ? await persistVerifiedOutputTaskOutcome(state, verified, persisted, outputArtifacts, signal) + : verified.kind === 'workspace' && persisted.kind === 'workspace' + ? await persistVerifiedWorkspaceTaskOutcome( + state, + verified, + persisted, + outputArtifacts, + signal, + ) + : undefined + if (!taskOutcome) throw new Error('persisted task outcome kind could not be narrowed') + return Object.freeze({ persistedCapture, taskOutcome }) +} + +interface VerifiedWorkspaceTaskCapture { + readonly kind: 'workspace' + readonly patch: Uint8Array + readonly archive: Uint8Array + readonly afterState: Extract< + AgentCandidateExecutorTaskOutcomeCapture, + { kind: 'workspace' } + >['afterState'] + readonly manifestBytes: Uint8Array + readonly resultCommit: string + readonly resultTree: string + readonly repository: AgentCandidateTaskRepository +} + +async function verifyWorkspaceTaskCapture( + state: PreparedCandidateState, + capture: Extract, + protectedValues: readonly string[], + signal?: AbortSignal, +): Promise { + const repository = state.benchmarkTask.repository + if (!repository) throw new Error('workspace task outcome is missing repository identity') const patch = Uint8Array.from(capture.gitDiff) const archive = Uint8Array.from(capture.archive) if (archive.byteLength === 0) throw new Error('candidate task archive cannot be empty') assertNoProtectedBytes(patch, protectedValues) assertNoProtectedBytes(archive, protectedValues) - const provisional = provisionalCandidateWorkspaceSnapshot(capture.afterState, archive) - const afterState = provisional.material - const repository = workspaceTaskRepository(state.executionPlan.value.material.task) + const afterState = immutableCandidateValue(capture.afterState) const verified = await verifyTaskOutcomePatch({ repositoryRoot: state.roots.staging.taskRoot, baseCommit: repository.baseCommit, @@ -141,24 +221,45 @@ export async function persistVerifiedCandidateTaskOutcome( afterState, }) signal?.throwIfAborted() - assertNoProtectedBytes(provisional.manifestBytes, protectedValues) - await verifyTaskOutcomeArchive(state, provisional.snapshot, archive, protectedValues) - signal?.throwIfAborted() - const snapshot = await persistCandidateWorkspaceSnapshot(outputArtifacts, { - executionId: state.executionId, + const manifestBytes = canonicalCandidateBytes(afterState) + assertNoProtectedBytes(manifestBytes, protectedValues) + const provisionalSnapshot = agentCandidateWorkspaceSnapshotEvidenceSchema.parse({ + kind: 'agent-candidate-workspace-snapshot', + digest: sha256Bytes(manifestBytes), material: afterState, - archive, - purpose: 'task', - signal, + manifest: embeddedCandidateArtifact(manifestBytes), + archive: embeddedCandidateArtifact(archive), }) - const gitDiff = await persistCandidateOutputArtifact(outputArtifacts, { - executionId: state.executionId, - purpose: 'task-patch', - bytes: patch, - signal, + await verifyTaskOutcomeArchive(state, provisionalSnapshot, archive, protectedValues) + signal?.throwIfAborted() + return { + kind: 'workspace', + patch, + archive, + afterState, + manifestBytes, + resultCommit: verified.resultCommit, + resultTree: verified.resultTree, + repository, + } +} + +async function persistVerifiedWorkspaceTaskOutcome( + state: PreparedCandidateState, + verified: VerifiedWorkspaceTaskCapture, + persisted: Extract, + outputArtifacts: AgentCandidateOutputArtifactPort, + signal?: AbortSignal, +): Promise { + const { afterState, manifestBytes, patch, repository } = verified + const snapshot = agentCandidateWorkspaceSnapshotEvidenceSchema.parse({ + kind: 'agent-candidate-workspace-snapshot', + digest: sha256Bytes(manifestBytes), + material: afterState, + manifest: persisted.manifest, + archive: persisted.archive, }) const material = { - schemaVersion: 2 as const, kind: 'agent-candidate-task-outcome-material' as const, executionPlanDigest: state.executionPlan.value.digest, outcome: { @@ -178,10 +279,92 @@ export async function persistVerifiedCandidateTaskOutcome( afterState: snapshot, gitDiff: { format: 'git-diff-binary' as const, - artifact: gitDiff, + artifact: persisted.gitDiff, }, }, + } satisfies AgentCandidateTaskOutcomeMaterial + const evidence = await persistTaskOutcomeEvidence(state, material, outputArtifacts, signal) + const storedPatch = Uint8Array.from(patch) + return Object.freeze({ + kind: 'workspace' as const, + evidence, + get patch(): Uint8Array { + return Uint8Array.from(storedPatch) + }, + [verifiedTaskOutcomeBrand]: true as const, + }) +} + +interface VerifiedOutputTaskCapture { + readonly kind: 'output' + readonly bytes: Uint8Array + readonly spec: AgentCandidateTaskOutputSpec +} + +function verifyOutputTaskCapture( + capturedBytes: Uint8Array, + expected: Extract, + protectedValues: readonly string[], +): VerifiedOutputTaskCapture { + if (capturedBytes.byteLength === 0) throw new Error('candidate task output cannot be empty') + if (capturedBytes.byteLength > expected.maxBytes) { + throw new Error(`candidate task output exceeds the frozen ${expected.maxBytes}-byte maximum`) } + const output = Uint8Array.from(capturedBytes) + assertNoProtectedBytes(output, protectedValues) + return { + kind: 'output', + bytes: output, + spec: { mediaType: expected.mediaType, maxBytes: expected.maxBytes }, + } +} + +async function persistVerifiedOutputTaskOutcome( + state: PreparedCandidateState, + verified: VerifiedOutputTaskCapture, + persisted: Extract, + outputArtifacts: AgentCandidateOutputArtifactPort, + signal?: AbortSignal, +): Promise { + if ( + persisted.spec.mediaType !== verified.spec.mediaType || + persisted.spec.maxBytes !== verified.spec.maxBytes + ) { + throw new Error('persisted task output changed the signed media constraints') + } + const material = { + kind: 'agent-candidate-task-outcome-material' as const, + executionPlanDigest: state.executionPlan.value.digest, + outcome: { + kind: 'output' as const, + spec: verified.spec, + artifact: persisted.artifact, + }, + } satisfies AgentCandidateTaskOutcomeMaterial + const evidence = await persistTaskOutcomeEvidence(state, material, outputArtifacts, signal) + const storedOutput = Uint8Array.from(verified.bytes) + return Object.freeze({ + kind: 'output' as const, + evidence, + spec: immutableCandidateValue(verified.spec), + get bytes(): Uint8Array { + return Uint8Array.from(storedOutput) + }, + [verifiedTaskOutcomeBrand]: true as const, + }) +} + +async function persistTaskOutcomeEvidence( + state: PreparedCandidateState, + material: Material, + outputArtifacts: AgentCandidateOutputArtifactPort, + signal?: AbortSignal, +): Promise< + Omit & { + material: Material + artifact: AgentCandidateArtifactRef + } +> { const bytes = canonicalCandidateBytes(material) const digest = sha256Bytes(bytes) const artifact = await persistCandidateOutputArtifact(outputArtifacts, { @@ -190,23 +373,17 @@ export async function persistVerifiedCandidateTaskOutcome( bytes, signal, }) - const evidence = immutableCandidateValue( + return immutableCandidateValue( agentCandidateTaskOutcomeEvidenceSchema.parse({ - schemaVersion: 2, kind: 'agent-candidate-task-outcome', digest, material, artifact, }), - ) as AgentCandidateTaskOutcomeEvidence & { artifact: AgentCandidateArtifactRef } - const storedPatch = Uint8Array.from(patch) - return Object.freeze({ - evidence, - get patch(): Uint8Array { - return Uint8Array.from(storedPatch) - }, - [verifiedTaskOutcomeBrand]: true as const, - }) + ) as Omit & { + material: Material + artifact: AgentCandidateArtifactRef + } } /** Grade only a runtime-verified outcome and persist both raw and normalized evidence. */ @@ -225,6 +402,7 @@ export async function persistCandidateBenchmarkResult( executionId: state.executionId, termination: frozenTermination, outcome, + expectedGrader: state.benchmarkTask.grader, grader, artifacts: outputArtifacts, signal, @@ -240,24 +418,13 @@ export async function persistCandidateBenchmarkResult( bytes: rawEvidence, signal, }) - const task = state.executionPlan.value.material.task const material = { - schemaVersion: 1 as const, kind: 'agent-candidate-benchmark-result-material' as const, executionPlanDigest: state.executionPlan.value.digest, taskOutcomeDigest: outcome.evidence.digest, - benchmark: { - name: task.benchmark, - version: task.benchmarkVersion, - taskId: task.taskId, - splitDigest: task.splitDigest, - }, - grader: { - name: graded.grader.name, - version: graded.grader.version, - artifact: graded.grader.artifact, - }, + grader: graded.grader, evidence: evidenceRef, + grading: graded.grading, score: evaluation.score, passed: evaluation.passed, dimensions: evaluation.dimensions, @@ -272,7 +439,6 @@ export async function persistCandidateBenchmarkResult( }) return immutableCandidateValue( agentCandidateBenchmarkResultEvidenceSchema.parse({ - schemaVersion: 1, kind: 'agent-candidate-benchmark-result', digest, material, @@ -302,6 +468,45 @@ async function verifyTaskOutcomeArchive( } } +async function verifyMemoryCapture( + state: PreparedCandidateState, + capture: SealedAgentCandidateExecutorFinalCapture, + protectedValues: readonly string[], + signal?: AbortSignal, +): Promise { + if (state.memory.mode === 'disabled') { + if (capture.memoryAfter) throw new Error('disabled memory cannot return an after-state') + return + } + const memory = capture.memoryAfter + if (!memory) throw new Error('isolated memory is missing its protected after-state') + if (memory.archive.byteLength === 0) throw new Error('isolated memory archive cannot be empty') + const manifestBytes = canonicalCandidateBytes(memory.afterState) + assertNoProtectedBytes(manifestBytes, protectedValues) + assertNoProtectedBytes(memory.archive, protectedValues) + const snapshot = agentCandidateWorkspaceSnapshotEvidenceSchema.parse({ + kind: 'agent-candidate-workspace-snapshot', + digest: sha256Bytes(manifestBytes), + material: memory.afterState, + manifest: embeddedCandidateArtifact(manifestBytes), + archive: embeddedCandidateArtifact(memory.archive), + }) + const root = await mkdtemp(join(tmpdir(), 'agent-candidate-memory-after-')) + try { + await state.ports.workspaces.materialize({ + role: 'memory', + snapshot, + archive: Uint8Array.from(memory.archive), + destination: root, + }) + const files = await readMaterializedWorkspaceFiles(root, memory.afterState) + for (const file of files) assertNoProtectedBytes(file.bytes, protectedValues) + signal?.throwIfAborted() + } finally { + await rm(root, { recursive: true, force: true }) + } +} + function normalizeEvaluation( evaluation: BenchmarkEvaluation, termination: AgentCandidateTermination, diff --git a/src/candidate-execution/prepare.ts b/src/candidate-execution/prepare.ts index 6045c36f..1b9bc081 100644 --- a/src/candidate-execution/prepare.ts +++ b/src/candidate-execution/prepare.ts @@ -11,21 +11,23 @@ import type { AgentCandidateMaterializationReceipt, AgentCandidateModelAccessNetwork, AgentCandidateResolvedModel, - HarnessType, } from '@tangle-network/agent-interface' import { + agentCandidateBenchmarkSuiteSchema, + agentCandidateBenchmarkTaskSchema, agentCandidateContainerSchema, agentCandidateExecutionLimitsSchema, agentCandidateExecutionPlanEvidenceSchema, agentCandidateExecutionPlanMaterialSchema, agentCandidateMaterializationReceiptSchema, agentCandidateModelAccessNetworkSchema, + agentCandidateRunCellSchema, + agentCandidateTaskOutcomeSpecSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, sha256DigestSchema, } from '@tangle-network/agent-interface' import { applyAgentCandidateWorkspacePlan, - type HarnessId, materializeCandidateProfile, } from '@tangle-network/agent-profile-materialize' @@ -48,14 +50,20 @@ import { canonicalCandidateDigest, canonicalCandidateDocument, embeddedCandidateArtifact, + omitTopLevelDigest, sha256Bytes, } from './digest' import { candidateExecutionOwnerWindowMs } from './execution-window' import { verifyTaskCheckout } from './git-materialize' -import { prepareAgentCandidateKnowledge } from './knowledge' +import { + CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV, + CANDIDATE_KNOWLEDGE_ROOT_ENV, + candidateKnowledgeExecutionPaths, + prepareAgentCandidateKnowledge, +} from './knowledge' import { sealAgentCandidateModelSettlement, usdToNanos } from './model-settlement' import { createPreparedCandidateExecution } from './prepared-state' -import { assertCandidateProfileExecutionSupport } from './profile' +import { candidateMaterializerHarness } from './profile' import { type AgentCandidateExecutionPorts, type AgentCandidateTaskExecution, @@ -71,21 +79,6 @@ import { verifiedResourceTextByDigest, } from './verify' -const MATERIALIZER_HARNESSES = new Set([ - 'claude-code', - 'claude', - 'claudish', - 'nanoclaw', - 'codex', - 'opencode', - 'kimi-code', - 'kimi', - 'pi', - 'gemini', - 'hermes', - 'openclaw', -]) - const MIN_RESERVATION_TTL_MS = 15 * 60_000 const PREPARED_HOLD_MARGIN_MS = 5 * 60_000 @@ -106,12 +99,23 @@ export async function prepareAgentCandidateExecution( const verifiedState = getVerifiedCandidateState(candidate) assertSameVerificationPorts(verifiedState.ports, ports) const bundle = candidate.bundle - assertCandidateProfileExecutionSupport(bundle.profile) - const harness = materializerHarness(bundle.execution.harness) + if (task.runCell.bundleDigest !== bundle.digest) { + throw new Error('candidate experiment cell does not bind the verified bundle') + } + const benchmarkTask = task.task + const attempt = { + number: task.runCell.attempt, + maxAttempts: benchmarkTask.attempt.maxAttempts, + retryPolicy: benchmarkTask.attempt.retryPolicy, + } as const + const harness = candidateMaterializerHarness(bundle.execution.harness) assertTaskInput(task, bundle.execution.instructionDelivery) - const resultTimeoutMs = candidateResultTimeout(options.resultTimeoutMs, task.limits.timeoutMs) + const resultTimeoutMs = candidateResultTimeout( + options.resultTimeoutMs, + benchmarkTask.limits.timeoutMs, + ) const ownerWindowMs = candidateExecutionOwnerWindowMs( - task.limits.timeoutMs, + benchmarkTask.limits.timeoutMs, cleanupTimeoutMs, resultTimeoutMs, ) @@ -121,23 +125,27 @@ export async function prepareAgentCandidateExecution( } assertDisjointHostStagingRoots(task) - const instructionBytes = Buffer.from(task.instruction, 'utf8') - const instructionDigest = sha256Bytes(instructionBytes) + const instructionBytes = Buffer.from(benchmarkTask.instruction, 'utf8') - const taskArtifacts = await verifyWorkspaceSnapshotArtifacts(task.workspace, ports.artifacts) + const taskArtifacts = await verifyWorkspaceSnapshotArtifacts( + benchmarkTask.workspace, + ports.artifacts, + ) await ports.workspaces.materialize({ role: 'task', - snapshot: task.workspace, + snapshot: benchmarkTask.workspace, archive: taskArtifacts.archive, destination: task.stagingRoots.taskRoot, }) - await verifyMaterializedWorkspace(task.stagingRoots.taskRoot, task.workspace.material, { + await verifyMaterializedWorkspace(task.stagingRoots.taskRoot, benchmarkTask.workspace.material, { ignoredProtectedRootEntries: ['.git', '.sidecar'], }) - await verifyTaskCheckout(task.stagingRoots.taskRoot, task.repository) + if (benchmarkTask.repository) { + await verifyTaskCheckout(task.stagingRoots.taskRoot, benchmarkTask.repository) + } const taskExecutorFiles = await readMaterializedWorkspaceFiles( task.stagingRoots.taskRoot, - task.workspace.material, + benchmarkTask.workspace.material, { ignoredProtectedRootEntries: ['.git', '.sidecar'] }, ) @@ -179,23 +187,26 @@ export async function prepareAgentCandidateExecution( ) await verifyMaterializedProfileWorkspace( task.stagingRoots.profileRoot, - profileApplication.profilePlan.material, + profileApplication.profileActivation.profilePlan.material, ) const profilePlanBytes = await readVerifiedArtifact( - profileApplication.profilePlan.artifact, + profileApplication.profileActivation.profilePlan.artifact, ports.artifacts, ) if ( !Buffer.from(profilePlanBytes).equals( - Buffer.from(canonicalCandidateBytes(profileApplication.profilePlan.material)), + Buffer.from( + canonicalCandidateBytes(profileApplication.profileActivation.profilePlan.material), + ), ) ) { throw new Error('profile materializer did not capture exact canonical plan bytes') } + const profileActivation = profileApplication.profileActivation const container = await resolveContainer(candidate, task, ports) const resolvedModel = await resolveModel(candidate, task, ports) - const preparationId = `candidate-preparation-v1.${randomBytes(32).toString('base64url')}` + const preparationId = `candidate-preparation.${randomBytes(32).toString('base64url')}` const reservationExpiresAtMs = Date.now() + Math.max(MIN_RESERVATION_TTL_MS, reservationWindowMs) const modelReservation = await withinCandidateCleanupDeadline( () => @@ -203,10 +214,10 @@ export async function prepareAgentCandidateExecution( executionId: task.executionId, preparationId, expiresAtMs: reservationExpiresAtMs, - attempt: task.attempt, + attempt, bundleDigest: bundle.digest, resolved: resolvedModel, - limits: modelLimits(task.limits), + limits: modelLimits(benchmarkTask.limits), }), candidateCleanupDeadline(cleanupTimeoutMs), 'protected model reservation', @@ -215,7 +226,7 @@ export async function prepareAgentCandidateExecution( try { validateProtectedModelReservation( modelReservation, - task.limits, + benchmarkTask.limits, preparationId, reservationExpiresAtMs, ) @@ -229,7 +240,12 @@ export async function prepareAgentCandidateExecution( ) const memory = preparedMemory.value const knowledge = await prepareAgentCandidateKnowledge(candidate, ports) - + const knowledgePaths = knowledge + ? candidateKnowledgeExecutionPaths( + task.executionRoots.taskRoot, + knowledge.retrievalConfig !== undefined, + ) + : undefined const baseLaunch = buildLaunch(candidate, task, profileApplication.flags) const publicEnv = mergePublicEnvironment( bundle.execution.env ?? {}, @@ -242,29 +258,28 @@ export async function prepareAgentCandidateExecution( }, } : {}, + knowledgePaths + ? { + [CANDIDATE_KNOWLEDGE_ROOT_ENV]: { + kind: 'public', + value: knowledgePaths.root, + }, + ...(knowledgePaths.retrievalConfig + ? { + [CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV]: { + kind: 'public' as const, + value: knowledgePaths.retrievalConfig, + }, + } + : {}), + } + : {}, ) - const routes = modelRoutes(bundle.profile, task.model.requested) + const routes = modelRoutes(bundle.profile, benchmarkTask.model.requested) const executionMaterial: AgentCandidateExecutionPlanMaterial = { - schemaVersion: 2, kind: 'agent-candidate-execution-plan-material', - bundleDigest: bundle.digest, + runCell: task.runCell, executionId: task.executionId, - attempt: task.attempt, - task: { - benchmark: task.benchmark, - benchmarkVersion: task.benchmarkVersion, - taskId: task.taskId, - splitDigest: task.splitDigest, - instruction: { - encoding: 'utf8', - sha256: instructionDigest, - byteLength: instructionBytes.byteLength, - delivery: bundle.execution.instructionDelivery, - }, - repository: task.repository, - outcome: { kind: 'workspace' }, - workspace: task.workspace, - }, workspaces: { taskRoot: task.executionRoots.taskRoot, ...(task.executionRoots.candidateRoot @@ -276,6 +291,8 @@ export async function prepareAgentCandidateExecution( profile: profileApplication.application, harness: bundle.execution.harness, harnessVersion: bundle.execution.harnessVersion, + instructionDelivery: bundle.execution.instructionDelivery, + limits: benchmarkTask.limits, container, model: { policy: 'single', @@ -287,7 +304,6 @@ export async function prepareAgentCandidateExecution( }, routes, }, - grader: task.grader, launch: { executable: baseLaunch.executable, args: baseLaunch.args, @@ -296,7 +312,6 @@ export async function prepareAgentCandidateExecution( }, ...(bundle.knowledge ? { knowledgeManifestDigest: bundle.knowledge.snapshot.digest } : {}), memory, - limits: task.limits, network: { mode: 'disabled' }, } agentCandidateExecutionPlanMaterialSchema.parse(executionMaterial) @@ -307,7 +322,6 @@ export async function prepareAgentCandidateExecution( } const executionPlan: AgentCandidateExecutionPlanEvidence = agentCandidateExecutionPlanEvidenceSchema.parse({ - schemaVersion: 2, kind: 'agent-candidate-execution-plan', digest: executionDigest, material: executionMaterial, @@ -317,11 +331,24 @@ export async function prepareAgentCandidateExecution( const entrypoint = candidateEntrypointReceipt(candidate) const materializationReceipt = canonicalCandidateDocument( { - schemaVersion: 2, kind: 'agent-candidate-materialization', digestAlgorithm: 'rfc8785-sha256', bundleDigest: bundle.digest, - profilePlan: profileApplication.profilePlan, + benchmark: { + suite: { + digest: task.benchmarkSuite.digest, + material: embeddedCandidateArtifact( + canonicalCandidateBytes(omitTopLevelDigest(task.benchmarkSuite)), + ), + }, + task: { + digest: benchmarkTask.digest, + material: embeddedCandidateArtifact( + canonicalCandidateBytes(omitTopLevelDigest(benchmarkTask)), + ), + }, + }, + profileActivation, executionPlan, ...(bundle.execution.workspace ? { candidateWorkspace: bundle.execution.workspace } : {}), codeKind: bundle.code.kind, @@ -336,7 +363,7 @@ export async function prepareAgentCandidateExecution( ) agentCandidateMaterializationReceiptSchema.parse(materializationReceipt.value) - const traceRunId = `${task.executionId}:attempt-${task.attempt.number}:${canonicalCandidateDigest({ preparationId }).slice(7, 23)}` + const traceRunId = `${task.executionId}:attempt-${attempt.number}:${canonicalCandidateDigest({ preparationId }).slice(7, 23)}` const traceTags = { [CANDIDATE_TRACE_TAGS.executionId]: task.executionId, [CANDIDATE_TRACE_TAGS.bundleDigest]: bundle.digest, @@ -355,16 +382,19 @@ export async function prepareAgentCandidateExecution( return createPreparedCandidateExecution({ ports, bundle, + benchmarkSuite: task.benchmarkSuite, + benchmarkTask, executionId: task.executionId, roots: { execution: { ...task.executionRoots }, staging: { ...task.stagingRoots }, }, profilePlan: { - value: profileApplication.profilePlan, + value: profileApplication.profileActivation.profilePlan, bytes: profilePlanBytes, written: [...profileApplication.application.mountPaths], }, + profileActivation, executionPlan: { value: executionPlan, bytes: executionBytes }, materializationReceipt, launch: { @@ -395,7 +425,7 @@ export async function prepareAgentCandidateExecution( ...(candidateExecutorFiles ? { candidateFiles: candidateExecutorFiles } : {}), profileFiles: exactProfileExecutorFiles( profileWorkspacePlan.files, - profileApplication.profilePlan.material.files, + profileApplication.profileActivation.profilePlan.material.files, ), }, ...(preparedMemory.accessDigest && preparedMemory.value.mode === 'isolated' @@ -495,34 +525,65 @@ function assertTaskInput( task: AgentCandidateTaskExecution, delivery: VerifiedAgentCandidate['bundle']['execution']['instructionDelivery'], ): void { + const benchmarkTask = agentCandidateBenchmarkTaskSchema.parse(task.task) + const benchmarkSuite = agentCandidateBenchmarkSuiteSchema.parse(task.benchmarkSuite) + const runCell = agentCandidateRunCellSchema.parse(task.runCell) const requiredStrings: Array<[string, string]> = [ ['executionId', task.executionId], - ['benchmark', task.benchmark], - ['benchmarkVersion', task.benchmarkVersion], - ['taskId', task.taskId], - ['repository identity', task.repository.identity], - ['repository root identity', task.repository.rootIdentity], + ['benchmark', benchmarkTask.benchmark.name], + ['benchmarkVersion', benchmarkTask.benchmark.version], + ['taskId', benchmarkTask.scenario.id], ] + if (benchmarkTask.outcome.kind === 'workspace' && !benchmarkTask.repository) { + throw new Error('workspace task outcome requires repository identity') + } + if (benchmarkTask.repository) { + requiredStrings.push( + ['repository identity', benchmarkTask.repository.identity], + ['repository root identity', benchmarkTask.repository.rootIdentity], + ) + } for (const [name, value] of requiredStrings) { if (!value.trim()) throw new Error(`${name} must be non-empty`) } if (!/^[A-Za-z0-9._:-]{1,200}$/.test(task.executionId)) { throw new Error('executionId must be a stable filesystem-neutral identifier') } - if (!task.instruction || !isWellFormedUnicode(task.instruction)) { + if (!benchmarkTask.instruction || !isWellFormedUnicode(benchmarkTask.instruction)) { throw new Error('task instruction must be non-empty well-formed Unicode') } - sha256DigestSchema.parse(task.splitDigest) - agentCandidateWorkspaceSnapshotEvidenceSchema.parse(task.workspace) - agentCandidateExecutionLimitsSchema.parse(task.limits) - if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(task.repository.baseCommit)) { - throw new Error('task repository base commit is not a full Git object id') + if ( + canonicalCandidateDigest(omitTopLevelDigest(benchmarkTask)) !== benchmarkTask.digest || + canonicalCandidateDigest(omitTopLevelDigest(benchmarkSuite)) !== benchmarkSuite.digest || + canonicalCandidateDigest(omitTopLevelDigest(runCell)) !== runCell.digest + ) { + throw new Error('candidate experiment cell contains an invalid content digest') } - if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(task.repository.baseTree)) { - throw new Error('task repository base tree is not a full Git object id') + const cellIndex = runCell.taskIndex * benchmarkSuite.reps + runCell.repetition + if ( + runCell.suiteDigest !== benchmarkSuite.digest || + runCell.taskDigest !== benchmarkTask.digest || + benchmarkSuite.taskDigests[runCell.taskIndex] !== benchmarkTask.digest || + runCell.repetition >= benchmarkSuite.reps || + benchmarkSuite.seeds[cellIndex] !== runCell.seed || + runCell.attempt > benchmarkTask.attempt.maxAttempts + ) { + throw new Error('candidate experiment cell does not match its signed suite and task') } - if (task.repository.baseCommit.length !== task.repository.baseTree.length) { - throw new Error('task repository Git object formats disagree') + sha256DigestSchema.parse(benchmarkTask.benchmark.splitDigest) + agentCandidateTaskOutcomeSpecSchema.parse(benchmarkTask.outcome) + agentCandidateWorkspaceSnapshotEvidenceSchema.parse(benchmarkTask.workspace) + agentCandidateExecutionLimitsSchema.parse(benchmarkTask.limits) + if (benchmarkTask.repository) { + if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(benchmarkTask.repository.baseCommit)) { + throw new Error('task repository base commit is not a full Git object id') + } + if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(benchmarkTask.repository.baseTree)) { + throw new Error('task repository base tree is not a full Git object id') + } + if (benchmarkTask.repository.baseCommit.length !== benchmarkTask.repository.baseTree.length) { + throw new Error('task repository Git object formats disagree') + } } for (const [name, root] of [ ['execution task root', task.executionRoots.taskRoot], @@ -538,15 +599,15 @@ function assertTaskInput( if (!canonical) throw new Error(`${name} must be a canonical absolute path`) } if ( - !Number.isInteger(task.attempt.number) || - !Number.isInteger(task.attempt.maxAttempts) || - task.attempt.number < 1 || - task.attempt.number > task.attempt.maxAttempts || - (task.attempt.retryPolicy === 'none' && task.attempt.maxAttempts !== 1) + !Number.isInteger(runCell.attempt) || + !Number.isInteger(benchmarkTask.attempt.maxAttempts) || + runCell.attempt < 1 || + runCell.attempt > benchmarkTask.attempt.maxAttempts || + (benchmarkTask.attempt.retryPolicy === 'none' && benchmarkTask.attempt.maxAttempts !== 1) ) { throw new Error('task attempt policy is invalid') } - const limits = task.limits + const limits = benchmarkTask.limits if ( !Number.isInteger(limits.timeoutMs) || limits.timeoutMs <= 0 || @@ -565,28 +626,33 @@ function assertTaskInput( throw new Error('task execution limits are invalid') } usdToNanos(limits.maxCostUsd, 'task maxCostUsd') - if (!task.model.requested.trim()) throw new Error('evaluator model request must be non-empty') - if (!task.grader.name.trim() || !task.grader.version.trim()) { + if (!benchmarkTask.model.requested.trim()) { + throw new Error('evaluator model request must be non-empty') + } + if (!benchmarkTask.grader.name.trim() || !benchmarkTask.grader.version.trim()) { throw new Error('evaluator benchmark grader identity must be non-empty') } - if (!Number.isInteger(task.grader.artifact.byteLength) || task.grader.artifact.byteLength <= 0) { + if ( + !Number.isInteger(benchmarkTask.grader.artifact.byteLength) || + benchmarkTask.grader.artifact.byteLength <= 0 + ) { throw new Error('evaluator benchmark grader artifact must be non-empty') } - sha256DigestSchema.parse(task.grader.artifact.sha256) - if (task.evaluatorTaskContainer) { + sha256DigestSchema.parse(benchmarkTask.grader.artifact.sha256) + if (benchmarkTask.evaluatorTaskContainer) { if ( - task.evaluatorTaskContainer.source !== 'evaluator-task-container' || - !task.evaluatorTaskContainer.image.trim() || - !task.evaluatorTaskContainer.platform.os.trim() || - !task.evaluatorTaskContainer.platform.architecture.trim() + benchmarkTask.evaluatorTaskContainer.source !== 'evaluator-task-container' || + !benchmarkTask.evaluatorTaskContainer.image.trim() || + !benchmarkTask.evaluatorTaskContainer.platform.os.trim() || + !benchmarkTask.evaluatorTaskContainer.platform.architecture.trim() ) { throw new Error('evaluator task container evidence is incomplete') } - sha256DigestSchema.parse(task.evaluatorTaskContainer.indexDigest) - sha256DigestSchema.parse(task.evaluatorTaskContainer.manifestDigest) + sha256DigestSchema.parse(benchmarkTask.evaluatorTaskContainer.indexDigest) + sha256DigestSchema.parse(benchmarkTask.evaluatorTaskContainer.manifestDigest) agentCandidateContainerSchema.parse({ - image: task.evaluatorTaskContainer.image, - indexDigest: task.evaluatorTaskContainer.indexDigest, + image: benchmarkTask.evaluatorTaskContainer.image, + indexDigest: benchmarkTask.evaluatorTaskContainer.indexDigest, }) } if (delivery.kind === 'utf8-file') { @@ -642,39 +708,31 @@ async function assertEmptyDirectory(path: string): Promise { } } -function materializerHarness(harness: HarnessType): HarnessId { - if (!MATERIALIZER_HARNESSES.has(harness)) { - throw new Error( - `sealed candidate profile materialization is unsupported for harness ${harness}`, - ) - } - return harness as HarnessId -} - async function resolveContainer( candidate: VerifiedAgentCandidate, task: AgentCandidateTaskExecution, ports: AgentCandidateExecutionPorts, ): Promise { const environment = candidate.bundle.execution.environment + const evaluatorTaskContainer = task.task.evaluatorTaskContainer const pinned = environment.kind === 'pinned-container' ? environment.container : undefined - if (environment.kind === 'evaluator-task-container' && !task.evaluatorTaskContainer) { + if (environment.kind === 'evaluator-task-container' && !evaluatorTaskContainer) { throw new Error('evaluator-task-container candidate requires an evaluator-owned task image') } - if (environment.kind === 'pinned-container' && task.evaluatorTaskContainer) { + if (environment.kind === 'pinned-container' && evaluatorTaskContainer) { throw new Error('pinned candidate containers cannot be replaced by a task image') } const resolved = await ports.containers.resolve({ candidate: pinned, - evaluatorTaskContainer: task.evaluatorTaskContainer, + evaluatorTaskContainer, }) if (resolved.source !== environment.kind) throw new Error('resolved container source drifted') if (pinned && (resolved.image !== pinned.image || resolved.indexDigest !== pinned.indexDigest)) { throw new Error('resolved pinned container does not match the candidate image index') } if ( - task.evaluatorTaskContainer && - JSON.stringify(resolved) !== JSON.stringify(task.evaluatorTaskContainer) + evaluatorTaskContainer && + canonicalCandidateDigest(resolved) !== canonicalCandidateDigest(evaluatorTaskContainer) ) { throw new Error('resolved task container does not match evaluator-owned image evidence') } @@ -687,25 +745,20 @@ async function resolveModel( ports: AgentCandidateExecutionPorts, ): Promise { const hints = candidate.bundle.profile.model - if (hints?.default !== undefined && hints.default !== task.model.requested) { + const expected = task.task.model + if (hints?.default !== undefined && hints.default !== expected.requested) { throw new Error('candidate model preference conflicts with the evaluator-owned model') } - if ( - hints?.reasoningEffort !== undefined && - hints.reasoningEffort !== task.model.reasoningEffort - ) { + if (hints?.reasoningEffort !== undefined && hints.reasoningEffort !== expected.reasoningEffort) { throw new Error('candidate reasoning effort conflicts with the evaluator-owned effort') } const resolved = await ports.models.resolve({ - requested: task.model.requested, + requested: expected.requested, harness: candidate.bundle.execution.harness, - reasoningEffort: task.model.reasoningEffort, + reasoningEffort: expected.reasoningEffort, }) - if ( - resolved.requested !== task.model.requested || - resolved.reasoningEffort !== task.model.reasoningEffort - ) { - throw new Error('model resolver drifted from the evaluator-owned request') + if (canonicalCandidateDigest(resolved) !== canonicalCandidateDigest(expected)) { + throw new Error('model resolver drifted from the signed benchmark model') } return resolved } @@ -725,7 +778,7 @@ async function prepareMemory( if (policy.mode === 'disabled') return { value: { mode: 'disabled' } } const seed = policy.seed ? await verifiedArtifactBytes(candidate, policy.seed) : undefined const executionSegment = canonicalCandidateDigest({ executionId: task.executionId }).slice(7) - const taskSegment = canonicalCandidateDigest({ taskId: task.taskId }).slice(7) + const taskSegment = canonicalCandidateDigest({ taskDigest: task.task.digest }).slice(7) const preparationSegment = canonicalCandidateDigest({ preparationId }).slice(7, 23) const effectiveNamespace = `candidate/${candidate.bundle.digest.slice(7, 23)}/${executionSegment}/${taskSegment}/${preparationSegment}` const reset = await withinCandidateCleanupDeadline( @@ -961,7 +1014,6 @@ function exactProfileExecutorFiles( return { path: expected.relPath, mode, bytes: Uint8Array.from(bytes) } }) } - function isWellFormedUnicode(value: string): boolean { for (let index = 0; index < value.length; index++) { const code = value.charCodeAt(index) diff --git a/src/candidate-execution/prepared-state.ts b/src/candidate-execution/prepared-state.ts index 97b1f033..aa63aef4 100644 --- a/src/candidate-execution/prepared-state.ts +++ b/src/candidate-execution/prepared-state.ts @@ -1,4 +1,9 @@ import { + type AgentCandidateBenchmarkSuite, + type AgentCandidateBenchmarkTask, + type AgentCandidateWorkspaceManifestMaterial, + agentCandidateBenchmarkSuiteSchema, + agentCandidateBenchmarkTaskSchema, agentCandidateExecutionPlanEvidenceSchema, agentCandidateMaterializationReceiptSchema, agentCandidateProfilePlanEvidenceSchema, @@ -14,22 +19,26 @@ import { sha256Bytes, } from './digest' import { verifyTaskCheckout } from './git-materialize' +import { parseAgentCandidateProfileActivation } from './profile' import type { AgentCandidateExecutionPorts, AgentCandidateExecutorRequest, + AgentCandidateExecutorWorkspaceFile, AgentCandidateProtectedModelActivation, AgentCandidateProtectedModelReservation, PreparedAgentCandidateExecution, } from './types' import { CANDIDATE_TRACE_ENV, CANDIDATE_TRACE_TAGS, preparedCandidateBrand } from './types' -import { workspaceTaskRepository } from './workspace-task' export interface PreparedCandidateState { ports: AgentCandidateExecutionPorts bundle: PreparedAgentCandidateExecution['bundle'] + benchmarkSuite: AgentCandidateBenchmarkSuite + benchmarkTask: AgentCandidateBenchmarkTask executionId: string roots: PreparedAgentCandidateExecution['roots'] profilePlan: PreparedAgentCandidateExecution['profilePlan'] + profileActivation: PreparedAgentCandidateExecution['profileActivation'] executionPlan: PreparedAgentCandidateExecution['executionPlan'] materializationReceipt: PreparedAgentCandidateExecution['materializationReceipt'] launch: PreparedAgentCandidateExecution['launch'] @@ -94,9 +103,11 @@ export function createPreparedCandidateExecution( assertPrivateCandidateIntegrity(state) const prepared = Object.freeze({ bundle: state.bundle, + benchmark: Object.freeze({ suite: state.benchmarkSuite, task: state.benchmarkTask }), executionId: state.executionId, roots: state.roots, profilePlan: evidenceView(state.profilePlan), + profileActivation: state.profileActivation, executionPlan: evidenceView(state.executionPlan), materializationReceipt: state.materializationReceipt, launch: state.launch, @@ -171,8 +182,9 @@ export function beginPreparedCandidateRun( const material = state.executionPlan.value.material const request = Object.freeze({ executionId: state.executionId, + benchmark: preparedBenchmark(state), inputs: Object.freeze({ - task: workspaceInputView(material.task.workspace, state.executorInputs.taskFiles), + task: workspaceInputView(state.benchmarkTask.workspace, state.executorInputs.taskFiles), ...(material.candidateWorkspace && state.executorInputs.candidateFiles ? { candidate: workspaceInputView( @@ -189,6 +201,7 @@ export function beginPreparedCandidateRun( }), roots: state.roots.execution, profilePlan: evidenceView(state.profilePlan), + profileActivation: state.profileActivation, executionPlan: evidenceView(state.executionPlan), materializationReceipt: state.materializationReceipt, launch: immutableCandidateValue({ ...state.launch, env: completeEnvironment }), @@ -203,6 +216,12 @@ export function beginPreparedCandidateRun( return { state, request } } +function preparedBenchmark( + state: PreparedCandidateState, +): PreparedAgentCandidateExecution['benchmark'] { + return Object.freeze({ suite: state.benchmarkSuite, task: state.benchmarkTask }) +} + export function consumePreparedCandidateExecution( prepared: PreparedAgentCandidateExecution, outcome: 'succeeded' | 'failed' | 'disposed' | 'disposal-failed' | 'cleanup-failed', @@ -223,10 +242,16 @@ export async function assertPreparedCandidateWorkspaces( state: PreparedCandidateState, ): Promise { const plan = state.executionPlan.value.material - await verifyMaterializedWorkspace(state.roots.staging.taskRoot, plan.task.workspace.material, { - ignoredProtectedRootEntries: ['.git', '.sidecar'], - }) - await verifyTaskCheckout(state.roots.staging.taskRoot, workspaceTaskRepository(plan.task)) + await verifyMaterializedWorkspace( + state.roots.staging.taskRoot, + state.benchmarkTask.workspace.material, + { + ignoredProtectedRootEntries: ['.git', '.sidecar'], + }, + ) + if (state.benchmarkTask.repository) { + await verifyTaskCheckout(state.roots.staging.taskRoot, state.benchmarkTask.repository) + } await verifyMaterializedProfileWorkspace( state.roots.staging.profileRoot, state.profilePlan.value.material, @@ -241,9 +266,19 @@ export async function assertPreparedCandidateWorkspaces( } function detachPreparedCandidateState(input: PreparedCandidateState): PreparedCandidateState { + const benchmarkSuite = immutableCandidateValue( + agentCandidateBenchmarkSuiteSchema.parse(input.benchmarkSuite), + ) + const benchmarkTask = immutableCandidateValue( + agentCandidateBenchmarkTaskSchema.parse(input.benchmarkTask), + ) const profilePlan = immutableCandidateValue( agentCandidateProfilePlanEvidenceSchema.parse(input.profilePlan.value), ) + const profileActivation = parseAgentCandidateProfileActivation( + input.profileActivation, + profilePlan.digest, + ) const executionPlan = immutableCandidateValue( agentCandidateExecutionPlanEvidenceSchema.parse(input.executionPlan.value), ) @@ -259,6 +294,8 @@ function detachPreparedCandidateState(input: PreparedCandidateState): PreparedCa return Object.freeze({ ports: input.ports, bundle: input.bundle, + benchmarkSuite, + benchmarkTask, executionId: input.executionId, roots: immutableCandidateValue(input.roots), profilePlan: Object.freeze({ @@ -266,6 +303,7 @@ function detachPreparedCandidateState(input: PreparedCandidateState): PreparedCa bytes: Uint8Array.from(input.profilePlan.bytes), written: Object.freeze([...input.profilePlan.written]), }), + profileActivation, executionPlan: Object.freeze({ value: executionPlan, bytes: Uint8Array.from(input.executionPlan.bytes), @@ -287,11 +325,7 @@ function detachPreparedCandidateState(input: PreparedCandidateState): PreparedCa ...(input.executorInputs.candidateFiles ? { candidateFiles: immutableExecutorFiles(input.executorInputs.candidateFiles) } : {}), - profileFiles: Object.freeze( - input.executorInputs.profileFiles.map((file) => - Object.freeze({ ...file, bytes: Uint8Array.from(file.bytes) }), - ), - ), + profileFiles: immutableExecutorFiles(input.executorInputs.profileFiles), }), ...(input.memoryReservation ? { memoryReservation: immutableCandidateValue(input.memoryReservation) } @@ -319,6 +353,7 @@ function assertPrivateCandidateIntegrity(state: PreparedCandidateState): void { throw new Error('prepared candidate bundle no longer matches its digest') } assertPlanEvidence(state.profilePlan.value, state.profilePlan.bytes, 'profile plan') + parseAgentCandidateProfileActivation(state.profileActivation, state.profilePlan.value.digest) assertPlanEvidence(state.executionPlan.value, state.executionPlan.bytes, 'execution plan') agentCandidateExecutionPlanEvidenceSchema.parse(state.executionPlan.value) @@ -333,16 +368,17 @@ function assertPrivateCandidateIntegrity(state: PreparedCandidateState): void { throw new Error('prepared materialization receipt no longer matches its canonical bytes') } - const instruction = state.executionPlan.value.material.task.instruction + assertSignedBenchmarkInput(state) + const instruction = Buffer.from(state.benchmarkTask.instruction, 'utf8') if ( - sha256Bytes(state.instruction.bytes) !== instruction.sha256 || - state.instruction.bytes.byteLength !== instruction.byteLength || - JSON.stringify(state.instruction.delivery) !== JSON.stringify(instruction.delivery) + !Buffer.from(state.instruction.bytes).equals(instruction) || + JSON.stringify(state.instruction.delivery) !== + JSON.stringify(state.executionPlan.value.material.instructionDelivery) ) { throw new Error('prepared instruction no longer matches the signed execution plan') } if ( - !/^candidate-preparation-v1\.[A-Za-z0-9_-]{43}$/.test(state.preparationId) || + !/^candidate-preparation\.[A-Za-z0-9_-]{43}$/.test(state.preparationId) || !Number.isSafeInteger(state.reservationExpiresAtMs) || state.reservationExpiresAtMs <= 0 || !Number.isSafeInteger(state.cleanupTimeoutMs) || @@ -397,7 +433,7 @@ function assertPrivateCandidateIntegrity(state: PreparedCandidateState): void { } if ( !state.trace.runId.startsWith( - `${state.executionId}:attempt-${state.executionPlan.value.material.attempt.number}:`, + `${state.executionId}:attempt-${state.executionPlan.value.material.runCell.attempt}:`, ) || canonicalCandidateDigest(state.trace.tags) !== canonicalCandidateDigest(expectedTags) || canonicalCandidateDigest(state.trace.env) !== canonicalCandidateDigest(expectedTraceEnvironment) @@ -465,14 +501,13 @@ function knowledgeView( : {}), }) } - function workspaceInputView( snapshot: AgentCandidateExecutorRequest['inputs']['task']['snapshot'], sourceFiles: PreparedCandidateState['executorInputs']['taskFiles'], ): AgentCandidateExecutorRequest['inputs']['task'] { return Object.freeze({ snapshot, - files: Object.freeze(sourceFiles.map((file) => profileFileView(file))), + files: Object.freeze(sourceFiles.map((file) => executorFileView(file))), }) } @@ -489,9 +524,25 @@ function profileFileView( }) } +function executorFileView( + source: PreparedCandidateState['executorInputs']['taskFiles'][number], +): AgentCandidateExecutorWorkspaceFile { + const bytes = Uint8Array.from(source.bytes) + return Object.freeze({ + path: source.path, + mode: source.mode, + get bytes(): Uint8Array { + return Uint8Array.from(bytes) + }, + }) +} + function assertExecutorInputs(state: PreparedCandidateState): void { const material = state.executionPlan.value.material - assertWorkspaceExecutorFiles(state.executorInputs.taskFiles, material.task.workspace.material) + assertWorkspaceExecutorFiles( + state.executorInputs.taskFiles, + state.benchmarkTask.workspace.material, + ) if (material.candidateWorkspace) { if (!state.executorInputs.candidateFiles) { throw new Error('prepared candidate executor files are missing') @@ -555,7 +606,7 @@ function assertPreparedKnowledge(state: PreparedCandidateState): void { function assertWorkspaceExecutorFiles( actualFiles: PreparedCandidateState['executorInputs']['taskFiles'], - expected: PreparedAgentCandidateExecution['executionPlan']['value']['material']['task']['workspace']['material'], + expected: AgentCandidateWorkspaceManifestMaterial, ): void { if (actualFiles.length !== expected.files.length) { throw new Error('prepared workspace executor files do not match the signed manifest') @@ -576,6 +627,42 @@ function assertWorkspaceExecutorFiles( } } +function assertSignedBenchmarkInput(state: PreparedCandidateState): void { + const plan = state.executionPlan.value.material + const cell = plan.runCell + const suite = state.benchmarkSuite + const task = state.benchmarkTask + const receipt = state.materializationReceipt.value + const cellIndex = cell.taskIndex * suite.reps + cell.repetition + if ( + canonicalCandidateDigest(omitTopLevelDigest(suite)) !== suite.digest || + canonicalCandidateDigest(omitTopLevelDigest(task)) !== task.digest || + canonicalCandidateDigest(omitTopLevelDigest(cell)) !== cell.digest || + cell.bundleDigest !== state.bundle.digest || + cell.suiteDigest !== suite.digest || + cell.taskDigest !== task.digest || + suite.taskDigests[cell.taskIndex] !== task.digest || + suite.seeds[cellIndex] !== cell.seed || + cell.repetition >= suite.reps || + cell.attempt < 1 || + cell.attempt > task.attempt.maxAttempts + ) { + throw new Error('prepared candidate no longer matches its signed experiment cell') + } + const suiteBytes = canonicalCandidateBytes(omitTopLevelDigest(suite)) + const taskBytes = canonicalCandidateBytes(omitTopLevelDigest(task)) + if ( + receipt.benchmark.suite.digest !== suite.digest || + receipt.benchmark.suite.material.sha256 !== sha256Bytes(suiteBytes) || + receipt.benchmark.suite.material.byteLength !== suiteBytes.byteLength || + receipt.benchmark.task.digest !== task.digest || + receipt.benchmark.task.material.sha256 !== sha256Bytes(taskBytes) || + receipt.benchmark.task.material.byteLength !== taskBytes.byteLength + ) { + throw new Error('materialization receipt no longer matches its signed benchmark input') + } +} + function immutableExecutorFiles( files: PreparedCandidateState['executorInputs']['taskFiles'], ): ReadonlyArray<{ path: string; mode: number; bytes: Uint8Array }> { diff --git a/src/candidate-execution/profile.ts b/src/candidate-execution/profile.ts index 7f814121..c4d187f0 100644 --- a/src/candidate-execution/profile.ts +++ b/src/candidate-execution/profile.ts @@ -1,25 +1,119 @@ import type { AgentCandidateConfigValue, AgentCandidateProfile, + AgentCandidateProfileActivation, + AgentCandidateProfilePlanEvidence, AgentCandidateResourceRef, AgentProfile, AgentProfileDiff, AgentProfileMcpServer, AgentProfileResourceRef, + HarnessType, } from '@tangle-network/agent-interface' import { + agentCandidateProfileActivationSchema, agentCandidateProfileSchema, agentProfileDiffSchema, agentProfileSchema, applyAgentProfileDiff, } from '@tangle-network/agent-interface' +import type { + AgentCandidateWorkspacePlan, + HarnessId, +} from '@tangle-network/agent-profile-materialize' import { canonicalCandidateBytes, canonicalCandidateDigest, + canonicalCandidateDocument, embeddedCandidateArtifact, + immutableCandidateValue, + omitTopLevelDigest, + sha256Bytes, } from './digest' +const MATERIALIZER_HARNESSES = new Set([ + 'claude-code', + 'claude', + 'claudish', + 'nanoclaw', + 'codex', + 'opencode', + 'kimi-code', + 'kimi', + 'pi', + 'gemini', + 'hermes', + 'openclaw', +]) + +export function candidateMaterializerHarness(harness: HarnessType): HarnessId { + if (!MATERIALIZER_HARNESSES.has(harness)) { + throw new Error( + `sealed candidate profile materialization is unsupported for harness ${harness}`, + ) + } + return harness as HarnessId +} + +/** Bind exact native profile text to the canonical plan captured during preparation. */ +export function createAgentCandidateProfileActivation( + plan: AgentCandidateWorkspacePlan, + profilePlan: AgentCandidateProfilePlanEvidence, +): AgentCandidateProfileActivation { + const sourceByPath = new Map(plan.files.map((file) => [file.relPath, file])) + if (sourceByPath.size !== plan.files.length) { + throw new Error('candidate profile activation contains duplicate native file paths') + } + const files = profilePlan.material.files.map((expected) => { + const source = sourceByPath.get(expected.relPath) + if (!source) { + throw new Error(`candidate profile activation is missing ${expected.relPath}`) + } + const mode = source.mode ?? 0o644 + if (!Number.isSafeInteger(mode) || mode < 0 || mode > 0o777) { + throw new Error(`candidate profile activation has invalid mode for ${expected.relPath}`) + } + return { path: expected.relPath, mode, content: source.content } + }) + if (files.length !== plan.files.length) { + throw new Error('candidate profile activation contains unplanned native files') + } + return parseAgentCandidateProfileActivation( + canonicalCandidateDocument({ + kind: 'agent-candidate-profile-activation', + profilePlan, + files, + }).value, + profilePlan.digest, + ) +} + +/** Parse and check every native file hash plus both canonical document digests. */ +export function parseAgentCandidateProfileActivation( + input: unknown, + expectedProfilePlanDigest?: AgentCandidateProfilePlanEvidence['digest'], +): AgentCandidateProfileActivation { + const activation = agentCandidateProfileActivationSchema.parse(input) + const planBytes = canonicalCandidateBytes(activation.profilePlan.material) + const profilePlanArtifact = activation.profilePlan.artifact + if ( + sha256Bytes(planBytes) !== activation.profilePlan.digest || + profilePlanArtifact.sha256 !== activation.profilePlan.digest || + profilePlanArtifact.byteLength !== planBytes.byteLength || + ('content' in profilePlanArtifact && + !Buffer.from(profilePlanArtifact.content, 'base64').equals(Buffer.from(planBytes))) || + (expectedProfilePlanDigest !== undefined && + activation.profilePlan.digest !== expectedProfilePlanDigest) + ) { + throw new Error('candidate profile activation has an invalid canonical profile plan') + } + if (canonicalCandidateDigest(omitTopLevelDigest(activation)) !== activation.digest) { + throw new Error('candidate profile activation digest does not match') + } + return immutableCandidateValue(activation) +} + const CANDIDATE_PROFILE_DIRECT_FIELDS = [ 'name', 'description', @@ -86,7 +180,7 @@ export function assertCandidateProfileBinding( if (measured.connections || measured.metadata || measured.extensions) { throw new Error('proposal candidate profile contains fields unsupported by sealed candidates') } - const normalized = candidateProfileAsAgentProfile(bundled) + const normalized = agentCandidateProfileAsAgentProfile(bundled) if (canonicalCandidateDigest(measured) !== canonicalCandidateDigest(normalized)) { throw new Error('proposal candidateProfile does not match candidateBundle.profile') } @@ -94,14 +188,24 @@ export function assertCandidateProfileBinding( /** Parse a complete profile without silently discarding unsupported fields. */ export function parseExactAgentProfile(input: unknown, label: string): AgentProfile { - const parsed = agentProfileSchema.parse(input) as AgentProfile + let parsed: AgentProfile + try { + parsed = agentProfileSchema.parse(input) as AgentProfile + } catch (cause) { + throw new Error(`${label} contains unsupported or non-canonical fields`, { cause }) + } assertCanonicalParse(input, parsed, label) return parsed } /** Parse a profile diff without silently discarding unsupported fields. */ export function parseExactAgentProfileDiff(input: unknown, label: string): AgentProfileDiff { - const parsed = agentProfileDiffSchema.parse(input) as AgentProfileDiff + let parsed: AgentProfileDiff + try { + parsed = agentProfileDiffSchema.parse(input) as AgentProfileDiff + } catch (cause) { + throw new Error(`${label} contains unsupported or non-canonical fields`, { cause }) + } assertCanonicalParse(input, parsed, label) return parsed } @@ -135,7 +239,9 @@ export function assertCandidateProfileExecutionSupport(profile: AgentCandidatePr } } -function candidateProfileAsAgentProfile(candidate: AgentCandidateProfile): AgentProfile { +export function agentCandidateProfileAsAgentProfile( + candidate: AgentCandidateProfile, +): AgentProfile { const output: Record = {} copyDirectProfileFields(output, candidate as unknown as Record) if (candidate.model) output.model = { ...candidate.model } diff --git a/src/candidate-execution/protected-model-port.ts b/src/candidate-execution/protected-model-port.ts index e69194e1..5111a9c7 100644 --- a/src/candidate-execution/protected-model-port.ts +++ b/src/candidate-execution/protected-model-port.ts @@ -204,7 +204,7 @@ export function createProtectedAgentCandidateModelPort( grantDigest: request.grantDigest, model: request.resolved.model, }) - if (state) assertWithinReservedLimits(sealed.fixedUsage, state) + if (state) assertWithinReservedLimits(sealed.usage, state) const settlementDigest = canonicalCandidateDigest(sealed.value) if (remembered?.settlementDigest && remembered.settlementDigest !== settlementDigest) { diff --git a/src/candidate-execution/recover.ts b/src/candidate-execution/recover.ts index cff11369..86e5576c 100644 --- a/src/candidate-execution/recover.ts +++ b/src/candidate-execution/recover.ts @@ -1,7 +1,17 @@ import type { TraceStore } from '@tangle-network/agent-eval' +import type { + AgentCandidateExecutionPlanMaterial, + AgentCandidateMaterializationReceipt, +} from '@tangle-network/agent-interface' +import { + agentCandidateExecutionPlanMaterialSchema, + agentCandidateMaterializationReceiptSchema, +} from '@tangle-network/agent-interface' +import { readVerifiedArtifact } from './artifacts' import type { AgentCandidateExecutionAttemptRef, + AgentCandidateExecutionClaim, AgentCandidateExecutionClaimStore, AgentCandidateExecutionFinishResult, } from './claim' @@ -10,9 +20,11 @@ import { candidateCleanupTimeout, withinCandidateCleanupDeadline, } from './cleanup' -import { sealAgentCandidateExecutorFinalCapture } from './executor-capture' +import { canonicalCandidateBytes } from './digest' +import { sealAgentCandidateExecutorStopAcknowledgement } from './executor-capture' import { sealAgentCandidateModelSettlement } from './model-settlement' import { persistCandidateModelSettlementEvidence } from './outcome-evidence' +import { persistCandidateOutputArtifact } from './output-artifacts' import { RecoveryAgentCandidateTraceStore } from './protected-trace-store' import type { AgentCandidateExecutionPorts, @@ -54,12 +66,15 @@ export async function recoverExpiredAgentCandidateExecution( } const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) - const controller = new AbortController() - controller.abort(new Error('recovering an expired candidate execution')) const recoveryTraceStore = new RecoveryAgentCandidateTraceStore(options.traceStore) + const preparation = withinCandidateCleanupDeadline( + () => readRecoveryPreparation(record.claim, options.outputArtifacts), + cleanupDeadlineAtMs, + 'expired candidate preparation evidence read', + ) const processClosure = withinCandidateCleanupDeadline( - async () => { - const stopped = await options.executor.stopAndCapture( + async (cleanupSignal) => { + const stopped = await options.executor.stop( { executionId: record.claim.executionId, executionPlanDigest: record.claim.executionPlanDigest, @@ -67,11 +82,23 @@ export async function recoverExpiredAgentCandidateExecution( { traceStore: recoveryTraceStore, reason: 'failed', - signal: controller.signal, - deadlineAtMs: record.claim.leaseExpiresAtMs, + signal: cleanupSignal, + deadlineAtMs: cleanupDeadlineAtMs, }, ) - sealAgentCandidateExecutorFinalCapture(stopped) + sealAgentCandidateExecutorStopAcknowledgement(stopped) + if (options.executor.dispose) { + const disposed = await options.executor.dispose( + { + executionId: record.claim.executionId, + executionPlanDigest: record.claim.executionPlanDigest, + }, + { signal: cleanupSignal }, + ) + if (disposed.disposed !== true || Object.keys(disposed).some((key) => key !== 'disposed')) { + throw new Error('expired candidate executor did not acknowledge resource disposal') + } + } return { stopped: true as const } }, cleanupDeadlineAtMs, @@ -120,16 +147,29 @@ export async function recoverExpiredAgentCandidateExecution( ) : undefined - const operations = [processClosure, modelClosure, ...(memoryClosure ? [memoryClosure] : [])] - const outcomes = await Promise.allSettled(operations) - const failures = outcomes + const closureOutcomes = Promise.allSettled([ + modelClosure, + memoryClosure ?? Promise.resolve(undefined), + ] as const) + const [preparationOutcome, processOutcome] = await Promise.allSettled([ + preparation, + processClosure, + ] as const) + const failures: unknown[] = [preparationOutcome, processOutcome] .filter((outcome): outcome is PromiseRejectedResult => outcome.status === 'rejected') .map((outcome) => outcome.reason) + const [modelOutcome, memoryOutcome] = await closureOutcomes + for (const outcome of [modelOutcome, memoryOutcome]) { + if (outcome.status === 'rejected') failures.push(outcome.reason) + } if (failures.length > 0) { throw new AggregateError(failures, 'expired candidate cleanup could not be proven') } - const model = await modelClosure + if (modelOutcome.status !== 'fulfilled') { + throw new Error('expired candidate model settlement was not recovered') + } + const model = modelOutcome.value const modelSettlement = await withinCandidateCleanupDeadline( () => persistCandidateModelSettlementEvidence( @@ -144,14 +184,38 @@ export async function recoverExpiredAgentCandidateExecution( cleanupDeadlineAtMs, 'expired candidate model-settlement persistence', ) + const failureClass = + record.phase === 'claimed' && model.usage.modelCalls === 0 + ? 'pre-model-infrastructure' + : 'unknown' + const failureEvidence = await withinCandidateCleanupDeadline( + () => + persistCandidateOutputArtifact(options.outputArtifacts, { + executionId: record.claim.executionId, + purpose: 'failure-evidence', + bytes: canonicalCandidateBytes({ + kind: 'agent-candidate-recovery-failure', + executionId: record.claim.executionId, + attempt: record.claim.attempt, + bundleDigest: record.claim.bundleDigest, + executionPlanDigest: record.claim.executionPlanDigest, + materializationReceiptDigest: + record.claim.preparationEvidence.materializationReceipt.sha256, + failureClass, + processStopped: true, + modelSettlementDigest: modelSettlement.digest, + memoryClosed: record.claim.cleanup.memory !== undefined, + }), + }), + cleanupDeadlineAtMs, + 'expired candidate failure-evidence persistence', + ) const memory = record.claim.cleanup.memory return await options.claimStore.recoverExpired(options.attempt, { - failureClass: - record.phase === 'claimed' && model.fixedUsage.modelCalls === 0 - ? 'pre-model-infrastructure' - : 'unknown', - usage: model.fixedUsage, + failureClass, + usage: model.usage, modelSettlement: modelSettlement.artifact, + failureEvidence, process: { stopped: true, executionPlanDigest: record.claim.executionPlanDigest, @@ -173,3 +237,52 @@ export async function recoverExpiredAgentCandidateExecution( : {}), }) } + +async function readRecoveryPreparation( + claim: AgentCandidateExecutionClaim, + artifacts: AgentCandidateOutputArtifactPort, +): Promise<{ + plan: AgentCandidateExecutionPlanMaterial + receipt: AgentCandidateMaterializationReceipt +}> { + const [planBytes, receiptBytes] = await Promise.all([ + readVerifiedArtifact(claim.preparationEvidence.executionPlan, artifacts), + readVerifiedArtifact(claim.preparationEvidence.materializationReceipt, artifacts), + ]) + const plan = agentCandidateExecutionPlanMaterialSchema.parse( + parseCanonicalBytes(planBytes, 'plan'), + ) + if (claim.preparationEvidence.executionPlan.sha256 !== claim.executionPlanDigest) { + throw new Error('recovery execution plan artifact does not match the claimed digest') + } + const receiptMaterial = parseCanonicalBytes(receiptBytes, 'materialization receipt') + const receipt = agentCandidateMaterializationReceiptSchema.parse({ + ...receiptMaterial, + digest: claim.preparationEvidence.materializationReceipt.sha256, + }) + if ( + receipt.bundleDigest !== claim.bundleDigest || + receipt.executionPlan.digest !== claim.executionPlanDigest || + plan.runCell.bundleDigest !== claim.bundleDigest || + plan.executionId !== claim.executionId + ) { + throw new Error('recovery preparation evidence does not match the durable claim') + } + return { plan, receipt } +} + +function parseCanonicalBytes(bytes: Uint8Array, label: string): Record { + let parsed: unknown + try { + parsed = JSON.parse(Buffer.from(bytes).toString('utf8')) + } catch (error) { + throw new Error(`candidate recovery ${label} is not JSON`, { cause: error }) + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`candidate recovery ${label} must be an object`) + } + if (!Buffer.from(canonicalCandidateBytes(parsed)).equals(Buffer.from(bytes))) { + throw new Error(`candidate recovery ${label} bytes are not canonical`) + } + return parsed as Record +} diff --git a/src/candidate-execution/types.ts b/src/candidate-execution/types.ts index f293a41b..2e98c7a1 100644 --- a/src/candidate-execution/types.ts +++ b/src/candidate-execution/types.ts @@ -2,28 +2,34 @@ import type { BenchmarkEvaluation, TraceStore } from '@tangle-network/agent-eval import type { AgentCandidateArtifactRef, AgentCandidateAttemptPolicy, + AgentCandidateBenchmarkSuite, + AgentCandidateBenchmarkTask, AgentCandidateBundle, AgentCandidateCapturedArtifact, AgentCandidateContainer, AgentCandidateEffectiveMemory, AgentCandidateExecutionLimits, AgentCandidateExecutionPlanEvidence, + AgentCandidateFixedSpend, AgentCandidateGitHubRepository, AgentCandidateInstructionDelivery, AgentCandidateKnowledgeRef, AgentCandidateMaterializationReceipt, AgentCandidateMemoryReceipt, AgentCandidateModelAccessNetwork, + AgentCandidateModelSettlementCall, AgentCandidateOciPlatform, + AgentCandidateProfileActivation, AgentCandidateProfilePlanEvidence, AgentCandidateResolvedModel, + AgentCandidateRunCell, AgentCandidateRunReceipt, - AgentCandidateSpend, AgentCandidateTaskOutcomeEvidence, + AgentCandidateTaskOutcomeMaterial, + AgentCandidateTaskOutputSpec, AgentCandidateTermination, AgentCandidateWorkspaceManifestMaterial, AgentCandidateWorkspaceSnapshotEvidence, - ReasoningEffort, Sha256Digest, } from '@tangle-network/agent-interface' @@ -37,11 +43,14 @@ export interface AgentCandidateArtifactPort { } export type AgentCandidateOutputPurpose = + | 'execution-plan' + | 'materialization-receipt' | 'candidate-workspace-manifest' | 'candidate-workspace-archive' | 'task-manifest' | 'task-archive' | 'task-patch' + | 'task-output' | 'task-outcome' | 'memory-after-manifest' | 'memory-after-archive' @@ -49,8 +58,11 @@ export type AgentCandidateOutputPurpose = | 'benchmark-result' | 'model-settlement' | 'trace' + | 'executor-native-evidence' | 'executor-capture' | 'run-receipt' + | 'knowledge-retrieval-config' + | 'knowledge-evaluation' | 'failure-evidence' /** Durable content-addressed evidence store controlled only by the evaluator. */ @@ -155,12 +167,6 @@ export type AgentCandidateModelLimits = Pick< 'maxModelCalls' | 'maxInputTokens' | 'maxOutputTokens' | 'maxCostUsd' > -export interface AgentCandidateBenchmarkGraderIdentity { - name: string - version: string - artifact: AgentCandidateArtifactRef -} - export interface AgentCandidateProtectedModelReservation { preparationId: string digest: Sha256Digest @@ -177,30 +183,11 @@ export interface AgentCandidateProtectedModelActivation { env: Readonly> } -/** One evaluator-gateway call in the final, revoked model-access ledger. */ -export interface AgentCandidateProtectedModelCall { - callId: string - /** Router-generated public response identity. */ - generationId: string - /** Exact protected agent-eval LLM span produced from the router ledger. */ - traceSpanId: string - status: 'succeeded' | 'failed' - model: string - startedAtMs: number - endedAtMs: number - inputTokens: number - outputTokens: number - cachedInputTokens: number - reasoningTokens: number - /** Integer billionths of one US dollar; avoids floating-point ledger drift. */ - costUsdNanos: number -} - export interface AgentCandidateProtectedModelSettlement { preparationId: string grantDigest: Sha256Digest closed: true - calls: readonly AgentCandidateProtectedModelCall[] + calls: readonly AgentCandidateModelSettlementCall[] } export interface AgentCandidateMemoryResetResult { @@ -258,26 +245,12 @@ export interface AgentCandidateExecutionPorts extends AgentCandidateVerification memory: AgentCandidateMemoryPort } +/** Runtime placement for one exact cell from a signed candidate experiment. */ export interface AgentCandidateTaskExecution { executionId: string - benchmark: string - benchmarkVersion: string - taskId: string - splitDigest: Sha256Digest - /** Exact agent-visible task instruction. The runtime rejects malformed Unicode. */ - instruction: string - repository: { - identity: string - rootIdentity: string - baseCommit: string - baseTree: string - } - attempt: AgentCandidateAttemptPolicy - model: { - requested: string - reasoningEffort: ReasoningEffort - } - grader: AgentCandidateBenchmarkGraderIdentity + runCell: AgentCandidateRunCell + benchmarkSuite: AgentCandidateBenchmarkSuite + task: AgentCandidateBenchmarkTask /** Absolute paths inside the evaluator-owned execution environment. */ executionRoots: { taskRoot: string @@ -289,9 +262,6 @@ export interface AgentCandidateTaskExecution { candidateRoot?: string profileRoot: string } - workspace: AgentCandidateWorkspaceSnapshotEvidence - evaluatorTaskContainer?: ResolvedAgentCandidateContainer - limits: AgentCandidateExecutionLimits } export interface VerifiedAgentCandidate { @@ -338,6 +308,10 @@ export interface PreparedAgentCandidateTrace { export interface PreparedAgentCandidateExecution { readonly bundle: AgentCandidateBundle + readonly benchmark: { + readonly suite: AgentCandidateBenchmarkSuite + readonly task: AgentCandidateBenchmarkTask + } readonly executionId: string readonly roots: { execution: { @@ -355,6 +329,7 @@ export interface PreparedAgentCandidateExecution { bytes: Uint8Array written: readonly string[] } + readonly profileActivation: AgentCandidateProfileActivation readonly executionPlan: { value: AgentCandidateExecutionPlanEvidence bytes: Uint8Array @@ -369,22 +344,31 @@ export interface PreparedAgentCandidateExecution { readonly [preparedCandidateBrand]: true } +export type { AgentCandidateBenchmarkGraderIdentity } from '@tangle-network/agent-interface' + export interface AgentCandidateProtectedRunCapture { executionId: string termination: AgentCandidateTermination } /** Raw evaluator capture made only after the candidate process is dead. */ -export interface AgentCandidateExecutorTaskOutcomeCapture { - /** Claimed final tree. The runtime recomputes it independently from `gitDiff`. */ - resultTree: string - /** Complete evaluator-captured workspace description after candidate execution. */ - afterState: AgentCandidateWorkspaceManifestMaterial - /** Reproducible workspace archive corresponding to `afterState`. */ - archive: Uint8Array - /** Exact binary patch from the signed task base to `afterState`. */ - gitDiff: Uint8Array -} +export type AgentCandidateExecutorTaskOutcomeCapture = + | { + readonly kind: 'workspace' + /** Claimed final tree. The runtime recomputes it independently from `gitDiff`. */ + readonly resultTree: string + /** Complete evaluator-captured workspace description after candidate execution. */ + readonly afterState: AgentCandidateWorkspaceManifestMaterial + /** Reproducible workspace archive corresponding to `afterState`. */ + readonly archive: Uint8Array + /** Exact binary patch from the signed task base to `afterState`. */ + readonly gitDiff: Uint8Array + } + | { + readonly kind: 'output' + /** Exact evaluator-captured final output bytes. */ + readonly bytes: Uint8Array + } /** Raw isolated-memory capture made only after access has been revoked. */ export interface AgentCandidateExecutorMemoryCapture { @@ -392,23 +376,40 @@ export interface AgentCandidateExecutorMemoryCapture { readonly archive: Uint8Array } -/** Idempotent executor result after process death and trace drain. */ +/** Replayable evaluator result captured only after process death and trace drain. */ export interface AgentCandidateExecutorFinalCapture { - readonly stopped: true readonly taskOutcome?: AgentCandidateExecutorTaskOutcomeCapture /** Required only when the prepared candidate uses isolated task memory. */ readonly memoryAfter?: AgentCandidateExecutorMemoryCapture + /** Executor-native bytes preserved when a fresh worker cannot reconstruct a verified outcome. */ + readonly evidence?: Uint8Array } -/** Branded task outcome that has survived independent patch and tree verification. */ -export interface VerifiedAgentCandidateTaskOutcome { - readonly evidence: AgentCandidateTaskOutcomeEvidence & { - readonly artifact: AgentCandidateArtifactRef +type PersistedTaskOutcomeEvidence< + Kind extends AgentCandidateTaskOutcomeMaterial['outcome']['kind'], +> = Omit & { + readonly artifact: AgentCandidateArtifactRef + readonly material: Omit & { + readonly outcome: Extract } - readonly patch: Uint8Array - readonly [verifiedTaskOutcomeBrand]: true } +/** Branded task outcome that has survived independent evaluator verification. */ +export type VerifiedAgentCandidateTaskOutcome = + | { + readonly kind: 'workspace' + readonly evidence: PersistedTaskOutcomeEvidence<'workspace'> + readonly patch: Uint8Array + readonly [verifiedTaskOutcomeBrand]: true + } + | { + readonly kind: 'output' + readonly evidence: PersistedTaskOutcomeEvidence<'output'> + readonly spec: AgentCandidateTaskOutputSpec + readonly bytes: Uint8Array + readonly [verifiedTaskOutcomeBrand]: true + } + /** * Evaluator-owned executable grader, pinned by immutable implementation bytes. * @@ -452,6 +453,7 @@ export interface AgentCandidateBenchmarkGraderPort { /** One detached request passed to the trusted environment-specific executor. */ export interface AgentCandidateExecutorRequest { readonly executionId: string + readonly benchmark: PreparedAgentCandidateExecution['benchmark'] /** Immutable bytes from which the executor creates fresh isolated workspaces. */ readonly inputs: { readonly task: AgentCandidateExecutorWorkspaceInput @@ -462,6 +464,7 @@ export interface AgentCandidateExecutorRequest { } readonly roots: PreparedAgentCandidateExecution['roots']['execution'] readonly profilePlan: PreparedAgentCandidateExecution['profilePlan'] + readonly profileActivation: AgentCandidateProfileActivation readonly executionPlan: PreparedAgentCandidateExecution['executionPlan'] readonly materializationReceipt: CanonicalCandidateDocument readonly launch: PreparedAgentCandidateLaunch @@ -471,7 +474,7 @@ export interface AgentCandidateExecutorRequest { readonly hardLimits: Pick /** Validity bound checked against protected traces; generic black-box executors cannot preempt it. */ readonly observedLimits: Pick - readonly knowledge?: PreparedAgentCandidateExecution['knowledge'] + readonly knowledge?: PreparedAgentCandidateKnowledge readonly trace: PreparedAgentCandidateTrace readonly memory: AgentCandidateEffectiveMemory } @@ -495,14 +498,8 @@ export interface AgentCandidateExecutorPort { deadlineAtMs: number }, ): Promise - /** - * Kill any process/container still associated with the request, drain trace - * writes, and capture the final task workspace before teardown. - * The runtime calls this on success, failure, and timeout before model settlement. - * Implementations must be idempotent and concurrency-safe for this exact - * execution/plan pair because a fresh worker may repeat crash recovery. - */ - stopAndCapture( + /** Kill the exact process/container and drain trace writes. Must be idempotent. */ + stop( request: AgentCandidateExecutorStopRequest, context: { traceStore: TraceStore @@ -512,7 +509,23 @@ export interface AgentCandidateExecutorPort { /** Absolute execution deadline; a later stop acknowledgement cannot produce success. */ deadlineAtMs: number }, + ): Promise<{ readonly stopped: true }> + /** Capture immutable final evidence after stop. Must be replayable by a fresh worker. */ + capture( + request: AgentCandidateExecutorStopRequest, + context: { + traceStore: TraceStore + /** Aborted at the frozen execution deadline or evaluator cleanup deadline. */ + signal: AbortSignal + }, ): Promise + /** Remove evaluator-owned execution resources after final capture. Must be idempotent. */ + dispose?( + request: AgentCandidateExecutorStopRequest, + context: { + signal: AbortSignal + }, + ): Promise<{ readonly disposed: true }> } /** Opaque process identity used for termination without re-exposing launch credentials. */ @@ -532,6 +545,7 @@ export interface AgentCandidateExecutorWorkspaceFile { readonly bytes: Uint8Array } +/** One exact profile file supplied to an evaluator-owned executor. */ export interface AgentCandidateExecutorProfileFile { readonly path: string readonly mode: number @@ -561,7 +575,7 @@ export type AgentCandidateRunFinalization = termination?: AgentCandidateTermination } /** Independent evaluator-gateway usage, even when execution or trace capture failed. */ - usage: AgentCandidateSpend | null + usage: AgentCandidateFixedSpend | null } /** Protected trace tags that bind a run to one prepared candidate execution. */ diff --git a/src/candidate-execution/verify.ts b/src/candidate-execution/verify.ts index 03eec2ad..ec95132d 100644 --- a/src/candidate-execution/verify.ts +++ b/src/candidate-execution/verify.ts @@ -19,6 +19,7 @@ import { omitTopLevelDigest, } from './digest' import { readCandidateGitHubResource, verifyCandidateCode } from './git-materialize' +import { assertCandidateProfileExecutionSupport } from './profile' import { type AgentCandidateVerificationPorts, type VerifiedAgentCandidate, @@ -33,6 +34,22 @@ interface VerifiedCandidateState { const verifiedCandidateState = new WeakMap() +/** Surfaces admitted by Runtime's verifier before an environment adapter is selected. */ +export const AGENT_CANDIDATE_EXECUTION_SUPPORT = Object.freeze({ + outcomes: Object.freeze(['workspace', 'output'] as const), + code: Object.freeze(['disabled', 'no-op', 'git-patch'] as const), + memory: Object.freeze(['disabled', 'isolated'] as const), + knowledge: true, + profile: Object.freeze({ + mcpTransports: Object.freeze(['stdio'] as const), + remoteMcp: false, + tools: false, + permissions: false, + modes: false, + confidential: false, + }), +}) + /** Verifies every digest, resource, workspace, and Git object in a candidate bundle. */ export async function verifyAgentCandidateBundle( input: unknown, @@ -46,6 +63,7 @@ export async function verifyAgentCandidateBundle( } const canonicalBytes = canonicalCandidateBytes(withoutDigest) verifyBytes(canonicalBytes, parsed.digest, canonicalBytes.byteLength, 'candidate bundle') + assertCandidateProfileExecutionSupport(parsed.profile) const artifactBytes = new Map() const readArtifact = async (artifact: AgentCandidateCapturedArtifact): Promise => { diff --git a/src/candidate-execution/workspace-archive.ts b/src/candidate-execution/workspace-archive.ts index 3f13a3ad..83a95733 100644 --- a/src/candidate-execution/workspace-archive.ts +++ b/src/candidate-execution/workspace-archive.ts @@ -73,7 +73,7 @@ const defaultLimits: AgentCandidateWorkspaceArchiveLimits = Object.freeze({ maxRepositoryBundleBytes: 128 * 1024 * 1024, }) -interface WorkspaceArchiveRepositoryV1 { +interface WorkspaceArchiveRepository { headCommit: string headTree: string bundle: { @@ -83,8 +83,7 @@ interface WorkspaceArchiveRepositoryV1 { } } -interface WorkspaceArchiveRepositoryMetadataV1 { - schemaVersion: 1 +interface WorkspaceArchiveRepositoryMetadata { kind: typeof archiveKind headCommit: string headTree: string @@ -96,7 +95,7 @@ interface WorkspaceArchiveRepositoryMetadataV1 { interface DecodedWorkspaceArchive { files: Array<{ path: string; mode: number; bytes: Uint8Array }> - repository?: WorkspaceArchiveRepositoryV1 + repository?: WorkspaceArchiveRepository } export interface CaptureAgentCandidateWorkspaceOptions { @@ -158,7 +157,7 @@ export async function captureAgentCandidateWorkspaceFiles( async function captureWorkspaceFiles( files: readonly AgentCandidateExecutorWorkspaceFile[], - repository: WorkspaceArchiveRepositoryV1 | undefined, + repository: WorkspaceArchiveRepository | undefined, limits: AgentCandidateWorkspaceArchiveLimits, artifactPersistence: CaptureAgentCandidateWorkspaceOptions['artifactPersistence'], ): Promise { @@ -177,7 +176,6 @@ async function captureWorkspaceFiles( const manifestArtifact = await captureWorkspaceArtifact('manifest', manifest, artifactPersistence) const archiveArtifact = await captureWorkspaceArtifact('archive', archive, artifactPersistence) const snapshot = deepFreezeCandidate({ - schemaVersion: 2, kind: 'agent-candidate-workspace-snapshot', digest: canonicalCandidateDigest(material), material, @@ -265,7 +263,7 @@ async function captureRepository( limits: AgentCandidateWorkspaceArchiveLimits, ): Promise<{ files: Array<{ path: string; mode: number; bytes: Uint8Array }> - repository: WorkspaceArchiveRepositoryV1 + repository: WorkspaceArchiveRepository }> { const stats = await lstat(root) if (!stats.isDirectory() || stats.isSymbolicLink() || (await realpath(root)) !== root) { @@ -338,7 +336,7 @@ async function captureRepository( async function encodeWorkspaceArchive( files: ReadonlyArray<{ path: string; mode: number; bytes: Uint8Array }>, - repository: WorkspaceArchiveRepositoryV1 | undefined, + repository: WorkspaceArchiveRepository | undefined, maxArchiveBytes: number, ): Promise { const archive = pack() @@ -356,7 +354,7 @@ async function encodeWorkspaceArchive( async function verifyCanonicalWorkspaceArchive( files: ReadonlyArray<{ path: string; mode: number; bytes: Uint8Array }>, - repository: WorkspaceArchiveRepositoryV1 | undefined, + repository: WorkspaceArchiveRepository | undefined, expected: Uint8Array, maxArchiveBytes: number, ): Promise { @@ -381,14 +379,13 @@ async function verifyCanonicalWorkspaceArchive( async function writeWorkspaceArchiveEntries( archive: Pack, files: ReadonlyArray<{ path: string; mode: number; bytes: Uint8Array }>, - repository: WorkspaceArchiveRepositoryV1 | undefined, + repository: WorkspaceArchiveRepository | undefined, ): Promise { for (const file of files) { await writeTarEntry(archive, `${workspaceEntryPrefix}${file.path}`, file.mode, file.bytes) } if (!repository) return const metadata = canonicalCandidateBytes({ - schemaVersion: 1, kind: archiveKind, headCommit: repository.headCommit, headTree: repository.headTree, @@ -396,7 +393,7 @@ async function writeWorkspaceArchiveEntries( sha256: repository.bundle.sha256, byteLength: repository.bundle.byteLength, }, - } satisfies WorkspaceArchiveRepositoryMetadataV1) + } satisfies WorkspaceArchiveRepositoryMetadata) await writeTarEntry(archive, repositoryMetadataEntry, 0o600, metadata) await writeTarEntry(archive, repositoryBundleEntry, 0o600, repository.bundle.bytes) } @@ -415,7 +412,7 @@ async function parseWorkspaceArchive( let totalBytes = 0 let retainedEntryBytes = 0 let previousPath: string | undefined - let repositoryMetadata: WorkspaceArchiveRepositoryMetadataV1 | undefined + let repositoryMetadata: WorkspaceArchiveRepositoryMetadata | undefined let repositoryBundle: Uint8Array | undefined const decoded = (async () => { for await (const entry of parser) { @@ -433,7 +430,7 @@ async function parseWorkspaceArchive( throw new Error('candidate workspace archive has an invalid file count') } const path = safeArchivePath(name.slice(workspaceEntryPrefix.length), limits.maxPathBytes) - if (!isRegularFileMode(entry.header.mode)) { + if (!isWorkspaceFileMode(entry.header.mode)) { throw new Error(`candidate workspace archive has an unsupported mode: ${path}`) } assertRetainedArchiveSize(bytes.byteLength, retainedEntryBytes, entry.header.size, limits) @@ -517,7 +514,7 @@ function assertRetainedArchiveSize( function parseRepositoryMetadata( bytes: Uint8Array, maxBundleBytes: number, -): WorkspaceArchiveRepositoryMetadataV1 { +): WorkspaceArchiveRepositoryMetadata { let value: unknown try { value = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) @@ -526,8 +523,7 @@ function parseRepositoryMetadata( } if ( !isRecord(value) || - Object.keys(value).sort().join(',') !== 'bundle,headCommit,headTree,kind,schemaVersion' || - value.schemaVersion !== 1 || + Object.keys(value).sort().join(',') !== 'bundle,headCommit,headTree,kind' || value.kind !== archiveKind || typeof value.headCommit !== 'string' || typeof value.headTree !== 'string' || @@ -550,7 +546,6 @@ function parseRepositoryMetadata( throw new Error('candidate repository HEAD and tree use different object formats') } return { - schemaVersion: 1, kind: archiveKind, headCommit: value.headCommit, headTree: value.headTree, @@ -562,9 +557,9 @@ function parseRepositoryMetadata( } function repositoryFromTar( - metadata: WorkspaceArchiveRepositoryMetadataV1, + metadata: WorkspaceArchiveRepositoryMetadata, bundle: Uint8Array, -): WorkspaceArchiveRepositoryV1 { +): WorkspaceArchiveRepository { verifyBytes(bundle, metadata.bundle.sha256, metadata.bundle.byteLength, 'candidate Git bundle') return { headCommit: metadata.headCommit, @@ -699,7 +694,7 @@ async function writeWorkspaceFiles( } async function materializeRepository( - repository: WorkspaceArchiveRepositoryV1, + repository: WorkspaceArchiveRepository, destination: string, ): Promise { const temporary = await mkdtemp(join(tmpdir(), 'agent-candidate-workspace-bundle-')) @@ -780,7 +775,7 @@ function assertWorkspaceFilesWithinLimits( 'candidate workspace paths must be unique and sorted', ) previousPath = path - if (!isRegularFileMode(file.mode)) { + if (!isWorkspaceFileMode(file.mode)) { throw new Error(`candidate workspace file has unsupported mode: ${path}`) } if (file.bytes.byteLength > limits.maxFileBytes) { @@ -822,7 +817,7 @@ function normalizeWorkspaceFiles( const path = safeArchivePath(descriptors.path?.value, limits.maxPathBytes) const mode = descriptors.mode?.value const inputBytes = descriptors.bytes?.value - if (!isRegularFileMode(mode)) { + if (!isWorkspaceFileMode(mode)) { throw new Error(`candidate workspace file has unsupported mode: ${path}`) } const view = inspectWorkspaceBytes(inputBytes, path) @@ -843,8 +838,8 @@ function normalizeWorkspaceFiles( return files } -function isRegularFileMode(value: unknown): value is number { - return Number.isInteger(value) && Number(value) >= 0 && Number(value) <= 0o777 +function isWorkspaceFileMode(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0 && (value as number) <= 0o777 } function workspaceLimits( diff --git a/src/candidate-execution/workspace-snapshot.ts b/src/candidate-execution/workspace-snapshot.ts deleted file mode 100644 index 19ade3e0..00000000 --- a/src/candidate-execution/workspace-snapshot.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { - AgentCandidateWorkspaceManifestMaterial, - AgentCandidateWorkspaceSnapshotEvidence, -} from '@tangle-network/agent-interface' -import { - agentCandidateWorkspaceManifestMaterialSchema, - agentCandidateWorkspaceSnapshotEvidenceSchema, -} from '@tangle-network/agent-interface' - -import { - canonicalCandidateBytes, - embeddedCandidateArtifact, - immutableCandidateValue, - sha256Bytes, -} from './digest' -import { persistCandidateOutputArtifact } from './output-artifacts' -import type { AgentCandidateOutputArtifactPort } from './types' - -export interface ProvisionalCandidateWorkspaceSnapshot { - readonly material: AgentCandidateWorkspaceManifestMaterial - readonly manifestBytes: Uint8Array - readonly snapshot: AgentCandidateWorkspaceSnapshotEvidence -} - -/** - * Build materialization-only evidence without parsing a large base64 field. - * Every field is evaluator-derived, and callers must not persist or expose this - * provisional value as receipt evidence. - */ -export function provisionalCandidateWorkspaceSnapshot( - materialInput: AgentCandidateWorkspaceManifestMaterial, - archive: Uint8Array, -): ProvisionalCandidateWorkspaceSnapshot { - const material = immutableCandidateValue( - agentCandidateWorkspaceManifestMaterialSchema.parse(materialInput), - ) - const manifestBytes = canonicalCandidateBytes(material) - const snapshot: AgentCandidateWorkspaceSnapshotEvidence = Object.freeze({ - schemaVersion: 2, - kind: 'agent-candidate-workspace-snapshot', - digest: sha256Bytes(manifestBytes), - material, - manifest: Object.freeze(embeddedCandidateArtifact(manifestBytes)), - archive: Object.freeze(embeddedCandidateArtifact(archive)), - }) - return Object.freeze({ material, manifestBytes, snapshot }) -} - -/** Persist verified workspace bytes and construct final reference-only evidence. */ -export async function persistCandidateWorkspaceSnapshot( - port: AgentCandidateOutputArtifactPort, - input: { - executionId: string - material: AgentCandidateWorkspaceManifestMaterial - archive: Uint8Array - purpose: 'task' | 'memory-after' - signal?: AbortSignal - }, -): Promise { - input.signal?.throwIfAborted() - const material = immutableCandidateValue(input.material) - const manifestBytes = canonicalCandidateBytes(material) - const [manifest, archive] = await Promise.all([ - persistCandidateOutputArtifact(port, { - executionId: input.executionId, - purpose: input.purpose === 'task' ? 'task-manifest' : 'memory-after-manifest', - bytes: manifestBytes, - ...(input.signal ? { signal: input.signal } : {}), - }), - persistCandidateOutputArtifact(port, { - executionId: input.executionId, - purpose: input.purpose === 'task' ? 'task-archive' : 'memory-after-archive', - bytes: input.archive, - ...(input.signal ? { signal: input.signal } : {}), - }), - ]) - return immutableCandidateValue( - agentCandidateWorkspaceSnapshotEvidenceSchema.parse({ - schemaVersion: 2, - kind: 'agent-candidate-workspace-snapshot', - digest: sha256Bytes(manifestBytes), - material, - manifest, - archive, - }), - ) -} diff --git a/src/candidate-execution/workspace-task.ts b/src/candidate-execution/workspace-task.ts deleted file mode 100644 index 0f378e60..00000000 --- a/src/candidate-execution/workspace-task.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { - AgentCandidateExecutionPlanMaterial, - AgentCandidateTaskRepository, -} from '@tangle-network/agent-interface' - -/** Narrow the runtime's workspace-only task contract after schema validation. */ -export function workspaceTaskRepository( - task: AgentCandidateExecutionPlanMaterial['task'], -): AgentCandidateTaskRepository { - if (task.outcome.kind !== 'workspace' || !task.repository) { - throw new Error('candidate runtime requires a workspace outcome with repository provenance') - } - return task.repository -} diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index 6e89c63b..cb31b9a5 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -81,7 +81,6 @@ export interface VerifyResult { export type Verifier = (worktreePath: string) => Promise | VerifyResult export interface AgenticGeneratorShotReceipt { - readonly schemaVersion: 1 readonly generation: number | null readonly candidateIndex: number | null /** One-based shot number within this candidate. */ @@ -250,7 +249,7 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG if (opts.codexReproducible) { if (!costLedger) { throw new Error( - 'agenticGenerator: reproducible Codex requires the run-wide CostLedger supplied by agent-eval 0.117+', + 'agenticGenerator: reproducible Codex requires the run-wide CostLedger supplied by agent-eval', ) } reproducibleCostLedger = costLedger @@ -674,7 +673,6 @@ function shotReceipt(input: { const result = input.result const costBasis = costBasisFor(input.costReceipt) return { - schemaVersion: 1, generation: input.generation ?? null, candidateIndex: input.candidateIndex ?? null, shot: input.shot + 1, diff --git a/src/improvement/cleanup.ts b/src/improvement/cleanup.ts new file mode 100644 index 00000000..bbe0d18d --- /dev/null +++ b/src/improvement/cleanup.ts @@ -0,0 +1,18 @@ +export async function rethrowAfterCleanup( + cause: unknown, + cleanup: () => Promise, + context: string, +): Promise { + const cleanupErrors: unknown[] = [] + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + await cleanup() + } catch (cleanupCause) { + cleanupErrors.push(cleanupCause) + continue + } + if (cleanupErrors.length === 0) throw cause + throw new AggregateError([cause, ...cleanupErrors], `${context}; cleanup retry succeeded`) + } + throw new AggregateError([cause, ...cleanupErrors], `${context}; cleanup failed`) +} diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index f259905f..93d73ccb 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -56,6 +56,7 @@ import { ConfigError } from '../errors' import type { LocalHarness } from '../mcp/local-harness' import { assertModelAllowed } from '../runtime/supervise/model-policy' import { agenticGenerator, type Verifier } from './agentic-generator' +import { rethrowAfterCleanup } from './cleanup' import { type CandidateGenerator, improvementDriver, @@ -348,29 +349,6 @@ interface PreparedCodeRun { cleanup(retainedWinner?: MutableSurface): Promise } -/** Preserve the primary failure while making two best-effort cleanup attempts. - * A failed first attempt is retained in the thrown AggregateError even when the - * retry succeeds, so callers can diagnose degraded cleanup without losing the - * error that caused cleanup to run. */ -async function rethrowAfterCleanup( - cause: unknown, - cleanup: () => Promise, - message: string, -): Promise { - const cleanupErrors: unknown[] = [] - for (let attempt = 0; attempt < 2; attempt += 1) { - try { - await cleanup() - } catch (cleanupCause) { - cleanupErrors.push(cleanupCause) - continue - } - if (cleanupErrors.length === 0) throw cause - throw new AggregateError([cause, ...cleanupErrors], `${message}; the cleanup retry succeeded`) - } - throw new AggregateError([cause, ...cleanupErrors], message) -} - async function discardPreparedBaseline( worktree: WorktreeAdapter, baselineWorktree: Worktree, @@ -379,7 +357,7 @@ async function discardPreparedBaseline( return rethrowAfterCleanup( cause, () => worktree.discard(baselineWorktree), - 'improve(): code preparation failed and its baseline worktree could not be cleaned', + 'improve(): code preparation failed', ) } @@ -628,7 +606,7 @@ export async function improve( return rethrowAfterCleanup( cause, () => preparedCode.cleanup(), - 'improve(): code improvement failed and its worktrees could not be cleaned', + 'improve(): code improvement failed', ) } diff --git a/src/improvement/improvement-driver.ts b/src/improvement/improvement-driver.ts index 6881c5e4..ec63c93b 100644 --- a/src/improvement/improvement-driver.ts +++ b/src/improvement/improvement-driver.ts @@ -31,6 +31,7 @@ import { type Worktree, type WorktreeAdapter, } from '@tangle-network/agent-eval/campaign' +import { rethrowAfterCleanup } from './cleanup' /** The byte-producing seam — the ONE thing that differs between the cheap * reflective path and the full agentic path. A generator makes (uncommitted) @@ -146,24 +147,14 @@ export function improvementDriver(opts: ImprovementDriverOptions): ManagedImprov owned.delete(wt.path) owned.set(surface.worktreeRef, wt) } catch (err) { - const cleanupErrors: unknown[] = [] - for (let attempt = 0; attempt < 2; attempt += 1) { - try { + const failure = err instanceof Error ? err.message : String(err) + return rethrowAfterCleanup( + err, + async () => { await opts.worktree.discard(wt) owned.delete(wt.path) - break - } catch (cause) { - cleanupErrors.push(cause) - } - } - if (cleanupErrors.length === 0) throw err - const failure = err instanceof Error ? err.message : String(err) - const cleanupSucceeded = !owned.has(wt.path) - throw new AggregateError( - [err, ...cleanupErrors], - cleanupSucceeded - ? `improvementDriver: ${failure}; candidate cleanup retry succeeded` - : `improvementDriver: ${failure}; candidate worktree could not be cleaned`, + }, + `improvementDriver: ${failure}`, ) } } diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 0b916825..2d626b8c 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -40,7 +40,6 @@ export { } from './improvement-driver' export { type McpServeSpec, mcpServeVerifier } from './mcp-serve-verifier' export { - type AgentProfileDiffProposal, type ProfileDiffProposerContext, type ProfileDiffProposerOptions, profileDiffProposer, diff --git a/src/improvement/profile-diff-proposer.test.ts b/src/improvement/profile-diff-proposer.test.ts index 504bfe21..2bb7bde6 100644 --- a/src/improvement/profile-diff-proposer.test.ts +++ b/src/improvement/profile-diff-proposer.test.ts @@ -25,31 +25,28 @@ describe('profileDiffProposer', () => { it('puts source-grounded profile diffs and generated mutations in one candidate pool', async () => { const discovered = profileDiffProposer({ proposeDiffs: () => [ - { - diff: defineAgentProfileDiff({ - schemaVersion: 1, - kind: 'agent-profile-diff', - id: 'github-review-skill', - title: 'Pinned review skill', - rationale: 'Existing skill covers the observed review failure.', - source: { - kind: 'frontier-author', - artifacts: ['https://github.com/example/skills/tree/0123456789abcdef/review'], - notes: ['MIT license verified by the research worker.'], - }, - set: { - resources: { - skills: [ - defineInlineResource( - 'review.SKILL.md', - '# Review\n\nRead the failing test before editing.', - ), - ], - failOnError: true, - }, + defineAgentProfileDiff({ + kind: 'agent-profile-diff', + id: 'github-review-skill', + title: 'Pinned review skill', + rationale: 'Existing skill covers the observed review failure.', + source: { + kind: 'frontier-author', + artifacts: ['https://github.com/example/skills/tree/0123456789abcdef/review'], + notes: ['MIT license verified by the research worker.'], + }, + set: { + resources: { + skills: [ + defineInlineResource( + 'review.SKILL.md', + '# Review\n\nRead the failing test before editing.', + ), + ], + failOnError: true, }, - }), - }, + }, + }), ], }) const generated: SurfaceProposer = { @@ -99,13 +96,10 @@ describe('profileDiffProposer', () => { const proposer = profileDiffProposer({ proposeDiffs: () => [ { - diff: { - schemaVersion: 1, - kind: 'agent-profile-diff', - set: { prompt: { systemPrompt: 'candidate' } }, - unknownField: true, - } as never, - }, + kind: 'agent-profile-diff', + set: { prompt: { systemPrompt: 'candidate' } }, + unknownField: true, + } as never, ], }) await expect(proposer.propose(context(1))).rejects.toThrow(/unsupported or non-canonical/) @@ -114,28 +108,19 @@ describe('profileDiffProposer', () => { it('drops no-op diffs and respects the shared population budget', async () => { const proposer = profileDiffProposer({ proposeDiffs: () => [ - { diff: defineAgentProfileDiff({ schemaVersion: 1, kind: 'agent-profile-diff' }) }, - { - diff: defineAgentProfileDiff({ - schemaVersion: 1, - kind: 'agent-profile-diff', - set: { prompt: { systemPrompt: 'baseline' } }, - }), - }, - { - diff: defineAgentProfileDiff({ - schemaVersion: 1, - kind: 'agent-profile-diff', - set: { prompt: { systemPrompt: 'first' } }, - }), - }, - { - diff: defineAgentProfileDiff({ - schemaVersion: 1, - kind: 'agent-profile-diff', - set: { prompt: { systemPrompt: 'first' } }, - }), - }, + defineAgentProfileDiff({ kind: 'agent-profile-diff' }), + defineAgentProfileDiff({ + kind: 'agent-profile-diff', + set: { prompt: { systemPrompt: 'baseline' } }, + }), + defineAgentProfileDiff({ + kind: 'agent-profile-diff', + set: { prompt: { systemPrompt: 'first' } }, + }), + defineAgentProfileDiff({ + kind: 'agent-profile-diff', + set: { prompt: { systemPrompt: 'first' } }, + }), ], }) const candidates = await proposer.propose(context(2)) diff --git a/src/improvement/profile-diff-proposer.ts b/src/improvement/profile-diff-proposer.ts index 8eed304c..90e2ddbc 100644 --- a/src/improvement/profile-diff-proposer.ts +++ b/src/improvement/profile-diff-proposer.ts @@ -17,12 +17,6 @@ import { parseExactAgentProfileDiff, } from '../candidate-execution/profile' -export interface AgentProfileDiffProposal { - diff: AgentProfileDiff - label?: string - rationale?: string -} - export type ProfileDiffProposerContext = ProposeContext & { profile: AgentProfile } @@ -30,7 +24,7 @@ export type ProfileDiffProposerContext = ProposeContext { proposeDiffs( context: ProfileDiffProposerContext, - ): Promise | readonly AgentProfileDiffProposal[] + ): Promise | readonly AgentProfileDiff[] } /** @@ -55,13 +49,13 @@ export function profileDiffProposer( const candidates: ProposedCandidate[] = [] const normalizedBaseline = applyExactAgentProfileDiff( profile, - { schemaVersion: 1, kind: 'agent-profile-diff' }, + { kind: 'agent-profile-diff' }, 'profile diff baseline', ) const seen = new Set([canonicalJson(normalizedBaseline)]) for (const [index, proposal] of proposals.entries()) { if (candidates.length >= ctx.populationSize || ctx.signal.aborted) break - const diff = parseExactAgentProfileDiff(proposal.diff, `profile diff proposal ${index}`) + const diff = parseExactAgentProfileDiff(proposal, `profile diff proposal ${index}`) const axes = changedAgentProfileAxes(diff) if (axes.length === 0) continue const candidate = applyExactAgentProfileDiff( @@ -74,8 +68,8 @@ export function profileDiffProposer( seen.add(surface) candidates.push({ surface, - label: proposal.label ?? diff.title ?? diff.id ?? `Change ${axes.join(', ')}`, - rationale: candidateRationale(proposal, diff, axes), + label: diff.title ?? diff.id ?? `Change ${axes.join(', ')}`, + rationale: candidateRationale(diff, axes), }) } return candidates @@ -83,12 +77,8 @@ export function profileDiffProposer( } } -function candidateRationale( - proposal: AgentProfileDiffProposal, - diff: AgentProfileDiff, - axes: readonly string[], -): string { - const reason = proposal.rationale ?? diff.rationale ?? `Changes profile axes: ${axes.join(', ')}.` +function candidateRationale(diff: AgentProfileDiff, axes: readonly string[]): string { + const reason = diff.rationale ?? `Changes profile axes: ${axes.join(', ')}.` const source = diff.source ? [diff.source.kind, ...(diff.source.artifacts ?? [])].join(' · ') : 'source unspecified' diff --git a/src/intelligence/delivery.test.ts b/src/intelligence/delivery.test.ts index f87f1798..a01f494c 100644 --- a/src/intelligence/delivery.test.ts +++ b/src/intelligence/delivery.test.ts @@ -83,7 +83,6 @@ describe('pullCertified', () => { it('deserializes the typed agentProfileDiffs the composed endpoint returns', async () => { const diff: AgentProfileDiff = { - schemaVersion: 1, kind: 'agent-profile-diff', set: { tools: { refund: true } }, } diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts index 69226b21..a9065cbc 100644 --- a/src/intelligence/improvement-cycle.ts +++ b/src/intelligence/improvement-cycle.ts @@ -1,22 +1,49 @@ import { type AnalystFinding, assertNoJudgeVerdict } from '@tangle-network/agent-eval/analyst' -import type { Scenario, SelfImproveResult } from '@tangle-network/agent-eval/contract' +import { + type CandidateExperimentExecutionInput, + type CompareCandidateExperimentOptions, + measuredComparisonFromCandidateExperiment, + runCandidateExperiment, + type Scenario, + verifyCandidateExperiment, + verifyCandidateExperimentComparison, +} from '@tangle-network/agent-eval/contract' import type { + AgentCandidateBenchmarkCellRef, AgentCandidateBundle, + AgentCandidateExperiment, + AgentCandidateExperimentMeasurement, + AgentCandidateRunCell, + AgentImprovementActivation, + AgentImprovementActivationTarget, + AgentImprovementMeasuredComparison, + AgentImprovementProposal, + AgentImprovementReview, + AgentImprovementReviewDecision, + AgentImprovementSurface, AgentProfile, + CandidateExecutionEvidence, Sha256Digest, } from '@tangle-network/agent-interface' -import { agentCandidateBundleSchema } from '@tangle-network/agent-interface' +import { + agentCandidateMaterializationReceiptSchema, + agentCandidateRunReceiptSchema, + agentImprovementActivationSchema, + agentImprovementProposalSchema, + agentImprovementReviewSchema, + candidateExecutionEvidenceSchema, +} from '@tangle-network/agent-interface' +import { materializeCandidateProfile } from '@tangle-network/agent-profile-materialize' + import { runAnalystLoop } from '../analyst-loop' import type { RunAnalystLoopOpts, RunAnalystLoopResult } from '../analyst-loop/types' import { - type AgentCandidateBundleInput, - sealAgentCandidateBundle, -} from '../candidate-execution/bundle' -import { + canonicalCandidateBytes, canonicalCandidateDigest, canonicalCandidateDocument, immutableCandidateValue, omitTopLevelDigest, + verifyCanonicalCandidateDocument, } from '../candidate-execution/digest' import { type ExecutePreparedAgentCandidateOptions, @@ -27,81 +54,104 @@ import { prepareAgentCandidateExecution, } from '../candidate-execution/prepare' import { - assertCandidateProfileBinding, - parseExactAgentProfile, + agentCandidateProfileAsAgentProfile, + candidateMaterializerHarness, + createAgentCandidateProfileActivation, + parseAgentCandidateProfileActivation, } from '../candidate-execution/profile' import type { AgentCandidateExecutionPorts, AgentCandidateRunFinalization, AgentCandidateTaskExecution, } from '../candidate-execution/types' -import { verifyAgentCandidateBundle } from '../candidate-execution/verify' import { - applyImprovementWinnerToProfile, - type ImproveOptions, - type ImproveResult, - type ImproveSurface, - improve, -} from '../improvement/improve' - -export type AgentImprovementEvaluation = Pick< - SelfImproveResult, - | 'baseline' - | 'winner' - | 'lift' - | 'diff' - | 'provenance' - | 'gateDecision' - | 'generationsExplored' - | 'durationMs' - | 'totalCostUsd' - | 'insight' - | 'power' -> - -export interface AgentImprovementProposal< - TScenario extends Scenario = Scenario, - TArtifact = unknown, -> { - schemaVersion: 1 - kind: 'agent-improvement-proposal' + verifiedResourceTextByDigest, + verifyAgentCandidateBundle, +} from '../candidate-execution/verify' +import { rethrowAfterCleanup } from '../improvement/cleanup' +import { type ImproveOptions, type ImproveResult, improve } from '../improvement/improve' + +export type { + AgentImprovementActivation, + AgentImprovementMeasuredComparison, + AgentImprovementProposal, + AgentImprovementReview, + AgentImprovementReviewDecision, + CandidateExecutionEvidence, +} from '@tangle-network/agent-interface' + +export interface AgentCandidateExperimentCellPlacement { + executionId: string + attempt?: number + executionRoots: AgentCandidateTaskExecution['executionRoots'] + stagingRoots: AgentCandidateTaskExecution['stagingRoots'] + ports: AgentCandidateExecutionPorts + preparation?: PrepareAgentCandidateExecutionOptions + execution: ExecutePreparedAgentCandidateOptions +} + +export interface RunAgentCandidateExperimentOptions + extends Omit { + experiment: AgentCandidateExperiment + placeCell: ( + input: CandidateExperimentExecutionInput, + ) => AgentCandidateExperimentCellPlacement | Promise + maxConcurrency?: number + signal?: AbortSignal +} + +export interface RunAgentCandidateExperimentResult { + experiment: AgentCandidateExperiment + measurements: AgentCandidateExperimentMeasurement[] + evaluation: AgentImprovementMeasuredComparison +} + +export interface ExecuteAgentCandidateExperimentCellOptions + extends CandidateExperimentExecutionInput, + AgentCandidateExperimentCellPlacement {} + +export interface VerifyCandidateExecutionEvidenceOptions { + experiment: AgentCandidateExperiment + arm: 'baseline' | 'candidate' + benchmarkCell: AgentCandidateBenchmarkCellRef + seed: number + attempt?: number + resolvedResources?: ReadonlyMap +} + +/** A failed baseline or candidate cell with its complete Runtime failure result. */ +export class AgentCandidateExperimentCellExecutionError extends Error { + readonly finalization: Extract + + constructor(finalization: Extract) { + super(`candidate experiment cell failed: ${finalization.reason}`) + this.name = 'AgentCandidateExperimentCellExecutionError' + this.finalization = finalization + } +} + +export interface CreateAgentImprovementProposalOptions { runId: string - surface: ImproveSurface - proposedAt: string - baselineProfile: AgentProfile - baselineProfileHash: string - candidateProfile: AgentProfile - candidateProfileHash: string - findings: AnalystFinding[] - evaluation: AgentImprovementEvaluation - candidateBundle?: AgentCandidateBundle - digest: Sha256Digest + findings: readonly AnalystFinding[] + evaluation: AgentImprovementMeasuredComparison + now?: () => Date } -export type AgentImprovementReviewDecision = 'approve' | 'reject' | 'request-changes' +export type CreateAgentImprovementMeasuredComparisonOptions = CompareCandidateExperimentOptions -export interface AgentImprovementReview { - schemaVersion: 1 - kind: 'agent-improvement-review' - proposalDigest: Sha256Digest - candidateBundleDigest?: Sha256Digest +export interface ReviewAgentImprovementInput { decision: AgentImprovementReviewDecision reviewedBy: string - reviewedAt: string reason: string feedback?: string - digest: Sha256Digest + now?: () => Date } -export interface CandidateExecutionEvidence { - proposalDigest: Sha256Digest - reviewDigest: Sha256Digest - bundleDigest: Sha256Digest - executionId: string - executionPlanDigest: Sha256Digest - materializationReceiptDigest: Sha256Digest - succeeded: boolean - runReceiptDigest?: Sha256Digest +export interface CreateAgentImprovementActivationOptions { + targets: [AgentImprovementActivationTarget, ...AgentImprovementActivationTarget[]] + fundingOwner: string + authorizedBy: string + now?: () => Date } export interface ProposeAgentImprovementOptions { @@ -109,90 +159,125 @@ export interface ProposeAgentImprovementOptions improvement: ImproveOptions - /** - * Optional environment adapter that freezes an executable bundle after the - * measured comparison recommends the candidate. Return the sealed output of - * `buildAgentCandidateBundle` directly, or a low-level digest-free input. - */ - buildCandidate?: (input: { + buildExperiment: (input: { analysis: RunAnalystLoopResult improvement: ImproveResult - }) => - | AgentCandidateBundleInput - | AgentCandidateBundle - | Promise + }) => AgentCandidateExperiment | Promise + placeCell: RunAgentCandidateExperimentOptions['placeCell'] + maxConcurrency?: number + signal?: AbortSignal + candidate?: AgentImprovementMeasuredComparison['candidate'] + metadata?: AgentImprovementMeasuredComparison['metadata'] now?: () => Date } export interface ProposeAgentImprovementResult { analysis: RunAnalystLoopResult improvement: ImproveResult - proposal: AgentImprovementProposal -} - -export interface CreateAgentImprovementProposalOptions { - runId: string - surface: ImproveSurface - baselineProfile: AgentProfile - candidateProfile: AgentProfile - findings: readonly AnalystFinding[] - evaluation: SelfImproveResult - candidateBundle?: AgentCandidateBundleInput | AgentCandidateBundle - now?: () => Date -} - -export interface ReviewAgentImprovementInput { - decision: AgentImprovementReviewDecision - reviewedBy: string - reason: string - feedback?: string - now?: () => Date + experiment: AgentCandidateExperiment + measurements: AgentCandidateExperimentMeasurement[] + proposal: AgentImprovementProposal } -export interface ExecuteApprovedAgentCandidateOptions { - proposal: AgentImprovementProposal - review: AgentImprovementReview - /** Product-owned authentication check for the persisted approval record. */ - authorizeReview: ( - review: AgentImprovementReview, - proposal: AgentImprovementProposal, - ) => boolean | Promise - task: AgentCandidateTaskExecution - ports: AgentCandidateExecutionPorts - preparation?: PrepareAgentCandidateExecutionOptions - execution: ExecutePreparedAgentCandidateOptions +type AnalystImprovementProposal = Omit & { + findings: AnalystFinding[] } -export interface ExecuteApprovedAgentCandidateResult { - finalization: AgentCandidateRunFinalization - evidence: CandidateExecutionEvidence +/** Execute both arms of one immutable experiment and derive its paired result. */ +export async function runAgentCandidateExperiment( + options: RunAgentCandidateExperimentOptions, +): Promise { + const experiment = verifyCandidateExperiment(options.experiment) + const measurements = await runCandidateExperiment({ + experiment, + ...(options.maxConcurrency === undefined ? {} : { maxConcurrency: options.maxConcurrency }), + ...(options.signal ? { signal: options.signal } : {}), + execute: async (input) => { + const placement = await options.placeCell(input) + return await executeAgentCandidateExperimentCell({ ...input, ...placement }) + }, + }) + const evaluation = createAgentImprovementMeasuredComparison({ + experiment, + measurements, + runId: options.runId, + ...(options.candidate ? { candidate: options.candidate } : {}), + ...(options.generationsExplored === undefined + ? {} + : { generationsExplored: options.generationsExplored }), + ...(options.searchDurationMs === undefined + ? {} + : { searchDurationMs: options.searchDurationMs }), + ...(options.searchCostUsd === undefined ? {} : { searchCostUsd: options.searchCostUsd }), + ...(options.metadata ? { metadata: options.metadata } : {}), + }) + return { experiment, measurements, evaluation } } -async function rethrowAfterImprovementDispose( - cause: unknown, - improvement: ImproveResult, -): Promise { - const disposeErrors: unknown[] = [] - for (let attempt = 0; attempt < 2; attempt += 1) { - try { - await improvement.dispose() - } catch (disposeCause) { - disposeErrors.push(disposeCause) - continue - } - if (disposeErrors.length === 0) throw cause - throw new AggregateError( - [cause, ...disposeErrors], - 'proposeAgentImprovement failed; the improvement cleanup retry succeeded', - ) +/** Execute one exact arm, task, repetition, seed, and attempt through Runtime. */ +export async function executeAgentCandidateExperimentCell( + options: ExecuteAgentCandidateExperimentCellOptions, +): Promise { + const experiment = verifyCandidateExperiment(options.experiment) + const bundle = experiment[options.arm] + assertExactExperimentInput(options, experiment, bundle) + const attempt = options.attempt ?? 1 + if (attempt > options.task.attempt.maxAttempts) { + throw new Error('candidate experiment attempt exceeds the signed task policy') } - throw new AggregateError( - [cause, ...disposeErrors], - 'proposeAgentImprovement failed and its improvement resources could not be cleaned', + const runCell = canonicalCandidateDocument({ + kind: 'agent-candidate-run-cell', + experimentDigest: experiment.digest, + arm: options.arm, + bundleDigest: bundle.digest, + suiteDigest: options.benchmarkCell.suiteDigest, + taskDigest: options.task.digest, + taskIndex: options.benchmarkCell.taskIndex, + repetition: options.benchmarkCell.repetition, + seed: options.seed, + attempt, + }).value + const verified = await verifyAgentCandidateBundle(bundle, options.ports) + const prepared = await prepareAgentCandidateExecution( + verified, + { + executionId: options.executionId, + runCell, + benchmarkSuite: experiment.benchmark.suite, + task: options.task, + executionRoots: options.executionRoots, + stagingRoots: options.stagingRoots, + }, + options.ports, + options.preparation, ) + const finalization = await executePreparedAgentCandidate(prepared, options.execution) + if (!finalization.succeeded) { + throw new AgentCandidateExperimentCellExecutionError(finalization) + } + const evidence = canonicalCandidateDocument({ + kind: 'agent-candidate-execution-evidence', + materializationReceipt: prepared.materializationReceipt.value, + receipt: finalization.receipt.value, + }).value + return verifyCandidateExecutionEvidence(evidence, { + experiment, + arm: options.arm, + benchmarkCell: options.benchmarkCell, + seed: options.seed, + attempt, + resolvedResources: verifiedResourceTextByDigest(verified), + }) +} + +/** Delegate all statistics and promotion checks to agent-eval's receipt-based comparison. */ +export function createAgentImprovementMeasuredComparison( + options: CreateAgentImprovementMeasuredComparisonOptions, +): AgentImprovementMeasuredComparison { + return verifyCandidateExperimentComparison(measuredComparisonFromCandidateExperiment(options)) } -/** Analyze one run and produce one measured, review-only improvement proposal. */ +/** Analyze, search, then remeasure the resulting exact candidate before proposing it. */ export async function proposeAgentImprovement( options: ProposeAgentImprovementOptions, ): Promise> { @@ -202,95 +287,89 @@ export async function proposeAgentImprovement improvement.dispose(), 'proposeAgentImprovement failed') } } -/** - * Freeze an already-measured improvement into the one reviewable proposal - * contract. Products that run analysis or evaluation in separate workers use - * this constructor instead of rerunning either phase or rebuilding digests. - */ -export function createAgentImprovementProposal( - options: CreateAgentImprovementProposalOptions, -): AgentImprovementProposal { +/** Create the reviewable record only from a complete, recomputable experiment result. */ +export function createAgentImprovementProposal( + options: CreateAgentImprovementProposalOptions, +): AgentImprovementProposal { const findings = assertNoJudgeVerdict( [...options.findings], 'createAgentImprovementProposal findings', ) - const candidateBundle = options.candidateBundle - ? sealBuiltCandidate(options.candidateBundle) - : undefined - const baselineProfile = parseExactAgentProfile( - options.baselineProfile, - 'proposal baseline profile', - ) - const candidateProfile = parseExactAgentProfile( - options.candidateProfile, - 'proposal candidate profile', - ) - assertMeasuredProfileBinding({ - surface: options.surface, - baselineProfile, - candidateProfile, - evaluation: options.evaluation, - hasExecutableBundle: candidateBundle !== undefined, - }) - if (candidateBundle) { - if (options.evaluation.gateDecision !== 'ship') { - throw new Error('executable candidate bundle requires a passing measured comparison') - } - assertCandidateProfileBinding(candidateProfile, candidateBundle.profile) + const evaluation = verifyCandidateExperimentComparison(options.evaluation) + if (evaluation.decision.outcome !== 'ship') { + throw new Error('agent improvement proposal requires a passing experiment') } - const withoutDigest = { - schemaVersion: 1 as const, - kind: 'agent-improvement-proposal' as const, - runId: options.runId, - surface: options.surface, - proposedAt: (options.now ?? (() => new Date()))().toISOString(), - baselineProfile, - baselineProfileHash: canonicalCandidateDigest(baselineProfile), - candidateProfile, - candidateProfileHash: canonicalCandidateDigest(candidateProfile), - findings: canonicalJsonValue([...findings]), - evaluation: canonicalJsonValue(improvementEvaluation(options.evaluation)), - ...(candidateBundle ? { candidateBundle } : {}), + if (options.runId !== evaluation.provenance.runId) { + throw new Error('proposal runId does not match its measured experiment') } - return canonicalCandidateDocument>(withoutDigest) - .value + const changedSurfaces = deriveChangedSurfaces( + evaluation.experiment.baseline, + evaluation.experiment.candidate, + ) + return agentImprovementProposalSchema.parse( + canonicalCandidateDocument({ + kind: 'agent-improvement-proposal', + runId: options.runId, + changedSurfaces, + proposedAt: (options.now ?? (() => new Date()))().toISOString(), + findings: [...findings], + evaluation, + }).value, + ) } -/** Persist an approve/reject/change-request decision bound to one exact proposal. */ +/** Persist a human or tenant-policy decision bound to one exact proposal. */ export function reviewAgentImprovementProposal( inputProposal: AgentImprovementProposal, input: ReviewAgentImprovementInput, @@ -298,250 +377,383 @@ export function reviewAgentImprovementProposal( const proposal = verifyAgentImprovementProposal(inputProposal) if (!input.reviewedBy.trim()) throw new Error('candidate review requires reviewedBy') if (!input.reason.trim()) throw new Error('candidate review requires a reason') - if (input.decision === 'approve') { - if (proposal.evaluation.gateDecision !== 'ship') { - throw new Error('candidate cannot be approved without a passing measured comparison') - } - if (!proposal.candidateBundle) { - throw new Error('candidate cannot be approved until an executable bundle is sealed') - } + if (input.decision === 'approve' && proposal.evaluation.decision.outcome !== 'ship') { + throw new Error('candidate cannot be approved without a passing experiment') } - const withoutDigest = { - schemaVersion: 1 as const, - kind: 'agent-improvement-review' as const, - proposalDigest: proposal.digest, - ...(proposal.candidateBundle ? { candidateBundleDigest: proposal.candidateBundle.digest } : {}), - decision: input.decision, - reviewedBy: input.reviewedBy, - reviewedAt: (input.now ?? (() => new Date()))().toISOString(), - reason: input.reason, - ...(input.feedback === undefined ? {} : { feedback: input.feedback }), + return agentImprovementReviewSchema.parse( + canonicalCandidateDocument({ + kind: 'agent-improvement-review', + proposalDigest: proposal.digest, + decision: input.decision, + reviewedBy: input.reviewedBy, + reviewedAt: (input.now ?? (() => new Date()))().toISOString(), + reason: input.reason, + ...(input.feedback === undefined ? {} : { feedback: input.feedback }), + }).value, + ) +} + +/** Authorize product-owned writes only after the exact candidate was measured and approved. */ +export function createAgentImprovementActivation( + inputProposal: AgentImprovementProposal, + inputReview: AgentImprovementReview, + options: CreateAgentImprovementActivationOptions, +): AgentImprovementActivation { + const proposal = verifyAgentImprovementProposal(inputProposal) + const review = verifyAgentImprovementReview(inputReview) + if (review.decision !== 'approve' || review.proposalDigest !== proposal.digest) { + throw new Error('candidate activation requires an approval for the exact proposal') } - return canonicalCandidateDocument(withoutDigest).value -} - -/** Verify, materialize, run, grade, and receipt only the exact approved bundle. */ -export async function executeApprovedAgentCandidate( - options: ExecuteApprovedAgentCandidateOptions, -): Promise { - const proposal = verifyAgentImprovementProposal(options.proposal) - const review = verifyAgentImprovementReview(options.review) - if (review.decision !== 'approve') throw new Error('candidate review is not an approval') - if (review.proposalDigest !== proposal.digest) { - throw new Error('candidate approval does not match the proposed improvement') + if (!options.fundingOwner.trim() || !options.authorizedBy.trim()) { + throw new Error('candidate activation authority must be non-empty') } - const bundle = proposal.candidateBundle - if (!bundle) throw new Error('approved proposal does not contain an executable candidate bundle') - if (review.candidateBundleDigest !== bundle.digest) { - throw new Error('candidate approval does not match the executable bundle') + const experiment = proposal.evaluation.experiment + assertActivationTargets(proposal.changedSurfaces, experiment, options.targets) + return agentImprovementActivationSchema.parse( + canonicalCandidateDocument({ + kind: 'agent-improvement-activation', + proposalDigest: proposal.digest, + reviewDigest: review.digest, + experimentDigest: experiment.digest, + candidateBundleDigest: experiment.candidate.digest, + targets: options.targets, + fundingOwner: options.fundingOwner, + authorizedBy: options.authorizedBy, + authorizedAt: (options.now ?? (() => new Date()))().toISOString(), + }).value, + ) +} + +/** Validate a proposal and recompute every binding to its measured experiment. */ +export function verifyAgentImprovementProposal(input: unknown): AgentImprovementProposal { + const proposal = verifyCanonicalCandidateDocument( + agentImprovementProposalSchema.parse(input), + 'agent improvement proposal', + ) + const evaluation = verifyCandidateExperimentComparison(proposal.evaluation) + if (evaluation.decision.outcome !== 'ship') { + throw new Error('agent improvement proposal does not contain a passing experiment') + } + if (proposal.runId !== evaluation.provenance.runId) { + throw new Error('proposal runId does not match its measured experiment') } - if (!(await options.authorizeReview(review, proposal))) { - throw new Error('candidate approval was not authorized by the configured authority') + const changedSurfaces = deriveChangedSurfaces( + evaluation.experiment.baseline, + evaluation.experiment.candidate, + ) + if (!sameOrderedValues(proposal.changedSurfaces, changedSurfaces)) { + throw new Error('proposal changed surfaces do not match its exact experiment') } + assertNoJudgeDerivedProposalFindings(proposal.findings) + return proposal +} - const verified = await verifyAgentCandidateBundle(bundle, options.ports) - const prepared = await prepareAgentCandidateExecution( - verified, - options.task, - options.ports, - options.preparation, +/** Validate the canonical identity and wire shape of an improvement review. */ +export function verifyAgentImprovementReview(input: unknown): AgentImprovementReview { + return verifyCanonicalCandidateDocument( + agentImprovementReviewSchema.parse(input), + 'agent improvement review', ) - const finalization = await executePreparedAgentCandidate(prepared, options.execution) - return { - finalization, - evidence: { - proposalDigest: proposal.digest, - reviewDigest: review.digest, - bundleDigest: bundle.digest, - executionId: prepared.executionId, - executionPlanDigest: prepared.executionPlan.value.digest, - materializationReceiptDigest: prepared.materializationReceipt.digest, - succeeded: finalization.succeeded, - ...(finalization.succeeded ? { runReceiptDigest: finalization.receipt.digest } : {}), - }, +} + +/** Validate activation authority against the exact proposal, review, experiment, and base state. */ +export function verifyAgentImprovementActivation(input: { + proposal: unknown + review: unknown + activation: unknown +}): AgentImprovementActivation { + const proposal = verifyAgentImprovementProposal(input.proposal) + const review = verifyAgentImprovementReview(input.review) + const activation = verifyCanonicalCandidateDocument( + agentImprovementActivationSchema.parse(input.activation), + 'agent improvement activation', + ) + const experiment = proposal.evaluation.experiment + if ( + review.decision !== 'approve' || + review.proposalDigest !== proposal.digest || + activation.proposalDigest !== proposal.digest || + activation.reviewDigest !== review.digest || + activation.experimentDigest !== experiment.digest || + activation.candidateBundleDigest !== experiment.candidate.digest + ) { + throw new Error('candidate activation does not bind the measured and approved candidate') } + assertActivationTargets(proposal.changedSurfaces, experiment, activation.targets) + return activation } -/** Validate a proposal's schema, profile, sealed bundle, and canonical digest. */ -export function verifyAgentImprovementProposal(input: unknown): AgentImprovementProposal { +/** Recheck one Runtime receipt against its exact signed experiment cell. */ +export function verifyCandidateExecutionEvidence( + input: unknown, + options: VerifyCandidateExecutionEvidenceOptions, +): CandidateExecutionEvidence { + const experiment = verifyCandidateExperiment(options.experiment) + const bundle = experiment[options.arm] + const task = experiment.benchmark.tasks[options.benchmarkCell.taskIndex] + const index = + options.benchmarkCell.taskIndex * experiment.benchmark.suite.reps + + options.benchmarkCell.repetition if ( - !isRecord(input) || - input.kind !== 'agent-improvement-proposal' || - input.schemaVersion !== 1 + !task || + options.benchmarkCell.suiteDigest !== experiment.benchmark.suite.digest || + options.seed !== experiment.benchmark.suite.seeds[index] ) { - throw new Error('invalid agent improvement proposal') + throw new Error('candidate execution evidence points outside its signed experiment') } - const proposal = input as unknown as AgentImprovementProposal - const parsedBaselineProfile = parseExactAgentProfile( - proposal.baselineProfile, - 'proposal baseline profile', + const evidence = verifyCanonicalCandidateDocument( + candidateExecutionEvidenceSchema.parse(input), + 'candidate execution evidence', ) - if (proposal.baselineProfileHash !== canonicalCandidateDigest(parsedBaselineProfile)) { - throw new Error('proposal baselineProfileHash does not match baselineProfile') - } - const parsedProfile = parseExactAgentProfile( - proposal.candidateProfile, - 'proposal candidate profile', + const materialization = verifyCanonicalCandidateDocument( + agentCandidateMaterializationReceiptSchema.parse(evidence.materializationReceipt), + 'candidate materialization receipt', ) - if (proposal.candidateProfileHash !== canonicalCandidateDigest(parsedProfile)) { - throw new Error('proposal candidateProfileHash does not match candidateProfile') - } - assertMeasuredProfileBinding({ - surface: proposal.surface, - baselineProfile: parsedBaselineProfile, - candidateProfile: parsedProfile, - evaluation: proposal.evaluation, - hasExecutableBundle: proposal.candidateBundle !== undefined, - }) - if (!Array.isArray(proposal.findings)) throw new Error('proposal findings must be an array') - if (!isSha256Digest(proposal.digest)) throw new Error('proposal digest is invalid') - if (proposal.candidateBundle) { - const parsedBundle = agentCandidateBundleSchema.parse(proposal.candidateBundle) - const sealed = sealAgentCandidateBundle(omitTopLevelDigest(parsedBundle)) - if (sealed.digest !== parsedBundle.digest) - throw new Error('proposal candidate bundle digest is invalid') - assertCandidateProfileBinding(parsedProfile, sealed.profile) - } - const actual = canonicalCandidateDigest(omitTopLevelDigest(proposal)) - if (actual !== proposal.digest) - throw new Error('agent improvement proposal digest does not match') - return immutableCandidateValue(proposal) -} - -function assertMeasuredProfileBinding(input: { - surface: ImproveSurface - baselineProfile: AgentProfile - candidateProfile: AgentProfile - evaluation: Pick, 'gateDecision' | 'winner'> - hasExecutableBundle: boolean -}): void { - const baselineDigest = canonicalCandidateDigest(input.baselineProfile) - if (input.evaluation.gateDecision !== 'ship') { - if (canonicalCandidateDigest(input.candidateProfile) !== baselineDigest) { - throw new Error('non-shipping evaluation must retain the baseline candidate profile') - } - return + const receipt = verifyCanonicalCandidateDocument( + agentCandidateRunReceiptSchema.parse(evidence.receipt), + 'candidate run receipt', + ) + const plan = materialization.executionPlan + const cell = plan.material.runCell + const attempt = options.attempt ?? 1 + if ( + cell.experimentDigest !== experiment.digest || + cell.arm !== options.arm || + cell.bundleDigest !== bundle.digest || + cell.suiteDigest !== experiment.benchmark.suite.digest || + cell.taskDigest !== task.digest || + cell.taskIndex !== options.benchmarkCell.taskIndex || + cell.repetition !== options.benchmarkCell.repetition || + cell.seed !== options.seed || + cell.attempt !== attempt || + canonicalCandidateDigest(omitTopLevelDigest(cell)) !== cell.digest + ) { + throw new Error('candidate execution receipt substituted its signed experiment cell') } - - const measured = measuredWinnerProfile( - input.baselineProfile, - input.surface, - input.evaluation.winner.surface, + assertCapturedInput( + materialization.benchmark.suite, + experiment.benchmark.suite, + 'benchmark suite', ) - if (!measured) { - if (input.hasExecutableBundle) { - throw new Error( - `executable '${input.surface}' candidate cannot be bound to its measured winner surface`, - ) - } - if (canonicalCandidateDigest(input.candidateProfile) !== baselineDigest) { - throw new Error( - `unbound '${input.surface}' improvement must retain the baseline candidate profile`, - ) - } - return + assertCapturedInput(materialization.benchmark.task, task, 'benchmark task') + assertEvidenceMaterialDigest(plan, 'candidate execution plan') + assertEvidenceMaterialDigest( + materialization.profileActivation.profilePlan, + 'candidate profile plan', + ) + const expectedProfilePlan = materializeCandidateProfile( + bundle.profile, + candidateMaterializerHarness(materialization.harness), + { resolvedResources: options.resolvedResources }, + ) + const activation = parseAgentCandidateProfileActivation( + materialization.profileActivation, + materialization.profileActivation.profilePlan.digest, + ) + const regeneratedActivation = createAgentCandidateProfileActivation( + expectedProfilePlan, + materialization.profileActivation.profilePlan, + ) + if (activation.digest !== regeneratedActivation.digest) { + throw new Error('candidate profile activation does not match the experiment bundle') } - if (canonicalCandidateDigest(input.candidateProfile) !== canonicalCandidateDigest(measured)) { - throw new Error('proposal candidate profile does not match the measured winner surface') + if ( + materialization.bundleDigest !== bundle.digest || + receipt.bundleDigest !== bundle.digest || + receipt.runCellDigest !== cell.digest || + receipt.materializationReceiptDigest !== materialization.digest || + receipt.executionPlanDigest !== plan.digest + ) { + throw new Error('candidate execution evidence does not bind one exact Runtime run') } + assertEvidenceMaterialDigest(receipt.modelSettlement, 'candidate model settlement') + assertEvidenceMaterialDigest(receipt.taskOutcome, 'candidate task outcome') + assertEvidenceMaterialDigest(receipt.benchmarkResult, 'candidate benchmark result') + return immutableCandidateValue(evidence) } -function measuredWinnerProfile( - baselineProfile: AgentProfile, - surface: ImproveSurface, - winner: unknown, -): AgentProfile | null { - if (surface === 'code' || surface === 'memory') return null - if (typeof winner !== 'string') { - throw new Error(`measured '${surface}' winner is not a serializable profile surface`) - } - if (surface !== 'skills') { - return applyImprovementWinnerToProfile(baselineProfile, surface, winner) - } - try { - return applyImprovementWinnerToProfile(baselineProfile, surface, winner) - } catch { - // A skill document is text stored outside the profile; a skill-ref array is - // JSON stored on the profile. Only the latter can bind to a profile bundle. - return null +function assertExactExperimentInput( + input: CandidateExperimentExecutionInput, + experiment: AgentCandidateExperiment, + bundle: AgentCandidateBundle, +): void { + const task = experiment.benchmark.tasks[input.benchmarkCell.taskIndex] + const index = + input.benchmarkCell.taskIndex * experiment.benchmark.suite.reps + input.benchmarkCell.repetition + if ( + input.experiment.digest !== experiment.digest || + input.bundle.digest !== bundle.digest || + !task || + input.task.digest !== task.digest || + input.benchmarkCell.suiteDigest !== experiment.benchmark.suite.digest || + input.seed !== experiment.benchmark.suite.seeds[index] + ) { + throw new Error('Runtime received a substituted candidate experiment cell') } } -function sealBuiltCandidate( - input: AgentCandidateBundleInput | AgentCandidateBundle, -): AgentCandidateBundle { - if (!('digest' in input)) return sealAgentCandidateBundle(input) - const parsed = agentCandidateBundleSchema.parse(input) - const sealed = sealAgentCandidateBundle(omitTopLevelDigest(parsed)) - if (sealed.digest !== parsed.digest) { - throw new Error('built candidate bundle digest is invalid') +function assertCapturedInput( + captured: { digest: Sha256Digest; material: { sha256: Sha256Digest; byteLength: number } }, + expected: { digest: Sha256Digest }, + label: string, +): void { + const bytes = canonicalCandidateBytes(omitTopLevelDigest(expected)) + if ( + captured.digest !== expected.digest || + captured.material.sha256 !== expected.digest || + captured.material.byteLength !== bytes.byteLength + ) { + throw new Error(`candidate materialization substituted its ${label}`) } - return sealed } -/** Validate a review's decision fields and canonical digest. */ -export function verifyAgentImprovementReview(input: unknown): AgentImprovementReview { - if (!isRecord(input) || input.kind !== 'agent-improvement-review' || input.schemaVersion !== 1) { - throw new Error('invalid agent improvement review') - } - const review = input as unknown as AgentImprovementReview - if (!isSha256Digest(review.digest) || !isSha256Digest(review.proposalDigest)) { - throw new Error('candidate review digest is invalid') - } - if (review.candidateBundleDigest !== undefined && !isSha256Digest(review.candidateBundleDigest)) { - throw new Error('candidate review bundle digest is invalid') +function assertEvidenceMaterialDigest( + evidence: { + digest: Sha256Digest + material: unknown + artifact: { sha256: Sha256Digest; byteLength: number } + }, + label: string, +): void { + const bytes = canonicalCandidateBytes(evidence.material) + if ( + canonicalCandidateDigest(evidence.material) !== evidence.digest || + evidence.artifact.sha256 !== evidence.digest || + evidence.artifact.byteLength !== bytes.byteLength + ) { + throw new Error(`${label} digest does not match its canonical material`) } - if (!['approve', 'reject', 'request-changes'].includes(review.decision)) { - throw new Error('candidate review decision is invalid') +} + +const CHANGED_SURFACE_ORDER: readonly AgentImprovementSurface[] = [ + 'prompt', + 'skills', + 'tools', + 'mcp', + 'hooks', + 'subagents', + 'agent-profile', + 'memory', + 'code', + 'knowledge', +] + +function deriveChangedSurfaces( + baselineBundle: AgentCandidateBundle, + candidateBundle: AgentCandidateBundle, +): [AgentImprovementSurface, ...AgentImprovementSurface[]] { + const baseline = improvementSurfaceValues(baselineBundle) + const candidate = improvementSurfaceValues(candidateBundle) + const changed = new Set() + for (const surface of CHANGED_SURFACE_ORDER) { + if ( + canonicalCandidateDigest(baseline[surface]) !== canonicalCandidateDigest(candidate[surface]) + ) { + changed.add(surface) + } } - const actual = canonicalCandidateDigest(omitTopLevelDigest(review)) - if (actual !== review.digest) throw new Error('agent improvement review digest does not match') - return immutableCandidateValue(review) + const ordered = CHANGED_SURFACE_ORDER.filter((surface) => changed.has(surface)) + if (ordered.length === 0) throw new Error('candidate experiment does not change an agent surface') + return ordered as [AgentImprovementSurface, ...AgentImprovementSurface[]] } -function improvementEvaluation( - result: SelfImproveResult, -): AgentImprovementEvaluation { +function improvementSurfaceValues( + bundle: AgentCandidateBundle, +): Record { + const profile = agentCandidateProfileAsAgentProfile(bundle.profile) return { - baseline: result.baseline, - winner: result.winner, - lift: result.lift, - diff: result.diff, - provenance: result.provenance, - gateDecision: result.gateDecision, - generationsExplored: result.generationsExplored, - durationMs: result.durationMs, - totalCostUsd: result.totalCostUsd, - insight: result.insight, - ...(result.power === undefined ? {} : { power: result.power }), + prompt: { + prompt: profile.prompt ?? null, + instructions: profile.resources?.instructions ?? null, + }, + skills: profile.resources?.skills ?? null, + tools: { + tools: profile.tools ?? null, + resources: profile.resources?.tools ?? null, + }, + mcp: profile.mcp ?? null, + hooks: profile.hooks ?? null, + subagents: { + subagents: profile.subagents ?? null, + resources: profile.resources?.agents ?? null, + }, + 'agent-profile': { profile: opaqueProfileSlice(profile), execution: bundle.execution }, + memory: bundle.memory, + code: bundle.code, + knowledge: bundle.knowledge ?? null, } } -function isRecord(value: unknown): value is Record { - return value !== null && typeof value === 'object' && !Array.isArray(value) +function opaqueProfileSlice(profile: AgentProfile): unknown { + const { + prompt: _prompt, + tools: _tools, + mcp: _mcp, + hooks: _hooks, + subagents: _subagents, + resources, + ...opaqueProfile + } = profile + const { + instructions: _instructions, + skills: _skills, + tools: _resourceTools, + agents: _agents, + ...opaqueResources + } = resources ?? {} + return { + ...opaqueProfile, + ...(Object.keys(opaqueResources).length > 0 ? { resources: opaqueResources } : {}), + } } -function isSha256Digest(value: unknown): value is Sha256Digest { - return typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value) +function assertActivationTargets( + surfaces: readonly AgentImprovementSurface[], + experiment: AgentCandidateExperiment, + targets: readonly AgentImprovementActivationTarget[], +): void { + const expected = new Set(surfaces) + const actual = new Set(targets.map((target) => target.surface)) + const baselineValues = improvementSurfaceValues(experiment.baseline) + if ( + targets.some((target) => !target.identity.trim()) || + targets.some( + (target) => + target.expectedBaseDigest !== + expectedActivationBaseDigest(experiment, target.surface, baselineValues), + ) || + expected.size !== actual.size || + [...expected].some((surface) => !actual.has(surface)) + ) { + throw new Error('candidate activation targets must cover exactly the changed surfaces') + } } -function canonicalJsonValue(value: T, path = '$'): T { - if (value === null || typeof value === 'string' || typeof value === 'boolean') return value - if (typeof value === 'number') { - if (!Number.isFinite(value)) throw new Error(`non-finite number at ${path}`) - return value - } - if (Array.isArray(value)) { - return value.map((entry, index) => { - if (entry === undefined) throw new Error(`undefined array entry at ${path}[${index}]`) - return canonicalJsonValue(entry, `${path}[${index}]`) - }) as T +function expectedActivationBaseDigest( + experiment: AgentCandidateExperiment, + surface: AgentImprovementSurface, + baselineValues: Record, +): Sha256Digest { + if (surface === 'knowledge' && experiment.candidate.knowledge) { + return experiment.candidate.knowledge.candidate.baseHash } - if (typeof value === 'object') { - const entries = Object.entries(value as Record) - .filter(([, entry]) => entry !== undefined) - .map(([key, entry]) => [key, canonicalJsonValue(entry, `${path}.${key}`)]) - return Object.fromEntries(entries) as T - } - throw new Error(`unsupported proposal value at ${path}`) + return canonicalCandidateDigest(baselineValues[surface]) +} + +function sameOrderedValues(left: readonly T[], right: readonly T[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]) +} + +function assertNoJudgeDerivedProposalFindings( + findings: AgentImprovementProposal['findings'], +): void { + const leaked = findings.filter((finding) => finding.derived_from_judge === true) + if (leaked.length === 0) return + const identifiers = leaked.map((finding) => + typeof finding.finding_id === 'string' ? finding.finding_id : '', + ) + throw new Error( + 'agent improvement proposal findings: judge-derived findings cannot steer an improvement: ' + + `[${identifiers.join(', ')}]`, + ) } diff --git a/src/intelligence/index.ts b/src/intelligence/index.ts index f81bf8f2..e4e0a73e 100644 --- a/src/intelligence/index.ts +++ b/src/intelligence/index.ts @@ -3,8 +3,9 @@ * Tangle Intelligence SDK — trace capture plus reviewable improvement. * * The client keeps live-agent trace delivery best-effort. The separate - * improvement-cycle exports analyze completed traces, measure one candidate, - * bind human review, and execute only an approved immutable bundle. + * improvement-cycle exports analyze completed traces, run a signed baseline + * versus candidate experiment, bind review to its result, and activate only + * the exact measured candidate. * * 1. OBSERVE — wrap a generic agent and export one trace span per call to * Tangle Intelligence, swallowing every export failure so a live agent @@ -19,7 +20,7 @@ */ import { contentHash } from '@tangle-network/agent-eval' -import type { AgentProfile } from '@tangle-network/agent-interface' +import type { AgentProfile, CandidateExecutionEvidence } from '@tangle-network/agent-interface' import { buildLoopOtelSpans, buildRuntimeEventOtelSpans, @@ -39,9 +40,17 @@ import { isIntelligenceOff, resolveEffort, } from './effort' -import type { CandidateExecutionEvidence } from './improvement-cycle' import { type Redactor, resolveRedactor } from './redact' +export type { + AgentCandidateProfileActivation, + AgentImprovementMeasuredComparison, + AgentImprovementProposal, + AgentImprovementReview, + AgentImprovementReviewDecision, + CandidateExecutionEvidence, +} from '@tangle-network/agent-interface' +export { parseAgentCandidateProfileActivation } from '../candidate-execution/profile' export type { CapabilityAuth, CapabilityInterface, @@ -94,25 +103,30 @@ export { resolveEffort, } from './effort' export type { - AgentImprovementEvaluation, - AgentImprovementProposal, - AgentImprovementReview, - AgentImprovementReviewDecision, - CandidateExecutionEvidence, + AgentCandidateExperimentCellPlacement, + CreateAgentImprovementActivationOptions, CreateAgentImprovementProposalOptions, - ExecuteApprovedAgentCandidateOptions, - ExecuteApprovedAgentCandidateResult, + ExecuteAgentCandidateExperimentCellOptions, ProposeAgentImprovementOptions, ProposeAgentImprovementResult, ReviewAgentImprovementInput, + RunAgentCandidateExperimentOptions, + RunAgentCandidateExperimentResult, + VerifyCandidateExecutionEvidenceOptions, } from './improvement-cycle' export { + AgentCandidateExperimentCellExecutionError, + createAgentImprovementActivation, + createAgentImprovementMeasuredComparison, createAgentImprovementProposal, - executeApprovedAgentCandidate, + executeAgentCandidateExperimentCell, proposeAgentImprovement, reviewAgentImprovementProposal, + runAgentCandidateExperiment, + verifyAgentImprovementActivation, verifyAgentImprovementProposal, verifyAgentImprovementReview, + verifyCandidateExecutionEvidence, } from './improvement-cycle' export type { Redactor } from './redact' export { defaultRedactor, resolveRedactor } from './redact' @@ -121,6 +135,15 @@ export { composeCertifiedProfile, composeCertifiedProfileFromWire, } from './resolver' +export type { + CreateSandboxCandidateExperimentExecutorOptions, + SandboxCandidateExperimentExecution, + SandboxCandidateExperimentExecutor, +} from './sandbox-approved-candidate' +export { + createSandboxCandidateExperimentExecutor, + sandboxCandidateExperimentExecutionSupport, +} from './sandbox-approved-candidate' export type { AppliedIntelligence, IntelligenceAgent, @@ -584,21 +607,21 @@ export function createIntelligenceClient(config: IntelligenceConfig): Intelligen : {}), ...(record.candidateExecution ? { - 'tangle.candidate.proposal_digest': record.candidateExecution.proposalDigest, - 'tangle.candidate.review_digest': record.candidateExecution.reviewDigest, - 'tangle.candidate.bundle_digest': record.candidateExecution.bundleDigest, - 'tangle.candidate.execution_id': record.candidateExecution.executionId, + 'tangle.candidate.bundle_digest': record.candidateExecution.receipt.bundleDigest, + 'tangle.candidate.experiment_digest': + record.candidateExecution.materializationReceipt.executionPlan.material.runCell + .experimentDigest, + 'tangle.candidate.execution_id': + record.candidateExecution.materializationReceipt.executionPlan.material.executionId, 'tangle.candidate.execution_plan_digest': - record.candidateExecution.executionPlanDigest, + record.candidateExecution.receipt.executionPlanDigest, 'tangle.candidate.materialization_receipt_digest': - record.candidateExecution.materializationReceiptDigest, - 'tangle.candidate.succeeded': record.candidateExecution.succeeded, - ...(record.candidateExecution.runReceiptDigest - ? { - 'tangle.candidate.run_receipt_digest': - record.candidateExecution.runReceiptDigest, - } - : {}), + record.candidateExecution.receipt.materializationReceiptDigest, + 'tangle.candidate.succeeded': + record.candidateExecution.receipt.termination.kind === 'exit' && + record.candidateExecution.receipt.termination.exitCode === 0 && + record.candidateExecution.receipt.benchmarkResult.material.passed, + 'tangle.candidate.run_receipt_digest': record.candidateExecution.receipt.digest, } : {}), } diff --git a/src/intelligence/sandbox-approved-candidate.ts b/src/intelligence/sandbox-approved-candidate.ts new file mode 100644 index 00000000..738fbce7 --- /dev/null +++ b/src/intelligence/sandbox-approved-candidate.ts @@ -0,0 +1,681 @@ +import { posix } from 'node:path' +import type { TraceStore } from '@tangle-network/agent-eval' +import type { CandidateExperimentExecutionInput } from '@tangle-network/agent-eval/contract' +import type { + AgentCandidateTermination, + CandidateExecutionEvidence, + Sha256Digest, +} from '@tangle-network/agent-interface' +import type { + CreateSandboxOptions, + ListSandboxOptions, + Process, + ProcessStatus, + SandboxClient, + SandboxInstance, + SandboxResources, +} from '@tangle-network/sandbox' + +import type { AgentCandidateExecutionClaimStore } from '../candidate-execution/claim' +import { canonicalCandidateBytes } from '../candidate-execution/digest' +import { + CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV, + CANDIDATE_KNOWLEDGE_ROOT_ENV, + candidateKnowledgeExecutionPaths, +} from '../candidate-execution/knowledge' +import type { PrepareAgentCandidateExecutionOptions } from '../candidate-execution/prepare' +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateExecutionPorts, + AgentCandidateExecutorPort, + AgentCandidateExecutorRequest, + AgentCandidateExecutorStopRequest, + AgentCandidateOutputArtifactPort, +} from '../candidate-execution/types' +import { AGENT_CANDIDATE_EXECUTION_SUPPORT } from '../candidate-execution/verify' +import { + type AgentCandidateExperimentCellPlacement, + executeAgentCandidateExperimentCell, +} from './improvement-cycle' + +type SandboxClientPort = Pick + +type SandboxExecutorOptions = NonNullable< + CreateSandboxCandidateExperimentExecutorOptions['sandbox'] +> + +interface SandboxRunState { + sandbox: SandboxInstance + output?: Uint8Array + termination?: AgentCandidateTermination + trace?: { + runId: string + scenarioId: string + startedAt: number + deadlineAtMs: number + timeoutMs: number + tags: Record + written: boolean + } +} + +/** Declares the exact candidate surfaces the sandbox executor can run. */ +export const sandboxCandidateExperimentExecutionSupport = Object.freeze({ + outcomes: Object.freeze(['output'] as const), + outputMediaTypes: Object.freeze(['text/*', 'application/json', '*+json'] as const), + code: Object.freeze(['disabled'] as const), + memory: Object.freeze(['disabled'] as const), + knowledge: true, + profile: AGENT_CANDIDATE_EXECUTION_SUPPORT.profile, + isolation: Object.freeze({ + freshSandbox: true, + exactProcess: true, + egress: Object.freeze(['blocked', 'strict'] as const), + }), +}) + +export interface CreateSandboxCandidateExperimentExecutorOptions { + client: SandboxClientPort + ports: AgentCandidateExecutionPorts + grader: AgentCandidateBenchmarkGraderPort + outputArtifacts: AgentCandidateOutputArtifactPort + traceStore: TraceStore + claimStore: AgentCandidateExecutionClaimStore + sandbox?: { + teamId?: string + resources?: SandboxResources + createTimeoutMs?: number + evidenceRetentionSeconds?: number + } + cleanupTimeoutMs?: number + resultTimeoutMs?: number +} + +export interface SandboxCandidateExperimentExecution extends CandidateExperimentExecutionInput { + executionId: string + attempt?: number + executionRoots: AgentCandidateExperimentCellPlacement['executionRoots'] + stagingRoots: AgentCandidateExperimentCellPlacement['stagingRoots'] + preparation?: PrepareAgentCandidateExecutionOptions +} + +export interface SandboxCandidateExperimentExecutor { + /** The same port is usable by Runtime's expired-claim recovery path. */ + readonly executor: AgentCandidateExecutorPort + execute(input: SandboxCandidateExperimentExecution): Promise +} + +/** Execute one signed experiment cell inside a fresh Tangle sandbox. */ +export function createSandboxCandidateExperimentExecutor( + options: CreateSandboxCandidateExperimentExecutorOptions, +): SandboxCandidateExperimentExecutor { + const executor = new SandboxAgentCandidateExecutor( + options.client, + options.sandbox, + options.resultTimeoutMs, + ) + return Object.freeze({ + executor, + async execute(input: SandboxCandidateExperimentExecution): Promise { + return await executeAgentCandidateExperimentCell({ + ...input, + attempt: input.attempt ?? 1, + executionId: input.executionId, + executionRoots: input.executionRoots, + stagingRoots: input.stagingRoots, + ports: options.ports, + ...(input.preparation ? { preparation: input.preparation } : {}), + execution: { + executor, + grader: options.grader, + outputArtifacts: options.outputArtifacts, + traceStore: options.traceStore, + claimStore: options.claimStore, + ...(options.cleanupTimeoutMs === undefined + ? {} + : { cleanupTimeoutMs: options.cleanupTimeoutMs }), + ...(options.resultTimeoutMs === undefined + ? {} + : { resultTimeoutMs: options.resultTimeoutMs }), + }, + }) + }, + }) +} + +class SandboxAgentCandidateExecutor implements AgentCandidateExecutorPort { + private readonly states = new Map() + + constructor( + private readonly client: SandboxClientPort, + private readonly options: SandboxExecutorOptions = {}, + private readonly captureTimeoutMs = 120_000, + ) {} + + async execute( + request: AgentCandidateExecutorRequest, + context: Parameters[1], + ) { + assertSupportedRequest(request) + const outcome = request.benchmark.task.outcome + if (outcome.kind !== 'output') throw new Error('sandbox executor requires an output task') + const key = executionKey(request.executionId, request.executionPlan.value.digest) + const sandbox = await this.client.create(sandboxCreateOptions(request, this.options), { + signal: context.signal, + timeoutMs: this.options.createTimeoutMs ?? 120_000, + }) + const state: SandboxRunState = { sandbox } + this.states.set(key, state) + if ((await sandbox.process.list()).length !== 0) { + throw new Error('fresh candidate sandbox already contains a process') + } + await materializeRequest(sandbox, request) + const launch = exactLaunch(request) + const startedAt = Date.now() + state.trace = { + runId: request.trace.runId, + scenarioId: request.benchmark.task.scenario.id, + startedAt, + deadlineAtMs: context.deadlineAtMs, + timeoutMs: request.hardLimits.timeoutMs, + tags: { ...request.trace.tags, sandboxId: sandbox.id }, + written: false, + } + const process = await sandbox.process.spawnExact(launch.executable, launch.args, { + cwd: request.launch.cwd, + env: { ...request.launch.env }, + ...(launch.stdin === undefined ? {} : { stdin: launch.stdin }), + timeoutMs: request.hardLimits.timeoutMs, + }) + const outputPromise = collectOutput(process, outcome.maxBytes) + let cancellation: Promise | undefined + let cancellationTermination: AgentCandidateTermination | undefined + let processExited = false + const cancel = () => { + if (processExited) return + cancellationTermination = + Date.now() >= context.deadlineAtMs + ? { kind: 'timeout', timeoutMs: request.hardLimits.timeoutMs } + : { kind: 'cancelled' } + cancellation ??= killProcess(process) + } + context.signal.addEventListener('abort', cancel, { once: true }) + if (context.signal.aborted) cancel() + try { + const waitedExitCode = await process.wait() + processExited = true + context.signal.removeEventListener('abort', cancel) + if (cancellation) await cancellation + const status = await process.status() + if (status.running || status.exitCode !== waitedExitCode) { + throw new Error('sandbox process returned an inconsistent terminal status') + } + state.output = await awaitOutput(outputPromise, context.signal, context.deadlineAtMs) + state.termination = cancellationTermination ?? processTermination(status) + await appendSandboxTrace(state, context.traceStore, status) + return { executionId: request.executionId, termination: state.termination } + } finally { + context.signal.removeEventListener('abort', cancel) + void outputPromise.catch(() => undefined) + } + } + + async stop( + request: AgentCandidateExecutorStopRequest, + context: Parameters[1], + ): Promise<{ readonly stopped: true }> { + context.signal.throwIfAborted() + const sandbox = await this.resolve(request, true) + if (!sandbox) return { stopped: true } + const statuses = await sandbox.process.list() + if (statuses.length > 1) throw new Error('candidate sandbox contains more than one process') + const status = statuses[0] + if (status?.running) { + const process = await sandbox.process.get(status.pid) + if (!process) throw new Error('candidate sandbox lost its running process handle') + await killProcess(process) + } + context.signal.throwIfAborted() + const remaining = await sandbox.process.list() + if (remaining.some((entry) => entry.running)) { + throw new Error('candidate sandbox process termination is not proven') + } + const state = this.states.get(executionKey(request.executionId, request.executionPlanDigest)) + if (state) { + if (context.reason === 'timeout') { + state.termination = { kind: 'timeout', timeoutMs: state.trace?.timeoutMs ?? 0 } + } + await appendSandboxTrace(state, context.traceStore, remaining[0]) + } + return { stopped: true } + } + + async capture( + request: AgentCandidateExecutorStopRequest, + context: Parameters[1], + ) { + context.signal.throwIfAborted() + const sandbox = await this.resolve(request, false) + const statuses = await sandbox.process.list() + if (statuses.length > 1 || statuses.some((entry) => entry.running)) { + throw new Error('candidate sandbox must contain one stopped process before capture') + } + const state = this.states.get(executionKey(request.executionId, request.executionPlanDigest)) + let output = state?.output + let termination = state?.termination + const status = statuses[0] + if (status && !output) { + const process = await sandbox.process.get(status.pid) + if (!process) throw new Error('candidate sandbox lost its captured process handle') + const expected = sandboxOutputSpec(sandbox) + output = await awaitOutput( + collectOutput(process, expected.maxBytes), + context.signal, + Date.now() + this.captureTimeoutMs, + ) + } + if (status && !termination) { + termination = processTermination(status) + } + context.signal.throwIfAborted() + const evidence = canonicalCandidateBytes({ + kind: 'sandbox-agent-candidate-capture', + sandboxId: sandbox.id, + executionId: request.executionId, + executionPlanDigest: request.executionPlanDigest, + ...(status + ? { + process: { + pid: status.pid, + exitCode: status.exitCode, + ...(status.exitSignal ? { exitSignal: status.exitSignal } : {}), + }, + } + : {}), + ...(termination ? { termination } : {}), + }) + return { + ...(output ? { taskOutcome: { kind: 'output' as const, bytes: output } } : {}), + evidence, + } + } + + async dispose( + request: AgentCandidateExecutorStopRequest, + context: { signal: AbortSignal }, + ): Promise<{ readonly disposed: true }> { + context.signal.throwIfAborted() + const sandbox = await this.resolve(request, true) + if (!sandbox) return { disposed: true } + try { + await sandbox.delete() + } catch (error) { + if (await this.client.get(sandbox.id)) throw error + } + context.signal.throwIfAborted() + this.states.delete(executionKey(request.executionId, request.executionPlanDigest)) + return { disposed: true } + } + + private async resolve( + request: AgentCandidateExecutorStopRequest, + missingIsDeleted: false, + ): Promise + private async resolve( + request: AgentCandidateExecutorStopRequest, + missingIsDeleted: true, + ): Promise + private async resolve( + request: AgentCandidateExecutorStopRequest, + missingIsDeleted: boolean, + ): Promise { + const key = executionKey(request.executionId, request.executionPlanDigest) + const active = this.states.get(key)?.sandbox + if (active) return active + const matches: SandboxInstance[] = [] + const scope: ListSandboxOptions['scope'] = this.options.teamId + ? `team:${this.options.teamId}` + : 'personal' + for (let offset = 0; ; offset += 100) { + const page = await this.client.list({ scope, limit: 100, offset }) + for (const candidate of page) { + if (matchesExecution(candidate, request)) matches.push(candidate) + } + if (page.length < 100) break + } + if (matches.length === 1) return matches[0] + if (matches.length > 1) throw new Error('multiple sandboxes match one candidate execution') + if (missingIsDeleted) return undefined + throw new Error('candidate sandbox evidence is unavailable') + } +} + +async function appendSandboxTrace( + state: SandboxRunState, + traceStore: TraceStore, + status?: ProcessStatus, +): Promise { + const trace = state.trace + if (!trace || trace.written) return + const endedAt = Math.max(trace.startedAt, Math.min(Date.now(), trace.deadlineAtMs)) + await traceStore.appendRun({ + runId: trace.runId, + scenarioId: trace.scenarioId, + startedAt: trace.startedAt, + endedAt, + status: + state.termination?.kind !== 'timeout' && status?.running === false && status.exitCode === 0 + ? 'completed' + : 'failed', + tags: trace.tags, + }) + trace.written = true +} + +function assertSupportedRequest(request: AgentCandidateExecutorRequest): void { + if (request.benchmark.task.outcome.kind !== 'output') { + throw new Error('sandbox candidate executor does not support workspace outcomes; use Pier') + } + if (request.executionPlan.value.material.codeKind !== 'disabled' || request.inputs.candidate) { + throw new Error('sandbox candidate executor does not support code workspaces; use Pier') + } + if (request.memory.mode !== 'disabled') { + throw new Error('sandbox candidate executor does not support isolated memory') + } + const mediaType = request.benchmark.task.outcome.mediaType.toLowerCase() + if ( + !( + mediaType.startsWith('text/') || + mediaType === 'application/json' || + mediaType.endsWith('+json') + ) + ) { + throw new Error('sandbox candidate executor supports only UTF-8 text and JSON outputs') + } +} + +function sandboxCreateOptions( + request: AgentCandidateExecutorRequest, + options: SandboxExecutorOptions = {}, +): CreateSandboxOptions { + const material = request.executionPlan.value.material + const retentionSeconds = options.evidenceRetentionSeconds ?? 900 + if (!Number.isSafeInteger(retentionSeconds) || retentionSeconds < 60) { + throw new Error('sandbox evidence retention must be at least 60 seconds') + } + const maxLifetimeSeconds = Math.ceil(request.hardLimits.timeoutMs / 1_000) + retentionSeconds + const network = material.model.access.network + const egressPolicy = + network.mode === 'disabled' + ? { mode: 'blocked' as const } + : { + mode: 'strict' as const, + allowDomains: [...network.domains], + includeImplicitDomains: false, + } + const value = { + image: exactImage(material.container.image, material.container.manifestDigest), + bare: true, + publicEdge: false, + ephemeral: true, + sshEnabled: false, + webTerminalEnabled: false, + secrets: [], + capabilities: [], + egressPolicy, + maxLifetimeSeconds, + idempotencyKey: `candidate-${request.executionPlan.value.digest.slice('sha256:'.length)}`, + metadata: { + kind: 'agent-candidate-execution', + executionId: request.executionId, + executionPlanDigest: request.executionPlan.value.digest, + outputMediaType: + request.benchmark.task.outcome.kind === 'output' + ? request.benchmark.task.outcome.mediaType + : '', + outputMaxBytes: + request.benchmark.task.outcome.kind === 'output' + ? request.benchmark.task.outcome.maxBytes + : 0, + }, + ...(options.teamId ? { teamId: options.teamId } : {}), + ...(options.resources ? { resources: options.resources } : {}), + } + return value +} + +async function materializeRequest( + sandbox: SandboxInstance, + request: AgentCandidateExecutorRequest, +): Promise { + const knowledgePaths = assertKnowledgeExecutionBinding(request) + for (const file of request.inputs.task.files) { + await writeExactFile(sandbox, beneath(request.roots.taskRoot, file.path), file.bytes, file.mode) + } + const profileRoot = + request.executionPlan.value.material.profile.targetWorkspace === 'task' + ? request.roots.taskRoot + : request.roots.candidateRoot + if (!profileRoot) throw new Error('candidate profile targets a missing workspace') + for (const file of request.profileActivation.files) { + await writeExactFile( + sandbox, + beneath(profileRoot, file.path), + Buffer.from(file.content, 'utf8'), + file.mode, + ) + } + if (request.knowledge && knowledgePaths) { + for (const file of request.knowledge.files) { + await writeExactFile(sandbox, beneath(knowledgePaths.root, file.path), file.bytes, file.mode) + } + if (request.knowledge.retrievalConfig && knowledgePaths.retrievalConfig) { + await writeExactFile( + sandbox, + knowledgePaths.retrievalConfig, + request.knowledge.retrievalConfig, + 0o644, + ) + } + } + if (request.instruction.delivery.kind === 'utf8-file') { + await writeExactFile( + sandbox, + request.instruction.delivery.path, + request.instruction.bytes, + 0o644, + ) + } +} + +function assertKnowledgeExecutionBinding( + request: AgentCandidateExecutorRequest, +): ReturnType | undefined { + const knowledge = request.knowledge + if (!knowledge) { + if ( + request.launch.env[CANDIDATE_KNOWLEDGE_ROOT_ENV] !== undefined || + request.launch.env[CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV] !== undefined + ) { + throw new Error('candidate launch declares knowledge paths without verified knowledge') + } + return undefined + } + const paths = candidateKnowledgeExecutionPaths( + request.roots.taskRoot, + knowledge.retrievalConfig !== undefined, + ) + if ( + request.launch.env[CANDIDATE_KNOWLEDGE_ROOT_ENV] !== paths.root || + request.launch.env[CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV] !== paths.retrievalConfig + ) { + throw new Error('candidate knowledge paths do not match the signed launch environment') + } + + const reserved = [paths.root, paths.retrievalConfig].filter( + (value): value is string => value !== undefined, + ) + const profileRoot = + request.executionPlan.value.material.profile.targetWorkspace === 'task' + ? request.roots.taskRoot + : request.roots.candidateRoot + const otherFiles = [ + ...request.inputs.task.files.map((file) => beneath(request.roots.taskRoot, file.path)), + ...(profileRoot + ? request.profileActivation.files.map((file) => beneath(profileRoot, file.path)) + : []), + ...(request.instruction.delivery.kind === 'utf8-file' + ? [request.instruction.delivery.path] + : []), + ] + if ( + otherFiles.some((path) => + reserved.some( + (reservedPath) => + path === reservedPath || + path.startsWith(`${reservedPath}/`) || + reservedPath.startsWith(`${path}/`), + ), + ) + ) { + throw new Error('candidate knowledge paths overlap other execution inputs') + } + return paths +} + +function exactLaunch(request: AgentCandidateExecutorRequest): { + executable: string + args: readonly string[] + stdin?: string +} { + const instruction = new TextDecoder('utf-8', { fatal: true }).decode(request.instruction.bytes) + switch (request.instruction.delivery.kind) { + case 'argv-append': + return { executable: request.launch.executable, args: [...request.launch.args, instruction] } + case 'stdin-utf8': + return { + executable: request.launch.executable, + args: request.launch.args, + stdin: instruction, + } + case 'utf8-file': + return { executable: request.launch.executable, args: request.launch.args } + } +} + +async function writeExactFile( + sandbox: SandboxInstance, + path: string, + bytes: Uint8Array, + mode: number, +): Promise { + await sandbox.fs.write(path, Buffer.from(bytes).toString('base64'), { + encoding: 'base64', + mode, + }) +} + +async function collectOutput(process: Process, maxBytes: number): Promise { + const chunks: Buffer[] = [] + let byteLength = 0 + for await (const chunk of process.stdout()) { + const bytes = Buffer.from(chunk, 'utf8') + byteLength += bytes.byteLength + if (byteLength > maxBytes) { + await killProcess(process) + throw new Error(`sandbox candidate output exceeds its ${maxBytes}-byte maximum`) + } + chunks.push(bytes) + } + return Uint8Array.from(Buffer.concat(chunks, byteLength)) +} + +async function awaitOutput( + output: Promise, + signal: AbortSignal, + deadlineAtMs: number, +): Promise { + signal.throwIfAborted() + const remainingMs = deadlineAtMs - Date.now() + if (remainingMs <= 0) throw new Error('sandbox candidate output deadline expired') + let timer: ReturnType | undefined + let onAbort: (() => void) | undefined + try { + return await Promise.race([ + output, + new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new Error('sandbox candidate output cancelled')) + signal.addEventListener('abort', onAbort, { once: true }) + timer = setTimeout( + () => reject(new Error('sandbox candidate output deadline expired')), + remainingMs, + ) + }), + ]) + } finally { + if (onAbort) signal.removeEventListener('abort', onAbort) + if (timer) clearTimeout(timer) + } +} + +async function killProcess(process: Process): Promise { + const before = await process.status() + if (!before.running) return + try { + await process.kill('SIGKILL', { tree: true }) + } catch (error) { + if ((await process.status()).running) throw error + } +} + +function processTermination(status: ProcessStatus): AgentCandidateTermination { + if (status.running) throw new Error('candidate process is still running') + if (status.exitSignal) { + if (!/^SIG[A-Z0-9]+$/.test(status.exitSignal)) { + throw new Error('sandbox returned an invalid process signal') + } + return { kind: 'signal', signal: status.exitSignal } + } + return { kind: 'exit', exitCode: status.exitCode } +} + +function matchesExecution( + sandbox: { metadata?: Record }, + request: AgentCandidateExecutorStopRequest, +): boolean { + return ( + sandbox.metadata?.kind === 'agent-candidate-execution' && + sandbox.metadata.executionId === request.executionId && + sandbox.metadata.executionPlanDigest === request.executionPlanDigest + ) +} + +function sandboxOutputSpec(sandbox: { metadata?: Record }): { maxBytes: number } { + const maxBytes = sandbox.metadata?.outputMaxBytes + if (!Number.isSafeInteger(maxBytes) || Number(maxBytes) < 1) { + throw new Error('candidate sandbox output bound is unavailable') + } + return { maxBytes: Number(maxBytes) } +} + +function executionKey(executionId: string, executionPlanDigest: Sha256Digest): string { + return `${executionId}\0${executionPlanDigest}` +} + +function exactImage(image: string, manifestDigest: Sha256Digest): string { + const marker = image.lastIndexOf('@') + if (marker < 0) return `${image}@${manifestDigest}` + if (image.slice(marker + 1) !== manifestDigest) { + throw new Error('candidate container image conflicts with its resolved manifest') + } + return image +} + +function beneath(root: string, relativePath: string): string { + if (posix.isAbsolute(relativePath)) throw new Error('candidate input path must be relative') + const path = posix.normalize(relativePath) + if (path === '..' || path.startsWith('../')) { + throw new Error('candidate input path escapes its execution root') + } + return posix.join(root, path) +} diff --git a/src/intelligence/with-intelligence.test.ts b/src/intelligence/with-intelligence.test.ts index e38e4478..a0a9d04d 100644 --- a/src/intelligence/with-intelligence.test.ts +++ b/src/intelligence/with-intelligence.test.ts @@ -1,4 +1,8 @@ -import type { AgentProfile, AgentProfileDiff } from '@tangle-network/agent-interface' +import type { + AgentProfile, + AgentProfileDiff, + CandidateExecutionEvidence, +} from '@tangle-network/agent-interface' import { applyAgentProfileDiff } from '@tangle-network/agent-interface' import { afterEach, describe, expect, it, vi } from 'vitest' import type { ProposedProfileDiff } from './delivery' @@ -8,7 +12,6 @@ import { withIntelligence } from './with-intelligence' /** A valid, promoted profile diff — the previously-DROPPED typed artifact the * composed endpoint returns alongside prompt/skill artifacts. */ const DIFF: AgentProfileDiff = { - schemaVersion: 1, kind: 'agent-profile-diff', id: 'diff-1', title: 'certified refund tool', @@ -243,15 +246,25 @@ describe('withIntelligence — SEND (a typed RunRecord to /v1/otlp)', () => { }, ], candidateExecution: { - proposalDigest: `sha256:${'1'.repeat(64)}`, - reviewDigest: `sha256:${'2'.repeat(64)}`, - bundleDigest: `sha256:${'3'.repeat(64)}`, - executionId: 'candidate-execution-1', - executionPlanDigest: `sha256:${'4'.repeat(64)}`, - materializationReceiptDigest: `sha256:${'5'.repeat(64)}`, - succeeded: true, - runReceiptDigest: `sha256:${'6'.repeat(64)}`, - }, + kind: 'agent-candidate-execution-evidence', + materializationReceipt: { + executionPlan: { + material: { + executionId: 'candidate-execution-1', + runCell: { experimentDigest: `sha256:${'1'.repeat(64)}` }, + }, + }, + }, + receipt: { + bundleDigest: `sha256:${'3'.repeat(64)}`, + executionPlanDigest: `sha256:${'4'.repeat(64)}`, + materializationReceiptDigest: `sha256:${'5'.repeat(64)}`, + termination: { kind: 'exit', exitCode: 0 }, + benchmarkResult: { material: { passed: true } }, + digest: `sha256:${'6'.repeat(64)}`, + }, + digest: `sha256:${'7'.repeat(64)}`, + } as CandidateExecutionEvidence, }) return 'answer' }, @@ -300,7 +313,7 @@ describe('withIntelligence — SEND (a typed RunRecord to /v1/otlp)', () => { expect(attrs['tool.name']).toBe('mcp__linear__linear_graphql') expect(String(attrs['tool.input'])).toContain(longInput) expect(attrs['tangle.candidate.execution_id']).toBe('candidate-execution-1') - expect(attrs['tangle.candidate.proposal_digest']).toBe(`sha256:${'1'.repeat(64)}`) + expect(attrs['tangle.candidate.experiment_digest']).toBe(`sha256:${'1'.repeat(64)}`) expect(attrs['tangle.candidate.run_receipt_digest']).toBe(`sha256:${'6'.repeat(64)}`) } finally { vi.useRealTimers() diff --git a/src/knowledge/improvement-job.ts b/src/knowledge/improvement-job.ts index c1f7e8da..80bf8782 100644 --- a/src/knowledge/improvement-job.ts +++ b/src/knowledge/improvement-job.ts @@ -1,12 +1,37 @@ +import { realpath } from 'node:fs/promises' +import { + type AgentCandidateCapturedArtifact, + type AgentCandidateKnowledge, + type AgentImprovementActivation, + type AgentImprovementProposal, + type AgentImprovementReview, + agentCandidateKnowledgeSchema, +} from '@tangle-network/agent-interface' import { type BuildEvalKnowledgeBundleOptions, evaluateKnowledgeBaseReadiness, + fromAgentCandidateKnowledgeRef, + hashKnowledgeBase, improveKnowledgeBase, type KnowledgeBaseQualityOptions, + type KnowledgeImprovementCandidateRef, type KnowledgeImprovementOptions, type KnowledgeImprovementResult, type KnowledgeReadinessSpec, + knowledgeImprovementCandidateRef, + promoteKnowledgeCandidate, + toAgentCandidateKnowledgeRef, + withKnowledgeImprovementCandidate, } from '@tangle-network/agent-knowledge' +import { canonicalCandidateBytes, embeddedCandidateArtifact } from '../candidate-execution/digest' +import { persistCandidateOutputArtifact } from '../candidate-execution/output-artifacts' +import type { AgentCandidateOutputArtifactPort } from '../candidate-execution/types' +import { captureAgentCandidateWorkspace } from '../candidate-execution/workspace-archive' +import { + verifyAgentImprovementActivation, + verifyAgentImprovementProposal, + verifyAgentImprovementReview, +} from '../intelligence/improvement-cycle' import type { ExecutorConfig } from '../runtime/supervise/runtime' import type { SuperviseOptions } from '../runtime/supervise/supervise' import type { SupervisorProfile } from '../runtime/supervise/supervisor-agent' @@ -39,9 +64,22 @@ export interface RunKnowledgeImprovementJobOptions task: unknown, opts: SuperviseOptions, ) => Promise> + candidateArtifacts?: AgentCandidateOutputArtifactPort + approval?: ApprovedKnowledgeImprovementCandidate onMeasurement?: (measurement: KnowledgeImprovementJobMeasurement) => Promise | void } +export interface ApprovedKnowledgeImprovementCandidate { + proposal: AgentImprovementProposal + review: AgentImprovementReview + activation: AgentImprovementActivation + authorizeActivation: ( + activation: AgentImprovementActivation, + proposal: AgentImprovementProposal, + review: AgentImprovementReview, + ) => boolean | Promise +} + export interface KnowledgeImprovementJobMeasurement { startedAt: string finishedAt: string @@ -52,6 +90,7 @@ export interface KnowledgeImprovementJobMeasurement { iterations: number inputTokens: number outputTokens: number + usdKnown: boolean usd: number ms: number } @@ -59,6 +98,7 @@ export interface KnowledgeImprovementJobMeasurement { export interface KnowledgeImprovementJobResult { improvement: KnowledgeImprovementResult + candidateKnowledge?: AgentCandidateKnowledge measurement: KnowledgeImprovementJobMeasurement promoted: boolean blocked: boolean @@ -104,7 +144,7 @@ export function createAgentKnowledgeReadinessCheck( } } -/** Run the full KB improvement job: candidate workspace, runtime supervisor update, readiness check, and promotion. */ +/** Produce a frozen KB candidate, and promote it only when an exact signed review is supplied. */ export async function runKnowledgeImprovementJob( options: RunKnowledgeImprovementJobOptions, ): Promise { @@ -112,9 +152,11 @@ export async function runKnowledgeImprovementJob( allowedModels, backend, budget, + candidateArtifacts, harness, makeWorkerAgent, onMeasurement, + approval, readinessCheck, runSupervised, supervisorModel, @@ -147,17 +189,50 @@ export async function runKnowledgeImprovementJob( runSupervised, } satisfies SupervisedKnowledgeUpdateOptions) - const resolvedImprovement = await improveKnowledgeBase({ - ...knowledgeOptions, - updateKnowledge: async (input) => { - const updateStartedAt = Date.now() - updateCalls += 1 - const result = await updateKnowledge(input) - updateDurationMs += Date.now() - updateStartedAt - addSpent(supervisedSpent, result.supervised) - return result - }, - } as KnowledgeImprovementOptions) + const instrumentedUpdateKnowledge: KnowledgeImprovementOptions['updateKnowledge'] = async ( + input, + ) => { + const updateStartedAt = Date.now() + updateCalls += 1 + const result = await updateKnowledge(input) + updateDurationMs += Date.now() - updateStartedAt + addSpent(supervisedSpent, result.supervised) + return result + } + let resolvedImprovement: KnowledgeImprovementResult + let candidateKnowledge: AgentCandidateKnowledge | undefined + if (approval) { + const approvedKnowledge = await approvedKnowledgeCandidate(approval) + const candidate = agentKnowledgeCandidateRef(approvedKnowledge) + knowledgeOptions.signal?.throwIfAborted() + resolvedImprovement = await promoteKnowledgeCandidate({ + root: options.root, + candidate, + ...(knowledgeOptions.ownerId ? { ownerId: knowledgeOptions.ownerId } : {}), + ...(knowledgeOptions.leaseTtlMs ? { leaseTtlMs: knowledgeOptions.leaseTtlMs } : {}), + ...(knowledgeOptions.now ? { now: knowledgeOptions.now } : {}), + ...(knowledgeOptions.onState ? { onState: knowledgeOptions.onState } : {}), + }) + knowledgeOptions.signal?.throwIfAborted() + const promotedHash = await hashKnowledgeBase(options.root) + if (!resolvedImprovement.promoted || promotedHash !== candidate.candidateHash) { + throw new Error('knowledge promotion did not activate the approved snapshot bytes') + } + candidateKnowledge = approvedKnowledge + } else { + resolvedImprovement = await improveKnowledgeBase({ + ...knowledgeOptions, + updateKnowledge: instrumentedUpdateKnowledge, + }) + if (resolvedImprovement.candidate) { + candidateKnowledge = await freezeKnowledgeCandidate( + options.root, + resolvedImprovement, + candidateArtifacts, + knowledgeOptions.signal, + ) + } + } const finishedAtMs = Date.now() const measurement: KnowledgeImprovementJobMeasurement = { startedAt, @@ -170,14 +245,106 @@ export async function runKnowledgeImprovementJob( await onMeasurement?.(measurement) return { improvement: resolvedImprovement, + ...(candidateKnowledge ? { candidateKnowledge } : {}), measurement, promoted: resolvedImprovement.promoted, blocked: resolvedImprovement.blocked, } } +async function freezeKnowledgeCandidate( + root: string, + improvement: KnowledgeImprovementResult, + artifacts: AgentCandidateOutputArtifactPort | undefined, + signal: AbortSignal | undefined, +): Promise { + const candidate = knowledgeImprovementCandidateRef(improvement) + return withKnowledgeImprovementCandidate({ root, candidate }, async (resolved) => { + const executionId = `knowledge-${candidate.candidateId}` + const candidateRef = toAgentCandidateKnowledgeRef(candidate) + const captured = await captureAgentCandidateWorkspace(await realpath(resolved.root), { + ...(artifacts + ? { + artifactPersistence: { + executionId, + outputArtifacts: artifacts, + ...(signal ? { signal } : {}), + }, + } + : {}), + }) + const evaluation = await captureKnowledgeEvidence( + canonicalCandidateBytes({ + kind: 'agent-knowledge-candidate-evaluation', + candidate: candidateRef, + metric: resolved.evaluation, + }), + 'knowledge-evaluation', + executionId, + artifacts, + signal, + ) + return agentCandidateKnowledgeSchema.parse({ + candidate: candidateRef, + snapshot: captured.snapshot, + evaluation, + }) + }) +} + +async function captureKnowledgeEvidence( + bytes: Uint8Array, + purpose: 'knowledge-evaluation', + executionId: string, + artifacts: AgentCandidateOutputArtifactPort | undefined, + signal: AbortSignal | undefined, +): Promise { + if (!artifacts) return embeddedCandidateArtifact(bytes) + return persistCandidateOutputArtifact(artifacts, { + executionId, + purpose, + bytes, + ...(signal ? { signal } : {}), + }) +} + +async function approvedKnowledgeCandidate( + approval: ApprovedKnowledgeImprovementCandidate, +): Promise { + const proposal = verifyAgentImprovementProposal(approval.proposal) + const review = verifyAgentImprovementReview(approval.review) + const activation = verifyAgentImprovementActivation({ + proposal, + review, + activation: approval.activation, + }) + const candidateBundle = proposal.evaluation.experiment.candidate + const knowledgeTarget = activation.targets.find((target) => target.surface === 'knowledge') + if ( + proposal.changedSurfaces.length !== 1 || + proposal.changedSurfaces[0] !== 'knowledge' || + !candidateBundle.knowledge || + !knowledgeTarget || + knowledgeTarget.expectedBaseDigest !== candidateBundle.knowledge.candidate.baseHash || + review.decision !== 'approve' || + review.proposalDigest !== proposal.digest + ) { + throw new Error('knowledge promotion requires an approved exact knowledge proposal') + } + if (!(await approval.authorizeActivation(activation, proposal, review))) { + throw new Error('knowledge candidate activation was not authorized') + } + return candidateBundle.knowledge +} + +function agentKnowledgeCandidateRef( + knowledge: AgentCandidateKnowledge, +): KnowledgeImprovementCandidateRef { + return fromAgentCandidateKnowledgeRef(knowledge.candidate) +} + function emptySpent(): KnowledgeImprovementJobMeasurement['supervisedSpent'] { - return { iterations: 0, inputTokens: 0, outputTokens: 0, usd: 0, ms: 0 } + return { iterations: 0, inputTokens: 0, outputTokens: 0, usdKnown: true, usd: 0, ms: 0 } } function addSpent( @@ -185,9 +352,10 @@ function addSpent( result: SupervisedResult, ): void { const spent = result.spentTotal - target.iterations += spent.iterations ?? 0 - target.inputTokens += spent.tokens?.input ?? 0 - target.outputTokens += spent.tokens?.output ?? 0 - target.usd += spent.usd ?? 0 - target.ms += spent.ms ?? 0 + target.iterations += spent.iterations + target.inputTokens += spent.tokens.input + target.outputTokens += spent.tokens.output + target.usdKnown = target.usdKnown && spent.usdKnown !== false + target.usd += spent.usd + target.ms += spent.ms } diff --git a/src/knowledge/index.ts b/src/knowledge/index.ts index 0297ca41..d9920117 100644 --- a/src/knowledge/index.ts +++ b/src/knowledge/index.ts @@ -1,5 +1,6 @@ export { type AgentKnowledgeReadinessCheckOptions, + type ApprovedKnowledgeImprovementCandidate, createAgentKnowledgeReadinessCheck, type KnowledgeImprovementJobMeasurement, type KnowledgeImprovementJobResult, diff --git a/src/knowledge/supervised-update.ts b/src/knowledge/supervised-update.ts index 9e7149e7..b1df286b 100644 --- a/src/knowledge/supervised-update.ts +++ b/src/knowledge/supervised-update.ts @@ -1,3 +1,4 @@ +import type { RagKnowledgeUpdateResult } from '@tangle-network/agent-knowledge' import { researcherProfile } from '../profiles/researcher' import type { DeliverableSpec } from '../runtime/supervise/completion-gate' import type { ExecutorConfig } from '../runtime/supervise/runtime' @@ -51,7 +52,7 @@ export interface SupervisedKnowledgeUpdateResult { applied: boolean summary: string supervised: SupervisedResult - metadata: Record + metadata: NonNullable } export interface SupervisedKnowledgeUpdateOptions { diff --git a/src/mcp/feedback-store.ts b/src/mcp/feedback-store.ts index 1ebbbfa6..50f6124a 100644 --- a/src/mcp/feedback-store.ts +++ b/src/mcp/feedback-store.ts @@ -2,11 +2,10 @@ * * Feedback persistence surface for the MCP layer. * - * The substrate cannot import `@tangle-network/agent-knowledge` (it would - * induce a dependency cycle), so the store is an abstract interface. The - * default implementation is in-memory; consumers wire their own adapter - * (a real KbStore-backed sink, an HTTP relay to gtm-agent's knowledge - * service, etc.) via `createMcpServer({ feedbackStore })`. + * Feedback storage is product policy, so the MCP layer depends on this narrow + * interface instead of choosing a knowledge store. The default implementation + * is in-memory; consumers wire their own durable adapter via + * `createMcpServer({ feedbackStore })`. * * Feedback events are append-only: every rating is a new event with a * fresh id, even when the same delegation is rated multiple times. The diff --git a/src/mcp/types.ts b/src/mcp/types.ts index 1cc95f95..576eb140 100644 --- a/src/mcp/types.ts +++ b/src/mcp/types.ts @@ -226,9 +226,9 @@ export interface DelegateUiAuditResult { } /** - * Loose shape of a research output over the wire — the substrate cannot - * import the `ResearchOutput` type from agent-knowledge without inducing - * a dependency cycle, so the MCP layer treats it structurally. + * Provider-neutral research output carried over the MCP boundary. The MCP + * layer accepts this structural shape instead of coupling its wire contract to + * one research implementation. * * @experimental */ diff --git a/src/mcp/worktree-harness.ts b/src/mcp/worktree-harness.ts index 4a5ab50c..7e1b22a0 100644 --- a/src/mcp/worktree-harness.ts +++ b/src/mcp/worktree-harness.ts @@ -218,6 +218,7 @@ export async function runWorktreeHarness( try { assertSupportedWorktreeProfile(opts.profile, opts.harness) + assertSafeProfileResourcePaths(opts.profile) const resourceInstructions = resolveResourceInstructions(opts.profile) const workspaceProfile = materializationOnlyProfile(opts.profile) const plan = materializeProfile(workspaceProfile, opts.harness) @@ -229,7 +230,7 @@ export async function runWorktreeHarness( ) } assertSafeMaterializedPaths(plan) - const applied = applyWorkspacePlan(plan, worktree.path) + const applied = applyWorkspacePlan(plan, worktree.path, { existingFiles: 'reject' }) if (applied.unsupported.length > 0) { throw new Error('runWorktreeHarness: applied profile unexpectedly retained unsupported rows') } @@ -437,6 +438,16 @@ function assertSafeMaterializedPaths(plan: WorkspacePlan): void { } } +function assertSafeProfileResourcePaths(profile: AgentProfile): void { + for (const file of profile.resources?.files ?? []) { + if (file.path.split('/').some(isGitMetadataSegment)) { + throw new Error( + `runWorktreeHarness: profile file cannot target reserved Git metadata: ${file.path}`, + ) + } + } +} + function isGitMetadataSegment(segment: string): boolean { const windowsCanonical = segment.replace(/[ .]+$/u, '').toLowerCase() return windowsCanonical === '.git' || windowsCanonical.startsWith('.git:') diff --git a/src/runtime/environment-provider.ts b/src/runtime/environment-provider.ts index c6a62a4b..adcdcfff 100644 --- a/src/runtime/environment-provider.ts +++ b/src/runtime/environment-provider.ts @@ -612,22 +612,25 @@ function sandboxInstanceAsEnvironment( }, } : {}), - ...(hasCheckpoint(box) - ? { - async checkpoint(options?: CheckpointRequest): Promise { - const result = await box.checkpoint(options as never) - return { id: checkpointIdFromResult(result), provider: providerName } - }, - } - : {}), - ...(hasFork(box) - ? { - async fork(checkpoint: CheckpointRef, options?: ForkRequest): Promise { - const forked = await box.fork(checkpoint.id, options as never) - return sandboxInstanceAsEnvironment(forked, providerName, client) - }, - } - : {}), + async checkpoint(options?: CheckpointRequest): Promise { + const result = await box.snapshot({ + ...(options?.name ? { tags: [options.name] } : {}), + }) + return { + id: result.snapshotId, + provider: providerName, + ...(options?.metadata ? { metadata: options.metadata } : {}), + } + }, + async fork(checkpoint: CheckpointRef, options?: ForkRequest): Promise { + const forked = await client.create({ + fromSnapshot: checkpoint.id, + fromSandboxId: String(box.id), + ...(options?.name ? { name: options.name } : {}), + ...(options?.metadata ? { metadata: options.metadata } : {}), + }) + return sandboxInstanceAsEnvironment(forked, providerName, client) + }, async placement(): Promise { return placementInfoFromLoopPlacement(client.describePlacement?.(box), box) }, @@ -1049,15 +1052,6 @@ function placementInfoFromLoopPlacement( } } -function checkpointIdFromResult(result: unknown): string { - const record = result && typeof result === 'object' ? (result as Record) : {} - const id = record.checkpointId ?? record.id - if (typeof id !== 'string' || id.length === 0) { - throw new ValidationError('sandboxClientAsProvider: checkpoint returned no checkpoint id') - } - return id -} - function defaultTangleSandboxCapabilities(): AgentEnvironmentCapabilities { return { profile: { @@ -1159,18 +1153,6 @@ function hasExec(box: SandboxInstance): box is SandboxInstance & { return typeof (box as { exec?: unknown }).exec === 'function' } -function hasCheckpoint( - box: SandboxInstance, -): box is SandboxInstance & { checkpoint(options?: unknown): Promise } { - return typeof (box as { checkpoint?: unknown }).checkpoint === 'function' -} - -function hasFork(box: SandboxInstance): box is SandboxInstance & { - fork(checkpointId: string, options?: unknown): Promise -} { - return typeof (box as { fork?: unknown }).fork === 'function' -} - interface SandboxSessionLike { readonly id: string status(): Promise diff --git a/tests/agent.test.ts b/tests/agent.test.ts index d1a52751..97953ebd 100644 --- a/tests/agent.test.ts +++ b/tests/agent.test.ts @@ -460,16 +460,15 @@ describe('createSurfaceImprovementAdapter — apply', () => { expect(r.warnings.join(' ')).toMatch(/mode=none/) }) - it('mode=open-pr without ghRepo fails loud (no silent fallback)', async () => { - const adapter = createSurfaceImprovementAdapter({ - surfaces, - repoRoot: tmpRoot, - draftPatch: async () => ({ patch: 'x', summary: 'y', rationale: 'z' }), - mode: 'open-pr', - }) - const r = await adapter.apply!([]) - expect(r.applied).toEqual([]) - expect(r.warnings.join(' ')).toMatch(/requires `ghRepo`/) + it('mode=open-pr without ghRepo fails at construction', () => { + expect(() => + createSurfaceImprovementAdapter({ + surfaces, + repoRoot: tmpRoot, + draftPatch: async () => ({ patch: 'x', summary: 'y', rationale: 'z' }), + mode: 'open-pr', + }), + ).toThrow(/requires `ghRepo`/) }) }) diff --git a/tests/agentic-generator.test.ts b/tests/agentic-generator.test.ts index 6c85d9e3..483266ae 100644 --- a/tests/agentic-generator.test.ts +++ b/tests/agentic-generator.test.ts @@ -194,7 +194,6 @@ describe('agenticGenerator — runs a harness in the worktree', () => { expect(out.applied).toBe(true) expect(receipts).toHaveLength(1) expect(receipts[0]).toMatchObject({ - schemaVersion: 1, generation: 4, candidateIndex: 2, shot: 1, diff --git a/tests/candidate-bundle-builder.test.ts b/tests/candidate-bundle-builder.test.ts index cd55b256..bf1266ad 100644 --- a/tests/candidate-bundle-builder.test.ts +++ b/tests/candidate-bundle-builder.test.ts @@ -3,7 +3,6 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { canonicalJson } from '@tangle-network/agent-eval' import { gitWorktreeAdapter } from '@tangle-network/agent-eval/campaign' import type { AgentCandidateArtifactRef, @@ -35,7 +34,7 @@ afterEach(() => { }) describe('public agent candidate bundle builder', () => { - it('binds profile diffs, verified CodeSurface bytes, knowledge, and memory into one executable candidate', async () => { + it('binds profile diffs, verified CodeSurface bytes, and memory into one executable candidate', async () => { const fixture = createCandidateExecutionFixture(true) const adapter = gitWorktreeAdapter({ repoRoot: fixture.task.stagingRoots.taskRoot, @@ -54,7 +53,6 @@ describe('public agent candidate bundle builder', () => { resources: { failOnError: true }, } const diff: AgentProfileDiff = { - schemaVersion: 1, kind: 'agent-profile-diff', id: 'add-review-skill', source: { kind: 'optimizer', artifacts: ['trace:run-1'] }, @@ -85,7 +83,6 @@ describe('public agent candidate bundle builder', () => { execution: fixture.bundle.execution, knowledge: { candidate: { - schemaVersion: 1, kind: 'knowledge-improvement-candidate', runId: 'knowledge-run-1', candidateId: 'knowledge-snapshot-1', @@ -100,16 +97,6 @@ describe('public agent candidate bundle builder', () => { evaluation: knowledgeEvaluation, }, memory: { mode: 'isolated', scope: 'task', seed: memorySeed }, - lineage: { - source: 'compound', - parentDigests: [ - fixture.bundle.lineage.parentDigests?.[0] ?? candidateSha('e'), - candidateSha('9'), - ], - runIds: ['optimizer-run-1'], - benchmark: fixture.bundle.lineage.benchmark, - spend: fixture.bundle.lineage.spend, - }, } try { @@ -136,9 +123,7 @@ describe('public agent candidate bundle builder', () => { name: 'review/SKILL.md', sha256: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), }) - expect(first.lineage.profileDiffIds).toEqual([ - `sha256:${createHash('sha256').update(canonicalJson(diff)).digest('hex')}`, - ]) + expect(first.knowledge).toEqual(input.knowledge) const verified = await verifyAgentCandidateBundle(first, { repositories: fixture.ports.repositories, @@ -181,7 +166,6 @@ describe('public agent candidate bundle builder', () => { }, execution: fixture.bundle.execution, memory: { mode: 'disabled' }, - lineage: { source: 'optimizer' }, }), ).toThrow(/CodeSurface|patch/i) @@ -195,10 +179,9 @@ describe('public agent candidate bundle builder', () => { it('fails closed when a generic profile would lose behavior or byte identity', () => { const fixture = createCandidateExecutionFixture(false) const base = { - code: { kind: 'disabled', reason: 'control' } as const, + code: { kind: 'disabled' } as const, execution: fixture.bundle.execution, memory: { mode: 'disabled' } as const, - lineage: { source: 'human' } as const, } expect(() => buildAgentCandidateBundle({ @@ -248,16 +231,6 @@ describe('public agent candidate bundle builder', () => { code: { kind: 'git-patch' } as unknown as BuildAgentCandidateBundleInput['code'], }), ).toThrow(/unsupported candidate code source/) - expect(() => - buildAgentCandidateBundle({ - ...base, - profile: { kind: 'profile', profile: simpleProfile() }, - lineage: { - source: 'human', - profileDiffIds: ['caller-controlled'], - } as unknown as BuildAgentCandidateBundleInput['lineage'], - }), - ).toThrow(/profileDiffIds.*derived/) }) it('round-trips every currently representable generic profile surface', () => { @@ -329,10 +302,9 @@ describe('public agent candidate bundle builder', () => { const bundle = buildAgentCandidateBundle({ profile: { kind: 'profile', profile }, - code: { kind: 'disabled', reason: 'control' }, + code: { kind: 'disabled' }, execution: fixture.bundle.execution, memory: { mode: 'disabled' }, - lineage: { source: 'human' }, }) expect(bundle.profile).toMatchObject({ @@ -363,10 +335,9 @@ describe('public agent candidate bundle builder', () => { }, }, }, - code: { kind: 'disabled', reason: 'control' }, + code: { kind: 'disabled' }, execution: fixture.bundle.execution, memory: { mode: 'disabled' }, - lineage: { source: 'human' }, }) expect(bundle.profile.hooks?.Stop?.[0]).toEqual({ executable: 'node', diff --git a/tests/candidate-execution-claim.test.ts b/tests/candidate-execution-claim.test.ts index 0f139045..699b6e31 100644 --- a/tests/candidate-execution-claim.test.ts +++ b/tests/candidate-execution-claim.test.ts @@ -2,7 +2,10 @@ import { spawn } from 'node:child_process' import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { resolve } from 'node:path' -import type { AgentCandidateArtifactRef } from '@tangle-network/agent-interface' +import type { + AgentCandidateArtifactRef, + AgentCandidateFixedSpend, +} from '@tangle-network/agent-interface' import { afterEach, describe, expect, it, vi } from 'vitest' import { @@ -12,7 +15,6 @@ import { type AgentCandidateExecutionLease, type AgentCandidateExecutionRecoveryEvidence, type AgentCandidateExecutionTerminalResult, - type AgentCandidateExecutionUsage, InMemoryAgentCandidateExecutionClaimStore, } from '../src/candidate-execution/claim' import { FileAgentCandidateExecutionClaimStore } from '../src/candidate-execution/claim-file-store' @@ -51,17 +53,17 @@ describe('candidate execution claim lifecycle', () => { const claimedAtMs = Date.now() vi.spyOn(Date, 'now').mockReturnValue(claimedAtMs) - const requested = candidateExecutionClaim(prepared) + const requested = candidateExecutionClaim(prepared, preparationEvidenceFor(prepared)) const ownerWindowMs = candidateExecutionOwnerWindowMs( - fixture.task.limits.timeoutMs, + fixture.task.task.limits.timeoutMs, cleanupTimeoutMs, - fixture.task.limits.timeoutMs, - fixture.task.limits.timeoutMs, + fixture.task.task.limits.timeoutMs, + fixture.task.task.limits.timeoutMs, ) expect(requested.leaseExpiresAtMs).toBe(claimedAtMs + ownerWindowMs) expect(requested.cleanup.cleanupTimeoutMs).toBe(cleanupTimeoutMs) - expect(requested.resultTimeoutMs).toBe(fixture.task.limits.timeoutMs) + expect(requested.resultTimeoutMs).toBe(fixture.task.task.limits.timeoutMs) expect(ownerWindowMs).toBeLessThan(15 * 60_000) }) @@ -81,14 +83,14 @@ describe('candidate execution claim lifecycle', () => { { cleanupTimeoutMs }, ) const ownerWindowMs = candidateExecutionOwnerWindowMs( - fixture.task.limits.timeoutMs, + fixture.task.task.limits.timeoutMs, cleanupTimeoutMs, - fixture.task.limits.timeoutMs, - fixture.task.limits.timeoutMs, + fixture.task.task.limits.timeoutMs, + fixture.task.task.limits.timeoutMs, ) vi.spyOn(Date, 'now').mockReturnValue(reservationExpiresAtMs - ownerWindowMs + 1) - expect(() => candidateExecutionClaim(prepared)).toThrow( + expect(() => candidateExecutionClaim(prepared, preparationEvidenceFor(prepared))).toThrow( /full execution and cleanup owner window/, ) }) @@ -172,7 +174,6 @@ describe('candidate execution claim lifecycle', () => { expect(terminal.usage).toEqual(result.usage) expect(terminal).toMatchObject({ - schemaVersion: 1, modelSettlement: result.modelSettlement, taskOutcome: result.taskOutcome, benchmarkResult: result.benchmarkResult, @@ -519,7 +520,7 @@ describe('candidate execution claim lifecycle', () => { ).toMatchObject({ acquired: false, detail: 'retry-lineage-mismatch' }) }) - it('persists claim7, pending1, terminal3, phase, staged, and full usage across stores', async () => { + it('persists claims, transitions, terminals, and full usage across stores', async () => { const directory = await tempDirectory() const store = new FileAgentCandidateExecutionClaimStore({ directory }) const acquired = await acquire(store, claim()) @@ -539,9 +540,7 @@ describe('candidate execution claim lifecycle', () => { staged: terminal, terminal, }) - expect(files.find(({ name }) => name.endsWith('.claim.json'))?.text).toContain('"version":7') - expect(files.find(({ name }) => name.includes('transition-2'))?.text).toContain('"version":1') - expect(files.find(({ name }) => name.endsWith('.terminal.json'))?.text).toContain('"version":3') + for (const file of files) expect(JSON.parse(file.text)).not.toHaveProperty('version') expect(files.map(({ text }) => text).join('\n')).not.toContain(acquired.lease.token) }) @@ -642,6 +641,10 @@ function claim( retryPolicy: 'none', bundleDigest: sha256('a'), executionPlanDigest: sha256('b'), + preparationEvidence: { + executionPlan: artifact('b'), + materializationReceipt: artifact('e'), + }, retryLineageDigest: sha256('c'), leaseExpiresAtMs: FUTURE_EXPIRY_MS, resultTimeoutMs: 60_000, @@ -656,10 +659,27 @@ function retryClaim( return claim({ maxAttempts: 3, retryPolicy: 'pre-model-infrastructure-only', ...overrides }) } +function preparationEvidenceFor( + prepared: Parameters[0], +): AgentCandidateExecutionClaim['preparationEvidence'] { + return { + executionPlan: { + locator: { kind: 's3', bucket: 'candidate-evidence', key: 'execution-plan.json' }, + sha256: prepared.executionPlan.value.digest, + byteLength: prepared.executionPlan.bytes.byteLength, + }, + materializationReceipt: { + locator: { kind: 's3', bucket: 'candidate-evidence', key: 'materialization-receipt.json' }, + sha256: prepared.materializationReceipt.digest, + byteLength: prepared.materializationReceipt.bytes.byteLength, + }, + } +} + function usage( modelCalls = 0, - overrides: Partial = {}, -): AgentCandidateExecutionUsage { + overrides: Partial = {}, +): AgentCandidateFixedSpend { return { costUsdNanos: modelCalls * 123_456, inputTokens: modelCalls * 101, @@ -677,7 +697,6 @@ function failed( overrides: Partial> = {}, ): Extract { return { - schemaVersion: 1, status: 'failed', failureClass, usage: usage(modelCalls), @@ -688,7 +707,6 @@ function failed( function succeeded(): Extract { return { - schemaVersion: 1, status: 'succeeded', usage: usage(2), modelSettlement: artifact('1'), @@ -731,14 +749,14 @@ function cleanupHandles(memory = false): AgentCandidateExecutionClaim['cleanup'] } function preparationId(character: string): string { - return `candidate-preparation-v1.${character.repeat(43)}` + return `candidate-preparation.${character.repeat(43)}` } function recoveryEvidence( requested: AgentCandidateExecutionClaim, overrides: { failureClass?: AgentCandidateExecutionFailureClass - usage?: AgentCandidateExecutionUsage + usage?: AgentCandidateFixedSpend modelSettlement?: AgentCandidateArtifactRef } = {}, ): AgentCandidateExecutionRecoveryEvidence { @@ -770,7 +788,7 @@ function sha256(character: string): `sha256:${string}` { } function leaseToken(character: string): string { - return `candidate-execution-lease-v1.${character.repeat(43)}` + return `candidate-execution-lease.${character.repeat(43)}` } async function acquire( diff --git a/tests/candidate-execution-cleanup.test.ts b/tests/candidate-execution-cleanup.test.ts index 36b431c6..dba378e0 100644 --- a/tests/candidate-execution-cleanup.test.ts +++ b/tests/candidate-execution-cleanup.test.ts @@ -41,9 +41,11 @@ describe('candidate cleanup timer bounds', () => { it('rejects results observed exactly at their frozen deadline', async () => { vi.useFakeTimers({ now: 100 }) + let observedCleanupSignal: AbortSignal | undefined await expect( withinCandidateCleanupDeadline( - async () => { + async (signal) => { + observedCleanupSignal = signal vi.setSystemTime(110) return 'ambiguous' }, @@ -51,6 +53,7 @@ describe('candidate cleanup timer bounds', () => { 'cleanup', ), ).rejects.toBeInstanceOf(CandidateCleanupTimeoutError) + expect(observedCleanupSignal?.aborted).toBe(true) let observedSignal: AbortSignal | undefined await expect( diff --git a/tests/candidate-execution-core.test.ts b/tests/candidate-execution-core.test.ts index a5351e68..751f7e63 100644 --- a/tests/candidate-execution-core.test.ts +++ b/tests/candidate-execution-core.test.ts @@ -102,7 +102,6 @@ describe('candidate canonical bytes and artifacts', () => { it('requires workspace manifest bytes to equal canonical material, not only a locator', async () => { const material: AgentCandidateWorkspaceManifestMaterial = { - schemaVersion: 2, kind: 'agent-candidate-workspace-manifest', files: [], } @@ -111,7 +110,6 @@ describe('candidate canonical bytes and artifacts', () => { await expect( verifyWorkspaceSnapshotArtifacts( { - schemaVersion: 2, kind: 'agent-candidate-workspace-snapshot', digest: manifest.sha256, material, @@ -124,7 +122,6 @@ describe('candidate canonical bytes and artifacts', () => { await expect( verifyWorkspaceSnapshotArtifacts( { - schemaVersion: 2, kind: 'agent-candidate-workspace-snapshot', digest: manifest.sha256, material, @@ -198,7 +195,6 @@ describe('materialized workspace identity', () => { chmodSync(join(root, 'run.js'), 0o664) const bytes = Buffer.from('ok\n') const material: AgentCandidateWorkspaceManifestMaterial = { - schemaVersion: 2, kind: 'agent-candidate-workspace-manifest', files: [ { @@ -233,7 +229,6 @@ describe('materialized workspace identity', () => { chmodSync(join(root, 'source'), 0o644) symlinkSync('source', join(root, 'symlink')) const material: AgentCandidateWorkspaceManifestMaterial = { - schemaVersion: 2, kind: 'agent-candidate-workspace-manifest', files: [], } diff --git a/tests/candidate-execution-dispose.test.ts b/tests/candidate-execution-dispose.test.ts index 89f7fe46..7a73be9a 100644 --- a/tests/candidate-execution-dispose.test.ts +++ b/tests/candidate-execution-dispose.test.ts @@ -36,7 +36,7 @@ describe('prepared candidate disposal', () => { disposed: true, }) expect(reasons).toHaveLength(1) - expect(reasons[0]).toMatch(/candidate-preparation-v1\..+:abandoned/) + expect(reasons[0]).toMatch(/candidate-preparation\..+:abandoned/) await expect( executePreparedAgentCandidate(prepared, { traceStore: {} as never, diff --git a/tests/candidate-execution-execute.test.ts b/tests/candidate-execution-execute.test.ts index 4940f5c4..f06e3c9d 100644 --- a/tests/candidate-execution-execute.test.ts +++ b/tests/candidate-execution-execute.test.ts @@ -21,13 +21,17 @@ import { createAgentCandidateWorkspacePort, } from '../src/candidate-execution/workspace-archive' import { + bindCandidateFixtureBundle, candidateBundle, candidateSha, cleanupCandidateFixtures, createCandidateExecutionFixture, + createCandidateOutputExecutionFixture, createCandidateOutputFixture, emptyCandidateSnapshot, redigestCandidateBundle, + replaceCandidateFixtureAttempt, + replaceCandidateFixtureTask, unchangedTaskOutcomeCapture, } from './helpers/candidate-execution-fixture' @@ -52,27 +56,48 @@ async function terminalTrace( }) } +interface TestExecutor { + execute: AgentCandidateExecutorPort['execute'] + stop: AgentCandidateExecutorPort['stop'] + capture?: AgentCandidateExecutorPort['capture'] + dispose?: AgentCandidateExecutorPort['dispose'] +} + function options( - executor: AgentCandidateExecutorPort, + executor: TestExecutor, traceStore = new InMemoryTraceStore(), claimStore = new InMemoryAgentCandidateExecutionClaimStore(), ) { const outputs = createCandidateOutputFixture() - let request: AgentCandidateExecutorRequest | undefined + const requests = new Map() return { executor: { execute: async (...args: Parameters) => { - request = args[0] + requests.set(args[0].executionId, args[0]) return await executor.execute(...args) }, - stopAndCapture: async (...args: Parameters) => { - const capture = await executor.stopAndCapture(...args) - if (capture.taskOutcome || !request) return capture + stop: executor.stop, + ...(executor.dispose !== undefined ? { dispose: executor.dispose } : {}), + capture: async ( + stopRequest: Parameters[0], + context: Parameters[1], + ) => { + const capture = executor.capture ? await executor.capture(stopRequest, context) : {} + if (capture.taskOutcome) return capture + const request = requests.get(stopRequest.executionId) + if (!request) throw new Error('fixture capture has no execution request') + const outcome = request.benchmark.task.outcome + if (outcome.kind !== 'workspace') { + throw new Error('fixture fallback requires a workspace outcome') + } + const repository = request.benchmark.task.repository + if (!repository) throw new Error('fixture fallback requires repository identity') return { ...capture, taskOutcome: { - resultTree: request.executionPlan.value.material.task.repository.baseTree, - afterState: request.executionPlan.value.material.task.workspace.material, + kind: 'workspace' as const, + resultTree: repository.baseTree, + afterState: request.benchmark.task.workspace.material, archive: Buffer.from('fixture unchanged task archive', 'utf8'), gitDiff: Buffer.alloc(0), }, @@ -88,9 +113,10 @@ function options( describe('atomic prepared candidate execution', () => { it('reveals credentials only to one trusted executor and returns a durable receipt', async () => { const fixture = createCandidateExecutionFixture() - fixture.bundle = candidateBundle({ - env: { PUBLIC_MODE: { kind: 'public', value: 'fixture' } }, - }) + bindCandidateFixtureBundle( + fixture, + candidateBundle({ env: { PUBLIC_MODE: { kind: 'public', value: 'fixture' } } }), + ) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), fixture.task, @@ -102,7 +128,7 @@ describe('atomic prepared candidate execution', () => { const traceStore = new InMemoryTraceStore() let observed: AgentCandidateExecutorRequest | undefined let stopped = 0 - const executor: AgentCandidateExecutorPort = { + const executor: TestExecutor = { execute: async (request, context) => { observed = request expect(context.traceStore).not.toBe(traceStore) @@ -114,8 +140,8 @@ describe('atomic prepared candidate execution', () => { TANGLE_CANDIDATE_EXECUTION_ID: prepared.executionId, }) expect(request.launch.env.PATH).toBeUndefined() - expect(request.hardLimits).toEqual({ timeoutMs: fixture.task.limits.timeoutMs }) - expect(request.observedLimits).toEqual({ maxSteps: fixture.task.limits.maxSteps }) + expect(request.hardLimits).toEqual({ timeoutMs: fixture.task.task.limits.timeoutMs }) + expect(request.observedLimits).toEqual({ maxSteps: fixture.task.task.limits.maxSteps }) expect(request.executionPlan.value.material.model.access.network).toEqual({ mode: 'gateway-only', domains: ['router.tangle.tools'], @@ -129,8 +155,11 @@ describe('atomic prepared candidate execution', () => { termination: { kind: 'exit', exitCode: 0 }, } }, - stopAndCapture: async () => { + stop: async () => { stopped++ + return { stopped: true } + }, + capture: async () => { if (!observed) throw new Error('candidate executor did not receive its request') const changedBytes = Buffer.from('export const value = 2\n') const taskRoot = fixture.task.stagingRoots.taskRoot @@ -151,8 +180,8 @@ describe('atomic prepared candidate execution', () => { })), ) return { - stopped: true, taskOutcome: { + kind: 'workspace', resultTree, afterState: captured.snapshot.material, archive: captured.archive, @@ -171,7 +200,9 @@ describe('atomic prepared candidate execution', () => { succeeded: true, receipt: { value: { - schemaVersion: 3, + executorCapture: expect.objectContaining({ + sha256: expect.stringMatching(/^sha256:/), + }), modelSettlement: { material: { usage: { modelCalls: 0, costUsdNanos: 0 } }, }, @@ -188,6 +219,54 @@ describe('atomic prepared candidate execution', () => { await expect( executionOptions.outputArtifacts.read(result.artifacts.runReceipt), ).resolves.toEqual(result.receipt.bytes) + expect(result.receipt.value.executorCapture).toEqual(result.artifacts.executorCapture) + }) + + it('grades and receipts exact normal-agent output through the shared execution path', async () => { + const fixture = createCandidateOutputExecutionFixture('application/json', 1_024) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const output = Buffer.from('{"recommendation":"add focused failure recovery"}\n', 'utf8') + const executionOptions = options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stop: async () => ({ stopped: true }), + capture: async () => ({ + taskOutcome: { kind: 'output', bytes: output }, + }), + }, + traceStore, + ) + let gradedOutput = Buffer.alloc(0) + executionOptions.grader = { + ...executionOptions.grader, + run: async (input) => { + if (input.outcome.kind !== 'output') throw new Error('expected output outcome') + gradedOutput = Buffer.from(input.outcome.bytes) + return await createCandidateOutputFixture().grader.run(input) + }, + } + + const result = await executePreparedAgentCandidate(prepared, executionOptions) + if (!result.succeeded) throw new Error(result.reason) + const captured = result.receipt.value.taskOutcome.material.outcome + if (captured.kind !== 'output') throw new Error('expected output receipt') + expect(gradedOutput).toEqual(output) + await expect(executionOptions.outputArtifacts.read(captured.artifact)).resolves.toEqual( + Uint8Array.from(output), + ) + expect(result.receipt.value.benchmarkResult.material).toMatchObject({ + taskOutcomeDigest: result.receipt.value.taskOutcome.digest, + score: 1, + passed: true, + }) }) it('publishes a referenced final receipt for a task archive larger than 28,254,720 bytes', async () => { @@ -207,7 +286,8 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => { + stop: async () => ({ stopped: true }), + capture: async () => { const taskRoot = fixture.task.stagingRoots.taskRoot writeFileSync(join(taskRoot, 'source.ts'), largeSource, { mode: 0o644 }) execFileSync('git', ['add', 'source.ts'], { cwd: taskRoot }) @@ -224,8 +304,8 @@ describe('atomic prepared candidate execution', () => { ) largeArchive = captured.archive return { - stopped: true, taskOutcome: { + kind: 'workspace', resultTree, afterState: captured.snapshot.material, archive: captured.archive, @@ -325,7 +405,7 @@ describe('atomic prepared candidate execution', () => { termination: { kind: 'exit', exitCode: 0 }, } }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }, traceStore, ), @@ -334,11 +414,12 @@ describe('atomic prepared candidate execution', () => { expect(result.receipt.value).toMatchObject({ modelSettlement: { material: { - schemaVersion: 2, usage: { modelCalls: 1, inputTokens: 10, outputTokens: 5, + cachedInputTokens: 2, + reasoningTokens: 1, costUsdNanos: 10_000_000, }, calls: [ @@ -398,7 +479,8 @@ describe('atomic prepared candidate execution', () => { executions++ throw new Error('must not execute') }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), + capture: async () => ({}), }), ) expect(result).toMatchObject({ @@ -431,7 +513,7 @@ describe('atomic prepared candidate execution', () => { const claimStore = { tryClaim: async (claim: Parameters[0]) => { const result = await baseStore.tryClaim(claim) - now += fixture.task.limits.timeoutMs + 1 + now += fixture.task.task.limits.timeoutMs + 1 return result }, getAttempt: (attempt: Parameters[0]) => @@ -458,7 +540,7 @@ describe('atomic prepared candidate execution', () => { execute: async () => { throw new Error('must not execute') }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }, new InMemoryTraceStore(), claimStore, @@ -476,7 +558,9 @@ describe('atomic prepared candidate execution', () => { let now = 1_000_000 vi.spyOn(Date, 'now').mockImplementation(() => now) const fixture = createCandidateExecutionFixture() - fixture.task.limits = { ...fixture.task.limits, timeoutMs: 100 } + replaceCandidateFixtureTask(fixture, { + limits: { ...fixture.task.task.limits, timeoutMs: 100 }, + }) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), fixture.task, @@ -490,7 +574,7 @@ describe('atomic prepared candidate execution', () => { baseStore.getAttempt(attempt), markCandidateMayRun: async (lease: Parameters[0]) => { const result = await baseStore.markCandidateMayRun(lease) - now += fixture.task.limits.timeoutMs + now += fixture.task.task.limits.timeoutMs return result }, stageTerminal: ( @@ -515,7 +599,7 @@ describe('atomic prepared candidate execution', () => { executions++ throw new Error('must not execute') }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }, new InMemoryTraceStore(), claimStore, @@ -538,7 +622,9 @@ describe('atomic prepared candidate execution', () => { vi.spyOn(Date, 'now').mockImplementation(() => now) const cleanupTimeoutMs = 100 const fixture = createCandidateExecutionFixture() - fixture.task.limits = { ...fixture.task.limits, timeoutMs: 1_000 } + replaceCandidateFixtureTask(fixture, { + limits: { ...fixture.task.task.limits, timeoutMs: 1_000 }, + }) fixture.ports.models.settleGrant = async ({ preparationId }) => { now += cleanupTimeoutMs - 1 return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } @@ -595,10 +681,11 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => { + stop: async () => { now += cleanupTimeoutMs - 1 - return { stopped: true, taskOutcome: unchangedTaskOutcomeCapture(fixture) } + return { stopped: true } }, + capture: async () => ({ taskOutcome: unchangedTaskOutcomeCapture(fixture) }), }, grader: { ...outputs.grader, @@ -621,7 +708,9 @@ describe('atomic prepared candidate execution', () => { it('allows executable grading to exceed cleanup time inside its frozen result budget', async () => { const fixture = createCandidateExecutionFixture() - fixture.task.limits = { ...fixture.task.limits, timeoutMs: 1_000 } + replaceCandidateFixtureTask(fixture, { + limits: { ...fixture.task.task.limits, timeoutMs: 1_000 }, + }) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), fixture.task, @@ -642,8 +731,8 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => ({ - stopped: true, + stop: async () => ({ stopped: true }), + capture: async () => ({ taskOutcome: unchangedTaskOutcomeCapture(fixture), }), }, @@ -669,7 +758,9 @@ describe('atomic prepared candidate execution', () => { it('cancels over-budget grading without any output appearing after terminal failure', async () => { const fixture = createCandidateExecutionFixture() - fixture.task.limits = { ...fixture.task.limits, timeoutMs: 1_000 } + replaceCandidateFixtureTask(fixture, { + limits: { ...fixture.task.task.limits, timeoutMs: 1_000 }, + }) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), fixture.task, @@ -699,8 +790,8 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => ({ - stopped: true, + stop: async () => ({ stopped: true }), + capture: async () => ({ taskOutcome: unchangedTaskOutcomeCapture(fixture), }), }, @@ -762,7 +853,8 @@ describe('atomic prepared candidate execution', () => { execute: async () => { throw new Error('must not execute') }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), + capture: async () => ({}), }), ) expect(result).toMatchObject({ @@ -804,7 +896,7 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }, traceStore, ), @@ -826,7 +918,8 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), + capture: async () => ({}), }, traceStore, claimStore: new InMemoryAgentCandidateExecutionClaimStore(), @@ -835,7 +928,7 @@ describe('atomic prepared candidate execution', () => { expect(result).toMatchObject({ succeeded: false, reason: expect.stringMatching(/without a captured task outcome/), - usage: { modelCalls: 0, costUsd: 0 }, + usage: { modelCalls: 0, costUsdNanos: 0 }, }) }) @@ -855,9 +948,9 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => + stop: async () => ({ stopped: true }), + capture: async () => ({ - stopped: true, taskOutcome: unchangedTaskOutcomeCapture(fixture), score: 1, }) as never, @@ -876,7 +969,7 @@ describe('atomic prepared candidate execution', () => { expect(result).toMatchObject({ succeeded: false, reason: expect.stringMatching(/unknown field score/), - usage: { modelCalls: 0, costUsd: 0 }, + usage: { modelCalls: 0, costUsdNanos: 0 }, }) expect(graderCalls).toBe(0) }) @@ -901,8 +994,8 @@ describe('atomic prepared candidate execution', () => { score: 1, } as never }, - stopAndCapture: async () => ({ - stopped: true, + stop: async () => ({ stopped: true }), + capture: async () => ({ taskOutcome: unchangedTaskOutcomeCapture(fixture), }), }, @@ -920,7 +1013,7 @@ describe('atomic prepared candidate execution', () => { expect(result).toMatchObject({ succeeded: false, reason: expect.stringMatching(/unknown field score/), - usage: { modelCalls: 0, costUsd: 0 }, + usage: { modelCalls: 0, costUsdNanos: 0 }, }) expect(graderCalls).toBe(0) }) @@ -943,7 +1036,7 @@ describe('atomic prepared candidate execution', () => { const didStart = new Promise((resolve) => { started = resolve }) - const executor: AgentCandidateExecutorPort = { + const executor: TestExecutor = { execute: async (request) => { executions++ started() @@ -951,7 +1044,7 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), } const first = executePreparedAgentCandidate(prepared, options(executor, traceStore, claimStore)) await didStart @@ -1027,14 +1120,14 @@ describe('atomic prepared candidate execution', () => { const didStart = new Promise((resolve) => { started = resolve }) - const executor: AgentCandidateExecutorPort = { + const executor: TestExecutor = { execute: async (request) => { started() await wait await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), } const first = executePreparedAgentCandidate( @@ -1101,13 +1194,13 @@ describe('atomic prepared candidate execution', () => { prepared, options({ execute: async () => Promise.reject(new Error(`container failed with ${secret}`)), - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }), ) expect(result).toMatchObject({ succeeded: false, reason: expect.not.stringContaining(secret), - usage: { modelCalls: 1, inputTokens: 10, outputTokens: 5, costUsd: 0.01 }, + usage: { modelCalls: 1, inputTokens: 10, outputTokens: 5, costUsdNanos: 10_000_000 }, }) expect(result.reason).toContain('[redacted:candidate-access]') expect({ closed, settlementReads }).toEqual({ closed: true, settlementReads: 1 }) @@ -1116,7 +1209,9 @@ describe('atomic prepared candidate execution', () => { it('owns the deadline, aborts, waits for process death, and then settles', async () => { const fixture = createCandidateExecutionFixture() - fixture.task.limits = { ...fixture.task.limits, timeoutMs: 15 } + replaceCandidateFixtureTask(fixture, { + limits: { ...fixture.task.task.limits, timeoutMs: 15 }, + }) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), fixture.task, @@ -1151,10 +1246,11 @@ describe('atomic prepared candidate execution', () => { }) }) }, - stopAndCapture: async (_request, context) => { + stop: async (_request, context) => { expect(context.reason).toBe('timeout') - expect(context.signal).toBe(observedSignal) - expect(context.deadlineAtMs).toBe(startedAt + 15) + expect(context.signal).not.toBe(observedSignal) + expect(context.signal.aborted).toBe(false) + expect(context.deadlineAtMs).toBe(startedAt + 15 + 30_000) stoppedAfterAbort = observedSignal?.aborted === true if (!timeoutRequest) throw new Error('timeout executor never received its request') await terminalTrace(timeoutRequest, traceStore, 115) @@ -1180,7 +1276,9 @@ describe('atomic prepared candidate execution', () => { it('cannot turn a stop acknowledgement after the frozen deadline into an exit success', async () => { const fixture = createCandidateExecutionFixture() - fixture.task.limits = { ...fixture.task.limits, timeoutMs: 15 } + replaceCandidateFixtureTask(fixture, { + limits: { ...fixture.task.task.limits, timeoutMs: 15 }, + }) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), fixture.task, @@ -1199,7 +1297,7 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore, 105) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => { + stop: async () => { markStopStarted() await new Promise((resolve) => setTimeout(resolve, 30)) return { stopped: true } @@ -1235,10 +1333,9 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => await new Promise(() => undefined), + stop: async () => await new Promise(() => undefined), }, traceStore, - claimStore, ), cleanupTimeoutMs: 20, }) @@ -1253,6 +1350,7 @@ describe('atomic prepared candidate execution', () => { it('withholds a receipt but still revokes model access when process death is unproven', async () => { const fixture = createCandidateExecutionFixture() let settlements = 0 + let disposals = 0 fixture.ports.models.settleGrant = async ({ preparationId }) => { settlements++ return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } @@ -1271,9 +1369,13 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => { + stop: async () => { throw new Error('container death not observed') }, + dispose: async () => { + disposals++ + return { disposed: true } + }, }, traceStore, ), @@ -1283,17 +1385,17 @@ describe('atomic prepared candidate execution', () => { reason: expect.stringMatching(/death not observed/), usage: { modelCalls: 0 }, }) - expect(settlements).toBe(1) + expect({ settlements, disposals }).toEqual({ settlements: 1, disposals: 1 }) }) it('leaves unknown settlement unfinished so a retry cannot guess zero spend', async () => { const claimStore = new InMemoryAgentCandidateExecutionClaimStore() const first = createCandidateExecutionFixture() - first.task.attempt = { + replaceCandidateFixtureAttempt(first, { number: 1, maxAttempts: 2, retryPolicy: 'pre-model-infrastructure-only', - } + }) first.ports.models.settleGrant = async () => { throw new Error('gateway settlement unavailable') } @@ -1309,7 +1411,7 @@ describe('atomic prepared candidate execution', () => { execute: async () => { throw new Error('infrastructure failed') }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }, new InMemoryTraceStore(), claimStore, @@ -1319,11 +1421,11 @@ describe('atomic prepared candidate execution', () => { rmSync(first.task.stagingRoots.profileRoot, { recursive: true }) mkdirSync(first.task.stagingRoots.profileRoot) - first.task.attempt = { + replaceCandidateFixtureAttempt(first, { number: 2, maxAttempts: 2, retryPolicy: 'pre-model-infrastructure-only', - } + }) const retryPrepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(first.bundle, first.ports), first.task, @@ -1338,7 +1440,7 @@ describe('atomic prepared candidate execution', () => { retryExecutions++ throw new Error('retry must not execute') }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }, new InMemoryTraceStore(), claimStore, @@ -1353,11 +1455,11 @@ describe('atomic prepared candidate execution', () => { it('admits a zero-call retry only for an explicit pre-model infrastructure failure', async () => { const fixture = createCandidateExecutionFixture() - fixture.task.attempt = { + replaceCandidateFixtureAttempt(fixture, { number: 1, maxAttempts: 2, retryPolicy: 'pre-model-infrastructure-only', - } + }) const claimStore = new InMemoryAgentCandidateExecutionClaimStore() let activations = 0 fixture.ports.models.activateGrant = async () => { @@ -1378,7 +1480,7 @@ describe('atomic prepared candidate execution', () => { execute: async () => { throw new Error('executor must not run before model activation') }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }, firstTraceStore, claimStore, @@ -1388,11 +1490,11 @@ describe('atomic prepared candidate execution', () => { rmSync(fixture.task.stagingRoots.profileRoot, { recursive: true }) mkdirSync(fixture.task.stagingRoots.profileRoot) - fixture.task.attempt = { + replaceCandidateFixtureAttempt(fixture, { number: 2, maxAttempts: 2, retryPolicy: 'pre-model-infrastructure-only', - } + }) const retryPrepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), fixture.task, @@ -1407,7 +1509,7 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }, traceStore, claimStore, @@ -1418,9 +1520,12 @@ describe('atomic prepared candidate execution', () => { it('closes isolated memory and persists a large after-state archive by reference', async () => { const fixture = createCandidateExecutionFixture() - fixture.bundle = redigestCandidateBundle(fixture.bundle, { - memory: { mode: 'isolated', scope: 'task' }, - }) + bindCandidateFixtureBundle( + fixture, + redigestCandidateBundle(fixture.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }), + ) const before = emptyCandidateSnapshot('before') const after = emptyCandidateSnapshot('after') const order: string[] = [] @@ -1451,32 +1556,36 @@ describe('atomic prepared candidate execution', () => { if (prepared.memory.mode !== 'isolated') throw new Error('isolated memory was not prepared') const traceStore = new InMemoryTraceStore() const largeMemoryArchive = Buffer.alloc(4_000_001, 0x5a) - const executionOptions = options( - { - execute: async (request) => { - expect(request.launch.env.TANGLE_MEMORY_NAMESPACE).toBe( - prepared.memory.effectiveNamespace, - ) - await terminalTrace(request, traceStore) - return { - executionId: request.executionId, - termination: { kind: 'exit', exitCode: 0 }, - } - }, - stopAndCapture: async () => { - order.push('process-stop') - return { - stopped: true, - memoryAfter: { - afterState: after.material, - archive: largeMemoryArchive, - }, - } + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async (request) => { + expect(request.launch.env.TANGLE_MEMORY_NAMESPACE).toBe( + prepared.memory.effectiveNamespace, + ) + await terminalTrace(request, traceStore) + return { + executionId: request.executionId, + termination: { kind: 'exit', exitCode: 0 }, + } + }, + stop: async () => { + order.push('process-stop') + return { stopped: true } + }, + capture: async () => { + return { + memoryAfter: { + afterState: after.material, + archive: largeMemoryArchive, + }, + } + }, }, - }, - traceStore, + traceStore, + ), ) - const result = await executePreparedAgentCandidate(prepared, executionOptions) expect(result.succeeded).toBe(true) expect(order).toEqual(['process-stop', 'memory-close', 'model-settle']) if (!result.succeeded || result.receipt.value.memory.mode !== 'isolated') return @@ -1491,9 +1600,12 @@ describe('atomic prepared candidate execution', () => { it('rejects a compressed protected value in isolated memory before persisting memory evidence', async () => { const fixture = createCandidateExecutionFixture() - fixture.bundle = redigestCandidateBundle(fixture.bundle, { - memory: { mode: 'isolated', scope: 'task' }, - }) + bindCandidateFixtureBundle( + fixture, + redigestCandidateBundle(fixture.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }), + ) const before = emptyCandidateSnapshot('secret-memory-before') fixture.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ preparationId, @@ -1516,7 +1628,6 @@ describe('atomic prepared candidate execution', () => { } const memoryBytes = Buffer.from(secret, 'utf8') const afterState = { - schemaVersion: 2 as const, kind: 'agent-candidate-workspace-manifest' as const, files: [ { @@ -1541,8 +1652,8 @@ describe('atomic prepared candidate execution', () => { await terminalTrace(request, traceStore) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => ({ - stopped: true, + stop: async () => ({ stopped: true }), + capture: async () => ({ taskOutcome: unchangedTaskOutcomeCapture(fixture), memoryAfter: { afterState, archive: gzipSync(memoryBytes) }, }), @@ -1615,7 +1726,7 @@ describe('atomic prepared candidate execution', () => { }) return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }, traceStore, ) @@ -1639,9 +1750,10 @@ describe('atomic prepared candidate execution', () => { it('rejects protected environment collisions before executor access', async () => { const fixture = createCandidateExecutionFixture() - fixture.bundle = candidateBundle({ - env: { PUBLIC_MODE: { kind: 'public', value: 'fixture' } }, - }) + bindCandidateFixtureBundle( + fixture, + candidateBundle({ env: { PUBLIC_MODE: { kind: 'public', value: 'fixture' } } }), + ) fixture.ports.models.activateGrant = async () => ({ env: { PUBLIC_MODE: 'secret-value' } }) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), @@ -1656,7 +1768,7 @@ describe('atomic prepared candidate execution', () => { executions++ throw new Error('must not execute') }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), }), ) expect(result).toMatchObject({ succeeded: false, reason: expect.stringMatching(/collides/) }) diff --git a/tests/candidate-execution-executor-capture.test.ts b/tests/candidate-execution-executor-capture.test.ts new file mode 100644 index 00000000..2303dc98 --- /dev/null +++ b/tests/candidate-execution-executor-capture.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' + +import { + sealAgentCandidateExecutorFinalCapture, + sealAgentCandidateExecutorStopAcknowledgement, +} from '../src/candidate-execution/executor-capture' + +describe('candidate executor capture', () => { + it('checks the signed output limit before detaching exact bytes', () => { + expect(() => + sealAgentCandidateExecutorFinalCapture( + { + taskOutcome: { kind: 'output', bytes: new Uint8Array(5) }, + }, + { kind: 'output', mediaType: 'application/octet-stream', maxBytes: 4 }, + ), + ).toThrow(/4-byte maximum/) + + const source = Uint8Array.from([1, 2, 3]) + const capture = sealAgentCandidateExecutorFinalCapture( + { taskOutcome: { kind: 'output', bytes: source } }, + { kind: 'output', mediaType: 'application/octet-stream', maxBytes: 3 }, + ) + source[0] = 9 + expect(capture.taskOutcome).toMatchObject({ kind: 'output', bytes: Uint8Array.from([1, 2, 3]) }) + }) + + it('rejects a capture that disagrees with the signed result kind', () => { + expect(() => + sealAgentCandidateExecutorFinalCapture( + { taskOutcome: { kind: 'output', bytes: Uint8Array.from([1]) } }, + { kind: 'workspace' }, + ), + ).toThrow(/signed outcome kind/) + }) + + it('seals stop independently from replayable capture evidence', () => { + expect(() => sealAgentCandidateExecutorStopAcknowledgement({ stopped: true })).not.toThrow() + expect(() => + sealAgentCandidateExecutorStopAcknowledgement({ stopped: true, taskOutcome: {} }), + ).toThrow(/unknown field taskOutcome/) + }) +}) diff --git a/tests/candidate-execution-finalize.test.ts b/tests/candidate-execution-finalize.test.ts index 29b3bf8a..afbf560a 100644 --- a/tests/candidate-execution-finalize.test.ts +++ b/tests/candidate-execution-finalize.test.ts @@ -1,6 +1,6 @@ import { InMemoryTraceStore } from '@tangle-network/agent-eval' import { afterEach, describe, expect, it } from 'vitest' - +import { sealAgentCandidateExecutorFinalCapture } from '../src/candidate-execution/executor-capture' import { finalizeAgentCandidateRun } from '../src/candidate-execution/finalize' import { type SealedAgentCandidateModelSettlement, @@ -9,7 +9,7 @@ import { import { persistCandidateBenchmarkResult, persistCandidateModelSettlement, - persistVerifiedCandidateTaskOutcome, + persistVerifiedAgentCandidateExecutorCapture, } from '../src/candidate-execution/outcome-evidence' import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' import { assertPreparedCandidateIntegrity } from '../src/candidate-execution/prepared-state' @@ -23,6 +23,7 @@ import { cleanupCandidateFixtures, createCandidateExecutionFixture, createCandidateOutputFixture, + replaceCandidateFixtureTask, } from './helpers/candidate-execution-fixture' afterEach(cleanupCandidateFixtures) @@ -45,7 +46,7 @@ async function prepared( }, ): Promise { const fixture = createCandidateExecutionFixture() - fixture.task.limits = limits + replaceCandidateFixtureTask(fixture, { limits }) return await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), fixture.task, @@ -94,24 +95,29 @@ async function finalizePrepared( }, ) const outputs = overrides.outputs ?? createCandidateOutputFixture() - const finalCapture = { - stopped: true as const, - taskOutcome: { - resultTree: state.executionPlan.value.material.task.repository.baseTree, - afterState: state.executionPlan.value.material.task.workspace.material, - archive: Buffer.from('fixture unchanged task archive', 'utf8'), - gitDiff: Buffer.alloc(0), - }, + const expectedOutcome = state.benchmarkTask.outcome + if (expectedOutcome.kind !== 'workspace') { + throw new Error('finalization fixture requires a workspace outcome') } - const [modelSettlement, taskOutcome] = await Promise.all([ + const repository = state.benchmarkTask.repository + if (!repository) throw new Error('finalization fixture requires repository identity') + const finalCapture = sealAgentCandidateExecutorFinalCapture( + { + taskOutcome: { + kind: 'workspace' as const, + resultTree: repository.baseTree, + afterState: state.benchmarkTask.workspace.material, + archive: Buffer.from('fixture unchanged task archive', 'utf8'), + gitDiff: Buffer.alloc(0), + }, + }, + expectedOutcome, + ) + const [modelSettlement, verifiedCapture] = await Promise.all([ persistCandidateModelSettlement(state, settlement, outputs.outputArtifacts), - persistVerifiedCandidateTaskOutcome( - state, - finalCapture.taskOutcome, - outputs.outputArtifacts, - [], - ), + persistVerifiedAgentCandidateExecutorCapture(state, finalCapture, outputs.outputArtifacts, []), ]) + const { persistedCapture, taskOutcome } = verifiedCapture const benchmarkResult = await persistCandidateBenchmarkResult( state, capture.termination, @@ -127,6 +133,7 @@ async function finalizePrepared( settlement, { finalCapture, + persistedCapture, modelSettlement, taskOutcome, benchmarkResult, @@ -194,7 +201,7 @@ async function traceStore( } describe('protected candidate run finalization', () => { - it('derives exact V3 evidence and durable trace bytes from tied agent-eval spans', async () => { + it('derives exact evidence and durable trace bytes from tied agent-eval spans', async () => { const execution = await prepared() const store = await traceStore(execution) const outputs = createCandidateOutputFixture() @@ -208,11 +215,10 @@ describe('protected candidate run finalization', () => { if (!result.succeeded) return expect(result.receipt.value.trace).toMatchObject({ eventCount: 3, modelCallCount: 1 }) expect(result.receipt.bytes.byteLength).toBeGreaterThan(0) - const task = assertPreparedCandidateIntegrity(execution).executionPlan.value.material.task - if (!task.repository) throw new Error('workspace task repository is missing') + const task = assertPreparedCandidateIntegrity(execution).benchmarkTask + if (task.outcome.kind !== 'workspace') throw new Error('expected workspace task outcome') + if (!task.repository) throw new Error('expected task repository identity') expect(result.receipt.value).toMatchObject({ - schemaVersion: 3, - executorCapture: result.artifacts.executorCapture, taskOutcome: { artifact: result.artifacts.taskOutcome, material: { @@ -252,6 +258,7 @@ describe('protected candidate run finalization', () => { inputTokens: 10, outputTokens: 5, cachedInputTokens: 2, + reasoningTokens: 0, modelCalls: 1, }, }, @@ -263,9 +270,13 @@ describe('protected candidate run finalization', () => { expect(outputs.outputArtifacts.read(result.artifacts.runReceipt)).resolves.toEqual( result.receipt.bytes, ), - expect(outputs.outputArtifacts.read(taskOutcome.gitDiff.artifact)).resolves.toEqual( - new Uint8Array(), - ), + expect( + outputs.outputArtifacts.read( + result.receipt.value.taskOutcome.material.outcome.kind === 'workspace' + ? result.receipt.value.taskOutcome.material.outcome.gitDiff.artifact + : result.receipt.value.taskOutcome.material.outcome.artifact, + ), + ).resolves.toEqual(new Uint8Array()), expect( outputs.outputArtifacts.read(result.receipt.value.benchmarkResult.material.evidence), ).resolves.toEqual(Uint8Array.from(Buffer.from('{"reward":1}\n', 'utf8'))), @@ -279,15 +290,11 @@ describe('protected candidate run finalization', () => { ), ) expect(executorCapture).toMatchObject({ - schemaVersion: 1, kind: 'agent-candidate-executor-capture', - executionId: execution.executionId, executionPlanDigest: execution.executionPlan.value.digest, - termination: { kind: 'exit', exitCode: 0 }, - stopped: true, taskOutcome: { - digest: result.receipt.value.taskOutcome.digest, - artifact: result.receipt.value.taskOutcome.artifact, + kind: 'workspace', + gitDiff: taskOutcome.gitDiff.artifact, }, }) expect(JSON.stringify(executorCapture)).not.toContain('content') diff --git a/tests/candidate-execution-model-port.test.ts b/tests/candidate-execution-model-port.test.ts index e2464ff5..5f590f8b 100644 --- a/tests/candidate-execution-model-port.test.ts +++ b/tests/candidate-execution-model-port.test.ts @@ -1,5 +1,9 @@ import { InMemoryTraceStore, type LlmSpan } from '@tangle-network/agent-eval' -import type { AgentCandidateResolvedModel, Sha256Digest } from '@tangle-network/agent-interface' +import type { + AgentCandidateModelSettlementCall, + AgentCandidateResolvedModel, + Sha256Digest, +} from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { appendAuthoritativeModelSettlementSpans, @@ -17,7 +21,6 @@ import { import type { AgentCandidateModelPort, AgentCandidateProtectedModelActivation, - AgentCandidateProtectedModelCall, AgentCandidateProtectedModelSettlement, } from '../src/candidate-execution/types' @@ -95,8 +98,8 @@ function settleInput( function modelCall( index: number, - overrides: Partial = {}, -): AgentCandidateProtectedModelCall { + overrides: Partial = {}, +): AgentCandidateModelSettlementCall { return { callId: `call-${index}`, generationId: `generation-${index}`, @@ -115,7 +118,7 @@ function modelCall( } function settlement( - calls: readonly AgentCandidateProtectedModelCall[] = [], + calls: readonly AgentCandidateModelSettlementCall[] = [], ): AgentCandidateProtectedModelSettlement { return { preparationId: 'preparation-1', diff --git a/tests/candidate-execution-outcome-evidence.test.ts b/tests/candidate-execution-outcome-evidence.test.ts index 05bdade4..0abd84de 100644 --- a/tests/candidate-execution-outcome-evidence.test.ts +++ b/tests/candidate-execution-outcome-evidence.test.ts @@ -4,9 +4,10 @@ import { gunzipSync, gzipSync } from 'node:zlib' import { afterEach, describe, expect, it } from 'vitest' import { sha256Bytes } from '../src/candidate-execution/digest' +import { sealAgentCandidateExecutorFinalCapture } from '../src/candidate-execution/executor-capture' import { persistCandidateBenchmarkResult, - persistVerifiedCandidateTaskOutcome, + persistVerifiedAgentCandidateExecutorCapture, } from '../src/candidate-execution/outcome-evidence' import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' import { assertPreparedCandidateIntegrity } from '../src/candidate-execution/prepared-state' @@ -14,6 +15,7 @@ import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' import { cleanupCandidateFixtures, createCandidateExecutionFixture, + createCandidateOutputExecutionFixture, createCandidateOutputFixture, unchangedTaskOutcomeCapture, } from './helpers/candidate-execution-fixture' @@ -21,6 +23,71 @@ import { afterEach(cleanupCandidateFixtures) describe('candidate outcome evidence', () => { + it('persists exact bounded media output and rejects invalid captures', async () => { + const fixture = createCandidateOutputExecutionFixture('application/json', 32) + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const source = Buffer.from('{"answer":42}\n', 'utf8') + const outcome = await persistTaskOutcome( + state, + { kind: 'output', bytes: source }, + outputs.outputArtifacts, + [], + ) + + source.fill(0) + expect(outcome).toMatchObject({ + kind: 'output', + spec: { mediaType: 'application/json', maxBytes: 32 }, + }) + if (outcome.kind !== 'output' || outcome.evidence.material.outcome.kind !== 'output') { + throw new Error('expected exact output evidence') + } + expect(Buffer.from(outcome.bytes).toString('utf8')).toBe('{"answer":42}\n') + await expect( + outputs.outputArtifacts.read(outcome.evidence.material.outcome.artifact), + ).resolves.toEqual(Uint8Array.from(Buffer.from('{"answer":42}\n', 'utf8'))) + + await expect( + persistTaskOutcome( + state, + { kind: 'output', bytes: new Uint8Array() }, + outputs.outputArtifacts, + [], + ), + ).rejects.toThrow(/cannot be empty/) + await expect( + persistTaskOutcome( + state, + { kind: 'output', bytes: Buffer.alloc(33) }, + outputs.outputArtifacts, + [], + ), + ).rejects.toThrow(/32-byte maximum/) + await expect( + persistTaskOutcome( + state, + { kind: 'output', bytes: Buffer.from('sk-output-secret-123456789') }, + outputs.outputArtifacts, + ['sk-output-secret-123456789'], + ), + ).rejects.toThrow(/protected value/) + await expect( + persistTaskOutcome( + state, + { + kind: 'workspace', + resultTree: '0'.repeat(40), + afterState: fixture.task.task.workspace.material, + archive: Buffer.from('archive'), + gitDiff: new Uint8Array(), + }, + outputs.outputArtifacts, + [], + ), + ).rejects.toThrow(/does not match the signed outcome kind/) + }) + it('rejects protected values in the canonical task manifest before any output write', async () => { const fixture = createCandidateExecutionFixture() const state = await preparedState(fixture) @@ -35,14 +102,61 @@ describe('candidate outcome evidence', () => { } await expect( - persistVerifiedCandidateTaskOutcome( + persistTaskOutcome(state, unchangedTaskOutcomeCapture(fixture), outputArtifacts, [ + 'source.ts', + ]), + ).rejects.toThrow(/protected value/) + expect(puts).toBe(0) + }) + + it('screens native executor evidence and stores it separately from the capture summary', async () => { + const fixture = createCandidateOutputExecutionFixture('text/plain', 32) + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const purposes: string[] = [] + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + purposes.push(input.purpose) + return await outputs.outputArtifacts.put(input) + }, + } + const capture = sealAgentCandidateExecutorFinalCapture( + { + taskOutcome: { kind: 'output', bytes: Buffer.from('answer', 'utf8') }, + evidence: Buffer.from('native evidence', 'utf8'), + }, + state.benchmarkTask.outcome, + ) + + await persistVerifiedAgentCandidateExecutorCapture(state, capture, outputArtifacts, []) + expect(purposes).toContain('executor-native-evidence') + expect(purposes).toContain('executor-capture') + + let protectedPuts = 0 + const secret = 'sk-native-evidence-secret-123456789' + const protectedCapture = sealAgentCandidateExecutorFinalCapture( + { + taskOutcome: { kind: 'output', bytes: Buffer.from('answer', 'utf8') }, + evidence: Buffer.from(secret, 'utf8'), + }, + state.benchmarkTask.outcome, + ) + await expect( + persistVerifiedAgentCandidateExecutorCapture( state, - unchangedTaskOutcomeCapture(fixture), - outputArtifacts, - ['source.ts'], + protectedCapture, + { + read: outputs.outputArtifacts.read, + put: async (input) => { + protectedPuts++ + return await outputs.outputArtifacts.put(input) + }, + }, + [secret], ), ).rejects.toThrow(/protected value/) - expect(puts).toBe(0) + expect(protectedPuts).toBe(0) }) it('validates task manifest material before any output write', async () => { @@ -62,7 +176,7 @@ describe('candidate outcome evidence', () => { if (!file) throw new Error('fixture task manifest is empty') await expect( - persistVerifiedCandidateTaskOutcome( + persistTaskOutcome( state, { ...capture, @@ -97,7 +211,7 @@ describe('candidate outcome evidence', () => { const secret = 'sk-compressed-before-put-123456789' await expect( - persistVerifiedCandidateTaskOutcome( + persistTaskOutcome( state, { ...unchangedTaskOutcomeCapture(fixture), @@ -133,12 +247,7 @@ describe('candidate outcome evidence', () => { } await expect( - persistVerifiedCandidateTaskOutcome( - state, - unchangedTaskOutcomeCapture(fixture), - outputArtifacts, - [], - ), + persistTaskOutcome(state, unchangedTaskOutcomeCapture(fixture), outputArtifacts, []), ).rejects.toThrow(/persisted candidate output digest/) expect(purposes.sort()).toEqual(['task-archive', 'task-manifest']) expect(purposes).not.toContain('task-patch') @@ -160,12 +269,7 @@ describe('candidate outcome evidence', () => { } await expect( - persistVerifiedCandidateTaskOutcome( - state, - unchangedTaskOutcomeCapture(fixture), - outputArtifacts, - [], - ), + persistTaskOutcome(state, unchangedTaskOutcomeCapture(fixture), outputArtifacts, []), ).rejects.toThrow(/does not identify the submitted bytes/) expect(purposes.sort()).toEqual(['task-archive', 'task-manifest']) expect(purposes).not.toContain('task-patch') @@ -179,7 +283,7 @@ describe('candidate outcome evidence', () => { const secret = 'sk-outcome-secret-123456789' const capture = unchangedTaskOutcomeCapture(fixture) await expect( - persistVerifiedCandidateTaskOutcome( + persistTaskOutcome( state, { ...capture, archive: Buffer.from(secret, 'utf8') }, outputs.outputArtifacts, @@ -187,12 +291,7 @@ describe('candidate outcome evidence', () => { ), ).rejects.toThrow(/protected value/) - const outcome = await persistVerifiedCandidateTaskOutcome( - state, - capture, - outputs.outputArtifacts, - [secret], - ) + const outcome = await persistTaskOutcome(state, capture, outputs.outputArtifacts, [secret]) await expect( persistCandidateBenchmarkResult( state, @@ -223,7 +322,7 @@ describe('candidate outcome evidence', () => { const fixture = createCandidateExecutionFixture() const state = await preparedState(fixture) const outputs = createCandidateOutputFixture() - const outcome = await persistVerifiedCandidateTaskOutcome( + const outcome = await persistTaskOutcome( state, unchangedTaskOutcomeCapture(fixture), outputs.outputArtifacts, @@ -253,7 +352,7 @@ describe('candidate outcome evidence', () => { const fixture = createCandidateExecutionFixture() const state = await preparedState(fixture) const outputs = createCandidateOutputFixture() - const outcome = await persistVerifiedCandidateTaskOutcome( + const outcome = await persistTaskOutcome( state, unchangedTaskOutcomeCapture(fixture), outputs.outputArtifacts, @@ -303,7 +402,7 @@ describe('candidate outcome evidence', () => { const fixture = createCandidateExecutionFixture() const state = await preparedState(fixture) const outputs = createCandidateOutputFixture() - const outcome = await persistVerifiedCandidateTaskOutcome( + const outcome = await persistTaskOutcome( state, unchangedTaskOutcomeCapture(fixture), outputs.outputArtifacts, @@ -364,3 +463,23 @@ async function preparedState(fixture: ReturnType[0], + taskOutcome: import('../src/candidate-execution/types').AgentCandidateExecutorTaskOutcomeCapture, + outputArtifacts: Parameters[2], + protectedValues: readonly string[], +) { + const capture = sealAgentCandidateExecutorFinalCapture( + { taskOutcome }, + state.benchmarkTask.outcome, + ) + return ( + await persistVerifiedAgentCandidateExecutorCapture( + state, + capture, + outputArtifacts, + protectedValues, + ) + ).taskOutcome +} diff --git a/tests/candidate-execution-prepare.test.ts b/tests/candidate-execution-prepare.test.ts index 4c54c0f2..7c1a1acd 100644 --- a/tests/candidate-execution-prepare.test.ts +++ b/tests/candidate-execution-prepare.test.ts @@ -1,313 +1,36 @@ -import { execFileSync } from 'node:child_process' -import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' +import { mkdirSync, writeFileSync } from 'node:fs' import { join } from 'node:path' -import type { - AgentCandidateBundle, - AgentCandidateExecution, - AgentCandidateWorkspaceSnapshotEvidence, - Sha256Digest, -} from '@tangle-network/agent-interface' +import type { AgentCandidateBundle } from '@tangle-network/agent-interface' import { afterEach, describe, expect, it } from 'vitest' import { MAX_CANDIDATE_TIMER_INTERVAL_MS } from '../src/candidate-execution/cleanup' import { - canonicalCandidateBytes, - canonicalCandidateDigest, + canonicalCandidateDocument, embeddedCandidateArtifact, } from '../src/candidate-execution/digest' +import { + CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV, + CANDIDATE_KNOWLEDGE_ROOT_ENV, +} from '../src/candidate-execution/knowledge' import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' -import type { - AgentCandidateExecutionPorts, - AgentCandidateTaskExecution, - ResolvedAgentCandidateContainer, -} from '../src/candidate-execution/types' +import { parseAgentCandidateProfileActivation } from '../src/candidate-execution/profile' +import type { AgentCandidateExecutionPorts } from '../src/candidate-execution/types' import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' - -const roots: string[] = [] -const sha = (character: string): Sha256Digest => `sha256:${character.repeat(64)}` +import { + bindCandidateFixtureBundle, + candidateBundle as bundle, + cleanupCandidateFixtures, + emptyCandidateSnapshot as emptySnapshot, + createCandidateExecutionFixture as fixture, + redigestCandidateBundle as redigestBundle, + replaceCandidateFixtureTask, + candidateSha as sha, +} from './helpers/candidate-execution-fixture' afterEach(() => { - for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) + cleanupCandidateFixtures() }) -function temporaryRoot(prefix: string): string { - const root = mkdtempSync(join(tmpdir(), prefix)) - roots.push(root) - return root -} - -function git(root: string, args: string[]): string { - return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim() -} - -function taskRepository(): { root: string; commit: string; tree: string } { - const root = temporaryRoot('candidate-task-') - git(root, ['init', '-b', 'main']) - git(root, ['config', 'user.email', 'test@example.com']) - git(root, ['config', 'user.name', 'Test']) - git(root, ['config', 'core.hooksPath', '/dev/null']) - git(root, ['remote', 'add', 'origin', 'git@github.com:owner/repo.git']) - writeFileSync(join(root, 'source.ts'), 'export const value = 1\n') - chmodSync(join(root, 'source.ts'), 0o644) - git(root, ['add', 'source.ts']) - git(root, ['commit', '-m', 'base']) - return { - root, - commit: git(root, ['rev-parse', 'HEAD']), - tree: git(root, ['rev-parse', 'HEAD^{tree}']), - } -} - -function snapshot( - root: string, - files: Array<{ path: string; mode: number }>, -): AgentCandidateWorkspaceSnapshotEvidence { - const material = { - schemaVersion: 2 as const, - kind: 'agent-candidate-workspace-manifest' as const, - files: files - .map((file) => { - const bytes = execFileSync('node', [ - '-e', - `process.stdout.write(require('fs').readFileSync(${JSON.stringify(join(root, file.path))}))`, - ]) - return { - path: file.path, - mode: file.mode, - sha256: embeddedCandidateArtifact(bytes).sha256, - byteLength: bytes.byteLength, - } - }) - .sort((a, b) => a.path.localeCompare(b.path)), - } - const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) - return { - schemaVersion: 2, - kind: 'agent-candidate-workspace-snapshot', - digest: manifest.sha256, - material, - manifest, - archive: embeddedCandidateArtifact(Buffer.from(`archive:${manifest.sha256}`)), - } -} - -function bundle( - execution: Partial = {}, - active?: { commit: string; tree: string; workspace: AgentCandidateWorkspaceSnapshotEvidence }, -): AgentCandidateBundle { - const value = { - schemaVersion: 2 as const, - kind: 'agent-candidate-bundle' as const, - digestAlgorithm: 'rfc8785-sha256' as const, - profile: { - name: 'candidate', - prompt: { instructions: ['Inspect the repository, implement the fix, and run tests.'] }, - model: { default: 'provider/model', reasoningEffort: 'high' as const }, - harness: 'codex' as const, - resources: { failOnError: true as const }, - }, - code: active - ? { - kind: 'no-op' as const, - reason: 'proposer-no-change' as const, - repository: { kind: 'github' as const, owner: 'owner', repo: 'repo' }, - baseCommit: active.commit, - baseTree: active.tree, - } - : { kind: 'disabled' as const, reason: 'control' as const }, - execution: { - harness: 'codex' as const, - harnessVersion: '1.2.3', - launch: active - ? { - kind: 'candidate-entrypoint' as const, - entrypoint: 'run.js', - interpreter: 'node' as const, - } - : { kind: 'container-command' as const, executable: 'codex' }, - instructionDelivery: { kind: 'stdin-utf8' as const }, - cwd: { workspace: 'task' as const, path: '.' }, - environment: { kind: 'evaluator-task-container' as const }, - ...(active ? { workspace: active.workspace } : {}), - isolation: { - network: 'disabled' as const, - remoteIntegrations: 'disabled' as const, - candidateSecrets: 'disabled' as const, - }, - ...execution, - }, - memory: { mode: 'disabled' as const }, - lineage: active - ? { - source: 'optimizer' as const, - parentDigests: [sha('e')], - runIds: ['optimizer-run-1'], - benchmark: { name: 'development', version: '1', splitDigest: sha('f') }, - spend: { - proposal: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, - evaluation: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, - }, - } - : { source: 'human' as const }, - } - return { ...value, digest: canonicalCandidateDigest(value) } -} - -function redigestBundle( - source: AgentCandidateBundle, - overrides: Partial>, -): AgentCandidateBundle { - const { digest: _digest, ...withoutDigest } = source - const value = { ...withoutDigest, ...overrides } - return { ...value, digest: canonicalCandidateDigest(value) } -} - -function emptySnapshot(label: string): AgentCandidateWorkspaceSnapshotEvidence { - const material = { - schemaVersion: 2 as const, - kind: 'agent-candidate-workspace-manifest' as const, - files: [], - } - const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) - return { - schemaVersion: 2, - kind: 'agent-candidate-workspace-snapshot', - digest: manifest.sha256, - material, - manifest, - archive: embeddedCandidateArtifact(Buffer.from(`empty:${label}`)), - } -} - -function fixture(active = false): { - bundle: AgentCandidateBundle - task: AgentCandidateTaskExecution - ports: AgentCandidateExecutionPorts - candidateRoot?: string -} { - const repository = taskRepository() - const taskWorkspace = snapshot(repository.root, [{ path: 'source.ts', mode: 0o644 }]) - let candidateRoot: string | undefined - let candidateWorkspace: AgentCandidateWorkspaceSnapshotEvidence | undefined - if (active) { - candidateRoot = temporaryRoot('candidate-built-') - writeFileSync(join(candidateRoot, 'run.js'), '#!/usr/bin/env node\n') - chmodSync(join(candidateRoot, 'run.js'), 0o755) - candidateWorkspace = snapshot(candidateRoot, [{ path: 'run.js', mode: 0o755 }]) - } - const profileRoot = temporaryRoot('candidate-profile-') - const selectedContainer: ResolvedAgentCandidateContainer = { - source: 'evaluator-task-container', - image: 'ghcr.io/example/task', - indexDigest: sha('a'), - manifestDigest: sha('b'), - platform: { os: 'linux', architecture: 'amd64' }, - } - const ports: AgentCandidateExecutionPorts = { - artifacts: { - read: async () => { - throw new Error('all fixture artifacts are embedded') - }, - }, - repositories: { resolve: async () => repository.root }, - workspaces: { materialize: async () => undefined }, - containers: { resolve: async () => selectedContainer }, - models: { - resolve: async ({ requested, reasoningEffort }) => ({ - requested, - provider: 'provider', - model: 'model-snapshot', - snapshot: 'model-snapshot-2026-07-01', - reasoningEffort, - }), - reserveGrant: async ({ preparationId, expiresAtMs, limits }) => ({ - preparationId, - digest: sha('c'), - expiresAtMs, - enforcedLimits: limits, - network: - limits.maxModelCalls === 0 - ? { mode: 'disabled' as const } - : { mode: 'gateway-only' as const, domains: ['router.tangle.tools'] }, - }), - activateGrant: async () => ({ env: { MODEL_GATEWAY_TOKEN: 'protected' } }), - settleGrant: async ({ preparationId }) => ({ - preparationId, - grantDigest: sha('c'), - closed: true, - calls: [], - }), - }, - memory: { - reset: async () => { - throw new Error('disabled memory must not reset') - }, - activate: async () => { - throw new Error('disabled memory must not activate') - }, - close: async () => { - throw new Error('disabled memory must not close') - }, - }, - } - const task: AgentCandidateTaskExecution = { - executionId: 'execution-1', - benchmark: 'repository-disjoint-smoke', - benchmarkVersion: '1', - taskId: 'owner-repo-1', - splitDigest: sha('d'), - instruction: 'Fix the failing behavior without changing the public API.', - repository: { - identity: 'github.com/owner/repo', - rootIdentity: 'owner/repo', - baseCommit: repository.commit, - baseTree: repository.tree, - }, - attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, - model: { requested: 'provider/model', reasoningEffort: 'high' }, - grader: { - name: 'fixture-executable-grader', - version: '1.0.0', - artifact: { - locator: { kind: 's3', bucket: 'candidate-test-artifacts', key: 'grader/fixture' }, - sha256: sha('a'), - byteLength: 1, - }, - }, - executionRoots: { - taskRoot: '/workspace/task', - ...(active ? { candidateRoot: '/opt/candidate' } : {}), - }, - stagingRoots: { - taskRoot: repository.root, - ...(candidateRoot ? { candidateRoot } : {}), - profileRoot, - }, - workspace: taskWorkspace, - evaluatorTaskContainer: selectedContainer, - limits: { - timeoutMs: 60_000, - maxSteps: 100, - maxModelCalls: 50, - maxInputTokens: 100_000, - maxOutputTokens: 50_000, - maxCostUsd: 5, - }, - } - return { - bundle: bundle( - {}, - active && candidateWorkspace - ? { commit: repository.commit, tree: repository.tree, workspace: candidateWorkspace } - : undefined, - ), - task, - ports, - ...(candidateRoot ? { candidateRoot } : {}), - } -} - describe('candidate execution preparation', () => { it('binds exact instruction, repository, profile, model, image, roots, and limits', async () => { const value = fixture() @@ -318,29 +41,28 @@ describe('candidate execution preparation', () => { path: '/tangle/input/task.txt', }, }) + bindCandidateFixtureBundle(value) const verified = await verifyAgentCandidateBundle(value.bundle, value.ports) const prepared = await prepareAgentCandidateExecution(verified, value.task, value.ports) const plan = prepared.executionPlan.value.material - expect(plan.task.instruction).toEqual({ - encoding: 'utf8', - sha256: embeddedCandidateArtifact(Buffer.from(value.task.instruction)).sha256, - byteLength: Buffer.byteLength(value.task.instruction), - delivery: value.bundle.execution.instructionDelivery, - }) - expect(plan.task.repository).toEqual(value.task.repository) - expect(plan.task.outcome).toEqual({ kind: 'workspace' }) + expect(plan.instructionDelivery).toEqual(value.bundle.execution.instructionDelivery) + expect(prepared.benchmark.task.outcome).toEqual(value.task.task.outcome) + expect(prepared.benchmark.task.repository).toEqual(value.task.task.repository) + expect(prepared.benchmark.task.outcome).toEqual({ kind: 'workspace' }) expect(plan.profile).toEqual({ planDigest: prepared.profilePlan.value.digest, targetWorkspace: 'task', mountPaths: ['AGENTS.md'], }) expect(plan.launch.env.TANGLE_CANDIDATE_TASK_PATH?.value).toBe('/tangle/input/task.txt') - expect(Buffer.from(prepared.instruction.bytes)).toEqual(Buffer.from(value.task.instruction)) + expect(Buffer.from(prepared.instruction.bytes)).toEqual( + Buffer.from(value.task.task.instruction), + ) expect(Buffer.from(prepared.executionPlan.bytes)).toEqual( Buffer.from(prepared.executionPlan.value.artifact.content, 'base64'), ) expect(prepared.materializationReceipt.bytes.byteLength).toBeGreaterThan(0) - expect(JSON.stringify(plan)).not.toContain(value.task.instruction) + expect(JSON.stringify(plan)).not.toContain(value.task.task.instruction) expect(JSON.stringify(prepared)).not.toContain('MODEL_GATEWAY_TOKEN') expect(JSON.stringify(prepared)).not.toContain('protected') expect(plan.model.access.network).toEqual({ @@ -349,15 +71,36 @@ describe('candidate execution preparation', () => { }) }) + it('rejects a self-rehashed profile activation whose native file differs from its plan', async () => { + const value = fixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ) + const { digest: _digest, ...activation } = prepared.profileActivation + const [first, ...rest] = activation.files + if (!first) throw new Error('fixture profile activation has no native files') + const forged = canonicalCandidateDocument({ + ...activation, + files: [{ ...first, content: `${first.content}\nforged` }, ...rest], + }).value + + expect(() => + parseAgentCandidateProfileActivation(forged, prepared.profilePlan.value.digest), + ).toThrow(/must match the canonical plan/) + }) + it('keeps argv task bytes out of fixed args and exposes deterministic delivery separately', async () => { const value = fixture() value.bundle = bundle({ instructionDelivery: { kind: 'argv-append' } }) + bindCandidateFixtureBundle(value) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(value.bundle, value.ports), value.task, value.ports, ) - expect(prepared.launch.args).not.toContain(value.task.instruction) + expect(prepared.launch.args).not.toContain(value.task.task.instruction) expect(prepared.instruction).toMatchObject({ delivery: { kind: 'argv-append' } }) }) @@ -395,7 +138,7 @@ describe('candidate execution preparation', () => { [field]: profileValue, } as AgentCandidateBundle['profile'], }) - const verified = await verifyAgentCandidateBundle(value.bundle, value.ports) + bindCandidateFixtureBundle(value) let stagingCalls = 0 let reservationCalls = 0 value.ports.workspaces.materialize = async () => { @@ -406,7 +149,7 @@ describe('candidate execution preparation', () => { throw new Error('candidate must fail before reserving protected access') } - await expect(prepareAgentCandidateExecution(verified, value.task, value.ports)).rejects.toThrow( + await expect(verifyAgentCandidateBundle(value.bundle, value.ports)).rejects.toThrow( new RegExp(`non-empty AgentProfile fields: ${field}`), ) expect(stagingCalls).toBe(0) @@ -421,9 +164,9 @@ describe('candidate execution preparation', () => { tools: {}, permissions: {}, modes: {}, - confidential: {}, }, }) + bindCandidateFixtureBundle(value) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(value.bundle, value.ports), @@ -435,7 +178,10 @@ describe('candidate execution preparation', () => { it('rejects task Git drift, dirty profile staging, and unenforced model limits', async () => { const gitDrift = fixture() - gitDrift.task.repository.baseTree = '0'.repeat(40) + if (!gitDrift.task.task.repository) throw new Error('expected repository identity') + replaceCandidateFixtureTask(gitDrift, { + repository: { ...gitDrift.task.task.repository, baseTree: '0'.repeat(40) }, + }) await expect( prepareAgentCandidateExecution( await verifyAgentCandidateBundle(gitDrift.bundle, gitDrift.ports), @@ -444,6 +190,20 @@ describe('candidate execution preparation', () => { ), ).rejects.toThrow(/base tree/) + const outputGitDrift = fixture() + if (!outputGitDrift.task.task.repository) throw new Error('expected repository identity') + replaceCandidateFixtureTask(outputGitDrift, { + outcome: { kind: 'output', mediaType: 'application/json', maxBytes: 1_024 }, + repository: { ...outputGitDrift.task.task.repository, baseTree: '0'.repeat(40) }, + }) + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(outputGitDrift.bundle, outputGitDrift.ports), + outputGitDrift.task, + outputGitDrift.ports, + ), + ).rejects.toThrow(/base tree/) + const profileDrift = fixture() writeFileSync(join(profileDrift.task.stagingRoots.profileRoot, 'stale'), 'stale') await expect( @@ -501,13 +261,15 @@ describe('candidate execution preparation', () => { expect(settledReservations).toBe(1) const zeroCall = fixture() - zeroCall.task.limits = { - ...zeroCall.task.limits, - maxModelCalls: 0, - maxInputTokens: 0, - maxOutputTokens: 0, - maxCostUsd: 0, - } + replaceCandidateFixtureTask(zeroCall, { + limits: { + ...zeroCall.task.task.limits, + maxModelCalls: 0, + maxInputTokens: 0, + maxOutputTokens: 0, + maxCostUsd: 0, + }, + }) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(zeroCall.bundle, zeroCall.ports), zeroCall.task, @@ -548,7 +310,7 @@ describe('candidate execution preparation', () => { network: { mode: 'gateway-only', domains: ['router.tangle.tools'] }, }) mismatched.ports.models.settleGrant = async () => ({ - preparationId: `candidate-preparation-v1.${'A'.repeat(43)}`, + preparationId: `candidate-preparation.${'A'.repeat(43)}`, grantDigest: sha('c'), closed: true, calls: [], @@ -564,7 +326,9 @@ describe('candidate execution preparation', () => { it('rejects a cost limit that cannot be represented by the protected integer ledger', async () => { const value = fixture() - value.task.limits = { ...value.task.limits, maxCostUsd: 10 ** 20 } + replaceCandidateFixtureTask(value, { + limits: { ...value.task.task.limits, maxCostUsd: 10 ** 20 }, + }) let reservations = 0 value.ports.models.reserveGrant = async () => { reservations++ @@ -582,10 +346,12 @@ describe('candidate execution preparation', () => { it('rejects a wall-time limit that Node would clamp to an immediate timer', async () => { const value = fixture() - value.task.limits = { - ...value.task.limits, - timeoutMs: MAX_CANDIDATE_TIMER_INTERVAL_MS + 1, - } + replaceCandidateFixtureTask(value, { + limits: { + ...value.task.task.limits, + timeoutMs: MAX_CANDIDATE_TIMER_INTERVAL_MS + 1, + }, + }) let reservations = 0 value.ports.models.reserveGrant = async () => { reservations++ @@ -615,7 +381,7 @@ describe('candidate execution preparation', () => { ).rejects.toThrow(/do not match|unsupported mode/) }) - it('resets task memory into an execution-scoped namespace and carries verified knowledge', async () => { + it('resets task memory into an execution-scoped namespace', async () => { const value = fixture() value.task.executionId = '..' const seedBytes = Buffer.from('seed-memory') @@ -632,7 +398,6 @@ describe('candidate execution preparation', () => { memory: { mode: 'isolated', scope: 'task', seed }, knowledge: { candidate: { - schemaVersion: 1, kind: 'knowledge-improvement-candidate', runId: 'knowledge-run-1', candidateId: 'knowledge-1', @@ -647,6 +412,7 @@ describe('candidate execution preparation', () => { evaluation, }, }) + bindCandidateFixtureBundle(value) value.ports.artifacts.read = async (ref) => { if (ref.sha256 === seed.sha256) return seedBytes throw new Error(`unexpected artifact ${ref.sha256}`) @@ -663,6 +429,7 @@ describe('candidate execution preparation', () => { beforeState: emptySnapshot('before'), } } + value.ports.memory.close = async () => ({ closed: true }) const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(value.bundle, value.ports), value.task, @@ -682,6 +449,45 @@ describe('candidate execution preparation', () => { files: [], }) expect(Buffer.from(prepared.knowledge?.retrievalConfig ?? [])).toEqual(knowledgeBytes) + expect(prepared.launch.env).toMatchObject({ + [CANDIDATE_KNOWLEDGE_ROOT_ENV]: '/workspace/task/.tangle/knowledge', + [CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV]: + '/workspace/task/.tangle/knowledge-retrieval-config.json', + }) + expect(prepared.executionPlan.value.material.launch.env).toMatchObject({ + [CANDIDATE_KNOWLEDGE_ROOT_ENV]: { + kind: 'public', + value: '/workspace/task/.tangle/knowledge', + }, + [CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV]: { + kind: 'public', + value: '/workspace/task/.tangle/knowledge-retrieval-config.json', + }, + }) + }) + + it('rejects snapshot-only knowledge before artifact access', async () => { + const value = fixture() + const bytes = Buffer.from('{"documents":[]}') + const manifest = { + locator: { kind: 's3' as const, bucket: 'test-artifacts', key: 'knowledge/manifest.json' }, + sha256: embeddedCandidateArtifact(bytes).sha256, + byteLength: bytes.byteLength, + } + value.bundle = redigestBundle(value.bundle, { + knowledge: { snapshotId: 'knowledge-1', manifest }, + }) + bindCandidateFixtureBundle(value) + let reads = 0 + value.ports.artifacts.read = async () => { + reads++ + return bytes + } + + await expect(verifyAgentCandidateBundle(value.bundle, value.ports)).rejects.toThrow( + /"knowledge"[\s\S]*"candidate"/, + ) + expect(reads).toBe(0) }) it('closes memory immediately when reset evidence fails before preparation can retain it', async () => { @@ -689,6 +495,7 @@ describe('candidate execution preparation', () => { value.bundle = redigestBundle(value.bundle, { memory: { mode: 'isolated', scope: 'task' }, }) + bindCandidateFixtureBundle(value) const before = emptySnapshot('malformed-reset') const closed: string[] = [] value.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ @@ -711,6 +518,6 @@ describe('candidate execution preparation', () => { ), ).rejects.toThrow(/not scoped/) expect(closed).toHaveLength(1) - expect(closed[0]).toMatch(/candidate-preparation-v1\..+:preparation-failed/) + expect(closed[0]).toMatch(/candidate-preparation\..+:preparation-failed/) }) }) diff --git a/tests/candidate-execution-recover.test.ts b/tests/candidate-execution-recover.test.ts index 4e3e380b..3434b881 100644 --- a/tests/candidate-execution-recover.test.ts +++ b/tests/candidate-execution-recover.test.ts @@ -11,14 +11,19 @@ import { } from '../src/candidate-execution/claim' import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' import { recoverExpiredAgentCandidateExecution } from '../src/candidate-execution/recover' -import type { AgentCandidateOutputArtifactPort } from '../src/candidate-execution/types' +import type { + AgentCandidateOutputArtifactPort, + PreparedAgentCandidateExecution, +} from '../src/candidate-execution/types' import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' import { + bindCandidateFixtureBundle, candidateSha, cleanupCandidateFixtures, createCandidateExecutionFixture, emptyCandidateSnapshot, redigestCandidateBundle, + replaceCandidateFixtureAttempt, } from './helpers/candidate-execution-fixture' afterEach(cleanupCandidateFixtures) @@ -33,19 +38,19 @@ describe('expired candidate recovery', () => { ) let now = Date.now() const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) - const claim = candidateExecutionClaim(prepared) + const outputArtifacts = outputArtifactPort() + const claim = await persistedClaim(prepared, outputArtifacts) const acquired = await claimStore.tryClaim(claim) expect(acquired.acquired).toBe(true) now = claim.leaseExpiresAtMs let stops = 0 + let disposals = 0 let settlements = 0 const originalSettle = fixture.ports.models.settleGrant fixture.ports.models.settleGrant = async (input) => { settlements++ return await originalSettle(input) } - const outputArtifacts = outputArtifactPort() - const result = await recoverExpiredAgentCandidateExecution({ attempt: { executionId: claim.executionId, attempt: claim.attempt }, claimStore, @@ -57,16 +62,26 @@ describe('expired candidate recovery', () => { execute: async () => { throw new Error('recovery must not execute') }, - stopAndCapture: async (request, context) => { + stop: async (request, context) => { stops++ expect(request).toEqual({ executionId: claim.executionId, executionPlanDigest: claim.executionPlanDigest, }) - expect(context.signal.aborted).toBe(true) - expect(context.deadlineAtMs).toBe(claim.leaseExpiresAtMs) + expect(context.signal.aborted).toBe(false) + expect(context.deadlineAtMs).toBeGreaterThan(Date.now()) return { stopped: true } }, + dispose: async (request, context) => { + disposals++ + expect(request).toEqual({ + executionId: claim.executionId, + executionPlanDigest: claim.executionPlanDigest, + }) + expect(context.signal.aborted).toBe(false) + return { disposed: true } + }, + capture: async () => ({ evidence: Buffer.from('official recovery evidence') }), }, }) expect(result).toMatchObject({ @@ -101,13 +116,16 @@ describe('expired candidate recovery', () => { execute: async () => { throw new Error('recovery must not execute') }, - stopAndCapture: async () => { + stop: async () => { throw new Error('terminal recovery must not stop twice') }, + capture: async () => { + throw new Error('terminal recovery must not capture twice') + }, }, }) expect(replay).toMatchObject({ finished: false, exactReplay: true }) - expect({ stops, settlements }).toEqual({ stops: 1, settlements: 1 }) + expect({ stops, disposals, settlements }).toEqual({ stops: 1, disposals: 1, settlements: 1 }) }) it('does no outward cleanup before expiry and records nothing when cleanup is unproven', async () => { @@ -119,10 +137,10 @@ describe('expired candidate recovery', () => { ) let now = Date.now() const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) - const claim = candidateExecutionClaim(prepared) + const outputArtifacts = outputArtifactPort() + const claim = await persistedClaim(prepared, outputArtifacts) await claimStore.tryClaim(claim) let stops = 0 - const outputArtifacts = outputArtifactPort() const recover = () => recoverExpiredAgentCandidateExecution({ attempt: { executionId: claim.executionId, attempt: claim.attempt }, @@ -136,10 +154,11 @@ describe('expired candidate recovery', () => { execute: async () => { throw new Error('recovery must not execute') }, - stopAndCapture: async () => { + stop: async () => { stops++ throw new Error('process still live') }, + capture: async () => ({}), }, }) @@ -155,11 +174,11 @@ describe('expired candidate recovery', () => { it('unlocks attempt two only when a recovered pre-run crash used zero model calls', async () => { const fixture = createCandidateExecutionFixture() - fixture.task.attempt = { + replaceCandidateFixtureAttempt(fixture, { number: 1, maxAttempts: 2, retryPolicy: 'pre-model-infrastructure-only', - } + }) const first = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), fixture.task, @@ -167,7 +186,8 @@ describe('expired candidate recovery', () => { ) let now = Date.now() const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) - const firstClaim = candidateExecutionClaim(first) + const outputArtifacts = outputArtifactPort() + const firstClaim = await persistedClaim(first, outputArtifacts) await claimStore.tryClaim(firstClaim) now = firstClaim.leaseExpiresAtMs const recovered = await recoverExpiredAgentCandidateExecution({ @@ -175,13 +195,14 @@ describe('expired candidate recovery', () => { claimStore, traceStore: new InMemoryTraceStore(), ports: fixture.ports, - outputArtifacts: outputArtifactPort(), + outputArtifacts, now: () => now, executor: { execute: async () => { throw new Error('recovery must not execute') }, - stopAndCapture: async () => ({ stopped: true }), + stop: async () => ({ stopped: true }), + capture: async () => ({}), }, }) expect(recovered).toMatchObject({ @@ -191,26 +212,29 @@ describe('expired candidate recovery', () => { rmSync(fixture.task.stagingRoots.profileRoot, { recursive: true, force: true }) mkdirSync(fixture.task.stagingRoots.profileRoot) - fixture.task.attempt = { + replaceCandidateFixtureAttempt(fixture, { number: 2, maxAttempts: 2, retryPolicy: 'pre-model-infrastructure-only', - } + }) const second = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), fixture.task, fixture.ports, ) - await expect(claimStore.tryClaim(candidateExecutionClaim(second))).resolves.toMatchObject({ - acquired: true, - }) + await expect( + claimStore.tryClaim(await persistedClaim(second, outputArtifacts)), + ).resolves.toMatchObject({ acquired: true }) }) it('repeats idempotent cleanup after a partial recovery failure', async () => { const fixture = createCandidateExecutionFixture() - fixture.bundle = redigestCandidateBundle(fixture.bundle, { - memory: { mode: 'isolated', scope: 'task' }, - }) + bindCandidateFixtureBundle( + fixture, + redigestCandidateBundle(fixture.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }), + ) const before = emptyCandidateSnapshot('recovery-before') fixture.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ preparationId, @@ -239,11 +263,20 @@ describe('expired candidate recovery', () => { ) let now = Date.now() const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) - const claim = candidateExecutionClaim(prepared) + const artifactStore = outputArtifactPort() + const persistedPurposes: string[] = [] + const outputArtifacts: AgentCandidateOutputArtifactPort = { + ...artifactStore, + put: async (input) => { + persistedPurposes.push(input.purpose) + return artifactStore.put(input) + }, + } + const claim = await persistedClaim(prepared, outputArtifacts) await claimStore.tryClaim(claim) now = claim.leaseExpiresAtMs let stops = 0 - const outputArtifacts = outputArtifactPort() + let captures = 0 const recover = () => recoverExpiredAgentCandidateExecution({ attempt: { executionId: claim.executionId, attempt: claim.attempt }, @@ -256,20 +289,26 @@ describe('expired candidate recovery', () => { execute: async () => { throw new Error('recovery must not execute') }, - stopAndCapture: async () => { + stop: async () => { stops++ return { stopped: true } }, + capture: async () => { + captures++ + return { evidence: Buffer.from(`capture-${captures}`) } + }, }, }) await expect(recover()).rejects.toThrow(/cleanup could not be proven/) + expect(persistedPurposes).not.toContain('executor-capture') expect( await claimStore.getAttempt({ executionId: claim.executionId, attempt: 1 }), ).not.toHaveProperty('terminal') await expect(recover()).resolves.toMatchObject({ finished: true }) - expect({ stops, settlements, memoryCloses }).toEqual({ + expect({ stops, captures, settlements, memoryCloses }).toEqual({ stops: 2, + captures: 0, settlements: 2, memoryCloses: 2, }) @@ -284,7 +323,8 @@ describe('expired candidate recovery', () => { ) let now = Date.now() const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) - const claim = candidateExecutionClaim(prepared) + const outputArtifacts = outputArtifactPort() + const claim = await persistedClaim(prepared, outputArtifacts) await claimStore.tryClaim(claim) now = claim.leaseExpiresAtMs const traceStore = new InMemoryTraceStore() @@ -295,13 +335,13 @@ describe('expired candidate recovery', () => { claimStore, traceStore, ports: fixture.ports, - outputArtifacts: outputArtifactPort(), + outputArtifacts, now: () => now, executor: { execute: async () => { throw new Error('recovery must not execute') }, - stopAndCapture: async (_request, context) => { + stop: async (_request, context) => { await context.traceStore.appendEvent({ runId: claim.cleanup.traceRunId, eventId: 'late-secret', @@ -311,6 +351,7 @@ describe('expired candidate recovery', () => { }) return { stopped: true } }, + capture: async () => ({}), }, }), ).rejects.toThrow(/cleanup could not be proven/) @@ -321,6 +362,25 @@ describe('expired candidate recovery', () => { }) }) +async function persistedClaim( + prepared: PreparedAgentCandidateExecution, + outputArtifacts: AgentCandidateOutputArtifactPort, +) { + const [executionPlan, materializationReceipt] = await Promise.all([ + outputArtifacts.put({ + executionId: prepared.executionId, + purpose: 'execution-plan', + bytes: prepared.executionPlan.bytes, + }), + outputArtifacts.put({ + executionId: prepared.executionId, + purpose: 'materialization-receipt', + bytes: prepared.materializationReceipt.bytes, + }), + ]) + return candidateExecutionClaim(prepared, { executionPlan, materializationReceipt }) +} + function outputArtifactPort(): AgentCandidateOutputArtifactPort { const stored = new Map() return { diff --git a/tests/candidate-execution-task-outcome.test.ts b/tests/candidate-execution-task-outcome.test.ts index 2a79e0f3..47b75ea8 100644 --- a/tests/candidate-execution-task-outcome.test.ts +++ b/tests/candidate-execution-task-outcome.test.ts @@ -50,7 +50,6 @@ function changedOutcome(root: string, baseTree: string) { const resultTree = git(root, ['write-tree'], undefined, environment) const resultBytes = Buffer.from('export const value = 2\n') const afterState = { - schemaVersion: 2 as const, kind: 'agent-candidate-workspace-manifest' as const, files: [ { @@ -67,15 +66,14 @@ function changedOutcome(root: string, baseTree: string) { describe('candidate task outcome verification', () => { it('proves a captured binary patch and after-state against the signed base tree', async () => { const fixture = createCandidateExecutionFixture() - const outcome = changedOutcome( - fixture.task.stagingRoots.taskRoot, - fixture.task.repository.baseTree, - ) + if (!fixture.task.task.repository) throw new Error('expected repository identity') + const repository = fixture.task.task.repository + const outcome = changedOutcome(fixture.task.stagingRoots.taskRoot, repository.baseTree) await expect( verifyTaskOutcomePatch({ repositoryRoot: fixture.task.stagingRoots.taskRoot, - baseCommit: fixture.task.repository.baseCommit, - baseTree: fixture.task.repository.baseTree, + baseCommit: repository.baseCommit, + baseTree: repository.baseTree, ...outcome, }), ).resolves.toMatchObject({ @@ -86,24 +84,23 @@ describe('candidate task outcome verification', () => { it('rejects a claimed result tree or after-state that does not match the patch', async () => { const fixture = createCandidateExecutionFixture() - const outcome = changedOutcome( - fixture.task.stagingRoots.taskRoot, - fixture.task.repository.baseTree, - ) + if (!fixture.task.task.repository) throw new Error('expected repository identity') + const repository = fixture.task.task.repository + const outcome = changedOutcome(fixture.task.stagingRoots.taskRoot, repository.baseTree) await expect( verifyTaskOutcomePatch({ repositoryRoot: fixture.task.stagingRoots.taskRoot, - baseCommit: fixture.task.repository.baseCommit, - baseTree: fixture.task.repository.baseTree, + baseCommit: repository.baseCommit, + baseTree: repository.baseTree, ...outcome, - resultTree: '0'.repeat(fixture.task.repository.baseTree.length), + resultTree: '0'.repeat(repository.baseTree.length), }), ).rejects.toThrow(/materialized tree/) await expect( verifyTaskOutcomePatch({ repositoryRoot: fixture.task.stagingRoots.taskRoot, - baseCommit: fixture.task.repository.baseCommit, - baseTree: fixture.task.repository.baseTree, + baseCommit: repository.baseCommit, + baseTree: repository.baseTree, ...outcome, afterState: { ...outcome.afterState, diff --git a/tests/helpers/candidate-execution-fixture.ts b/tests/helpers/candidate-execution-fixture.ts index 5b77563f..07ed7533 100644 --- a/tests/helpers/candidate-execution-fixture.ts +++ b/tests/helpers/candidate-execution-fixture.ts @@ -2,9 +2,13 @@ import { execFileSync } from 'node:child_process' import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' - +import { + sealCandidateBenchmarkSuite, + sealCandidateBenchmarkTask, +} from '@tangle-network/agent-eval/contract' import type { AgentCandidateArtifactRef, + AgentCandidateBenchmarkTask, AgentCandidateBundle, AgentCandidateExecution, AgentCandidateWorkspaceSnapshotEvidence, @@ -13,6 +17,7 @@ import type { import { canonicalCandidateBytes, canonicalCandidateDigest, + canonicalCandidateDocument, embeddedCandidateArtifact, sha256Bytes, } from '../../src/candidate-execution/digest' @@ -68,7 +73,6 @@ function snapshot( files: Array<{ path: string; mode: number }>, ): AgentCandidateWorkspaceSnapshotEvidence { const material = { - schemaVersion: 2 as const, kind: 'agent-candidate-workspace-manifest' as const, files: files .map((file) => { @@ -87,7 +91,6 @@ function snapshot( } const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) return { - schemaVersion: 2, kind: 'agent-candidate-workspace-snapshot', digest: manifest.sha256, material, @@ -105,7 +108,6 @@ export function candidateBundle( }, ): AgentCandidateBundle { const value = { - schemaVersion: 2 as const, kind: 'agent-candidate-bundle' as const, digestAlgorithm: 'rfc8785-sha256' as const, profile: { @@ -123,7 +125,7 @@ export function candidateBundle( baseCommit: active.commit, baseTree: active.tree, } - : { kind: 'disabled' as const, reason: 'control' as const }, + : { kind: 'disabled' as const }, execution: { harness: 'codex' as const, harnessVersion: '1.2.3', @@ -146,22 +148,6 @@ export function candidateBundle( ...execution, }, memory: { mode: 'disabled' as const }, - lineage: active - ? { - source: 'optimizer' as const, - parentDigests: [candidateSha('e')], - runIds: ['optimizer-run-1'], - benchmark: { - name: 'development', - version: '1', - splitDigest: candidateSha('f'), - }, - spend: { - proposal: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, - evaluation: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, - }, - } - : { source: 'human' as const }, } return { ...value, digest: canonicalCandidateDigest(value) } } @@ -175,15 +161,70 @@ export function redigestCandidateBundle( return { ...value, digest: canonicalCandidateDigest(value) } } +export function bindCandidateFixtureBundle( + fixture: CandidateExecutionFixture, + bundle: AgentCandidateBundle = fixture.bundle, +): void { + const { digest: _digest, ...cell } = fixture.task.runCell + fixture.bundle = bundle + fixture.task = { + ...fixture.task, + runCell: canonicalCandidateDocument({ ...cell, bundleDigest: bundle.digest }).value, + } +} + +export function replaceCandidateFixtureTask( + fixture: CandidateExecutionFixture, + overrides: Partial>, +): void { + const material = { ...omitTaskDigest(fixture.task.task), ...overrides } + const task = sealCandidateBenchmarkTask(material) + const benchmark = sealCandidateBenchmarkSuite({ + tasks: [task], + reps: fixture.task.benchmarkSuite.reps, + seeds: fixture.task.benchmarkSuite.seeds, + }) + const { digest: _digest, ...cell } = fixture.task.runCell + fixture.task = { + ...fixture.task, + task, + benchmarkSuite: benchmark.suite, + runCell: canonicalCandidateDocument({ + ...cell, + suiteDigest: benchmark.suite.digest, + taskDigest: task.digest, + }).value, + } +} + +export function replaceCandidateFixtureAttempt( + fixture: CandidateExecutionFixture, + attempt: { + number: number + maxAttempts: number + retryPolicy: 'none' | 'pre-model-infrastructure-only' + }, +): void { + replaceCandidateFixtureTask(fixture, { + attempt: { + maxAttempts: attempt.maxAttempts, + retryPolicy: attempt.retryPolicy, + }, + }) + const { digest: _digest, ...cell } = fixture.task.runCell + fixture.task = { + ...fixture.task, + runCell: canonicalCandidateDocument({ ...cell, attempt: attempt.number }).value, + } +} + export function emptyCandidateSnapshot(label: string): AgentCandidateWorkspaceSnapshotEvidence { const material = { - schemaVersion: 2 as const, kind: 'agent-candidate-workspace-manifest' as const, files: [], } const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) return { - schemaVersion: 2, kind: 'agent-candidate-workspace-snapshot', digest: manifest.sha256, material, @@ -202,10 +243,17 @@ export interface CandidateExecutionFixture { export function unchangedTaskOutcomeCapture( fixture: CandidateExecutionFixture, ): AgentCandidateExecutorTaskOutcomeCapture { + if (fixture.task.task.outcome.kind !== 'workspace') { + throw new Error('unchanged workspace capture requires a workspace outcome') + } + if (!fixture.task.task.repository) { + throw new Error('workspace task is missing repository identity') + } return { - resultTree: fixture.task.repository.baseTree, - afterState: fixture.task.workspace.material, - archive: Buffer.from(`task-archive:${fixture.task.workspace.digest}`, 'utf8'), + kind: 'workspace', + resultTree: fixture.task.task.repository.baseTree, + afterState: fixture.task.task.workspace.material, + archive: Buffer.from(`task-archive:${fixture.task.task.workspace.digest}`, 'utf8'), gitDiff: Buffer.alloc(0), } } @@ -294,6 +342,7 @@ export function createCandidateExecutionFixture(active = false): CandidateExecut } return } + if (role === 'knowledge' && workspace.material.files.length === 0) return const source = role === 'task' ? repository.root : candidateRoot if (!source) throw new Error(`fixture ${role} workspace source is missing`) if (source === destination) return @@ -344,12 +393,25 @@ export function createCandidateExecutionFixture(active = false): CandidateExecut }, }, } - const task: AgentCandidateTaskExecution = { - executionId: 'execution-1', - benchmark: 'repository-disjoint-smoke', - benchmarkVersion: '1', - taskId: 'owner-repo-1', - splitDigest: candidateSha('d'), + const bundle = candidateBundle( + {}, + active && candidateWorkspace + ? { commit: repository.commit, tree: repository.tree, workspace: candidateWorkspace } + : undefined, + ) + const benchmarkTask = sealCandidateBenchmarkTask({ + kind: 'agent-candidate-benchmark-task', + digestAlgorithm: 'rfc8785-sha256', + benchmark: { + name: 'repository-disjoint-smoke', + version: '1', + splitDigest: candidateSha('d'), + }, + scenario: { + id: 'owner-repo-1', + kind: 'coding', + scenarioDigest: candidateSha('8'), + }, instruction: 'Fix the failing behavior without changing the public API.', repository: { identity: 'github.com/owner/repo', @@ -357,26 +419,29 @@ export function createCandidateExecutionFixture(active = false): CandidateExecut baseCommit: repository.commit, baseTree: repository.tree, }, - attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, - model: { requested: 'provider/model', reasoningEffort: 'high' }, + outcome: { kind: 'workspace' }, + attempt: { maxAttempts: 1, retryPolicy: 'none' }, + model: { + requested: 'provider/model', + provider: 'provider', + model: 'model-snapshot', + snapshot: 'model-snapshot-2026-07-01', + reasoningEffort: 'high', + }, grader: { name: 'fixture-executable-grader', version: '1.0.0', + format: 'tangle-grader', artifact: { - locator: { kind: 's3', bucket: 'candidate-test-artifacts', key: 'grader/fixture' }, + locator: { + kind: 's3', + bucket: 'candidate-test-artifacts', + key: `grader/${sha256Bytes(fixtureGraderBytes).slice('sha256:'.length)}`, + }, sha256: sha256Bytes(fixtureGraderBytes), byteLength: fixtureGraderBytes.byteLength, }, }, - executionRoots: { - taskRoot: '/workspace/task', - ...(active ? { candidateRoot: '/opt/candidate' } : {}), - }, - stagingRoots: { - taskRoot: repository.root, - ...(candidateRoot ? { candidateRoot } : {}), - profileRoot, - }, workspace: taskWorkspace, evaluatorTaskContainer: selectedContainer, limits: { @@ -387,16 +452,106 @@ export function createCandidateExecutionFixture(active = false): CandidateExecut maxOutputTokens: 50_000, maxCostUsd: 5, }, + }) + const benchmark = sealCandidateBenchmarkSuite({ + tasks: [benchmarkTask], + reps: 1, + seeds: [42], + }) + const task: AgentCandidateTaskExecution = { + executionId: 'execution-1', + runCell: canonicalCandidateDocument({ + kind: 'agent-candidate-run-cell' as const, + experimentDigest: candidateSha('9'), + arm: 'candidate' as const, + bundleDigest: bundle.digest, + suiteDigest: benchmark.suite.digest, + taskDigest: benchmarkTask.digest, + taskIndex: 0, + repetition: 0, + seed: 42, + attempt: 1, + }).value, + benchmarkSuite: benchmark.suite, + task: benchmarkTask, + executionRoots: { + taskRoot: '/workspace/task', + ...(active ? { candidateRoot: '/opt/candidate' } : {}), + }, + stagingRoots: { + taskRoot: repository.root, + ...(candidateRoot ? { candidateRoot } : {}), + profileRoot, + }, } return { - bundle: candidateBundle( - {}, - active && candidateWorkspace - ? { commit: repository.commit, tree: repository.tree, workspace: candidateWorkspace } - : undefined, - ), + bundle, task, ports, ...(candidateRoot ? { candidateRoot } : {}), } } + +export function createCandidateOutputExecutionFixture( + mediaType = 'application/json', + maxBytes = 1_048_576, + withRepository = false, +): CandidateExecutionFixture { + const fixture = createCandidateExecutionFixture() + if (withRepository) { + return withBenchmarkTask(fixture, { + ...omitTaskDigest(fixture.task.task), + outcome: { kind: 'output', mediaType, maxBytes }, + }) + } + const taskRoot = temporaryRoot('candidate-output-task-') + const { repository: _repository, ...taskWithoutRepository } = omitTaskDigest(fixture.task.task) + return withBenchmarkTask( + { + ...fixture, + task: { + ...fixture.task, + stagingRoots: { ...fixture.task.stagingRoots, taskRoot }, + }, + }, + { + ...taskWithoutRepository, + outcome: { kind: 'output', mediaType, maxBytes }, + workspace: emptyCandidateSnapshot('output-task'), + }, + ) +} + +function omitTaskDigest( + task: AgentCandidateBenchmarkTask, +): Omit { + const { digest: _digest, ...material } = task + return material +} + +function withBenchmarkTask( + fixture: CandidateExecutionFixture, + material: Omit, +): CandidateExecutionFixture { + const task = sealCandidateBenchmarkTask(material) + const benchmark = sealCandidateBenchmarkSuite({ + tasks: [task], + reps: fixture.task.benchmarkSuite.reps, + seeds: fixture.task.benchmarkSuite.seeds, + }) + const { digest: _cellDigest, ...cell } = fixture.task.runCell + const runCell = canonicalCandidateDocument({ + ...cell, + suiteDigest: benchmark.suite.digest, + taskDigest: task.digest, + }).value + return { + ...fixture, + task: { + ...fixture.task, + runCell, + benchmarkSuite: benchmark.suite, + task, + }, + } +} diff --git a/tests/helpers/candidate-experiment-fixture.ts b/tests/helpers/candidate-experiment-fixture.ts new file mode 100644 index 00000000..ee31da70 --- /dev/null +++ b/tests/helpers/candidate-experiment-fixture.ts @@ -0,0 +1,199 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { InMemoryTraceStore } from '@tangle-network/agent-eval' +import { + type CandidateExperimentExecutionInput, + sealCandidateBenchmarkSuite, + sealCandidateExperiment, +} from '@tangle-network/agent-eval/contract' +import type { + AgentCandidateBundle, + AgentCandidateExperiment, + CandidateExecutionEvidence, +} from '@tangle-network/agent-interface' + +import { InMemoryAgentCandidateExecutionClaimStore } from '../../src/candidate-execution/claim' +import { sha256Bytes } from '../../src/candidate-execution/digest' +import type { AgentCandidateExperimentCellPlacement } from '../../src/intelligence/improvement-cycle' +import { + type CandidateExecutionFixture, + candidateSha, + createCandidateOutputExecutionFixture, + createCandidateOutputFixture, + emptyCandidateSnapshot, + redigestCandidateBundle, +} from './candidate-execution-fixture' + +const roots: string[] = [] + +export interface CandidateExperimentFixture { + fixture: CandidateExecutionFixture + experiment: AgentCandidateExperiment + placeCell(input: CandidateExperimentExecutionInput): AgentCandidateExperimentCellPlacement +} + +export interface CreateCandidateExperimentFixtureOptions { + baseline?: AgentCandidateBundle + candidate?: AgentCandidateBundle + candidateSurface?: 'prompt' | 'memory' + reps?: number + scoreFor?: (input: CandidateExperimentExecutionInput) => number + configureFixture?: (fixture: CandidateExecutionFixture) => void +} + +export function cleanupCandidateExperimentFixtures(): void { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +} + +export function createCandidateExperimentFixture( + options: CreateCandidateExperimentFixtureOptions = {}, +): CandidateExperimentFixture { + const fixture = createCandidateOutputExecutionFixture('text/plain', 1_024) + options.configureFixture?.(fixture) + const baseline = options.baseline ?? fixture.bundle + const candidate = + options.candidate ?? + (options.candidateSurface === 'memory' + ? redigestCandidateBundle(baseline, { memory: { mode: 'isolated', scope: 'task' } }) + : redigestCandidateBundle(baseline, { + profile: { + ...baseline.profile, + prompt: { + ...baseline.profile.prompt, + systemPrompt: 'Return the exact measured answer.', + }, + }, + })) + const reps = options.reps ?? 3 + const seeds = Array.from({ length: reps }, (_, index) => 101 + index) as [number, ...number[]] + const experiment = sealCandidateExperiment({ + kind: 'agent-candidate-experiment', + digestAlgorithm: 'rfc8785-sha256', + baseline, + candidate, + candidateLineage: { source: 'human' }, + benchmark: sealCandidateBenchmarkSuite({ tasks: [fixture.task.task], reps, seeds }), + policy: { + confidenceLevel: 0.95, + resamples: 500, + bootstrapSeed: 1_337, + deltaThreshold: 0, + minProductiveRuns: 3, + budgetUsd: 1, + criticalDimensions: ['quality'], + regressionTolerance: 0.05, + }, + }) + const memoryBefore = emptyCandidateSnapshot('experiment-memory-before') + fixture.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ + preparationId, + accessDigest: candidateSha('8'), + expiresAtMs, + evidence: memoryBefore.manifest, + emptyStateDigest: memoryBefore.digest, + beforeState: memoryBefore, + }) + fixture.ports.memory.activate = async ({ effectiveNamespace }) => ({ + env: { TANGLE_MEMORY_NAMESPACE: effectiveNamespace }, + }) + fixture.ports.memory.close = async () => ({ closed: true }) + + return { + fixture, + experiment, + placeCell(input) { + const taskRoot = temporaryRoot('candidate-experiment-task-') + const profileRoot = temporaryRoot('candidate-experiment-profile-') + const traceStore = new InMemoryTraceStore() + const outputs = createCandidateOutputFixture() + const score = options.scoreFor?.(input) ?? (input.arm === 'candidate' ? 1 : 0) + const output = Buffer.from(score === 1 ? 'strong' : 'weak', 'utf8') + return { + executionId: [ + 'experiment', + input.arm, + input.benchmarkCell.taskIndex, + input.benchmarkCell.repetition, + ].join('-'), + executionRoots: { taskRoot: '/workspace/task' }, + stagingRoots: { taskRoot, profileRoot }, + ports: fixture.ports, + execution: { + executor: { + execute: async (request, context) => { + const startedAt = 1_000 + input.benchmarkCell.repetition * 100 + await context.traceStore.appendRun({ + runId: request.trace.runId, + scenarioId: request.benchmark.task.scenario.id, + startedAt, + endedAt: startedAt + 50, + status: 'completed', + tags: { ...request.trace.tags }, + }) + return { + executionId: request.executionId, + termination: { kind: 'exit' as const, exitCode: 0 }, + } + }, + stop: async () => ({ stopped: true }), + capture: async () => ({ + taskOutcome: { kind: 'output' as const, bytes: output }, + ...(input.bundle.memory.mode === 'isolated' + ? { + memoryAfter: { + afterState: memoryBefore.material, + archive: Buffer.from('empty experiment memory', 'utf8'), + }, + } + : {}), + }), + }, + grader: { + ...outputs.grader, + run: async ({ implementation, outcome, signal }) => { + signal.throwIfAborted() + if (outcome.kind !== 'output') throw new Error('fixture grader requires output') + const measured = Buffer.from(outcome.bytes).toString('utf8') === 'strong' ? 1 : 0 + const evidence = Buffer.from(JSON.stringify({ score: measured }), 'utf8') + return { + evaluation: { + score: measured, + passed: measured === 1, + dimensions: { quality: measured }, + raw: {}, + }, + evidence, + binding: { + implementationDigest: sha256Bytes(implementation.bytes), + taskOutcomeDigest: outcome.evidence.digest, + outputDigest: sha256Bytes(evidence), + }, + } + }, + }, + outputArtifacts: outputs.outputArtifacts, + traceStore, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + }, + } + }, + } +} + +export async function executeCandidateExperimentInput( + input: CandidateExperimentExecutionInput, + placeCell: CandidateExperimentFixture['placeCell'], + execute: ( + input: CandidateExperimentExecutionInput & AgentCandidateExperimentCellPlacement, + ) => Promise, +): Promise { + return await execute({ ...input, ...placeCell(input) }) +} + +function temporaryRoot(prefix: string): string { + const root = mkdtempSync(join(tmpdir(), prefix)) + roots.push(root) + return root +} diff --git a/tests/improvement-cycle.test.ts b/tests/improvement-cycle.test.ts index dae36255..77d1df13 100644 --- a/tests/improvement-cycle.test.ts +++ b/tests/improvement-cycle.test.ts @@ -1,44 +1,25 @@ -import { type AnalystFinding, InMemoryTraceStore } from '@tangle-network/agent-eval' -import type { - DispatchContext, - JudgeConfig, - MutableSurface, - Scenario, - SurfaceProposer, -} from '@tangle-network/agent-eval/contract' -import { afterEach, describe, expect, it, vi } from 'vitest' -import type { AnalystRegistryLike } from '../src/analyst-loop/types' -import { buildAgentCandidateBundle } from '../src/candidate-execution/builder' -import { InMemoryAgentCandidateExecutionClaimStore } from '../src/candidate-execution/claim' -import { assertCandidateProfileBinding } from '../src/candidate-execution/profile' -import type { - AgentCandidateExecutorPort, - AgentCandidateExecutorRequest, -} from '../src/candidate-execution/types' +import type { AnalystFinding } from '@tangle-network/agent-eval' +import { afterEach, describe, expect, it } from 'vitest' + +import { canonicalCandidateDigest } from '../src/candidate-execution/digest' +import { agentCandidateProfileAsAgentProfile } from '../src/candidate-execution/profile' import { + createAgentImprovementActivation, createAgentImprovementProposal, - executeApprovedAgentCandidate, proposeAgentImprovement, reviewAgentImprovementProposal, + runAgentCandidateExperiment, + verifyAgentImprovementActivation, verifyAgentImprovementProposal, verifyAgentImprovementReview, + verifyCandidateExecutionEvidence, } from '../src/intelligence/improvement-cycle' +import { cleanupCandidateFixtures } from './helpers/candidate-execution-fixture' import { - candidateSha, - cleanupCandidateFixtures, - createCandidateExecutionFixture, - createCandidateOutputFixture, - unchangedTaskOutcomeCapture, -} from './helpers/candidate-execution-fixture' - -interface DemoScenario extends Scenario { - kind: 'demo' -} - -const scenarios: DemoScenario[] = Array.from({ length: 12 }, (_, index) => ({ - id: `scenario-${index}`, - kind: 'demo' as const, -})) + type CandidateExperimentFixture, + cleanupCandidateExperimentFixtures, + createCandidateExperimentFixture, +} from './helpers/candidate-experiment-fixture' const finding: AnalystFinding = { schema_version: '1.0.0', @@ -47,527 +28,240 @@ const finding: AnalystFinding = { produced_at: '2026-07-10T00:00:00.000Z', severity: 'high', area: 'prompt', - claim: 'The agent omits the required marker.', + claim: 'The agent omits the required answer.', evidence_refs: [{ kind: 'span', id: 'span-1' }], - recommended_action: 'Add the measured marker.', + recommended_action: 'Return the measured answer.', confidence: 0.9, subject: 'agent-profile:prompt.systemPrompt', } -const registry = (findings: AnalystFinding[]): AnalystRegistryLike => ({ - list: () => [{ id: 'improvement' }], - run: async (runId) => ({ - run_id: runId, - correlation_id: `correlation-${runId}`, - started_at: '2026-07-10T00:00:00.000Z', - ended_at: '2026-07-10T00:00:01.000Z', - findings, - per_analyst: [ - { - analyst_id: 'improvement', - status: 'ok', - findings_count: findings.length, - latency_ms: 1, - cost_usd: 0, - }, - ], - total_cost_usd: 0, - }), -}) - -const proposer: SurfaceProposer = { - kind: 'scripted', - propose: async () => [ - { surface: 'PROMOTED', label: 'measured winner', rationale: 'addresses finding-1' }, - ], -} - -const judge: JudgeConfig = { - name: 'literal-marker', - dimensions: [{ key: 'marker', description: 'Contains the required marker.' }], - score: ({ artifact }) => { - const composite = artifact.includes('PROMOTED') ? 1 : 0 - return { dimensions: { marker: composite }, composite, notes: '' } - }, -} - -async function agent( - surface: MutableSurface, - _scenario: DemoScenario, - context: DispatchContext, -): Promise { - const paid = await context.cost.runPaidCall({ - channel: 'agent', - actor: 'fixture', - model: 'fixture-model', - maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, - execute: async () => String(surface), - receipt: () => ({ - model: 'fixture-model', - inputTokens: 1, - outputTokens: 1, - actualCostUsd: 0.0001, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value -} - -function fixtureProfile() { - return { - name: 'candidate', - prompt: { - systemPrompt: 'BASELINE', - instructions: ['Inspect the repository, implement the fix, and run tests.'], - }, - model: { default: 'provider/model', reasoningEffort: 'high' as const }, - harness: 'codex' as const, - resources: { failOnError: true as const }, - } -} - -function alignedBundle( - bundle: Omit['bundle'], 'digest'>, - profile: { prompt?: { systemPrompt?: string } }, -) { - return { - ...bundle, - profile: { - ...bundle.profile, - prompt: { - ...bundle.profile.prompt, - systemPrompt: profile.prompt?.systemPrompt, - }, - }, - } -} - -function alignedSealedBundle( - bundle: Omit['bundle'], 'digest'>, - profile: { prompt?: { systemPrompt?: string } }, -) { - const aligned = alignedBundle(bundle, profile) - const { profileDiffIds: _profileDiffIds, ...lineage } = aligned.lineage - return buildAgentCandidateBundle({ - profile: { kind: 'candidate-profile', profile: aligned.profile }, - code: aligned.code, - execution: aligned.execution, - ...(aligned.knowledge ? { knowledge: aligned.knowledge } : {}), - memory: aligned.memory, - lineage, - }) -} - afterEach(() => { + cleanupCandidateExperimentFixtures() cleanupCandidateFixtures() - vi.restoreAllMocks() }) describe('agent improvement lifecycle', () => { - it('refuses memory persistence before human approval', async () => { - const writeBack = vi.fn() - await expect( - proposeAgentImprovement({ - runId: 'analysis-run-memory-writeback', - profile: fixtureProfile(), - analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, - improvement: { - surface: 'memory', - memory: { - document: '# Durable memory\n', - writeBack, - }, - generator: proposer, - scenarios, - judge, - agent, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.5 }, - }, - }), - ).rejects.toThrow('cannot write memory before human approval') - expect(writeBack).not.toHaveBeenCalled() - }) - - it('disposes a retained code winner when candidate construction fails', async () => { - const { execFileSync } = await import('node:child_process') - const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') - const { tmpdir } = await import('node:os') - const { join } = await import('node:path') - const repoRoot = mkdtempSync(join(tmpdir(), 'improvement-cycle-code-cleanup-')) - const git = (...args: string[]) => - execFileSync('git', args, { cwd: repoRoot, encoding: 'utf8', stdio: 'pipe' }) - try { - git('init', '-q', '-b', 'main') - git('config', 'user.email', 'improvement-cycle@test.local') - git('config', 'user.name', 'improvement-cycle-test') - writeFileSync(join(repoRoot, 'module.txt'), 'BASELINE\n') - git('add', 'module.txt') - git('commit', '-qm', 'baseline') - - const buildCandidate = vi.fn(async () => { - throw new Error('candidate construction failed') - }) - await expect( - proposeAgentImprovement({ - runId: 'analysis-run-code-build-failure', - profile: fixtureProfile(), - analysis: { - registry: registry([finding]), - inputs: {}, - findingsStore: null, - log: () => {}, - }, - improvement: { - surface: 'code', - scenarios, - judge: { - name: 'code-marker', - dimensions: [{ key: 'marker', description: 'Code contains the promoted marker.' }], - score: ({ artifact }) => { - const composite = artifact.includes('PROMOTED') ? 1 : 0 - return { dimensions: { marker: composite }, composite, notes: '' } - }, - }, - agent: async (surface, _scenario, context) => { - const paid = await context.cost.runPaidCall({ - channel: 'agent', - actor: 'fixture', - model: 'fixture-model', - maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, - execute: async () => { - if (typeof surface === 'string') throw new Error('expected code surface') - return readFileSync(join(surface.worktreeRef, 'module.txt'), 'utf8') - }, - receipt: () => ({ - model: 'fixture-model', - inputTokens: 1, - outputTokens: 1, - actualCostUsd: 0.0001, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value - }, - code: { - repoRoot, - generator: { - kind: 'fixture', - async generate({ worktreePath }) { - writeFileSync(join(worktreePath, 'module.txt'), 'PROMOTED\n') - return { applied: true, summary: 'write promoted marker' } - }, - }, - }, - promotionGate: { - name: 'fixture-ship', - decide: async () => ({ - decision: 'ship', - reasons: ['candidate passes the fixture comparison'], - contributingGates: [], - }), - }, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, - }, - buildCandidate, - }), - ).rejects.toThrow('candidate construction failed') - - expect(buildCandidate).toHaveBeenCalledOnce() - expect(git('worktree', 'list', '--porcelain').match(/^worktree /gm)).toHaveLength(1) - } finally { - rmSync(repoRoot, { recursive: true, force: true }) - } - }) + it('measures the exact paired matrix before review and activation', async () => { + const rig = createCandidateExperimentFixture() + const executed: string[] = [] + const result = await runAgentCandidateExperiment({ + experiment: rig.experiment, + runId: 'paired-run-1', + maxConcurrency: 2, + placeCell: (input) => { + executed.push(`${input.arm}:${input.benchmarkCell.repetition}`) + return rig.placeCell(input) + }, + }) - it('binds typed candidate hooks to their equivalent measured profile commands', () => { - expect(() => - assertCandidateProfileBinding( - { - hooks: { - beforeTool: [ - { - command: "node 'path with space.js'", - timeoutMs: 1_000, - env: { MODE: 'check' }, - }, - ], - }, - }, - { - hooks: { - beforeTool: [ - { - executable: 'node', - args: [{ kind: 'public', value: 'path with space.js' }], - timeoutMs: 1_000, - env: { MODE: { kind: 'public', value: 'check' } }, - }, - ], - }, - }, - ), - ).not.toThrow() - }) + expect(executed.sort()).toEqual([ + 'baseline:0', + 'baseline:1', + 'baseline:2', + 'candidate:0', + 'candidate:1', + 'candidate:2', + ]) + expect(result.measurements).toHaveLength(3) + expect(result.evaluation).toMatchObject({ + experiment: { digest: rig.experiment.digest }, + overall: { baseline: 0, candidate: 1, delta: 1, n: 3 }, + decision: { outcome: 'ship' }, + evaluation: { executionCostUsd: 0, searchCostUsd: 0, totalCostUsd: 0 }, + }) - it('analyzes, measures, approves, executes, grades, and links one exact receipt', async () => { - const fixture = createCandidateExecutionFixture() - const { digest: _digest, ...bundleInput } = fixture.bundle - const profile = fixtureProfile() - const proposed = await proposeAgentImprovement({ - runId: 'analysis-run-1', - profile, - analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, - improvement: { - surface: 'prompt', - generator: proposer, - scenarios, - judge, - agent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, - }, - buildCandidate: ({ improvement }) => alignedSealedBundle(bundleInput, improvement.profile), + const proposal = createAgentImprovementProposal({ + runId: 'paired-run-1', + findings: [finding], + evaluation: result.evaluation, now: () => new Date('2026-07-10T01:00:00.000Z'), }) + expect(proposal.changedSurfaces).toEqual(['prompt']) + expect(verifyAgentImprovementProposal(proposal)).toEqual(proposal) + const { digest: _proposalDigest, ...proposalWithoutDigest } = proposal + const judgeDerivedProposalWithoutDigest = { + ...proposalWithoutDigest, + findings: proposal.findings.map((item) => ({ ...item, derived_from_judge: true })), + } + const judgeDerivedProposal = { + ...judgeDerivedProposalWithoutDigest, + digest: canonicalCandidateDigest(judgeDerivedProposalWithoutDigest), + } + expect(() => verifyAgentImprovementProposal(judgeDerivedProposal)).toThrow(/judge-derived/) - expect(proposed.proposal.evaluation.gateDecision).toBe('ship') - expect(proposed.proposal.evaluation.lift).toBeGreaterThan(0) - expect(proposed.proposal.findings).toEqual([finding]) - expect(proposed.proposal.candidateProfile.prompt?.systemPrompt).toBe('PROMOTED') - expect(proposed.proposal.candidateBundle?.digest).toMatch(/^sha256:[a-f0-9]{64}$/) - expect(verifyAgentImprovementProposal(proposed.proposal)).toEqual(proposed.proposal) - expect( - createAgentImprovementProposal({ - runId: 'analysis-run-1', - surface: 'prompt', - baselineProfile: profile, - candidateProfile: proposed.improvement.profile, - findings: proposed.analysis.analystResult.findings, - evaluation: proposed.improvement.raw, - candidateBundle: proposed.proposal.candidateBundle, - now: () => new Date('2026-07-10T01:00:00.000Z'), - }), - ).toEqual(proposed.proposal) - expect(() => - createAgentImprovementProposal({ - runId: 'analysis-run-unmeasured', - surface: 'prompt', - baselineProfile: profile, - candidateProfile: { - ...proposed.improvement.profile, - prompt: { systemPrompt: 'UNMEASURED' }, - }, - findings: proposed.analysis.analystResult.findings, - evaluation: proposed.improvement.raw, - }), - ).toThrow(/does not match the measured winner surface/) - expect(() => - createAgentImprovementProposal({ - runId: 'analysis-run-malformed-profile', - surface: 'agent-profile', - baselineProfile: profile, - candidateProfile: profile, - findings: proposed.analysis.analystResult.findings, - evaluation: { - ...proposed.improvement.raw, - winner: { - ...proposed.improvement.raw.winner, - surface: '{not-json', - }, - }, - }), - ).toThrow(/not valid JSON/) - - const review = reviewAgentImprovementProposal(proposed.proposal, { + const review = reviewAgentImprovementProposal(proposal, { decision: 'approve', reviewedBy: 'operator@example.com', - reason: 'Measured winner passed the held-back scenarios.', - now: () => new Date('2026-07-10T02:00:00.000Z'), + reason: 'All three paired tasks improved.', + now: () => new Date('2026-07-10T01:01:00.000Z'), }) expect(verifyAgentImprovementReview(review)).toEqual(review) - const traceStore = new InMemoryTraceStore() - let request: AgentCandidateExecutorRequest | undefined - const executor: AgentCandidateExecutorPort = { - execute: async (input) => { - request = input - await traceStore.appendRun({ - runId: input.trace.runId, - scenarioId: 'approved-candidate', - startedAt: 100, - endedAt: 200, - status: 'completed', - tags: { ...input.trace.tags }, - }) - return { executionId: input.executionId, termination: { kind: 'exit', exitCode: 0 } } - }, - stopAndCapture: async () => ({ - stopped: true, - taskOutcome: unchangedTaskOutcomeCapture(fixture), - }), - } - const outputs = createCandidateOutputFixture() - const executed = await executeApprovedAgentCandidate({ - proposal: proposed.proposal, - review, - authorizeReview: async (candidateReview) => candidateReview.digest === review.digest, - task: fixture.task, - ports: fixture.ports, - execution: { - executor, - traceStore, - claimStore: new InMemoryAgentCandidateExecutionClaimStore(), - ...outputs, - }, + const activation = createAgentImprovementActivation(proposal, review, { + targets: [ + { + surface: 'prompt', + identity: 'tenant/default/profile', + expectedBaseDigest: promptSurfaceDigest(rig), + }, + ], + fundingOwner: 'tenant/default', + authorizedBy: 'operator@example.com', + now: () => new Date('2026-07-10T01:02:00.000Z'), }) + expect(verifyAgentImprovementActivation({ proposal, review, activation })).toEqual(activation) + }) - expect(request).toBeDefined() - expect(executed.finalization.succeeded).toBe(true) - expect(executed.evidence).toMatchObject({ - proposalDigest: proposed.proposal.digest, - reviewDigest: review.digest, - bundleDigest: proposed.proposal.candidateBundle?.digest, - executionId: fixture.task.executionId, - succeeded: true, - runReceiptDigest: expect.stringMatching(/^sha256:/), + it('runs isolated memory as a real candidate surface', async () => { + const rig = createCandidateExperimentFixture({ candidateSurface: 'memory' }) + const result = await runAgentCandidateExperiment({ + experiment: rig.experiment, + runId: 'memory-run-1', + placeCell: rig.placeCell, + }) + const proposal = createAgentImprovementProposal({ + runId: 'memory-run-1', + findings: [finding], + evaluation: result.evaluation, }) + + expect(proposal.changedSurfaces).toEqual(['memory']) + expect(result.measurements).toHaveLength(3) + for (const measurement of result.measurements) { + expect(measurement.baseline.receipt.memory.mode).toBe('disabled') + expect(measurement.candidate.receipt.memory.mode).toBe('isolated') + } }) - it('records rejection and refuses to execute it', async () => { - const fixture = createCandidateExecutionFixture() - const { digest: _digest, ...bundleInput } = fixture.bundle - const proposed = await proposeAgentImprovement({ - runId: 'analysis-run-2', - profile: fixtureProfile(), - analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, - improvement: { - surface: 'prompt', - generator: proposer, - scenarios, - judge, - agent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, - }, - buildCandidate: ({ improvement }) => alignedBundle(bundleInput, improvement.profile), + it('rejects receipt substitution and stale activation targets', async () => { + const rig = createCandidateExperimentFixture() + const result = await runAgentCandidateExperiment({ + experiment: rig.experiment, + runId: 'binding-run-1', + placeCell: rig.placeCell, }) - const review = reviewAgentImprovementProposal(proposed.proposal, { - decision: 'reject', + const evidence = result.measurements[0]?.candidate + if (!evidence) throw new Error('expected candidate evidence') + const benchmarkCell = { + suiteDigest: rig.experiment.benchmark.suite.digest, + taskIndex: 0, + repetition: 0, + } + expect( + verifyCandidateExecutionEvidence(evidence, { + experiment: rig.experiment, + arm: 'candidate', + benchmarkCell, + seed: 101, + }), + ).toEqual(evidence) + expect(() => + verifyCandidateExecutionEvidence(evidence, { + experiment: rig.experiment, + arm: 'baseline', + benchmarkCell, + seed: 101, + }), + ).toThrow(/substituted|bind/) + + const proposal = createAgentImprovementProposal({ + runId: 'binding-run-1', + findings: [], + evaluation: result.evaluation, + }) + const review = reviewAgentImprovementProposal(proposal, { + decision: 'approve', reviewedBy: 'operator@example.com', - reason: 'The change is not appropriate for this deployment.', - feedback: 'Keep the baseline behavior.', + reason: 'Exact measured candidate approved.', }) - - await expect( - executeApprovedAgentCandidate({ - proposal: proposed.proposal, - review, - authorizeReview: async () => true, - task: fixture.task, - ports: fixture.ports, - execution: { - executor: { - execute: async () => { - throw new Error('must not execute') - }, - stopAndCapture: async () => ({ stopped: true }), + expect(() => + createAgentImprovementActivation(proposal, review, { + targets: [ + { + surface: 'prompt', + identity: 'tenant/default/profile', + expectedBaseDigest: canonicalCandidateDigest('stale-profile'), }, - traceStore: new InMemoryTraceStore(), - claimStore: new InMemoryAgentCandidateExecutionClaimStore(), - ...createCandidateOutputFixture(), - }, + ], + fundingOwner: 'tenant/default', + authorizedBy: 'operator@example.com', }), - ).rejects.toThrow('not an approval') + ).toThrow(/activation targets/) }) - it('rejects a sealed build candidate whose digest was tampered', async () => { - const fixture = createCandidateExecutionFixture() - const { digest: _digest, ...bundleInput } = fixture.bundle + it('does not create a proposal from an inconclusive comparison', async () => { + const rig = createCandidateExperimentFixture({ scoreFor: () => 1 }) + const result = await runAgentCandidateExperiment({ + experiment: rig.experiment, + runId: 'no-lift-run', + placeCell: rig.placeCell, + }) - await expect( - proposeAgentImprovement({ - runId: 'analysis-run-tampered-bundle', - profile: fixtureProfile(), - analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, - improvement: { - surface: 'prompt', - generator: proposer, - scenarios, - judge, - agent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, - }, - buildCandidate: ({ improvement }) => ({ - ...alignedSealedBundle(bundleInput, improvement.profile), - digest: candidateSha('0'), - }), + expect(result.evaluation.decision.outcome).not.toBe('ship') + expect(() => + createAgentImprovementProposal({ + runId: 'no-lift-run', + findings: [], + evaluation: result.evaluation, }), - ).rejects.toThrow('built candidate bundle digest is invalid') + ).toThrow(/passing experiment/) }) - it('refuses a structurally valid approval that its authority does not recognize', async () => { - const fixture = createCandidateExecutionFixture() - const { digest: _digest, ...bundleInput } = fixture.bundle - const proposed = await proposeAgentImprovement({ - runId: 'analysis-run-unauthorized', - profile: fixtureProfile(), - analysis: { registry: registry([finding]), inputs: {}, findingsStore: null, log: () => {} }, - improvement: { - surface: 'prompt', - generator: proposer, - scenarios, - judge, - agent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, - }, - buildCandidate: ({ improvement }) => alignedBundle(bundleInput, improvement.profile), + it('does not activate a rejected proposal', async () => { + const rig = createCandidateExperimentFixture() + const result = await runAgentCandidateExperiment({ + experiment: rig.experiment, + runId: 'rejected-run', + placeCell: rig.placeCell, }) - const review = reviewAgentImprovementProposal(proposed.proposal, { - decision: 'approve', - reviewedBy: 'forged@example.com', - reason: 'Self-authored approval.', + const proposal = createAgentImprovementProposal({ + runId: 'rejected-run', + findings: [], + evaluation: result.evaluation, + }) + const review = reviewAgentImprovementProposal(proposal, { + decision: 'reject', + reviewedBy: 'operator@example.com', + reason: 'Not suitable for this deployment.', }) - await expect( - executeApprovedAgentCandidate({ - proposal: proposed.proposal, - review, - authorizeReview: async () => false, - task: fixture.task, - ports: fixture.ports, - execution: { - executor: { - execute: async () => { - throw new Error('must not execute') - }, - stopAndCapture: async () => ({ stopped: true }), + expect(() => + createAgentImprovementActivation(proposal, review, { + targets: [ + { + surface: 'prompt', + identity: 'tenant/default/profile', + expectedBaseDigest: promptSurfaceDigest(rig), }, - traceStore: new InMemoryTraceStore(), - claimStore: new InMemoryAgentCandidateExecutionClaimStore(), - ...createCandidateOutputFixture(), - }, + ], + fundingOwner: 'tenant/default', + authorizedBy: 'operator@example.com', }), - ).rejects.toThrow('not authorized') + ).toThrow(/requires an approval/) }) - it('rejects judge-derived findings before they can steer a proposal', async () => { + it('blocks optimizer write-back before measurement and approval', async () => { await expect( proposeAgentImprovement({ - runId: 'analysis-run-3', - profile: fixtureProfile(), - analysis: { - registry: registry([{ ...finding, derived_from_judge: true }]), - inputs: {}, - findingsStore: null, - log: () => {}, + runId: 'unsafe-memory-write', + profile: { name: 'fixture' }, + analysis: {} as never, + improvement: { memory: { writeBack: async () => undefined } } as never, + buildExperiment: async () => { + throw new Error('must not build an experiment') }, - improvement: { - surface: 'prompt', - generator: proposer, - scenarios, - judge, - agent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, + placeCell: () => { + throw new Error('must not place a cell') }, }), - ).rejects.toThrow(/judge/i) + ).rejects.toThrow(/cannot write memory before approval/) }) }) + +function promptSurfaceDigest(rig: CandidateExperimentFixture): `sha256:${string}` { + const profile = agentCandidateProfileAsAgentProfile(rig.experiment.baseline.profile) + return canonicalCandidateDigest({ + prompt: profile.prompt ?? null, + instructions: profile.resources?.instructions ?? null, + }) +} diff --git a/tests/knowledge-improvement-job.test.ts b/tests/knowledge-improvement-job.test.ts index 9e120534..232aa829 100644 --- a/tests/knowledge-improvement-job.test.ts +++ b/tests/knowledge-improvement-job.test.ts @@ -1,16 +1,42 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' +import { dirname, join, relative } from 'node:path' +import type { AgentImprovementActivation } from '@tangle-network/agent-interface' import { addSourceText, defineReadinessSpec, + hashKnowledgeBase, initKnowledgeBase, + knowledgeImprovementCandidateRef, + withKnowledgeImprovementCandidate, } from '@tangle-network/agent-knowledge' -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it } from 'vitest' +import { createAgentCandidateWorkspacePort } from '../src/candidate-execution/workspace-archive' +import { + createAgentImprovementActivation, + createAgentImprovementProposal, + reviewAgentImprovementProposal, + runAgentCandidateExperiment, +} from '../src/intelligence/improvement-cycle' import { runKnowledgeImprovementJob } from '../src/knowledge' import type { SuperviseOptions } from '../src/runtime/supervise/supervise' import type { SupervisorProfile } from '../src/runtime/supervise/supervisor-agent' import type { SupervisedResult } from '../src/runtime/supervise/types' +import { + candidateBundle, + cleanupCandidateFixtures, + createCandidateOutputFixture, + redigestCandidateBundle, +} from './helpers/candidate-execution-fixture' +import { + cleanupCandidateExperimentFixtures, + createCandidateExperimentFixture, +} from './helpers/candidate-experiment-fixture' + +afterEach(() => { + cleanupCandidateExperimentFixtures() + cleanupCandidateFixtures() +}) async function withKb(fn: (root: string) => Promise): Promise { const root = await mkdtemp(join(tmpdir(), 'agent-runtime-knowledge-job-')) @@ -28,7 +54,13 @@ function winner(): SupervisedResult { out: { ok: true }, outRef: 'out:1', tree: { nodes: [] }, - spentTotal: { iterations: 2, tokens: { input: 30, output: 12 }, usd: 0.004, ms: 75 }, + spentTotal: { + iterations: 2, + tokens: { input: 30, output: 12 }, + usdKnown: false, + usd: 0.004, + ms: 75, + }, } as unknown as SupervisedResult } @@ -64,105 +96,317 @@ async function writeRuntimeJobPage(root: string): Promise { ) } +async function liveKnowledgeBytes(root: string): Promise> { + const paths = [join(root, 'knowledge'), join(root, 'raw')] + const sourceRegistry = join(root, '.agent-knowledge', 'sources.json') + const output: Record = {} + for (const path of paths) await collectFiles(root, path, output) + try { + output[relative(root, sourceRegistry)] = (await readFile(sourceRegistry)).toString('base64') + } catch (error) { + if (!isMissingFile(error)) throw error + } + return output +} + +async function collectFiles( + root: string, + path: string, + output: Record, +): Promise { + let info: Awaited> + try { + info = await stat(path) + } catch (error) { + if (isMissingFile(error)) return + throw error + } + if (info.isFile()) { + output[relative(root, path)] = (await readFile(path)).toString('base64') + return + } + for (const entry of (await readdir(path)).sort()) { + await collectFiles(root, join(path, entry), output) + } +} + +function isMissingFile(error: unknown): boolean { + return error instanceof Error && 'code' in error && error.code === 'ENOENT' +} + +const KNOWLEDGE_IMPROVEMENT_JOB_TEST_TIMEOUT_MS = 15_000 + describe('runKnowledgeImprovementJob', () => { - it('runs agent-knowledge improvement through a supervised runtime updater', async () => { - await withKb(async (root) => { - const measurements: unknown[] = [] - let captured: - | { - profile: SupervisorProfile - task: unknown - opts: SuperviseOptions - } - | undefined - - const result = await runKnowledgeImprovementJob({ - root, - goal: 'Add runtime job knowledge', - runId: 'runtime-job', - strict: true, - budget: { maxIterations: 2, maxTokens: 1000 }, - readinessCheck: async ({ root: candidateRoot }) => { - try { - const text = await readFile(join(candidateRoot, 'knowledge', 'runtime-job.md'), 'utf8') - return { ready: text.includes('candidate workspace') } - } catch { - return { ready: false, summary: 'runtime job page missing' } - } - }, - runSupervised: async (profile, task, opts) => { - captured = { profile, task, opts } - const candidateRoot = rootFromTask(task) - const page = join(candidateRoot, 'knowledge', 'runtime-job.md') - await mkdir(dirname(page), { recursive: true }) - await writeFile( - page, - [ - '---', - 'id: runtime-job', - 'title: Runtime Job', - '---', - '# Runtime Job', - 'The runtime job updates the candidate workspace.', - ].join('\n'), - 'utf8', - ) - await expect(opts.deliverable?.check({})).resolves.toBe(true) - return winner() - }, - onMeasurement: (measurement) => measurements.push(measurement), - }) + it( + 'leaves the live knowledge base byte-identical until approval', + async () => { + await withKb(async (root) => { + const measurements: unknown[] = [] + const before = await liveKnowledgeBytes(root) + let captured: + | { + profile: SupervisorProfile + task: unknown + opts: SuperviseOptions + } + | undefined + + const result = await runKnowledgeImprovementJob({ + root, + goal: 'Add runtime job knowledge', + runId: 'runtime-job', + strict: true, + budget: { maxIterations: 2, maxTokens: 1000 }, + readinessCheck: async ({ root: candidateRoot }) => { + try { + const text = await readFile( + join(candidateRoot, 'knowledge', 'runtime-job.md'), + 'utf8', + ) + return { ready: text.includes('candidate workspace') } + } catch { + return { ready: false, summary: 'runtime job page missing' } + } + }, + runSupervised: async (profile, task, opts) => { + captured = { profile, task, opts } + const candidateRoot = rootFromTask(task) + const page = join(candidateRoot, 'knowledge', 'runtime-job.md') + await mkdir(dirname(page), { recursive: true }) + await writeFile( + page, + [ + '---', + 'id: runtime-job', + 'title: Runtime Job', + '---', + '# Runtime Job', + 'The runtime job updates the candidate workspace.', + ].join('\n'), + 'utf8', + ) + await expect(opts.deliverable?.check({})).resolves.toBe(true) + return winner() + }, + onMeasurement: (measurement) => measurements.push(measurement), + }) - expect(result.promoted).toBe(true) - expect(result.improvement.state.status).toBe('promoted') - expect(captured?.profile.name).toBe('knowledge-research-supervisor') - expect(captured?.task).toContain('Goal: Add runtime job knowledge') - expect(captured?.task).toContain('Knowledge base root:') - await expect(readFile(join(root, 'knowledge', 'runtime-job.md'), 'utf8')).resolves.toContain( - 'candidate workspace', - ) - expect(result.measurement.updateCalls).toBe(1) - expect(result.measurement.supervisedSpent).toMatchObject({ - iterations: 2, - inputTokens: 30, - outputTokens: 12, - usd: 0.004, - ms: 75, + expect(result.promoted).toBe(false) + expect(result.improvement.state.status).toBe('candidate-ready') + expect(captured?.profile.name).toBe('knowledge-research-supervisor') + expect(captured?.task).toContain('Goal: Add runtime job knowledge') + expect(captured?.task).toContain('Knowledge base root:') + await expect( + readFile(join(root, 'knowledge', 'runtime-job.md'), 'utf8'), + ).rejects.toMatchObject({ + code: 'ENOENT', + }) + expect(await liveKnowledgeBytes(root)).toEqual(before) + expect(result.candidateKnowledge?.candidate.candidateId).toBe( + result.improvement.candidate?.candidateId, + ) + await withKnowledgeImprovementCandidate( + { root, candidate: knowledgeImprovementCandidateRef(result.improvement) }, + async ({ root: candidateRoot }) => { + expect(result.candidateKnowledge?.candidate.candidateHash).toBe( + `sha256:${await hashKnowledgeBase(candidateRoot)}`, + ) + }, + ) + expect(result.candidateKnowledge?.snapshot.material.files).toEqual( + expect.arrayContaining([expect.objectContaining({ path: 'knowledge/runtime-job.md' })]), + ) + expect(result.measurement.updateCalls).toBe(1) + expect(result.measurement.supervisedSpent).toMatchObject({ + iterations: 2, + inputTokens: 30, + outputTokens: 12, + usdKnown: false, + usd: 0.004, + ms: 75, + }) + expect(measurements).toHaveLength(1) }) - expect(measurements).toHaveLength(1) - }) - }) + }, + KNOWLEDGE_IMPROVEMENT_JOB_TEST_TIMEOUT_MS, + ) + + it( + 'uses the built-in agent-knowledge readiness check by default', + async () => { + await withKb(async (root) => { + const readinessSpec = defineReadinessSpec({ + id: 'runtime-job', + description: 'runtime knowledge improvement job is documented', + query: 'runtime knowledge improvement job candidate workspaces', + requiredFor: ['RuntimeJob'], + confidenceNeeded: 0.1, + minSources: 1, + minHits: 1, + }) + + const result = await runKnowledgeImprovementJob({ + root, + goal: 'Add runtime job knowledge', + runId: 'runtime-job-default-readiness', + strict: true, + readinessSpecs: [readinessSpec], + budget: { maxIterations: 2, maxTokens: 1000 }, + runSupervised: async (_profile, task, opts) => { + await writeRuntimeJobPage(rootFromTask(task)) + await expect(opts.deliverable?.check({})).resolves.toBe(true) + return winner() + }, + }) - it('uses the built-in agent-knowledge readiness check by default', async () => { - await withKb(async (root) => { - const readinessSpec = defineReadinessSpec({ - id: 'runtime-job', - description: 'runtime knowledge improvement job is documented', - query: 'runtime knowledge improvement job candidate workspaces', - requiredFor: ['RuntimeJob'], - confidenceNeeded: 0.1, - minSources: 1, - minHits: 1, + expect(result.promoted).toBe(false) + await withKnowledgeImprovementCandidate( + { root, candidate: knowledgeImprovementCandidateRef(result.improvement) }, + async ({ root: candidateRoot }) => { + await expect( + readFile(join(candidateRoot, 'knowledge', 'runtime-job.md'), 'utf8'), + ).resolves.toContain('source-backed evidence') + }, + ) + await expect( + readFile(join(root, 'knowledge', 'runtime-job.md'), 'utf8'), + ).rejects.toMatchObject({ + code: 'ENOENT', + }) }) + }, + KNOWLEDGE_IMPROVEMENT_JOB_TEST_TIMEOUT_MS, + ) - const result = await runKnowledgeImprovementJob({ - root, - goal: 'Add runtime job knowledge', - runId: 'runtime-job-default-readiness', - strict: true, - readinessSpecs: [readinessSpec], - budget: { maxIterations: 2, maxTokens: 1000 }, - runSupervised: async (_profile, task, opts) => { + it( + 'promotes only the frozen candidate bytes after an exact approved review', + async () => { + await withKb(async (root) => { + const artifacts = createCandidateOutputFixture().outputArtifacts + const update = async ( + _profile: SupervisorProfile, + task: unknown, + opts: SuperviseOptions, + ) => { await writeRuntimeJobPage(rootFromTask(task)) await expect(opts.deliverable?.check({})).resolves.toBe(true) return winner() - }, - }) + } + const proposed = await runKnowledgeImprovementJob({ + root, + goal: 'Add runtime job knowledge', + runId: 'runtime-job-approved', + strict: true, + budget: { maxIterations: 2, maxTokens: 1000 }, + readinessCheck: async ({ root: candidateRoot }) => ({ + ready: await readFile(join(candidateRoot, 'knowledge', 'runtime-job.md'), 'utf8') + .then((text) => text.includes('source-backed evidence')) + .catch(() => false), + }), + runSupervised: update, + candidateArtifacts: artifacts, + }) + const knowledge = proposed.candidateKnowledge + if (!knowledge) throw new Error('expected frozen knowledge candidate') + const candidateRef = knowledgeImprovementCandidateRef(proposed.improvement) + const candidateBytes = await withKnowledgeImprovementCandidate( + { root, candidate: candidateRef }, + ({ root: candidateRoot }) => liveKnowledgeBytes(candidateRoot), + ) + const liveBeforeApproval = await liveKnowledgeBytes(root) + const baseBundle = candidateBundle() + const bundle = redigestCandidateBundle(baseBundle, { knowledge }) + const rig = createCandidateExperimentFixture({ + baseline: baseBundle, + candidate: bundle, + configureFixture: (fixture) => { + fixture.ports.artifacts = artifacts + const materialize = fixture.ports.workspaces.materialize + const archivedWorkspaces = createAgentCandidateWorkspacePort() + fixture.ports.workspaces.materialize = async (input) => + input.role === 'knowledge' + ? archivedWorkspaces.materialize(input) + : materialize(input) + }, + }) + const measured = await runAgentCandidateExperiment({ + experiment: rig.experiment, + runId: 'runtime-job-approved', + placeCell: rig.placeCell, + }) + const proposal = createAgentImprovementProposal({ + runId: 'runtime-job-approved', + findings: [], + evaluation: measured.evaluation, + now: () => new Date('2026-07-13T01:00:00.000Z'), + }) + const review = reviewAgentImprovementProposal(proposal, { + decision: 'approve', + reviewedBy: 'operator@example.com', + reason: 'Approve the exact frozen knowledge candidate.', + now: () => new Date('2026-07-13T01:01:00.000Z'), + }) + const activation = createAgentImprovementActivation(proposal, review, { + targets: [ + { + surface: 'knowledge', + identity: root, + expectedBaseDigest: knowledge.candidate.baseHash, + }, + ], + fundingOwner: 'tenant/default', + authorizedBy: 'operator@example.com', + now: () => new Date('2026-07-13T01:02:00.000Z'), + }) - expect(result.promoted).toBe(true) - await expect(readFile(join(root, 'knowledge', 'runtime-job.md'), 'utf8')).resolves.toContain( - 'source-backed evidence', - ) - }) - }) + expect(await liveKnowledgeBytes(root)).toEqual(liveBeforeApproval) + const approvedUpdate = async () => { + throw new Error('candidate-ready promotion must not rerun the updater') + } + const runApproved = ( + approvedProposal: typeof proposal, + approvedReview: typeof review, + approvedActivation: AgentImprovementActivation, + ) => + runKnowledgeImprovementJob({ + root, + goal: 'Add runtime job knowledge', + runId: 'runtime-job-approved', + strict: true, + budget: { maxIterations: 2, maxTokens: 1000 }, + readinessCheck: async () => ({ ready: true }), + runSupervised: approvedUpdate, + candidateArtifacts: artifacts, + approval: { + proposal: approvedProposal, + review: approvedReview, + activation: approvedActivation, + authorizeActivation: async (candidateActivation) => + candidateActivation.digest === approvedActivation.digest, + }, + }) + const promoteApproved = () => runApproved(proposal, review, activation) + + await expect( + withKnowledgeImprovementCandidate( + { root, candidate: candidateRef }, + async ({ root: candidateRoot }) => { + const frozenPage = join(candidateRoot, 'knowledge', 'runtime-job.md') + await writeFile(frozenPage, 'tampered frozen snapshot', 'utf8') + }, + ), + ).rejects.toThrow(/snapshot changed during use/) + expect(await liveKnowledgeBytes(root)).toEqual(liveBeforeApproval) + + const promoted = await promoteApproved() + + expect(promoted.promoted).toBe(true) + expect(promoted.improvement.state.status).toBe('promoted') + expect(promoted.measurement.updateCalls).toBe(0) + expect(await liveKnowledgeBytes(root)).toEqual(candidateBytes) + expect(`sha256:${await hashKnowledgeBase(root)}`).toBe(knowledge.candidate.candidateHash) + }) + }, + KNOWLEDGE_IMPROVEMENT_JOB_TEST_TIMEOUT_MS, + ) }) diff --git a/tests/mcp/worktree-harness.test.ts b/tests/mcp/worktree-harness.test.ts index df2337a6..5f31b7b7 100644 --- a/tests/mcp/worktree-harness.test.ts +++ b/tests/mcp/worktree-harness.test.ts @@ -1,6 +1,7 @@ import { execFileSync, spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' import { + chmodSync, existsSync, mkdirSync, mkdtempSync, @@ -77,20 +78,17 @@ describe('runWorktreeHarness profile materialization', () => { } }) - it('delivers mounted inputs, captures the worker edit, and excludes tracked and untracked inputs', async () => { - const repoRoot = initializeRepository({ - 'src/value.ts': 'export const value = 1\n', - 'profile-tracked.txt': 'repository version\n', - }) + it('delivers mounted inputs, captures the worker edit, and excludes profile inputs', async () => { const runId = 'profile-real-path' const newInputPath = '.agent-profile/[literal]*:context.txt' - const trackedInputPath = 'profile-tracked.txt' const newInputMarker = 'PROFILE_NEW_INPUT_8f08e9f8' - const trackedInputMarker = 'PROFILE_TRACKED_INPUT_c521cd4d' const systemMarker = 'SYSTEM_PROMPT_2ef237df' const promptInstructionMarker = 'PROMPT_INSTRUCTION_6dcc8c4e' const resourceInstructionMarker = 'RESOURCE_INSTRUCTION_75b98d68' const taskMarker = 'TASK_PROMPT_d5ade1ac' + const repoRoot = initializeRepository({ + 'src/value.ts': 'export const value = 1\n', + }) const profile: AgentProfile = { model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, prompt: { @@ -103,14 +101,6 @@ describe('runWorktreeHarness profile materialization', () => { path: newInputPath, resource: { kind: 'inline', name: 'new-context', content: newInputMarker }, }, - { - path: trackedInputPath, - resource: { - kind: 'inline', - name: 'tracked-context', - content: trackedInputMarker, - }, - }, ], instructions: { kind: 'inline', @@ -131,7 +121,6 @@ describe('runWorktreeHarness profile materialization', () => { codexReproducible: true, runHarness: async (options: RunLocalHarnessOptions) => { expect(readFileSync(join(options.cwd, newInputPath), 'utf8')).toBe(newInputMarker) - expect(readFileSync(join(options.cwd, trackedInputPath), 'utf8')).toBe(trackedInputMarker) const prompt = options.invocation?.args[1] ?? '' expect(options.codexReproducible).toBe(true) expect(options.invocation?.args).toContain('project_doc_max_bytes=0') @@ -146,10 +135,6 @@ describe('runWorktreeHarness profile materialization', () => { writeFileSync(join(options.cwd, 'src/value.ts'), 'export const value = 2\n') writeFileSync(join(options.cwd, newInputPath), 'worker changed new profile input\n') - writeFileSync( - join(options.cwd, trackedInputPath), - 'worker changed tracked profile input\n', - ) return successfulHarnessResult() }, }) @@ -159,13 +144,11 @@ describe('runWorktreeHarness profile materialization', () => { expect(execution.out.patch).toContain('diff --git a/src/value.ts b/src/value.ts') expect(execution.out.patch).toContain('+export const value = 2') expect(execution.out.patch).not.toContain(newInputPath) - expect(execution.out.patch).not.toContain(trackedInputPath) expect(execution.out.patch).not.toContain(newInputMarker) - expect(execution.out.patch).not.toContain(trackedInputMarker) expect(execution.out.stats).toEqual({ filesChanged: 1, insertions: 1, deletions: 1 }) expect(execution.out.profileMaterialization).toMatchObject({ workspacePlanDigest: expect.stringMatching(/^sha256:[a-f0-9]{64}$/u), - writtenPaths: [newInputPath, trackedInputPath], + writtenPaths: [newInputPath], unsupported: [], environmentNames: [], flags: [], @@ -188,6 +171,51 @@ describe('runWorktreeHarness profile materialization', () => { } }) + it('rejects profile inputs that shadow repository files', async () => { + const repoRoot = initializeRepository({ + 'profile-tracked.txt': 'repository version\n', + 'src/value.ts': 'export const value = 1\n', + }) + const runId = 'profile-existing-file' + const runHarness = vi.fn() + chmodSync(join(repoRoot, 'profile-tracked.txt'), 0o755) + git(repoRoot, ['add', 'profile-tracked.txt']) + git(repoRoot, ['commit', '-q', '-m', 'make profile fixture executable']) + const previousUmask = process.umask(0o022) + try { + await expect( + runWorktreeHarness({ + repoRoot, + profile: { + resources: { + files: [ + { + path: 'profile-tracked.txt', + executable: true, + resource: { + kind: 'inline', + name: 'tracked-context', + content: 'repository version\n', + }, + }, + ], + }, + }, + harness: 'codex', + taskPrompt: 'task', + runId, + runHarness, + }), + ).rejects.toThrow(/Refusing to replace existing workspace file: profile-tracked\.txt/u) + expect(runHarness).not.toHaveBeenCalled() + expect(existsSync(join(repoRoot, '.agent-worktrees', runId))).toBe(false) + expect(git(repoRoot, ['branch', '--list', `delegate/${runId}`])).toBe('') + } finally { + process.umask(previousUmask) + rmSync(repoRoot, { recursive: true, force: true }) + } + }) + it('materializes every Claude-supported structural axis and delivers instructions once', async () => { const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) const runId = 'supported-structural-axes' @@ -198,8 +226,8 @@ describe('runWorktreeHarness profile materialization', () => { nativeAgent: '.claude/agents/native-reviewer.md', command: '.claude/commands/audit.md', subagent: '.claude/agents/helper.md', - mcp: '.mcp.json', - settings: '.claude/settings.json', + mcp: '.tangle/claude-mcp.json', + settings: '.tangle/claude-settings.json', } as const try { const run = await runWorktreeHarness({ @@ -276,8 +304,16 @@ describe('runWorktreeHarness profile materialization', () => { const settings = JSON.parse( readFileSync(join(options.cwd, paths.settings), 'utf8'), ) as Record - expect(settings).toHaveProperty('enabledMcpjsonServers') expect(settings).toHaveProperty('hooks') + expect(options.invocation?.args).toEqual( + expect.arrayContaining([ + '--mcp-config', + paths.mcp, + '--strict-mcp-config', + '--settings', + paths.settings, + ]), + ) const prompt = options.invocation?.args[1] ?? '' expect(count(prompt, resourceInstructionMarker)).toBe(1) expect(count(prompt, 'DIRECT_SYSTEM_a3563f03')).toBe(1) diff --git a/tests/sandbox-approved-candidate.test.ts b/tests/sandbox-approved-candidate.test.ts new file mode 100644 index 00000000..dba08b7e --- /dev/null +++ b/tests/sandbox-approved-candidate.test.ts @@ -0,0 +1,587 @@ +import { chmodSync, mkdirSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' + +import { InMemoryTraceStore } from '@tangle-network/agent-eval' +import { + type CandidateExperimentExecutionInput, + sealCandidateBenchmarkSuite, + sealCandidateExperiment, +} from '@tangle-network/agent-eval/contract' +import type { + AgentCandidateBundle, + AgentCandidateExperiment, +} from '@tangle-network/agent-interface' +import type { + CreateSandboxOptions, + Process, + ProcessStatus, + SandboxInstance, +} from '@tangle-network/sandbox' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { InMemoryAgentCandidateExecutionClaimStore } from '../src/candidate-execution/claim' +import { + canonicalCandidateBytes, + embeddedCandidateArtifact, +} from '../src/candidate-execution/digest' +import { + CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV, + CANDIDATE_KNOWLEDGE_ROOT_ENV, +} from '../src/candidate-execution/knowledge' +import type { AgentCandidateExecutorRequest } from '../src/candidate-execution/types' +import { + createSandboxCandidateExperimentExecutor, + sandboxCandidateExperimentExecutionSupport, +} from '../src/intelligence/sandbox-approved-candidate' +import { + candidateSha, + cleanupCandidateFixtures, + createCandidateOutputExecutionFixture, + createCandidateOutputFixture, + redigestCandidateBundle, + replaceCandidateFixtureTask, +} from './helpers/candidate-execution-fixture' + +afterEach(cleanupCandidateFixtures) + +describe('sandbox candidate experiment executor', () => { + it('runs one exact bounded-output cell with profile and knowledge bytes', async () => { + const fixture = createCandidateOutputExecutionFixture('application/json', 1_024) + const baseline = fixture.bundle + const knowledgeBytes = Buffer.from('# Exact runbook\nUse the verified endpoint.\n', 'utf8') + const knowledgeMaterial = { + kind: 'agent-candidate-workspace-manifest' as const, + files: [ + { + path: 'runbook.md', + mode: 0o644, + sha256: embeddedCandidateArtifact(knowledgeBytes).sha256, + byteLength: knowledgeBytes.byteLength, + }, + ], + } + const knowledgeManifest = embeddedCandidateArtifact(canonicalCandidateBytes(knowledgeMaterial)) + const retrievalConfigBytes = Buffer.from('{"topK":4}\n', 'utf8') + const candidate = redigestCandidateBundle(baseline, { + knowledge: { + candidate: { + kind: 'knowledge-improvement-candidate', + runId: 'knowledge-run-1', + candidateId: 'knowledge-candidate-1', + goalHash: candidateSha('1'), + baseHash: candidateSha('2'), + candidateHash: candidateSha('3'), + evidenceHash: candidateSha('4'), + promotionPlanHash: candidateSha('5'), + }, + snapshot: { + kind: 'agent-candidate-workspace-snapshot', + digest: knowledgeManifest.sha256, + material: knowledgeMaterial, + manifest: knowledgeManifest, + archive: embeddedCandidateArtifact(Buffer.from('knowledge-archive', 'utf8')), + }, + retrievalConfig: embeddedCandidateArtifact(retrievalConfigBytes), + evaluation: embeddedCandidateArtifact(Buffer.from('{"score":1}\n', 'utf8')), + }, + }) + const experiment = candidateExperiment(fixture, candidate) + const materialize = fixture.ports.workspaces.materialize + fixture.ports.workspaces.materialize = async (input) => { + if (input.role !== 'knowledge') return await materialize(input) + const path = join(input.destination, 'runbook.md') + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, knowledgeBytes) + chmodSync(path, 0o644) + } + const process = completedProcess('{"accepted":true}\n') + const writes: Array<{ path: string; content: string; mode: number }> = [] + let spawned = false + let deleted = false + const sandbox = { + id: 'sandbox-candidate-1', + metadata: undefined, + fs: { + write: vi.fn(async (path: string, content: string, options: { mode: number }) => { + writes.push({ path, content, mode: options.mode }) + }), + }, + process: { + spawnExact: vi.fn(async () => { + spawned = true + return process + }), + list: vi.fn(async () => (spawned ? [stoppedStatus(0)] : [])), + get: vi.fn(async () => process), + }, + delete: vi.fn(async () => { + deleted = true + }), + } as unknown as SandboxInstance + const create = vi.fn(async () => sandbox) + const outputs = createCandidateOutputFixture() + const adapter = createSandboxCandidateExperimentExecutor({ + client: { + create, + get: vi.fn(async () => (deleted ? null : sandbox)), + list: vi.fn(async () => (deleted ? [] : [sandbox])), + }, + ports: fixture.ports, + ...outputs, + traceStore: new InMemoryTraceStore(), + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + }) + + const input = candidateCell(experiment, fixture) + const evidence = await adapter.execute(input) + + expect(create).toHaveBeenCalledWith( + expect.objectContaining({ + image: `ghcr.io/example/task@sha256:${'b'.repeat(64)}`, + publicEdge: false, + ephemeral: true, + bare: true, + egressPolicy: { + mode: 'strict', + allowDomains: ['router.tangle.tools'], + includeImplicitDomains: false, + }, + metadata: expect.objectContaining({ + kind: 'agent-candidate-execution', + executionId: input.executionId, + executionPlanDigest: evidence.receipt.executionPlanDigest, + outputMediaType: 'application/json', + outputMaxBytes: 1_024, + }), + }), + expect.objectContaining({ signal: expect.any(AbortSignal), timeoutMs: 120_000 }), + ) + const spawnExact = sandbox.process.spawnExact as ReturnType + expect(spawnExact).toHaveBeenCalledOnce() + const [executable, args, launch] = spawnExact.mock.calls[0]! + expect(executable).toBe('codex') + expect(args).toEqual(expect.any(Array)) + expect(launch).toMatchObject({ + cwd: '/workspace/task', + stdin: fixture.task.task.instruction, + timeoutMs: fixture.task.task.limits.timeoutMs, + }) + expect(launch.env).not.toHaveProperty('PATH') + expect(launch.env).toHaveProperty('MODEL_GATEWAY_TOKEN', 'protected') + expect(launch.env).toMatchObject({ + [CANDIDATE_KNOWLEDGE_ROOT_ENV]: '/workspace/task/.tangle/knowledge', + [CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV]: + '/workspace/task/.tangle/knowledge-retrieval-config.json', + }) + expect(deleted).toBe(true) + expect(evidence).toMatchObject({ + materializationReceipt: { + profileActivation: { files: expect.any(Array) }, + }, + receipt: { + bundleDigest: candidate.digest, + executorCapture: expect.objectContaining({ sha256: expect.stringMatching(/^sha256:/) }), + taskOutcome: { + material: { + outcome: { + kind: 'output', + spec: { mediaType: 'application/json', maxBytes: 1_024 }, + }, + }, + }, + }, + }) + for (const file of evidence.materializationReceipt.profileActivation.files) { + const written = writes.find((entry) => entry.path === `/workspace/task/${file.path}`) + expect(written?.mode).toBe(file.mode) + expect(Buffer.from(written?.content ?? '', 'base64').toString('utf8')).toBe(file.content) + } + expect( + Buffer.from( + writes.find((entry) => entry.path.endsWith('/knowledge/runbook.md'))?.content ?? '', + 'base64', + ), + ).toEqual(knowledgeBytes) + expect( + Buffer.from( + writes.find((entry) => entry.path.endsWith('knowledge-retrieval-config.json'))?.content ?? + '', + 'base64', + ), + ).toEqual(retrievalConfigBytes) + }) + + it('declares and enforces its bounded capability', async () => { + expect(sandboxCandidateExperimentExecutionSupport).toEqual({ + outcomes: ['output'], + outputMediaTypes: ['text/*', 'application/json', '*+json'], + code: ['disabled'], + memory: ['disabled'], + knowledge: true, + profile: { + mcpTransports: ['stdio'], + remoteMcp: false, + tools: false, + permissions: false, + modes: false, + confidential: false, + }, + isolation: { + freshSandbox: true, + exactProcess: true, + egress: ['blocked', 'strict'], + }, + }) + const adapter = createSandboxCandidateExperimentExecutor({ + client: { + create: vi.fn(async () => { + throw new Error('unsupported requests must fail before sandbox creation') + }), + get: vi.fn(async () => null), + list: vi.fn(async () => []), + }, + ports: {} as never, + grader: {} as never, + outputArtifacts: {} as never, + traceStore: new InMemoryTraceStore(), + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + }) + const request = minimalExecutorRequest() + + await expect( + adapter.executor.execute( + withRequest(request, { taskOutcome: { kind: 'workspace' } }), + {} as never, + ), + ).rejects.toThrow(/workspace outcomes; use Pier/) + await expect( + adapter.executor.execute(withRequest(request, { codeKind: 'no-op' }), {} as never), + ).rejects.toThrow(/code workspaces; use Pier/) + await expect( + adapter.executor.execute(withRequest(request, { memoryMode: 'isolated' }), {} as never), + ).rejects.toThrow(/isolated memory/) + await expect( + adapter.executor.execute( + withRequest(request, { mediaType: 'application/octet-stream' }), + {} as never, + ), + ).rejects.toThrow(/only UTF-8 text and JSON/) + }) + + it('blocks all sandbox egress when the signed task disables model calls', async () => { + const fixture = createCandidateOutputExecutionFixture('application/json', 32) + replaceCandidateFixtureTask(fixture, { + limits: { ...fixture.task.task.limits, maxModelCalls: 0 }, + }) + const experiment = candidateExperiment(fixture) + const process = completedProcess('{"accepted":true}\n') + const { adapter, create } = sandboxAdapter(fixture, process, () => stoppedStatus(0)) + + await adapter.execute(candidateCell(experiment, fixture)) + + expect(create).toHaveBeenCalledWith( + expect.objectContaining({ egressPolicy: { mode: 'blocked' } }), + expect.any(Object), + ) + }) + + it('recovers bounded output from sandbox metadata in a fresh worker', async () => { + const executionPlanDigest = candidateSha('a') + const process = completedProcess('{"recovered":true}\n') + let deleted = false + const sandbox = { + id: 'sandbox-recovery', + metadata: { + kind: 'agent-candidate-execution', + executionId: 'recovery-execution', + executionPlanDigest, + outputMaxBytes: 64, + }, + process: { + list: vi.fn(async () => [stoppedStatus(0)]), + get: vi.fn(async () => process), + }, + delete: vi.fn(async () => { + deleted = true + }), + } as unknown as SandboxInstance + const adapter = createSandboxCandidateExperimentExecutor({ + client: { + create: vi.fn(async () => sandbox), + get: vi.fn(async () => (deleted ? null : sandbox)), + list: vi.fn(async () => (deleted ? [] : [sandbox])), + }, + ports: {} as never, + grader: {} as never, + outputArtifacts: {} as never, + traceStore: new InMemoryTraceStore(), + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + resultTimeoutMs: 100, + }) + const request = { executionId: 'recovery-execution', executionPlanDigest } + const signal = new AbortController().signal + + const capture = await adapter.executor.capture(request, { + traceStore: new InMemoryTraceStore(), + signal, + }) + await adapter.executor.dispose?.(request, { signal }) + + expect(Buffer.from(capture.taskOutcome?.bytes ?? []).toString('utf8')).toBe( + '{"recovered":true}\n', + ) + expect(deleted).toBe(true) + }) + + it('kills oversized output and deletes the failed sandbox', async () => { + const fixture = createCandidateOutputExecutionFixture('application/json', 4) + const experiment = candidateExperiment(fixture) + let running = true + let resolveWait!: (exitCode: number) => void + const wait = new Promise((resolve) => { + resolveWait = resolve + }) + const status = () => ({ ...stoppedStatus(137), running }) + const kill = vi.fn(async () => { + running = false + resolveWait(137) + }) + const process = { + wait: vi.fn(async () => wait), + status: vi.fn(async () => status()), + stdout: async function* () { + yield 'too-long' + }, + kill, + } as unknown as Process + const { adapter, deleted } = sandboxAdapter(fixture, process, status) + + await expect(adapter.execute(candidateCell(experiment, fixture))).rejects.toThrow( + /output exceeds/, + ) + expect(kill).toHaveBeenCalledWith('SIGKILL', { tree: true }) + expect(deleted()).toBe(true) + }) + + it('records timeout evidence and deletes the sandbox', async () => { + const fixture = createCandidateOutputExecutionFixture('application/json', 32) + replaceCandidateFixtureTask(fixture, { + limits: { ...fixture.task.task.limits, timeoutMs: 100 }, + }) + const experiment = candidateExperiment(fixture) + let running = true + let resolveWait!: (exitCode: number) => void + const wait = new Promise((resolve) => { + resolveWait = resolve + }) + const status = () => ({ ...stoppedStatus(137), running }) + const kill = vi.fn(async () => { + running = false + resolveWait(137) + }) + const process = { + wait: vi.fn(async () => wait), + status: vi.fn(async () => status()), + stdout: async function* () { + yield 'partial' + }, + kill, + } as unknown as Process + const { adapter, deleted } = sandboxAdapter(fixture, process, status) + + const evidence = await adapter.execute(candidateCell(experiment, fixture)) + expect(evidence.receipt.termination).toEqual({ kind: 'timeout', timeoutMs: 100 }) + expect(kill).toHaveBeenCalledWith('SIGKILL', { tree: true }) + expect(deleted()).toBe(true) + }) + + it('fails before durable success when sandbox deletion is not proven', async () => { + const fixture = createCandidateOutputExecutionFixture('application/json', 32) + const experiment = candidateExperiment(fixture) + const process = completedProcess('{"accepted":true}\n') + const status = () => stoppedStatus(0) + const { adapter, sandbox } = sandboxAdapter(fixture, process, status, true) + + await expect(adapter.execute(candidateCell(experiment, fixture))).rejects.toThrow( + /resource disposal|sandbox deletion failed/, + ) + expect(sandbox.delete).toHaveBeenCalledOnce() + }) +}) + +function candidateExperiment( + fixture: ReturnType, + candidate: AgentCandidateBundle = redigestCandidateBundle(fixture.bundle, { + profile: { + ...fixture.bundle.profile, + prompt: { ...fixture.bundle.profile.prompt, systemPrompt: 'Candidate prompt.' }, + }, + }), +): AgentCandidateExperiment { + return sealCandidateExperiment({ + kind: 'agent-candidate-experiment', + digestAlgorithm: 'rfc8785-sha256', + baseline: fixture.bundle, + candidate, + candidateLineage: { source: 'human' }, + benchmark: sealCandidateBenchmarkSuite({ + tasks: [fixture.task.task], + reps: 1, + seeds: [101], + }), + policy: { + confidenceLevel: 0.95, + resamples: 500, + bootstrapSeed: 1_337, + deltaThreshold: 0, + minProductiveRuns: 3, + budgetUsd: 1, + criticalDimensions: [], + regressionTolerance: 0.05, + }, + }) +} + +function candidateCell( + experiment: AgentCandidateExperiment, + fixture: ReturnType, +) { + const input: CandidateExperimentExecutionInput & { + executionId: string + executionRoots: typeof fixture.task.executionRoots + stagingRoots: typeof fixture.task.stagingRoots + } = { + experiment, + arm: 'candidate', + bundle: experiment.candidate, + task: experiment.benchmark.tasks[0]!, + benchmarkCell: { + suiteDigest: experiment.benchmark.suite.digest, + taskIndex: 0, + repetition: 0, + }, + seed: 101, + executionId: 'sandbox-execution-1', + executionRoots: fixture.task.executionRoots, + stagingRoots: fixture.task.stagingRoots, + } + return input +} + +function completedProcess(output: string): Process { + return { + wait: vi.fn(async () => 0), + status: vi.fn(async () => stoppedStatus(0)), + stdout: async function* () { + yield output + }, + kill: vi.fn(async () => undefined), + } as unknown as Process +} + +function stoppedStatus(exitCode: number): ProcessStatus { + return { + pid: 42, + command: 'codex', + cwd: '/workspace/task', + running: false, + exitCode, + startedAt: new Date('2026-07-13T12:02:00.000Z'), + exitedAt: new Date('2026-07-13T12:02:01.000Z'), + } +} + +function sandboxAdapter( + fixture: ReturnType, + process: Process, + currentStatus: () => ProcessStatus, + failDelete = false, +) { + let spawned = false + let wasDeleted = false + const sandbox = { + id: 'sandbox-candidate-test', + metadata: undefined, + fs: { write: vi.fn(async () => undefined) }, + process: { + spawnExact: vi.fn(async () => { + spawned = true + return process + }), + list: vi.fn(async () => (spawned ? [currentStatus()] : [])), + get: vi.fn(async () => process), + }, + delete: vi.fn(async () => { + if (failDelete) throw new Error('sandbox deletion failed') + wasDeleted = true + }), + } as unknown as SandboxInstance + const outputs = createCandidateOutputFixture() + const create = vi.fn(async (options: CreateSandboxOptions) => { + ;(sandbox as unknown as { metadata: Record }).metadata = options.metadata ?? {} + return sandbox + }) + const adapter = createSandboxCandidateExperimentExecutor({ + client: { + create, + get: vi.fn(async () => (wasDeleted ? null : sandbox)), + list: vi.fn(async () => (wasDeleted ? [] : [sandbox])), + }, + ports: fixture.ports, + ...outputs, + traceStore: new InMemoryTraceStore(), + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + }) + return { adapter, sandbox, create, deleted: () => wasDeleted } +} + +function minimalExecutorRequest(): AgentCandidateExecutorRequest { + return { + benchmark: { + task: { outcome: { kind: 'output', mediaType: 'text/plain', maxBytes: 10 } }, + }, + executionPlan: { value: { material: { codeKind: 'disabled' } } }, + inputs: {}, + memory: { mode: 'disabled' }, + } as unknown as AgentCandidateExecutorRequest +} + +function withRequest( + request: AgentCandidateExecutorRequest, + change: { + taskOutcome?: { kind: 'workspace' } + codeKind?: 'no-op' + memoryMode?: 'isolated' + mediaType?: string + }, +): AgentCandidateExecutorRequest { + const outcome = request.benchmark.task.outcome + return { + ...request, + benchmark: { + ...request.benchmark, + task: { + ...request.benchmark.task, + outcome: + change.taskOutcome ?? + (outcome.kind === 'output' + ? { ...outcome, mediaType: change.mediaType ?? outcome.mediaType } + : outcome), + }, + }, + executionPlan: { + ...request.executionPlan, + value: { + ...request.executionPlan.value, + material: { + ...request.executionPlan.value.material, + codeKind: change.codeKind ?? request.executionPlan.value.material.codeKind, + }, + }, + }, + memory: + change.memoryMode === 'isolated' + ? ({ mode: 'isolated' } as AgentCandidateExecutorRequest['memory']) + : request.memory, + } +} diff --git a/vitest.config.ts b/vitest.config.ts index af6d0d43..f5f6454c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,11 +3,16 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ resolve: { - // The package's own public subpath, resolvable from dynamically imported authored - // modules under test (the repo does not self-link in node_modules). - alias: { - '@tangle-network/agent-runtime/loops': resolve(__dirname, 'src/runtime/index.ts'), - }, + alias: [ + { + find: /^@tangle-network\/agent-runtime$/, + replacement: resolve(__dirname, 'src/index.ts'), + }, + { + find: /^@tangle-network\/agent-runtime\/loops$/, + replacement: resolve(__dirname, 'src/runtime/index.ts'), + }, + ], }, test: { exclude: ['**/node_modules/**', 'dist/**', 'bench/**', '**/.claude/worktrees/**'],