feat(candidate): measure exact candidates before activation

.github/workflows/ci.yml

@@ -42,9 +42,6 @@ jobs:
 
   agent-bench:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: bench
     steps:
       - uses: actions/checkout@v4
 
@@ -54,16 +51,13 @@ jobs:
         with:
           node-version: 22
           cache: pnpm
-          cache-dependency-path: bench/pnpm-lock.yaml
+          cache-dependency-path: pnpm-lock.yaml
 
-      - name: Install published dependencies
+      - name: Install package set
         run: pnpm install --frozen-lockfile
 
-      - name: Typecheck public entrypoint against installed packages
-        run: pnpm run typecheck:public
-
-      - name: Test deterministic package paths
-        run: pnpm test
+      - name: Build current agent-runtime
+        run: pnpm run build
 
-      - name: Verify packed package in a clean consumer
-        run: pnpm run verify:package
+      - name: Verify agent-bench against current agent-runtime
+        run: pnpm run verify:bench

.github/workflows/publish.yml

@@ -40,6 +40,9 @@ jobs:
       - name: Verify packed package exports
         run: pnpm run verify:package
 
+      - name: Verify agent-bench against this release
+        run: pnpm run verify:bench
+
       - name: Verify tag/version lock
         run: |
           NPM_VERSION=$(node -p "require('./package.json').version")
@@ -98,9 +101,6 @@ jobs:
     # tag/version locking stays intact and the root package is never co-published.
     if: startsWith(github.ref, 'refs/tags/agent-bench-v')
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: bench
     steps:
       - uses: actions/checkout@v4
 
@@ -110,23 +110,17 @@ jobs:
         with:
           node-version: 22
           cache: pnpm
-          cache-dependency-path: bench/pnpm-lock.yaml
+          cache-dependency-path: pnpm-lock.yaml
 
       - name: Install deps
         run: pnpm install --frozen-lockfile
 
-      - name: Typecheck public entrypoint against installed packages
-        run: pnpm run typecheck:public
-
-      - name: Test deterministic package paths
-        run: pnpm test
-
-      - name: Verify packed package in a clean consumer
-        run: pnpm run verify:package
+      - name: Verify packed agent-bench against published dependencies
+        run: pnpm run verify:bench:published
 
       - name: Verify tag/version lock
         run: |
-          NPM_VERSION=$(node -p "require('./package.json').version")
+          NPM_VERSION=$(node -p "require('./bench/package.json').version")
           TAG_VERSION="${GITHUB_REF#refs/tags/agent-bench-v}"
           if [ "$TAG_VERSION" != "$NPM_VERSION" ]; then
             echo "::error::Tag/version mismatch: tag=$TAG_VERSION package=$NPM_VERSION."
@@ -141,9 +135,6 @@ jobs:
     permissions:
       contents: read
       id-token: write
-    defaults:
-      run:
-        working-directory: bench
     steps:
       - uses: actions/checkout@v4
 
@@ -153,7 +144,7 @@ jobs:
         with:
           node-version: 22
           cache: pnpm
-          cache-dependency-path: bench/pnpm-lock.yaml
+          cache-dependency-path: pnpm-lock.yaml
 
       - run: pnpm install --frozen-lockfile
 
@@ -162,10 +153,14 @@ jobs:
           # Requires the npmjs Trusted Publisher on @tangle-network/agent-bench:
           # org tangle-network, repo agent-runtime, workflow publish.yml.
           npm install -g npm@11
-          NAME=$(node -p "require('./package.json').name")
-          VERSION=$(node -p "require('./package.json').version")
+          NAME=$(node -p "require('./bench/package.json').name")
+          VERSION=$(node -p "require('./bench/package.json').version")
           if npm view "$NAME@$VERSION" version >/dev/null 2>&1; then
             echo "$NAME@$VERSION already on registry; skipping publish"
           else
-            npm publish --provenance --access public
+            mkdir -p "$RUNNER_TEMP/agent-bench-package"
+            pnpm --dir bench pack --pack-destination "$RUNNER_TEMP/agent-bench-package"
+            package="$(find "$RUNNER_TEMP/agent-bench-package" -maxdepth 1 -name '*.tgz' -print -quit)"
+            test -n "$package"
+            npm publish "$package" --provenance --access public
           fi

bench/package.json

@@ -24,23 +24,19 @@
     "test": "node scripts/run-package-tests.mjs",
     "typecheck:public": "tsc -p tsconfig.public.json",
     "verify:package": "node scripts/verify-packed-consumer.mjs",
+    "verify:package:local-runtime": "node scripts/verify-packed-consumer.mjs --local-runtime",
     "verify:pier": "tsx scripts/verify-pier-pair.mts"
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "^0.117.1",
-    "@tangle-network/agent-interface": "^0.25.0",
-    "@tangle-network/agent-runtime": "0.94.9",
-    "@tangle-network/sandbox": "^0.10.3"
+    "@tangle-network/agent-eval": "0.122.1",
+    "@tangle-network/agent-interface": "0.30.0",
+    "@tangle-network/agent-runtime": "workspace:*",
+    "@tangle-network/sandbox": "^0.11.1"
   },
   "devDependencies": {
-    "@types/node": "^25.0.0",
-    "tsx": "^4.19.0",
-    "typescript": "^6.0.3"
-  },
-  "pnpm": {
-    "overrides": {
-      "@tangle-network/agent-eval": "0.117.1"
-    }
+    "@types/node": "^25.9.3",
+    "tsx": "^4.22.4",
+    "typescript": "^5.7.0"
   },
   "files": [
     "src",

bench/pier_agents/candidate_contract.py

@@ -107,6 +107,17 @@ def _object(value: Any, label: str) -> dict[str, Any]:
     return value
 
 
+def _contract_object(
+    value: Any, label: str, *, kind: str | None = None
+) -> dict[str, Any]:
+    obj = _object(value, label)
+    if "schemaVersion" in obj or "version" in obj:
+        raise CandidateContractError(f"{label} contains obsolete version fields")
+    if kind is not None and obj.get("kind") != kind:
+        raise CandidateContractError(f"{label} has the wrong kind")
+    return obj
+
+
 def _string(value: Any, label: str) -> str:
     if not isinstance(value, str) or not value or "\0" in value:
         raise CandidateContractError(f"{label} must be a non-empty string without NUL")
@@ -236,26 +247,22 @@ def _embedded_bytes(value: Any, label: str) -> bytes:
 
 
 def _workspace_files(value: Any, label: str) -> tuple[WorkspaceFile, ...]:
-    snapshot = _object(value, label)
-    if (
-        snapshot.get("schemaVersion") != 1
-        or snapshot.get("kind") != "agent-candidate-workspace-snapshot"
-    ):
-        raise CandidateContractError(f"{label} has the wrong snapshot kind")
-    material = _object(snapshot.get("material"), f"{label}.material")
-    if (
-        material.get("schemaVersion") != 1
-        or material.get("kind") != "agent-candidate-workspace-manifest"
-    ):
-        raise CandidateContractError(f"{label}.material has the wrong manifest kind")
+    snapshot = _contract_object(
+        value, label, kind="agent-candidate-workspace-snapshot"
+    )
+    material = _contract_object(
+        snapshot.get("material"),
+        f"{label}.material",
+        kind="agent-candidate-workspace-manifest",
+    )
     values = material.get("files")
     if not isinstance(values, list):
         raise CandidateContractError(f"{label}.material.files must be an array")
     files: list[WorkspaceFile] = []
     for index, value in enumerate(values):
         obj = _object(value, f"{label}.material.files[{index}]")
         mode = _integer(obj.get("mode"), f"{label}.material.files[{index}].mode")
-        if mode not in {0o644, 0o755}:
+        if mode > 0o777:
             raise CandidateContractError(
                 f"{label} contains unsupported file mode {mode:o}"
             )
@@ -287,7 +294,7 @@ def _profile_files(value: Any) -> tuple[ProfileFile, ...]:
     for index, value in enumerate(values):
         obj = _object(value, f"profilePlan.material.files[{index}]")
         mode = _integer(obj.get("mode"), f"profilePlan.material.files[{index}].mode")
-        if mode not in {0o644, 0o755}:
+        if mode > 0o777:
             raise CandidateContractError(
                 f"profile plan contains unsupported mode {mode:o}"
             )
@@ -359,26 +366,23 @@ def load_prepared_candidate_contract(
         raise CandidateContractError(
             "materialization receipt bytes do not match the expected digest"
         )
-    if plan.get("schemaVersion") != 1 or plan.get("kind") != _PLAN_KIND:
-        raise CandidateContractError("execution plan has the wrong schema or kind")
-    if receipt.get("schemaVersion") != 1 or receipt.get("kind") != _RECEIPT_KIND:
-        raise CandidateContractError(
-            "materialization receipt has the wrong schema or kind"
-        )
+    plan = _contract_object(plan, "execution plan", kind=_PLAN_KIND)
+    receipt = _contract_object(
+        receipt, "materialization receipt", kind=_RECEIPT_KIND
+    )
     if receipt.get("digestAlgorithm") != "rfc8785-sha256" or "digest" in receipt:
         raise CandidateContractError(
             "materialization receipt must be exact digest-free canonical material"
         )
 
-    execution_evidence = _object(receipt.get("executionPlan"), "receipt.executionPlan")
+    execution_evidence = _contract_object(
+        receipt.get("executionPlan"),
+        "receipt.executionPlan",
+        kind=_EXECUTION_EVIDENCE_KIND,
+    )
     plan_digest = _digest(
         execution_evidence.get("digest"), "receipt.executionPlan.digest"
     )
-    if (
-        execution_evidence.get("schemaVersion") != 1
-        or execution_evidence.get("kind") != _EXECUTION_EVIDENCE_KIND
-    ):
-        raise CandidateContractError("receipt execution evidence has the wrong kind")
     if (
         sha256_bytes(plan_raw) != plan_digest
         or execution_evidence.get("material") != plan
@@ -402,11 +406,16 @@ def load_prepared_candidate_contract(
     execution_id = _string(plan.get("executionId"), "plan.executionId")
 
     task = _object(plan.get("task"), "plan.task")
+    outcome = _object(task.get("outcome"), "plan.task.outcome")
+    if outcome.get("kind") != "workspace":
+        raise CandidateContractError("Pier requires a workspace task outcome")
     repository = _object(task.get("repository"), "plan.task.repository")
     base_commit = _git_object(
         repository.get("baseCommit"), "plan.task.repository.baseCommit"
     )
-    base_tree = _git_object(repository.get("baseTree"), "plan.task.repository.baseTree")
+    base_tree = _git_object(
+        repository.get("baseTree"), "plan.task.repository.baseTree"
+    )
     if len(base_commit) != len(base_tree):
         raise CandidateContractError("task Git objects use different hash formats")
     instruction = _instruction(task.get("instruction"))
@@ -471,12 +480,11 @@ def load_prepared_candidate_contract(
     if receipt.get("candidateWorkspace") != candidate_snapshot:
         raise CandidateContractError("receipt and plan candidate workspaces differ")
 
-    profile_evidence = _object(receipt.get("profilePlan"), "receipt.profilePlan")
-    if (
-        profile_evidence.get("schemaVersion") != 1
-        or profile_evidence.get("kind") != _PROFILE_PLAN_KIND
-    ):
-        raise CandidateContractError("receipt profile evidence has the wrong kind")
+    profile_evidence = _contract_object(
+        receipt.get("profilePlan"),
+        "receipt.profilePlan",
+        kind=_PROFILE_PLAN_KIND,
+    )
     profile_digest = _digest(
         profile_evidence.get("digest"), "receipt.profilePlan.digest"
     )
@@ -489,7 +497,7 @@ def load_prepared_candidate_contract(
         profile_artifact_material = json.loads(profile_raw)
     except (UnicodeDecodeError, json.JSONDecodeError) as exc:
         raise CandidateContractError("profile artifact is not UTF-8 JSON") from exc
-    profile_material = _object(
+    profile_material = _contract_object(
         profile_evidence.get("material"), "receipt.profilePlan.material"
     )
     if profile_artifact_material != profile_material: