Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 2 additions & 46 deletions scripts/check-full-eval-pilots.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import json
import re
import stat
import subprocess
import sys
from pathlib import Path, PurePosixPath
from typing import Any, NoReturn, cast
Expand Down Expand Up @@ -726,43 +725,6 @@ def full_reject_host_paths(value: object, label: str) -> None:
full_reject_host_paths(item, f"{label}[{index}]")


def full_git_blob(commit: str, relative: str, field: str) -> bytes:
full_safe_relative(relative, field)
try:
tree = subprocess.run(
["git", "ls-tree", "-z", commit, "--", relative],
cwd=ROOT,
capture_output=True,
check=True,
).stdout
except (OSError, subprocess.CalledProcessError) as exc:
fail(f"{field}: cannot inspect immutable Git tree for {commit}: {exc}")
matches: list[tuple[str, str]] = []
for entry in (item for item in tree.split(b"\0") if item):
try:
metadata, path_bytes = entry.split(b"\t", 1)
mode, object_type, _object_id = metadata.split()
path = path_bytes.decode("utf-8")
except (UnicodeError, ValueError) as exc:
fail(f"{field}: malformed Git tree entry: {exc}")
if path == relative:
matches.append((mode.decode("ascii"), object_type.decode("ascii")))
if len(matches) != 1:
fail(f"{field}: {relative!r} must resolve to exactly one Git tree entry")
mode, object_type = matches[0]
if object_type != "blob" or mode == "120000":
fail(f"{field}: {relative!r} must be a non-symlink Git blob")
try:
return subprocess.run(
["git", "show", f"{commit}:{relative}"],
cwd=ROOT,
capture_output=True,
check=True,
).stdout
except (OSError, subprocess.CalledProcessError) as exc:
fail(f"{field}: cannot read immutable Git blob: {exc}")


def full_canonical_prompt(scenario_id: str, invocation_cwd: str) -> str:
return (
f"Pilot=reflect-repo-local-safety; Scenario={scenario_id}; Fixture CWD={invocation_cwd}; Plugin checkout=plugin-root. "
Expand Down Expand Up @@ -1291,12 +1253,9 @@ def full_validate_source(
if blobs != FULL_CONTRACT_BLOBS:
fail(f"{label}.plugin.contract_blobs must match the approved contract set")
for relative, expected_sha in FULL_CONTRACT_BLOBS.items():
immutable = full_git_blob(FULL_PLUGIN_COMMIT, relative, f"{label}.plugin.contract_blobs.{relative}")
if hashlib.sha256(immutable).hexdigest() != expected_sha:
fail(f"{label}.plugin contract blob changed at the recorded commit: {relative}")
_, live = full_repo_file(relative, f"{label}.plugin.live.{relative}")
if hashlib.sha256(live.read_bytes()).hexdigest() != expected_sha:
fail(f"{label}.plugin live blob differs from the recorded commit: {relative}")
fail(f"{label}.plugin live blob differs from the recorded evidence hash: {relative}")

command = source.get("command")
if not isinstance(command, dict):
Expand Down Expand Up @@ -1559,12 +1518,9 @@ def gap_validate_contract_blobs(value: object, field: str) -> None:
if value != GAP_CONTRACT_BLOB_LIST:
fail(f"{field} must match the approved ordered contract blob set")
for relative, expected_sha in GAP_CONTRACT_BLOBS.items():
immutable = full_git_blob(GAP_PLUGIN_COMMIT, relative, f"{field}.{relative}")
if hashlib.sha256(immutable).hexdigest() != expected_sha:
fail(f"{field}: immutable Git contract blob changed at {relative}")
_, live = full_repo_file(relative, f"{field}.live.{relative}")
if hashlib.sha256(live.read_bytes()).hexdigest() != expected_sha:
fail(f"{field}: live contract blob differs from the immutable Git blob at {relative}")
fail(f"{field}: live contract blob differs from the recorded evidence hash at {relative}")


def gap_validate_source_refs(
Expand Down
17 changes: 2 additions & 15 deletions scripts/check-operator-evidence-handoffs.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

import hashlib
import json
import subprocess
import sys
from pathlib import Path
from typing import Any, NoReturn
Expand Down Expand Up @@ -125,17 +124,6 @@ def exact_file_hash(path: str) -> str:
return hashlib.sha256(candidate.read_bytes()).hexdigest()


def immutable_blob(path: str) -> bytes:
completed = subprocess.run(
["git", "show", f"{PLUGIN_COMMIT}:{path}"],
cwd=ROOT,
capture_output=True,
check=False,
)
require(completed.returncode == 0, f"cannot read plugin blob at {PLUGIN_COMMIT}:{path}")
return completed.stdout


def reject_host_paths(value: object, field: str) -> None:
if isinstance(value, str):
for marker in ("/home/", "/Users/", "/tmp/", "/root/"):
Expand Down Expand Up @@ -173,8 +161,7 @@ def validate_plugin(result: dict[str, Any]) -> None:
require(actual == CONTRACT_BLOBS, "plugin.contract_blobs must match the recorded contract set")
for path, expected_sha in CONTRACT_BLOBS.items():
require(json_digest([path]) != "", "deterministic blob check is enabled")
require(digest(immutable_blob(path).decode("utf-8")) == expected_sha, f"immutable plugin blob changed: {path}")
require(exact_file_hash(path) == expected_sha, f"live plugin blob differs from recorded commit: {path}")
require(exact_file_hash(path) == expected_sha, f"live plugin blob differs from recorded evidence hash: {path}")

stale = plugin.get("stale_invalid_choice_contract_blobs")
require(isinstance(stale, list), "stale_invalid_choice_contract_blobs must be a list")
Expand All @@ -184,7 +171,7 @@ def validate_plugin(result: dict[str, Any]) -> None:
record = stale_map[path]
require(record.get("sha256") == CONTRACT_BLOBS[path], f"stale contract hash mismatch: {path}")
require(record.get("required_text") == snippet, f"stale invalid-choice text mismatch: {path}")
require(snippet in immutable_blob(path).decode("utf-8"), f"immutable stale invalid-choice text missing: {path}")
require(snippet in (ROOT / path).read_text(encoding="utf-8"), f"live stale invalid-choice text missing: {path}")


def validate_missing(result: dict[str, Any]) -> None:
Expand Down
2 changes: 1 addition & 1 deletion scripts/test-check-full-eval-pilots.py
Original file line number Diff line number Diff line change
Expand Up @@ -583,7 +583,7 @@ def gap_state_honor_status_mutation(result: dict[str, Any], checkout: Path) -> N
("GAP Git status mutation", gap_git_status_mutation, "git_before.status_porcelain must be an empty list for the read-only run"),
("GAP packet/write injection", gap_packet_write_injection, "fixture.fallback_writes must be empty"),
("GAP source path traversal", gap_source_path_traversal, "source_path must be a normalized POSIX path relative to the checkout"),
("GAP live contract blob mismatch", gap_live_contract_blob_mismatch, "live contract blob differs from the immutable Git blob"),
("GAP live contract blob mismatch", gap_live_contract_blob_mismatch, "live contract blob differs from the recorded evidence hash"),
("GAP malformed source ref", gap_malformed_source_ref, "gap_observation.source_refs[0] has malformed fields"),
("GAP unrecorded runtime-home delta", gap_runtime_home_unrecorded_delta, "runtime_home.changed_paths must match the complete before/after delta"),
("GAP false write-free claim", gap_false_write_free_claim, "session.write_free must be false when runtime_home.changed_paths is non-empty"),
Expand Down
Loading