From d49d346aaad35286f775d63e917f8a1b42463d7b Mon Sep 17 00:00:00 2001 From: Zach Wentz <4832+zkwentz@users.noreply.github.com> Date: Sun, 12 Jul 2026 13:23:57 -0400 Subject: [PATCH] fix(validation): bind author reports to policy and source --- pyproject.toml | 7 +- src/openenv/validation/__init__.py | 7 +- src/openenv/validation/checks.py | 1 + src/openenv/validation/executor.py | 4 + src/openenv/validation/models.py | 45 +++++++- src/openenv/validation/planner.py | 32 +++++- src/openenv/validation/remote.py | 119 +++++++++++++++++++-- tests/test_validation/test_executor.py | 25 +++++ tests/test_validation/test_planner.py | 70 +++++++++++++ tests/test_validation/test_remote.py | 140 ++++++++++++++++++++----- 10 files changed, 408 insertions(+), 42 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8877c3b82..07c543f6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "typer>=0.9.0", "rich>=13.0.0", "pyyaml>=6.0", - "huggingface_hub>=0.20.0", + "huggingface_hub>=1.22.0", "openai>=2.7.2", "tomli>=2.3.0", "tomli-w>=1.2.0", @@ -55,9 +55,8 @@ modal = [ inspect = [ "inspect-ai>=0.3.0", ] -hf-sandbox = [ - "huggingface_hub>=1.22.0", -] +# Kept as a compatibility extra; Sandbox support is required by Hub pushes. +hf-sandbox = [] [project.scripts] openenv = "openenv.cli.__main__:main" diff --git a/src/openenv/validation/__init__.py b/src/openenv/validation/__init__.py index 190100149..b8c5edb12 100644 --- a/src/openenv/validation/__init__.py +++ b/src/openenv/validation/__init__.py @@ -27,7 +27,11 @@ OPENENV_VALIDATION_POLICY, VALIDATION_POLICY_VERSION, ) -from .remote import RemoteValidationError, run_remote_validation +from .remote import ( + RemoteValidationError, + run_remote_validation, + validation_source_digest, +) from .security import ensure_official_hf_sandbox __all__ = [ @@ -56,4 +60,5 @@ "format_shared_validation_report", "run_local_validation", "run_remote_validation", + "validation_source_digest", ] diff --git a/src/openenv/validation/checks.py b/src/openenv/validation/checks.py index 16e4a7e51..9584268d6 100644 --- a/src/openenv/validation/checks.py +++ b/src/openenv/validation/checks.py @@ -920,6 +920,7 @@ def get_openenv_checks() -> tuple[ValidationCheck, ...]: } ), profiles=_RUNTIME_PROFILES, + severity=ValidationSeverity.ADVISORY, timeout_s=600.0, built_in=False, requirement_binding=ValidationRequirementBinding.VERIFIER, diff --git a/src/openenv/validation/executor.py b/src/openenv/validation/executor.py index def7042b6..11b32daf9 100644 --- a/src/openenv/validation/executor.py +++ b/src/openenv/validation/executor.py @@ -195,6 +195,10 @@ def _execute_check( not diagnostics and check.built_in and outcome.status is not ValidationStatus.PASS + and not ( + check.severity is ValidationSeverity.ADVISORY + and outcome.status is ValidationStatus.SKIP + ) ): diagnostics = ( ValidationDiagnostic( diff --git a/src/openenv/validation/models.py b/src/openenv/validation/models.py index e78db50ee..6ed5d8041 100644 --- a/src/openenv/validation/models.py +++ b/src/openenv/validation/models.py @@ -16,6 +16,7 @@ _STABLE_IDENTIFIER = re.compile(r"^[a-z0-9][a-z0-9._-]{0,127}$") +_SHA256_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") _TRUSTED_METADATA_STRING_KEYS = frozenset( { "id", @@ -385,6 +386,8 @@ class ValidationPolicy: version: str supported_subjects: frozenset[tuple[str, ExecutionModel]] checks: tuple[ValidationCheck, ...] + supported_spec_versions: frozenset[tuple[str, str]] = frozenset() + supported_adapters: frozenset[tuple[str, str]] = frozenset() def __post_init__(self) -> None: if not _STABLE_IDENTIFIER.fullmatch(self.version): @@ -395,6 +398,20 @@ def __post_init__(self) -> None: for spec_id, model in self.supported_subjects ): raise ValueError("ValidationPolicy must declare supported subject pairs") + if any( + not _STABLE_IDENTIFIER.fullmatch(spec_id) + or not version.strip() + or len(version) > 128 + for spec_id, version in self.supported_spec_versions + ): + raise ValueError("ValidationPolicy contains invalid spec-version bindings") + if any( + not _STABLE_IDENTIFIER.fullmatch(adapter_id) + or not version.strip() + or len(version) > 128 + for adapter_id, version in self.supported_adapters + ): + raise ValueError("ValidationPolicy contains invalid adapter bindings") criterion_ids = [check.criterion_id for check in self.checks] if len(criterion_ids) != len(set(criterion_ids)): raise ValueError("ValidationPolicy contains duplicate criterion IDs") @@ -405,9 +422,27 @@ def supports(self, subject: ValidationSubject) -> bool: def supports_identity(self, identity: SpecIdentity) -> bool: """Return whether this policy applies to a detected spec identity.""" - return bool( - (identity.spec_id, identity.execution_model) in self.supported_subjects + subject_supported = ( + identity.spec_id, + identity.execution_model, + ) in self.supported_subjects + version_supported = ( + not self.supported_spec_versions + or ( + identity.spec_id, + identity.spec_version, + ) + in self.supported_spec_versions + ) + adapter_supported = ( + not self.supported_adapters + or ( + identity.adapter.adapter_id, + identity.adapter.adapter_version, + ) + in self.supported_adapters ) + return bool(subject_supported and version_supported and adapter_supported) @dataclass @@ -573,6 +608,7 @@ class ValidationReport: spec: ValidationSubject | None = None spec_identity: SpecIdentity | None = None repo_sha: str | None = None + source_digest: str | None = None image_digest: str | None = None certified: bool = False certification_eligible: bool = False @@ -589,6 +625,10 @@ def __post_init__(self) -> None: and self.spec_identity != self.spec.spec ): raise ValueError("ValidationReport spec identity must match its subject") + if self.source_digest is not None and not _SHA256_DIGEST.fullmatch( + self.source_digest + ): + raise ValueError("ValidationReport source digest must be lowercase SHA-256") @property def passed(self) -> bool: @@ -678,6 +718,7 @@ def to_dict(self) -> dict[str, Any]: "certified": self.certified, "certification_eligible": self.certification_eligible, "repo_sha": self.repo_sha, + "source_digest": self.source_digest, "image_digest": self.image_digest, "started_at": self.started_at, "finished_at": self.finished_at, diff --git a/src/openenv/validation/planner.py b/src/openenv/validation/planner.py index a296f0f2c..4c5892661 100644 --- a/src/openenv/validation/planner.py +++ b/src/openenv/validation/planner.py @@ -39,12 +39,32 @@ version=VALIDATION_POLICY_VERSION, supported_subjects=frozenset({("openenv", ExecutionModel.SERVED)}), checks=get_openenv_checks(), + supported_spec_versions=frozenset({("openenv", "1")}), + supported_adapters=frozenset({("openenv-yaml", "1")}), ) _POLICY_ATTESTATION = object() def _repo_sha(path: Path) -> str | None: try: + status = subprocess.run( + [ + "git", + "-C", + str(path), + "status", + "--porcelain", + "--untracked-files=all", + "--", + ".", + ], + check=False, + capture_output=True, + text=True, + timeout=3.0, + ) + if status.returncode != 0 or status.stdout.strip(): + return None completed = subprocess.run( ["git", "-C", str(path), "rev-parse", "HEAD"], check=False, @@ -143,7 +163,17 @@ def _policy_for_subject( ) -> tuple[ValidationPolicy, bool]: selected = policy or OPENENV_VALIDATION_POLICY identity = spec_load.spec - if identity is not None and not selected.supports_identity(identity): + incompatible_subject = bool( + identity is not None + and (identity.spec_id, identity.execution_model) + not in selected.supported_subjects + ) + incompatible_loaded_version = bool( + spec_load.subject is not None + and identity is not None + and not selected.supports_identity(identity) + ) + if identity is not None and (incompatible_subject or incompatible_loaded_version): raise ValueError( f"Validation policy {selected.version!r} does not support " f"spec {identity.spec_id!r} with execution model " diff --git a/src/openenv/validation/remote.py b/src/openenv/validation/remote.py index f9f65035e..c81aa4fbc 100644 --- a/src/openenv/validation/remote.py +++ b/src/openenv/validation/remote.py @@ -4,6 +4,7 @@ from __future__ import annotations +import hashlib import json import re import tarfile @@ -27,6 +28,7 @@ ValidationSeverity, ValidationStatus, ) +from .planner import build_validation_plan, VALIDATION_POLICY_VERSION from .specs import ( AdapterIdentity, DetectionMode, @@ -67,7 +69,14 @@ ".netrc", ".npmrc", ".pypirc", + "credentials", "credentials.json", + "id_dsa", + "id_ed25519", + "id_ecdsa", + "id_rsa", + "secrets.json", + "secrets.toml", "validation-report.json", } ) @@ -78,16 +87,58 @@ class RemoteValidationError(RuntimeError): def _archive_path_allowed(relative: Path) -> bool: - if any(part in _EXCLUDED_PARTS for part in relative.parts): + if any(part in _EXCLUDED_PARTS or part.startswith(".") for part in relative.parts): return False name = relative.name - if name in _EXCLUDED_NAMES or name == ".env" or name.startswith(".env."): + if name in _EXCLUDED_NAMES or relative.suffix.lower() in { + ".key", + ".p12", + ".pem", + ".pfx", + }: return False if relative.as_posix() == ".openenv/validation-report.json": return False return True +def _validate_snapshot_tree(root: Path) -> None: + for candidate in root.rglob("*"): + if candidate.is_symlink(): + raise RemoteValidationError( + "Remote validation source archives cannot contain symbolic links" + ) + + +def validation_source_digest(root: str | Path) -> str: + """Hash the exact regular source files included in a validation snapshot.""" + root_path = Path(root).resolve() + if not root_path.is_dir(): + raise RemoteValidationError("Validation source digest requires a directory") + _validate_snapshot_tree(root_path) + digest = hashlib.sha256() + for candidate in sorted(root_path.rglob("*")): + relative = candidate.relative_to(root_path) + if not _archive_path_allowed(relative) or not candidate.is_file(): + continue + encoded_path = relative.as_posix().encode("utf-8") + digest.update(len(encoded_path).to_bytes(8, "big")) + digest.update(encoded_path) + with candidate.open("rb") as source_file: + while chunk := source_file.read(1024 * 1024): + digest.update(len(chunk).to_bytes(8, "big")) + digest.update(chunk) + digest.update((0).to_bytes(8, "big")) + return f"sha256:{digest.hexdigest()}" + + +def _snapshot_layout(source: Path) -> tuple[Path, str]: + parent_manifest = source.parent / "task.toml" + if source.name == "environment" and parent_manifest.is_file(): + return source.parent, "environment" + return source, "." + + def _add_tree(archive: tarfile.TarFile, root: Path, *, prefix: Path) -> None: def normalize(info: tarfile.TarInfo) -> tarfile.TarInfo: info.uid = 0 @@ -114,6 +165,7 @@ def normalize(info: tarfile.TarInfo) -> tarfile.TarInfo: def _create_revision_archive(source: Path, destination: Path) -> None: + _validate_snapshot_tree(source) with tarfile.open(destination, mode="w:gz", format=tarfile.PAX_FORMAT) as archive: _add_tree(archive, source, prefix=Path(".")) @@ -184,7 +236,10 @@ def _create_validator_archive(destination: Path) -> None: def _remote_command( - *, profile: ValidationProfile, runtime_timeout_s: float + *, + profile: ValidationProfile, + runtime_timeout_s: float, + target_relative: str, ) -> list[str]: script = r""" set -eu @@ -193,7 +248,9 @@ def _remote_command( report_path="$3" profile="$4" runtime_timeout="$5" -environment_root=/workspace/environment +target_relative="$6" +environment_root=/workspace/source +validation_target="$environment_root/$target_relative" validator_root=/workspace/validator trusted_root=/workspace/trusted subject_root=/workspace/subject @@ -212,13 +269,13 @@ def _remote_command( chown -R "$subject_uid:$subject_gid" "$subject_root" || exit 70 setpriv --reuid="$subject_uid" --regid="$subject_gid" --clear-groups \ env HOME="$subject_home" python -m pip install --disable-pip-version-check \ - --no-input --target "$subject_site" "$environment_root" || exit 71 + --no-input --target "$subject_site" "$validation_target" || exit 71 set +e OPENENV_VALIDATION_SUBJECT_UID="$subject_uid" \ OPENENV_VALIDATION_SUBJECT_GID="$subject_gid" \ PYTHONPATH="$validator_root/src:$subject_site" \ python -m openenv.cli.__main__ validate \ - "$environment_root" --profile "$profile" --json --output "$report_path" \ + "$validation_target" --profile "$profile" --json --output "$report_path" \ --timeout "$runtime_timeout" validation_status=$? set -e @@ -235,6 +292,7 @@ def _remote_command( _REMOTE_REPORT, profile.value, f"{runtime_timeout_s:g}", + target_relative, ] @@ -475,6 +533,7 @@ def _parse_report( spec=subject, spec_identity=identity, repo_sha=document.get("repo_sha"), + source_digest=document.get("source_digest"), image_digest=document.get("image_digest"), certified=document.get("certified") is True, certification_eligible=document.get("certification_eligible") is True, @@ -485,6 +544,43 @@ def _parse_report( return report +def _validate_report_contract( + report: ValidationReport, + *, + source: Path, + expected_profile: ValidationProfile, +) -> None: + capabilities = RunnerCapabilities( + runner="hf-sandbox", + available=frozenset( + {ValidationCapability.SOURCE, ValidationCapability.RUNTIME} + ), + official=False, + isolation_mode="dedicated", + ) + expected_plan, _ = build_validation_plan( + source, + profile=expected_profile, + capabilities=capabilities, + ) + if report.policy_version != VALIDATION_POLICY_VERSION: + raise ValueError("report policy version is not the initiating policy") + if report.spec_identity != expected_plan.spec_identity: + raise ValueError("report spec identity does not match the source snapshot") + if len(report.results) != len(expected_plan.checks): + raise ValueError("report criteria do not match the initiating plan") + for result, check in zip(report.results, expected_plan.checks, strict=True): + if ( + result.criterion_id != check.criterion_id + or result.requirement != check.requirement + or result.severity is not check.severity + or result.required_capabilities != check.capabilities + or result.built_in is not check.built_in + or result.timeout_s != check.timeout_s + ): + raise ValueError("report criterion metadata does not match the policy") + + def run_remote_validation( source: str | Path, *, @@ -512,6 +608,8 @@ def run_remote_validation( raise RemoteValidationError("Remote validation timeouts must be positive") if not isinstance(flavor, str) or not flavor.strip(): raise RemoteValidationError("Remote validation hardware flavor is invalid") + snapshot_root, target_relative = _snapshot_layout(source_path) + source_digest = validation_source_digest(snapshot_root) try: from huggingface_hub import Sandbox @@ -526,7 +624,7 @@ def run_remote_validation( revision_archive = temporary_path / "revision.tar.gz" validator_archive = temporary_path / "validator.tar.gz" downloaded_report = temporary_path / "validation-report.json" - _create_revision_archive(source_path, revision_archive) + _create_revision_archive(snapshot_root, revision_archive) _create_validator_archive(validator_archive) try: @@ -543,6 +641,7 @@ def run_remote_validation( _remote_command( profile=selected_profile, runtime_timeout_s=runtime_timeout_s, + target_relative=target_relative, ), shell=False, timeout=sandbox_timeout_s, @@ -556,6 +655,11 @@ def run_remote_validation( try: payload = json.loads(downloaded_report.read_text(encoding="utf-8")) report = _parse_report(payload, expected_profile=selected_profile) + _validate_report_contract( + report, + source=source_path, + expected_profile=selected_profile, + ) except (OSError, ValueError, TypeError, json.JSONDecodeError) as exc: raise RemoteValidationError( "Remote validation report is invalid or malformed" @@ -580,6 +684,7 @@ def run_remote_validation( isolation_mode="dedicated", ), repo_sha=repo_sha or report.repo_sha, + source_digest=source_digest, certified=False, certification_eligible=False, ) diff --git a/tests/test_validation/test_executor.py b/tests/test_validation/test_executor.py index 34a52e565..3155b87fb 100644 --- a/tests/test_validation/test_executor.py +++ b/tests/test_validation/test_executor.py @@ -118,6 +118,31 @@ def test_existing_profiles_keep_blocking_skips_nonfatal(tmp_path: Path) -> None: assert report.passed is True +def test_advisory_skip_does_not_suggest_environment_remediation( + tmp_path: Path, +) -> None: + check = ValidationCheck( + criterion_id="test.optional", + requirement="An optional declaration", + capabilities=frozenset(), + severity=ValidationSeverity.ADVISORY, + timeout_s=1.0, + evaluator=lambda _context: CheckOutcome.skip(message="Not declared"), + ) + plan = ValidationPlan( + target=str(tmp_path), + profile=ValidationProfile.STATIC, + policy_version="test-policy", + capabilities=RunnerCapabilities(runner="local"), + checks=(check,), + ) + context = ValidationContext(target=tmp_path, spec_load=_absent_spec()) + + report = execute_validation_plan(plan, context) + + assert report.results[0].diagnostics == () + + def test_echo_env_local_and_remote_executors_share_criterion_results() -> None: env_dir = Path(__file__).resolve().parents[2] / "envs" / "echo_env" local_plan, local_context = build_validation_plan( diff --git a/tests/test_validation/test_planner.py b/tests/test_validation/test_planner.py index 68f4db100..c2eeff600 100644 --- a/tests/test_validation/test_planner.py +++ b/tests/test_validation/test_planner.py @@ -6,6 +6,7 @@ import hashlib import json +import subprocess from dataclasses import replace from pathlib import Path @@ -42,6 +43,38 @@ from ._helpers import write_harbor_task, write_valid_env +def test_plan_only_reports_repo_sha_for_clean_source(tmp_path: Path) -> None: + env_dir = tmp_path / "env" + write_valid_env(env_dir) + subprocess.run(["git", "init", str(tmp_path)], check=True, capture_output=True) + subprocess.run( + ["git", "-C", str(tmp_path), "add", "."], check=True, capture_output=True + ) + subprocess.run( + [ + "git", + "-C", + str(tmp_path), + "-c", + "user.name=OpenEnv Tests", + "-c", + "user.email=tests@openenv.invalid", + "commit", + "-m", + "fixture", + ], + check=True, + capture_output=True, + ) + + _, clean_context = build_validation_plan(env_dir) + assert clean_context.repo_sha is not None + + (env_dir / "uncommitted.txt").write_text("dirty\n") + _, dirty_context = build_validation_plan(env_dir) + assert dirty_context.repo_sha is None + + def test_plan_carries_sanitized_harbor_execution_requirements(tmp_path: Path) -> None: env_dir = tmp_path / "test_env" write_valid_env(env_dir) @@ -94,6 +127,32 @@ def test_plan_carries_sanitized_harbor_execution_requirements(tmp_path: Path) -> assert payload["spec"]["requirements"]["document_digest"].startswith("sha256:") +@pytest.mark.parametrize( + ("spec_versions", "adapters"), + [ + (frozenset({("openenv", "2")}), frozenset({("openenv-yaml", "1")})), + (frozenset({("openenv", "1")}), frozenset({("openenv-yaml", "2")})), + ], +) +def test_policy_binding_rejects_unpinned_spec_or_adapter_versions( + tmp_path: Path, + spec_versions: frozenset[tuple[str, str]], + adapters: frozenset[tuple[str, str]], +) -> None: + env_dir = tmp_path / "env" + write_valid_env(env_dir) + policy = ValidationPolicy( + version="pinned-policy", + supported_subjects=frozenset({("openenv", ExecutionModel.SERVED)}), + checks=(), + supported_spec_versions=spec_versions, + supported_adapters=adapters, + ) + + with pytest.raises(ValueError, match="does not support spec"): + build_validation_plan(env_dir, policy=policy) + + def test_plan_includes_harbor_step_requirements_and_sibling_verifier( tmp_path: Path, ) -> None: @@ -115,6 +174,17 @@ def test_plan_includes_harbor_step_requirements_and_sibling_verifier( "/workspace/result.json" ) + publish_plan, _ = build_validation_plan( + environment, + profile=ValidationProfile.PUBLISH, + ) + verifier = next( + check + for check in publish_plan.checks + if check.criterion_id == "custom.verifier" + ) + assert verifier.severity is ValidationSeverity.ADVISORY + class _OneShotAdapter: spec_id = "one-shot-test" diff --git a/tests/test_validation/test_remote.py b/tests/test_validation/test_remote.py index 005c5e2ae..6fdd0f2f6 100644 --- a/tests/test_validation/test_remote.py +++ b/tests/test_validation/test_remote.py @@ -8,20 +8,19 @@ import io import json import tarfile +from dataclasses import replace from pathlib import Path from types import ModuleType, SimpleNamespace from unittest.mock import MagicMock, patch import pytest from huggingface_hub import Sandbox, SandboxPool +from openenv.validation import build_validation_plan, execute_validation_plan from openenv.validation.models import ( RunnerCapabilities, ValidationCapability, ValidationProfile, ValidationReport, - ValidationResult, - ValidationSeverity, - ValidationStatus, ) from ._helpers import write_valid_env @@ -50,32 +49,23 @@ def _remote_module() -> ModuleType: def _report_payload(target: Path) -> dict[str, object]: - result = ValidationResult( - criterion_id="source.validation_spec", - requirement="Detect the validation spec", - status=ValidationStatus.PASS, - severity=ValidationSeverity.BLOCKING, - evidence={"state": "loaded"}, - duration_s=0.01, - timeout_s=10.0, - required_capabilities=frozenset({ValidationCapability.SOURCE}), + capabilities = RunnerCapabilities( + runner="hf-sandbox", + available=frozenset( + {ValidationCapability.SOURCE, ValidationCapability.RUNTIME} + ), + official=False, + isolation_mode="dedicated", ) - report = ValidationReport( - target=str(target), + plan, context = build_validation_plan( + target, profile=ValidationProfile.PUBLISH, - policy_version="rfc008-v1", - runner=RunnerCapabilities( - runner="hf-sandbox", - available=frozenset( - {ValidationCapability.SOURCE, ValidationCapability.RUNTIME} - ), - official=False, - isolation_mode="dedicated", - ), - results=(result,), - duration_s=0.1, - started_at="2026-07-12T12:00:00+00:00", - finished_at="2026-07-12T12:00:00.100000+00:00", + capabilities=capabilities, + ) + report = execute_validation_plan(plan, context) + report = replace( + report, + runner=capabilities, repo_sha=_REVISION, certified=False, certification_eligible=False, @@ -124,6 +114,9 @@ def test_remote_validation_uses_dedicated_sandbox_and_returns_report( marker = source / "revision-marker.txt" marker.write_text("exact revision contents\n") (source / ".env").write_text("HF_TOKEN=hf_do_not_upload\n") + (source / ".envrc").write_text("export TOKEN=do-not-upload\n") + (source / ".netrc").write_text("password do-not-upload\n") + (source / "credentials.json").write_text('{"token": "do-not-upload"}\n') (source / ".git").mkdir() (source / ".git" / "config").write_text("credential = do-not-upload\n") (source / ".openenv").mkdir() @@ -176,6 +169,9 @@ def write_download(_remote_path: str, local_path: str | Path) -> None: assert not any(name.endswith(".env") for name in source_members) assert not any(".git" in Path(name).parts for name in source_members) assert not any(name.endswith("validation-report.json") for name in source_members) + assert not any(name.endswith(".envrc") for name in source_members) + assert not any(name.endswith(".netrc") for name in source_members) + assert not any(name.endswith("credentials.json") for name in source_members) validator_source = Path(__file__).parents[2] / "src" / "openenv" / "validation" assert ( _archive_member(validator_upload, "openenv/validation/models.py") @@ -203,10 +199,66 @@ def write_download(_remote_path: str, local_path: str | Path) -> None: assert report.certified is False assert report.certification_eligible is False assert report.results[0].criterion_id == "source.validation_spec" + assert report.source_digest is not None + assert report.source_digest.startswith("sha256:") sandbox.kill.assert_called_once_with() sandbox.close.assert_called_once_with() +def test_remote_validation_preserves_parent_harbor_composition( + tmp_path: Path, +) -> None: + remote = _remote_module() + task_root = tmp_path / "task" + source = task_root / "environment" + write_valid_env(source) + (task_root / "task.toml").write_text('schema_version = "1.1"\n') + (task_root / "tests").mkdir() + (task_root / "tests" / "test.sh").write_text("#!/bin/sh\nexit 0\n") + sandbox = _sandbox() + uploads: dict[str, bytes] = {} + + def capture_upload(local_path: str | Path, remote_path: str, **_: object) -> None: + uploads[remote_path] = Path(local_path).read_bytes() + + def write_download(_remote_path: str, local_path: str | Path) -> None: + Path(local_path).write_text(json.dumps(_report_payload(source))) + + sandbox.files.upload.side_effect = capture_upload + sandbox.files.download.side_effect = write_download + with patch.object(Sandbox, "create", return_value=sandbox): + report = remote.run_remote_validation( + source, + profile=ValidationProfile.PUBLISH, + repo_sha=_REVISION, + sandbox_image=_SANDBOX_IMAGE, + ) + + revision = next(payload for path, payload in uploads.items() if "revision" in path) + members = _archive_names(revision) + assert "task.toml" in members + assert "tests/test.sh" in members + command = sandbox.run.call_args.args[0] + assert command[-1] == "environment" + assert report.spec is not None + assert report.spec.requirements.state.value == "loaded" + + +def test_remote_validation_rejects_source_symlinks(tmp_path: Path) -> None: + remote = _remote_module() + source = tmp_path / "environment" + write_valid_env(source) + outside = tmp_path / "outside.txt" + outside.write_text("secret\n") + (source / "linked-secret.txt").symlink_to(outside) + + with patch.object(Sandbox, "create") as create: + with pytest.raises(remote.RemoteValidationError, match="symbolic links"): + remote.run_remote_validation(source) + + create.assert_not_called() + + def test_remote_validation_command_failure_is_clean_and_closes_sandbox( tmp_path: Path, ) -> None: @@ -270,3 +322,37 @@ def write_download(_remote_path: str, local_path: str | Path) -> None: assert "not-json" not in message sandbox.kill.assert_called_once_with() sandbox.close.assert_called_once_with() + + +@pytest.mark.parametrize("malformation", ["empty_criteria", "wrong_policy"]) +def test_remote_validation_rejects_structurally_forged_report( + tmp_path: Path, + malformation: str, +) -> None: + remote = _remote_module() + source = tmp_path / "environment" + write_valid_env(source) + sandbox = _sandbox() + payload = _report_payload(source) + if malformation == "empty_criteria": + payload["criteria"] = [] + payload["passed"] = True + payload["status"] = "pass" + else: + payload["policy_version"] = "attacker-policy" + + def write_download(_remote_path: str, local_path: str | Path) -> None: + Path(local_path).write_text(json.dumps(payload)) + + sandbox.files.download.side_effect = write_download + with patch.object(Sandbox, "create", return_value=sandbox): + with pytest.raises(remote.RemoteValidationError, match="report"): + remote.run_remote_validation( + source, + profile=ValidationProfile.PUBLISH, + repo_sha=_REVISION, + sandbox_image=_SANDBOX_IMAGE, + ) + + sandbox.kill.assert_called_once_with() + sandbox.close.assert_called_once_with()