Validation: bind author reports to policy and source

pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
     "typer>=0.9.0",
     "rich>=13.0.0",
     "pyyaml>=6.0",
-    "huggingface_hub>=0.20.0",
+    "huggingface_hub>=1.22.0",
     "openai>=2.7.2",
     "tomli>=2.3.0",
     "tomli-w>=1.2.0",
@@ -55,9 +55,8 @@ modal = [
 inspect = [
     "inspect-ai>=0.3.0",
 ]
-hf-sandbox = [
-    "huggingface_hub>=1.22.0",
-]
+# Kept as a compatibility extra; Sandbox support is required by Hub pushes.
+hf-sandbox = []
 
 [project.scripts]
 openenv = "openenv.cli.__main__:main"

src/openenv/validation/__init__.py

@@ -27,7 +27,11 @@
     OPENENV_VALIDATION_POLICY,
     VALIDATION_POLICY_VERSION,
 )
-from .remote import RemoteValidationError, run_remote_validation
+from .remote import (
+    RemoteValidationError,
+    run_remote_validation,
+    validation_source_digest,
+)
 from .security import ensure_official_hf_sandbox
 
 __all__ = [
@@ -56,4 +60,5 @@
     "format_shared_validation_report",
     "run_local_validation",
     "run_remote_validation",
+    "validation_source_digest",
 ]

src/openenv/validation/checks.py

@@ -920,6 +920,7 @@ def get_openenv_checks() -> tuple[ValidationCheck, ...]:
                     }
                 ),
                 profiles=_RUNTIME_PROFILES,
+                severity=ValidationSeverity.ADVISORY,
                 timeout_s=600.0,
                 built_in=False,
                 requirement_binding=ValidationRequirementBinding.VERIFIER,

src/openenv/validation/executor.py

@@ -195,6 +195,10 @@ def _execute_check(
         not diagnostics
         and check.built_in
         and outcome.status is not ValidationStatus.PASS
+        and not (
+            check.severity is ValidationSeverity.ADVISORY
+            and outcome.status is ValidationStatus.SKIP
+        )
     ):
         diagnostics = (
             ValidationDiagnostic(

src/openenv/validation/models.py

@@ -16,6 +16,7 @@
 
 
 _STABLE_IDENTIFIER = re.compile(r"^[a-z0-9][a-z0-9._-]{0,127}$")
+_SHA256_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
 _TRUSTED_METADATA_STRING_KEYS = frozenset(
     {
         "id",
@@ -385,6 +386,8 @@ class ValidationPolicy:
     version: str
     supported_subjects: frozenset[tuple[str, ExecutionModel]]
     checks: tuple[ValidationCheck, ...]
+    supported_spec_versions: frozenset[tuple[str, str]] = frozenset()
+    supported_adapters: frozenset[tuple[str, str]] = frozenset()
 
     def __post_init__(self) -> None:
         if not _STABLE_IDENTIFIER.fullmatch(self.version):
@@ -395,6 +398,20 @@ def __post_init__(self) -> None:
             for spec_id, model in self.supported_subjects
         ):
             raise ValueError("ValidationPolicy must declare supported subject pairs")
+        if any(
+            not _STABLE_IDENTIFIER.fullmatch(spec_id)
+            or not version.strip()
+            or len(version) > 128
+            for spec_id, version in self.supported_spec_versions
+        ):
+            raise ValueError("ValidationPolicy contains invalid spec-version bindings")
+        if any(
+            not _STABLE_IDENTIFIER.fullmatch(adapter_id)
+            or not version.strip()
+            or len(version) > 128
+            for adapter_id, version in self.supported_adapters
+        ):
+            raise ValueError("ValidationPolicy contains invalid adapter bindings")
         criterion_ids = [check.criterion_id for check in self.checks]
         if len(criterion_ids) != len(set(criterion_ids)):
             raise ValueError("ValidationPolicy contains duplicate criterion IDs")
@@ -405,9 +422,27 @@ def supports(self, subject: ValidationSubject) -> bool:
 
     def supports_identity(self, identity: SpecIdentity) -> bool:
         """Return whether this policy applies to a detected spec identity."""
-        return bool(
-            (identity.spec_id, identity.execution_model) in self.supported_subjects
+        subject_supported = (
+            identity.spec_id,
+            identity.execution_model,
+        ) in self.supported_subjects
+        version_supported = (
+            not self.supported_spec_versions
+            or (
+                identity.spec_id,
+                identity.spec_version,
+            )
+            in self.supported_spec_versions
+        )
+        adapter_supported = (
+            not self.supported_adapters
+            or (
+                identity.adapter.adapter_id,
+                identity.adapter.adapter_version,
+            )
+            in self.supported_adapters
         )
+        return bool(subject_supported and version_supported and adapter_supported)
 
 
 @dataclass
@@ -573,6 +608,7 @@ class ValidationReport:
     spec: ValidationSubject | None = None
     spec_identity: SpecIdentity | None = None
     repo_sha: str | None = None
+    source_digest: str | None = None
     image_digest: str | None = None
     certified: bool = False
     certification_eligible: bool = False
@@ -589,6 +625,10 @@ def __post_init__(self) -> None:
             and self.spec_identity != self.spec.spec
         ):
             raise ValueError("ValidationReport spec identity must match its subject")
+        if self.source_digest is not None and not _SHA256_DIGEST.fullmatch(
+            self.source_digest
+        ):
+            raise ValueError("ValidationReport source digest must be lowercase SHA-256")
 
     @property
     def passed(self) -> bool:
@@ -678,6 +718,7 @@ def to_dict(self) -> dict[str, Any]:
             "certified": self.certified,
             "certification_eligible": self.certification_eligible,
             "repo_sha": self.repo_sha,
+            "source_digest": self.source_digest,
             "image_digest": self.image_digest,
             "started_at": self.started_at,
             "finished_at": self.finished_at,

src/openenv/validation/planner.py

@@ -39,12 +39,32 @@
     version=VALIDATION_POLICY_VERSION,
     supported_subjects=frozenset({("openenv", ExecutionModel.SERVED)}),
     checks=get_openenv_checks(),
+    supported_spec_versions=frozenset({("openenv", "1")}),
+    supported_adapters=frozenset({("openenv-yaml", "1")}),
 )
 _POLICY_ATTESTATION = object()
 
 
 def _repo_sha(path: Path) -> str | None:
     try:
+        status = subprocess.run(
+            [
+                "git",
+                "-C",
+                str(path),
+                "status",
+                "--porcelain",
+                "--untracked-files=all",
+                "--",
+                ".",
+            ],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=3.0,
+        )
+        if status.returncode != 0 or status.stdout.strip():
+            return None
         completed = subprocess.run(
             ["git", "-C", str(path), "rev-parse", "HEAD"],
             check=False,
@@ -143,7 +163,17 @@ def _policy_for_subject(
 ) -> tuple[ValidationPolicy, bool]:
     selected = policy or OPENENV_VALIDATION_POLICY
     identity = spec_load.spec
-    if identity is not None and not selected.supports_identity(identity):
+    incompatible_subject = bool(
+        identity is not None
+        and (identity.spec_id, identity.execution_model)
+        not in selected.supported_subjects
+    )
+    incompatible_loaded_version = bool(
+        spec_load.subject is not None
+        and identity is not None
+        and not selected.supports_identity(identity)
+    )
+    if identity is not None and (incompatible_subject or incompatible_loaded_version):
         raise ValueError(
             f"Validation policy {selected.version!r} does not support "
             f"spec {identity.spec_id!r} with execution model "