From a99705a476551088e477cf058d0bfeae70b13c7e Mon Sep 17 00:00:00 2001 From: Zach Wentz <4832+zkwentz@users.noreply.github.com> Date: Sun, 12 Jul 2026 13:00:58 -0400 Subject: [PATCH] feat(validation): add structured author guidance --- src/openenv/cli/commands/validate.py | 2 +- src/openenv/validation/__init__.py | 6 + src/openenv/validation/checks.py | 217 ++++++++++++++++++++++++- src/openenv/validation/executor.py | 37 +++++ src/openenv/validation/local.py | 51 +++++- src/openenv/validation/models.py | 148 +++++++++++++++++ tests/test_cli/test_validate.py | 96 +++++++++++ tests/test_validation/test_guidance.py | 109 +++++++++++++ 8 files changed, 659 insertions(+), 7 deletions(-) create mode 100644 tests/test_validation/test_guidance.py diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index 8d3f7b90b..50213ff31 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -203,7 +203,7 @@ def validate( if json_output or (runtime_target is not None and profile is None): typer.echo(serialized) else: - typer.echo(format_shared_validation_report(report)) + typer.echo(format_shared_validation_report(report, verbose=verbose)) if output is not None: typer.echo(f"Report written to {output}") diff --git a/src/openenv/validation/__init__.py b/src/openenv/validation/__init__.py index 9e889e0b7..42a80aa01 100644 --- a/src/openenv/validation/__init__.py +++ b/src/openenv/validation/__init__.py @@ -6,13 +6,16 @@ from .local import format_shared_validation_report, run_local_validation from .models import ( CheckOutcome, + DiagnosticLocation, RunnerCapabilities, ValidationCapability, ValidationCheck, ValidationContext, + ValidationDiagnostic, ValidationPlan, ValidationPolicy, ValidationProfile, + ValidationRemediation, ValidationReport, ValidationRequirementBinding, ValidationResult, @@ -28,16 +31,19 @@ __all__ = [ "CheckOutcome", + "DiagnosticLocation", "OPENENV_VALIDATION_POLICY", "RunnerCapabilities", "VALIDATION_POLICY_VERSION", "ValidationCapability", "ValidationCheck", "ValidationContext", + "ValidationDiagnostic", "ValidationPlan", "ValidationPolicy", "ValidationProfile", "ValidationReport", + "ValidationRemediation", "ValidationRequirementBinding", "ValidationResult", "ValidationSeverity", diff --git a/src/openenv/validation/checks.py b/src/openenv/validation/checks.py index 77eaebbb0..16e4a7e51 100644 --- a/src/openenv/validation/checks.py +++ b/src/openenv/validation/checks.py @@ -14,10 +14,13 @@ from .models import ( CheckOutcome, + DiagnosticLocation, ValidationCapability, ValidationCheck, ValidationContext, + ValidationDiagnostic, ValidationProfile, + ValidationRemediation, ValidationRequirementBinding, ValidationSeverity, ValidationStatus, @@ -80,6 +83,20 @@ def _validation_spec(context: ValidationContext) -> CheckOutcome: return CheckOutcome.fail( loaded.to_dict(), message="No supported validation spec was detected", + diagnostics=( + ValidationDiagnostic( + code="validation_spec_missing", + message="Add a supported environment manifest", + location=DiagnosticLocation(path="openenv.yaml"), + ), + ), + remediation=( + ValidationRemediation( + kind="edit", + message="Create openenv.yaml with the environment identity and runtime settings", + path="openenv.yaml", + ), + ), ) if loaded.state in { SpecLoadState.INVALID, @@ -89,6 +106,20 @@ def _validation_spec(context: ValidationContext) -> CheckOutcome: return CheckOutcome.fail( loaded.to_dict(), message="The validation spec could not be loaded", + diagnostics=( + ValidationDiagnostic( + code="validation_spec_invalid", + message="Fix the OpenEnv manifest before validation can continue", + location=DiagnosticLocation(path="openenv.yaml"), + ), + ), + remediation=( + ValidationRemediation( + kind="edit", + message="Correct the invalid fields or syntax in openenv.yaml", + path="openenv.yaml", + ), + ), ) if loaded.subject is None: return CheckOutcome.error( @@ -103,6 +134,13 @@ def _validation_requirements(context: ValidationContext) -> CheckOutcome: return CheckOutcome.skip( {"requirements_present": False, "reason": "spec_unavailable"}, message="Requirements cannot be loaded until the source spec is valid", + diagnostics=( + ValidationDiagnostic( + code="requirements_blocked_by_spec", + message="Requirements cannot be inspected until the environment manifest is valid", + location=DiagnosticLocation(path="openenv.yaml"), + ), + ), ) loaded = subject.requirements if loaded.state is RequirementsState.ABSENT: @@ -112,11 +150,41 @@ def _validation_requirements(context: ValidationContext) -> CheckOutcome: "migration": "Missing requirements are locally non-failing in rfc008-v1", }, message="No requirements envelope found; policy defaults apply during migration", + diagnostics=( + ValidationDiagnostic( + code="requirements_missing", + message="A publish-ready environment must declare its execution requirements", + location=DiagnosticLocation(path="task.toml"), + ), + ), + remediation=( + ValidationRemediation( + kind="edit", + message='Add a Harbor schema 1.1 requirements envelope; start with schema_version = "1.1"', + path="task.toml", + pointer="/schema_version", + ), + ), ) if loaded.state in {RequirementsState.INVALID, RequirementsState.UNSUPPORTED}: return CheckOutcome.fail( loaded.to_evidence(), message="The requirements envelope could not be loaded", + diagnostics=( + ValidationDiagnostic( + code="requirements_invalid", + message="Fix unsupported fields or invalid TOML in the requirements envelope", + location=DiagnosticLocation(path="task.toml"), + ), + ), + remediation=( + ValidationRemediation( + kind="edit", + message='Use the supported Harbor schema_version "1.1" configuration', + path="task.toml", + pointer="/schema_version", + ), + ), ) if loaded.requirements is None: return CheckOutcome.error( @@ -132,6 +200,20 @@ def _openenv_manifest(context: ValidationContext) -> CheckOutcome: return CheckOutcome.fail( {"path": str(path), "missing_or_unsafe": True}, message="Missing or unsafe openenv.yaml", + diagnostics=( + ValidationDiagnostic( + code="manifest_missing", + message="The environment root must contain a regular openenv.yaml file", + location=DiagnosticLocation(path="openenv.yaml"), + ), + ), + remediation=( + ValidationRemediation( + kind="edit", + message="Create a regular openenv.yaml manifest in the environment root", + path="openenv.yaml", + ), + ), ) try: manifest = yaml.safe_load(path.read_text(encoding="utf-8")) @@ -139,6 +221,20 @@ def _openenv_manifest(context: ValidationContext) -> CheckOutcome: return CheckOutcome.fail( {"path": str(path), "error_type": type(exc).__name__}, message="Unable to parse openenv.yaml", + diagnostics=( + ValidationDiagnostic( + code="manifest_syntax_invalid", + message="openenv.yaml contains invalid YAML syntax", + location=DiagnosticLocation(path="openenv.yaml"), + ), + ), + remediation=( + ValidationRemediation( + kind="edit", + message="Correct the YAML syntax in openenv.yaml", + path="openenv.yaml", + ), + ), ) if not isinstance(manifest, dict): return CheckOutcome.fail( @@ -151,6 +247,25 @@ def _openenv_manifest(context: ValidationContext) -> CheckOutcome: return CheckOutcome.fail( {"path": str(path), "missing_fields": missing}, message="openenv.yaml is missing required fields", + diagnostics=tuple( + ValidationDiagnostic( + code="manifest_field_missing", + message=f"Add the required `{field}` setting", + location=DiagnosticLocation( + path="openenv.yaml", pointer=f"/{field}" + ), + ) + for field in missing + ), + remediation=tuple( + ValidationRemediation( + kind="edit", + message=f"Configure `{field}` in openenv.yaml", + path="openenv.yaml", + pointer=f"/{field}", + ) + for field in missing + ), ) invalid_fields: dict[str, Any] = {} spec_version = manifest.get("spec_version") @@ -170,6 +285,25 @@ def _openenv_manifest(context: ValidationContext) -> CheckOutcome: return CheckOutcome.fail( {"path": str(path), "invalid_fields": invalid_fields}, message="openenv.yaml contains unsupported or invalid field values", + diagnostics=tuple( + ValidationDiagnostic( + code="manifest_field_invalid", + message=f"Configure a supported value for `{field}`", + location=DiagnosticLocation( + path="openenv.yaml", pointer=f"/{field}" + ), + ) + for field in invalid_fields + ), + remediation=tuple( + ValidationRemediation( + kind="edit", + message=f"Correct `{field}` in openenv.yaml", + path="openenv.yaml", + pointer=f"/{field}", + ) + for field in invalid_fields + ), ) return CheckOutcome.pass_( { @@ -222,7 +356,25 @@ def _project_layout(context: ValidationContext) -> CheckOutcome: "issues": issues, } if issues: - return CheckOutcome.fail(evidence, message=f"{len(issues)} layout issue(s)") + return CheckOutcome.fail( + evidence, + message=f"{len(issues)} layout issue(s)", + diagnostics=tuple( + ValidationDiagnostic( + code="project_layout_issue", + message=issue, + location=DiagnosticLocation(path="."), + ) + for issue in issues + ), + remediation=( + ValidationRemediation( + kind="edit", + message="Add the missing project files or correct the server entry point", + path="pyproject.toml", + ), + ), + ) return CheckOutcome.pass_(evidence) @@ -254,6 +406,23 @@ def _dependencies(context: ValidationContext) -> CheckOutcome: "dockerfile_installs_openenv": False, }, message="Missing required dependency: openenv>=0.2.0", + diagnostics=( + ValidationDiagnostic( + code="openenv_dependency_missing", + message="The project must install the OpenEnv runtime dependency", + location=DiagnosticLocation( + path="pyproject.toml", pointer="/project/dependencies" + ), + ), + ), + remediation=( + ValidationRemediation( + kind="command", + message="Add the OpenEnv runtime dependency and refresh the lockfile", + argv=("uv", "add", "openenv>=0.2.0"), + cwd=".", + ), + ), ) return CheckOutcome.pass_( { @@ -270,6 +439,21 @@ def _lockfile(context: ValidationContext) -> CheckOutcome: return CheckOutcome.fail( {"path": str(lockfile), "missing_or_unsafe": True}, message="Missing or unsafe uv.lock - run 'uv lock' to generate it", + diagnostics=( + ValidationDiagnostic( + code="lockfile_missing", + message="A publish-ready environment must include a regular uv.lock file", + location=DiagnosticLocation(path="uv.lock"), + ), + ), + remediation=( + ValidationRemediation( + kind="command", + message="Generate the dependency lockfile", + argv=("uv", "lock"), + cwd=".", + ), + ), ) try: with lockfile.open("rb") as lockfile_handle: @@ -321,6 +505,23 @@ def _schema_sources(context: ValidationContext) -> CheckOutcome: return CheckOutcome.fail( {"checked": checked, "errors": errors}, message="Python schema/application sources do not compile", + diagnostics=tuple( + ValidationDiagnostic( + code="python_source_invalid", + message="Correct the Python syntax error in this source file", + location=DiagnosticLocation( + path=str(error["path"]), line=error.get("line") + ), + ) + for error in errors + ), + remediation=( + ValidationRemediation( + kind="edit", + message="Fix each reported Python syntax or file-safety error", + path=str(errors[0]["path"]), + ), + ), ) return CheckOutcome.pass_({"checked": checked}) @@ -368,6 +569,20 @@ def _dockerfile(context: ValidationContext) -> CheckOutcome: return CheckOutcome.fail( {"searched": ["server/Dockerfile", "Dockerfile"]}, message="Missing Dockerfile and normalized container-image declaration", + diagnostics=( + ValidationDiagnostic( + code="container_declaration_missing", + message="Declare a container image or add a Dockerfile", + location=DiagnosticLocation(path="server/Dockerfile"), + ), + ), + remediation=( + ValidationRemediation( + kind="edit", + message="Add server/Dockerfile with a supported FROM instruction", + path="server/Dockerfile", + ), + ), ) diff --git a/src/openenv/validation/executor.py b/src/openenv/validation/executor.py index 5f978fac3..def7042b6 100644 --- a/src/openenv/validation/executor.py +++ b/src/openenv/validation/executor.py @@ -15,8 +15,10 @@ from .models import ( CheckOutcome, ValidationContext, + ValidationDiagnostic, ValidationPlan, ValidationProfile, + ValidationRemediation, ValidationReport, ValidationResult, ValidationSeverity, @@ -98,6 +100,15 @@ def _execute_check( required_capabilities=check.capabilities, built_in=check.built_in, message="Runner does not provide the capabilities required by this check", + diagnostics=( + ValidationDiagnostic( + code="runner_capability_unavailable", + message=( + "This blocking check could not run with the selected " + "validation runner" + ), + ), + ), ) started = time.perf_counter() @@ -127,6 +138,14 @@ def _execute_check( not isinstance(outcome.status, ValidationStatus) or not isinstance(outcome.evidence, Mapping) or (outcome.message is not None and not isinstance(outcome.message, str)) + or not isinstance(outcome.diagnostics, tuple) + or not all( + isinstance(item, ValidationDiagnostic) for item in outcome.diagnostics + ) + or not isinstance(outcome.remediation, tuple) + or not all( + isinstance(item, ValidationRemediation) for item in outcome.remediation + ) ): outcome = CheckOutcome.error( {"reason": "invalid_outcome_shape"}, @@ -171,6 +190,22 @@ def _execute_check( ) evidence = json_safe(outcome.evidence) + diagnostics = outcome.diagnostics + if ( + not diagnostics + and check.built_in + and outcome.status is not ValidationStatus.PASS + ): + diagnostics = ( + ValidationDiagnostic( + code=f"criterion_{outcome.status.value}", + message=( + "Review this criterion's safe evidence and correct the environment " + "configuration before retrying" + ), + ), + ) + return ValidationResult( criterion_id=check.criterion_id, requirement=check.requirement, @@ -186,6 +221,8 @@ def _execute_check( message=( redact_string(outcome.message) if outcome.message is not None else None ), + diagnostics=diagnostics, + remediation=outcome.remediation, ) diff --git a/src/openenv/validation/local.py b/src/openenv/validation/local.py index fda581946..94526424c 100644 --- a/src/openenv/validation/local.py +++ b/src/openenv/validation/local.py @@ -5,7 +5,9 @@ from __future__ import annotations import errno +import json import os +import shlex import shutil import signal import socket @@ -503,7 +505,9 @@ def run_local_validation( ) -def format_shared_validation_report(report: ValidationReport) -> str: +def format_shared_validation_report( + report: ValidationReport, *, verbose: bool = False +) -> str: """Render a concise human-readable shared report.""" marker = "OK" if report.passed else "FAIL" lines = [ @@ -524,8 +528,45 @@ def format_shared_validation_report(report: ValidationReport) -> str: ) if result.message: lines.append(f" {result.message}") - issues = result.evidence.get("issues") - if isinstance(issues, list): - lines.extend(f" {issue}" for issue in issues if isinstance(issue, str)) - lines.append("Certification: not claimed by local validation") + for diagnostic in result.diagnostics: + safe_diagnostic = diagnostic.to_dict() + location = "" + if diagnostic.location is not None: + location = diagnostic.location.path + if diagnostic.location.pointer is not None: + location += diagnostic.location.pointer + if diagnostic.location.line is not None: + location += f":{diagnostic.location.line}" + location = f" ({location})" + lines.append( + f" Diagnostic [{diagnostic.code}]{location}: " + f"{safe_diagnostic['message']}" + ) + for remediation in result.remediation: + safe_remediation = remediation.to_dict() + lines.append(f" Fix: {safe_remediation['message']}") + if remediation.argv: + lines.append(f" $ {shlex.join(remediation.argv)}") + elif remediation.path is not None: + destination = remediation.path + if remediation.pointer is not None: + destination += remediation.pointer + lines.append(f" Edit: {destination}") + elif remediation.url is not None: + lines.append(f" Docs: {remediation.url}") + if verbose and result.evidence: + rendered_evidence = json.dumps( + result.evidence, indent=2, sort_keys=True, default=str + ) + lines.append(" Evidence:") + lines.extend(f" {line}" for line in rendered_evidence.splitlines()) + if report.profile is ValidationProfile.PUBLISH and not report.passed: + blockers = report.to_dict()["summary"] + blocking_ids = [ + *blockers["blocking_failed_criteria"], + *blockers["blocking_skipped_criteria"], + ] + if blocking_ids: + lines.append(f"Publish blockers: {', '.join(blocking_ids)}") + lines.append("Certification: not claimed by author validation") return "\n".join(lines) diff --git a/src/openenv/validation/models.py b/src/openenv/validation/models.py index 58704c5db..e78db50ee 100644 --- a/src/openenv/validation/models.py +++ b/src/openenv/validation/models.py @@ -9,6 +9,7 @@ from enum import Enum from pathlib import Path from typing import Any, Callable, Mapping +from urllib.parse import urlsplit from .serialization import json_safe, redact_string from .specs.base import ExecutionModel, SpecIdentity, SpecLoad, ValidationSubject @@ -86,6 +87,131 @@ class ValidationRequirementBinding(str, Enum): NETWORK = "network" +def _validate_relative_path(value: str, *, field_name: str) -> None: + path = Path(value) + if ( + not value + or len(value) > 4096 + or "\x00" in value + or path.is_absolute() + or ".." in path.parts + ): + raise ValueError(f"{field_name} must be a bounded relative path") + + +@dataclass(frozen=True) +class DiagnosticLocation: + """Repository-relative location for an author-facing diagnostic.""" + + path: str + pointer: str | None = None + line: int | None = None + column: int | None = None + + def __post_init__(self) -> None: + _validate_relative_path(self.path, field_name="diagnostic path") + if self.pointer is not None and ( + not self.pointer.startswith("/") + or len(self.pointer) > 4096 + or "\x00" in self.pointer + ): + raise ValueError("diagnostic pointer must be a bounded document pointer") + for name, value in (("line", self.line), ("column", self.column)): + if value is not None and (not isinstance(value, int) or value < 1): + raise ValueError(f"diagnostic {name} must be a positive integer") + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-safe source location.""" + return { + "path": self.path, + "pointer": self.pointer, + "line": self.line, + "column": self.column, + } + + +@dataclass(frozen=True) +class ValidationDiagnostic: + """Trusted, typed explanation of a validation problem.""" + + code: str + message: str + location: DiagnosticLocation | None = None + + def __post_init__(self) -> None: + if not _STABLE_IDENTIFIER.fullmatch(self.code): + raise ValueError("diagnostic code must be a stable lowercase identifier") + if not self.message.strip() or len(self.message) > 4096: + raise ValueError("diagnostic message must be non-empty and bounded") + + def to_dict(self) -> dict[str, Any]: + """Return the report representation with free text redacted.""" + return { + "code": self.code, + "message": redact_string(self.message), + "location": self.location.to_dict() if self.location is not None else None, + } + + +@dataclass(frozen=True) +class ValidationRemediation: + """Display-only, policy-authored guidance for resolving a diagnostic.""" + + kind: str + message: str + argv: tuple[str, ...] = () + cwd: str | None = None + path: str | None = None + pointer: str | None = None + url: str | None = None + + def __post_init__(self) -> None: + if self.kind not in {"command", "documentation", "edit", "retry"}: + raise ValueError("unsupported remediation kind") + if not self.message.strip() or len(self.message) > 4096: + raise ValueError("remediation message must be non-empty and bounded") + if any( + not isinstance(argument, str) + or not argument + or len(argument) > 4096 + or "\x00" in argument + for argument in self.argv + ): + raise ValueError("remediation argv must contain bounded strings") + if self.kind == "command" and not self.argv: + raise ValueError("command remediation requires argv") + if self.kind != "command" and self.argv: + raise ValueError("only command remediation may declare argv") + if self.cwd is not None: + _validate_relative_path(self.cwd, field_name="remediation cwd") + if self.path is not None: + _validate_relative_path(self.path, field_name="remediation path") + if self.pointer is not None and ( + not self.pointer.startswith("/") + or len(self.pointer) > 4096 + or "\x00" in self.pointer + ): + raise ValueError("remediation pointer must be a bounded document pointer") + if self.url is not None: + parsed = urlsplit(self.url) + if parsed.scheme != "https" or not parsed.hostname or parsed.username: + raise ValueError( + "remediation URL must be an HTTPS URL without credentials" + ) + + def to_dict(self) -> dict[str, Any]: + """Return structured guidance; argv is never interpreted as a shell string.""" + return { + "kind": self.kind, + "message": redact_string(self.message), + "argv": [redact_string(argument) for argument in self.argv], + "cwd": self.cwd, + "path": self.path, + "pointer": self.pointer, + "url": redact_string(self.url) if self.url is not None else None, + } + + @dataclass(frozen=True) class RunnerCapabilities: """Capabilities and trust properties exposed by a validation runner.""" @@ -125,6 +251,8 @@ class CheckOutcome: evidence: Mapping[str, Any] = field(default_factory=dict) message: str | None = None severity: ValidationSeverity | None = None + diagnostics: tuple[ValidationDiagnostic, ...] = () + remediation: tuple[ValidationRemediation, ...] = () @classmethod def pass_( @@ -133,6 +261,8 @@ def pass_( *, message: str | None = None, severity: ValidationSeverity | None = None, + diagnostics: tuple[ValidationDiagnostic, ...] = (), + remediation: tuple[ValidationRemediation, ...] = (), ) -> "CheckOutcome": """Build a passing outcome.""" return cls( @@ -140,6 +270,8 @@ def pass_( evidence or {}, message=message, severity=severity, + diagnostics=diagnostics, + remediation=remediation, ) @classmethod @@ -149,6 +281,8 @@ def fail( *, message: str | None = None, severity: ValidationSeverity | None = None, + diagnostics: tuple[ValidationDiagnostic, ...] = (), + remediation: tuple[ValidationRemediation, ...] = (), ) -> "CheckOutcome": """Build a failing outcome.""" return cls( @@ -156,6 +290,8 @@ def fail( evidence or {}, message=message, severity=severity, + diagnostics=diagnostics, + remediation=remediation, ) @classmethod @@ -165,6 +301,8 @@ def skip( *, message: str | None = None, severity: ValidationSeverity | None = None, + diagnostics: tuple[ValidationDiagnostic, ...] = (), + remediation: tuple[ValidationRemediation, ...] = (), ) -> "CheckOutcome": """Build a skipped outcome.""" return cls( @@ -172,6 +310,8 @@ def skip( evidence or {}, message=message, severity=severity, + diagnostics=diagnostics, + remediation=remediation, ) @classmethod @@ -181,6 +321,8 @@ def error( *, message: str | None = None, severity: ValidationSeverity | None = None, + diagnostics: tuple[ValidationDiagnostic, ...] = (), + remediation: tuple[ValidationRemediation, ...] = (), ) -> "CheckOutcome": """Build an infrastructure-error outcome.""" return cls( @@ -188,6 +330,8 @@ def error( evidence or {}, message=message, severity=severity, + diagnostics=diagnostics, + remediation=remediation, ) @@ -377,6 +521,8 @@ class ValidationResult: required_capabilities: frozenset[ValidationCapability] built_in: bool = True message: str | None = None + diagnostics: tuple[ValidationDiagnostic, ...] = () + remediation: tuple[ValidationRemediation, ...] = () @property def passed(self) -> bool: @@ -401,6 +547,8 @@ def to_dict(self) -> dict[str, Any]: "duration_s": round(self.duration_s, 6), "duration_ms": round(self.duration_s * 1000, 3), "timeout_s": self.timeout_s, + "diagnostics": [item.to_dict() for item in self.diagnostics], + "remediation": [item.to_dict() for item in self.remediation], } if self.message is not None: payload["details"] = redact_string(self.message) diff --git a/tests/test_cli/test_validate.py b/tests/test_cli/test_validate.py index d19f36d4f..417705aff 100644 --- a/tests/test_cli/test_validate.py +++ b/tests/test_cli/test_validate.py @@ -60,6 +60,61 @@ def _write_minimal_valid_env( (env_dir / "server" / "Dockerfile").write_text("FROM python:3.12-slim\n") +def _diagnostic_failure_report(target: Path): + from openenv.validation.models import ( + DiagnosticLocation, + RunnerCapabilities, + ValidationCapability, + ValidationDiagnostic, + ValidationProfile, + ValidationRemediation, + ValidationReport, + ValidationResult, + ValidationSeverity, + ValidationStatus, + ) + + diagnostic = ValidationDiagnostic( + code="missing_dependency", + message="Missing OpenEnv runtime dependency", + location=DiagnosticLocation(path="pyproject.toml"), + ) + remediation = ValidationRemediation( + kind="command", + message="Add the OpenEnv runtime dependency", + argv=("uv", "add", "openenv>=0.2.0"), + cwd=".", + ) + result = ValidationResult( + criterion_id="source.dependencies", + requirement="The environment declares the OpenEnv runtime dependency", + status=ValidationStatus.FAIL, + severity=ValidationSeverity.BLOCKING, + evidence={ + "project_dependency_count": 0, + "dockerfile_installs_openenv": False, + }, + duration_s=0.01, + timeout_s=10.0, + required_capabilities=frozenset({ValidationCapability.SOURCE}), + diagnostics=(diagnostic,), + remediation=(remediation,), + ) + return ValidationReport( + target=str(target), + profile=ValidationProfile.STATIC, + policy_version="test-policy", + runner=RunnerCapabilities( + runner="local", + available=frozenset({ValidationCapability.SOURCE}), + ), + results=(result,), + duration_s=0.01, + started_at="2026-01-01T00:00:00+00:00", + finished_at="2026-01-01T00:00:00.010000+00:00", + ) + + def test_validate_running_environment_success() -> None: """Runtime validator returns passing criteria for a conforming server.""" @@ -243,6 +298,47 @@ def test_unqualified_local_validate_uses_shared_static_semantics( assert "The validation spec could not be loaded" in result.output +def test_shared_human_report_surfaces_typed_remediation(tmp_path: Path) -> None: + env_dir = tmp_path / "test_env" + _write_minimal_valid_env(env_dir) + report = _diagnostic_failure_report(env_dir) + + with patch( + "openenv.cli.commands.validate.run_local_validation", + return_value=report, + ): + result = runner.invoke( + app, + ["validate", str(env_dir), "--profile", "static"], + ) + + assert result.exit_code == 1 + assert "Missing OpenEnv runtime dependency" in result.output + assert "pyproject.toml" in result.output + assert "Add the OpenEnv runtime dependency" in result.output + assert "uv add 'openenv>=0.2.0'" in result.output + assert "project_dependency_count" not in result.output + + +def test_shared_verbose_report_includes_structured_evidence(tmp_path: Path) -> None: + env_dir = tmp_path / "test_env" + _write_minimal_valid_env(env_dir) + report = _diagnostic_failure_report(env_dir) + + with patch( + "openenv.cli.commands.validate.run_local_validation", + return_value=report, + ): + result = runner.invoke( + app, + ["validate", str(env_dir), "--profile", "static", "--verbose"], + ) + + assert result.exit_code == 1 + assert "project_dependency_count" in result.output + assert "dockerfile_installs_openenv" in result.output + + def test_validate_command_local_json_output(tmp_path: Path) -> None: """CLI can emit JSON report for local validation via --json.""" env_dir = tmp_path / "test_env" diff --git a/tests/test_validation/test_guidance.py b/tests/test_validation/test_guidance.py new file mode 100644 index 000000000..e845d9e89 --- /dev/null +++ b/tests/test_validation/test_guidance.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for typed, display-only validation guidance.""" + +from __future__ import annotations + +import pytest +from openenv.validation import ( + DiagnosticLocation, + RunnerCapabilities, + ValidationDiagnostic, + ValidationProfile, + ValidationRemediation, + ValidationReport, + ValidationResult, + ValidationSeverity, + ValidationStatus, +) + + +def test_guidance_serializes_as_typed_report_data() -> None: + result = ValidationResult( + criterion_id="source.lockfile", + requirement="A lockfile is required", + status=ValidationStatus.FAIL, + severity=ValidationSeverity.BLOCKING, + evidence={"missing": True}, + duration_s=0.01, + timeout_s=1.0, + required_capabilities=frozenset(), + diagnostics=( + ValidationDiagnostic( + code="lockfile_missing", + message="Generate a lockfile", + location=DiagnosticLocation(path="uv.lock", line=1), + ), + ), + remediation=( + ValidationRemediation( + kind="command", + message="Generate the lockfile", + argv=("uv", "lock"), + cwd=".", + ), + ), + ) + report = ValidationReport( + target=".", + profile=ValidationProfile.PUBLISH, + policy_version="test-policy", + runner=RunnerCapabilities(runner="local"), + results=(result,), + duration_s=0.01, + started_at="2026-01-01T00:00:00+00:00", + finished_at="2026-01-01T00:00:00.010000+00:00", + ) + + criterion = report.to_dict()["criteria"][0] + + assert criterion["diagnostics"] == [ + { + "code": "lockfile_missing", + "message": "Generate a lockfile", + "location": { + "path": "uv.lock", + "pointer": None, + "line": 1, + "column": None, + }, + } + ] + assert criterion["remediation"] == [ + { + "kind": "command", + "message": "Generate the lockfile", + "argv": ["uv", "lock"], + "cwd": ".", + "path": None, + "pointer": None, + "url": None, + } + ] + + +@pytest.mark.parametrize("path", ["/tmp/secret", "../secret", "dir/../../secret"]) +def test_guidance_locations_must_be_repository_relative(path: str) -> None: + with pytest.raises(ValueError, match="relative path"): + DiagnosticLocation(path=path) + + +def test_command_guidance_requires_argv_and_never_accepts_a_shell_string() -> None: + with pytest.raises(ValueError, match="requires argv"): + ValidationRemediation(kind="command", message="Run the fix") + + with pytest.raises(TypeError): + ValidationRemediation( # type: ignore[call-arg] + kind="command", + message="Run the fix", + command="uv lock", + ) + + +def test_documentation_guidance_requires_a_credential_free_https_url() -> None: + with pytest.raises(ValueError, match="HTTPS URL"): + ValidationRemediation( + kind="documentation", + message="Read the guide", + url="http://user:password@example.com/guide", + )