From 2dcab803c1ec79d6f7efb69641ced9722d0865a6 Mon Sep 17 00:00:00 2001 From: Zach Wentz <4832+zkwentz@users.noreply.github.com> Date: Sun, 12 Jul 2026 07:49:01 -0400 Subject: [PATCH] feat(cli): add validation profiles and shared reports --- docs/source/reference/cli.md | 22 ++++++ src/openenv/cli/commands/validate.py | 103 ++++++++++++++++++++++++++- tests/test_cli/test_validate.py | 56 +++++++++++++-- 3 files changed, 176 insertions(+), 5 deletions(-) diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index a6a968ab1..72a5f08b6 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -40,6 +40,28 @@ openenv import path/to/source --name my_env --output-dir ./envs --env-class MyEn ## `openenv validate` +Use an explicit profile to produce the RFC 008 validation plan and shared +report schema: + +```bash +openenv validate path/to/env --profile static +openenv validate path/to/env --profile runtime --json +openenv validate --url http://127.0.0.1:8000 --profile runtime --json +openenv validate path/to/env --profile full --output validation.json +``` + +`static` checks source and packaging, `runtime` adds a launched or connected +server, and `full` records every policy criterion while marking unavailable +remote capabilities as skipped. Local reports never claim official +certification. Automatic local launch executes the environment checkout as the +current user and is intended only for trusted development source; use `--url` +for a server you already isolated. + +This command intentionally targets the served OpenEnv spec. Shared reports +record spec, adapter, and execution-model provenance, but `openenv validate` +does not auto-dispatch external task-package formats. A separate spec-selected +task workflow is tracked in [issue #898](https://github.com/huggingface/OpenEnv/issues/898). + [[autodoc]] openenv.cli.commands.validate.validate ## `openenv push` diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index b3926a769..a706757f5 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -19,6 +19,11 @@ validate_multi_mode_deployment, validate_running_environment, ) +from openenv.validation import ( + format_shared_validation_report, + run_local_validation, + ValidationProfile, +) def _looks_like_url(value: str) -> bool: @@ -48,9 +53,23 @@ def validate( bool, typer.Option( "--json", - help="Output local validation report as JSON (runtime validation is JSON by default)", + help="Output the RFC 008 shared validation report as JSON", ), ] = False, + profile: Annotated[ + str | None, + typer.Option( + "--profile", + help="Validation profile: static, runtime, or full", + ), + ] = None, + output: Annotated[ + Path | None, + typer.Option( + "--output", + help="Write the shared JSON validation report to this path", + ), + ] = None, timeout: Annotated[ float, typer.Option( @@ -74,6 +93,11 @@ def validate( Runtime validation checks if a live OpenEnv server conforms to the versioned runtime API contract and returns a criteria-based JSON report. + Explicit static, runtime, and full profiles emit the RFC 008 shared report. + Reports identify the served OpenEnv spec and pinned adapter; external task + package formats are intentionally outside this command's dispatch surface. + Automatic runtime launch is intended for trusted local source; connect to + an already isolated server with `--url` for untrusted environments. Examples: @@ -91,6 +115,9 @@ def validate( # Validate specific environment $ openenv validate envs/echo_env + + # Run every locally available check and record remote-only skips + $ openenv validate envs/echo_env --profile full --output report.json ``` """ runtime_target = url @@ -114,6 +141,80 @@ def validate( raise typer.Exit(1) runtime_target = target + # Machine-readable and explicit-profile invocations use the RFC 008 shared + # report. Only the unqualified human rendering stays on the legacy path. + if profile is not None or output is not None or json_output: + if profile is None: + selected_profile = ( + ValidationProfile.RUNTIME + if runtime_target is not None + else ValidationProfile.STATIC + ) + else: + try: + selected_profile = ValidationProfile(profile.lower()) + except ValueError as exc: + typer.echo( + "Error: --profile must be one of: static, runtime, full", + err=True, + ) + raise typer.Exit(1) from exc + + if runtime_target is not None and selected_profile is ValidationProfile.STATIC: + typer.echo( + "Error: The static profile requires a local source directory", + err=True, + ) + raise typer.Exit(1) + + if runtime_target is not None: + shared_target: str | Path = runtime_target + shared_runtime_url = runtime_target + else: + shared_target = Path.cwd() if target is None else Path(target) + shared_runtime_url = None + if not shared_target.exists(): + typer.echo(f"Error: Path does not exist: {shared_target}", err=True) + raise typer.Exit(1) + if not shared_target.is_dir(): + typer.echo(f"Error: Path is not a directory: {shared_target}", err=True) + raise typer.Exit(1) + + try: + report = run_local_validation( + shared_target, + profile=selected_profile, + runtime_url=shared_runtime_url, + timeout_s=timeout, + ) + except ValueError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(1) from exc + payload = report.to_dict() + serialized = json.dumps(payload, indent=2) + + if output is not None: + try: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(f"{serialized}\n", encoding="utf-8") + except OSError as exc: + typer.echo( + f"Error: Unable to write validation report: {type(exc).__name__}", + err=True, + ) + raise typer.Exit(1) from exc + + if json_output: + typer.echo(serialized) + else: + typer.echo(format_shared_validation_report(report)) + if output is not None: + typer.echo(f"Report written to {output}") + + if not report.passed: + raise typer.Exit(1) + return + if runtime_target is not None: try: report = validate_running_environment(runtime_target, timeout_s=timeout) diff --git a/tests/test_cli/test_validate.py b/tests/test_cli/test_validate.py index cf710d4e1..80c797d5d 100644 --- a/tests/test_cli/test_validate.py +++ b/tests/test_cli/test_validate.py @@ -41,7 +41,9 @@ def _write_minimal_valid_env( (env_dir / "openenv.yaml").write_text( "spec_version: 1\nname: test_env\ntype: space\nruntime: fastapi\napp: server.app:app\nport: 8000\n" ) - (env_dir / "uv.lock").write_text("") + (env_dir / "uv.lock").write_text( + 'version = 1\nrevision = 1\nrequires-python = ">=3.10"\n' + ) (env_dir / "pyproject.toml").write_text( "[project]\n" 'name = "test-env"\n' @@ -54,6 +56,7 @@ def _write_minimal_valid_env( (env_dir / "server" / "app.py").write_text( f"{main_signature}\n return None\n\nif __name__ == '__main__':\n {main_invocation}\n" ) + (env_dir / "server" / "Dockerfile").write_text("FROM python:3.12-slim\n") def test_validate_running_environment_success() -> None: @@ -194,6 +197,16 @@ def test_validate_command_runtime_target_outputs_json() -> None: mock_validate.assert_called_once_with("https://example.com", timeout_s=5.0) +def test_validate_shared_profile_reports_invalid_runtime_url() -> None: + result = runner.invoke( + app, + ["validate", "--url", "http://[::1", "--profile", "runtime", "--json"], + ) + + assert result.exit_code == 1 + assert "Error: Invalid runtime URL" in result.output + + def test_validate_command_local_path_still_works(tmp_path: Path) -> None: """CLI local validation remains backward compatible.""" env_dir = tmp_path / "test_env" @@ -214,13 +227,47 @@ def test_validate_command_local_json_output(tmp_path: Path) -> None: assert result.exit_code == 0 payload = json.loads(result.output) - assert payload["validation_type"] == "local_environment" + assert payload["validation_type"] == "openenv_validation" + assert payload["report_schema_version"] == "1.0" + assert payload["profile"] == "static" + assert payload["spec"]["id"] == "openenv" + assert payload["spec"]["adapter"] == { + "id": "openenv-yaml", + "version": "1", + } + assert payload["spec"]["execution_model"] == "served" assert payload["passed"] is True - assert payload["summary"]["passed_count"] == 1 - assert payload["summary"]["total_count"] == 1 + assert payload["summary"]["passed_count"] >= 1 + assert payload["summary"]["total_count"] >= 1 assert payload["summary"]["failed_criteria"] == [] +def test_profile_json_can_be_written_to_output_file(tmp_path: Path) -> None: + env_dir = tmp_path / "test_env" + report_path = tmp_path / "reports" / "validation.json" + _write_minimal_valid_env(env_dir) + + result = runner.invoke( + app, + [ + "validate", + str(env_dir), + "--profile", + "static", + "--json", + "--output", + str(report_path), + ], + ) + + assert result.exit_code == 0, result.output + stdout_payload = json.loads(result.output) + file_payload = json.loads(report_path.read_text()) + assert file_payload == stdout_payload + assert file_payload["profile"] == "static" + assert file_payload["report_schema_version"] == "1.0" + + def test_validate_command_rejects_environment_package_as_runtime_dependency( tmp_path: Path, ) -> None: @@ -259,6 +306,7 @@ def test_validate_command_accepts_dockerfile_managed_openenv_runtime( 'server = "server.app:main"\n' ) (env_dir / "server" / "Dockerfile").write_text( + "FROM python:3.12-slim\n" 'RUN pip install --no-cache-dir --no-deps "openenv[core]>=0.2.2"\n' )