diff --git a/.github/workflows/test_validate_readme_commands.py b/.github/workflows/test_validate_readme_commands.py new file mode 100644 index 0000000..c89dc2d --- /dev/null +++ b/.github/workflows/test_validate_readme_commands.py @@ -0,0 +1,1482 @@ +import io +import json +import subprocess +import tempfile +import textwrap +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +import sys +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from validate_readme_commands import ( + DEFAULT_LABS, + CommandExecutionError, + CommandSpec, + DockerWarmupResult, + LabExecutionResult, + PreflightCheckResult, + PreflightRequirements, + RunReport, + _expected_output_matches, + determine_preflight_requirements, + derive_final_cleanup_commands, + load_run_report, + main, + parse_readme_commands, + print_command_mapping, + print_run_report, + run_preflight, + warm_docker_images, + write_run_report, + reset_lab_changes, + resolve_readme_paths, + run_single_readme, + run_cleanup_commands, + run_command_specs, + snapshot_repo_state, + should_skip_command, + _remove_path, +) + + +ROOT_DIR = Path(__file__).resolve().parents[2] +SCRIPT_PATH = ROOT_DIR / ".github" / "workflows" / "validate_readme_commands.py" + + +class GitRepoTestCase(unittest.TestCase): + def _init_git_repo(self, repo_path: Path) -> None: + self._git(repo_path, "init") + self._git(repo_path, "config", "user.name", "Test User") + self._git(repo_path, "config", "user.email", "test@example.com") + + def _git(self, cwd: Path, *args: str) -> None: + completed = subprocess.run( + ["git", *args], + cwd=str(cwd), + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(completed.returncode, 0, msg=completed.stderr) + + +class ParseReadmeCommandsTests(GitRepoTestCase): + def test_collects_terminaloutput_blocks_until_next_shell_block(self) -> None: + readme_path = self._write_readme( + """ + Intro + + ```shell + echo first + ``` + + prose between blocks + + ```terminaloutput + first + ``` + + ```terminaloutput + second + ``` + + ```shell + echo third + ``` + """ + ) + + commands = parse_readme_commands(readme_path) + + self.assertEqual( + commands, + [ + CommandSpec( + command="echo first\n", + expected_outputs=["first\n", "second\n"], + ), + CommandSpec(command="echo third\n", expected_outputs=[]), + ], + ) + + def test_ignores_terminaloutput_before_first_shell_block(self) -> None: + readme_path = self._write_readme( + """ + ```terminaloutput + orphan + ``` + + ```shell + echo ok + ``` + """ + ) + + commands = parse_readme_commands(readme_path) + + self.assertEqual(commands, [CommandSpec(command="echo ok\n", expected_outputs=[])]) + + def test_repo_readme_parses_with_expected_shape(self) -> None: + commands = parse_readme_commands(ROOT_DIR / "api-coverage" / "README.md") + + self.assertEqual(len(commands), 7) + self.assertEqual( + commands[0].command, + "docker compose up test --build --abort-on-container-exit\n", + ) + self.assertEqual(len(commands[0].expected_outputs), 3) + self.assertEqual(commands[1].command, "docker compose down -v\n") + self.assertEqual(commands[1].expected_outputs, []) + + def _write_readme(self, content: str) -> Path: + temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(temp_dir.cleanup) + readme_path = Path(temp_dir.name) / "README.md" + readme_path.write_text(textwrap.dedent(content).lstrip(), encoding="utf-8") + return readme_path + + +class RunCommandSpecsTests(GitRepoTestCase): + def test_zero_exit_with_matching_output_passes(self) -> None: + results = run_command_specs( + [CommandSpec(command="printf 'hello\\n'", expected_outputs=["hello\n"])], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertEqual(len(results.results), 1) + self.assertEqual(results.results[0].returncode, 0) + + def test_non_zero_exit_with_matching_output_passes(self) -> None: + results = run_command_specs( + [CommandSpec(command="printf 'expected\\n'; exit 3", expected_outputs=["expected\n"])], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertEqual(len(results.results), 1) + self.assertFalse(results.results[0].skipped) + self.assertEqual(results.results[0].returncode, 3) + + def test_non_zero_exit_without_expected_output_fails(self) -> None: + with self.assertRaises(CommandExecutionError) as ctx: + run_command_specs( + [CommandSpec(command="printf 'wrong\\n'; exit 4", expected_outputs=["expected\n"])], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertIn("missing expected terminaloutput block #1", str(ctx.exception)) + self.assertIn("Exit code: 4", str(ctx.exception)) + + def test_non_zero_exit_without_terminaloutput_blocks_fails(self) -> None: + with self.assertRaises(CommandExecutionError) as ctx: + run_command_specs( + [CommandSpec(command="exit 7", expected_outputs=[])], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertIn("command exited non-zero without any expected terminaloutput blocks", str(ctx.exception)) + + def test_failure_message_shows_expected_and_closest_actual_line(self) -> None: + with self.assertRaises(CommandExecutionError) as ctx: + run_command_specs( + [ + CommandSpec( + command="printf 'Tests run: 1, Successes: 1, Failures: 0, WIP: 0, Errors: 0\\n'", + expected_outputs=["Tests run: 1, Successes: 1, Failures: 0, Errors: 0\n"], + ) + ], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + message = str(ctx.exception) + self.assertIn("Mismatch Detail", message) + self.assertIn("Expected line", message) + self.assertIn("Tests run: 1, Successes: 1, Failures: 0, Errors: 0", message) + self.assertIn("Closest actual line", message) + self.assertIn("Tests run: 1, Successes: 1, Failures: 0, WIP: 0, Errors: 0", message) + + def test_line_by_line_matching_allows_prefixed_actual_lines(self) -> None: + expected_output = ( + "Failed the following API Coverage Report success criteria:\n" + "Total API coverage: 50% is less than the specified minimum threshold of 100%.\n" + "Total missed operations: 1 is greater than the maximum threshold of 0.\n" + ) + actual_output = ( + "api-coverage-openapi-test | Failed the following API Coverage Report success criteria:\n" + "api-coverage-openapi-test | Total API coverage: 50% is less than the specified minimum threshold of 100%.\n" + "api-coverage-openapi-test | Total missed operations: 1 is greater than the maximum threshold of 0.\n" + ) + + self.assertTrue(_expected_output_matches(expected_output, actual_output)) + + +class RemovePathTests(unittest.TestCase): + @patch("validate_readme_commands.subprocess.run") + @patch("validate_readme_commands.Path.unlink") + def test_remove_path_uses_docker_fallback_on_permission_error( + self, + unlink_mock, + subprocess_run_mock, + ) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_root = Path(temp_dir) + target = repo_root / "quick-start-mock" / "specs" / "service_examples" / "pets.json" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("data", encoding="utf-8") + + unlink_mock.side_effect = PermissionError("permission denied") + subprocess_run_mock.return_value = subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + + _remove_path(target, repo_root) + + subprocess_run_mock.assert_called_once() + docker_command = subprocess_run_mock.call_args.args[0] + self.assertEqual(docker_command[:4], ["docker", "run", "--rm", "-v"]) + self.assertIn(f"{repo_root}:/workspace", docker_command) + self.assertIn("rm -rf '/workspace/quick-start-mock/specs/service_examples/pets.json'", docker_command) + + def test_line_by_line_matching_does_not_require_order(self) -> None: + expected_output = "first line\nsecond line\n" + actual_output = "prefix second line\nprefix first line\n" + + self.assertTrue(_expected_output_matches(expected_output, actual_output)) + + def test_line_by_line_matching_trims_expected_line_whitespace(self) -> None: + expected_output = " first line \n\tsecond line\t\n" + actual_output = "prefix first line\nprefix second line\n" + + self.assertTrue(_expected_output_matches(expected_output, actual_output)) + + def test_timeout_is_reported_cleanly(self) -> None: + with self.assertRaises(CommandExecutionError) as ctx: + run_command_specs( + [CommandSpec(command="sleep 2", expected_outputs=[])], + cwd=ROOT_DIR, + timeout_seconds=0.1, + ) + + self.assertIn("timed out after 0.1 seconds", str(ctx.exception)) + + def test_streams_command_output_in_realtime_to_terminal(self) -> None: + stdout_buffer = io.StringIO() + + with redirect_stdout(stdout_buffer), redirect_stderr(io.StringIO()): + results = run_command_specs( + [CommandSpec(command="printf 'streamed\\n'", expected_outputs=["streamed\n"])], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertEqual(len(results.results), 1) + self.assertIn("streamed\n", stdout_buffer.getvalue()) + + def test_skips_commands_containing_docker_and_studio(self) -> None: + results = run_command_specs( + [ + CommandSpec( + command="docker compose --profile studio up --build", + expected_outputs=[], + ) + ], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertEqual(len(results.results), 1) + self.assertTrue(results.results[0].skipped) + self.assertEqual(results.results[0].returncode, 0) + + def test_runs_cleanup_commands_after_failure_but_skips_non_cleanup_zero_output_commands(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + cleanup_marker = temp_path / "cleanup.txt" + non_cleanup_marker = temp_path / "non_cleanup.txt" + stderr_buffer = io.StringIO() + + with redirect_stderr(stderr_buffer): + with self.assertRaises(CommandExecutionError) as ctx: + run_command_specs( + [ + CommandSpec(command="printf 'actual\\n'", expected_outputs=["expected\n"]), + CommandSpec( + command=f"kill -0 $$ >/dev/null 2>&1; printf cleanup > {cleanup_marker.name}", + expected_outputs=[], + ), + CommandSpec(command=f"printf mutate > {non_cleanup_marker.name}", expected_outputs=[]), + CommandSpec(command="printf 'later\\n'", expected_outputs=["later\n"]), + ], + cwd=temp_path, + timeout_seconds=5, + ) + + self.assertIn("Cleanup commands executed", str(ctx.exception)) + self.assertIn( + "Validation failed at command #1. Skipping remaining commands and running cleanup commands...", + stderr_buffer.getvalue(), + ) + self.assertTrue(cleanup_marker.exists()) + self.assertFalse(non_cleanup_marker.exists()) + + +class CleanupCommandTests(GitRepoTestCase): + def test_derive_final_cleanup_commands_preserves_profiles(self) -> None: + cleanup_commands = derive_final_cleanup_commands( + [ + CommandSpec( + command="docker compose --profile test up test --build --abort-on-container-exit\n", + expected_outputs=[], + ), + CommandSpec( + command="docker compose run --rm mcp-test --enable-resiliency-tests\n", + expected_outputs=[], + ), + ] + ) + + self.assertEqual( + cleanup_commands, + [ + "docker compose --profile test down -v --remove-orphans", + "docker compose down -v --remove-orphans", + ], + ) + + def test_runs_only_cleanup_commands_before_next_expected_output_command(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + cleanup_marker = temp_path / "cleanup.txt" + later_marker = temp_path / "later.txt" + + results = run_cleanup_commands( + [ + CommandSpec( + command=f"kill -0 $$ >/dev/null 2>&1; printf done > {cleanup_marker.name}", + expected_outputs=[], + ), + CommandSpec(command=f"printf skip > {later_marker.name}", expected_outputs=[]), + CommandSpec(command="printf 'stop here\\n'", expected_outputs=["stop here\n"]), + ], + cwd=temp_path, + timeout_seconds=5, + ) + + self.assertEqual(len(results), 1) + self.assertTrue(cleanup_marker.exists()) + self.assertFalse(later_marker.exists()) + + def test_reset_lab_changes_restores_new_tracked_changes_only(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = Path(temp_dir) + self._init_git_repo(repo_path) + tracked_path = repo_path / "tracked.txt" + preserved_path = repo_path / "preserved.txt" + tracked_path.write_text("original\n", encoding="utf-8") + preserved_path.write_text("keep me dirty\n", encoding="utf-8") + self._git(repo_path, "add", "tracked.txt", "preserved.txt") + self._git(repo_path, "commit", "-m", "init") + preserved_path.write_text("user edit\n", encoding="utf-8") + + baseline = snapshot_repo_state(repo_path) + tracked_path.write_text("lab edit\n", encoding="utf-8") + created_path = repo_path / "created.txt" + created_path.write_text("new file\n", encoding="utf-8") + + reset_summary = reset_lab_changes(repo_path, baseline) + + self.assertEqual(tracked_path.read_text(encoding="utf-8"), "original\n") + self.assertEqual(preserved_path.read_text(encoding="utf-8"), "user edit\n") + self.assertFalse(created_path.exists()) + self.assertEqual(reset_summary.restored, [Path("tracked.txt")]) + self.assertEqual(reset_summary.removed, [Path("created.txt")]) + + +class SkipCommandTests(GitRepoTestCase): + def test_skip_rule_requires_both_docker_and_studio(self) -> None: + self.assertTrue(should_skip_command("docker compose --profile studio up --build")) + self.assertFalse(should_skip_command("docker compose up test --build")) + self.assertFalse(should_skip_command("open specmatic studio")) + + +class PreflightTests(GitRepoTestCase): + def test_determine_preflight_requirements_for_non_docker_lab(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) + (lab_path / "README.md").write_text( + textwrap.dedent( + """ + ```shell + printf 'ok\\n' + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + requirements = determine_preflight_requirements([lab_path / "README.md"]) + + self.assertEqual( + requirements, + PreflightRequirements( + docker_cli=False, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + def test_determine_preflight_requirements_for_repo_lab(self) -> None: + requirements = determine_preflight_requirements([ROOT_DIR / "api-coverage" / "README.md"]) + + self.assertEqual( + requirements, + PreflightRequirements( + docker_cli=True, + docker_compose=True, + license_validation=True, + remote_contract_access=False, + ), + ) + + def test_run_preflight_returns_empty_when_no_checks_required(self) -> None: + results = run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=False, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + self.assertEqual(results, []) + + def test_run_preflight_reports_missing_docker(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + side_effect=OSError("No such file or directory: 'docker'"), + ): + results = run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=True, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[0].name, "docker") + self.assertFalse(results[0].passed) + self.assertIn("No such file or directory", results[0].detail or "") + + def test_run_preflight_reports_missing_docker_compose(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + return_value=subprocess.CompletedProcess( + args=["docker", "compose", "version"], + returncode=1, + stdout="", + stderr="docker: 'compose' is not a docker command", + ), + ): + results = run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=False, + docker_compose=True, + license_validation=False, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[0].name, "docker compose") + self.assertFalse(results[0].passed) + self.assertIn("not a docker command", results[0].detail or "") + + def test_run_preflight_reports_unreachable_docker_daemon(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + side_effect=[ + subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "info"], returncode=1, stdout="", stderr="Cannot connect to the Docker daemon"), + ], + ): + results = run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=True, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[1].name, "docker daemon") + self.assertFalse(results[1].passed) + self.assertIn("Cannot connect to the Docker daemon", results[1].detail or "") + + def test_run_preflight_reports_missing_license_file(self) -> None: + with ( + patch( + "validate_readme_commands.subprocess.run", + side_effect=[ + subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "compose", "version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "info"], returncode=0, stdout="", stderr=""), + ], + ), + patch("validate_readme_commands.Path.is_file", return_value=False), + ): + results = run_preflight( + [ROOT_DIR / "api-coverage" / "README.md"], + PreflightRequirements( + docker_cli=True, + docker_compose=True, + license_validation=True, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[3].name, "specmatic license file exists") + self.assertFalse(results[3].passed) + self.assertEqual(results[4].name, "specmatic license validation") + self.assertFalse(results[4].passed) + self.assertTrue(results[4].skipped) + + def test_run_preflight_reports_invalid_license(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + side_effect=[ + subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "compose", "version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "info"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "run"], returncode=1, stdout="", stderr="License expired"), + ], + ) as mocked_run, patch("validate_readme_commands.Path.is_file", return_value=True): + results = run_preflight( + [ROOT_DIR / "api-coverage" / "README.md"], + PreflightRequirements( + docker_cli=True, + docker_compose=True, + license_validation=True, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[4].name, "specmatic license validation") + self.assertFalse(results[4].passed) + self.assertIn("License expired", results[4].detail or "") + license_call = mocked_run.call_args_list[3] + self.assertEqual( + license_call.kwargs["cwd"], + str(ROOT_DIR), + ) + self.assertEqual( + license_call.args[0], + [ + "docker", + "run", + "--rm", + "-v", + "./license.txt:/specmatic/specmatic-license.txt:ro", + "-e", + "SPECMATIC_LICENSE_PATH=/specmatic/specmatic-license.txt", + "specmatic/enterprise:latest", + "show-license", + ], + ) + + def test_run_preflight_reports_expired_license_even_when_show_license_exits_zero(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + side_effect=[ + subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "compose", "version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "info"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess( + args=["docker", "run"], + returncode=0, + stdout=( + "WARNING: License loaded from /specmatic/specmatic-license.txt is expired as of May 16, 2026 at 6:30:32 AM UTC\n" + "Using Specmatic Trial license initialized from jar:file:/usr/local/share/enterprise/enterprise.jar!/specmatic-default-trial-license.txt\n" + "License details:\n" + ), + stderr="", + ), + ], + ), patch("validate_readme_commands.Path.is_file", return_value=True): + results = run_preflight( + [ROOT_DIR / "api-coverage" / "README.md"], + PreflightRequirements( + docker_cli=True, + docker_compose=True, + license_validation=True, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[4].name, "specmatic license validation") + self.assertFalse(results[4].passed) + self.assertIn("is expired", results[4].detail or "") + self.assertIn("initialized from jar:file:", results[4].detail or "") + + def test_run_preflight_skips_license_validation_when_not_required(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + return_value=subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + ) as mocked_run: + results = run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=True, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + self.assertTrue(all(result.name != "specmatic license validation" for result in results)) + self.assertEqual(mocked_run.call_count, 2) + + def test_run_preflight_reports_remote_contract_access_failure(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + return_value=subprocess.CompletedProcess(args=["git", "ls-remote"], returncode=128, stdout="", stderr="Could not resolve host: github.com"), + ): + results = run_preflight( + [ROOT_DIR / "response-templating" / "README.md"], + PreflightRequirements( + docker_cli=False, + docker_compose=False, + license_validation=False, + remote_contract_access=True, + ), + ) + + self.assertEqual(results[0].name, "labs-contracts access") + self.assertFalse(results[0].passed) + self.assertIn("Could not resolve host", results[0].detail or "") + + def test_run_preflight_skips_remote_contract_check_when_not_required(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + return_value=subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + ) as mocked_run: + run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=True, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + commands = [call.args[0] for call in mocked_run.call_args_list] + self.assertNotIn( + ["git", "ls-remote", "--exit-code", "https://github.com/specmatic/labs-contracts.git", "HEAD"], + commands, + ) + + def test_warm_docker_images_skips_non_compose_lab(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) + (lab_path / "README.md").write_text( + textwrap.dedent( + """ + ```shell + printf 'ok\\n' + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + results = warm_docker_images([lab_path / "README.md"]) + + self.assertEqual(results, []) + + def test_warm_docker_images_runs_compose_pull_with_extended_timeout(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + return_value=subprocess.CompletedProcess(args=["docker", "compose", "pull"], returncode=0, stdout="", stderr=""), + ) as mocked_run: + results = warm_docker_images([ROOT_DIR / "api-coverage" / "README.md"]) + + self.assertEqual(len(results), 1) + self.assertEqual(results[0].lab_name, "api-coverage") + self.assertTrue(results[0].passed) + warmup_call = mocked_run.call_args_list[0] + self.assertEqual( + warmup_call.args[0], + ["docker", "compose", "pull", "--ignore-buildable"], + ) + self.assertEqual(warmup_call.kwargs["cwd"], str(ROOT_DIR / "api-coverage")) + self.assertEqual(warmup_call.kwargs["timeout"], 300.0) + + def test_warm_docker_images_reports_timeout(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd="docker compose pull", timeout=300), + ): + results = warm_docker_images([ROOT_DIR / "api-coverage" / "README.md"]) + + self.assertEqual(results[0].lab_name, "api-coverage") + self.assertFalse(results[0].passed) + self.assertIn("timed out after 300 seconds", results[0].detail or "") + + +class MainTests(GitRepoTestCase): + def test_resolve_readme_paths_uses_default_labs_when_no_arg(self) -> None: + readme_paths = resolve_readme_paths(None) + + self.assertEqual(len(readme_paths), len(DEFAULT_LABS)) + self.assertTrue(str(readme_paths[0]).endswith("/api-coverage/README.md")) + self.assertTrue(str(readme_paths[-1]).endswith("/workflow-in-same-spec/README.md")) + + def test_resolve_readme_paths_appends_readme_to_lab_directory(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) / "sample-lab" + lab_path.mkdir() + + readme_paths = resolve_readme_paths(str(lab_path)) + + self.assertEqual(readme_paths, [(lab_path / "README.md").resolve()]) + + def test_main_returns_zero_for_valid_readme(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) + readme_path = lab_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf 'ok\\n' + ``` + + ```terminaloutput + ok + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + with patch("validate_readme_commands.run_preflight", return_value=[]): + with patch("validate_readme_commands.warm_docker_images", return_value=[]): + exit_code = main([str(lab_path), "--timeout", "5"]) + + self.assertEqual(exit_code, 0) + + def test_main_single_lab_skips_preflight(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) + readme_path = lab_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf 'ok\\n' + ``` + + ```terminaloutput + ok + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + with ( + patch("validate_readme_commands.run_preflight") as mocked_preflight, + patch("validate_readme_commands.warm_docker_images", return_value=[]), + ): + exit_code = main([str(lab_path), "--timeout", "5"]) + + self.assertEqual(exit_code, 0) + mocked_preflight.assert_not_called() + + def test_dry_run_prints_command_mapping(self) -> None: + stdout_buffer = io.StringIO() + command_specs = [ + CommandSpec(command="echo one\n", expected_outputs=["one\n", "two\n"]), + CommandSpec(command="echo two\n", expected_outputs=[]), + ] + + with redirect_stdout(stdout_buffer): + print_command_mapping(command_specs) + + output = stdout_buffer.getvalue() + self.assertIn("Command #1", output) + self.assertIn("Shell", output) + self.assertIn("expected terminaloutput blocks: 2", output) + self.assertIn(" echo one", output) + self.assertIn(" echo one\n\nterminaloutput #1", output) + self.assertIn("terminaloutput #1", output) + self.assertIn(" one", output) + self.assertIn("terminaloutput #2", output) + self.assertIn(" two", output) + self.assertIn("Command #2", output) + self.assertIn("terminaloutput none", output) + + def test_cli_invocation_reports_failure(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) + readme_path = lab_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf 'actual\\n' + ``` + + ```terminaloutput + expected + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + ["python3", str(SCRIPT_PATH), str(lab_path)], + cwd=str(ROOT_DIR), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(completed.returncode, 1) + self.assertIn("missing expected terminaloutput block #1", completed.stderr) + self.assertTrue(completed.stdout.rstrip().endswith("FAIL")) + + def test_cli_invocation_prints_failure_summary(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) + readme_path = lab_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf 'actual\\n' + ``` + + ```terminaloutput + expected + ``` + + ```shell + printf 'later\\n' + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + ["python3", str(SCRIPT_PATH), str(lab_path)], + cwd=str(ROOT_DIR), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(completed.returncode, 1) + self.assertIn("FAIL command #1", completed.stdout) + self.assertIn("SKIP command #2", completed.stdout) + self.assertTrue(completed.stdout.rstrip().endswith("FAIL")) + + def test_dry_run_does_not_execute_commands(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) + readme_path = lab_path / "README.md" + marker_path = lab_path / "marker.txt" + readme_path.write_text( + textwrap.dedent( + f""" + ```shell + printf executed > {marker_path.name} + ``` + + ```terminaloutput + executed + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + [ + "python3", + str(SCRIPT_PATH), + "--dry-run", + str(lab_path), + ], + cwd=str(lab_path), + capture_output=True, + text=True, + check=False, + ) + + self.assertFalse(marker_path.exists()) + + self.assertEqual(completed.returncode, 0) + self.assertIn("Command #1", completed.stdout) + + def test_cli_invocation_prints_pass_at_end(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = Path(temp_dir) + self._init_git_repo(repo_path) + readme_path = repo_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf ok + ``` + + ```terminaloutput + ok + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + ["python3", str(SCRIPT_PATH), str(repo_path)], + cwd=str(repo_path), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(completed.returncode, 0) + self.assertIn("Command #1", completed.stdout) + self.assertIn("expected terminaloutput blocks: 1", completed.stdout) + self.assertTrue(completed.stdout.rstrip().endswith("PASS")) + + def test_cli_invocation_prints_command_execution_banners(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = Path(temp_dir) + self._init_git_repo(repo_path) + readme_path = repo_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf 'hello\\n' + ``` + + ```terminaloutput + hello + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + ["python3", str(SCRIPT_PATH), str(repo_path)], + cwd=str(repo_path), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(completed.returncode, 0) + self.assertIn("BEGIN command #1", completed.stdout) + self.assertIn("Command #1 output:", completed.stdout) + self.assertIn("END command #1 (exit 0)", completed.stdout) + + def test_main_resets_lab_changed_files_by_default(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = Path(temp_dir) + self._init_git_repo(repo_path) + target_path = repo_path / "tracked.txt" + target_path.write_text("original\n", encoding="utf-8") + self._git(repo_path, "add", "tracked.txt") + self._git(repo_path, "commit", "-m", "init") + readme_path = repo_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf changed > tracked.txt + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + ["python3", str(SCRIPT_PATH), str(repo_path)], + cwd=str(repo_path), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(target_path.read_text(encoding="utf-8"), "original\n") + + self.assertEqual(completed.returncode, 0) + self.assertIn("RESET (restored: 1, removed: 0)", completed.stdout) + + def test_main_skip_reset_preserves_lab_changed_files(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = Path(temp_dir) + self._init_git_repo(repo_path) + target_path = repo_path / "tracked.txt" + target_path.write_text("original\n", encoding="utf-8") + self._git(repo_path, "add", "tracked.txt") + self._git(repo_path, "commit", "-m", "init") + readme_path = repo_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf changed > tracked.txt + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + [ + "python3", + str(SCRIPT_PATH), + "--skip-reset", + str(repo_path), + ], + cwd=str(repo_path), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(target_path.read_text(encoding="utf-8"), "changed") + + self.assertEqual(completed.returncode, 0) + self.assertNotIn("RESET", completed.stdout) + + def test_main_without_readme_runs_all_default_labs_and_continues_after_failure(self) -> None: + fake_readmes = [ + Path("/tmp/lab-one/README.md"), + Path("/tmp/lab-two/README.md"), + Path("/tmp/lab-three/README.md"), + ] + stdout_buffer = io.StringIO() + calls: list[Path] = [] + + def fake_run_single_readme( + *, + readme_path: Path, + dry_run: bool, + skip_reset: bool, + timeout_seconds: float, + ) -> LabExecutionResult: + calls.append(readme_path) + if readme_path == fake_readmes[0]: + return LabExecutionResult( + name="lab-one", + exit_code=0, + duration_seconds=0.0, + validated_commands=5, + total_commands=5, + skipped_commands=0, + ) + if readme_path == fake_readmes[1]: + return LabExecutionResult( + name="lab-two", + exit_code=1, + duration_seconds=0.0, + validated_commands=2, + total_commands=4, + skipped_commands=2, + ) + return LabExecutionResult( + name="lab-three", + exit_code=0, + duration_seconds=0.0, + validated_commands=3, + total_commands=5, + skipped_commands=2, + ) + + with ( + patch("validate_readme_commands.resolve_readme_paths", return_value=fake_readmes), + patch("pathlib.Path.is_file", return_value=True), + patch("validate_readme_commands.run_preflight", return_value=[]), + patch("validate_readme_commands.warm_docker_images", return_value=[]) as mocked_warmup, + patch("validate_readme_commands.run_single_readme", side_effect=fake_run_single_readme), + patch( + "validate_readme_commands.time.perf_counter", + side_effect=[0.0, 1.5, 1.5, 4.0, 4.0, 7.25], + ), + redirect_stdout(stdout_buffer), + ): + exit_code = main([]) + + self.assertEqual(exit_code, 1) + warmed_readmes = [call.args[0][0] for call in mocked_warmup.call_args_list] + self.assertEqual(warmed_readmes, fake_readmes) + self.assertEqual(calls, fake_readmes) + output = stdout_buffer.getvalue() + self.assertIn("===== lab-one =====", output) + self.assertIn("===== lab-two =====", output) + self.assertIn("===== lab-three =====", output) + self.assertIn("===== Summary =====", output) + self.assertIn("PASS labs: 2", output) + self.assertIn("FAIL labs: 1", output) + self.assertIn("Lab", output) + self.assertIn("Status", output) + self.assertIn("Duration", output) + self.assertIn("Validated", output) + self.assertIn("Skipped", output) + self.assertIn("lab-one PASS", output) + self.assertIn("1.50s", output) + self.assertIn("5/5", output) + self.assertIn("lab-three PASS", output) + self.assertIn("3.25s", output) + self.assertIn("3/5", output) + self.assertIn("lab-two FAIL", output) + self.assertIn("2.50s", output) + self.assertIn("2/4", output) + + def test_main_aborts_before_running_labs_when_preflight_fails(self) -> None: + stdout_buffer = io.StringIO() + + with ( + patch( + "validate_readme_commands.resolve_readme_paths", + return_value=[ROOT_DIR / "api-coverage" / "README.md"], + ), + patch("pathlib.Path.is_file", return_value=True), + patch( + "validate_readme_commands.run_preflight", + return_value=[ + PreflightCheckResult( + name="docker daemon", + passed=False, + skipped=False, + detail="Docker is not running", + suggestion="Start Docker Desktop.", + ) + ], + ), + patch("validate_readme_commands.warm_docker_images") as mocked_warmup, + patch("validate_readme_commands.run_single_readme") as mocked_run_single_readme, + redirect_stdout(stdout_buffer), + ): + exit_code = main([]) + + self.assertEqual(exit_code, 1) + mocked_warmup.assert_not_called() + mocked_run_single_readme.assert_not_called() + output = stdout_buffer.getvalue() + self.assertIn("===== Preflight =====", output) + self.assertIn("Preflight failed. No labs were executed.", output) + self.assertNotIn("===== api-coverage =====", output) + + def test_print_preflight_results_uses_skip_for_dependent_checks(self) -> None: + stdout_buffer = io.StringIO() + + with redirect_stdout(stdout_buffer): + from validate_readme_commands import print_preflight_results + + print_preflight_results( + [ + PreflightCheckResult(name="docker", passed=True), + PreflightCheckResult( + name="docker daemon", + passed=False, + detail="Docker is not running", + suggestion="Start Docker Desktop.", + ), + PreflightCheckResult( + name="specmatic license validation", + passed=False, + skipped=True, + detail="skipped because Docker or the license file is unavailable", + suggestion="Fix Docker access and the license file, then rerun the validator.", + ), + ] + ) + + output = stdout_buffer.getvalue() + self.assertIn("PASS docker", output) + self.assertIn("FAIL docker daemon: Docker is not running", output) + self.assertIn("SKIP specmatic license validation: skipped because Docker or the license file is unavailable", output) + + def test_main_prints_preflight_once_then_runs_labs(self) -> None: + stdout_buffer = io.StringIO() + fake_readmes = [Path("/tmp/lab-one/README.md")] + + with ( + patch("validate_readme_commands.resolve_readme_paths", return_value=fake_readmes), + patch("pathlib.Path.is_file", return_value=True), + patch( + "validate_readme_commands.run_preflight", + return_value=[PreflightCheckResult(name="docker", passed=True)], + ), + patch("validate_readme_commands.warm_docker_images", return_value=[]), + patch( + "validate_readme_commands.run_single_readme", + return_value=LabExecutionResult( + name="lab-one", + exit_code=0, + duration_seconds=0.0, + validated_commands=1, + total_commands=1, + skipped_commands=0, + ), + ), + patch("validate_readme_commands.time.perf_counter", side_effect=[0.0, 1.0]), + redirect_stdout(stdout_buffer), + ): + exit_code = main([]) + + self.assertEqual(exit_code, 0) + output = stdout_buffer.getvalue() + self.assertEqual(output.count("===== Preflight ====="), 1) + self.assertIn("PASS docker", output) + + def test_main_marks_current_lab_failed_when_docker_warmup_fails(self) -> None: + stdout_buffer = io.StringIO() + fake_readmes = [Path("/tmp/lab-one/README.md"), Path("/tmp/lab-two/README.md")] + + with ( + patch("validate_readme_commands.resolve_readme_paths", return_value=fake_readmes), + patch("pathlib.Path.is_file", return_value=True), + patch("validate_readme_commands.run_preflight", return_value=[]), + patch( + "validate_readme_commands.warm_docker_images", + side_effect=[ + [DockerWarmupResult(lab_name="lab-one", passed=False, detail="timed out after 300 seconds")], + [], + ], + ), + patch( + "validate_readme_commands.run_single_readme", + return_value=LabExecutionResult( + name="lab-two", + exit_code=0, + duration_seconds=0.0, + validated_commands=1, + total_commands=1, + skipped_commands=0, + ), + ) as mocked_run_single_readme, + patch("validate_readme_commands.time.perf_counter", side_effect=[0.0, 1.0, 1.0, 2.0]), + redirect_stdout(stdout_buffer), + ): + exit_code = main([]) + + self.assertEqual(exit_code, 1) + mocked_run_single_readme.assert_called_once_with( + readme_path=fake_readmes[1], + dry_run=False, + skip_reset=False, + timeout_seconds=120.0, + ) + output = stdout_buffer.getvalue() + self.assertIn("===== lab-one =====", output) + self.assertIn("===== Docker Warmup =====", output) + self.assertIn("FAIL lab-one: timed out after 300 seconds", output) + self.assertIn("===== lab-two =====", output) + self.assertIn("PASS labs: 1", output) + self.assertIn("FAIL labs: 1", output) + + def test_main_dry_run_summary_does_not_report_pass_or_fail(self) -> None: + fake_readmes = [ + Path("/tmp/lab-one/README.md"), + Path("/tmp/lab-two/README.md"), + ] + stdout_buffer = io.StringIO() + + def fake_run_single_readme( + *, + readme_path: Path, + dry_run: bool, + skip_reset: bool, + timeout_seconds: float, + ) -> LabExecutionResult: + self.assertTrue(dry_run) + if readme_path == fake_readmes[0]: + return LabExecutionResult( + name="lab-one", + exit_code=0, + duration_seconds=0.0, + validated_commands=0, + total_commands=5, + skipped_commands=0, + ) + return LabExecutionResult( + name="lab-two", + exit_code=0, + duration_seconds=0.0, + validated_commands=0, + total_commands=3, + skipped_commands=0, + ) + + with ( + patch("validate_readme_commands.resolve_readme_paths", return_value=fake_readmes), + patch("pathlib.Path.is_file", return_value=True), + patch("validate_readme_commands.run_single_readme", side_effect=fake_run_single_readme), + patch("validate_readme_commands.time.perf_counter", side_effect=[0.0, 1.0, 1.0, 2.0]), + redirect_stdout(stdout_buffer), + ): + exit_code = main(["--dry-run"]) + + self.assertEqual(exit_code, 0) + output = stdout_buffer.getvalue() + self.assertIn("===== Summary =====", output) + self.assertIn("DRY RUN labs: 2", output) + self.assertIn("Lab", output) + self.assertIn("Status", output) + self.assertIn("Duration", output) + self.assertIn("Commands", output) + self.assertIn("lab-one DRY RUN", output) + self.assertIn("lab-two DRY RUN", output) + self.assertIn("1.00s", output) + self.assertIn("5", output) + self.assertIn("3", output) + self.assertNotIn("PASS labs:", output) + self.assertNotIn("FAIL labs:", output) + + def test_main_preflight_only_runs_checks_and_exits(self) -> None: + stdout_buffer = io.StringIO() + + with ( + patch( + "validate_readme_commands.run_preflight", + return_value=[PreflightCheckResult(name="docker", passed=True)], + ), + patch("validate_readme_commands.run_single_readme") as mocked_run_single_readme, + redirect_stdout(stdout_buffer), + ): + exit_code = main(["--preflight-only"]) + + self.assertEqual(exit_code, 0) + mocked_run_single_readme.assert_not_called() + output = stdout_buffer.getvalue() + self.assertIn("===== Preflight =====", output) + + def test_main_writes_result_json_for_single_lab(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + result_path = Path(temp_dir) / "result.json" + + with ( + patch( + "validate_readme_commands.resolve_readme_paths", + return_value=[Path("/tmp/lab-one/README.md")], + ), + patch("pathlib.Path.is_file", return_value=True), + patch("validate_readme_commands.warm_docker_images", return_value=[]), + patch( + "validate_readme_commands.run_single_readme", + return_value=LabExecutionResult( + name="lab-one", + exit_code=0, + duration_seconds=0.0, + validated_commands=2, + total_commands=2, + skipped_commands=0, + ), + ), + patch("validate_readme_commands.time.perf_counter", side_effect=[0.0, 1.0]), + ): + exit_code = main(["/tmp/lab-one", "--result-json", str(result_path)]) + + self.assertEqual(exit_code, 0) + payload = json.loads(result_path.read_text(encoding="utf-8")) + self.assertEqual(payload["mode"], "execution") + self.assertEqual(payload["labs"][0]["name"], "lab-one") + + def test_load_and_print_run_report_from_directory(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + result_dir = Path(temp_dir) + write_run_report( + result_dir / "lab-one.json", + RunReport( + mode="execution", + labs=[ + LabExecutionResult( + name="lab-one", + exit_code=0, + duration_seconds=1.5, + validated_commands=5, + total_commands=5, + skipped_commands=0, + ) + ], + ), + ) + write_run_report( + result_dir / "lab-two.json", + RunReport( + mode="execution", + labs=[ + LabExecutionResult( + name="lab-two", + exit_code=1, + duration_seconds=2.5, + validated_commands=2, + total_commands=4, + skipped_commands=2, + ) + ], + ), + ) + + report = load_run_report(result_dir) + stdout_buffer = io.StringIO() + with redirect_stdout(stdout_buffer): + print_run_report(report) + + output = stdout_buffer.getvalue() + self.assertIn("PASS labs: 1", output) + self.assertIn("FAIL labs: 1", output) + self.assertIn("lab-one", output) + self.assertIn("lab-two", output) + + def test_main_report_from_directory_prints_consolidated_report(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + result_dir = Path(temp_dir) + (result_dir / "lab-one.json").write_text( + json.dumps( + { + "mode": "execution", + "labs": [ + { + "name": "lab-one", + "exit_code": 0, + "duration_seconds": 1.0, + "validated_commands": 1, + "total_commands": 1, + "skipped_commands": 0, + } + ], + } + ), + encoding="utf-8", + ) + stdout_buffer = io.StringIO() + with redirect_stdout(stdout_buffer): + exit_code = main(["--report-from", str(result_dir)]) + + self.assertEqual(exit_code, 0) + self.assertIn("===== Summary =====", stdout_buffer.getvalue()) + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/validate-readmes.yaml b/.github/workflows/validate-readmes.yaml new file mode 100644 index 0000000..00f3a59 --- /dev/null +++ b/.github/workflows/validate-readmes.yaml @@ -0,0 +1,131 @@ +name: Validate Lab ReadMes + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +jobs: + preflight: + name: Shared Preflight + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Run validator unit tests + run: python3 -m unittest discover -s .github/workflows -p 'test_validate_readme_commands.py' -q + + - name: Run shared preflight + run: python3 .github/workflows/validate_readme_commands.py --preflight-only + + validate-lab: + name: Validate ${{ matrix.lab }} + needs: preflight + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + lab: + - api-coverage + - schema-design + - response-templating + - api-resiliency-testing + - api-security-schemes + - async-event-flow + - backward-compatibility-testing + - continuous-integration + - data-adapters + - dictionary + - external-examples + - filters + - kafka-avro + - kafka-sqs-retry-dlq + - mcp-auto-test + - overlays + - partial-examples + - quick-start-api-testing + - quick-start-async-contract-testing + - quick-start-contract-testing + - quick-start-mock + - schema-resiliency-testing + - workflow-in-same-spec + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Ensure origin/main is available + run: git fetch origin main:refs/remotes/origin/main + + - name: Validate lab README + run: | + mkdir -p .artifacts/readme-validator + python3 .github/workflows/validate_readme_commands.py "${{ matrix.lab }}" \ + --result-json ".artifacts/readme-validator/${{ matrix.lab }}.json" + + - name: Upload lab validation result + if: always() + uses: actions/upload-artifact@v6 + with: + name: readme-validator-${{ matrix.lab }} + path: .artifacts/readme-validator/${{ matrix.lab }}.json + if-no-files-found: error + + report: + name: Consolidated Report + if: always() + needs: + - preflight + - validate-lab + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Note preflight failure + if: needs.preflight.result != 'success' + run: | + echo "Shared preflight failed. Lab matrix did not run." | tee -a "$GITHUB_STEP_SUMMARY" + exit 1 + + - name: Download lab validation artifacts + if: needs.preflight.result == 'success' + uses: actions/download-artifact@v7 + with: + pattern: readme-validator-* + path: .artifacts/readme-validator + merge-multiple: true + + - name: Print consolidated report + if: needs.preflight.result == 'success' + run: | + python3 .github/workflows/validate_readme_commands.py \ + --report-from ".artifacts/readme-validator" | tee .artifacts/readme-validator/summary.txt + { + echo '```text' + cat .artifacts/readme-validator/summary.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/validate_readme_commands.py b/.github/workflows/validate_readme_commands.py new file mode 100644 index 0000000..a52d92f --- /dev/null +++ b/.github/workflows/validate_readme_commands.py @@ -0,0 +1,1734 @@ +#!/usr/bin/env python3 +"""Validate shell commands and expected terminal output in a README.""" + +from __future__ import annotations + +import argparse +import difflib +import json +import queue +import shutil +import shlex +import subprocess +import sys +import threading +import time +import traceback +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Sequence + +ANSI_RESET = "\033[0m" +ANSI_BOLD = "\033[1m" +ANSI_CYAN = "\033[36m" +ANSI_GREEN = "\033[32m" +ANSI_RED = "\033[31m" +ANSI_YELLOW = "\033[33m" +ANSI_DIM = "\033[2m" +DOCKER_WARMUP_TIMEOUT_SECONDS = 300.0 + +DEFAULT_LABS = [ + "api-coverage", + "schema-design", + "response-templating", + "api-resiliency-testing", + "api-security-schemes", + "async-event-flow", + "backward-compatibility-testing", + "continuous-integration", + "data-adapters", + "dictionary", + "external-examples", + "filters", + "kafka-avro", + "kafka-sqs-retry-dlq", + "mcp-auto-test", + "overlays", + "partial-examples", + "quick-start-api-testing", + "quick-start-async-contract-testing", + "quick-start-contract-testing", + "quick-start-mock", + "schema-resiliency-testing", + "workflow-in-same-spec", +] + + +def _find_repo_root_from_script() -> Path: + script_path = Path(__file__).resolve() + for candidate in script_path.parents: + if (candidate / ".git").exists(): + return candidate + return script_path.parents[2] + + +REPO_ROOT = _find_repo_root_from_script() + + +@dataclass(frozen=True) +class ValidationRunSummary: + results: list["CommandResult"] + failure_message: str | None = None + failed_index: int | None = None + + +@dataclass(frozen=True) +class CommandSpec: + command: str + expected_outputs: list[str] + + +@dataclass(frozen=True) +class CommandResult: + index: int + command: str + expected_outputs: list[str] + cwd: Path + skipped: bool + returncode: int + stdout: str + stderr: str + + @property + def combined_output(self) -> str: + return f"{self.stdout}{self.stderr}" + + +@dataclass(frozen=True) +class FencedBlock: + language: str + content: str + + +@dataclass(frozen=True) +class RepoSnapshot: + repo_root: Path + scope: Path + tracked_dirty: set[Path] + untracked: set[Path] + + +@dataclass(frozen=True) +class ResetSummary: + restored: list[Path] + removed: list[Path] + + +@dataclass(frozen=True) +class LabExecutionResult: + name: str + exit_code: int + duration_seconds: float + validated_commands: int + total_commands: int + skipped_commands: int + + +@dataclass(frozen=True) +class PreflightRequirements: + docker_cli: bool + docker_compose: bool + license_validation: bool + remote_contract_access: bool + + @property + def any_required(self) -> bool: + return any( + ( + self.docker_cli, + self.docker_compose, + self.license_validation, + self.remote_contract_access, + ) + ) + + +@dataclass(frozen=True) +class PreflightCheckResult: + name: str + passed: bool + skipped: bool = False + detail: str | None = None + suggestion: str | None = None + + +@dataclass(frozen=True) +class DockerWarmupResult: + lab_name: str + passed: bool + detail: str | None = None + + +@dataclass(frozen=True) +class RunReport: + mode: str + labs: list[LabExecutionResult] + + +class ReadmeValidationError(Exception): + """Base error for README command validation.""" + + +class ReadmeParseError(ReadmeValidationError): + """Raised when the README structure is invalid.""" + + +class CommandExecutionError(ReadmeValidationError): + """Raised when a command times out or output validation fails.""" + + +class CommandValidationFailure(CommandExecutionError): + """Raised when a command output does not satisfy expectations.""" + + +class ValidationStopped(CommandExecutionError): + """Raised when validation stops early and a summary should still be printed.""" + + def __init__(self, summary: ValidationRunSummary) -> None: + super().__init__(summary.failure_message or "Validation stopped.") + self.summary = summary + + +class GitInteractionError(ReadmeValidationError): + """Raised when git state cannot be inspected or restored.""" + + +def _supports_color() -> bool: + return sys.stdout.isatty() + + +def _style(text: str, *codes: str) -> str: + if not _supports_color() or not codes: + return text + return f"{''.join(codes)}{text}{ANSI_RESET}" + + +def should_skip_command(command: str) -> bool: + normalized_command = command.lower() + return "docker" in normalized_command and "studio" in normalized_command + + +def parse_readme_commands(readme_path: Path) -> list[CommandSpec]: + lines = readme_path.read_text(encoding="utf-8").splitlines(keepends=True) + blocks = _parse_fenced_blocks(lines) + + commands: list[CommandSpec] = [] + for block_index, block in enumerate(blocks): + if block.language != "shell": + continue + + expected_outputs: list[str] = [] + next_shell_index = len(blocks) + for later_index in range(block_index + 1, len(blocks)): + if _is_command_block_language(blocks[later_index].language): + next_shell_index = later_index + break + + for later_block in blocks[block_index + 1 : next_shell_index]: + if later_block.language == "terminaloutput": + expected_outputs.append(later_block.content) + + commands.append( + CommandSpec( + command=block.content, + expected_outputs=expected_outputs, + ) + ) + + return commands + + +def _readme_command_specs(readme_path: Path) -> list[CommandSpec]: + try: + return parse_readme_commands(readme_path) + except (OSError, ReadmeValidationError): + return [] + + +def _is_command_block_language(language: str) -> bool: + return language in {"shell", "powershell"} + + +def _parse_fenced_blocks(lines: Sequence[str]) -> list[FencedBlock]: + blocks: list[FencedBlock] = [] + current_language: str | None = None + current_lines: list[str] = [] + + for line in lines: + stripped = line.strip() + if current_language is None: + if stripped.startswith("```"): + current_language = stripped[3:].strip() + current_lines = [] + continue + + if stripped == "```": + blocks.append(FencedBlock(language=current_language, content="".join(current_lines))) + current_language = None + current_lines = [] + continue + + current_lines.append(line) + + if current_language is not None: + raise ReadmeParseError("Unterminated fenced code block in README.") + + return blocks + + +def run_command_specs( + command_specs: Sequence[CommandSpec], + cwd: Path, + timeout_seconds: float, +) -> ValidationRunSummary: + results: list[CommandResult] = [] + + for index, command_spec in enumerate(command_specs, start=1): + if should_skip_command(command_spec.command): + print_command_skip_banner(index=index, command=command_spec.command, cwd=cwd) + results.append(_build_result(index=index, command_spec=command_spec, cwd=cwd, skipped=True)) + continue + + print_command_execution_start( + index=index, + command=command_spec.command, + cwd=cwd, + expected_outputs=command_spec.expected_outputs, + ) + try: + completed = _run_command( + command=command_spec.command, + cwd=cwd, + timeout_seconds=timeout_seconds, + ) + except subprocess.TimeoutExpired as exc: + results.append( + _build_result( + index=index, + command_spec=command_spec, + cwd=cwd, + returncode=-1, + stdout=exc.stdout or "", + stderr=exc.stderr or "", + ) + ) + raise ValidationStopped( + ValidationRunSummary( + results=results, + failure_message=_format_failure_message( + index=index, + command=command_spec.command, + expected_outputs=command_spec.expected_outputs, + cwd=cwd, + returncode=None, + reason=f"timed out after {timeout_seconds} seconds", + ), + failed_index=index, + ) + ) from exc + + result = _build_result( + index=index, + command_spec=command_spec, + cwd=cwd, + returncode=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + ) + print_command_execution_end(result) + + try: + _assert_command_result(result) + except CommandValidationFailure as exc: + results.append(result) + print( + f"Validation failed at command #{index}. " + "Skipping remaining commands and running cleanup commands...", + file=sys.stderr, + flush=True, + ) + cleanup_results = run_cleanup_commands( + command_specs=command_specs[index:], + cwd=cwd, + timeout_seconds=timeout_seconds, + ) + if cleanup_results: + raise ValidationStopped( + ValidationRunSummary( + results=results, + failure_message=( + f"{exc}\nCleanup commands executed:\n" + f"{_format_cleanup_results(cleanup_results)}" + ), + failed_index=index, + ) + ) from exc + raise ValidationStopped( + ValidationRunSummary( + results=results, + failure_message=str(exc), + failed_index=index, + ) + ) from exc + + results.append(result) + + return ValidationRunSummary(results=results) + + +def _assert_command_result(result: CommandResult) -> None: + if result.expected_outputs: + for output_index, expected_output in enumerate(result.expected_outputs, start=1): + if not _expected_output_matches(expected_output, result.combined_output): + mismatch_detail = _describe_output_mismatch(expected_output, result.combined_output) + raise CommandValidationFailure( + _format_failure_message( + index=result.index, + command=result.command, + expected_outputs=result.expected_outputs, + cwd=result.cwd, + returncode=result.returncode, + reason=f"missing expected terminaloutput block #{output_index}", + detail=mismatch_detail, + ) + ) + return + + if result.returncode != 0: + raise CommandValidationFailure( + _format_failure_message( + index=result.index, + command=result.command, + expected_outputs=result.expected_outputs, + cwd=result.cwd, + returncode=result.returncode, + reason="command exited non-zero without any expected terminaloutput blocks", + ) + ) + + +def _expected_output_matches(expected_output: str, actual_output: str) -> bool: + expected_lines = [line.strip() for line in expected_output.splitlines() if line.strip()] + actual_lines = actual_output.splitlines() + + if not expected_lines: + return True + + for expected_line in expected_lines: + if not any(expected_line in actual_line for actual_line in actual_lines): + return False + + return True + + +def _describe_output_mismatch(expected_output: str, actual_output: str) -> str | None: + expected_lines = [line.strip() for line in expected_output.splitlines() if line.strip()] + actual_lines = actual_output.splitlines() + + if not expected_lines: + return None + + for expected_line in expected_lines: + if not any(expected_line in actual_line for actual_line in actual_lines): + closest_line = _find_closest_line(expected_line, actual_lines) + divider = _style("-" * 48, ANSI_DIM) + detail_lines = [ + "", + divider, + _style("Mismatch Detail", ANSI_BOLD, ANSI_RED), + "", + _style("Expected line", ANSI_BOLD, ANSI_GREEN), + f" {expected_line}", + ] + if closest_line is not None: + detail_lines.extend( + [ + "", + _style("Closest actual line", ANSI_BOLD, ANSI_YELLOW), + f" {closest_line}", + ] + ) + else: + detail_lines.extend( + [ + "", + _style("Closest actual line", ANSI_BOLD, ANSI_YELLOW), + " none", + ] + ) + detail_lines.extend(["", divider]) + return "\n".join(detail_lines) + + return None + + +def _find_closest_line(expected_line: str, actual_lines: Sequence[str]) -> str | None: + best_line: str | None = None + best_ratio = 0.0 + + for actual_line in actual_lines: + ratio = difflib.SequenceMatcher(None, expected_line, actual_line).ratio() + if ratio > best_ratio: + best_ratio = ratio + best_line = actual_line + + if best_ratio == 0.0: + return None + + return best_line + + +def _run_command( + *, + command: str, + cwd: Path, + timeout_seconds: float, +) -> subprocess.CompletedProcess[str]: + process = subprocess.Popen( + command, + shell=True, + cwd=str(cwd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + assert process.stdout is not None + assert process.stderr is not None + + stdout_chunks: list[str] = [] + stderr_chunks: list[str] = [] + output_queue: queue.Queue[tuple[str, str | None]] = queue.Queue() + + def _reader(stream_name: str, stream, chunks: list[str]) -> None: + try: + for line in iter(stream.readline, ""): + chunks.append(line) + output_queue.put((stream_name, line)) + finally: + stream.close() + output_queue.put((stream_name, None)) + + stdout_thread = threading.Thread( + target=_reader, + args=("stdout", process.stdout, stdout_chunks), + daemon=True, + ) + stderr_thread = threading.Thread( + target=_reader, + args=("stderr", process.stderr, stderr_chunks), + daemon=True, + ) + stdout_thread.start() + stderr_thread.start() + + finished_streams = 0 + timed_out = False + deadline = time.monotonic() + timeout_seconds + + try: + while finished_streams < 2: + if process.poll() is None and time.monotonic() > deadline: + timed_out = True + process.kill() + process.wait() + raise subprocess.TimeoutExpired( + cmd=command, + timeout=timeout_seconds, + output="".join(stdout_chunks), + stderr="".join(stderr_chunks), + ) + + try: + stream_name, chunk = output_queue.get(timeout=0.1) + except queue.Empty: + if process.poll() is not None: + continue + continue + + if chunk is None: + finished_streams += 1 + continue + + target_stream = sys.stdout if stream_name == "stdout" else sys.stderr + print(chunk, end="", file=target_stream, flush=True) + except Exception: + process.kill() + process.wait() + raise + + try: + returncode = process.wait(timeout=0) + finally: + stdout_thread.join(timeout=1) + stderr_thread.join(timeout=1) + if not timed_out and process.poll() is None: + process.kill() + process.wait() + + return subprocess.CompletedProcess( + args=command, + returncode=returncode, + stdout="".join(stdout_chunks), + stderr="".join(stderr_chunks), + ) + + +def run_cleanup_commands( + command_specs: Sequence[CommandSpec], + cwd: Path, + timeout_seconds: float, +) -> list[CommandResult]: + cleanup_results: list[CommandResult] = [] + + for cleanup_index, command_spec in enumerate(command_specs, start=1): + if command_spec.expected_outputs: + break + if not _is_cleanup_command(command_spec.command): + continue + + try: + completed = _run_command( + command=command_spec.command, + cwd=cwd, + timeout_seconds=timeout_seconds, + ) + except subprocess.TimeoutExpired: + continue + cleanup_results.append( + _build_result( + index=cleanup_index, + command_spec=command_spec, + cwd=cwd, + returncode=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + ) + ) + + return cleanup_results + + +def run_final_cleanup_commands( + command_specs: Sequence[CommandSpec], + cwd: Path, + timeout_seconds: float, +) -> list[CommandResult]: + cleanup_results: list[CommandResult] = [] + + for cleanup_index, cleanup_command in enumerate(derive_final_cleanup_commands(command_specs), start=1): + try: + completed = _run_command( + command=cleanup_command, + cwd=cwd, + timeout_seconds=timeout_seconds, + ) + except subprocess.TimeoutExpired: + continue + cleanup_results.append( + CommandResult( + index=cleanup_index, + command=cleanup_command, + expected_outputs=[], + cwd=cwd, + skipped=False, + returncode=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + ) + ) + + return cleanup_results + + +def derive_final_cleanup_commands(command_specs: Sequence[CommandSpec]) -> list[str]: + cleanup_commands: list[str] = [] + seen: set[str] = set() + + for command_spec in command_specs: + cleanup_command = _derive_cleanup_command(command_spec.command) + if cleanup_command is None or cleanup_command in seen: + continue + seen.add(cleanup_command) + cleanup_commands.append(cleanup_command) + + return cleanup_commands + + +def _derive_cleanup_command(command: str) -> str | None: + try: + tokens = shlex.split(command) + except ValueError: + return None + + if len(tokens) >= 2 and tokens[0] == "docker" and tokens[1] == "compose": + compose_start_index = 2 + elif tokens and tokens[0] == "docker-compose": + compose_start_index = 1 + else: + return None + + subcommand_index = None + for index in range(compose_start_index, len(tokens)): + if tokens[index] in {"up", "run"}: + subcommand_index = index + break + + if subcommand_index is None: + return None + + cleanup_tokens = tokens[:subcommand_index] + ["down", "-v", "--remove-orphans"] + return shlex.join(cleanup_tokens) + + +def _is_cleanup_command(command: str) -> bool: + normalized_command = " ".join(command.lower().split()) + cleanup_patterns = ( + "docker compose down", + "docker-compose down", + "docker stop", + "docker rm", + "docker container rm", + "kubectl delete", + "pkill ", + "kill ", + ) + return any(pattern in normalized_command for pattern in cleanup_patterns) + + +def _format_cleanup_results(cleanup_results: Sequence[CommandResult]) -> str: + lines: list[str] = [] + for result in cleanup_results: + lines.append(f"- `{result.command.strip()}` exited with {result.returncode}") + return "\n".join(lines) + + +def print_final_cleanup_summary(cleanup_results: Sequence[CommandResult]) -> None: + if not cleanup_results: + return + + print("FINAL CLEANUP") + for result in cleanup_results: + print(f" {result.command}") + + +def _format_failure_message( + *, + index: int, + command: str, + expected_outputs: Sequence[str], + cwd: Path | None, + returncode: int | None, + reason: str, + detail: str | None = None, +) -> str: + returncode_text = "n/a" if returncode is None else str(returncode) + lines = [ + f"Command #{index} failed: {reason}", + f"Exit code: {returncode_text}", + ] + if cwd is not None: + lines.append(f"Working directory: {cwd}") + lines.extend( + [ + "Command:", + command.rstrip("\n"), + f"Expected terminaloutput blocks: {len(expected_outputs)}", + ] + ) + if detail: + lines.extend(["", detail]) + return "\n".join(lines) + + +def _build_result( + *, + index: int, + command_spec: CommandSpec, + cwd: Path, + skipped: bool = False, + returncode: int = 0, + stdout: str = "", + stderr: str = "", +) -> CommandResult: + return CommandResult( + index=index, + command=command_spec.command, + expected_outputs=command_spec.expected_outputs, + cwd=cwd, + skipped=skipped, + returncode=returncode, + stdout=stdout, + stderr=stderr, + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Validate README shell commands against terminaloutput blocks." + ) + parser.add_argument( + "readme", + nargs="?", + help="Lab directory to validate. If omitted, runs the built-in lab README list.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Parse the README and print command-to-terminaloutput mappings without running commands.", + ) + parser.add_argument( + "--skip-reset", + action="store_true", + help="Do not reset lab-changed files back to their original git state at the end of the run.", + ) + parser.add_argument( + "--timeout", + type=float, + default=120.0, + help="Timeout in seconds for each shell command. Default: 120.", + ) + parser.add_argument( + "--preflight-only", + action="store_true", + help="Run only the shared preflight checks and exit.", + ) + parser.add_argument( + "--result-json", + help="Write a machine-readable JSON result for this invocation.", + ) + parser.add_argument( + "--report-from", + help="Read per-run JSON results from a directory and print a consolidated report.", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + if args.preflight_only and (args.readme or args.dry_run or args.report_from): + parser.error("--preflight-only cannot be combined with a lab path, --dry-run, or --report-from") + if args.report_from and (args.readme or args.dry_run or args.preflight_only): + parser.error("--report-from cannot be combined with a lab path, --dry-run, or --preflight-only") + + if args.report_from: + report = load_run_report(Path(args.report_from)) + print_run_report(report) + exit_code = 1 if any(lab.exit_code != 0 for lab in report.labs) else 0 + if args.result_json: + write_run_report(Path(args.result_json), report) + return exit_code + + readme_paths = resolve_readme_paths(args.readme) + multiple_labs = len(readme_paths) > 1 + for readme_path in readme_paths: + if not readme_path.is_file(): + parser.error(f"README file does not exist: {readme_path}") + + should_run_preflight = not args.dry_run and (args.preflight_only or args.readme is None) + if should_run_preflight: + requirements = determine_preflight_requirements(readme_paths) + streamed_preflight = requirements.any_required + if streamed_preflight: + print("===== Preflight =====") + preflight_results = run_preflight( + readme_paths, + requirements, + on_result=_print_preflight_result, + ) + if preflight_results: + if not streamed_preflight: + print_preflight_results(preflight_results) + if any(not result.passed and not result.skipped for result in preflight_results): + print("Preflight failed. No labs were executed.") + if args.result_json: + write_run_report(Path(args.result_json), RunReport(mode="preflight-only", labs=[])) + return 1 + if args.preflight_only: + if args.result_json: + write_run_report(Path(args.result_json), RunReport(mode="preflight-only", labs=[])) + return 0 + + overall_exit_code = 0 + passed_labs: list[LabExecutionResult] = [] + failed_labs: list[LabExecutionResult] = [] + for index, readme_path in enumerate(readme_paths, start=1): + if multiple_labs: + if index > 1: + print() + print(f"===== {readme_path.parent.name} =====") + start_time = time.perf_counter() + if not args.dry_run: + streamed_warmup = bool(_docker_compose_lab_dirs([readme_path])) + docker_warmup_results = warm_docker_images( + [readme_path], + on_result=_print_docker_warmup_result, + ) + if docker_warmup_results: + if not streamed_warmup: + print_docker_warmup_results(docker_warmup_results) + if any(not result.passed for result in docker_warmup_results): + print("FAIL") + overall_exit_code = 1 + failed_labs.append( + LabExecutionResult( + name=readme_path.parent.name, + exit_code=1, + duration_seconds=time.perf_counter() - start_time, + validated_commands=0, + total_commands=0, + skipped_commands=0, + ) + ) + continue + lab_result = run_single_readme( + readme_path=readme_path, + dry_run=args.dry_run, + skip_reset=args.skip_reset, + timeout_seconds=args.timeout, + ) + lab_result = LabExecutionResult( + name=lab_result.name, + exit_code=lab_result.exit_code, + duration_seconds=time.perf_counter() - start_time, + validated_commands=lab_result.validated_commands, + total_commands=lab_result.total_commands, + skipped_commands=lab_result.skipped_commands, + ) + if lab_result.exit_code != 0: + overall_exit_code = lab_result.exit_code + failed_labs.append(lab_result) + else: + passed_labs.append(lab_result) + + if multiple_labs: + print() + print_multi_lab_summary(passed_labs, failed_labs, dry_run=args.dry_run) + + report_mode = "dry-run" if args.dry_run else "execution" + if args.result_json: + write_run_report(Path(args.result_json), RunReport(mode=report_mode, labs=[*passed_labs, *failed_labs])) + + return overall_exit_code + + +def run_single_readme( + *, + readme_path: Path, + dry_run: bool, + skip_reset: bool, + timeout_seconds: float, +) -> LabExecutionResult: + repo_snapshot = None if skip_reset else snapshot_repo_state(readme_path.parent) + lab_name = readme_path.parent.name + try: + command_specs = parse_readme_commands(readme_path) + print_command_mapping(command_specs) + if dry_run: + return LabExecutionResult( + name=lab_name, + exit_code=0, + duration_seconds=0.0, + validated_commands=0, + total_commands=len(command_specs), + skipped_commands=0, + ) + summary = run_command_specs( + command_specs=command_specs, + cwd=readme_path.parent, + timeout_seconds=timeout_seconds, + ) + exit_code = 0 + except ValidationStopped as exc: + summary = exc.summary + exit_code = 1 + except ReadmeValidationError as exc: + print(f"README: {readme_path}", file=sys.stderr) + print(exc, file=sys.stderr) + return LabExecutionResult( + name=lab_name, + exit_code=1, + duration_seconds=0.0, + validated_commands=0, + total_commands=0, + skipped_commands=0, + ) + + print_run_summary(summary, total_commands=len(command_specs)) + + if summary.failure_message is not None: + print(f"README: {readme_path}", file=sys.stderr) + print(summary.failure_message, file=sys.stderr) + + try: + final_cleanup_results = run_final_cleanup_commands( + command_specs=command_specs, + cwd=readme_path.parent, + timeout_seconds=timeout_seconds, + ) + if final_cleanup_results: + print_final_cleanup_summary(final_cleanup_results) + + if repo_snapshot is not None: + print_reset_summary(reset_lab_changes(readme_path.parent, repo_snapshot)) + except Exception: + exit_code = 1 + print(f"README: {readme_path}", file=sys.stderr) + print("Cleanup/reset failed:", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + + if exit_code == 0: + print(f"Validated {len(summary.results)} command(s) in {readme_path}") + print("PASS") + else: + print("FAIL") + executed_skips = sum(1 for result in summary.results if result.skipped) + skipped_commands = total_commands_skipped(summary, len(command_specs)) + return LabExecutionResult( + name=lab_name, + exit_code=exit_code, + duration_seconds=0.0, + validated_commands=len(summary.results) - executed_skips, + total_commands=len(command_specs), + skipped_commands=skipped_commands, + ) + + +def resolve_readme_paths(readme_arg: str | None) -> list[Path]: + if readme_arg: + return [(Path(readme_arg).expanduser().resolve() / "README.md")] + + return [(REPO_ROOT / lab / "README.md").resolve() for lab in DEFAULT_LABS] + + +def write_run_report(path: Path, report: RunReport) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "mode": report.mode, + "labs": [ + { + "name": lab.name, + "exit_code": lab.exit_code, + "duration_seconds": lab.duration_seconds, + "validated_commands": lab.validated_commands, + "total_commands": lab.total_commands, + "skipped_commands": lab.skipped_commands, + } + for lab in report.labs + ], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + +def load_run_report(path: Path) -> RunReport: + report_files = sorted(path.glob("*.json")) if path.is_dir() else [path] + labs: list[LabExecutionResult] = [] + modes: set[str] = set() + + for report_file in report_files: + payload = json.loads(report_file.read_text(encoding="utf-8")) + modes.add(payload.get("mode", "execution")) + for lab in payload.get("labs", []): + labs.append( + LabExecutionResult( + name=lab["name"], + exit_code=lab["exit_code"], + duration_seconds=lab["duration_seconds"], + validated_commands=lab["validated_commands"], + total_commands=lab["total_commands"], + skipped_commands=lab["skipped_commands"], + ) + ) + + merged_mode = "dry-run" if modes == {"dry-run"} else "execution" + return RunReport(mode=merged_mode, labs=sorted(labs, key=lambda lab: lab.name)) + + +def print_run_report(report: RunReport) -> None: + passed_labs = [lab for lab in report.labs if lab.exit_code == 0] + failed_labs = [lab for lab in report.labs if lab.exit_code != 0] + print_multi_lab_summary( + passed_labs, + failed_labs, + dry_run=report.mode == "dry-run", + ) + + +def determine_preflight_requirements(readme_paths: Sequence[Path]) -> PreflightRequirements: + docker_cli = False + docker_compose = False + license_validation = False + remote_contract_access = False + + for readme_path in readme_paths: + command_specs = _readme_command_specs(readme_path) + if any("docker" in command_spec.command.lower() for command_spec in command_specs): + docker_cli = True + if any( + "docker compose" in command_spec.command.lower() + or "docker-compose" in command_spec.command.lower() + for command_spec in command_specs + ): + docker_compose = True + + for config_path in _related_config_paths(readme_path.parent): + if not config_path.is_file(): + continue + try: + config_text = config_path.read_text(encoding="utf-8") + except OSError: + continue + if "/specmatic/specmatic-license.txt" in config_text: + license_validation = True + if "https://github.com/specmatic/labs-contracts.git" in config_text: + remote_contract_access = True + + return PreflightRequirements( + docker_cli=docker_cli, + docker_compose=docker_compose, + license_validation=license_validation, + remote_contract_access=remote_contract_access, + ) + + +def _related_config_paths(lab_dir: Path) -> list[Path]: + return [ + lab_dir / "docker-compose.yaml", + lab_dir / "specmatic.yaml", + ] + + +def warm_docker_images( + readme_paths: Sequence[Path], + timeout_seconds: float = DOCKER_WARMUP_TIMEOUT_SECONDS, + on_result=None, +) -> list[DockerWarmupResult]: + results: list[DockerWarmupResult] = [] + + def _emit(result: DockerWarmupResult) -> None: + results.append(result) + if on_result is not None: + on_result(result) + + for lab_dir in _docker_compose_lab_dirs(readme_paths): + if on_result is not None and not results: + print("===== Docker Warmup =====") + try: + completed = subprocess.run( + ["docker", "compose", "pull", "--ignore-buildable"], + cwd=str(lab_dir), + capture_output=True, + text=True, + check=False, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired: + _emit( + DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=f"timed out after {int(timeout_seconds)} seconds", + ) + ) + continue + except OSError as exc: + _emit( + DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=str(exc), + ) + ) + continue + + if completed.returncode == 0: + _emit(DockerWarmupResult(lab_name=lab_dir.name, passed=True)) + continue + + error_output = completed.stderr.strip() or completed.stdout.strip() or "docker compose pull failed" + _emit( + DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=error_output, + ) + ) + + return results + + +def _docker_compose_lab_dirs(readme_paths: Sequence[Path]) -> list[Path]: + lab_dirs: list[Path] = [] + seen: set[Path] = set() + + for readme_path in readme_paths: + lab_dir = readme_path.parent + docker_compose_path = lab_dir / "docker-compose.yaml" + if lab_dir in seen or not docker_compose_path.is_file(): + continue + command_specs = _readme_command_specs(readme_path) + if not any( + "docker compose" in command_spec.command.lower() + or "docker-compose" in command_spec.command.lower() + for command_spec in command_specs + ): + continue + seen.add(lab_dir) + lab_dirs.append(lab_dir) + + return lab_dirs + + +def run_preflight( + readme_paths: Sequence[Path], + requirements: PreflightRequirements | None = None, + on_result=None, +) -> list[PreflightCheckResult]: + requirements = requirements or determine_preflight_requirements(readme_paths) + if not requirements.any_required: + return [] + + results: list[PreflightCheckResult] = [] + license_path = REPO_ROOT / "license.txt" + + def _emit(result: PreflightCheckResult) -> None: + results.append(result) + if on_result is not None: + on_result(result) + + if requirements.docker_cli: + _emit( + _run_preflight_command( + name="docker", + command=["docker", "--version"], + failure_detail="docker CLI is not available", + suggestion="Install Docker and make sure `docker` is on PATH.", + ) + ) + + docker_ready = not results or results[-1].passed + if requirements.docker_compose: + _emit( + _run_preflight_command( + name="docker compose", + command=["docker", "compose", "version"], + failure_detail="docker compose is not available", + suggestion="Install a Docker version that includes `docker compose`.", + ) + ) + docker_ready = docker_ready and results[-1].passed + + if requirements.docker_cli and docker_ready: + _emit( + _run_preflight_command( + name="docker daemon", + command=["docker", "info"], + failure_detail="Docker daemon is not reachable", + suggestion="Start Docker Desktop or the Docker daemon, then rerun the validator.", + ) + ) + docker_ready = docker_ready and results[-1].passed + elif requirements.docker_cli: + _emit( + PreflightCheckResult( + name="docker daemon", + passed=False, + skipped=True, + detail="skipped because Docker CLI/Compose is unavailable", + suggestion="Fix the Docker installation first.", + ) + ) + + if requirements.license_validation: + if license_path.is_file(): + _emit(PreflightCheckResult(name="specmatic license file exists", passed=True)) + else: + _emit( + PreflightCheckResult( + name="specmatic license file exists", + passed=False, + detail=f"missing {license_path}", + suggestion="Add a valid `license.txt` at the labs repo root.", + ) + ) + + if docker_ready and license_path.is_file(): + _emit(_validate_specmatic_license(REPO_ROOT)) + else: + _emit( + PreflightCheckResult( + name="specmatic license validation", + passed=False, + skipped=True, + detail="skipped because Docker or the license file is unavailable", + suggestion="Fix Docker access and the license file, then rerun the validator.", + ) + ) + + if requirements.remote_contract_access: + _emit( + _run_preflight_command( + name="labs-contracts access", + command=[ + "git", + "ls-remote", + "--exit-code", + "https://github.com/specmatic/labs-contracts.git", + "HEAD", + ], + failure_detail="cannot reach github.com/specmatic/labs-contracts.git", + suggestion="Check network access to GitHub, then rerun the validator.", + ) + ) + + return results + + +def _run_preflight_command( + *, + name: str, + command: Sequence[str], + failure_detail: str, + suggestion: str, + cwd: Path | None = None, +) -> PreflightCheckResult: + try: + completed = subprocess.run( + list(command), + cwd=str(cwd) if cwd is not None else None, + capture_output=True, + text=True, + check=False, + ) + except OSError as exc: + return PreflightCheckResult( + name=name, + passed=False, + detail=f"{failure_detail}: {exc}", + suggestion=suggestion, + ) + + if completed.returncode == 0: + return PreflightCheckResult(name=name, passed=True) + + error_output = completed.stderr.strip() or completed.stdout.strip() or failure_detail + return PreflightCheckResult( + name=name, + passed=False, + detail=error_output, + suggestion=suggestion, + ) + + +def _validate_specmatic_license(repo_root: Path) -> PreflightCheckResult: + completed = subprocess.run( + [ + "docker", + "run", + "--rm", + "-v", + "./license.txt:/specmatic/specmatic-license.txt:ro", + "-e", + "SPECMATIC_LICENSE_PATH=/specmatic/specmatic-license.txt", + "specmatic/enterprise:latest", + "show-license", + ], + cwd=str(repo_root), + capture_output=True, + text=True, + check=False, + ) + output = "\n".join(part for part in [completed.stdout.strip(), completed.stderr.strip()] if part) + + loaded_expected_license = "initialized from /specmatic/specmatic-license.txt" in output + expired_license = "is expired" in output + fell_back_to_default = "initialized from jar:file:" in output + + if completed.returncode == 0 and loaded_expected_license and not expired_license and not fell_back_to_default: + return PreflightCheckResult(name="specmatic license validation", passed=True) + + return PreflightCheckResult( + name="specmatic license validation", + passed=False, + detail=output or "Specmatic license validation failed.", + suggestion="Replace `license.txt` with a valid, unexpired Specmatic license.", + ) + + +def print_preflight_results(results: Sequence[PreflightCheckResult]) -> None: + if not results: + return + + print("===== Preflight =====") + for result in results: + _print_preflight_result(result) + + +def print_docker_warmup_results(results: Sequence[DockerWarmupResult]) -> None: + if not results: + return + + print("===== Docker Warmup =====") + for result in results: + _print_docker_warmup_result(result) + + +def _print_preflight_result(result: PreflightCheckResult) -> None: + if result.skipped: + status = "SKIP" + else: + status = "PASS" if result.passed else "FAIL" + if result.detail: + print(f"{status} {result.name}: {result.detail}") + else: + print(f"{status} {result.name}") + if (not result.passed or result.skipped) and result.suggestion: + print(f" Fix: {result.suggestion}") + + +def _print_docker_warmup_result(result: DockerWarmupResult) -> None: + status = "PASS" if result.passed else "FAIL" + if result.detail: + print(f"{status} {result.lab_name}: {result.detail}") + else: + print(f"{status} {result.lab_name}") + + +def print_command_mapping(command_specs: Sequence[CommandSpec]) -> None: + separator = _style("=" * 72, ANSI_DIM) + + for index, command_spec in enumerate(command_specs, start=1): + print(separator) + print(_style(f"Command #{index}", ANSI_BOLD, ANSI_CYAN)) + print( + _style("Shell", ANSI_BOLD, ANSI_YELLOW) + + f" {_style(f'(expected terminaloutput blocks: {len(command_spec.expected_outputs)})', ANSI_DIM)}" + ) + _print_indented_block(command_spec.command) + for output_index, expected_output in enumerate(command_spec.expected_outputs, start=1): + print() + print(_style(f"terminaloutput #{output_index}", ANSI_BOLD, ANSI_GREEN)) + _print_indented_block(expected_output) + if not command_spec.expected_outputs: + print() + print(_style("terminaloutput none", ANSI_DIM)) + print() + if command_specs: + print(separator) + + +def print_command_execution_start( + *, + index: int, + command: str, + cwd: Path, + expected_outputs: Sequence[str], +) -> None: + separator = "=" * 72 + print(separator) + print(f"BEGIN command #{index}") + print(f"Working directory: {cwd}") + print(f"Expected terminaloutput blocks: {len(expected_outputs)}") + print("Command:") + _print_indented_block(command) + print("-" * 72) + print(f"Command #{index} output:") + print("-" * 72) + + +def print_command_execution_end(result: CommandResult) -> None: + print("-" * 72) + print(f"END command #{result.index} (exit {result.returncode})") + print("=" * 72) + + +def print_command_skip_banner(*, index: int, command: str, cwd: Path) -> None: + separator = "=" * 72 + print(separator) + print(f"SKIP command #{index}") + print(f"Working directory: {cwd}") + print("Command:") + _print_indented_block(command) + print("=" * 72) + + +def _print_indented_block(content: str) -> None: + for line in content.rstrip("\n").splitlines(): + print(f" {line}") + + +def snapshot_repo_state(cwd: Path) -> RepoSnapshot | None: + repo_root = _get_repo_root(cwd) + if repo_root is None: + return None + + scope = cwd.resolve().relative_to(repo_root) + + return RepoSnapshot( + repo_root=repo_root, + scope=scope, + tracked_dirty=_git_path_set( + repo_root, + ["git", "status", "--porcelain=v1", "--untracked-files=no", "-z", "--", str(scope)], + ), + untracked=_git_path_set( + repo_root, + ["git", "ls-files", "-o", "--exclude-standard", "-z", "--", str(scope)], + ), + ) + + +def reset_lab_changes(cwd: Path, baseline: RepoSnapshot | None) -> ResetSummary: + if baseline is None: + return ResetSummary(restored=[], removed=[]) + + current_snapshot = snapshot_repo_state(cwd) + if current_snapshot is None: + return ResetSummary(restored=[], removed=[]) + + paths_to_restore = sorted(current_snapshot.tracked_dirty - baseline.tracked_dirty) + paths_to_remove = sorted(current_snapshot.untracked - baseline.untracked) + + if paths_to_restore: + _run_git_command( + baseline.repo_root, + ["git", "restore", "--worktree", "--source=HEAD", "--", *[str(path) for path in paths_to_restore]], + ) + + removed: list[Path] = [] + for path in paths_to_remove: + absolute_path = baseline.repo_root / path + if not absolute_path.exists(): + continue + _remove_path(absolute_path, baseline.repo_root) + removed.append(path) + + return ResetSummary(restored=paths_to_restore, removed=removed) + + +def _remove_path(path: Path, repo_root: Path) -> None: + try: + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + return + except PermissionError: + pass + + relative_path = path.resolve().relative_to(repo_root.resolve()) + container_target = PurePosixPath("/workspace") / PurePosixPath(relative_path.as_posix()) + + command = [ + "docker", + "run", + "--rm", + "-v", + f"{repo_root}:/workspace", + "--entrypoint", + "sh", + "specmatic/enterprise:latest", + "-lc", + f"rm -rf '{container_target}'", + ] + + completed = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0 and path.exists(): + error_output = completed.stderr.strip() or completed.stdout.strip() or "docker rm failed" + raise PermissionError(f"failed to remove {path}: {error_output}") + + +def print_reset_summary(reset_summary: ResetSummary) -> None: + if not reset_summary.restored and not reset_summary.removed: + return + + print( + "RESET " + f"(restored: {len(reset_summary.restored)}, removed: {len(reset_summary.removed)})" + ) + + +def print_run_summary(summary: ValidationRunSummary, total_commands: int) -> None: + for result in summary.results: + if result.skipped: + print( + f"SKIP command #{result.index} " + f"(contains both 'docker' and 'studio')" + ) + else: + status = "FAIL" if summary.failed_index == result.index else "PASS" + print( + f"{status} command #{result.index} (exit {result.returncode}, " + f"expected blocks: {len(result.expected_outputs)})" + ) + + for index in range(len(summary.results) + 1, total_commands + 1): + print(f"SKIP command #{index}") + + +def _format_duration(duration_seconds: float) -> str: + if duration_seconds < 60: + return f"{duration_seconds:.2f}s" + + minutes, seconds = divmod(duration_seconds, 60) + if duration_seconds < 3600: + return f"{int(minutes)}m {seconds:.2f}s" + + hours, minutes = divmod(minutes, 60) + return f"{int(hours)}h {int(minutes)}m {seconds:.2f}s" + + +def total_commands_skipped(summary: ValidationRunSummary, total_commands: int) -> int: + executed_skips = sum(1 for result in summary.results if result.skipped) + remaining_skips = total_commands - len(summary.results) + return executed_skips + remaining_skips + + +def print_multi_lab_summary( + passed_labs: Sequence[LabExecutionResult], + failed_labs: Sequence[LabExecutionResult], + dry_run: bool = False, +) -> None: + print("===== Summary =====") + if dry_run: + dry_run_labs = [*passed_labs, *failed_labs] + print(f"DRY RUN labs: {len(dry_run_labs)}") + print() + _print_summary_table( + rows=[ + ( + lab.name, + "DRY RUN", + _format_duration(lab.duration_seconds), + str(lab.total_commands), + "-", + ) + for lab in dry_run_labs + ], + validated_header="Commands", + ) + return + + print(f"PASS labs: {len(passed_labs)}") + print(f"FAIL labs: {len(failed_labs)}") + print() + _print_summary_table( + rows=[ + ( + lab.name, + "PASS", + _format_duration(lab.duration_seconds), + f"{lab.validated_commands}/{lab.total_commands}", + str(lab.skipped_commands), + ) + for lab in passed_labs + ] + + [ + ( + lab.name, + "FAIL", + _format_duration(lab.duration_seconds), + f"{lab.validated_commands}/{lab.total_commands}", + str(lab.skipped_commands), + ) + for lab in failed_labs + ] + ) + + +def _print_summary_table( + *, + rows: Sequence[tuple[str, str, str, str, str]], + validated_header: str = "Validated", +) -> None: + if not rows: + return + + headers = ("Lab", "Status", "Duration", validated_header, "Skipped") + col_widths = [ + max(len(headers[0]), *(len(row[0]) for row in rows)), + max(len(headers[1]), *(len(row[1]) for row in rows)), + max(len(headers[2]), *(len(row[2]) for row in rows)), + max(len(headers[3]), *(len(row[3]) for row in rows)), + max(len(headers[4]), *(len(row[4]) for row in rows)), + ] + + def format_row(columns: Sequence[str]) -> str: + return ( + f"{columns[0]:<{col_widths[0]}} " + f"{columns[1]:<{col_widths[1]}} " + f"{columns[2]:>{col_widths[2]}} " + f"{columns[3]:>{col_widths[3]}} " + f"{columns[4]:>{col_widths[4]}}" + ) + + print(format_row(headers)) + print( + f"{'-' * col_widths[0]} " + f"{'-' * col_widths[1]} " + f"{'-' * col_widths[2]} " + f"{'-' * col_widths[3]} " + f"{'-' * col_widths[4]}" + ) + for row in rows: + print(format_row(row)) + + +def _get_repo_root(cwd: Path) -> Path | None: + completed = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=str(cwd), + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + return None + return Path(completed.stdout.strip()) + + +def _git_path_set(cwd: Path, command: list[str]) -> set[Path]: + completed = _run_git_command(cwd, command) + if command[1:3] == ["status", "--porcelain=v1"]: + return _parse_status_paths(completed.stdout) + return {Path(path) for path in completed.stdout.split("\0") if path} + + +def _parse_status_paths(output: str) -> set[Path]: + paths: set[Path] = set() + for entry in output.split("\0"): + if entry and len(entry) >= 4: + paths.add(Path(entry[3:])) + return paths + + +def _run_git_command(cwd: Path, command: list[str]) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + command, + cwd=str(cwd), + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + stderr = completed.stderr.strip() or "unknown git error" + raise GitInteractionError(f"Git command failed: {' '.join(command)}\n{stderr}") + return completed + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/api-coverage/README.md b/api-coverage/README.md index f7a9c5c..f3e6c16 100644 --- a/api-coverage/README.md +++ b/api-coverage/README.md @@ -90,15 +90,17 @@ Expected behavior: Expected result summary: ```terminaloutput -Tests run: 2, Successes: 1, Failures: 1, Errors: 0 +Tests run: 2, Successes: 1, Failures: 1, WIP: 0, Errors: 0 ``` Expected coverage highlight: ```terminaloutput -100% /pets/{petId} GET 200 1 covered -0% /pets/search GET 200 1 not implemented -0% /pets/find GET 0 0 missing in spec +| coverage | path | method | requestContentType | response | responseContentType | remarks | result | +|----------|---------------|--------|--------------------|----------|---------------------|------------------|--------| +| 100% | /pets/{petId} | GET | NA | 200 | application/json | covered | 1p | +| 0% | /pets/search | GET | NA | 200 | application/json | not implemented | 1f | +| 0% | /pets/find | GET | NA | 0 | NA | missing in spec* | | ``` Expected gate failure highlight: @@ -146,6 +148,12 @@ Change it to: Do not change anything else in the operation. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's|/pets/search:|/pets/find:|' specs/service.yaml" +``` + ## 3. Re-run the tests and coverage check Run the same command again: @@ -156,7 +164,7 @@ docker compose up test --build --abort-on-container-exit Expected final result: ```terminaloutput -Tests run: 2, Successes: 2, Failures: 0, Errors: 0 +Tests run: 2, Successes: 2, Failures: 0, WIP: 0, Errors: 0 ``` Expected coverage outcome: diff --git a/api-resiliency-testing/README.md b/api-resiliency-testing/README.md index cd45b07..95c836c 100644 --- a/api-resiliency-testing/README.md +++ b/api-resiliency-testing/README.md @@ -76,7 +76,7 @@ docker compose --profile test up --abort-on-container-exit Expected baseline result: ```terminaloutput -Tests run: 5, Successes: 3, Failures: 2, Errors: 0 +Tests run: 5, Successes: 3, Failures: 2, WIP: 0, Errors: 0 ``` The two failing scenarios should be: @@ -108,6 +108,12 @@ Keep: - query `to-date` as `2025-11-15` - the `200 OK` downstream response body +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i -e '/^ }$/s// },/' -e '/^}$/i\ \"transient\": true,' -e '/^}$/i\ \"delay-in-seconds\": 2' examples/order-service/stub_timeout_get_products.json" +``` + Re-run: ```shell @@ -117,7 +123,7 @@ docker compose --profile test up --abort-on-container-exit Expected checkpoint result: ```terminaloutput -Tests run: 5, Successes: 4, Failures: 1, Errors: 0 +Tests run: 5, Successes: 4, Failures: 1, WIP: 0, Errors: 0 ``` At this point: @@ -144,6 +150,12 @@ Keep: - the existing header matchers - the downstream `201 Created` response +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i -e '/^ }$/s// },/' -e '/^}$/i\ \"transient\": true,' -e '/^}$/i\ \"delay-in-seconds\": 2' examples/order-service/stub_timeout_post_product.json" +``` + Re-run: ```shell @@ -153,7 +165,7 @@ docker compose --profile test up --abort-on-container-exit Expected checkpoint result: ```terminaloutput -Tests run: 5, Successes: 5, Failures: 0, Errors: 0 +Tests run: 5, Successes: 5, Failures: 0, WIP: 0, Errors: 0 ``` At this point: @@ -179,6 +191,12 @@ to: schemaResiliencyTests: all ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's/schemaResiliencyTests: none/schemaResiliencyTests: all/' specmatic.yaml" +``` + Re-run: ```shell @@ -202,7 +220,7 @@ Expected failure direction: Expected Task C checkpoint result before the matcher fix: ```terminaloutput -Tests run: 249, Successes: 238, Failures: 11, Errors: 0 +Tests run: 249, Successes: 238, Failures: 11, WIP: 0, Errors: 0 ``` Why this is useful: @@ -247,6 +265,12 @@ Why `value:each` is the right matcher here: Specmatic documentation for this matcher behavior: - [Matchers: step-by-step example for `value: each`](https://docs.specmatic.io/features/matchers#step-by-step-example-for-value-each) +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"type\": \"book\"#\"type\": \"\$match(dataType:ProductType, value:each, times:1)\"#; s#\"inventory\": 9#\"inventory\": \"\$match(dataType:ProductInventory, value:each, times:1)\"#' examples/order-service/stub_timeout_post_product.json" +``` + Keep: - `"transient": true` - `"delay-in-seconds": 2` @@ -266,7 +290,7 @@ Expected outcome: Final expected result: ```terminaloutput -Tests run: 249, Successes: 249, Failures: 0, Errors: 0 +Tests run: 249, Successes: 249, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: diff --git a/api-security-schemes/README.md b/api-security-schemes/README.md index 6a0e2d1..90950b9 100644 --- a/api-security-schemes/README.md +++ b/api-security-schemes/README.md @@ -107,7 +107,7 @@ docker compose up specmatic-test --abort-on-container-exit Expected failing result: ```terminaloutput -Tests run: 171, Successes: 0, Failures: 171, Errors: 0 +Tests run: 171, Successes: 0, Failures: 171, WIP: 0, Errors: 0 ``` - The compose command exits with a non-zero code. @@ -132,6 +132,12 @@ Update [`specmatic.yaml`](specmatic.yaml) to valid values again: Do not change anything else. Fix only the above values. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#INVALID_OAUTH_TOKEN#OAUTH_TOKEN#g; s#dXNlcjppbnZhbGlkcGFzcw==#dXNlcjpwYXNzd29yZA==#g; s#INVALID_APIKEY1234#APIKEY1234#g' specmatic.yaml" +``` + ## Verify the fix Re-run: @@ -143,7 +149,7 @@ docker compose up specmatic-test --abort-on-container-exit Expected result: ```terminaloutput -Tests run: 171, Successes: 171, Failures: 0, Errors: 0 +Tests run: 171, Successes: 171, Failures: 0, WIP: 0, Errors: 0 ``` Cleanup after run: diff --git a/async-event-flow/.backup/acceptOrder-with-before.json b/async-event-flow/.backup/acceptOrder-with-before.json new file mode 100644 index 0000000..9977f4a --- /dev/null +++ b/async-event-flow/.backup/acceptOrder-with-before.json @@ -0,0 +1,36 @@ +{ + "id": "accept-order", + "name": "ACCEPT_ORDER", + "before": [ + { + "type": "http", + "wait": "PT1S", + "http-request": { + "baseUrl": "http://sut:8080", + "path": "/orders", + "method": "PUT", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "id": 123, + "status": "ACCEPTED", + "timestamp": "2025-04-12T14:30:00Z" + } + }, + "http-response": { + "status": 200 + }, + "timeout": "PT30S" + } + ], + "send": { + "topic": "accepted-orders", + "key": 123, + "payload": { + "id": 123, + "status": "ACCEPTED", + "timestamp": "2025-04-12T14:30:00Z" + } + } +} diff --git a/async-event-flow/.backup/outForDeliveryOrder-with-before.json b/async-event-flow/.backup/outForDeliveryOrder-with-before.json new file mode 100644 index 0000000..329a697 --- /dev/null +++ b/async-event-flow/.backup/outForDeliveryOrder-with-before.json @@ -0,0 +1,49 @@ +{ + "id": "order-out-for-delivery", + "name": "ORDER_OUT_FOR_DELIVERY", + "receive": { + "topic": "out-for-delivery-orders", + "key": 456, + "payload": { + "orderId": 456, + "deliveryAddress": "1234 Elm Street, Springfield", + "deliveryDate": "2025-04-14" + } + }, + "after": [ + { + "type": "http", + "wait": "PT2S", + "http-request": { + "baseUrl": "http://sut:8080", + "path": "/orders/456?status=SHIPPED", + "method": "GET" + }, + "http-response": { + "status": 200, + "body": { + "id": "$match(exact: 456)", + "orderItems": [], + "lastUpdatedDate": "$match(dataType: datetime)", + "status": "$match(exact: SHIPPED)" + } + }, + "timeout": "PT10S" + }, + { + "type": "http", + "http-request": { + "baseUrl": "http://localhost:8090", + "path": "/_specmatic/verify?exampleIds=tax-invoice-for-order-456", + "method": "GET" + }, + "http-response": { + "status": 200, + "body": { + "tax-invoice-for-order-456": "$match(exact: 1)" + } + }, + "timeout": "PT20S" + } + ] +} diff --git a/async-event-flow/README.md b/async-event-flow/README.md index 87d5bd7..65ddc67 100644 --- a/async-event-flow/README.md +++ b/async-event-flow/README.md @@ -75,15 +75,21 @@ Together, `receive`/`send` plus `before`/`after` fixtures let you express full e ## Run the contract tests using Specmatic Studio 1. Start Kafka, the sample service, and Specmatic Studio. ```shell -docker compose up +docker compose up -d ``` 2. Go to [Studio](http://127.0.0.1:9000/_specmatic/studio) and open the [specmatic.yaml](specmatic.yaml) file from the left sidebar, click on "Run Suite", and use the checked-out contract under `.specmatic/repos/labs-contracts/asyncapi/async-event-flow/async-order-service.yaml` if you want to inspect the loaded AsyncAPI file in Studio. You should first see 2 passing tests and 2 failing tests: +Alternatively, just run the following commands: + +```shell +docker compose exec -T studio specmatic run-suite +``` + ```terminaloutput -Tests run: 4, Successes: 2, Failures: 2, Errors: 0 +Tests run: 4, Successes: 2, Failures: 2, WIP: 0, Errors: 0 ``` 3. Fix the examples: @@ -126,17 +132,31 @@ To: "tax-invoice-for-order-456": "$match(exact: 1)" ``` +Alternatively, just run the following commands: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp .backup/acceptOrder-with-before.json examples/async-order-service/acceptOrder.json && cp .backup/outForDeliveryOrder-with-before.json examples/async-order-service/outForDeliveryOrder.json' +``` + 4. Restart Docker Containers + ```shell -docker compose down -v && docker compose up +docker compose down -v +docker compose up -d ``` 5. Re-run the suite from Studio. +Alternatively, just run the following commands: + +```shell +docker compose exec -T studio specmatic run-suite +``` + You should now see: ```terminaloutput -Tests run: 4, Successes: 4, Failures: 0, Errors: 0 +Tests run: 4, Successes: 4, Failures: 0, WIP: 0, Errors: 0 ``` ### Cleanup diff --git a/async-event-flow/specmatic.yaml b/async-event-flow/specmatic.yaml index aec4cc0..204d64d 100644 --- a/async-event-flow/specmatic.yaml +++ b/async-event-flow/specmatic.yaml @@ -7,7 +7,7 @@ systemUnderTest: data: examples: - directories: - - "${EX_DIR:examples}" + - "${EX_DIR:examples/async-order-service}" dependencies: services: @@ -18,7 +18,7 @@ dependencies: data: examples: - directories: - - "${EX_DIR:examples}" + - "${EX_DIR:examples/tax-service}" components: sources: @@ -46,7 +46,7 @@ components: asyncapi: type: test servers: - - host: "${KAFKA_BROKER_HOST:localhost:9092}" + - host: "${KAFKA_BROKER_HOST:kafka:9092}" protocol: kafka taxServiceMock: openapi: diff --git a/backward-compatibility-testing/README.md b/backward-compatibility-testing/README.md index d4b9bc2..68f9b57 100644 --- a/backward-compatibility-testing/README.md +++ b/backward-compatibility-testing/README.md @@ -89,14 +89,21 @@ properties: You now have an uncommitted change in a tracked contract file. Specmatic will compare it to the version on `origin/main`. +Alternatively, just run the following commands: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc "sed -i 's/version: 1.0.0/version: 1.1.0/' products.yaml; sed -i '/properties:/,/sku:/s/type: string/type: number/' products.yaml; sed -i '/^ sku:$/i\\ category:\\n type: string' products.yaml" +``` + ## Part B: Run the backward compatibility check Run: *Unix/Mac: ```shell docker run --rm \ - -v ..:/workspace \ - -v ../license.txt:/specmatic/specmatic-license.txt:ro \ + --user "$(id -u):$(id -g)" \ + -v "${PWD}/..:/workspace" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ backward-compatibility-check \ @@ -105,23 +112,20 @@ docker run --rm \ ``` ```terminaloutput +Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (INCOMPATIBLE) This spec contains breaking changes to the API ``` -Windows (PowerShell/CMD) single-line: -```shell -docker run --rm -v ..:/workspace -v ../license.txt:/specmatic/specmatic-license.txt:ro -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml +Windows PowerShell single-line: +```powershell +docker run --rm --user "$(id -u):$(id -g)" -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml ``` ```terminaloutput +Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (INCOMPATIBLE) This spec contains breaking changes to the API ``` -Why the command is structured this way: -- `-v ..:/workspace` mounts the `labs` repository root, not just this lab folder, so Specmatic can access the git repository metadata. -- `--base-branch origin/main` tells Specmatic which tracked baseline to compare against. -- `--target-path backward-compatibility-testing/products.yaml` tells Specmatic to compare the working tree version of this file with the tracked version on `origin/main`. - Expected failure highlights: ```terminaloutput @@ -135,11 +139,12 @@ The Incompatibility Report: This is number in the new specification response but string in the old specification ``` -Expected verdict: +Why the command is structured this way: +- `-v "${PWD}/..:/workspace"` mounts the `labs` repository root, not just this lab folder, so Specmatic can access the git repository metadata. +- `--user "$(id -u):$(id -g)"` runs the container as your host user, which avoids git ownership issues when the mounted repository is inspected inside the container. +- `--base-branch origin/main` tells Specmatic which tracked baseline to compare against. +- `--target-path backward-compatibility-testing/products.yaml` tells Specmatic to compare the working tree version of this file with the tracked version on `origin/main`. -```terminaloutput -(INCOMPATIBLE) This spec contains breaking changes to the API -``` Why this fails: - Adding optional `category` is safe. @@ -165,14 +170,21 @@ name: Keep the new `category` field. Keep version `1.1.0`. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc 'sed -i "/properties:/,/sku:/s/type: number/type: string/" products.yaml' +``` + ## Part D: Re-run the check Run the same command again: *Unix/Mac: ```shell docker run --rm \ - -v ..:/workspace \ - -v ../license.txt:/specmatic/specmatic-license.txt:ro \ + --user "$(id -u):$(id -g)" \ + -v "${PWD}/..:/workspace" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ backward-compatibility-check \ @@ -185,9 +197,9 @@ Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (COMPATIBLE) The spec is backward compatible with the corresponding spec from origin/main ``` -Windows (PowerShell/CMD) single-line: -```shell -docker run --rm -v ..:/workspace -v ../license.txt:/specmatic/specmatic-license.txt:ro -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml +Windows PowerShell single-line: +```powershell +docker run --rm --user "$(id -u):$(id -g)" -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml ``` ```terminaloutput @@ -195,13 +207,6 @@ Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (COMPATIBLE) The spec is backward compatible with the corresponding spec from origin/main ``` -Expected passing output: - -```terminaloutput -Verdict for spec /workspace/backward-compatibility-testing/products.yaml: - (COMPATIBLE) The spec is backward compatible with the corresponding spec from origin/main -``` - ## Clean up Restore the tracked file: @@ -209,10 +214,6 @@ Restore the tracked file: git restore products.yaml ``` -```terminaloutput -products.yaml restored. -``` - ## Check backward compatibility in Specmatic Studio before saving Start Studio from `labs/backward-compatibility-testing`: diff --git a/continuous-integration/README.md b/continuous-integration/README.md index 0525880..0e75389 100644 --- a/continuous-integration/README.md +++ b/continuous-integration/README.md @@ -99,7 +99,7 @@ Expected behavior: Expected failure highlight: ```terminaloutput -(INCOMPATIBLE) This spec contains breaking changes to the API +(INCOMPATIBLE) The changes to the spec are NOT backward compatible with the corresponding spec from main ``` Why this fails: @@ -139,6 +139,12 @@ Keep: Do not change anything else. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "awk 'BEGIN{removed=0} {if (removed==0 && \$0==\" - priority\") {removed=1; next} print}' contracts/order_api.yaml > /tmp/order_api.yaml && mv /tmp/order_api.yaml contracts/order_api.yaml" +``` + ## Part C: Re-run the CI simulation Run the same command again: diff --git a/data-adapters/README.md b/data-adapters/README.md index 332e537..c40c996 100644 --- a/data-adapters/README.md +++ b/data-adapters/README.md @@ -42,7 +42,7 @@ This lab will help you understand how to identify such mismatches and use Specma Run: ```shell -docker compose up +docker compose up -d ``` ## 2. Trigger the mismatch from browser (intentional failure) @@ -101,11 +101,17 @@ Run: chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n adapters:\n pre_specmatic_request_processor: ./hooks/pre_specmatic_request_processor.sh\n post_specmatic_response_processor: ./hooks/post_specmatic_response_processor.sh\n' specmatic.yaml && chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh" +``` + ## 6. Restart mock + UI Run: ```shell -docker compose up +docker compose up -d ``` ## 7. Trigger the matching request/response from browser @@ -144,7 +150,7 @@ git update-index --chmod=+x hooks/pre_specmatic_request_processor.sh hooks/post_ - Ensure hook files use LF line endings (not CRLF). In Git Bash: -```shell +```powershell sed -i 's/\r$//' hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh ``` diff --git a/data-adapters/docker-compose.yaml b/data-adapters/docker-compose.yaml index bafe655..1f08f77 100644 --- a/data-adapters/docker-compose.yaml +++ b/data-adapters/docker-compose.yaml @@ -41,6 +41,10 @@ services: condition: service_healthy entrypoint: ["curl"] command: + - --connect-timeout + - "10" + - --max-time + - "20" - -i - -X - POST diff --git a/dictionary/README.md b/dictionary/README.md index 86bd0fd..c0b2dfc 100644 --- a/dictionary/README.md +++ b/dictionary/README.md @@ -25,7 +25,7 @@ docker compose up suite --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 3, Successes: 0, Failures: 3, Errors: 0 +Tests run: 3, Successes: 0, Failures: 3, WIP: 0, Errors: 0 ``` ### Root cause of the failed tests @@ -42,11 +42,11 @@ docker compose down -v Generate dictionary data from existing examples: ```shell -docker run --rm -v .:/usr/src/app specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml +docker run --rm -v "${PWD}:/usr/src/app" specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml ``` ```terminaloutput -Generated dictionary file at specs/dictionary.yaml +Dictionary file written to /usr/src/app/specs/dictionary.yaml ``` Open and understand the [generated dictionary file](specs/dictionary.yaml) @@ -59,6 +59,13 @@ data: path: specs/dictionary.yaml ``` +Alternatively, just run the following commands: + +```shell +docker run --rm -v "${PWD}:/usr/src/app" specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n dictionary:\n path: specs/dictionary.yaml\n' specmatic.yaml" +``` + ## 3. Re-run the suite after configuring dictionary ```shell @@ -68,7 +75,7 @@ docker compose up suite --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 3, Successes: 3, Failures: 0, Errors: 0 +Tests run: 3, Successes: 3, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: @@ -78,8 +85,8 @@ docker compose down -v ``` ## Pass Criteria -- Baseline run fails with `Tests run: 3, Successes: 0, Failures: 3, Errors: 0`. -- After dictionary configuration, run passes with `Tests run: 3, Successes: 3, Failures: 0, Errors: 0`. +- Baseline run fails with `Tests run: 3, Successes: 0, Failures: 3, WIP: 0, Errors: 0`. +- After dictionary configuration, run passes with `Tests run: 3, Successes: 3, Failures: 0, WIP: 0, Errors: 0`. ## Additional Resources - [Specmatic Dictionary Documentation](https://docs.specmatic.io/features/dictionary) diff --git a/external-examples/.backup/createOrder_application_json_201_application_json_1.json b/external-examples/.backup/createOrder_application_json_201_application_json_1.json new file mode 100644 index 0000000..33c51e0 --- /dev/null +++ b/external-examples/.backup/createOrder_application_json_201_application_json_1.json @@ -0,0 +1,25 @@ +{ + "http-request": { + "path": "/orders", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "productid": 442, + "count": 325 + }, + "description": "This is an example of a request to create a new entity at the /orders endpoint " + }, + "http-response": { + "status": 201, + "body": { + "id": 915 + }, + "status-text": "Created", + "headers": { + "Content-Type": "application/json" + }, + "description": "This is an example of a response after creating a new entity at the /orders endpoint " + } +} \ No newline at end of file diff --git a/external-examples/.backup/createProduct_application_json_201_application_json_1.json b/external-examples/.backup/createProduct_application_json_201_application_json_1.json new file mode 100644 index 0000000..2481334 --- /dev/null +++ b/external-examples/.backup/createProduct_application_json_201_application_json_1.json @@ -0,0 +1,26 @@ +{ + "http-request": { + "path": "/products", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "name": "Dr. Brendon Mosciski", + "type": "book", + "inventory": 67 + }, + "description": "This is an example of a request to create a new entity at the /products endpoint " + }, + "http-response": { + "status": 201, + "body": { + "id": 124 + }, + "status-text": "Created", + "headers": { + "Content-Type": "application/json" + }, + "description": "This is an example of a response after creating a new entity at the /products endpoint " + } +} \ No newline at end of file diff --git a/external-examples/README.md b/external-examples/README.md index dfd73cb..7f5a581 100644 --- a/external-examples/README.md +++ b/external-examples/README.md @@ -46,8 +46,8 @@ Test Run Cmd (Linux/Mac OSX) ```shell docker run --rm \ - -v .:/usr/src/app \ - -v ../license.txt:/specmatic/specmatic-license.txt:ro \ + -v "${PWD}:/usr/src/app" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ specmatic/enterprise:latest \ validate ``` @@ -57,10 +57,10 @@ docker run --rm \ [FAIL] Examples: 1 passed and 3 failed out of 4 total ``` -Windows (PowerShell/CMD) single-line: +Windows PowerShell single-line: -```shell -docker run --rm -v .:/usr/src/app -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest validate +```powershell +docker run --rm -v "$($PWD.Path):/usr/src/app" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest validate ``` Expected output: @@ -93,14 +93,28 @@ In Studio, update the failing examples: 3. `examples/test_accepted_order_request.json` - Add missing required property `count` (for example `2`) in request body. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc ' +sed -i "s/\"to-date\": \"today\"/\"to-date\": \"2025-11-28\"/" examples/test_find_available_products_book_200.json && +sed -i "s/\"type\": \"movie\"/\"type\": \"book\"/" examples/test_accepted_product_request.json && +sed -i "s/\"inventory\": \"five\"/\"inventory\": 5/" examples/test_accepted_product_request.json && +sed -i "s/\"productid\": 1234/\"productid\": 1234,/" examples/test_accepted_order_request.json && +sed -i "/\"productid\": 1234,/a\\ \"count\": 2" examples/test_accepted_order_request.json +' +``` + #### 4. Generate missing examples in the same Studio flow Still in Studio, generate examples for: - `POST /products` with response `201` - `POST /orders` with response `201` -Expected output: -```terminaloutput -[OK] Examples: 6 passed and 0 failed out of 6 total + +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp .backup/* examples/' ``` ### Final Phase @@ -111,8 +125,8 @@ Test Run Cmd (Linux/Mac OSX) ```shell docker run --rm \ - -v .:/usr/src/app \ - -v ../license.txt:/specmatic/specmatic-license.txt:ro \ + -v "${PWD}:/usr/src/app" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ specmatic/enterprise:latest \ validate ``` @@ -122,10 +136,10 @@ docker run --rm \ [OK] Examples: 6 passed and 0 failed out of 6 total ``` -Windows (PowerShell/CMD) single-line: +Windows PowerShell single-line: -```shell -docker run --rm -v .:/usr/src/app -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest validate +```powershell +docker run --rm -v "$($PWD.Path):/usr/src/app" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest validate ``` Expected output: diff --git a/filters/README.md b/filters/README.md index bd6c2f2..72ff384 100644 --- a/filters/README.md +++ b/filters/README.md @@ -32,7 +32,7 @@ docker compose up --abort-on-container-exit Expected baseline output: ```terminaloutput -Tests run: 136, Successes: 20, Failures: 114, Errors: 2 +Tests run: 136, Successes: 18, Failures: 114, WIP: 4, Errors: 0 ``` Clean up: @@ -87,6 +87,12 @@ Stop Studio: docker compose --profile studio down -v ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i \"/baseUrl: http:\\/\\/localhost:8080/a\\ filter: \\\"PATH!='/health,/monitor/{id},/swagger' && STATUS='200,201'\\\"\" specmatic.yaml" +``` + ## 5. Verify from CLI (with persisted filters) Run: @@ -97,7 +103,7 @@ docker compose up --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 20, Successes: 20, Failures: 0, Errors: 0 +Tests run: 20, Successes: 18, Failures: 0, WIP: 2, Errors: 0 ``` Clean up: @@ -109,7 +115,7 @@ docker compose down -v ## Pass Criteria - Baseline run shows `136` tests with many failures. - After applying and exporting filters, CLI run shows: - - `Tests run: 20, Successes: 20, Failures: 0, Errors: 0` + - `Tests run: 20, Successes: 18, Failures: 0, WIP: 2, Errors: 0` ## Why this lab matters - Filters help teams focus on critical scenarios while they triage known failures. diff --git a/kafka-avro/README.md b/kafka-avro/README.md index 09dea26..75cd58d 100644 --- a/kafka-avro/README.md +++ b/kafka-avro/README.md @@ -76,18 +76,28 @@ docker compose up specmatic-test --abort-on-container-exit Expected failure signal: ```terminaloutput -Timeout waiting for a message on topic 'wip-orders'. -Refer to Message Count Report to verify the message counts on different topics. - -Tests run: 2, Successes: 0, Failures: 2, Errors: 0 +Unsuccessful Scenarios: + "Upon receiving a message on 'new-orders' channel, should send a message on 'wip-orders' channel. Example: PLACE_MACBOOK_ORDER FAILED" + Reason: + Cannot convert value: 600.0 to an Avro Integer + + "Upon receiving a message on 'new-orders' channel, should send a message on 'wip-orders' channel. Example: PLACE_IPHONE_ORDER FAILED" + Reason: + Cannot convert value: 500.0 to an Avro Integer + +Tests run: 2, Successes: 0, Failures: 2, WIP: 0, Errors: 0 ``` The message count report should show: ```terminaloutput -| topic | Actual | Expected | -| new-orders | 2 | 2 | -| wip-orders | 0 | 2 | ++------------+-------------------------+----------+ +| Topic | No of messages received | ++------------+-------------------------+----------+ +| | Actual | Expected | ++------------+-------------------------+----------+ +| new-orders | 2 | 2 | ++------------+-------------------------+----------+ ``` Clean up before making changes: @@ -187,6 +197,20 @@ Replace `docker-config/avro/WipOrders.avsc` with: } ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc "sed -i -e '/^ \"name\": \"id\",$/{n;s/^ \"type\": \"int\"$/&,\\ + \"x-minimum\": 1,\\ + \"x-maximum\": 100/;}' /usr/src/app/docker-config/avro/WipOrders.avsc && sed -i -e '/^ \"name\": \"id\",$/{n;s/^ \"type\": \"int\"$/&,\\ + \"x-minimum\": 1,\\ + \"x-maximum\": 100/;}' -e '/^ \"name\": \"name\",$/{n;s/^ \"type\": \"string\"$/&,\\ + \"x-minLength\": 2,\\ + \"x-maxLength\": 10,\\ + \"x-regex\": \"^[A-Za-z]{2,10}$\"/;}' -e '/^ \"name\": \"price\",$/{n;s/^ \"type\": \"int\"$/&,\\ + \"x-minimum\": 1000/;}' /usr/src/app/docker-config/avro/NewOrders.avsc" +``` + ### Step 2: Update the examples In both files under `api-specs/order-service-async-avro-v3_0_0_examples/`, replace the current invalid values with values that satisfy the new schema constraints. @@ -258,6 +282,12 @@ Replace `api-specs/order-service-async-avro-v3_0_0_examples/PLACE_MACBOOK_ORDER. } ``` +Alternatively, just run the following commands: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc "sed -i 's/\"id\": 101/\"id\": 1/; s/iPhone 14 Pro Max/iPhone/; s/\"price\": 500.00/\"price\": 5000/; s/exact:101/exact:1/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_IPHONE_ORDER.json && sed -i 's/\"id\": 102/\"id\": 2/; s/Macbook Mini Pro M5/Macbook/; s/\"price\": 600.00/\"price\": 6000/; s/exact:102/exact:2/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_MACBOOK_ORDER.json" +``` + ## Verify the fix Re-run the contract tests: @@ -269,15 +299,20 @@ docker compose up specmatic-test --abort-on-container-exit Expected passing output: ```terminaloutput -Tests run: 2, Successes: 2, Failures: 0, Errors: 0 +Tests run: 2, Successes: 2, Failures: 0, WIP: 0, Errors: 0 ``` The message count report should now show: ```terminaloutput -| topic | Actual | Expected | -| new-orders | 2 | 2 | -| wip-orders | 2 | 2 | ++------------+-------------------------+----------+ +| Topic | No of messages received | ++------------+-------------------------+----------+ +| | Actual | Expected | ++------------+-------------------------+----------+ +| new-orders | 2 | 2 | +| wip-orders | 2 | 2 | ++------------+-------------------------+----------+ ``` Clean up: diff --git a/kafka-avro/docker-config/avro/NewOrders.avsc b/kafka-avro/docker-config/avro/NewOrders.avsc index 86cbc92..80d92f4 100644 --- a/kafka-avro/docker-config/avro/NewOrders.avsc +++ b/kafka-avro/docker-config/avro/NewOrders.avsc @@ -15,10 +15,22 @@ "type": "record", "name": "Item", "fields": [ - { "name": "id", "type": "int" }, - { "name": "name", "type": "string" }, - { "name": "quantity", "type": "int" }, - { "name": "price", "type": "int" } + { + "name": "id", + "type": "int" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "quantity", + "type": "int" + }, + { + "name": "price", + "type": "int" + } ] } } diff --git a/kafka-sqs-retry-dlq/README.md b/kafka-sqs-retry-dlq/README.md index 10db3cb..0a4af62 100644 --- a/kafka-sqs-retry-dlq/README.md +++ b/kafka-sqs-retry-dlq/README.md @@ -67,7 +67,7 @@ Expected failure shape: Preferred failure summary: ```terminaloutput -Tests run: 6, Successes: 4, Failures: 2, Errors: 0 +Tests run: 6, Successes: 4, Failures: 2, WIP: 0, Errors: 0 ``` The failing scenarios should be: @@ -98,6 +98,12 @@ What to look for: Do not change the contract, examples, or Compose wiring. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's/# threading.Thread/threading.Thread/' service/app.py" +``` + ## Pass criteria Re-run: @@ -109,17 +115,22 @@ docker compose up contract-test --build --abort-on-container-exit Expected passing output: ```terminaloutput -Tests run: 6, Successes: 6, Failures: 0, Errors: 0 +Tests run: 6, Successes: 6, Failures: 0, WIP: 0, Errors: 0 ``` The message count report should show: ```terminaloutput -| Topic/queue name | Actual | Expected | -| place-order-topic | 6 | 6 | -| place-order-queue | 4 | 4 | -| place-order-retry-topic | 2 | 2 | -| place-order-dlq-topic | 2 | 2 | ++-------------------------+-------------------------+----------+ +| Topic | No of messages received | ++-------------------------+-------------------------+----------+ +| | Actual | Expected | ++-------------------------+-------------------------+----------+ +| place-order-topic | 6 | 6 | +| place-order-queue | 4 | 4 | +| place-order-dlq-topic | 2 | 2 | +| place-order-retry-topic | 2 | 2 | ++-------------------------+-------------------------+----------+ ``` Clean up: diff --git a/mcp-auto-test/README.md b/mcp-auto-test/README.md index 6482440..9223d25 100644 --- a/mcp-auto-test/README.md +++ b/mcp-auto-test/README.md @@ -98,6 +98,12 @@ Make these two fixes: Do not change anything else. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"damage\": 0.0#\"damaged\": 0.0#; s#order\\[\"shipment\"\\]#order[\"shipmentStatus\"]#' service/order_service.py" +``` + ## Part C: Re-run tests (expected to pass) Run: @@ -141,6 +147,12 @@ Overall Success Rate: 94.3% This is not required for the lab goal. It is a follow-up to explore how Specmatic mutates valid tool inputs to probe validation boundaries. Try to pass all 35 tests. +Clean up: + +```shell +docker compose down -v +``` + ## Pass criteria - Baseline run fails with two tool execution failures. - After fixing only `service/order_service.py`, both tests pass. diff --git a/overlays/.backup/path-prefix.overlay.yaml b/overlays/.backup/path-prefix.overlay.yaml new file mode 100644 index 0000000..1eb79db --- /dev/null +++ b/overlays/.backup/path-prefix.overlay.yaml @@ -0,0 +1,18 @@ +overlay: 1.0.0 +info: + title: Add /api/v1 prefix for deployed provider compatibility + version: "1.0" +actions: + - target: "$.paths" + update: + /api/v1/users/{id}: + get: + summary: Get user by id + parameters: + - $ref: "#/components/parameters/UserIdPathParam" + responses: + "200": + $ref: "#/components/responses/User200" + + - target: "$.paths['/api/users/{id}']" + remove: true diff --git a/overlays/README.md b/overlays/README.md index 01d6319..866197b 100644 --- a/overlays/README.md +++ b/overlays/README.md @@ -54,7 +54,7 @@ docker compose up test --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 1, Successes: 0, Failures: 1, Errors: 0 +Tests run: 1, Successes: 0, Failures: 1, WIP: 0, Errors: 0 ``` - The test run fails because requests are generated for `/api/users/{id}` while the provider serves `/api/v1/users/{id}`. @@ -108,6 +108,12 @@ Uncomment this line: overlayFilePath: ./overlays/path-prefix.overlay.yaml ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp .backup/path-prefix.overlay.yaml overlays/path-prefix.overlay.yaml && sed -i "s/^# overlayFilePath:/ overlayFilePath:/" specmatic.yaml' +``` + ## Pass verification Run: @@ -118,7 +124,7 @@ docker compose up test --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 1, Successes: 1, Failures: 0, Errors: 0 +Tests run: 1, Successes: 1, Failures: 0, WIP: 0, Errors: 0 ``` - Contract tests pass, because Specmatic applies the overlay before running tests. diff --git a/partial-examples/README.md b/partial-examples/README.md index 099c65b..b324769 100644 --- a/partial-examples/README.md +++ b/partial-examples/README.md @@ -41,12 +41,12 @@ Some APIs may have a lot of transient, but mandatory fields which does not matte Validate the original incomplete examples and observe the intentional failures. -Test Run Cmd (Linux/Mac OSX) +Validate Command (Linux/Mac OSX) ```shell docker run --rm \ - -v .:/usr/src/app \ - -v ../license.txt:/specmatic/specmatic-license.txt:ro \ + -v "${PWD}:/usr/src/app" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ specmatic/enterprise:latest \ validate ``` @@ -56,10 +56,10 @@ docker run --rm \ [FAIL] Examples: 0 passed and 3 failed out of 3 total ``` -Windows (PowerShell/CMD) single-line +Validate Command (Windows PowerShell) -```shell -docker run --rm -v .:/usr/src/app -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest validate +```powershell +docker run --rm -v "$($PWD.Path):/usr/src/app" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest validate ``` ```terminaloutput @@ -98,14 +98,43 @@ Stop Studio after the examples are saved: docker compose --profile studio down -v ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc ' +sed -i "2i\\ \\\"partial\\\": {" examples/test_accepted_order_request.json && +sed -i "\$i\\ }" examples/test_accepted_order_request.json && +sed -i "2i\\ \\\"partial\\\": {" examples/test_accepted_product_request.json && +sed -i "\$i\\ }" examples/test_accepted_product_request.json && +sed -i "s#\\\"path\\\": \\\"/findAvailableProducts\\\"#\\\"path\\\": \\\"/findAvailableProducts?type=book\\\"#" examples/test_find_available_products_book_200.json && +sed -i "/\\\"query\\\": {/a\\ \\\"from-date\\\": \\\"2025-10-01\\\",\\n \\\"to-date\\\": \\\"2025-10-15\\\"" examples/test_find_available_products_book_200.json && +sed -i "/\\\"type\\\": \\\"book\\\"/d" examples/test_find_available_products_book_200.json +' +``` + ### Final Phase -Re-run validation with the Windows single-line command after the Studio fixes are saved. +Re-run validation after the Studio fixes are saved. -Windows (PowerShell/CMD) single-line +Validate Command (Linux/Mac OSX) ```shell -docker run --rm -v .:/usr/src/app -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest validate +docker run --rm \ + -v "${PWD}:/usr/src/app" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ + specmatic/enterprise:latest \ + validate +``` + +```terminaloutput +[OK] Specification product_search_bff_v6.yaml: PASSED +[OK] Examples: 3 passed and 0 failed out of 3 total +``` + +Validate Command (Windows PowerShell) + +```powershell +docker run --rm -v "$($PWD.Path):/usr/src/app" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest validate ``` ```terminaloutput @@ -144,7 +173,7 @@ docker compose up --abort-on-container-exit This runs the suite, starts the dependency mocks, and executes the tests. You should see: ```terminaloutput -Tests run: 7, Successes: 7, Failures: 0, Errors: 0 +Tests run: 7, Successes: 6, Failures: 0, WIP: 1, Errors: 0 ``` ```shell @@ -153,7 +182,7 @@ docker compose down -v ## Pass Criteria - Validation shows: `3 passed and 0 failed out of 3 total`. -- Loop test shows: `Tests run: 7, Successes: 7, Failures: 0, Errors: 0`. +- Loop test shows: `Tests run: 7, Successes: 6, Failures: 0, WIP: 1, Errors: 0`. ## What you learned diff --git a/quick-start-api-testing/README.md b/quick-start-api-testing/README.md index 6694d33..db34f35 100644 --- a/quick-start-api-testing/README.md +++ b/quick-start-api-testing/README.md @@ -74,7 +74,7 @@ docker compose up api-test --build --abort-on-container-exit * Expected console output ```terminaloutput -Tests run: 4, Successes: 2, Failures: 2, Errors: 0 +Tests run: 4, Successes: 2, Failures: 2, WIP: 0, Errors: 0 ``` Why the baseline fails: @@ -99,6 +99,12 @@ In `http-response.body`, change: Do not change any other fields. +Alternatively, just run the following command for Task A: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#[$]match(exact: approved)#\$match(pattern: approved\|verified)#' examples/test_finance_user_11.json" +``` + Re-run: ```shell @@ -108,7 +114,7 @@ docker compose up api-test --build --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 4, Successes: 3, Failures: 1, Errors: 0 +Tests run: 4, Successes: 3, Failures: 1, WIP: 0, Errors: 0 ``` Clean up: @@ -128,6 +134,12 @@ In `http-response.body`, change: Keep `handledBy` and `decision` as exact matches. +Alternatively, just run the following command for Final Phase: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#[$]match(exact: VRF-123456)#\$match(pattern: VRF-[0-9]{6})#; s#[$]match(exact: 2026-03-17)#\$match(dataType: date)#' examples/test_support_user_55.json" +``` + Run: ```shell @@ -137,7 +149,7 @@ docker compose up api-test --build --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 4, Successes: 4, Failures: 0, Errors: 0 +Tests run: 4, Successes: 4, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: diff --git a/quick-start-async-contract-testing/README.md b/quick-start-async-contract-testing/README.md index 5950bbb..2f66d62 100644 --- a/quick-start-async-contract-testing/README.md +++ b/quick-start-async-contract-testing/README.md @@ -45,7 +45,7 @@ docker compose up contract-test --build --abort-on-container-exit ``` ```terminaloutput -Tests run: 1, Successes: 0, Failures: 1, Errors: 0 +Tests run: 1, Successes: 0, Failures: 1, WIP: 0, Errors: 0 ``` Expected failure signal: @@ -73,6 +73,12 @@ To: "status": "INITIATED", ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"status\": \"STARTED\"#\"status\": \"INITIATED\"#' service/processor.py" +``` + ## Pass criteria Re-run: @@ -83,7 +89,7 @@ docker compose up contract-test --build --abort-on-container-exit Expected pass signal: ```terminaloutput -Tests run: 1, Successes: 1, Failures: 0, Errors: 0 +Tests run: 1, Successes: 1, Failures: 0, WIP: 0, Errors: 0 ``` Then clean up: diff --git a/quick-start-contract-testing/README.md b/quick-start-contract-testing/README.md index 90edf76..99b5d46 100644 --- a/quick-start-contract-testing/README.md +++ b/quick-start-contract-testing/README.md @@ -47,7 +47,7 @@ docker compose up test --build --abort-on-container-exit ``` ```terminaloutput -Request to http://petstore:8080 at +Request to http://petstore:8080 at GET /pets/1 Specmatic-Response-Code: 200 Host: petstore:8080 @@ -55,10 +55,10 @@ Request to http://petstore:8080 at Accept: */* Content-Type: NOT SENT -Response at +Response at 200 OK - Server: BaseHTTP/0.6 Python/3.14.3 - Date: + Server: BaseHTTP/0.6 Python/ + Date: Content-Type: application/json Content-Length: 79 @@ -74,7 +74,7 @@ Scenario: GET /pets/(petid:number) -> 200 with the request from the example 'SCO Expected output: ```terminaloutput -Tests run: 1, Successes: 0, Failures: 1, Errors: 0 +Tests run: 1, Successes: 0, Failures: 1, WIP: 0, Errors: 0 ``` How did Specmatic generate the GET request for /pets with ID `1`? @@ -108,6 +108,12 @@ with: Do not change anything else. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's/"petType"/"type"/g' service/server.py" +``` + ## Part C: Re-run tests (expected to pass) Run: @@ -117,7 +123,7 @@ docker compose up test --build --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 1, Successes: 1, Failures: 0, Errors: 0 +Tests run: 1, Successes: 1, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: diff --git a/quick-start-mock/.backup/pets_242_GET_200_1.json b/quick-start-mock/.backup/pets_242_GET_200_1.json new file mode 100644 index 0000000..2393a97 --- /dev/null +++ b/quick-start-mock/.backup/pets_242_GET_200_1.json @@ -0,0 +1,19 @@ +{ + "http-request": { + "path": "/pets/242", + "method": "GET" + }, + "http-response": { + "status": 200, + "body": { + "id": 242, + "name": "Bolt", + "type": "Mixed Breed", + "status": "Available" + }, + "status-text": "OK", + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/quick-start-mock/README.md b/quick-start-mock/README.md index bb41ed2..74ce16a 100644 --- a/quick-start-mock/README.md +++ b/quick-start-mock/README.md @@ -45,7 +45,7 @@ In real projects, consumer teams are often blocked because a dependency is late, Start only the consumer: ```shell -docker compose up consumer --build +docker compose up -d --wait consumer ``` Open [http://127.0.0.1:8081](http://127.0.0.1:8081). @@ -62,11 +62,17 @@ Why this fails: - Consumer is running. - Provider at `:9100` is not running yet. +Alternatively, just run the following commands: + +```shell +docker run --rm --network quick-start-mock_default --entrypoint sh specmatic/enterprise:latest -lc 'curl --fail --silent --show-error http://127.0.0.1:9100/pets/1 >/dev/null && exit 1 || echo "Request failed as expected"' +``` + ## Part B: Start contract-generated mock (consumer unblocked) -Keep Part A terminal running. In a second terminal, run: +With consumer still running, run: ```shell -docker compose --profile mock up mock +docker compose --profile mock up -d --wait mock ``` Go back to consumer UI and click **Load Pet** again. @@ -83,9 +89,43 @@ Try additional IDs: - `2` -> success with contract-compliant dynamic data generated by Specmatic Mock. Notice that each time you try the same ID, you get a different response. This shows that the mock is generating data from the contract rather than just returning a static example. - `abc` -> error response (`400`) because path parameter must be numeric -## Part C: Stop only the mock and observe fallback -In the terminal where mock is running, press `Ctrl+C`. +Alternatively, just run the following commands: +```shell +expected='{"id":1,"name":"Scooby","type":"GoldenRetriever","status":"Adopted"}' +actual="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/1 | tr -d '\n ' )" +[ "$actual" = "$expected" ] || { + echo "Unexpected payload: $actual" + exit 1 +} +first="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://mock:9100/pets/2)" +second="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://mock:9100/pets/2)" +first_body="$(printf '%s\n' "$first" | sed '$d')" +first_status="$(printf '%s\n' "$first" | tail -n 1)" +second_body="$(printf '%s\n' "$second" | sed '$d')" +second_status="$(printf '%s\n' "$second" | tail -n 1)" +[ "$first_status" = "200" ] || { + echo "First call expected 200, got $first_status" + exit 1 +} +[ "$second_status" = "200" ] || { + echo "Second call expected 200, got $second_status" + exit 1 +} +[ "$first_body" != "$second_body" ] || { + echo "Expected different response bodies, but both were:" + printf '%s\n' "$first_body" + exit 1 +} +echo "Both calls returned 200 and different bodies" +status="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -o /dev/null -w '%{http_code}' http://mock:9100/pets/abc)" +[ "$status" = "400" ] || { + echo "Expected 400, got $status" + exit 1 +} +``` + +## Part C: Stop only the mock and observe fallback Clean up: ```shell @@ -97,11 +137,17 @@ Go back to consumer UI and click **Load Pet** again. Expected output: - Status returns to `Service unavailable`. +Alternatively, just run the following commands: + +```shell +docker run --rm --network quick-start-mock_default --entrypoint sh specmatic/enterprise:latest -lc 'curl --fail --silent --show-error http://127.0.0.1:9100/pets/1 >/dev/null && exit 1 || echo "Request failed as expected"' +``` + ## Part D: Run mock from Studio and inspect traffic Start Studio in a new terminal: ```shell -docker compose --profile studio up studio +docker compose --profile studio up -d --wait studio ``` Open [http://127.0.0.1:9000/_specmatic/studio](http://127.0.0.1:9000/_specmatic/studio). @@ -129,6 +175,33 @@ To inspect mock traffic in Studio: 2. Open `Scenario: GET /pets/(petid:number) -> 200`. 3. Review request and response details and confirm they match the generated example. +Alternatively, just run the following commands: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'mkdir -p specs/service_examples && cp .backup/pets_242_GET_200_1.json specs/service_examples/pets_242_GET_200_1.json' +docker compose exec -T studio sh -lc 'specmatic mock /usr/src/app/specs/service.yaml --port 9100 >/tmp/specmatic-mock.log 2>&1 &' +first="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://studio:9100/pets/242)" +second="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://studio:9100/pets/242)" +first_body="$(printf '%s\n' "$first" | sed '$d')" +first_status="$(printf '%s\n' "$first" | tail -n 1)" +second_body="$(printf '%s\n' "$second" | sed '$d')" +second_status="$(printf '%s\n' "$second" | tail -n 1)" +[ "$first_status" = "200" ] || { + echo "First call expected 200, got $first_status" + exit 1 +} +[ "$second_status" = "200" ] || { + echo "Second call expected 200, got $second_status" + exit 1 +} +[ "$first_body" = "$second_body" ] || { + echo "Expected same response bodies, but both were different:" + printf 'First: %s\n\nSecond: %s' "$first_body" "$second_body" + exit 1 +} +echo "Both calls returned 200 and the same response body" +``` + ## Pass criteria - Without mock: consumer shows `Service unavailable`. - With mock running: consumer shows `Success` and returns JSON. diff --git a/quick-start-mock/specs/service.yaml b/quick-start-mock/specs/service.yaml index 808a82a..7641441 100644 --- a/quick-start-mock/specs/service.yaml +++ b/quick-start-mock/specs/service.yaml @@ -40,5 +40,5 @@ paths: value: id: 1 name: Scooby - type: Golden Retriever + type: GoldenRetriever status: Adopted \ No newline at end of file diff --git a/response-templating/.backup/test_find_available_products_book_200.json b/response-templating/.backup/test_find_available_products_book_200.json new file mode 100644 index 0000000..f9335f6 --- /dev/null +++ b/response-templating/.backup/test_find_available_products_book_200.json @@ -0,0 +1,44 @@ +{ + "http-request": { + "path": "/findAvailableProducts", + "method": "GET", + "query": { + "from-date": "2026-02-15", + "to-date": "2026-02-15", + "type": "(TYPE:ProductType)" + }, + "headers": { + "pageSize": "10" + } + }, + "http-response": { + "status": 200, + "body": { + "id": "$(dataLookup.products[TYPE].id)", + "name": "$(dataLookup.products[TYPE].name)", + "type": "$(TYPE)", + "inventory": "$(dataLookup.products[TYPE].inventory)", + "createdOn": "$(dataLookup.products[TYPE].createdOn)" + }, + "status-text": "OK", + "headers": { + "Content-Type": "application/json" + } + }, + "dataLookup": { + "products": { + "book": { + "id": 1, + "name": "Larry Potter", + "inventory": 100, + "createdOn": "2026-02-15" + }, + "gadget": { + "id": 2, + "name": "iPhone", + "inventory": 500, + "createdOn": "2026-02-15" + } + } + } +} diff --git a/response-templating/README.md b/response-templating/README.md index 26592f5..92dcd61 100644 --- a/response-templating/README.md +++ b/response-templating/README.md @@ -58,7 +58,7 @@ docker compose up --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 4, Successes: 1, Failures: 3, Errors: 0 +Tests run: 4, Successes: 1, Failures: 3, WIP: 0, Errors: 0 ``` Clean up: @@ -81,14 +81,21 @@ Change response templating so: Keep `id` as-is. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"productid\": 1#\"productid\": \"(PRODUCTID:number)\"#; s#\"count\": 100#\"count\": \"(COUNT:number)\"#; s#\"productid\": \"\$match(dataType:number)\"#\"productid\": \"\$(PRODUCTID)\"#; s#\"count\": 1#\"count\": \"\$(COUNT)\"#' examples/mock/test_accepted_order_request.json" +``` + ### Checkpoint after Task A Run: ```shell docker compose up --abort-on-container-exit ``` + Expected checkpoint output: ```terminaloutput -Tests run: 4, Successes: 2, Failures: 2, Errors: 0 +Tests run: 4, Successes: 2, Failures: 2, WIP: 0, Errors: 0 ``` Clean up: ```shell @@ -104,7 +111,14 @@ Configure lookup logic based on request query `type` so that: - for `type=gadget` return response values matching test expectation (`id=2`, `name=iPhone`, `type=gadget`, `inventory=500`, `createdOn` as valid date) Note: -- You can keep this as one lookup-driven mock example instead of creating duplicate mock files. +- You MUST keep this as one lookup-driven mock example instead of creating duplicate mock files. +- type's data type should be ProductType not string. type is an enum which allows only specific values and not any string. + +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp .backup/test_find_available_products_book_200.json examples/mock/test_find_available_products_book_200.json' +``` ## 4. Final verification Run: @@ -114,7 +128,7 @@ docker compose up --abort-on-container-exit Expected final output: ```terminaloutput -Tests run: 4, Successes: 4, Failures: 0, Errors: 0 +Tests run: 4, Successes: 4, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: diff --git a/schema-design/.backup/payments-request.yaml b/schema-design/.backup/payments-request.yaml new file mode 100644 index 0000000..ac9190d --- /dev/null +++ b/schema-design/.backup/payments-request.yaml @@ -0,0 +1,94 @@ +openapi: 3.0.0 +info: + title: Payment API + version: "1.0" +servers: + - url: http://localhost:8080 +paths: + /payments: + post: + summary: Create a payment + operationId: createPayment + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PaymentRequest" + responses: + "201": + description: Payment accepted + content: + application/json: + schema: + $ref: "#/components/schemas/PaymentAccepted" + "400": + description: Bad request + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" +components: + schemas: + CardPaymentRequest: + type: object + required: + - paymentType + - cardNumber + - cardExpiry + - cardCvv + properties: + paymentType: + type: string + enum: [ card ] + cardNumber: + type: string + cardExpiry: + type: string + cardCvv: + type: string + BankTransferPaymentRequest: + type: object + required: + - paymentType + - bankAccountNumber + - bankRoutingNumber + - bankAccountHolder + properties: + paymentType: + type: string + enum: [ bank_transfer ] + bankAccountNumber: + type: string + bankRoutingNumber: + type: string + bankAccountHolder: + type: string + PaymentRequest: + oneOf: + - $ref: "#/components/schemas/CardPaymentRequest" + - $ref: "#/components/schemas/BankTransferPaymentRequest" + discriminator: + propertyName: paymentType + mapping: + card: "#/components/schemas/CardPaymentRequest" + bank_transfer: "#/components/schemas/BankTransferPaymentRequest" + PaymentAccepted: + type: object + additionalProperties: false + properties: + id: + type: integer + status: + type: string + required: + - id + - status + ErrorResponse: + type: object + additionalProperties: false + properties: + message: + type: string + required: + - message diff --git a/schema-design/README.md b/schema-design/README.md index 7e06b76..82f6124 100644 --- a/schema-design/README.md +++ b/schema-design/README.md @@ -45,7 +45,7 @@ docker compose up contract-test --build --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 4, Successes: 2, Failures: 2, Errors: 0 +Tests run: 4, Successes: 2, Failures: 2, WIP: 0, Errors: 0 ``` Failure reason points to `POST /payments` where requests like `{ "paymentType": "card" }` and `{ "paymentType": "bank_transfer" }` are contract-valid but service returns `400` instead of expected `201`. @@ -120,6 +120,12 @@ Open `specs/payment-api.yaml` and update `PaymentRequest` to this shape: bank_transfer: "#/components/schemas/BankTransferPaymentRequest" ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp .backup/payments-request.yaml specs/payment-api.yaml' +``` + ## 3. Re-run contract tests Run: @@ -129,7 +135,7 @@ docker compose up contract-test --build --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 2, Successes: 2, Failures: 0, Errors: 0 +Tests run: 2, Successes: 2, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: diff --git a/schema-resiliency-testing/README.md b/schema-resiliency-testing/README.md index df1e1b2..0510e91 100644 --- a/schema-resiliency-testing/README.md +++ b/schema-resiliency-testing/README.md @@ -33,7 +33,7 @@ Then go to the Test tab, set url as `http://localhost:8080` and click on the "Ru You should see ```terminaloutput -Tests run: 3, Successes: 3, Failures: 0, Errors: 0 +Tests run: 3, Successes: 3, Failures: 0, WIP: 0, Errors: 0 ``` Stop Studio before moving to the next steps: @@ -54,7 +54,7 @@ This will run the suite, start the dependency mock, and run the tests against it Expected console output: ```terminaloutput -Tests run: 3, Successes: 3, Failures: 0, Errors: 0 +Tests run: 3, Successes: 3, Failures: 0, WIP: 0, Errors: 0 ``` Clean up @@ -69,6 +69,12 @@ The goal of this lab is to try different schema resiliency testing levels and se ### Positive Only Tests In `specmatic.yaml` change `schemaResiliencyTests: none` to `schemaResiliencyTests: positiveOnly` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: none#schemaResiliencyTests: positiveOnly#' specmatic.yaml" +``` + #### Run Positive only Tests Start docker containers @@ -80,7 +86,7 @@ docker compose up --abort-on-container-exit Expected console output: ```terminaloutput -Tests run: 42, Successes: 42, Failures: 0, Errors: 0 +Tests run: 42, Successes: 42, Failures: 0, WIP: 0, Errors: 0 ``` Clean up @@ -92,6 +98,12 @@ docker compose down -v ### Positive and Negative Tests (ALL) In `specmatic.yaml` change `schemaResiliencyTests: positiveOnly` to `schemaResiliencyTests: all` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: positiveOnly#schemaResiliencyTests: all#' specmatic.yaml" +``` + #### Run all Tests Start docker containers diff --git a/workflow-in-same-spec/README.md b/workflow-in-same-spec/README.md index 2f47f9f..9faad74 100644 --- a/workflow-in-same-spec/README.md +++ b/workflow-in-same-spec/README.md @@ -50,7 +50,7 @@ docker compose --profile test up test --build --abort-on-container-exit Expected baseline result: ```terminaloutput -Tests run: 4, Successes: 1, Failures: 3, Errors: 0 +Tests run: 4, Successes: 1, Failures: 3, WIP: 0, Errors: 0 ``` You should see failures for: @@ -105,6 +105,12 @@ workflow: use: "PATH.task_id" ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ workflow:\n ids:\n \"POST /tasks -> 200\":\n extract: \"BODY.tasks.[0].id\"\n \"GET /tasks/(task_id:string) -> 200\":\n use: \"PATH.task_id\"\n \"PUT /tasks/(task_id:string) -> 200\":\n use: \"PATH.task_id\"\n \"DELETE /tasks/(task_id:string) -> 204\":\n use: \"PATH.task_id\"\n' specmatic.yaml" +``` + Re-run: ```shell @@ -114,7 +120,7 @@ docker compose --profile test up test --build --abort-on-container-exit Expected passing result: ```terminaloutput -Tests run: 4, Successes: 4, Failures: 0, Errors: 0 +Tests run: 4, Successes: 4, Failures: 0, WIP: 0, Errors: 0 ``` Cleanup: