From 373dce60bc98c5d30a1359f71f6bc6384c41e313 Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Wed, 13 May 2026 23:05:28 +0530 Subject: [PATCH 01/71] feat: add validate readme command to extract command and validate the terminal output from console with the one in readme file --- api-coverage/README.md | 16 +- tests/test_validate_readme_commands.py | 528 +++++++++++++++++++ validate_readme_commands.py | 701 +++++++++++++++++++++++++ 3 files changed, 1241 insertions(+), 4 deletions(-) create mode 100644 tests/test_validate_readme_commands.py create mode 100755 validate_readme_commands.py diff --git a/api-coverage/README.md b/api-coverage/README.md index f7a9c5c..cb2b988 100644 --- a/api-coverage/README.md +++ b/api-coverage/README.md @@ -96,9 +96,11 @@ Tests run: 2, Successes: 1, Failures: 1, Errors: 0 Expected coverage highlight: ```terminaloutput -100% /pets/{petId} GET 200 1 covered -0% /pets/search GET 200 1 not implemented -0% /pets/find GET 0 0 missing in spec +| coverage | path | method | requestContentType | response | responseContentType | remarks | result | +|----------|---------------|--------|--------------------|----------|---------------------|------------------|--------| +| 100% | /pets/{petId} | GET | NA | 200 | application/json | covered | 1p | +| 0% | /pets/search | GET | NA | 200 | application/json | not implemented | 1f | +| 0% | /pets/find | GET | NA | 0 | NA | missing in spec* | | ``` Expected gate failure highlight: @@ -146,7 +148,13 @@ Change it to: Do not change anything else in the operation. -## 3. Re-run the tests and coverage check +Alternatively, just run the following command + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's|/pets/search:$|/pets/find:|' specs/service.yaml" +``` + +## Final Phase Run the same command again: ```shell diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py new file mode 100644 index 0000000..57b0e6a --- /dev/null +++ b/tests/test_validate_readme_commands.py @@ -0,0 +1,528 @@ +import io +import subprocess +import tempfile +import textwrap +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +from validate_readme_commands import ( + CommandExecutionError, + CommandSpec, + _expected_output_matches, + main, + parse_readme_commands, + print_dry_run, + reset_lab_changes, + run_cleanup_commands, + run_command_specs, + snapshot_repo_state, + should_skip_command, +) + + +ROOT_DIR = Path(__file__).resolve().parents[1] + + +class GitRepoTestCase(unittest.TestCase): + def _init_git_repo(self, repo_path: Path) -> None: + self._git(repo_path, "init") + self._git(repo_path, "config", "user.name", "Test User") + self._git(repo_path, "config", "user.email", "test@example.com") + + def _git(self, cwd: Path, *args: str) -> None: + completed = subprocess.run( + ["git", *args], + cwd=str(cwd), + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(completed.returncode, 0, msg=completed.stderr) + + +class ParseReadmeCommandsTests(GitRepoTestCase): + def test_collects_terminaloutput_blocks_until_next_shell_block(self) -> None: + readme_path = self._write_readme( + """ + Intro + + ```shell + echo first + ``` + + prose between blocks + + ```terminaloutput + first + ``` + + ```terminaloutput + second + ``` + + ```shell + echo third + ``` + """ + ) + + commands = parse_readme_commands(readme_path) + + self.assertEqual( + commands, + [ + CommandSpec( + command="echo first\n", + expected_outputs=["first\n", "second\n"], + ), + CommandSpec(command="echo third\n", expected_outputs=[]), + ], + ) + + def test_ignores_terminaloutput_before_first_shell_block(self) -> None: + readme_path = self._write_readme( + """ + ```terminaloutput + orphan + ``` + + ```shell + echo ok + ``` + """ + ) + + commands = parse_readme_commands(readme_path) + + self.assertEqual(commands, [CommandSpec(command="echo ok\n", expected_outputs=[])]) + + def test_repo_readme_parses_with_expected_shape(self) -> None: + commands = parse_readme_commands(ROOT_DIR / "api-coverage" / "README.md") + + self.assertEqual(len(commands), 7) + self.assertEqual( + commands[0].command, + "docker compose up test --build --abort-on-container-exit\n", + ) + self.assertEqual(len(commands[0].expected_outputs), 3) + self.assertEqual(commands[1].command, "docker compose down -v\n") + self.assertEqual(commands[1].expected_outputs, []) + + def _write_readme(self, content: str) -> Path: + temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(temp_dir.cleanup) + readme_path = Path(temp_dir.name) / "README.md" + readme_path.write_text(textwrap.dedent(content).lstrip(), encoding="utf-8") + return readme_path + + +class RunCommandSpecsTests(GitRepoTestCase): + def test_zero_exit_with_matching_output_passes(self) -> None: + results = run_command_specs( + [CommandSpec(command="printf 'hello\\n'", expected_outputs=["hello\n"])], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertEqual(len(results.results), 1) + self.assertEqual(results.results[0].returncode, 0) + + def test_non_zero_exit_with_matching_output_passes(self) -> None: + results = run_command_specs( + [CommandSpec(command="printf 'expected\\n'; exit 3", expected_outputs=["expected\n"])], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertEqual(len(results.results), 1) + self.assertFalse(results.results[0].skipped) + self.assertEqual(results.results[0].returncode, 3) + + def test_non_zero_exit_without_expected_output_fails(self) -> None: + with self.assertRaises(CommandExecutionError) as ctx: + run_command_specs( + [CommandSpec(command="printf 'wrong\\n'; exit 4", expected_outputs=["expected\n"])], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertIn("missing expected terminaloutput block #1", str(ctx.exception)) + self.assertIn("Exit code: 4", str(ctx.exception)) + + def test_non_zero_exit_without_terminaloutput_blocks_fails(self) -> None: + with self.assertRaises(CommandExecutionError) as ctx: + run_command_specs( + [CommandSpec(command="exit 7", expected_outputs=[])], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertIn("command exited non-zero without any expected terminaloutput blocks", str(ctx.exception)) + + def test_line_by_line_matching_allows_prefixed_actual_lines(self) -> None: + expected_output = ( + "Failed the following API Coverage Report success criteria:\n" + "Total API coverage: 50% is less than the specified minimum threshold of 100%.\n" + "Total missed operations: 1 is greater than the maximum threshold of 0.\n" + ) + actual_output = ( + "api-coverage-openapi-test | Failed the following API Coverage Report success criteria:\n" + "api-coverage-openapi-test | Total API coverage: 50% is less than the specified minimum threshold of 100%.\n" + "api-coverage-openapi-test | Total missed operations: 1 is greater than the maximum threshold of 0.\n" + ) + + self.assertTrue(_expected_output_matches(expected_output, actual_output)) + + def test_line_by_line_matching_preserves_order(self) -> None: + expected_output = "first line\nsecond line\n" + actual_output = "prefix second line\nprefix first line\n" + + self.assertFalse(_expected_output_matches(expected_output, actual_output)) + + def test_timeout_is_reported_cleanly(self) -> None: + with self.assertRaises(CommandExecutionError) as ctx: + run_command_specs( + [CommandSpec(command="sleep 2", expected_outputs=[])], + cwd=ROOT_DIR, + timeout_seconds=0.1, + ) + + self.assertIn("timed out after 0.1 seconds", str(ctx.exception)) + + def test_streams_command_output_in_realtime_to_terminal(self) -> None: + stdout_buffer = io.StringIO() + + with redirect_stdout(stdout_buffer), redirect_stderr(io.StringIO()): + results = run_command_specs( + [CommandSpec(command="printf 'streamed\\n'", expected_outputs=["streamed\n"])], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertEqual(len(results.results), 1) + self.assertIn("streamed\n", stdout_buffer.getvalue()) + + def test_skips_commands_containing_docker_and_studio(self) -> None: + results = run_command_specs( + [ + CommandSpec( + command="docker compose --profile studio up --build", + expected_outputs=[], + ) + ], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + self.assertEqual(len(results.results), 1) + self.assertTrue(results.results[0].skipped) + self.assertEqual(results.results[0].returncode, 0) + + def test_runs_cleanup_commands_after_failure_but_skips_non_cleanup_zero_output_commands(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + cleanup_marker = temp_path / "cleanup.txt" + non_cleanup_marker = temp_path / "non_cleanup.txt" + stderr_buffer = io.StringIO() + + with redirect_stderr(stderr_buffer): + with self.assertRaises(CommandExecutionError) as ctx: + run_command_specs( + [ + CommandSpec(command="printf 'actual\\n'", expected_outputs=["expected\n"]), + CommandSpec( + command=f"kill -0 $$ >/dev/null 2>&1; printf cleanup > {cleanup_marker.name}", + expected_outputs=[], + ), + CommandSpec(command=f"printf mutate > {non_cleanup_marker.name}", expected_outputs=[]), + CommandSpec(command="printf 'later\\n'", expected_outputs=["later\n"]), + ], + cwd=temp_path, + timeout_seconds=5, + ) + + self.assertIn("Cleanup commands executed", str(ctx.exception)) + self.assertIn( + "Validation failed at command #1. Skipping remaining commands and running cleanup commands...", + stderr_buffer.getvalue(), + ) + self.assertTrue(cleanup_marker.exists()) + self.assertFalse(non_cleanup_marker.exists()) + + +class CleanupCommandTests(GitRepoTestCase): + def test_runs_only_cleanup_commands_before_next_expected_output_command(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + cleanup_marker = temp_path / "cleanup.txt" + later_marker = temp_path / "later.txt" + + results = run_cleanup_commands( + [ + CommandSpec( + command=f"kill -0 $$ >/dev/null 2>&1; printf done > {cleanup_marker.name}", + expected_outputs=[], + ), + CommandSpec(command=f"printf skip > {later_marker.name}", expected_outputs=[]), + CommandSpec(command="printf 'stop here\\n'", expected_outputs=["stop here\n"]), + ], + cwd=temp_path, + timeout_seconds=5, + ) + + self.assertEqual(len(results), 1) + self.assertTrue(cleanup_marker.exists()) + self.assertFalse(later_marker.exists()) + + def test_reset_lab_changes_restores_new_tracked_changes_only(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = Path(temp_dir) + self._init_git_repo(repo_path) + tracked_path = repo_path / "tracked.txt" + preserved_path = repo_path / "preserved.txt" + tracked_path.write_text("original\n", encoding="utf-8") + preserved_path.write_text("keep me dirty\n", encoding="utf-8") + self._git(repo_path, "add", "tracked.txt", "preserved.txt") + self._git(repo_path, "commit", "-m", "init") + preserved_path.write_text("user edit\n", encoding="utf-8") + + baseline = snapshot_repo_state(repo_path) + tracked_path.write_text("lab edit\n", encoding="utf-8") + created_path = repo_path / "created.txt" + created_path.write_text("new file\n", encoding="utf-8") + + reset_summary = reset_lab_changes(repo_path, baseline) + + self.assertEqual(tracked_path.read_text(encoding="utf-8"), "original\n") + self.assertEqual(preserved_path.read_text(encoding="utf-8"), "user edit\n") + self.assertFalse(created_path.exists()) + self.assertEqual(reset_summary.restored, [Path("tracked.txt")]) + self.assertEqual(reset_summary.removed, [Path("created.txt")]) + + +class SkipCommandTests(GitRepoTestCase): + def test_skip_rule_requires_both_docker_and_studio(self) -> None: + self.assertTrue(should_skip_command("docker compose --profile studio up --build")) + self.assertFalse(should_skip_command("docker compose up test --build")) + self.assertFalse(should_skip_command("open specmatic studio")) + + +class MainTests(GitRepoTestCase): + def test_main_returns_zero_for_valid_readme(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + readme_path = Path(temp_dir) / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf 'ok\\n' + ``` + + ```terminaloutput + ok + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + exit_code = main([str(readme_path), "--timeout", "5"]) + + self.assertEqual(exit_code, 0) + + def test_dry_run_prints_command_mapping(self) -> None: + stdout_buffer = io.StringIO() + command_specs = [ + CommandSpec(command="echo one\n", expected_outputs=["one\n", "two\n"]), + CommandSpec(command="echo two\n", expected_outputs=[]), + ] + + with redirect_stdout(stdout_buffer): + print_dry_run(command_specs) + + output = stdout_buffer.getvalue() + self.assertIn("Command #1:", output) + self.assertIn("echo one", output) + self.assertIn("Expected terminaloutput blocks: 2", output) + self.assertIn("terminaloutput #1:", output) + self.assertIn("one", output) + self.assertIn("terminaloutput #2:", output) + self.assertIn("two", output) + self.assertIn("Command #2:", output) + self.assertIn("Expected terminaloutput blocks: 0", output) + + def test_cli_invocation_reports_failure(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + readme_path = Path(temp_dir) / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf 'actual\\n' + ``` + + ```terminaloutput + expected + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(readme_path)], + cwd=str(ROOT_DIR), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(completed.returncode, 1) + self.assertIn("missing expected terminaloutput block #1", completed.stderr) + + def test_cli_invocation_prints_failure_summary(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + readme_path = Path(temp_dir) / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf 'actual\\n' + ``` + + ```terminaloutput + expected + ``` + + ```shell + printf 'later\\n' + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(readme_path)], + cwd=str(ROOT_DIR), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(completed.returncode, 1) + self.assertIn("FAIL command #1", completed.stdout) + self.assertIn("SKIP command #2", completed.stdout) + + def test_dry_run_does_not_execute_commands(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + readme_path = temp_path / "README.md" + marker_path = temp_path / "marker.txt" + readme_path.write_text( + textwrap.dedent( + f""" + ```shell + printf executed > {marker_path.name} + ``` + + ```terminaloutput + executed + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + [ + "python3", + str(ROOT_DIR / "validate_readme_commands.py"), + "--dry-run", + str(readme_path), + ], + cwd=str(temp_path), + capture_output=True, + text=True, + check=False, + ) + + self.assertFalse(marker_path.exists()) + + self.assertEqual(completed.returncode, 0) + self.assertIn("Command #1:", completed.stdout) + + def test_main_resets_lab_changed_files_by_default(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = Path(temp_dir) + self._init_git_repo(repo_path) + target_path = repo_path / "tracked.txt" + target_path.write_text("original\n", encoding="utf-8") + self._git(repo_path, "add", "tracked.txt") + self._git(repo_path, "commit", "-m", "init") + readme_path = repo_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf changed > tracked.txt + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(readme_path)], + cwd=str(repo_path), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(target_path.read_text(encoding="utf-8"), "original\n") + + self.assertEqual(completed.returncode, 0) + self.assertIn("RESET (restored: 1, removed: 0)", completed.stdout) + + def test_main_skip_reset_preserves_lab_changed_files(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = Path(temp_dir) + self._init_git_repo(repo_path) + target_path = repo_path / "tracked.txt" + target_path.write_text("original\n", encoding="utf-8") + self._git(repo_path, "add", "tracked.txt") + self._git(repo_path, "commit", "-m", "init") + readme_path = repo_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf changed > tracked.txt + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + [ + "python3", + str(ROOT_DIR / "validate_readme_commands.py"), + "--skip-reset", + str(readme_path), + ], + cwd=str(repo_path), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(target_path.read_text(encoding="utf-8"), "changed") + + self.assertEqual(completed.returncode, 0) + self.assertNotIn("RESET", completed.stdout) + +if __name__ == "__main__": + unittest.main() diff --git a/validate_readme_commands.py b/validate_readme_commands.py new file mode 100755 index 0000000..d2708eb --- /dev/null +++ b/validate_readme_commands.py @@ -0,0 +1,701 @@ +#!/usr/bin/env python3 +"""Validate shell commands and expected terminal output in a README.""" + +from __future__ import annotations + +import argparse +import queue +import shutil +import subprocess +import sys +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + + +@dataclass(frozen=True) +class ValidationRunSummary: + results: list["CommandResult"] + failure_message: str | None = None + failed_index: int | None = None + + +@dataclass(frozen=True) +class CommandSpec: + command: str + expected_outputs: list[str] + + +@dataclass(frozen=True) +class CommandResult: + index: int + command: str + expected_outputs: list[str] + cwd: Path + skipped: bool + returncode: int + stdout: str + stderr: str + + @property + def combined_output(self) -> str: + return f"{self.stdout}{self.stderr}" + + +@dataclass(frozen=True) +class FencedBlock: + language: str + content: str + + +@dataclass(frozen=True) +class RepoSnapshot: + repo_root: Path + scope: Path + tracked_dirty: set[Path] + untracked: set[Path] + + +@dataclass(frozen=True) +class ResetSummary: + restored: list[Path] + removed: list[Path] + + +class ReadmeValidationError(Exception): + """Base error for README command validation.""" + + +class ReadmeParseError(ReadmeValidationError): + """Raised when the README structure is invalid.""" + + +class CommandExecutionError(ReadmeValidationError): + """Raised when a command times out or output validation fails.""" + + +class CommandValidationFailure(CommandExecutionError): + """Raised when a command output does not satisfy expectations.""" + + +class ValidationStopped(CommandExecutionError): + """Raised when validation stops early and a summary should still be printed.""" + + def __init__(self, summary: ValidationRunSummary) -> None: + super().__init__(summary.failure_message or "Validation stopped.") + self.summary = summary + + +class GitInteractionError(ReadmeValidationError): + """Raised when git state cannot be inspected or restored.""" + + +def should_skip_command(command: str) -> bool: + normalized_command = command.lower() + return "docker" in normalized_command and "studio" in normalized_command + + +def parse_readme_commands(readme_path: Path) -> list[CommandSpec]: + lines = readme_path.read_text(encoding="utf-8").splitlines(keepends=True) + blocks = _parse_fenced_blocks(lines) + + commands: list[CommandSpec] = [] + for block_index, block in enumerate(blocks): + if block.language != "shell": + continue + + expected_outputs: list[str] = [] + next_shell_index = len(blocks) + for later_index in range(block_index + 1, len(blocks)): + if blocks[later_index].language == "shell": + next_shell_index = later_index + break + + for later_block in blocks[block_index + 1 : next_shell_index]: + if later_block.language == "terminaloutput": + expected_outputs.append(later_block.content) + + commands.append( + CommandSpec( + command=block.content, + expected_outputs=expected_outputs, + ) + ) + + return commands + + +def _parse_fenced_blocks(lines: Sequence[str]) -> list[FencedBlock]: + blocks: list[FencedBlock] = [] + current_language: str | None = None + current_lines: list[str] = [] + + for line in lines: + stripped = line.strip() + if current_language is None: + if stripped.startswith("```"): + current_language = stripped[3:].strip() + current_lines = [] + continue + + if stripped == "```": + blocks.append(FencedBlock(language=current_language, content="".join(current_lines))) + current_language = None + current_lines = [] + continue + + current_lines.append(line) + + if current_language is not None: + raise ReadmeParseError("Unterminated fenced code block in README.") + + return blocks + + +def run_command_specs( + command_specs: Sequence[CommandSpec], + cwd: Path, + timeout_seconds: float, +) -> ValidationRunSummary: + results: list[CommandResult] = [] + + for index, command_spec in enumerate(command_specs, start=1): + if should_skip_command(command_spec.command): + results.append(_build_result(index=index, command_spec=command_spec, cwd=cwd, skipped=True)) + continue + + try: + completed = _run_command( + command=command_spec.command, + cwd=cwd, + timeout_seconds=timeout_seconds, + ) + except subprocess.TimeoutExpired as exc: + results.append( + _build_result( + index=index, + command_spec=command_spec, + cwd=cwd, + returncode=-1, + stdout=exc.stdout or "", + stderr=exc.stderr or "", + ) + ) + raise ValidationStopped( + ValidationRunSummary( + results=results, + failure_message=_format_failure_message( + index=index, + command=command_spec.command, + expected_outputs=command_spec.expected_outputs, + cwd=cwd, + returncode=None, + reason=f"timed out after {timeout_seconds} seconds", + ), + failed_index=index, + ) + ) from exc + + result = _build_result( + index=index, + command_spec=command_spec, + cwd=cwd, + returncode=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + ) + + try: + _assert_command_result(result) + except CommandValidationFailure as exc: + results.append(result) + print( + f"Validation failed at command #{index}. " + "Skipping remaining commands and running cleanup commands...", + file=sys.stderr, + flush=True, + ) + cleanup_results = run_cleanup_commands( + command_specs=command_specs[index:], + cwd=cwd, + timeout_seconds=timeout_seconds, + ) + if cleanup_results: + raise ValidationStopped( + ValidationRunSummary( + results=results, + failure_message=( + f"{exc}\nCleanup commands executed:\n" + f"{_format_cleanup_results(cleanup_results)}" + ), + failed_index=index, + ) + ) from exc + raise ValidationStopped( + ValidationRunSummary( + results=results, + failure_message=str(exc), + failed_index=index, + ) + ) from exc + + results.append(result) + + return ValidationRunSummary(results=results) + + +def _assert_command_result(result: CommandResult) -> None: + if result.expected_outputs: + for output_index, expected_output in enumerate(result.expected_outputs, start=1): + if not _expected_output_matches(expected_output, result.combined_output): + raise CommandValidationFailure( + _format_failure_message( + index=result.index, + command=result.command, + expected_outputs=result.expected_outputs, + cwd=result.cwd, + returncode=result.returncode, + reason=f"missing expected terminaloutput block #{output_index}", + ) + ) + return + + if result.returncode != 0: + raise CommandValidationFailure( + _format_failure_message( + index=result.index, + command=result.command, + expected_outputs=result.expected_outputs, + cwd=result.cwd, + returncode=result.returncode, + reason="command exited non-zero without any expected terminaloutput blocks", + ) + ) + + +def _expected_output_matches(expected_output: str, actual_output: str) -> bool: + expected_lines = [line for line in expected_output.splitlines() if line.strip()] + actual_lines = actual_output.splitlines() + + if not expected_lines: + return True + + actual_index = 0 + for expected_line in expected_lines: + while actual_index < len(actual_lines): + if expected_line in actual_lines[actual_index]: + actual_index += 1 + break + actual_index += 1 + else: + return False + + return True + + +def _run_command( + *, + command: str, + cwd: Path, + timeout_seconds: float, +) -> subprocess.CompletedProcess[str]: + process = subprocess.Popen( + command, + shell=True, + cwd=str(cwd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + assert process.stdout is not None + assert process.stderr is not None + + stdout_chunks: list[str] = [] + stderr_chunks: list[str] = [] + output_queue: queue.Queue[tuple[str, str | None]] = queue.Queue() + + def _reader(stream_name: str, stream, chunks: list[str]) -> None: + try: + for line in iter(stream.readline, ""): + chunks.append(line) + output_queue.put((stream_name, line)) + finally: + stream.close() + output_queue.put((stream_name, None)) + + stdout_thread = threading.Thread( + target=_reader, + args=("stdout", process.stdout, stdout_chunks), + daemon=True, + ) + stderr_thread = threading.Thread( + target=_reader, + args=("stderr", process.stderr, stderr_chunks), + daemon=True, + ) + stdout_thread.start() + stderr_thread.start() + + finished_streams = 0 + timed_out = False + deadline = time.monotonic() + timeout_seconds + + try: + while finished_streams < 2: + if process.poll() is None and time.monotonic() > deadline: + timed_out = True + process.kill() + process.wait() + raise subprocess.TimeoutExpired( + cmd=command, + timeout=timeout_seconds, + output="".join(stdout_chunks), + stderr="".join(stderr_chunks), + ) + + try: + stream_name, chunk = output_queue.get(timeout=0.1) + except queue.Empty: + if process.poll() is not None: + continue + continue + + if chunk is None: + finished_streams += 1 + continue + + target_stream = sys.stdout if stream_name == "stdout" else sys.stderr + print(chunk, end="", file=target_stream, flush=True) + except Exception: + process.kill() + process.wait() + raise + + try: + returncode = process.wait(timeout=0) + finally: + stdout_thread.join(timeout=1) + stderr_thread.join(timeout=1) + if not timed_out and process.poll() is None: + process.kill() + process.wait() + + return subprocess.CompletedProcess( + args=command, + returncode=returncode, + stdout="".join(stdout_chunks), + stderr="".join(stderr_chunks), + ) + + +def run_cleanup_commands( + command_specs: Sequence[CommandSpec], + cwd: Path, + timeout_seconds: float, +) -> list[CommandResult]: + cleanup_results: list[CommandResult] = [] + + for cleanup_index, command_spec in enumerate(command_specs, start=1): + if command_spec.expected_outputs: + break + if not _is_cleanup_command(command_spec.command): + continue + + try: + completed = _run_command( + command=command_spec.command, + cwd=cwd, + timeout_seconds=timeout_seconds, + ) + except subprocess.TimeoutExpired: + continue + cleanup_results.append( + _build_result( + index=cleanup_index, + command_spec=command_spec, + cwd=cwd, + returncode=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + ) + ) + + return cleanup_results + + +def _is_cleanup_command(command: str) -> bool: + normalized_command = " ".join(command.lower().split()) + cleanup_patterns = ( + "docker compose down", + "docker-compose down", + "docker stop", + "docker rm", + "docker container rm", + "kubectl delete", + "pkill ", + "kill ", + ) + return any(pattern in normalized_command for pattern in cleanup_patterns) + + +def _format_cleanup_results(cleanup_results: Sequence[CommandResult]) -> str: + lines: list[str] = [] + for result in cleanup_results: + lines.append(f"- `{result.command.strip()}` exited with {result.returncode}") + return "\n".join(lines) + + +def _format_failure_message( + *, + index: int, + command: str, + expected_outputs: Sequence[str], + cwd: Path | None, + returncode: int | None, + reason: str, +) -> str: + returncode_text = "n/a" if returncode is None else str(returncode) + lines = [ + f"Command #{index} failed: {reason}", + f"Exit code: {returncode_text}", + ] + if cwd is not None: + lines.append(f"Working directory: {cwd}") + lines.extend( + [ + "Command:", + command.rstrip("\n"), + f"Expected terminaloutput blocks: {len(expected_outputs)}", + ] + ) + return "\n".join(lines) + + +def _build_result( + *, + index: int, + command_spec: CommandSpec, + cwd: Path, + skipped: bool = False, + returncode: int = 0, + stdout: str = "", + stderr: str = "", +) -> CommandResult: + return CommandResult( + index=index, + command=command_spec.command, + expected_outputs=command_spec.expected_outputs, + cwd=cwd, + skipped=skipped, + returncode=returncode, + stdout=stdout, + stderr=stderr, + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Validate README shell commands against terminaloutput blocks." + ) + parser.add_argument("readme", help="Path to the README.md file to validate.") + parser.add_argument( + "--dry-run", + action="store_true", + help="Parse the README and print command-to-terminaloutput mappings without running commands.", + ) + parser.add_argument( + "--skip-reset", + action="store_true", + help="Do not reset lab-changed files back to their original git state at the end of the run.", + ) + parser.add_argument( + "--timeout", + type=float, + default=120.0, + help="Timeout in seconds for each shell command. Default: 120.", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + readme_path = Path(args.readme).expanduser().resolve() + if not readme_path.is_file(): + parser.error(f"README file does not exist: {readme_path}") + + repo_snapshot = None if args.skip_reset else snapshot_repo_state(readme_path.parent) + + try: + command_specs = parse_readme_commands(readme_path) + if args.dry_run: + print_dry_run(command_specs) + return 0 + summary = run_command_specs( + command_specs=command_specs, + cwd=readme_path.parent, + timeout_seconds=args.timeout, + ) + exit_code = 0 + except ValidationStopped as exc: + summary = exc.summary + exit_code = 1 + except ReadmeValidationError as exc: + print(f"README: {readme_path}", file=sys.stderr) + print(exc, file=sys.stderr) + return 1 + + print_run_summary(summary, total_commands=len(command_specs)) + + if summary.failure_message is not None: + print(f"README: {readme_path}", file=sys.stderr) + print(summary.failure_message, file=sys.stderr) + + if repo_snapshot is not None: + print_reset_summary(reset_lab_changes(readme_path.parent, repo_snapshot)) + + if exit_code == 0: + print(f"Validated {len(summary.results)} command(s) in {readme_path}") + return exit_code + + +def print_dry_run(command_specs: Sequence[CommandSpec]) -> None: + for index, command_spec in enumerate(command_specs, start=1): + print(f"Command #{index}:") + print(command_spec.command.rstrip("\n")) + print(f"Expected terminaloutput blocks: {len(command_spec.expected_outputs)}") + for output_index, expected_output in enumerate(command_spec.expected_outputs, start=1): + print(f"terminaloutput #{output_index}:") + print(expected_output.rstrip("\n")) + print() + + +def snapshot_repo_state(cwd: Path) -> RepoSnapshot | None: + repo_root = _get_repo_root(cwd) + if repo_root is None: + return None + + scope = cwd.resolve().relative_to(repo_root) + + return RepoSnapshot( + repo_root=repo_root, + scope=scope, + tracked_dirty=_git_path_set( + repo_root, + ["git", "status", "--porcelain=v1", "--untracked-files=no", "-z", "--", str(scope)], + ), + untracked=_git_path_set( + repo_root, + ["git", "ls-files", "-o", "--exclude-standard", "-z", "--", str(scope)], + ), + ) + + +def reset_lab_changes(cwd: Path, baseline: RepoSnapshot | None) -> ResetSummary: + if baseline is None: + return ResetSummary(restored=[], removed=[]) + + current_snapshot = snapshot_repo_state(cwd) + if current_snapshot is None: + return ResetSummary(restored=[], removed=[]) + + paths_to_restore = sorted(current_snapshot.tracked_dirty - baseline.tracked_dirty) + paths_to_remove = sorted(current_snapshot.untracked - baseline.untracked) + + if paths_to_restore: + _run_git_command( + baseline.repo_root, + ["git", "restore", "--worktree", "--source=HEAD", "--", *[str(path) for path in paths_to_restore]], + ) + + removed: list[Path] = [] + for path in paths_to_remove: + absolute_path = baseline.repo_root / path + if not absolute_path.exists(): + continue + if absolute_path.is_dir(): + shutil.rmtree(absolute_path) + else: + absolute_path.unlink() + removed.append(path) + + return ResetSummary(restored=paths_to_restore, removed=removed) + + +def print_reset_summary(reset_summary: ResetSummary) -> None: + if not reset_summary.restored and not reset_summary.removed: + return + + print( + "RESET " + f"(restored: {len(reset_summary.restored)}, removed: {len(reset_summary.removed)})" + ) + + +def print_run_summary(summary: ValidationRunSummary, total_commands: int) -> None: + for result in summary.results: + if result.skipped: + print( + f"SKIP command #{result.index} " + f"(contains both 'docker' and 'studio')" + ) + else: + status = "FAIL" if summary.failed_index == result.index else "PASS" + print( + f"{status} command #{result.index} (exit {result.returncode}, " + f"expected blocks: {len(result.expected_outputs)})" + ) + + for index in range(len(summary.results) + 1, total_commands + 1): + print(f"SKIP command #{index}") + + +def _get_repo_root(cwd: Path) -> Path | None: + completed = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=str(cwd), + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + return None + return Path(completed.stdout.strip()) + + +def _git_path_set(cwd: Path, command: list[str]) -> set[Path]: + completed = _run_git_command(cwd, command) + if command[1:3] == ["status", "--porcelain=v1"]: + return _parse_status_paths(completed.stdout) + return {Path(path) for path in completed.stdout.split("\0") if path} + + +def _parse_status_paths(output: str) -> set[Path]: + paths: set[Path] = set() + for entry in output.split("\0"): + if entry and len(entry) >= 4: + paths.add(Path(entry[3:])) + return paths + + +def _run_git_command(cwd: Path, command: list[str]) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + command, + cwd=str(cwd), + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + stderr = completed.stderr.strip() or "unknown git error" + raise GitInteractionError(f"Git command failed: {' '.join(command)}\n{stderr}") + return completed + + +if __name__ == "__main__": + sys.exit(main()) From 1200480cc35dcb211d24b4b0670945bc1a91af4f Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Wed, 13 May 2026 23:09:48 +0530 Subject: [PATCH 02/71] chore: add PASS/FAIL status at the end of the console output --- tests/test_validate_readme_commands.py | 33 ++++++++++++++++++++++++++ validate_readme_commands.py | 3 +++ 2 files changed, 36 insertions(+) diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index 57b0e6a..68e2b23 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -380,6 +380,7 @@ def test_cli_invocation_reports_failure(self) -> None: self.assertEqual(completed.returncode, 1) self.assertIn("missing expected terminaloutput block #1", completed.stderr) + self.assertTrue(completed.stdout.rstrip().endswith("FAIL")) def test_cli_invocation_prints_failure_summary(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: @@ -414,6 +415,7 @@ def test_cli_invocation_prints_failure_summary(self) -> None: self.assertEqual(completed.returncode, 1) self.assertIn("FAIL command #1", completed.stdout) self.assertIn("SKIP command #2", completed.stdout) + self.assertTrue(completed.stdout.rstrip().endswith("FAIL")) def test_dry_run_does_not_execute_commands(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: @@ -453,6 +455,37 @@ def test_dry_run_does_not_execute_commands(self) -> None: self.assertEqual(completed.returncode, 0) self.assertIn("Command #1:", completed.stdout) + def test_cli_invocation_prints_pass_at_end(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = Path(temp_dir) + self._init_git_repo(repo_path) + readme_path = repo_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf ok + ``` + + ```terminaloutput + ok + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(readme_path)], + cwd=str(repo_path), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(completed.returncode, 0) + self.assertTrue(completed.stdout.rstrip().endswith("PASS")) + def test_main_resets_lab_changed_files_by_default(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: repo_path = Path(temp_dir) diff --git a/validate_readme_commands.py b/validate_readme_commands.py index d2708eb..1639ce8 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -561,6 +561,9 @@ def main(argv: Sequence[str] | None = None) -> int: if exit_code == 0: print(f"Validated {len(summary.results)} command(s) in {readme_path}") + print("PASS") + else: + print("FAIL") return exit_code From e934320829b51d0d6baa5ca0a4fd2bcca4a1a19c Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Thu, 14 May 2026 07:51:44 +0530 Subject: [PATCH 03/71] Improve README validator output and reset behavior --- tests/test_validate_readme_commands.py | 28 ++++++++------ validate_readme_commands.py | 53 ++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index 68e2b23..fce1951 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -12,7 +12,7 @@ _expected_output_matches, main, parse_readme_commands, - print_dry_run, + print_command_mapping, reset_lab_changes, run_cleanup_commands, run_command_specs, @@ -339,18 +339,20 @@ def test_dry_run_prints_command_mapping(self) -> None: ] with redirect_stdout(stdout_buffer): - print_dry_run(command_specs) + print_command_mapping(command_specs) output = stdout_buffer.getvalue() - self.assertIn("Command #1:", output) - self.assertIn("echo one", output) - self.assertIn("Expected terminaloutput blocks: 2", output) - self.assertIn("terminaloutput #1:", output) - self.assertIn("one", output) - self.assertIn("terminaloutput #2:", output) - self.assertIn("two", output) - self.assertIn("Command #2:", output) - self.assertIn("Expected terminaloutput blocks: 0", output) + self.assertIn("Command #1", output) + self.assertIn("Shell", output) + self.assertIn("expected terminaloutput blocks: 2", output) + self.assertIn(" echo one", output) + self.assertIn(" echo one\n\nterminaloutput #1", output) + self.assertIn("terminaloutput #1", output) + self.assertIn(" one", output) + self.assertIn("terminaloutput #2", output) + self.assertIn(" two", output) + self.assertIn("Command #2", output) + self.assertIn("terminaloutput none", output) def test_cli_invocation_reports_failure(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: @@ -453,7 +455,7 @@ def test_dry_run_does_not_execute_commands(self) -> None: self.assertFalse(marker_path.exists()) self.assertEqual(completed.returncode, 0) - self.assertIn("Command #1:", completed.stdout) + self.assertIn("Command #1", completed.stdout) def test_cli_invocation_prints_pass_at_end(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: @@ -484,6 +486,8 @@ def test_cli_invocation_prints_pass_at_end(self) -> None: ) self.assertEqual(completed.returncode, 0) + self.assertIn("Command #1", completed.stdout) + self.assertIn("expected terminaloutput blocks: 1", completed.stdout) self.assertTrue(completed.stdout.rstrip().endswith("PASS")) def test_main_resets_lab_changed_files_by_default(self) -> None: diff --git a/validate_readme_commands.py b/validate_readme_commands.py index 1639ce8..751cffa 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -14,6 +14,13 @@ from pathlib import Path from typing import Sequence +ANSI_RESET = "\033[0m" +ANSI_BOLD = "\033[1m" +ANSI_CYAN = "\033[36m" +ANSI_GREEN = "\033[32m" +ANSI_YELLOW = "\033[33m" +ANSI_DIM = "\033[2m" + @dataclass(frozen=True) class ValidationRunSummary: @@ -92,6 +99,16 @@ class GitInteractionError(ReadmeValidationError): """Raised when git state cannot be inspected or restored.""" +def _supports_color() -> bool: + return sys.stdout.isatty() + + +def _style(text: str, *codes: str) -> str: + if not _supports_color() or not codes: + return text + return f"{''.join(codes)}{text}{ANSI_RESET}" + + def should_skip_command(command: str) -> bool: normalized_command = command.lower() return "docker" in normalized_command and "studio" in normalized_command @@ -534,8 +551,9 @@ def main(argv: Sequence[str] | None = None) -> int: try: command_specs = parse_readme_commands(readme_path) if args.dry_run: - print_dry_run(command_specs) + print_command_mapping(command_specs) return 0 + print_command_mapping(command_specs) summary = run_command_specs( command_specs=command_specs, cwd=readme_path.parent, @@ -567,15 +585,36 @@ def main(argv: Sequence[str] | None = None) -> int: return exit_code -def print_dry_run(command_specs: Sequence[CommandSpec]) -> None: +def print_command_mapping(command_specs: Sequence[CommandSpec]) -> None: + separator = _style("=" * 72, ANSI_DIM) + for index, command_spec in enumerate(command_specs, start=1): - print(f"Command #{index}:") - print(command_spec.command.rstrip("\n")) - print(f"Expected terminaloutput blocks: {len(command_spec.expected_outputs)}") + print(separator) + print(_style(f"Command #{index}", ANSI_BOLD, ANSI_CYAN)) + print( + _style("Shell", ANSI_BOLD, ANSI_YELLOW) + + f" {_style(f'(expected terminaloutput blocks: {len(command_spec.expected_outputs)})', ANSI_DIM)}" + ) + _print_indented_block(command_spec.command) for output_index, expected_output in enumerate(command_spec.expected_outputs, start=1): - print(f"terminaloutput #{output_index}:") - print(expected_output.rstrip("\n")) + print() + print(_style(f"terminaloutput #{output_index}", ANSI_BOLD, ANSI_GREEN)) + _print_indented_block(expected_output) + if not command_spec.expected_outputs: + print() + print(_style("terminaloutput none", ANSI_DIM)) print() + if command_specs: + print(separator) + + +def print_dry_run(command_specs: Sequence[CommandSpec]) -> None: + print_command_mapping(command_specs) + + +def _print_indented_block(content: str) -> None: + for line in content.rstrip("\n").splitlines(): + print(f" {line}") def snapshot_repo_state(cwd: Path) -> RepoSnapshot | None: From cf668115feb296e2ebede38db17e617ecb47fe69 Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Thu, 14 May 2026 09:03:53 +0530 Subject: [PATCH 04/71] feat: run all labs sequentially. add alternative commands --- api-coverage/README.md | 2 +- api-security-schemes/README.md | 6 ++ continuous-integration/README.md | 6 ++ data-adapters/README.md | 6 ++ dictionary/README.md | 7 ++ kafka-sqs-retry-dlq/README.md | 6 ++ mcp-auto-test/README.md | 6 ++ quick-start-api-testing/README.md | 12 +++ quick-start-async-contract-testing/README.md | 6 ++ quick-start-contract-testing/README.md | 6 ++ schema-resiliency-testing/README.md | 12 +++ tests/test_validate_readme_commands.py | 45 ++++++++++ validate_readme_commands.py | 86 ++++++++++++++++++-- workflow-in-same-spec/README.md | 6 ++ 14 files changed, 204 insertions(+), 8 deletions(-) diff --git a/api-coverage/README.md b/api-coverage/README.md index cb2b988..83bde80 100644 --- a/api-coverage/README.md +++ b/api-coverage/README.md @@ -148,7 +148,7 @@ Change it to: Do not change anything else in the operation. -Alternatively, just run the following command +Alternatively, just run the following command: ```shell docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's|/pets/search:$|/pets/find:|' specs/service.yaml" diff --git a/api-security-schemes/README.md b/api-security-schemes/README.md index 6a0e2d1..ed54875 100644 --- a/api-security-schemes/README.md +++ b/api-security-schemes/README.md @@ -132,6 +132,12 @@ Update [`specmatic.yaml`](specmatic.yaml) to valid values again: Do not change anything else. Fix only the above values. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#INVALID_OAUTH_TOKEN#OAUTH_TOKEN#g; s#dXNlcjppbnZhbGlkcGFzcw==#dXNlcjpwYXNzd29yZA==#g; s#INVALID_APIKEY1234#APIKEY1234#g' specmatic.yaml" +``` + ## Verify the fix Re-run: diff --git a/continuous-integration/README.md b/continuous-integration/README.md index 0525880..12966c0 100644 --- a/continuous-integration/README.md +++ b/continuous-integration/README.md @@ -139,6 +139,12 @@ Keep: Do not change anything else. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "awk 'BEGIN{done=0} !done && \$0==\" - priority\" {done=1; next} {print}' contracts/order_api.yaml > /tmp/order_api.yaml && mv /tmp/order_api.yaml contracts/order_api.yaml" +``` + ## Part C: Re-run the CI simulation Run the same command again: diff --git a/data-adapters/README.md b/data-adapters/README.md index 332e537..5f7ddae 100644 --- a/data-adapters/README.md +++ b/data-adapters/README.md @@ -94,6 +94,12 @@ Why both hooks are needed: - `pre_specmatic_request_processor`: adapts PascalCase request fields to camelCase before sending to mock. - `post_specmatic_response_processor`: adapts camelCase mock response fields back to PascalCase before assertion. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n adapters:\n pre_specmatic_request_processor: ./hooks/pre_specmatic_request_processor.sh\n post_specmatic_response_processor: ./hooks/post_specmatic_response_processor.sh\n' specmatic.yaml && chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh" +``` + ## 5. Ensure hook scripts are executable Run: diff --git a/dictionary/README.md b/dictionary/README.md index 86bd0fd..51b596c 100644 --- a/dictionary/README.md +++ b/dictionary/README.md @@ -59,6 +59,13 @@ data: path: specs/dictionary.yaml ``` +Alternatively, just run the following commands: + +```shell +docker run --rm -v "$PWD:/usr/src/app" specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n dictionary:\n path: specs/dictionary.yaml\n' specmatic.yaml" +``` + ## 3. Re-run the suite after configuring dictionary ```shell diff --git a/kafka-sqs-retry-dlq/README.md b/kafka-sqs-retry-dlq/README.md index 10db3cb..7e09abe 100644 --- a/kafka-sqs-retry-dlq/README.md +++ b/kafka-sqs-retry-dlq/README.md @@ -98,6 +98,12 @@ What to look for: Do not change the contract, examples, or Compose wiring. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's/# threading.Thread/threading.Thread/' service/app.py" +``` + ## Pass criteria Re-run: diff --git a/mcp-auto-test/README.md b/mcp-auto-test/README.md index 6482440..55c032c 100644 --- a/mcp-auto-test/README.md +++ b/mcp-auto-test/README.md @@ -98,6 +98,12 @@ Make these two fixes: Do not change anything else. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"damage\": 0.0#\"damaged\": 0.0#; s#order\\[\"shipment\"\\]#order[\"shipmentStatus\"]#' service/order_service.py" +``` + ## Part C: Re-run tests (expected to pass) Run: diff --git a/quick-start-api-testing/README.md b/quick-start-api-testing/README.md index 6694d33..68d234f 100644 --- a/quick-start-api-testing/README.md +++ b/quick-start-api-testing/README.md @@ -99,6 +99,12 @@ In `http-response.body`, change: Do not change any other fields. +Alternatively, just run the following command for Task A: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#[$]match(exact: approved)#\$match(pattern: approved\|verified)#' examples/test_finance_user_11.json" +``` + Re-run: ```shell @@ -128,6 +134,12 @@ In `http-response.body`, change: Keep `handledBy` and `decision` as exact matches. +Alternatively, just run the following command for Final Phase: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#[$]match(exact: VRF-123456)#\$match(pattern: VRF-[0-9]{6})#; s#[$]match(exact: 2026-03-17)#\$match(dataType: date)#' examples/test_support_user_55.json" +``` + Run: ```shell diff --git a/quick-start-async-contract-testing/README.md b/quick-start-async-contract-testing/README.md index 5950bbb..71c8c5c 100644 --- a/quick-start-async-contract-testing/README.md +++ b/quick-start-async-contract-testing/README.md @@ -73,6 +73,12 @@ To: "status": "INITIATED", ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"status\": \"STARTED\"#\"status\": \"INITIATED\"#' service/processor.py" +``` + ## Pass criteria Re-run: diff --git a/quick-start-contract-testing/README.md b/quick-start-contract-testing/README.md index 90edf76..b6c160f 100644 --- a/quick-start-contract-testing/README.md +++ b/quick-start-contract-testing/README.md @@ -108,6 +108,12 @@ with: Do not change anything else. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's/"petType"/"type"/g' service/server.py" +``` + ## Part C: Re-run tests (expected to pass) Run: diff --git a/schema-resiliency-testing/README.md b/schema-resiliency-testing/README.md index df1e1b2..a1b2686 100644 --- a/schema-resiliency-testing/README.md +++ b/schema-resiliency-testing/README.md @@ -69,6 +69,12 @@ The goal of this lab is to try different schema resiliency testing levels and se ### Positive Only Tests In `specmatic.yaml` change `schemaResiliencyTests: none` to `schemaResiliencyTests: positiveOnly` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: none#schemaResiliencyTests: positiveOnly#' specmatic.yaml" +``` + #### Run Positive only Tests Start docker containers @@ -92,6 +98,12 @@ docker compose down -v ### Positive and Negative Tests (ALL) In `specmatic.yaml` change `schemaResiliencyTests: positiveOnly` to `schemaResiliencyTests: all` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: positiveOnly#schemaResiliencyTests: all#' specmatic.yaml" +``` + #### Run all Tests Start docker containers diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index fce1951..cb7c3ed 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -5,8 +5,10 @@ import unittest from contextlib import redirect_stderr, redirect_stdout from pathlib import Path +from unittest.mock import patch from validate_readme_commands import ( + DEFAULT_LABS, CommandExecutionError, CommandSpec, _expected_output_matches, @@ -14,6 +16,8 @@ parse_readme_commands, print_command_mapping, reset_lab_changes, + resolve_readme_paths, + run_single_readme, run_cleanup_commands, run_command_specs, snapshot_repo_state, @@ -309,6 +313,13 @@ def test_skip_rule_requires_both_docker_and_studio(self) -> None: class MainTests(GitRepoTestCase): + def test_resolve_readme_paths_uses_default_labs_when_no_arg(self) -> None: + readme_paths = resolve_readme_paths(None) + + self.assertEqual(len(readme_paths), len(DEFAULT_LABS)) + self.assertTrue(str(readme_paths[0]).endswith("/api-coverage/README.md")) + self.assertTrue(str(readme_paths[-1]).endswith("/workflow-in-same-spec/README.md")) + def test_main_returns_zero_for_valid_readme(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: readme_path = Path(temp_dir) / "README.md" @@ -561,5 +572,39 @@ def test_main_skip_reset_preserves_lab_changed_files(self) -> None: self.assertEqual(completed.returncode, 0) self.assertNotIn("RESET", completed.stdout) + def test_main_without_readme_runs_all_default_labs_and_continues_after_failure(self) -> None: + fake_readmes = [ + Path("/tmp/lab-one/README.md"), + Path("/tmp/lab-two/README.md"), + Path("/tmp/lab-three/README.md"), + ] + stdout_buffer = io.StringIO() + calls: list[Path] = [] + + def fake_run_single_readme(*, readme_path: Path, dry_run: bool, skip_reset: bool, timeout_seconds: float) -> int: + calls.append(readme_path) + return 1 if readme_path == fake_readmes[1] else 0 + + with ( + patch("validate_readme_commands.resolve_readme_paths", return_value=fake_readmes), + patch("pathlib.Path.is_file", return_value=True), + patch("validate_readme_commands.run_single_readme", side_effect=fake_run_single_readme), + redirect_stdout(stdout_buffer), + ): + exit_code = main([]) + + self.assertEqual(exit_code, 1) + self.assertEqual(calls, fake_readmes) + output = stdout_buffer.getvalue() + self.assertIn("===== lab-one =====", output) + self.assertIn("===== lab-two =====", output) + self.assertIn("===== lab-three =====", output) + self.assertIn("===== Summary =====", output) + self.assertIn("PASS labs: 2", output) + self.assertIn(" lab-one", output) + self.assertIn(" lab-three", output) + self.assertIn("FAIL labs: 1", output) + self.assertIn(" lab-two", output) + if __name__ == "__main__": unittest.main() diff --git a/validate_readme_commands.py b/validate_readme_commands.py index 751cffa..df15fc5 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -21,6 +21,21 @@ ANSI_YELLOW = "\033[33m" ANSI_DIM = "\033[2m" +DEFAULT_LABS = [ + "api-coverage", + "api-security-schemes", + "continuous-integration", + "data-adapters", + "dictionary", + "kafka-sqs-retry-dlq", + "mcp-auto-test", + "quick-start-api-testing", + "quick-start-async-contract-testing", + "quick-start-contract-testing", + "schema-resiliency-testing", + "workflow-in-same-spec", +] + @dataclass(frozen=True) class ValidationRunSummary: @@ -518,7 +533,11 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Validate README shell commands against terminaloutput blocks." ) - parser.add_argument("readme", help="Path to the README.md file to validate.") + parser.add_argument( + "readme", + nargs="?", + help="Path to the README.md file to validate. If omitted, runs the built-in lab README list.", + ) parser.add_argument( "--dry-run", action="store_true", @@ -542,22 +561,57 @@ def main(argv: Sequence[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) - readme_path = Path(args.readme).expanduser().resolve() - if not readme_path.is_file(): - parser.error(f"README file does not exist: {readme_path}") + readme_paths = resolve_readme_paths(args.readme) + multiple_labs = len(readme_paths) > 1 + for readme_path in readme_paths: + if not readme_path.is_file(): + parser.error(f"README file does not exist: {readme_path}") + + overall_exit_code = 0 + passed_labs: list[str] = [] + failed_labs: list[str] = [] + for index, readme_path in enumerate(readme_paths, start=1): + if multiple_labs: + if index > 1: + print() + print(f"===== {readme_path.parent.name} =====") + exit_code = run_single_readme( + readme_path=readme_path, + dry_run=args.dry_run, + skip_reset=args.skip_reset, + timeout_seconds=args.timeout, + ) + if exit_code != 0: + overall_exit_code = exit_code + failed_labs.append(readme_path.parent.name) + else: + passed_labs.append(readme_path.parent.name) + + if multiple_labs: + print() + print_multi_lab_summary(passed_labs, failed_labs) + + return overall_exit_code - repo_snapshot = None if args.skip_reset else snapshot_repo_state(readme_path.parent) +def run_single_readme( + *, + readme_path: Path, + dry_run: bool, + skip_reset: bool, + timeout_seconds: float, +) -> int: + repo_snapshot = None if skip_reset else snapshot_repo_state(readme_path.parent) try: command_specs = parse_readme_commands(readme_path) - if args.dry_run: + if dry_run: print_command_mapping(command_specs) return 0 print_command_mapping(command_specs) summary = run_command_specs( command_specs=command_specs, cwd=readme_path.parent, - timeout_seconds=args.timeout, + timeout_seconds=timeout_seconds, ) exit_code = 0 except ValidationStopped as exc: @@ -585,6 +639,14 @@ def main(argv: Sequence[str] | None = None) -> int: return exit_code +def resolve_readme_paths(readme_arg: str | None) -> list[Path]: + if readme_arg: + return [Path(readme_arg).expanduser().resolve()] + + repo_root = Path(__file__).resolve().parent + return [(repo_root / lab / "README.md").resolve() for lab in DEFAULT_LABS] + + def print_command_mapping(command_specs: Sequence[CommandSpec]) -> None: separator = _style("=" * 72, ANSI_DIM) @@ -697,6 +759,16 @@ def print_run_summary(summary: ValidationRunSummary, total_commands: int) -> Non print(f"SKIP command #{index}") +def print_multi_lab_summary(passed_labs: Sequence[str], failed_labs: Sequence[str]) -> None: + print("===== Summary =====") + print(f"PASS labs: {len(passed_labs)}") + for lab in passed_labs: + print(f" {lab}") + print(f"FAIL labs: {len(failed_labs)}") + for lab in failed_labs: + print(f" {lab}") + + def _get_repo_root(cwd: Path) -> Path | None: completed = subprocess.run( ["git", "rev-parse", "--show-toplevel"], diff --git a/workflow-in-same-spec/README.md b/workflow-in-same-spec/README.md index 2f47f9f..f2f5291 100644 --- a/workflow-in-same-spec/README.md +++ b/workflow-in-same-spec/README.md @@ -105,6 +105,12 @@ workflow: use: "PATH.task_id" ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ workflow:\n ids:\n \"POST /tasks -> 200\":\n extract: \"BODY.tasks.[0].id\"\n \"GET /tasks/(task_id:string) -> 200\":\n use: \"PATH.task_id\"\n \"PUT /tasks/(task_id:string) -> 200\":\n use: \"PATH.task_id\"\n \"DELETE /tasks/(task_id:string) -> 204\":\n use: \"PATH.task_id\"\n' specmatic.yaml" +``` + Re-run: ```shell From 3e02dc8f43db1747441051cfc1281b41e70f2090 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 10:43:48 +0530 Subject: [PATCH 05/71] Fixed README command --- continuous-integration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-integration/README.md b/continuous-integration/README.md index 12966c0..4de5f04 100644 --- a/continuous-integration/README.md +++ b/continuous-integration/README.md @@ -142,7 +142,7 @@ Do not change anything else. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "awk 'BEGIN{done=0} !done && \$0==\" - priority\" {done=1; next} {print}' contracts/order_api.yaml > /tmp/order_api.yaml && mv /tmp/order_api.yaml contracts/order_api.yaml" +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "awk 'BEGIN{removed=0} {if (removed==0 && \$0==\" - priority\") {removed=1; next} print}' contracts/order_api.yaml > /tmp/order_api.yaml && mv /tmp/order_api.yaml contracts/order_api.yaml" ``` ## Part C: Re-run the CI simulation From 11b1ab425d709566131ad346d0a4549ab600f0c6 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 10:55:18 +0530 Subject: [PATCH 06/71] updated command in dictionary and added absolute path of /specs/dictionary.yaml in terminaloutput of command - relative path not present in console log --- dictionary/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dictionary/README.md b/dictionary/README.md index 51b596c..61da81b 100644 --- a/dictionary/README.md +++ b/dictionary/README.md @@ -42,11 +42,11 @@ docker compose down -v Generate dictionary data from existing examples: ```shell -docker run --rm -v .:/usr/src/app specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml +docker run --rm -v "$PWD:/usr/src/app" specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml ``` ```terminaloutput -Generated dictionary file at specs/dictionary.yaml +Dictionary file written to /usr/src/app/specs/dictionary.yaml ``` Open and understand the [generated dictionary file](specs/dictionary.yaml) From 3cb008c4c4ff12c93a84f7786e1ff14c022ed0e7 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 12:21:08 +0530 Subject: [PATCH 07/71] Updated README message in failure for continuous integration - previous message was incorrect --- continuous-integration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-integration/README.md b/continuous-integration/README.md index 4de5f04..bdd4b63 100644 --- a/continuous-integration/README.md +++ b/continuous-integration/README.md @@ -99,7 +99,7 @@ Expected behavior: Expected failure highlight: ```terminaloutput -(INCOMPATIBLE) This spec contains breaking changes to the API +(INCOMPATIBLE) The changes to the spec are NOT backward compatible with the corresponding spec from main ``` Why this fails: From f687c815e1184d84a2c4b00465e370a6aae3ccd1 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 12:32:38 +0530 Subject: [PATCH 08/71] Running docker compose up in detached state, to avoid timeout error during validation --- data-adapters/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-adapters/README.md b/data-adapters/README.md index 5f7ddae..c237e54 100644 --- a/data-adapters/README.md +++ b/data-adapters/README.md @@ -42,7 +42,7 @@ This lab will help you understand how to identify such mismatches and use Specma Run: ```shell -docker compose up +docker compose up -d ``` ## 2. Trigger the mismatch from browser (intentional failure) From 437debc443ee1d9f25ac15813def2de707784217 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 12:33:19 +0530 Subject: [PATCH 09/71] Running docker compose up in detached state, to avoid timeout error during validation --- data-adapters/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-adapters/README.md b/data-adapters/README.md index c237e54..d4d04d3 100644 --- a/data-adapters/README.md +++ b/data-adapters/README.md @@ -111,7 +111,7 @@ chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_ Run: ```shell -docker compose up +docker compose up -d ``` ## 7. Trigger the matching request/response from browser From e877282922878a68111976d267f53bae7b7cd8ef Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 12:42:11 +0530 Subject: [PATCH 10/71] Moved alternative command to location after chmod command - post action, fenced windows command with --- data-adapters/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/data-adapters/README.md b/data-adapters/README.md index d4d04d3..eb9804b 100644 --- a/data-adapters/README.md +++ b/data-adapters/README.md @@ -94,17 +94,17 @@ Why both hooks are needed: - `pre_specmatic_request_processor`: adapts PascalCase request fields to camelCase before sending to mock. - `post_specmatic_response_processor`: adapts camelCase mock response fields back to PascalCase before assertion. -Alternatively, just run the following command: +## 5. Ensure hook scripts are executable +Run: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n adapters:\n pre_specmatic_request_processor: ./hooks/pre_specmatic_request_processor.sh\n post_specmatic_response_processor: ./hooks/post_specmatic_response_processor.sh\n' specmatic.yaml && chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh" +chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh ``` -## 5. Ensure hook scripts are executable -Run: +Alternatively, just run the following command: ```shell -chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n adapters:\n pre_specmatic_request_processor: ./hooks/pre_specmatic_request_processor.sh\n post_specmatic_response_processor: ./hooks/post_specmatic_response_processor.sh\n' specmatic.yaml && chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh" ``` ## 6. Restart mock + UI @@ -150,7 +150,7 @@ git update-index --chmod=+x hooks/pre_specmatic_request_processor.sh hooks/post_ - Ensure hook files use LF line endings (not CRLF). In Git Bash: -```shell +```powershell sed -i 's/\r$//' hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh ``` From b7691ae516f1deedfadaca8d9215b6d5bab28d46 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 12:57:24 +0530 Subject: [PATCH 11/71] Updated Actual/Expected messages table to newer format - failed previously --- kafka-sqs-retry-dlq/README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/kafka-sqs-retry-dlq/README.md b/kafka-sqs-retry-dlq/README.md index 7e09abe..e9bc17a 100644 --- a/kafka-sqs-retry-dlq/README.md +++ b/kafka-sqs-retry-dlq/README.md @@ -121,11 +121,16 @@ Tests run: 6, Successes: 6, Failures: 0, Errors: 0 The message count report should show: ```terminaloutput -| Topic/queue name | Actual | Expected | -| place-order-topic | 6 | 6 | -| place-order-queue | 4 | 4 | -| place-order-retry-topic | 2 | 2 | -| place-order-dlq-topic | 2 | 2 | ++-------------------------+-------------------------+----------+ +| Topic | No of messages received | ++-------------------------+-------------------------+----------+ +| | Actual | Expected | ++-------------------------+-------------------------+----------+ +| place-order-topic | 6 | 6 | +| place-order-queue | 4 | 4 | +| place-order-dlq-topic | 2 | 2 | +| place-order-retry-topic | 2 | 2 | ++-------------------------+-------------------------+----------+ ``` Clean up: From 9ac1f6224207e30149ad8c2e9645d0e05aa835af Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Thu, 14 May 2026 13:26:35 +0530 Subject: [PATCH 12/71] remove transient info from terminal output --- quick-start-contract-testing/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/quick-start-contract-testing/README.md b/quick-start-contract-testing/README.md index b6c160f..1d32e3e 100644 --- a/quick-start-contract-testing/README.md +++ b/quick-start-contract-testing/README.md @@ -47,7 +47,7 @@ docker compose up test --build --abort-on-container-exit ``` ```terminaloutput -Request to http://petstore:8080 at +Request to http://petstore:8080 at GET /pets/1 Specmatic-Response-Code: 200 Host: petstore:8080 @@ -55,10 +55,10 @@ Request to http://petstore:8080 at Accept: */* Content-Type: NOT SENT -Response at +Response at 200 OK - Server: BaseHTTP/0.6 Python/3.14.3 - Date: + Server: BaseHTTP/0.6 Python/ + Date: Content-Type: application/json Content-Length: 79 From 8a0ef755d6bce9cb9a27c5b2cd76ffc6292e73f5 Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Thu, 14 May 2026 14:00:20 +0530 Subject: [PATCH 13/71] fix: quick-start-contract-testing readme's terminal output --- api-coverage/README.md | 2 +- quick-start-contract-testing/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api-coverage/README.md b/api-coverage/README.md index 83bde80..1ea4703 100644 --- a/api-coverage/README.md +++ b/api-coverage/README.md @@ -151,7 +151,7 @@ Do not change anything else in the operation. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's|/pets/search:$|/pets/find:|' specs/service.yaml" +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's|/pets/search:|/pets/find:|' specs/service.yaml" ``` ## Final Phase diff --git a/quick-start-contract-testing/README.md b/quick-start-contract-testing/README.md index 1d32e3e..5318d68 100644 --- a/quick-start-contract-testing/README.md +++ b/quick-start-contract-testing/README.md @@ -74,7 +74,7 @@ Scenario: GET /pets/(petid:number) -> 200 with the request from the example 'SCO Expected output: ```terminaloutput -Tests run: 1, Successes: 0, Failures: 1, Errors: 0 +Tests run: 1, Successes: 0, Failures: 1, WIP: 0, Errors: 0 ``` How did Specmatic generate the GET request for /pets with ID `1`? @@ -123,7 +123,7 @@ docker compose up test --build --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 1, Successes: 1, Failures: 0, Errors: 0 +Tests run: 1, Successes: 1, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: From 16122027a76bf8473045d1ae5d14d195d2147475 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 15:52:50 +0530 Subject: [PATCH 14/71] Removed WIP tag from expected output --- quick-start-contract-testing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quick-start-contract-testing/README.md b/quick-start-contract-testing/README.md index 5318d68..10892c3 100644 --- a/quick-start-contract-testing/README.md +++ b/quick-start-contract-testing/README.md @@ -74,7 +74,7 @@ Scenario: GET /pets/(petid:number) -> 200 with the request from the example 'SCO Expected output: ```terminaloutput -Tests run: 1, Successes: 0, Failures: 1, WIP: 0, Errors: 0 +Tests run: 1, Successes: 0, Failures: 1, Errors: 0 ``` How did Specmatic generate the GET request for /pets with ID `1`? From 7ee1180162fe6530101e15bc6eb2ac3d6ba6c4b9 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 16:06:59 +0530 Subject: [PATCH 15/71] Added WIP Tag --- quick-start-contract-testing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quick-start-contract-testing/README.md b/quick-start-contract-testing/README.md index 10892c3..5318d68 100644 --- a/quick-start-contract-testing/README.md +++ b/quick-start-contract-testing/README.md @@ -74,7 +74,7 @@ Scenario: GET /pets/(petid:number) -> 200 with the request from the example 'SCO Expected output: ```terminaloutput -Tests run: 1, Successes: 0, Failures: 1, Errors: 0 +Tests run: 1, Successes: 0, Failures: 1, WIP: 0, Errors: 0 ``` How did Specmatic generate the GET request for /pets with ID `1`? From ee62b39a75760bb80624d9c9e98cdff45c79c842 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 16:21:50 +0530 Subject: [PATCH 16/71] Updated WIP count --- quick-start-api-testing/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/quick-start-api-testing/README.md b/quick-start-api-testing/README.md index 68d234f..9001463 100644 --- a/quick-start-api-testing/README.md +++ b/quick-start-api-testing/README.md @@ -74,7 +74,7 @@ docker compose up api-test --build --abort-on-container-exit * Expected console output ```terminaloutput -Tests run: 4, Successes: 2, Failures: 2, Errors: 0 +Tests run: 4, Successes: 2, Failures: 2, WIP: 0, Errors: 0 ``` Why the baseline fails: @@ -114,7 +114,7 @@ docker compose up api-test --build --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 4, Successes: 3, Failures: 1, Errors: 0 +Tests run: 4, Successes: 3, Failures: 1, WIP: 0, Errors: 0 ``` Clean up: @@ -149,7 +149,7 @@ docker compose up api-test --build --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 4, Successes: 4, Failures: 0, Errors: 0 +Tests run: 4, Successes: 4, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: From c1066d778a9cf92fb8187321e7e8bd18ba5cc10f Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 16:22:40 +0530 Subject: [PATCH 17/71] api-coverage: Updated WIP counts --- api-coverage/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-coverage/README.md b/api-coverage/README.md index 1ea4703..1145e8c 100644 --- a/api-coverage/README.md +++ b/api-coverage/README.md @@ -90,7 +90,7 @@ Expected behavior: Expected result summary: ```terminaloutput -Tests run: 2, Successes: 1, Failures: 1, Errors: 0 +Tests run: 2, Successes: 1, Failures: 1, WIP: 0, Errors: 0 ``` Expected coverage highlight: @@ -164,7 +164,7 @@ docker compose up test --build --abort-on-container-exit Expected final result: ```terminaloutput -Tests run: 2, Successes: 2, Failures: 0, Errors: 0 +Tests run: 2, Successes: 2, Failures: 0, WIP: 0, Errors: 0 ``` Expected coverage outcome: From fc8ee7805af325f0f86de6777c23cfeabe825749 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 16:23:51 +0530 Subject: [PATCH 18/71] api-security-schemes: Updated WIP counts --- api-security-schemes/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-security-schemes/README.md b/api-security-schemes/README.md index ed54875..47598b2 100644 --- a/api-security-schemes/README.md +++ b/api-security-schemes/README.md @@ -107,7 +107,7 @@ docker compose up specmatic-test --abort-on-container-exit Expected failing result: ```terminaloutput -Tests run: 171, Successes: 0, Failures: 171, Errors: 0 +Tests run: 171, Successes: 0, Failures: 171, WIP: 0, Errors: 0 ``` - The compose command exits with a non-zero code. @@ -149,7 +149,7 @@ docker compose up specmatic-test --abort-on-container-exit Expected result: ```terminaloutput -Tests run: 171, Successes: 171, Failures: 0, Errors: 0 +Tests run: 171, Successes: 171, Failures: 0, WIP: 0, Errors: 0 ``` Cleanup after run: From 7f9d4c1e703eab27ad27feaa12ccb8de7b6a4e4a Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 16:25:12 +0530 Subject: [PATCH 19/71] dictionary: Updated WIP counts --- dictionary/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dictionary/README.md b/dictionary/README.md index 61da81b..50dfa07 100644 --- a/dictionary/README.md +++ b/dictionary/README.md @@ -25,7 +25,7 @@ docker compose up suite --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 3, Successes: 0, Failures: 3, Errors: 0 +Tests run: 3, Successes: 0, Failures: 3, WIP: 0, Errors: 0 ``` ### Root cause of the failed tests @@ -75,7 +75,7 @@ docker compose up suite --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 3, Successes: 3, Failures: 0, Errors: 0 +Tests run: 3, Successes: 3, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: @@ -85,8 +85,8 @@ docker compose down -v ``` ## Pass Criteria -- Baseline run fails with `Tests run: 3, Successes: 0, Failures: 3, Errors: 0`. -- After dictionary configuration, run passes with `Tests run: 3, Successes: 3, Failures: 0, Errors: 0`. +- Baseline run fails with `Tests run: 3, Successes: 0, Failures: 3, WIP: 0, Errors: 0`. +- After dictionary configuration, run passes with `Tests run: 3, Successes: 3, Failures: 0, WIP: 0, Errors: 0`. ## Additional Resources - [Specmatic Dictionary Documentation](https://docs.specmatic.io/features/dictionary) From 60189af71beb2cbcd01a95b82b2193316737d794 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 16:27:54 +0530 Subject: [PATCH 20/71] kafka-sqs-retry-dlq : Updated WIP counts --- kafka-sqs-retry-dlq/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka-sqs-retry-dlq/README.md b/kafka-sqs-retry-dlq/README.md index e9bc17a..eecae8b 100644 --- a/kafka-sqs-retry-dlq/README.md +++ b/kafka-sqs-retry-dlq/README.md @@ -67,7 +67,7 @@ Expected failure shape: Preferred failure summary: ```terminaloutput -Tests run: 6, Successes: 4, Failures: 2, Errors: 0 +Tests run: 6, Successes: 4, Failures: 2, WIP: 0, Errors: 0 ``` The failing scenarios should be: @@ -115,7 +115,7 @@ docker compose up contract-test --build --abort-on-container-exit Expected passing output: ```terminaloutput -Tests run: 6, Successes: 6, Failures: 0, Errors: 0 +Tests run: 6, Successes: 6, Failures: 0, WIP: 0, Errors: 0 ``` The message count report should show: From f23878e946882a70b940351f26b3d2c47ab8ec53 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 16:28:25 +0530 Subject: [PATCH 21/71] workflow-in-same-spec : Updated WIP counts --- workflow-in-same-spec/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow-in-same-spec/README.md b/workflow-in-same-spec/README.md index f2f5291..3a6ef59 100644 --- a/workflow-in-same-spec/README.md +++ b/workflow-in-same-spec/README.md @@ -50,7 +50,7 @@ docker compose --profile test up test --build --abort-on-container-exit Expected baseline result: ```terminaloutput -Tests run: 4, Successes: 1, Failures: 3, Errors: 0 +Tests run: 4, Successes: 1, Failures: 3, WIP: 0, Errors: 0 ``` You should see failures for: @@ -120,7 +120,7 @@ docker compose --profile test up test --build --abort-on-container-exit Expected passing result: ```terminaloutput -Tests run: 4, Successes: 4, Failures: 0, Errors: 0 +Tests run: 4, Successes: 4, Failures: 0, WIP: 0, Errors: 0 ``` Cleanup: From 74b3a02035eec4890c1407bbb68c99ad1b7f2020 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 17:27:49 +0530 Subject: [PATCH 22/71] api-resiliency-testing: Added commands + WIP counts --- api-resiliency-testing/README.md | 35 +++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/api-resiliency-testing/README.md b/api-resiliency-testing/README.md index cd45b07..7bf61ec 100644 --- a/api-resiliency-testing/README.md +++ b/api-resiliency-testing/README.md @@ -76,7 +76,7 @@ docker compose --profile test up --abort-on-container-exit Expected baseline result: ```terminaloutput -Tests run: 5, Successes: 3, Failures: 2, Errors: 0 +Tests run: 5, Successes: 3, Failures: 2, WIP: 0, Errors: 0 ``` The two failing scenarios should be: @@ -108,6 +108,12 @@ Keep: - query `to-date` as `2025-11-15` - the `200 OK` downstream response body +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i -e '/^ }$/s// },/' -e '/^}$/i\ \"transient\": true,' -e '/^}$/i\ \"delay-in-seconds\": 2' examples/order-service/stub_timeout_get_products.json" +``` + Re-run: ```shell @@ -117,7 +123,7 @@ docker compose --profile test up --abort-on-container-exit Expected checkpoint result: ```terminaloutput -Tests run: 5, Successes: 4, Failures: 1, Errors: 0 +Tests run: 5, Successes: 4, Failures: 1, WIP: 0, Errors: 0 ``` At this point: @@ -144,6 +150,12 @@ Keep: - the existing header matchers - the downstream `201 Created` response +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i -e '/^ }$/s// },/' -e '/^}$/i\ \"transient\": true,' -e '/^}$/i\ \"delay-in-seconds\": 2' examples/order-service/stub_timeout_post_product.json" +``` + Re-run: ```shell @@ -153,7 +165,7 @@ docker compose --profile test up --abort-on-container-exit Expected checkpoint result: ```terminaloutput -Tests run: 5, Successes: 5, Failures: 0, Errors: 0 +Tests run: 5, Successes: 5, Failures: 0, WIP: 0, Errors: 0 ``` At this point: @@ -179,6 +191,12 @@ to: schemaResiliencyTests: all ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's/schemaResiliencyTests: none/schemaResiliencyTests: all/' specmatic.yaml" +``` + Re-run: ```shell @@ -202,7 +220,7 @@ Expected failure direction: Expected Task C checkpoint result before the matcher fix: ```terminaloutput -Tests run: 249, Successes: 238, Failures: 11, Errors: 0 +Tests run: 249, Successes: 238, Failures: 11, WIP: 0, Errors: 0 ``` Why this is useful: @@ -237,6 +255,7 @@ To fix this, update `examples/order-service/stub_timeout_post_product.json` so t } ``` + Why `value:each` is the right matcher here: - `value:each` tracks matcher exhaustion separately for each distinct value - that means each valid `type` value gets its own one-time transient timeout match @@ -247,6 +266,12 @@ Why `value:each` is the right matcher here: Specmatic documentation for this matcher behavior: - [Matchers: step-by-step example for `value: each`](https://docs.specmatic.io/features/matchers#step-by-step-example-for-value-each) +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"type\": \"book\"#\"type\": \"\$match(dataType:ProductType, value:each, times:1)\"#; s#\"inventory\": 9#\"inventory\": \"\$match(dataType:ProductInventory, value:each, times:1)\"#' examples/order-service/stub_timeout_post_product.json" +``` + Keep: - `"transient": true` - `"delay-in-seconds": 2` @@ -266,7 +291,7 @@ Expected outcome: Final expected result: ```terminaloutput -Tests run: 249, Successes: 249, Failures: 0, Errors: 0 +Tests run: 249, Successes: 249, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: From d78f98d229e8d0bd5addba64b218d472a524301a Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 17:44:58 +0530 Subject: [PATCH 23/71] schema-design : Added commands + added file for replacement data (payments-request.yaml) + WIP counts --- schema-design/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/schema-design/README.md b/schema-design/README.md index 7e06b76..b51d561 100644 --- a/schema-design/README.md +++ b/schema-design/README.md @@ -45,7 +45,7 @@ docker compose up contract-test --build --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 4, Successes: 2, Failures: 2, Errors: 0 +Tests run: 4, Successes: 2, Failures: 2, WIP: 0, Errors: 0 ``` Failure reason points to `POST /payments` where requests like `{ "paymentType": "card" }` and `{ "paymentType": "bank_transfer" }` are contract-valid but service returns `400` instead of expected `201`. @@ -120,6 +120,12 @@ Open `specs/payment-api.yaml` and update `PaymentRequest` to this shape: bank_transfer: "#/components/schemas/BankTransferPaymentRequest" ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc 'cp payments-request.yaml specs/payment-api.yaml' +``` + ## 3. Re-run contract tests Run: @@ -129,7 +135,7 @@ docker compose up contract-test --build --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 2, Successes: 2, Failures: 0, Errors: 0 +Tests run: 2, Successes: 2, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: From 3791b81028aec9c5ee35e5255ef710ac4ccbbaa2 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 18:20:24 +0530 Subject: [PATCH 24/71] response-templating : Added commands + added file for replacement data (test_find_available_products_book_200.json) + WIP counts --- response-templating/README.md | 19 ++++++-- ...test_find_available_products_book_200.json | 44 +++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 response-templating/test_find_available_products_book_200.json diff --git a/response-templating/README.md b/response-templating/README.md index 26592f5..154c784 100644 --- a/response-templating/README.md +++ b/response-templating/README.md @@ -58,7 +58,7 @@ docker compose up --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 4, Successes: 1, Failures: 3, Errors: 0 +Tests run: 4, Successes: 1, Failures: 3, WIP: 0, Errors: 0 ``` Clean up: @@ -81,14 +81,21 @@ Change response templating so: Keep `id` as-is. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"productid\": 1#\"productid\": \"(PRODUCTID:number)\"#; s#\"count\": 100#\"count\": \"(COUNT:number)\"#; s#\"productid\": \"\$match(dataType:number)\"#\"productid\": \"\$(PRODUCTID)\"#; s#\"count\": 1#\"count\": \"\$(COUNT)\"#' examples/mock/test_accepted_order_request.json" +``` + ### Checkpoint after Task A Run: ```shell docker compose up --abort-on-container-exit ``` + Expected checkpoint output: ```terminaloutput -Tests run: 4, Successes: 2, Failures: 2, Errors: 0 +Tests run: 4, Successes: 2, Failures: 2, WIP: 0, Errors: 0 ``` Clean up: ```shell @@ -106,6 +113,12 @@ Configure lookup logic based on request query `type` so that: Note: - You can keep this as one lookup-driven mock example instead of creating duplicate mock files. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc 'cp test_find_available_products_book_200.json examples/mock/test_find_available_products_book_200.json' +``` + ## 4. Final verification Run: ```shell @@ -114,7 +127,7 @@ docker compose up --abort-on-container-exit Expected final output: ```terminaloutput -Tests run: 4, Successes: 4, Failures: 0, Errors: 0 +Tests run: 4, Successes: 4, Failures: 0, WIP: 0, Errors: 0 ``` Clean up: diff --git a/response-templating/test_find_available_products_book_200.json b/response-templating/test_find_available_products_book_200.json new file mode 100644 index 0000000..f9335f6 --- /dev/null +++ b/response-templating/test_find_available_products_book_200.json @@ -0,0 +1,44 @@ +{ + "http-request": { + "path": "/findAvailableProducts", + "method": "GET", + "query": { + "from-date": "2026-02-15", + "to-date": "2026-02-15", + "type": "(TYPE:ProductType)" + }, + "headers": { + "pageSize": "10" + } + }, + "http-response": { + "status": 200, + "body": { + "id": "$(dataLookup.products[TYPE].id)", + "name": "$(dataLookup.products[TYPE].name)", + "type": "$(TYPE)", + "inventory": "$(dataLookup.products[TYPE].inventory)", + "createdOn": "$(dataLookup.products[TYPE].createdOn)" + }, + "status-text": "OK", + "headers": { + "Content-Type": "application/json" + } + }, + "dataLookup": { + "products": { + "book": { + "id": 1, + "name": "Larry Potter", + "inventory": 100, + "createdOn": "2026-02-15" + }, + "gadget": { + "id": 2, + "name": "iPhone", + "inventory": 500, + "createdOn": "2026-02-15" + } + } + } +} From b07638367ee9db6d52e6a7a7caca9ff08deae504 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 18:20:48 +0530 Subject: [PATCH 25/71] schema-design : payments-request.yaml (Drop in replacement file) --- schema-design/payments-request.yaml | 94 +++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 schema-design/payments-request.yaml diff --git a/schema-design/payments-request.yaml b/schema-design/payments-request.yaml new file mode 100644 index 0000000..ac9190d --- /dev/null +++ b/schema-design/payments-request.yaml @@ -0,0 +1,94 @@ +openapi: 3.0.0 +info: + title: Payment API + version: "1.0" +servers: + - url: http://localhost:8080 +paths: + /payments: + post: + summary: Create a payment + operationId: createPayment + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PaymentRequest" + responses: + "201": + description: Payment accepted + content: + application/json: + schema: + $ref: "#/components/schemas/PaymentAccepted" + "400": + description: Bad request + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" +components: + schemas: + CardPaymentRequest: + type: object + required: + - paymentType + - cardNumber + - cardExpiry + - cardCvv + properties: + paymentType: + type: string + enum: [ card ] + cardNumber: + type: string + cardExpiry: + type: string + cardCvv: + type: string + BankTransferPaymentRequest: + type: object + required: + - paymentType + - bankAccountNumber + - bankRoutingNumber + - bankAccountHolder + properties: + paymentType: + type: string + enum: [ bank_transfer ] + bankAccountNumber: + type: string + bankRoutingNumber: + type: string + bankAccountHolder: + type: string + PaymentRequest: + oneOf: + - $ref: "#/components/schemas/CardPaymentRequest" + - $ref: "#/components/schemas/BankTransferPaymentRequest" + discriminator: + propertyName: paymentType + mapping: + card: "#/components/schemas/CardPaymentRequest" + bank_transfer: "#/components/schemas/BankTransferPaymentRequest" + PaymentAccepted: + type: object + additionalProperties: false + properties: + id: + type: integer + status: + type: string + required: + - id + - status + ErrorResponse: + type: object + additionalProperties: false + properties: + message: + type: string + required: + - message From a2b61bb82f065a62d7fa45c5daa762728ad12a58 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 18:38:01 +0530 Subject: [PATCH 26/71] Added 3 tested labs --- validate_readme_commands.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validate_readme_commands.py b/validate_readme_commands.py index df15fc5..6a92a13 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -23,6 +23,9 @@ DEFAULT_LABS = [ "api-coverage", + "schema-design", + "response-templating", + "api-resiliency-testing", "api-security-schemes", "continuous-integration", "data-adapters", From 13dfb9ab829634c09a36d91af914c14dcc24ac6b Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 18:46:55 +0530 Subject: [PATCH 27/71] filters : added alternative run command --- filters/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/filters/README.md b/filters/README.md index bd6c2f2..7a05acf 100644 --- a/filters/README.md +++ b/filters/README.md @@ -87,6 +87,12 @@ Stop Studio: docker compose --profile studio down -v ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i \"/baseUrl: http:\\/\\/localhost:8080/a\\ filter: \\\"PATH!='/health,/monitor/{id},/swagger' && STATUS='200,201'\\\"\" specmatic.yaml" +``` + ## 5. Verify from CLI (with persisted filters) Run: From 8234bccdea8a856ce6bbcbedab0c052105cdc2e4 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 18:56:37 +0530 Subject: [PATCH 28/71] filters: Added WIP counts, passes --- filters/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/filters/README.md b/filters/README.md index 7a05acf..a829189 100644 --- a/filters/README.md +++ b/filters/README.md @@ -32,7 +32,7 @@ docker compose up --abort-on-container-exit Expected baseline output: ```terminaloutput -Tests run: 136, Successes: 20, Failures: 114, Errors: 2 +Tests run: 136, Successes: 18, Failures: 114, WIP: 4, Errors: 0 ``` Clean up: @@ -103,7 +103,7 @@ docker compose up --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 20, Successes: 20, Failures: 0, Errors: 0 +Tests run: 20, Successes: 18, Failures: 0, WIP: 2, Errors: 0 ``` Clean up: @@ -115,7 +115,7 @@ docker compose down -v ## Pass Criteria - Baseline run shows `136` tests with many failures. - After applying and exporting filters, CLI run shows: - - `Tests run: 20, Successes: 20, Failures: 0, Errors: 0` + - `Tests run: 20, Successes: 18, Failures: 0, WIP: 2, Errors: 0` ## Why this lab matters - Filters help teams focus on critical scenarios while they triage known failures. From 8a72af8f9bc880713bf794c30d070960cee34117 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 19:09:55 +0530 Subject: [PATCH 29/71] overlays : added alternative run command + file for overlay changes --- overlays/path-prefix.overlay.yaml | 18 ++++++++++++++++++ overlays/specmatic.yaml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 overlays/path-prefix.overlay.yaml diff --git a/overlays/path-prefix.overlay.yaml b/overlays/path-prefix.overlay.yaml new file mode 100644 index 0000000..1eb79db --- /dev/null +++ b/overlays/path-prefix.overlay.yaml @@ -0,0 +1,18 @@ +overlay: 1.0.0 +info: + title: Add /api/v1 prefix for deployed provider compatibility + version: "1.0" +actions: + - target: "$.paths" + update: + /api/v1/users/{id}: + get: + summary: Get user by id + parameters: + - $ref: "#/components/parameters/UserIdPathParam" + responses: + "200": + $ref: "#/components/responses/User200" + + - target: "$.paths['/api/users/{id}']" + remove: true diff --git a/overlays/specmatic.yaml b/overlays/specmatic.yaml index 487d5ed..b08342b 100644 --- a/overlays/specmatic.yaml +++ b/overlays/specmatic.yaml @@ -25,7 +25,7 @@ systemUnderTest: id: my-service baseUrl: http://myservice:8080 # uncomment the below line to add a path prefix to all paths in the spec. This is useful when the service is running behind an API gateway that adds a path prefix. -# overlayFilePath: ./overlays/path-prefix.overlay.yaml + overlayFilePath: ./overlays/path-prefix.overlay.yaml specmatic: license: From 3f17de7fdf9bc9a243f2da96cb16999b7f29d72a Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 19:11:49 +0530 Subject: [PATCH 30/71] Commented line in , restored to original --- overlays/specmatic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overlays/specmatic.yaml b/overlays/specmatic.yaml index b08342b..487d5ed 100644 --- a/overlays/specmatic.yaml +++ b/overlays/specmatic.yaml @@ -25,7 +25,7 @@ systemUnderTest: id: my-service baseUrl: http://myservice:8080 # uncomment the below line to add a path prefix to all paths in the spec. This is useful when the service is running behind an API gateway that adds a path prefix. - overlayFilePath: ./overlays/path-prefix.overlay.yaml +# overlayFilePath: ./overlays/path-prefix.overlay.yaml specmatic: license: From 6449786e73c3fc3dd7a5a230727e55a3ab58193a Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 19:12:21 +0530 Subject: [PATCH 31/71] Added alternative run command in overlays --- overlays/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/overlays/README.md b/overlays/README.md index 01d6319..53bdf75 100644 --- a/overlays/README.md +++ b/overlays/README.md @@ -108,6 +108,12 @@ Uncomment this line: overlayFilePath: ./overlays/path-prefix.overlay.yaml ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc 'cp path-prefix.overlay.yaml overlays/path-prefix.overlay.yaml && sed -i "s/^# overlayFilePath:/ overlayFilePath:/" specmatic.yaml' +``` + ## Pass verification Run: From 56516101617e78d87edc8ae5a286ba282f2746b1 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 19:12:56 +0530 Subject: [PATCH 32/71] overlays: added WIP counts --- overlays/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/overlays/README.md b/overlays/README.md index 53bdf75..5efa44a 100644 --- a/overlays/README.md +++ b/overlays/README.md @@ -54,7 +54,7 @@ docker compose up test --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 1, Successes: 0, Failures: 1, Errors: 0 +Tests run: 1, Successes: 0, Failures: 1, WIP: 0, Errors: 0 ``` - The test run fails because requests are generated for `/api/users/{id}` while the provider serves `/api/v1/users/{id}`. @@ -124,7 +124,7 @@ docker compose up test --abort-on-container-exit Expected output: ```terminaloutput -Tests run: 1, Successes: 1, Failures: 0, Errors: 0 +Tests run: 1, Successes: 1, Failures: 0, WIP: 0, Errors: 0 ``` - Contract tests pass, because Specmatic applies the overlay before running tests. From 18746ec85c4575f1a4d785566456873e0fa4af91 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 19:33:24 +0530 Subject: [PATCH 33/71] backward compatibility: File copy based approach. Added alternative run commands --- .../products-breaking.yaml | 30 +++++++++++++++++++ .../products-fixed.yaml | 30 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 backward-compatibility-testing/products-breaking.yaml create mode 100644 backward-compatibility-testing/products-fixed.yaml diff --git a/backward-compatibility-testing/products-breaking.yaml b/backward-compatibility-testing/products-breaking.yaml new file mode 100644 index 0000000..4fc9384 --- /dev/null +++ b/backward-compatibility-testing/products-breaking.yaml @@ -0,0 +1,30 @@ +openapi: 3.0.0 +info: + title: Sample Product API + version: 1.1.0 +paths: + /products/{id}: + get: + summary: Get product by id + parameters: + - in: path + name: id + required: true + schema: + type: number + responses: + "200": + description: Product details + content: + application/json: + schema: + type: object + required: + - name + properties: + name: + type: number + sku: + type: string + category: + type: string diff --git a/backward-compatibility-testing/products-fixed.yaml b/backward-compatibility-testing/products-fixed.yaml new file mode 100644 index 0000000..2a45e5d --- /dev/null +++ b/backward-compatibility-testing/products-fixed.yaml @@ -0,0 +1,30 @@ +openapi: 3.0.0 +info: + title: Sample Product API + version: 1.1.0 +paths: + /products/{id}: + get: + summary: Get product by id + parameters: + - in: path + name: id + required: true + schema: + type: number + responses: + "200": + description: Product details + content: + application/json: + schema: + type: object + required: + - name + properties: + name: + type: string + sku: + type: string + category: + type: string From eecf6e2122350c418243c30159418cb539e8085d Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Thu, 14 May 2026 20:16:59 +0530 Subject: [PATCH 34/71] external examples, added alternate command --- external-examples/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/external-examples/README.md b/external-examples/README.md index dfd73cb..79eaade 100644 --- a/external-examples/README.md +++ b/external-examples/README.md @@ -103,6 +103,12 @@ Expected output: [OK] Examples: 6 passed and 0 failed out of 6 total ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'cp -R external-examples-fixed/. examples/' +``` + ### Final Phase #### 5. Re-run validation and verify pass state From e196f629308537e9cb9095297395802f0286fbee Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Thu, 14 May 2026 21:21:01 +0530 Subject: [PATCH 35/71] chore: remove unnecessary extra line break --- api-resiliency-testing/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/api-resiliency-testing/README.md b/api-resiliency-testing/README.md index 7bf61ec..4d66c0c 100644 --- a/api-resiliency-testing/README.md +++ b/api-resiliency-testing/README.md @@ -255,7 +255,6 @@ To fix this, update `examples/order-service/stub_timeout_post_product.json` so t } ``` - Why `value:each` is the right matcher here: - `value:each` tracks matcher exhaustion separately for each distinct value - that means each valid `type` value gets its own one-time transient timeout match From 739411038afa77b63dc1da16748bc16abf2802c7 Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Thu, 14 May 2026 21:52:22 +0530 Subject: [PATCH 36/71] Add multi-lab mode and smarter cleanup --- mcp-auto-test/README.md | 6 ++ tests/test_validate_readme_commands.py | 23 +++++++ validate_readme_commands.py | 90 ++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) diff --git a/mcp-auto-test/README.md b/mcp-auto-test/README.md index 55c032c..0f57edd 100644 --- a/mcp-auto-test/README.md +++ b/mcp-auto-test/README.md @@ -147,6 +147,12 @@ Overall Success Rate: 94.3% This is not required for the lab goal. It is a follow-up to explore how Specmatic mutates valid tool inputs to probe validation boundaries. Try to pass all 35 tests. +Clean up: + +```shell +docker compose down -v +``` + ## Pass criteria - Baseline run fails with two tool execution failures. - After fixing only `service/order_service.py`, both tests pass. diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index cb7c3ed..a5818dc 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -12,6 +12,7 @@ CommandExecutionError, CommandSpec, _expected_output_matches, + derive_final_cleanup_commands, main, parse_readme_commands, print_command_mapping, @@ -256,6 +257,28 @@ def test_runs_cleanup_commands_after_failure_but_skips_non_cleanup_zero_output_c class CleanupCommandTests(GitRepoTestCase): + def test_derive_final_cleanup_commands_preserves_profiles(self) -> None: + cleanup_commands = derive_final_cleanup_commands( + [ + CommandSpec( + command="docker compose --profile test up test --build --abort-on-container-exit\n", + expected_outputs=[], + ), + CommandSpec( + command="docker compose run --rm mcp-test --enable-resiliency-tests\n", + expected_outputs=[], + ), + ] + ) + + self.assertEqual( + cleanup_commands, + [ + "docker compose --profile test down -v --remove-orphans", + "docker compose down -v --remove-orphans", + ], + ) + def test_runs_only_cleanup_commands_before_next_expected_output_command(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) diff --git a/validate_readme_commands.py b/validate_readme_commands.py index 6a92a13..6afbf5f 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -6,6 +6,7 @@ import argparse import queue import shutil +import shlex import subprocess import sys import threading @@ -462,6 +463,78 @@ def run_cleanup_commands( return cleanup_results +def run_final_cleanup_commands( + command_specs: Sequence[CommandSpec], + cwd: Path, + timeout_seconds: float, +) -> list[CommandResult]: + cleanup_results: list[CommandResult] = [] + + for cleanup_index, cleanup_command in enumerate(derive_final_cleanup_commands(command_specs), start=1): + try: + completed = _run_command( + command=cleanup_command, + cwd=cwd, + timeout_seconds=timeout_seconds, + ) + except subprocess.TimeoutExpired: + continue + cleanup_results.append( + CommandResult( + index=cleanup_index, + command=cleanup_command, + expected_outputs=[], + cwd=cwd, + skipped=False, + returncode=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + ) + ) + + return cleanup_results + + +def derive_final_cleanup_commands(command_specs: Sequence[CommandSpec]) -> list[str]: + cleanup_commands: list[str] = [] + seen: set[str] = set() + + for command_spec in command_specs: + cleanup_command = _derive_cleanup_command(command_spec.command) + if cleanup_command is None or cleanup_command in seen: + continue + seen.add(cleanup_command) + cleanup_commands.append(cleanup_command) + + return cleanup_commands + + +def _derive_cleanup_command(command: str) -> str | None: + try: + tokens = shlex.split(command) + except ValueError: + return None + + if len(tokens) >= 2 and tokens[0] == "docker" and tokens[1] == "compose": + compose_start_index = 2 + elif tokens and tokens[0] == "docker-compose": + compose_start_index = 1 + else: + return None + + subcommand_index = None + for index in range(compose_start_index, len(tokens)): + if tokens[index] in {"up", "run"}: + subcommand_index = index + break + + if subcommand_index is None: + return None + + cleanup_tokens = tokens[:subcommand_index] + ["down", "-v", "--remove-orphans"] + return shlex.join(cleanup_tokens) + + def _is_cleanup_command(command: str) -> bool: normalized_command = " ".join(command.lower().split()) cleanup_patterns = ( @@ -484,6 +557,15 @@ def _format_cleanup_results(cleanup_results: Sequence[CommandResult]) -> str: return "\n".join(lines) +def print_final_cleanup_summary(cleanup_results: Sequence[CommandResult]) -> None: + if not cleanup_results: + return + + print("FINAL CLEANUP") + for result in cleanup_results: + print(f" {result.command}") + + def _format_failure_message( *, index: int, @@ -631,6 +713,14 @@ def run_single_readme( print(f"README: {readme_path}", file=sys.stderr) print(summary.failure_message, file=sys.stderr) + final_cleanup_results = run_final_cleanup_commands( + command_specs=command_specs, + cwd=readme_path.parent, + timeout_seconds=timeout_seconds, + ) + if final_cleanup_results: + print_final_cleanup_summary(final_cleanup_results) + if repo_snapshot is not None: print_reset_summary(reset_lab_changes(readme_path.parent, repo_snapshot)) From 91f88eba958ec692e43c4e8ebb519fbf881f6ac5 Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Thu, 14 May 2026 21:57:45 +0530 Subject: [PATCH 37/71] chore: update validate_readme_commands.py so the positional argument is now a lab folder, not a README path. --- tests/test_validate_readme_commands.py | 40 +++++++++++++++++--------- validate_readme_commands.py | 4 +-- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index a5818dc..cd30a41 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -343,9 +343,19 @@ def test_resolve_readme_paths_uses_default_labs_when_no_arg(self) -> None: self.assertTrue(str(readme_paths[0]).endswith("/api-coverage/README.md")) self.assertTrue(str(readme_paths[-1]).endswith("/workflow-in-same-spec/README.md")) + def test_resolve_readme_paths_appends_readme_to_lab_directory(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) / "sample-lab" + lab_path.mkdir() + + readme_paths = resolve_readme_paths(str(lab_path)) + + self.assertEqual(readme_paths, [(lab_path / "README.md").resolve()]) + def test_main_returns_zero_for_valid_readme(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - readme_path = Path(temp_dir) / "README.md" + lab_path = Path(temp_dir) + readme_path = lab_path / "README.md" readme_path.write_text( textwrap.dedent( """ @@ -361,7 +371,7 @@ def test_main_returns_zero_for_valid_readme(self) -> None: encoding="utf-8", ) - exit_code = main([str(readme_path), "--timeout", "5"]) + exit_code = main([str(lab_path), "--timeout", "5"]) self.assertEqual(exit_code, 0) @@ -390,7 +400,8 @@ def test_dry_run_prints_command_mapping(self) -> None: def test_cli_invocation_reports_failure(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - readme_path = Path(temp_dir) / "README.md" + lab_path = Path(temp_dir) + readme_path = lab_path / "README.md" readme_path.write_text( textwrap.dedent( """ @@ -407,7 +418,7 @@ def test_cli_invocation_reports_failure(self) -> None: ) completed = subprocess.run( - ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(readme_path)], + ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(lab_path)], cwd=str(ROOT_DIR), capture_output=True, text=True, @@ -420,7 +431,8 @@ def test_cli_invocation_reports_failure(self) -> None: def test_cli_invocation_prints_failure_summary(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - readme_path = Path(temp_dir) / "README.md" + lab_path = Path(temp_dir) + readme_path = lab_path / "README.md" readme_path.write_text( textwrap.dedent( """ @@ -441,7 +453,7 @@ def test_cli_invocation_prints_failure_summary(self) -> None: ) completed = subprocess.run( - ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(readme_path)], + ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(lab_path)], cwd=str(ROOT_DIR), capture_output=True, text=True, @@ -455,9 +467,9 @@ def test_cli_invocation_prints_failure_summary(self) -> None: def test_dry_run_does_not_execute_commands(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - temp_path = Path(temp_dir) - readme_path = temp_path / "README.md" - marker_path = temp_path / "marker.txt" + lab_path = Path(temp_dir) + readme_path = lab_path / "README.md" + marker_path = lab_path / "marker.txt" readme_path.write_text( textwrap.dedent( f""" @@ -478,9 +490,9 @@ def test_dry_run_does_not_execute_commands(self) -> None: "python3", str(ROOT_DIR / "validate_readme_commands.py"), "--dry-run", - str(readme_path), + str(lab_path), ], - cwd=str(temp_path), + cwd=str(lab_path), capture_output=True, text=True, check=False, @@ -512,7 +524,7 @@ def test_cli_invocation_prints_pass_at_end(self) -> None: ) completed = subprocess.run( - ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(readme_path)], + ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(repo_path)], cwd=str(repo_path), capture_output=True, text=True, @@ -545,7 +557,7 @@ def test_main_resets_lab_changed_files_by_default(self) -> None: ) completed = subprocess.run( - ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(readme_path)], + ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(repo_path)], cwd=str(repo_path), capture_output=True, text=True, @@ -582,7 +594,7 @@ def test_main_skip_reset_preserves_lab_changed_files(self) -> None: "python3", str(ROOT_DIR / "validate_readme_commands.py"), "--skip-reset", - str(readme_path), + str(repo_path), ], cwd=str(repo_path), capture_output=True, diff --git a/validate_readme_commands.py b/validate_readme_commands.py index 6afbf5f..a400fad 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -621,7 +621,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "readme", nargs="?", - help="Path to the README.md file to validate. If omitted, runs the built-in lab README list.", + help="Lab directory to validate. If omitted, runs the built-in lab README list.", ) parser.add_argument( "--dry-run", @@ -734,7 +734,7 @@ def run_single_readme( def resolve_readme_paths(readme_arg: str | None) -> list[Path]: if readme_arg: - return [Path(readme_arg).expanduser().resolve()] + return [(Path(readme_arg).expanduser().resolve() / "README.md")] repo_root = Path(__file__).resolve().parent return [(repo_root / lab / "README.md").resolve() for lab in DEFAULT_LABS] From 1e7d0ef1e8628054e459692db6a0d41143736ee7 Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Thu, 14 May 2026 22:09:23 +0530 Subject: [PATCH 38/71] Improve README validator diagnostics and defaults --- quick-start-async-contract-testing/README.md | 4 +- tests/test_validate_readme_commands.py | 20 ++++++ validate_readme_commands.py | 70 ++++++++++++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/quick-start-async-contract-testing/README.md b/quick-start-async-contract-testing/README.md index 71c8c5c..8b2d5b0 100644 --- a/quick-start-async-contract-testing/README.md +++ b/quick-start-async-contract-testing/README.md @@ -45,7 +45,7 @@ docker compose up contract-test --build --abort-on-container-exit ``` ```terminaloutput -Tests run: 1, Successes: 0, Failures: 1, Errors: 0 +Tests run: 1, Successes: 0, Failures: 1, WIP: 0, Errors: 0 ``` Expected failure signal: @@ -89,7 +89,7 @@ docker compose up contract-test --build --abort-on-container-exit Expected pass signal: ```terminaloutput -Tests run: 1, Successes: 1, Failures: 0, Errors: 0 +Tests run: 1, Successes: 1, Failures: 0, WIP: 0, Errors: 0 ``` Then clean up: diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index cd30a41..1287a04 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -165,6 +165,26 @@ def test_non_zero_exit_without_terminaloutput_blocks_fails(self) -> None: self.assertIn("command exited non-zero without any expected terminaloutput blocks", str(ctx.exception)) + def test_failure_message_shows_expected_and_closest_actual_line(self) -> None: + with self.assertRaises(CommandExecutionError) as ctx: + run_command_specs( + [ + CommandSpec( + command="printf 'Tests run: 1, Successes: 1, Failures: 0, WIP: 0, Errors: 0\\n'", + expected_outputs=["Tests run: 1, Successes: 1, Failures: 0, Errors: 0\n"], + ) + ], + cwd=ROOT_DIR, + timeout_seconds=5, + ) + + message = str(ctx.exception) + self.assertIn("Mismatch Detail", message) + self.assertIn("Expected line", message) + self.assertIn("Tests run: 1, Successes: 1, Failures: 0, Errors: 0", message) + self.assertIn("Closest actual line", message) + self.assertIn("Tests run: 1, Successes: 1, Failures: 0, WIP: 0, Errors: 0", message) + def test_line_by_line_matching_allows_prefixed_actual_lines(self) -> None: expected_output = ( "Failed the following API Coverage Report success criteria:\n" diff --git a/validate_readme_commands.py b/validate_readme_commands.py index a400fad..aef378c 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import difflib import queue import shutil import shlex @@ -19,6 +20,7 @@ ANSI_BOLD = "\033[1m" ANSI_CYAN = "\033[36m" ANSI_GREEN = "\033[32m" +ANSI_RED = "\033[31m" ANSI_YELLOW = "\033[33m" ANSI_DIM = "\033[2m" @@ -286,6 +288,7 @@ def _assert_command_result(result: CommandResult) -> None: if result.expected_outputs: for output_index, expected_output in enumerate(result.expected_outputs, start=1): if not _expected_output_matches(expected_output, result.combined_output): + mismatch_detail = _describe_output_mismatch(expected_output, result.combined_output) raise CommandValidationFailure( _format_failure_message( index=result.index, @@ -294,6 +297,7 @@ def _assert_command_result(result: CommandResult) -> None: cwd=result.cwd, returncode=result.returncode, reason=f"missing expected terminaloutput block #{output_index}", + detail=mismatch_detail, ) ) return @@ -331,6 +335,69 @@ def _expected_output_matches(expected_output: str, actual_output: str) -> bool: return True +def _describe_output_mismatch(expected_output: str, actual_output: str) -> str | None: + expected_lines = [line for line in expected_output.splitlines() if line.strip()] + actual_lines = actual_output.splitlines() + + if not expected_lines: + return None + + actual_index = 0 + for expected_line in expected_lines: + while actual_index < len(actual_lines): + if expected_line in actual_lines[actual_index]: + actual_index += 1 + break + actual_index += 1 + else: + closest_line = _find_closest_line(expected_line, actual_lines) + divider = _style("-" * 48, ANSI_DIM) + detail_lines = [ + "", + divider, + _style("Mismatch Detail", ANSI_BOLD, ANSI_RED), + "", + _style("Expected line", ANSI_BOLD, ANSI_GREEN), + f" {expected_line}", + ] + if closest_line is not None: + detail_lines.extend( + [ + "", + _style("Closest actual line", ANSI_BOLD, ANSI_YELLOW), + f" {closest_line}", + ] + ) + else: + detail_lines.extend( + [ + "", + _style("Closest actual line", ANSI_BOLD, ANSI_YELLOW), + " none", + ] + ) + detail_lines.extend(["", divider]) + return "\n".join(detail_lines) + + return None + + +def _find_closest_line(expected_line: str, actual_lines: Sequence[str]) -> str | None: + best_line: str | None = None + best_ratio = 0.0 + + for actual_line in actual_lines: + ratio = difflib.SequenceMatcher(None, expected_line, actual_line).ratio() + if ratio > best_ratio: + best_ratio = ratio + best_line = actual_line + + if best_ratio == 0.0: + return None + + return best_line + + def _run_command( *, command: str, @@ -574,6 +641,7 @@ def _format_failure_message( cwd: Path | None, returncode: int | None, reason: str, + detail: str | None = None, ) -> str: returncode_text = "n/a" if returncode is None else str(returncode) lines = [ @@ -589,6 +657,8 @@ def _format_failure_message( f"Expected terminaloutput blocks: {len(expected_outputs)}", ] ) + if detail: + lines.extend(["", detail]) return "\n".join(lines) From dda5e283561cb6c81c0a95d50ad3a15f160425bc Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Thu, 14 May 2026 23:27:41 +0530 Subject: [PATCH 39/71] WIP fix for schema-resiliency-testing lab --- schema-resiliency-testing/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schema-resiliency-testing/README.md b/schema-resiliency-testing/README.md index a1b2686..474f2c6 100644 --- a/schema-resiliency-testing/README.md +++ b/schema-resiliency-testing/README.md @@ -33,7 +33,7 @@ Then go to the Test tab, set url as `http://localhost:8080` and click on the "Ru You should see ```terminaloutput -Tests run: 3, Successes: 3, Failures: 0, Errors: 0 +Tests run: 3, Successes: 3, Failures: 0, WIP: 0, Errors: 0 ``` Stop Studio before moving to the next steps: @@ -54,7 +54,7 @@ This will run the suite, start the dependency mock, and run the tests against it Expected console output: ```terminaloutput -Tests run: 3, Successes: 3, Failures: 0, Errors: 0 +Tests run: 3, Successes: 3, Failures: 0, WIP: 0, Errors: 0 ``` Clean up @@ -86,7 +86,7 @@ docker compose up --abort-on-container-exit Expected console output: ```terminaloutput -Tests run: 42, Successes: 42, Failures: 0, Errors: 0 +Tests run: 42, Successes: 42, Failures: 0, WIP: 0, Errors: 0 ``` Clean up From 0e938914c967c852d3759108bab63a3a41e0918a Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Thu, 14 May 2026 23:36:15 +0530 Subject: [PATCH 40/71] add filters and overlays to default_labs list --- validate_readme_commands.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/validate_readme_commands.py b/validate_readme_commands.py index aef378c..b16bc6f 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -40,6 +40,8 @@ "quick-start-contract-testing", "schema-resiliency-testing", "workflow-in-same-spec", + "filters", + "overlays" ] From 31483b41001ddedd7ff30ebea3f06caae0a71f19 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Fri, 15 May 2026 10:35:45 +0530 Subject: [PATCH 41/71] Backward compatibility readme fixed --- backward-compatibility-testing/README.md | 29 ++++++++++-------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/backward-compatibility-testing/README.md b/backward-compatibility-testing/README.md index d4b9bc2..5e7364d 100644 --- a/backward-compatibility-testing/README.md +++ b/backward-compatibility-testing/README.md @@ -89,6 +89,12 @@ properties: You now have an uncommitted change in a tracked contract file. Specmatic will compare it to the version on `origin/main`. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/workspace" -w /workspace specmatic/enterprise:latest -lc 'cp products-breaking.yaml products.yaml' +``` + ## Part B: Run the backward compatibility check Run: @@ -135,12 +141,6 @@ The Incompatibility Report: This is number in the new specification response but string in the old specification ``` -Expected verdict: - -```terminaloutput -(INCOMPATIBLE) This spec contains breaking changes to the API -``` - Why this fails: - Adding optional `category` is safe. - Changing `name` from `string` to `number` is a breaking change for existing consumers. @@ -165,6 +165,12 @@ name: Keep the new `category` field. Keep version `1.1.0`. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/workspace" -w /workspace specmatic/enterprise:latest -lc 'cp products-fixed.yaml products.yaml' +``` + ## Part D: Re-run the check Run the same command again: @@ -195,13 +201,6 @@ Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (COMPATIBLE) The spec is backward compatible with the corresponding spec from origin/main ``` -Expected passing output: - -```terminaloutput -Verdict for spec /workspace/backward-compatibility-testing/products.yaml: - (COMPATIBLE) The spec is backward compatible with the corresponding spec from origin/main -``` - ## Clean up Restore the tracked file: @@ -209,10 +208,6 @@ Restore the tracked file: git restore products.yaml ``` -```terminaloutput -products.yaml restored. -``` - ## Check backward compatibility in Specmatic Studio before saving Start Studio from `labs/backward-compatibility-testing`: From 4ec96d208ad345baa606d4fcf8f6b4dd318736c0 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Fri, 15 May 2026 15:26:28 +0530 Subject: [PATCH 42/71] Fixed labs: async-event-flow, external-examples, kafka-avro, partial-examples, quick-start-mock --- async-event-flow/README.md | 33 +++++++++-- .../fixed/acceptOrder-with-before.json | 36 ++++++++++++ .../outForDeliveryOrder-with-before.json | 48 +++++++++++++++ async-event-flow/run-suite-config.yaml | 58 +++++++++++++++++++ external-examples/README.md | 18 ++++-- ...plication_json_201_application_json_1.json | 25 ++++++++ ...plication_json_201_application_json_1.json | 26 +++++++++ kafka-avro/README.md | 50 ++++++++++++---- .../fixed-avro/NewOrders.fixed.avsc | 39 +++++++++++++ .../fixed-avro/WipOrders.fixed.avsc | 21 +++++++ partial-examples/README.md | 18 +++++- quick-start-mock/README.md | 42 ++++++++++++-- .../pets_242_GET_200_1.json | 19 ++++++ 13 files changed, 403 insertions(+), 30 deletions(-) create mode 100644 async-event-flow/examples/fixed/acceptOrder-with-before.json create mode 100644 async-event-flow/examples/fixed/outForDeliveryOrder-with-before.json create mode 100644 async-event-flow/run-suite-config.yaml create mode 100644 external-examples/external-examples-generated/createOrder_application_json_201_application_json_1.json create mode 100644 external-examples/external-examples-generated/createProduct_application_json_201_application_json_1.json create mode 100644 kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc create mode 100644 kafka-avro/docker-config/fixed-avro/WipOrders.fixed.avsc create mode 100644 quick-start-mock/quick-start-mock-generated/pets_242_GET_200_1.json diff --git a/async-event-flow/README.md b/async-event-flow/README.md index 87d5bd7..922937f 100644 --- a/async-event-flow/README.md +++ b/async-event-flow/README.md @@ -75,15 +75,21 @@ Together, `receive`/`send` plus `before`/`after` fixtures let you express full e ## Run the contract tests using Specmatic Studio 1. Start Kafka, the sample service, and Specmatic Studio. ```shell -docker compose up +docker compose up -d ``` 2. Go to [Studio](http://127.0.0.1:9000/_specmatic/studio) and open the [specmatic.yaml](specmatic.yaml) file from the left sidebar, click on "Run Suite", and use the checked-out contract under `.specmatic/repos/labs-contracts/asyncapi/async-event-flow/async-order-service.yaml` if you want to inspect the loaded AsyncAPI file in Studio. You should first see 2 passing tests and 2 failing tests: +Alternatively, just run the following commands: + +```shell +docker run --rm --network async-event-flow_default -v "$PWD:/usr/src/app" -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest run-suite --config=/usr/src/app/run-suite-config.yaml +``` + ```terminaloutput -Tests run: 4, Successes: 2, Failures: 2, Errors: 0 +Tests run: 4, Successes: 2, Failures: 2, WIP: 0, Errors: 0 ``` 3. Fix the examples: @@ -114,7 +120,7 @@ Tests run: 4, Successes: 2, Failures: 2, Errors: 0 } ], ``` -- In `examples/async-order-service/outForDeliveryOrder.json`, fix the `after` fixture by changing: +- In `examples/async-order-service/outForDeliveryOrder.json`, add a `before` fixture that seeds order `456` before the event is published, and fix the `after` fixture by changing: ```json "tax-invoice-for-order-456": "$match(exact: 2)" @@ -126,17 +132,32 @@ To: "tax-invoice-for-order-456": "$match(exact: 1)" ``` +Alternatively, just run the following commands: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'cp examples/fixed/acceptOrder-with-before.json examples/async-order-service/acceptOrder.json' +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'cp examples/fixed/outForDeliveryOrder-with-before.json examples/async-order-service/outForDeliveryOrder.json' +``` + 4. Restart Docker Containers + ```shell -docker compose down -v && docker compose up +docker compose down -v +docker compose up -d ``` -5. Re-run the suite from Studio. +Re-run the suite from Studio. + +Alternatively, just run the following commands: + +```shell +docker run --rm --network async-event-flow_default -v "$PWD:/usr/src/app" -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest run-suite --config=/usr/src/app/run-suite-config.yaml +``` You should now see: ```terminaloutput -Tests run: 4, Successes: 4, Failures: 0, Errors: 0 +Tests run: 4, Successes: 4, Failures: 0, WIP: 0, Errors: 0 ``` ### Cleanup diff --git a/async-event-flow/examples/fixed/acceptOrder-with-before.json b/async-event-flow/examples/fixed/acceptOrder-with-before.json new file mode 100644 index 0000000..9977f4a --- /dev/null +++ b/async-event-flow/examples/fixed/acceptOrder-with-before.json @@ -0,0 +1,36 @@ +{ + "id": "accept-order", + "name": "ACCEPT_ORDER", + "before": [ + { + "type": "http", + "wait": "PT1S", + "http-request": { + "baseUrl": "http://sut:8080", + "path": "/orders", + "method": "PUT", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "id": 123, + "status": "ACCEPTED", + "timestamp": "2025-04-12T14:30:00Z" + } + }, + "http-response": { + "status": 200 + }, + "timeout": "PT30S" + } + ], + "send": { + "topic": "accepted-orders", + "key": 123, + "payload": { + "id": 123, + "status": "ACCEPTED", + "timestamp": "2025-04-12T14:30:00Z" + } + } +} diff --git a/async-event-flow/examples/fixed/outForDeliveryOrder-with-before.json b/async-event-flow/examples/fixed/outForDeliveryOrder-with-before.json new file mode 100644 index 0000000..f3a37b7 --- /dev/null +++ b/async-event-flow/examples/fixed/outForDeliveryOrder-with-before.json @@ -0,0 +1,48 @@ +{ + "id": "order-out-for-delivery", + "name": "ORDER_OUT_FOR_DELIVERY", + "receive": { + "topic": "out-for-delivery-orders", + "payload": { + "orderId": 456, + "deliveryAddress": "1234 Elm Street, Springfield", + "deliveryDate": "2025-04-14" + } + }, + "after": [ + { + "type": "http", + "wait": "PT2S", + "http-request": { + "baseUrl": "http://sut:8080", + "path": "/orders/456?status=SHIPPED", + "method": "GET" + }, + "http-response": { + "status": 200, + "body": { + "id": "$match(exact: 456)", + "orderItems": [], + "lastUpdatedDate": "$match(dataType: datetime)", + "status": "$match(exact: SHIPPED)" + } + }, + "timeout": "PT10S" + }, + { + "type": "http", + "http-request": { + "baseUrl": "http://localhost:8090", + "path": "/_specmatic/verify?exampleIds=tax-invoice-for-order-456", + "method": "GET" + }, + "http-response": { + "status": 200, + "body": { + "tax-invoice-for-order-456": "$match(exact: 1)" + } + }, + "timeout": "PT20S" + } + ] +} diff --git a/async-event-flow/run-suite-config.yaml b/async-event-flow/run-suite-config.yaml new file mode 100644 index 0000000..5fbcc74 --- /dev/null +++ b/async-event-flow/run-suite-config.yaml @@ -0,0 +1,58 @@ +version: 3 +systemUnderTest: + service: + $ref: "#/components/services/orderAsyncService" + runOptions: + $ref: "#/components/runOptions/orderAsyncServiceTest" + data: + examples: + - directories: + - examples/async-order-service + +dependencies: + services: + - service: + $ref: "#/components/services/taxService" + runOptions: + $ref: "#/components/runOptions/taxServiceMock" + data: + examples: + - directories: + - examples/tax-service + +components: + sources: + labsContracts: + git: + url: https://github.com/specmatic/labs-contracts.git + branch: main + services: + orderAsyncService: + definitions: + - definition: + source: + $ref: "#/components/sources/labsContracts" + specs: + - asyncapi/async-event-flow/async-order-service.yaml + taxService: + definitions: + - definition: + source: + $ref: "#/components/sources/labsContracts" + specs: + - openapi/async-event-flow/tax-service.yaml + runOptions: + orderAsyncServiceTest: + asyncapi: + type: test + servers: + - host: "${KAFKA_BROKER_HOST:kafka:9092}" + protocol: kafka + taxServiceMock: + openapi: + type: mock + baseUrl: "${TAX_SERVICE_URL:http://0.0.0.0:8090}" + +specmatic: + license: + path: /specmatic/specmatic-license.txt diff --git a/external-examples/README.md b/external-examples/README.md index 79eaade..7151b20 100644 --- a/external-examples/README.md +++ b/external-examples/README.md @@ -93,20 +93,28 @@ In Studio, update the failing examples: 3. `examples/test_accepted_order_request.json` - Add missing required property `count` (for example `2`) in request body. +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc ' +sed -i "s/\"to-date\": \"today\"/\"to-date\": \"2025-11-28\"/" examples/test_find_available_products_book_200.json && +sed -i "s/\"type\": \"movie\"/\"type\": \"book\"/" examples/test_accepted_product_request.json && +sed -i "s/\"inventory\": \"five\"/\"inventory\": 5/" examples/test_accepted_product_request.json && +sed -i "s/\"productid\": 1234/\"productid\": 1234,/" examples/test_accepted_order_request.json && +sed -i "/\"productid\": 1234,/a\\ \"count\": 2" examples/test_accepted_order_request.json +' +``` + #### 4. Generate missing examples in the same Studio flow Still in Studio, generate examples for: - `POST /products` with response `201` - `POST /orders` with response `201` -Expected output: -```terminaloutput -[OK] Examples: 6 passed and 0 failed out of 6 total -``` Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'cp -R external-examples-fixed/. examples/' +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'cp external-examples-generated/* examples/' ``` ### Final Phase diff --git a/external-examples/external-examples-generated/createOrder_application_json_201_application_json_1.json b/external-examples/external-examples-generated/createOrder_application_json_201_application_json_1.json new file mode 100644 index 0000000..33c51e0 --- /dev/null +++ b/external-examples/external-examples-generated/createOrder_application_json_201_application_json_1.json @@ -0,0 +1,25 @@ +{ + "http-request": { + "path": "/orders", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "productid": 442, + "count": 325 + }, + "description": "This is an example of a request to create a new entity at the /orders endpoint " + }, + "http-response": { + "status": 201, + "body": { + "id": 915 + }, + "status-text": "Created", + "headers": { + "Content-Type": "application/json" + }, + "description": "This is an example of a response after creating a new entity at the /orders endpoint " + } +} \ No newline at end of file diff --git a/external-examples/external-examples-generated/createProduct_application_json_201_application_json_1.json b/external-examples/external-examples-generated/createProduct_application_json_201_application_json_1.json new file mode 100644 index 0000000..2481334 --- /dev/null +++ b/external-examples/external-examples-generated/createProduct_application_json_201_application_json_1.json @@ -0,0 +1,26 @@ +{ + "http-request": { + "path": "/products", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "name": "Dr. Brendon Mosciski", + "type": "book", + "inventory": 67 + }, + "description": "This is an example of a request to create a new entity at the /products endpoint " + }, + "http-response": { + "status": 201, + "body": { + "id": 124 + }, + "status-text": "Created", + "headers": { + "Content-Type": "application/json" + }, + "description": "This is an example of a response after creating a new entity at the /products endpoint " + } +} \ No newline at end of file diff --git a/kafka-avro/README.md b/kafka-avro/README.md index 09dea26..4416d89 100644 --- a/kafka-avro/README.md +++ b/kafka-avro/README.md @@ -76,18 +76,28 @@ docker compose up specmatic-test --abort-on-container-exit Expected failure signal: ```terminaloutput -Timeout waiting for a message on topic 'wip-orders'. -Refer to Message Count Report to verify the message counts on different topics. - -Tests run: 2, Successes: 0, Failures: 2, Errors: 0 +Unsuccessful Scenarios: + "Upon receiving a message on 'new-orders' channel, should send a message on 'wip-orders' channel. Example: PLACE_MACBOOK_ORDER FAILED" + Reason: + Cannot convert value: 600.0 to an Avro Integer + + "Upon receiving a message on 'new-orders' channel, should send a message on 'wip-orders' channel. Example: PLACE_IPHONE_ORDER FAILED" + Reason: + Cannot convert value: 500.0 to an Avro Integer + +Tests run: 2, Successes: 0, Failures: 2, WIP: 0, Errors: 0 ``` The message count report should show: ```terminaloutput -| topic | Actual | Expected | -| new-orders | 2 | 2 | -| wip-orders | 0 | 2 | ++------------+-------------------------+----------+ +| Topic | No of messages received | ++------------+-------------------------+----------+ +| | Actual | Expected | ++------------+-------------------------+----------+ +| new-orders | 2 | 2 | ++------------+-------------------------+----------+ ``` Clean up before making changes: @@ -187,6 +197,12 @@ Replace `docker-config/avro/WipOrders.avsc` with: } ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'cp docker-config/fixed-avro/NewOrders.fixed.avsc docker-config/avro/NewOrders.avsc && cp docker-config/fixed-avro/WipOrders.fixed.avsc docker-config/avro/WipOrders.avsc' +``` + ### Step 2: Update the examples In both files under `api-specs/order-service-async-avro-v3_0_0_examples/`, replace the current invalid values with values that satisfy the new schema constraints. @@ -258,6 +274,13 @@ Replace `api-specs/order-service-async-avro-v3_0_0_examples/PLACE_MACBOOK_ORDER. } ``` +Alternatively, just run the following commands: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc "sed -i 's/\"id\": 101/\"id\": 1/; s/iPhone 14 Pro Max/iPhone/; s/\"price\": 500.00/\"price\": 5000/; s/exact:101/exact:1/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_IPHONE_ORDER.json" +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc "sed -i 's/\"id\": 102/\"id\": 2/; s/Macbook Mini Pro M5/Macbook/; s/\"price\": 600.00/\"price\": 6000/; s/exact:102/exact:2/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_MACBOOK_ORDER.json" +``` + ## Verify the fix Re-run the contract tests: @@ -269,15 +292,20 @@ docker compose up specmatic-test --abort-on-container-exit Expected passing output: ```terminaloutput -Tests run: 2, Successes: 2, Failures: 0, Errors: 0 +Tests run: 2, Successes: 2, Failures: 0, WIP: 0, Errors: 0 ``` The message count report should now show: ```terminaloutput -| topic | Actual | Expected | -| new-orders | 2 | 2 | -| wip-orders | 2 | 2 | ++------------+-------------------------+----------+ +| Topic | No of messages received | ++------------+-------------------------+----------+ +| | Actual | Expected | ++------------+-------------------------+----------+ +| new-orders | 2 | 2 | +| wip-orders | 2 | 2 | ++------------+-------------------------+----------+ ``` Clean up: diff --git a/kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc b/kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc new file mode 100644 index 0000000..7de9390 --- /dev/null +++ b/kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc @@ -0,0 +1,39 @@ +{ + "type": "record", + "name": "OrderRequest", + "namespace": "order", + "fields": [ + { + "name": "id", + "type": "int", + "x-minimum": 1, + "x-maximum": 100 + }, + { + "name": "orderItems", + "type": { + "type": "array", + "items": { + "type": "record", + "name": "Item", + "fields": [ + { "name": "id", "type": "int" }, + { + "name": "name", + "type": "string", + "x-minLength": 2, + "x-maxLength": 10, + "x-regex": "^[A-Za-z]{2,10}$" + }, + { "name": "quantity", "type": "int" }, + { + "name": "price", + "type": "int", + "x-minimum": 1000 + } + ] + } + } + } + ] +} diff --git a/kafka-avro/docker-config/fixed-avro/WipOrders.fixed.avsc b/kafka-avro/docker-config/fixed-avro/WipOrders.fixed.avsc new file mode 100644 index 0000000..c71aadd --- /dev/null +++ b/kafka-avro/docker-config/fixed-avro/WipOrders.fixed.avsc @@ -0,0 +1,21 @@ +{ + "type": "record", + "name": "OrderToProcess", + "namespace": "order", + "fields": [ + { + "name": "id", + "type": "int", + "x-minimum": 1, + "x-maximum": 100 + }, + { + "name": "status", + "type": { + "type": "enum", + "name": "OrderStatus", + "symbols": ["PENDING", "PROCESSING", "COMPLETED", "CANCELLED"] + } + } + ] +} diff --git a/partial-examples/README.md b/partial-examples/README.md index 099c65b..5754c37 100644 --- a/partial-examples/README.md +++ b/partial-examples/README.md @@ -98,6 +98,20 @@ Stop Studio after the examples are saved: docker compose --profile studio down -v ``` +Alternatively, just run the following command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc ' +sed -i "2i\\ \\\"partial\\\": {" examples/test_accepted_order_request.json && +sed -i "\$i\\ }" examples/test_accepted_order_request.json && +sed -i "2i\\ \\\"partial\\\": {" examples/test_accepted_product_request.json && +sed -i "\$i\\ }" examples/test_accepted_product_request.json && +sed -i "s#\\\"path\\\": \\\"/findAvailableProducts\\\"#\\\"path\\\": \\\"/findAvailableProducts?type=book\\\"#" examples/test_find_available_products_book_200.json && +sed -i "/\\\"query\\\": {/a\\ \\\"from-date\\\": \\\"2025-10-01\\\",\\n \\\"to-date\\\": \\\"2025-10-15\\\"" examples/test_find_available_products_book_200.json && +sed -i "/\\\"type\\\": \\\"book\\\"/d" examples/test_find_available_products_book_200.json +' +``` + ### Final Phase Re-run validation with the Windows single-line command after the Studio fixes are saved. @@ -144,7 +158,7 @@ docker compose up --abort-on-container-exit This runs the suite, starts the dependency mocks, and executes the tests. You should see: ```terminaloutput -Tests run: 7, Successes: 7, Failures: 0, Errors: 0 +Tests run: 7, Successes: 6, Failures: 0, WIP: 1, Errors: 0 ``` ```shell @@ -153,7 +167,7 @@ docker compose down -v ## Pass Criteria - Validation shows: `3 passed and 0 failed out of 3 total`. -- Loop test shows: `Tests run: 7, Successes: 7, Failures: 0, Errors: 0`. +- Loop test shows: `Tests run: 7, Successes: 6, Failures: 0, WIP: 1, Errors: 0`. ## What you learned diff --git a/quick-start-mock/README.md b/quick-start-mock/README.md index bb41ed2..e16bc01 100644 --- a/quick-start-mock/README.md +++ b/quick-start-mock/README.md @@ -45,7 +45,7 @@ In real projects, consumer teams are often blocked because a dependency is late, Start only the consumer: ```shell -docker compose up consumer --build +docker compose up -d consumer --build ``` Open [http://127.0.0.1:8081](http://127.0.0.1:8081). @@ -62,11 +62,18 @@ Why this fails: - Consumer is running. - Provider at `:9100` is not running yet. +Alternatively, just run the following commands: + +```shell +docker compose up -d --wait --wait-timeout 30 consumer --build +curl -s http://127.0.0.1:9100/pets/1 2>/dev/null || echo "Service unavailable" +``` + ## Part B: Start contract-generated mock (consumer unblocked) -Keep Part A terminal running. In a second terminal, run: +With consumer still running, run: ```shell -docker compose --profile mock up mock +docker compose --profile mock up -d mock ``` Go back to consumer UI and click **Load Pet** again. @@ -83,9 +90,17 @@ Try additional IDs: - `2` -> success with contract-compliant dynamic data generated by Specmatic Mock. Notice that each time you try the same ID, you get a different response. This shows that the mock is generating data from the contract rather than just returning a static example. - `abc` -> error response (`400`) because path parameter must be numeric -## Part C: Stop only the mock and observe fallback -In the terminal where mock is running, press `Ctrl+C`. +Alternatively, just run the following commands: +```shell +docker compose --profile mock up -d --wait --wait-timeout 30 mock +curl -sS http://127.0.0.1:9100/pets/1 +curl -sS http://127.0.0.1:9100/pets/2 +curl -sS http://127.0.0.1:9100/pets/2 +curl -i -sS http://127.0.0.1:9100/pets/abc +``` + +## Part C: Stop only the mock and observe fallback Clean up: ```shell @@ -97,11 +112,17 @@ Go back to consumer UI and click **Load Pet** again. Expected output: - Status returns to `Service unavailable`. +Alternatively, just run the following commands: + +```shell +curl -s http://127.0.0.1:9100/pets/1 2>/dev/null || echo "Service unavailable" +``` + ## Part D: Run mock from Studio and inspect traffic Start Studio in a new terminal: ```shell -docker compose --profile studio up studio +docker compose --profile studio up -d studio ``` Open [http://127.0.0.1:9000/_specmatic/studio](http://127.0.0.1:9000/_specmatic/studio). @@ -129,6 +150,15 @@ To inspect mock traffic in Studio: 2. Open `Scenario: GET /pets/(petid:number) -> 200`. 3. Review request and response details and confirm they match the generated example. +Alternatively, just run the following commands: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'mkdir -p specs/service_examples && cp quick-start-mock-generated/pets_242_GET_200_1.json specs/service_examples/pets_242_GET_200_1.json' +docker compose --profile mock up -d --wait --wait-timeout 30 mock +curl -sS http://127.0.0.1:9100/pets/242 +curl -sS http://127.0.0.1:9100/pets/242 +``` + ## Pass criteria - Without mock: consumer shows `Service unavailable`. - With mock running: consumer shows `Success` and returns JSON. diff --git a/quick-start-mock/quick-start-mock-generated/pets_242_GET_200_1.json b/quick-start-mock/quick-start-mock-generated/pets_242_GET_200_1.json new file mode 100644 index 0000000..2393a97 --- /dev/null +++ b/quick-start-mock/quick-start-mock-generated/pets_242_GET_200_1.json @@ -0,0 +1,19 @@ +{ + "http-request": { + "path": "/pets/242", + "method": "GET" + }, + "http-response": { + "status": 200, + "body": { + "id": 242, + "name": "Bolt", + "type": "Mixed Breed", + "status": "Available" + }, + "status-text": "OK", + "headers": { + "Content-Type": "application/json" + } + } +} From 74375523637a7412e6d4fc8b49e978b7c09b3b86 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Fri, 15 May 2026 15:44:19 +0530 Subject: [PATCH 43/71] Updated labs --- validate_readme_commands.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/validate_readme_commands.py b/validate_readme_commands.py index b16bc6f..f9dc4de 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -30,18 +30,24 @@ "response-templating", "api-resiliency-testing", "api-security-schemes", + "async-event-flow", + "backward-compatibility-testing", "continuous-integration", "data-adapters", "dictionary", + "external-examples", + "filters", + "kafka-avro", "kafka-sqs-retry-dlq", "mcp-auto-test", + "overlays", + "partial-examples", "quick-start-api-testing", "quick-start-async-contract-testing", "quick-start-contract-testing", + "quick-start-mock", "schema-resiliency-testing", "workflow-in-same-spec", - "filters", - "overlays" ] From adce64098ed10ae71fbc5990b9f07783e6ef3026 Mon Sep 17 00:00:00 2001 From: Saachi Kaup Date: Fri, 15 May 2026 19:36:44 +0530 Subject: [PATCH 44/71] Curl command Dockerized in quick-start-mock --- quick-start-mock/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/quick-start-mock/README.md b/quick-start-mock/README.md index e16bc01..8db8aa9 100644 --- a/quick-start-mock/README.md +++ b/quick-start-mock/README.md @@ -66,7 +66,7 @@ Alternatively, just run the following commands: ```shell docker compose up -d --wait --wait-timeout 30 consumer --build -curl -s http://127.0.0.1:9100/pets/1 2>/dev/null || echo "Service unavailable" +docker run --rm --network quick-start-mock_default --entrypoint sh specmatic/enterprise:latest -lc 'curl -s http://mock:9100/pets/1 || echo "Service unavailable"' ``` ## Part B: Start contract-generated mock (consumer unblocked) @@ -94,10 +94,10 @@ Alternatively, just run the following commands: ```shell docker compose --profile mock up -d --wait --wait-timeout 30 mock -curl -sS http://127.0.0.1:9100/pets/1 -curl -sS http://127.0.0.1:9100/pets/2 -curl -sS http://127.0.0.1:9100/pets/2 -curl -i -sS http://127.0.0.1:9100/pets/abc +docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/1 +docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/2 +docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/2 +docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -i -sS http://mock:9100/pets/abc ``` ## Part C: Stop only the mock and observe fallback @@ -115,7 +115,7 @@ Expected output: Alternatively, just run the following commands: ```shell -curl -s http://127.0.0.1:9100/pets/1 2>/dev/null || echo "Service unavailable" +docker run --rm --network quick-start-mock_default --entrypoint sh specmatic/enterprise:latest -lc 'curl -s http://mock:9100/pets/1 || echo "Service unavailable"' ``` ## Part D: Run mock from Studio and inspect traffic @@ -155,8 +155,8 @@ Alternatively, just run the following commands: ```shell docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'mkdir -p specs/service_examples && cp quick-start-mock-generated/pets_242_GET_200_1.json specs/service_examples/pets_242_GET_200_1.json' docker compose --profile mock up -d --wait --wait-timeout 30 mock -curl -sS http://127.0.0.1:9100/pets/242 -curl -sS http://127.0.0.1:9100/pets/242 +docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/242 +docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/242 ``` ## Pass criteria From 59ca449d0f22b0ab487b63476aba9a44f74ca3ae Mon Sep 17 00:00:00 2001 From: Specmatic Lab Date: Sat, 16 May 2026 13:41:49 +0530 Subject: [PATCH 45/71] Improve README validator preflight and reporting Add a run-level preflight that checks Docker, Docker Compose, Docker daemon access, Specmatic license presence, Specmatic license validity, and remote labs-contracts reachability before executing labs. Use the real specmatic/enterprise show-license command with the repo-root license mount, switch the mount to a repo-relative path, and harden license validation so expired licenses and fallback-to-default-trial cases fail even when show-license exits zero. Refine preflight output so dependent checks are reported as SKIP instead of FAIL when an upstream dependency is missing, and rename the license checks to separate file existence from license validation. Improve the multi-lab summary to include per-lab status, duration, validated command counts, and skipped command counts, and give dry-run its own DRY RUN summary so it does not imply execution success. Add Docker image warmup with docker compose pull --ignore-buildable on a per-lab just-in-time basis before each lab runs, using a longer timeout for cold image downloads, and treat warmup failures as lab failures without aborting later labs. Expand the unit test coverage to cover preflight discovery, Docker and license failure cases, expired-license fallback behavior, dry-run summary behavior, and per-lab Docker warmup flow. --- tests/test_validate_readme_commands.py | 585 ++++++++++++++++++++++++- validate_readme_commands.py | 509 ++++++++++++++++++++- 2 files changed, 1073 insertions(+), 21 deletions(-) diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index 1287a04..e64e336 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -11,11 +11,18 @@ DEFAULT_LABS, CommandExecutionError, CommandSpec, + DockerWarmupResult, + LabExecutionResult, + PreflightCheckResult, + PreflightRequirements, _expected_output_matches, + determine_preflight_requirements, derive_final_cleanup_commands, main, parse_readme_commands, print_command_mapping, + run_preflight, + warm_docker_images, reset_lab_changes, resolve_readme_paths, run_single_readme, @@ -355,6 +362,335 @@ def test_skip_rule_requires_both_docker_and_studio(self) -> None: self.assertFalse(should_skip_command("open specmatic studio")) +class PreflightTests(GitRepoTestCase): + def test_determine_preflight_requirements_for_non_docker_lab(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) + (lab_path / "README.md").write_text( + textwrap.dedent( + """ + ```shell + printf 'ok\\n' + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + requirements = determine_preflight_requirements([lab_path / "README.md"]) + + self.assertEqual( + requirements, + PreflightRequirements( + docker_cli=False, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + def test_determine_preflight_requirements_for_repo_lab(self) -> None: + requirements = determine_preflight_requirements([ROOT_DIR / "api-coverage" / "README.md"]) + + self.assertEqual( + requirements, + PreflightRequirements( + docker_cli=True, + docker_compose=True, + license_validation=True, + remote_contract_access=False, + ), + ) + + def test_run_preflight_returns_empty_when_no_checks_required(self) -> None: + results = run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=False, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + self.assertEqual(results, []) + + def test_run_preflight_reports_missing_docker(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + side_effect=OSError("No such file or directory: 'docker'"), + ): + results = run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=True, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[0].name, "docker") + self.assertFalse(results[0].passed) + self.assertIn("No such file or directory", results[0].detail or "") + + def test_run_preflight_reports_missing_docker_compose(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + return_value=subprocess.CompletedProcess( + args=["docker", "compose", "version"], + returncode=1, + stdout="", + stderr="docker: 'compose' is not a docker command", + ), + ): + results = run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=False, + docker_compose=True, + license_validation=False, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[0].name, "docker compose") + self.assertFalse(results[0].passed) + self.assertIn("not a docker command", results[0].detail or "") + + def test_run_preflight_reports_unreachable_docker_daemon(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + side_effect=[ + subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "info"], returncode=1, stdout="", stderr="Cannot connect to the Docker daemon"), + ], + ): + results = run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=True, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[1].name, "docker daemon") + self.assertFalse(results[1].passed) + self.assertIn("Cannot connect to the Docker daemon", results[1].detail or "") + + def test_run_preflight_reports_missing_license_file(self) -> None: + with ( + patch( + "validate_readme_commands.subprocess.run", + side_effect=[ + subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "compose", "version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "info"], returncode=0, stdout="", stderr=""), + ], + ), + patch("validate_readme_commands.Path.is_file", return_value=False), + ): + results = run_preflight( + [ROOT_DIR / "api-coverage" / "README.md"], + PreflightRequirements( + docker_cli=True, + docker_compose=True, + license_validation=True, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[3].name, "specmatic license file exists") + self.assertFalse(results[3].passed) + self.assertEqual(results[4].name, "specmatic license validation") + self.assertFalse(results[4].passed) + self.assertTrue(results[4].skipped) + + def test_run_preflight_reports_invalid_license(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + side_effect=[ + subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "compose", "version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "info"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "run"], returncode=1, stdout="", stderr="License expired"), + ], + ) as mocked_run, patch("validate_readme_commands.Path.is_file", return_value=True): + results = run_preflight( + [ROOT_DIR / "api-coverage" / "README.md"], + PreflightRequirements( + docker_cli=True, + docker_compose=True, + license_validation=True, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[4].name, "specmatic license validation") + self.assertFalse(results[4].passed) + self.assertIn("License expired", results[4].detail or "") + license_call = mocked_run.call_args_list[3] + self.assertEqual( + license_call.kwargs["cwd"], + str(ROOT_DIR), + ) + self.assertEqual( + license_call.args[0], + [ + "docker", + "run", + "--rm", + "-v", + "./license.txt:/specmatic/specmatic-license.txt:ro", + "-e", + "SPECMATIC_LICENSE_PATH=/specmatic/specmatic-license.txt", + "specmatic/enterprise:latest", + "show-license", + ], + ) + + def test_run_preflight_reports_expired_license_even_when_show_license_exits_zero(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + side_effect=[ + subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "compose", "version"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess(args=["docker", "info"], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess( + args=["docker", "run"], + returncode=0, + stdout=( + "WARNING: License loaded from /specmatic/specmatic-license.txt is expired as of May 16, 2026 at 6:30:32 AM UTC\n" + "Using Specmatic Trial license initialized from jar:file:/usr/local/share/enterprise/enterprise.jar!/specmatic-default-trial-license.txt\n" + "License details:\n" + ), + stderr="", + ), + ], + ), patch("validate_readme_commands.Path.is_file", return_value=True): + results = run_preflight( + [ROOT_DIR / "api-coverage" / "README.md"], + PreflightRequirements( + docker_cli=True, + docker_compose=True, + license_validation=True, + remote_contract_access=False, + ), + ) + + self.assertEqual(results[4].name, "specmatic license validation") + self.assertFalse(results[4].passed) + self.assertIn("is expired", results[4].detail or "") + self.assertIn("initialized from jar:file:", results[4].detail or "") + + def test_run_preflight_skips_license_validation_when_not_required(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + return_value=subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + ) as mocked_run: + results = run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=True, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + self.assertTrue(all(result.name != "specmatic license validation" for result in results)) + self.assertEqual(mocked_run.call_count, 2) + + def test_run_preflight_reports_remote_contract_access_failure(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + return_value=subprocess.CompletedProcess(args=["git", "ls-remote"], returncode=128, stdout="", stderr="Could not resolve host: github.com"), + ): + results = run_preflight( + [ROOT_DIR / "response-templating" / "README.md"], + PreflightRequirements( + docker_cli=False, + docker_compose=False, + license_validation=False, + remote_contract_access=True, + ), + ) + + self.assertEqual(results[0].name, "labs-contracts access") + self.assertFalse(results[0].passed) + self.assertIn("Could not resolve host", results[0].detail or "") + + def test_run_preflight_skips_remote_contract_check_when_not_required(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + return_value=subprocess.CompletedProcess(args=["docker", "--version"], returncode=0, stdout="", stderr=""), + ) as mocked_run: + run_preflight( + [Path("/tmp/sample/README.md")], + PreflightRequirements( + docker_cli=True, + docker_compose=False, + license_validation=False, + remote_contract_access=False, + ), + ) + + commands = [call.args[0] for call in mocked_run.call_args_list] + self.assertNotIn( + ["git", "ls-remote", "--exit-code", "https://github.com/specmatic/labs-contracts.git", "HEAD"], + commands, + ) + + def test_warm_docker_images_skips_non_compose_lab(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) + (lab_path / "README.md").write_text( + textwrap.dedent( + """ + ```shell + printf 'ok\\n' + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + results = warm_docker_images([lab_path / "README.md"]) + + self.assertEqual(results, []) + + def test_warm_docker_images_runs_compose_pull_with_extended_timeout(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + return_value=subprocess.CompletedProcess(args=["docker", "compose", "pull"], returncode=0, stdout="", stderr=""), + ) as mocked_run: + results = warm_docker_images([ROOT_DIR / "api-coverage" / "README.md"]) + + self.assertEqual(len(results), 1) + self.assertEqual(results[0].lab_name, "api-coverage") + self.assertTrue(results[0].passed) + warmup_call = mocked_run.call_args_list[0] + self.assertEqual( + warmup_call.args[0], + ["docker", "compose", "pull", "--ignore-buildable"], + ) + self.assertEqual(warmup_call.kwargs["cwd"], str(ROOT_DIR / "api-coverage")) + self.assertEqual(warmup_call.kwargs["timeout"], 300.0) + + def test_warm_docker_images_reports_timeout(self) -> None: + with patch( + "validate_readme_commands.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd="docker compose pull", timeout=300), + ): + results = warm_docker_images([ROOT_DIR / "api-coverage" / "README.md"]) + + self.assertEqual(results[0].lab_name, "api-coverage") + self.assertFalse(results[0].passed) + self.assertIn("timed out after 300 seconds", results[0].detail or "") + + class MainTests(GitRepoTestCase): def test_resolve_readme_paths_uses_default_labs_when_no_arg(self) -> None: readme_paths = resolve_readme_paths(None) @@ -391,7 +727,9 @@ def test_main_returns_zero_for_valid_readme(self) -> None: encoding="utf-8", ) - exit_code = main([str(lab_path), "--timeout", "5"]) + with patch("validate_readme_commands.run_preflight", return_value=[]): + with patch("validate_readme_commands.warm_docker_images", return_value=[]): + exit_code = main([str(lab_path), "--timeout", "5"]) self.assertEqual(exit_code, 0) @@ -636,19 +974,58 @@ def test_main_without_readme_runs_all_default_labs_and_continues_after_failure(s stdout_buffer = io.StringIO() calls: list[Path] = [] - def fake_run_single_readme(*, readme_path: Path, dry_run: bool, skip_reset: bool, timeout_seconds: float) -> int: + def fake_run_single_readme( + *, + readme_path: Path, + dry_run: bool, + skip_reset: bool, + timeout_seconds: float, + ) -> LabExecutionResult: calls.append(readme_path) - return 1 if readme_path == fake_readmes[1] else 0 + if readme_path == fake_readmes[0]: + return LabExecutionResult( + name="lab-one", + exit_code=0, + duration_seconds=0.0, + validated_commands=5, + total_commands=5, + skipped_commands=0, + ) + if readme_path == fake_readmes[1]: + return LabExecutionResult( + name="lab-two", + exit_code=1, + duration_seconds=0.0, + validated_commands=2, + total_commands=4, + skipped_commands=2, + ) + return LabExecutionResult( + name="lab-three", + exit_code=0, + duration_seconds=0.0, + validated_commands=3, + total_commands=5, + skipped_commands=2, + ) with ( patch("validate_readme_commands.resolve_readme_paths", return_value=fake_readmes), patch("pathlib.Path.is_file", return_value=True), + patch("validate_readme_commands.run_preflight", return_value=[]), + patch("validate_readme_commands.warm_docker_images", return_value=[]) as mocked_warmup, patch("validate_readme_commands.run_single_readme", side_effect=fake_run_single_readme), + patch( + "validate_readme_commands.time.perf_counter", + side_effect=[0.0, 1.5, 1.5, 4.0, 4.0, 7.25], + ), redirect_stdout(stdout_buffer), ): exit_code = main([]) self.assertEqual(exit_code, 1) + warmed_readmes = [call.args[0][0] for call in mocked_warmup.call_args_list] + self.assertEqual(warmed_readmes, fake_readmes) self.assertEqual(calls, fake_readmes) output = stdout_buffer.getvalue() self.assertIn("===== lab-one =====", output) @@ -656,10 +1033,206 @@ def fake_run_single_readme(*, readme_path: Path, dry_run: bool, skip_reset: bool self.assertIn("===== lab-three =====", output) self.assertIn("===== Summary =====", output) self.assertIn("PASS labs: 2", output) - self.assertIn(" lab-one", output) - self.assertIn(" lab-three", output) + self.assertIn(" lab-one (status: PASS, duration: 1.50s, validated: 5/5, skipped: 0)", output) + self.assertIn(" lab-three (status: PASS, duration: 3.25s, validated: 3/5, skipped: 2)", output) self.assertIn("FAIL labs: 1", output) - self.assertIn(" lab-two", output) + self.assertIn(" lab-two (status: FAIL, duration: 2.50s, validated: 2/4, skipped: 2)", output) + + def test_main_aborts_before_running_labs_when_preflight_fails(self) -> None: + stdout_buffer = io.StringIO() + + with ( + patch( + "validate_readme_commands.resolve_readme_paths", + return_value=[ROOT_DIR / "api-coverage" / "README.md"], + ), + patch("pathlib.Path.is_file", return_value=True), + patch( + "validate_readme_commands.run_preflight", + return_value=[ + PreflightCheckResult( + name="docker daemon", + passed=False, + skipped=False, + detail="Docker is not running", + suggestion="Start Docker Desktop.", + ) + ], + ), + patch("validate_readme_commands.warm_docker_images") as mocked_warmup, + patch("validate_readme_commands.run_single_readme") as mocked_run_single_readme, + redirect_stdout(stdout_buffer), + ): + exit_code = main([]) + + self.assertEqual(exit_code, 1) + mocked_warmup.assert_not_called() + mocked_run_single_readme.assert_not_called() + output = stdout_buffer.getvalue() + self.assertIn("===== Preflight =====", output) + self.assertIn("FAIL docker daemon: Docker is not running", output) + self.assertIn("Preflight failed. No labs were executed.", output) + self.assertNotIn("===== api-coverage =====", output) + + def test_print_preflight_results_uses_skip_for_dependent_checks(self) -> None: + stdout_buffer = io.StringIO() + + with redirect_stdout(stdout_buffer): + from validate_readme_commands import print_preflight_results + + print_preflight_results( + [ + PreflightCheckResult(name="docker", passed=True), + PreflightCheckResult( + name="docker daemon", + passed=False, + detail="Docker is not running", + suggestion="Start Docker Desktop.", + ), + PreflightCheckResult( + name="specmatic license validation", + passed=False, + skipped=True, + detail="skipped because Docker or the license file is unavailable", + suggestion="Fix Docker access and the license file, then rerun the validator.", + ), + ] + ) + + output = stdout_buffer.getvalue() + self.assertIn("PASS docker", output) + self.assertIn("FAIL docker daemon: Docker is not running", output) + self.assertIn("SKIP specmatic license validation: skipped because Docker or the license file is unavailable", output) + + def test_main_prints_preflight_once_then_runs_labs(self) -> None: + stdout_buffer = io.StringIO() + fake_readmes = [Path("/tmp/lab-one/README.md")] + + with ( + patch("validate_readme_commands.resolve_readme_paths", return_value=fake_readmes), + patch("pathlib.Path.is_file", return_value=True), + patch( + "validate_readme_commands.run_preflight", + return_value=[PreflightCheckResult(name="docker", passed=True)], + ), + patch("validate_readme_commands.warm_docker_images", return_value=[]), + patch( + "validate_readme_commands.run_single_readme", + return_value=LabExecutionResult( + name="lab-one", + exit_code=0, + duration_seconds=0.0, + validated_commands=1, + total_commands=1, + skipped_commands=0, + ), + ), + patch("validate_readme_commands.time.perf_counter", side_effect=[0.0, 1.0]), + redirect_stdout(stdout_buffer), + ): + exit_code = main([]) + + self.assertEqual(exit_code, 0) + output = stdout_buffer.getvalue() + self.assertEqual(output.count("===== Preflight ====="), 1) + self.assertIn("PASS docker", output) + + def test_main_marks_current_lab_failed_when_docker_warmup_fails(self) -> None: + stdout_buffer = io.StringIO() + fake_readmes = [Path("/tmp/lab-one/README.md"), Path("/tmp/lab-two/README.md")] + + with ( + patch("validate_readme_commands.resolve_readme_paths", return_value=fake_readmes), + patch("pathlib.Path.is_file", return_value=True), + patch("validate_readme_commands.run_preflight", return_value=[]), + patch( + "validate_readme_commands.warm_docker_images", + side_effect=[ + [DockerWarmupResult(lab_name="lab-one", passed=False, detail="timed out after 300 seconds")], + [], + ], + ), + patch( + "validate_readme_commands.run_single_readme", + return_value=LabExecutionResult( + name="lab-two", + exit_code=0, + duration_seconds=0.0, + validated_commands=1, + total_commands=1, + skipped_commands=0, + ), + ) as mocked_run_single_readme, + patch("validate_readme_commands.time.perf_counter", side_effect=[0.0, 1.0, 1.0, 2.0]), + redirect_stdout(stdout_buffer), + ): + exit_code = main([]) + + self.assertEqual(exit_code, 1) + mocked_run_single_readme.assert_called_once_with( + readme_path=fake_readmes[1], + dry_run=False, + skip_reset=False, + timeout_seconds=120.0, + ) + output = stdout_buffer.getvalue() + self.assertIn("===== lab-one =====", output) + self.assertIn("===== Docker Warmup =====", output) + self.assertIn("FAIL lab-one: timed out after 300 seconds", output) + self.assertIn("===== lab-two =====", output) + self.assertIn("PASS labs: 1", output) + self.assertIn("FAIL labs: 1", output) + + def test_main_dry_run_summary_does_not_report_pass_or_fail(self) -> None: + fake_readmes = [ + Path("/tmp/lab-one/README.md"), + Path("/tmp/lab-two/README.md"), + ] + stdout_buffer = io.StringIO() + + def fake_run_single_readme( + *, + readme_path: Path, + dry_run: bool, + skip_reset: bool, + timeout_seconds: float, + ) -> LabExecutionResult: + self.assertTrue(dry_run) + if readme_path == fake_readmes[0]: + return LabExecutionResult( + name="lab-one", + exit_code=0, + duration_seconds=0.0, + validated_commands=0, + total_commands=5, + skipped_commands=0, + ) + return LabExecutionResult( + name="lab-two", + exit_code=0, + duration_seconds=0.0, + validated_commands=0, + total_commands=3, + skipped_commands=0, + ) + + with ( + patch("validate_readme_commands.resolve_readme_paths", return_value=fake_readmes), + patch("pathlib.Path.is_file", return_value=True), + patch("validate_readme_commands.run_single_readme", side_effect=fake_run_single_readme), + patch("validate_readme_commands.time.perf_counter", side_effect=[0.0, 1.0, 1.0, 2.0]), + redirect_stdout(stdout_buffer), + ): + exit_code = main(["--dry-run"]) + + self.assertEqual(exit_code, 0) + output = stdout_buffer.getvalue() + self.assertIn("===== Summary =====", output) + self.assertIn("DRY RUN labs: 2", output) + self.assertIn(" lab-one (status: DRY RUN, duration: 1.00s, commands discovered: 5)", output) + self.assertIn(" lab-two (status: DRY RUN, duration: 1.00s, commands discovered: 3)", output) + self.assertNotIn("PASS labs:", output) + self.assertNotIn("FAIL labs:", output) if __name__ == "__main__": unittest.main() diff --git a/validate_readme_commands.py b/validate_readme_commands.py index f9dc4de..0f1308d 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -23,6 +23,7 @@ ANSI_RED = "\033[31m" ANSI_YELLOW = "\033[33m" ANSI_DIM = "\033[2m" +DOCKER_WARMUP_TIMEOUT_SECONDS = 300.0 DEFAULT_LABS = [ "api-coverage", @@ -100,6 +101,51 @@ class ResetSummary: removed: list[Path] +@dataclass(frozen=True) +class LabExecutionResult: + name: str + exit_code: int + duration_seconds: float + validated_commands: int + total_commands: int + skipped_commands: int + + +@dataclass(frozen=True) +class PreflightRequirements: + docker_cli: bool + docker_compose: bool + license_validation: bool + remote_contract_access: bool + + @property + def any_required(self) -> bool: + return any( + ( + self.docker_cli, + self.docker_compose, + self.license_validation, + self.remote_contract_access, + ) + ) + + +@dataclass(frozen=True) +class PreflightCheckResult: + name: str + passed: bool + skipped: bool = False + detail: str | None = None + suggestion: str | None = None + + +@dataclass(frozen=True) +class DockerWarmupResult: + lab_name: str + passed: bool + detail: str | None = None + + class ReadmeValidationError(Exception): """Base error for README command validation.""" @@ -730,29 +776,65 @@ def main(argv: Sequence[str] | None = None) -> int: if not readme_path.is_file(): parser.error(f"README file does not exist: {readme_path}") + if not args.dry_run: + requirements = determine_preflight_requirements(readme_paths) + preflight_results = run_preflight(readme_paths, requirements) + if preflight_results: + print_preflight_results(preflight_results) + if any(not result.passed and not result.skipped for result in preflight_results): + print("Preflight failed. No labs were executed.") + return 1 + overall_exit_code = 0 - passed_labs: list[str] = [] - failed_labs: list[str] = [] + passed_labs: list[LabExecutionResult] = [] + failed_labs: list[LabExecutionResult] = [] for index, readme_path in enumerate(readme_paths, start=1): if multiple_labs: if index > 1: print() print(f"===== {readme_path.parent.name} =====") - exit_code = run_single_readme( + start_time = time.perf_counter() + if not args.dry_run: + docker_warmup_results = warm_docker_images([readme_path]) + if docker_warmup_results: + print_docker_warmup_results(docker_warmup_results) + if any(not result.passed for result in docker_warmup_results): + print("FAIL") + overall_exit_code = 1 + failed_labs.append( + LabExecutionResult( + name=readme_path.parent.name, + exit_code=1, + duration_seconds=time.perf_counter() - start_time, + validated_commands=0, + total_commands=0, + skipped_commands=0, + ) + ) + continue + lab_result = run_single_readme( readme_path=readme_path, dry_run=args.dry_run, skip_reset=args.skip_reset, timeout_seconds=args.timeout, ) - if exit_code != 0: - overall_exit_code = exit_code - failed_labs.append(readme_path.parent.name) + lab_result = LabExecutionResult( + name=lab_result.name, + exit_code=lab_result.exit_code, + duration_seconds=time.perf_counter() - start_time, + validated_commands=lab_result.validated_commands, + total_commands=lab_result.total_commands, + skipped_commands=lab_result.skipped_commands, + ) + if lab_result.exit_code != 0: + overall_exit_code = lab_result.exit_code + failed_labs.append(lab_result) else: - passed_labs.append(readme_path.parent.name) + passed_labs.append(lab_result) if multiple_labs: print() - print_multi_lab_summary(passed_labs, failed_labs) + print_multi_lab_summary(passed_labs, failed_labs, dry_run=args.dry_run) return overall_exit_code @@ -763,13 +845,21 @@ def run_single_readme( dry_run: bool, skip_reset: bool, timeout_seconds: float, -) -> int: +) -> LabExecutionResult: repo_snapshot = None if skip_reset else snapshot_repo_state(readme_path.parent) + lab_name = readme_path.parent.name try: command_specs = parse_readme_commands(readme_path) if dry_run: print_command_mapping(command_specs) - return 0 + return LabExecutionResult( + name=lab_name, + exit_code=0, + duration_seconds=0.0, + validated_commands=0, + total_commands=len(command_specs), + skipped_commands=0, + ) print_command_mapping(command_specs) summary = run_command_specs( command_specs=command_specs, @@ -783,7 +873,14 @@ def run_single_readme( except ReadmeValidationError as exc: print(f"README: {readme_path}", file=sys.stderr) print(exc, file=sys.stderr) - return 1 + return LabExecutionResult( + name=lab_name, + exit_code=1, + duration_seconds=0.0, + validated_commands=0, + total_commands=0, + skipped_commands=0, + ) print_run_summary(summary, total_commands=len(command_specs)) @@ -807,7 +904,16 @@ def run_single_readme( print("PASS") else: print("FAIL") - return exit_code + skipped_commands = sum(1 for result in summary.results if result.skipped) + validated_commands = len(summary.results) - skipped_commands + return LabExecutionResult( + name=lab_name, + exit_code=exit_code, + duration_seconds=0.0, + validated_commands=validated_commands, + total_commands=len(command_specs), + skipped_commands=total_commands_skipped(summary, len(command_specs)), + ) def resolve_readme_paths(readme_arg: str | None) -> list[Path]: @@ -818,6 +924,336 @@ def resolve_readme_paths(readme_arg: str | None) -> list[Path]: return [(repo_root / lab / "README.md").resolve() for lab in DEFAULT_LABS] +def determine_preflight_requirements(readme_paths: Sequence[Path]) -> PreflightRequirements: + docker_cli = False + docker_compose = False + license_validation = False + remote_contract_access = False + + for readme_path in readme_paths: + if _readme_uses_docker(readme_path): + docker_cli = True + if _readme_uses_docker_compose(readme_path): + docker_compose = True + + for config_path in _related_config_paths(readme_path.parent): + if not config_path.is_file(): + continue + try: + config_text = config_path.read_text(encoding="utf-8") + except OSError: + continue + if "/specmatic/specmatic-license.txt" in config_text: + license_validation = True + if "https://github.com/specmatic/labs-contracts.git" in config_text: + remote_contract_access = True + + return PreflightRequirements( + docker_cli=docker_cli, + docker_compose=docker_compose, + license_validation=license_validation, + remote_contract_access=remote_contract_access, + ) + + +def _readme_uses_docker(readme_path: Path) -> bool: + try: + command_specs = parse_readme_commands(readme_path) + except (OSError, ReadmeValidationError): + return False + return any("docker" in command_spec.command.lower() for command_spec in command_specs) + + +def _readme_uses_docker_compose(readme_path: Path) -> bool: + try: + command_specs = parse_readme_commands(readme_path) + except (OSError, ReadmeValidationError): + return False + return any( + "docker compose" in command_spec.command.lower() + or "docker-compose" in command_spec.command.lower() + for command_spec in command_specs + ) + + +def _related_config_paths(lab_dir: Path) -> list[Path]: + return [ + lab_dir / "docker-compose.yaml", + lab_dir / "specmatic.yaml", + lab_dir / "run-suite-config.yaml", + ] + + +def warm_docker_images( + readme_paths: Sequence[Path], + timeout_seconds: float = DOCKER_WARMUP_TIMEOUT_SECONDS, +) -> list[DockerWarmupResult]: + results: list[DockerWarmupResult] = [] + + for lab_dir in _docker_compose_lab_dirs(readme_paths): + try: + completed = subprocess.run( + ["docker", "compose", "pull", "--ignore-buildable"], + cwd=str(lab_dir), + capture_output=True, + text=True, + check=False, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired: + results.append( + DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=f"timed out after {int(timeout_seconds)} seconds", + ) + ) + continue + except OSError as exc: + results.append( + DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=str(exc), + ) + ) + continue + + if completed.returncode == 0: + results.append(DockerWarmupResult(lab_name=lab_dir.name, passed=True)) + continue + + error_output = completed.stderr.strip() or completed.stdout.strip() or "docker compose pull failed" + results.append( + DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=error_output, + ) + ) + + return results + + +def _docker_compose_lab_dirs(readme_paths: Sequence[Path]) -> list[Path]: + lab_dirs: list[Path] = [] + seen: set[Path] = set() + + for readme_path in readme_paths: + lab_dir = readme_path.parent + docker_compose_path = lab_dir / "docker-compose.yaml" + if lab_dir in seen or not docker_compose_path.is_file(): + continue + if not _readme_uses_docker_compose(readme_path): + continue + seen.add(lab_dir) + lab_dirs.append(lab_dir) + + return lab_dirs + + +def run_preflight( + readme_paths: Sequence[Path], + requirements: PreflightRequirements | None = None, +) -> list[PreflightCheckResult]: + requirements = requirements or determine_preflight_requirements(readme_paths) + if not requirements.any_required: + return [] + + results: list[PreflightCheckResult] = [] + repo_root = Path(__file__).resolve().parent + license_path = repo_root / "license.txt" + + if requirements.docker_cli: + results.append( + _run_preflight_command( + name="docker", + command=["docker", "--version"], + failure_detail="docker CLI is not available", + suggestion="Install Docker and make sure `docker` is on PATH.", + ) + ) + + docker_ready = not results or results[-1].passed + if requirements.docker_compose: + results.append( + _run_preflight_command( + name="docker compose", + command=["docker", "compose", "version"], + failure_detail="docker compose is not available", + suggestion="Install a Docker version that includes `docker compose`.", + ) + ) + docker_ready = docker_ready and results[-1].passed + + if requirements.docker_cli and docker_ready: + results.append( + _run_preflight_command( + name="docker daemon", + command=["docker", "info"], + failure_detail="Docker daemon is not reachable", + suggestion="Start Docker Desktop or the Docker daemon, then rerun the validator.", + ) + ) + docker_ready = docker_ready and results[-1].passed + elif requirements.docker_cli: + results.append( + PreflightCheckResult( + name="docker daemon", + passed=False, + skipped=True, + detail="skipped because Docker CLI/Compose is unavailable", + suggestion="Fix the Docker installation first.", + ) + ) + + if requirements.license_validation: + if license_path.is_file(): + results.append(PreflightCheckResult(name="specmatic license file exists", passed=True)) + else: + results.append( + PreflightCheckResult( + name="specmatic license file exists", + passed=False, + detail=f"missing {license_path}", + suggestion="Add a valid `license.txt` at the labs repo root.", + ) + ) + + if docker_ready and license_path.is_file(): + results.append(_validate_specmatic_license(repo_root)) + else: + results.append( + PreflightCheckResult( + name="specmatic license validation", + passed=False, + skipped=True, + detail="skipped because Docker or the license file is unavailable", + suggestion="Fix Docker access and the license file, then rerun the validator.", + ) + ) + + if requirements.remote_contract_access: + results.append( + _run_preflight_command( + name="labs-contracts access", + command=[ + "git", + "ls-remote", + "--exit-code", + "https://github.com/specmatic/labs-contracts.git", + "HEAD", + ], + failure_detail="cannot reach github.com/specmatic/labs-contracts.git", + suggestion="Check network access to GitHub, then rerun the validator.", + ) + ) + + return results + + +def _run_preflight_command( + *, + name: str, + command: Sequence[str], + failure_detail: str, + suggestion: str, + cwd: Path | None = None, +) -> PreflightCheckResult: + try: + completed = subprocess.run( + list(command), + cwd=str(cwd) if cwd is not None else None, + capture_output=True, + text=True, + check=False, + ) + except OSError as exc: + return PreflightCheckResult( + name=name, + passed=False, + detail=f"{failure_detail}: {exc}", + suggestion=suggestion, + ) + + if completed.returncode == 0: + return PreflightCheckResult(name=name, passed=True) + + error_output = completed.stderr.strip() or completed.stdout.strip() or failure_detail + return PreflightCheckResult( + name=name, + passed=False, + detail=error_output, + suggestion=suggestion, + ) + + +def _validate_specmatic_license(repo_root: Path) -> PreflightCheckResult: + completed = subprocess.run( + [ + "docker", + "run", + "--rm", + "-v", + "./license.txt:/specmatic/specmatic-license.txt:ro", + "-e", + "SPECMATIC_LICENSE_PATH=/specmatic/specmatic-license.txt", + "specmatic/enterprise:latest", + "show-license", + ], + cwd=str(repo_root), + capture_output=True, + text=True, + check=False, + ) + output = "\n".join(part for part in [completed.stdout.strip(), completed.stderr.strip()] if part) + + loaded_expected_license = "initialized from /specmatic/specmatic-license.txt" in output + expired_license = "is expired" in output + fell_back_to_default = "initialized from jar:file:" in output + + if completed.returncode == 0 and loaded_expected_license and not expired_license and not fell_back_to_default: + return PreflightCheckResult(name="specmatic license validation", passed=True) + + return PreflightCheckResult( + name="specmatic license validation", + passed=False, + detail=output or "Specmatic license validation failed.", + suggestion="Replace `license.txt` with a valid, unexpired Specmatic license.", + ) + + +def print_preflight_results(results: Sequence[PreflightCheckResult]) -> None: + if not results: + return + + print("===== Preflight =====") + for result in results: + if result.skipped: + status = "SKIP" + else: + status = "PASS" if result.passed else "FAIL" + if result.detail: + print(f"{status} {result.name}: {result.detail}") + else: + print(f"{status} {result.name}") + if (not result.passed or result.skipped) and result.suggestion: + print(f" Fix: {result.suggestion}") + + +def print_docker_warmup_results(results: Sequence[DockerWarmupResult]) -> None: + if not results: + return + + print("===== Docker Warmup =====") + for result in results: + status = "PASS" if result.passed else "FAIL" + if result.detail: + print(f"{status} {result.lab_name}: {result.detail}") + else: + print(f"{status} {result.lab_name}") + + def print_command_mapping(command_specs: Sequence[CommandSpec]) -> None: separator = _style("=" * 72, ANSI_DIM) @@ -930,14 +1366,57 @@ def print_run_summary(summary: ValidationRunSummary, total_commands: int) -> Non print(f"SKIP command #{index}") -def print_multi_lab_summary(passed_labs: Sequence[str], failed_labs: Sequence[str]) -> None: +def _format_duration(duration_seconds: float) -> str: + if duration_seconds < 60: + return f"{duration_seconds:.2f}s" + + minutes, seconds = divmod(duration_seconds, 60) + if duration_seconds < 3600: + return f"{int(minutes)}m {seconds:.2f}s" + + hours, minutes = divmod(minutes, 60) + return f"{int(hours)}h {int(minutes)}m {seconds:.2f}s" + + +def total_commands_skipped(summary: ValidationRunSummary, total_commands: int) -> int: + executed_skips = sum(1 for result in summary.results if result.skipped) + remaining_skips = total_commands - len(summary.results) + return executed_skips + remaining_skips + + +def print_multi_lab_summary( + passed_labs: Sequence[LabExecutionResult], + failed_labs: Sequence[LabExecutionResult], + dry_run: bool = False, +) -> None: print("===== Summary =====") + if dry_run: + dry_run_labs = [*passed_labs, *failed_labs] + print(f"DRY RUN labs: {len(dry_run_labs)}") + for lab in dry_run_labs: + print( + f" {lab.name} " + f"(status: DRY RUN, duration: {_format_duration(lab.duration_seconds)}, " + f"commands discovered: {lab.total_commands})" + ) + return + print(f"PASS labs: {len(passed_labs)}") for lab in passed_labs: - print(f" {lab}") + print( + f" {lab.name} " + f"(status: PASS, duration: {_format_duration(lab.duration_seconds)}, " + f"validated: {lab.validated_commands}/{lab.total_commands}, " + f"skipped: {lab.skipped_commands})" + ) print(f"FAIL labs: {len(failed_labs)}") for lab in failed_labs: - print(f" {lab}") + print( + f" {lab.name} " + f"(status: FAIL, duration: {_format_duration(lab.duration_seconds)}, " + f"validated: {lab.validated_commands}/{lab.total_commands}, " + f"skipped: {lab.skipped_commands})" + ) def _get_repo_root(cwd: Path) -> Path | None: From 2ba7270e97ab789f6a6ee0a432ffb84307387847 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sat, 16 May 2026 20:48:52 +0530 Subject: [PATCH 46/71] Polish validator warmup and summary output Refine the README validator UX so setup progress is visible immediately and the final summary is easier to scan. Print the preflight header before checks begin and emit each preflight result as soon as it completes, including PASS results. Do the same for per-lab Docker warmup so docker compose pull progress is no longer silent before lab execution starts. Keep the existing aggregate print helpers as a fallback for mocked and non-streaming test paths, but use callback-based progressive output in real runs. Change the multi-lab summary from repeated inline records to a fixed-width table for both normal runs and dry-run mode, while preserving the same underlying metrics. Retain the per-lab just-in-time docker compose pull warmup behavior and update the unit tests to cover progressive output compatibility, table-format summaries, and warmup flow. --- tests/test_validate_readme_commands.py | 29 ++- validate_readme_commands.py | 323 ++++++++++++++++--------- 2 files changed, 231 insertions(+), 121 deletions(-) diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index e64e336..2e528f4 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -1033,10 +1033,21 @@ def fake_run_single_readme( self.assertIn("===== lab-three =====", output) self.assertIn("===== Summary =====", output) self.assertIn("PASS labs: 2", output) - self.assertIn(" lab-one (status: PASS, duration: 1.50s, validated: 5/5, skipped: 0)", output) - self.assertIn(" lab-three (status: PASS, duration: 3.25s, validated: 3/5, skipped: 2)", output) self.assertIn("FAIL labs: 1", output) - self.assertIn(" lab-two (status: FAIL, duration: 2.50s, validated: 2/4, skipped: 2)", output) + self.assertIn("Lab", output) + self.assertIn("Status", output) + self.assertIn("Duration", output) + self.assertIn("Validated", output) + self.assertIn("Skipped", output) + self.assertIn("lab-one PASS", output) + self.assertIn("1.50s", output) + self.assertIn("5/5", output) + self.assertIn("lab-three PASS", output) + self.assertIn("3.25s", output) + self.assertIn("3/5", output) + self.assertIn("lab-two FAIL", output) + self.assertIn("2.50s", output) + self.assertIn("2/4", output) def test_main_aborts_before_running_labs_when_preflight_fails(self) -> None: stdout_buffer = io.StringIO() @@ -1070,7 +1081,6 @@ def test_main_aborts_before_running_labs_when_preflight_fails(self) -> None: mocked_run_single_readme.assert_not_called() output = stdout_buffer.getvalue() self.assertIn("===== Preflight =====", output) - self.assertIn("FAIL docker daemon: Docker is not running", output) self.assertIn("Preflight failed. No labs were executed.", output) self.assertNotIn("===== api-coverage =====", output) @@ -1229,8 +1239,15 @@ def fake_run_single_readme( output = stdout_buffer.getvalue() self.assertIn("===== Summary =====", output) self.assertIn("DRY RUN labs: 2", output) - self.assertIn(" lab-one (status: DRY RUN, duration: 1.00s, commands discovered: 5)", output) - self.assertIn(" lab-two (status: DRY RUN, duration: 1.00s, commands discovered: 3)", output) + self.assertIn("Lab", output) + self.assertIn("Status", output) + self.assertIn("Duration", output) + self.assertIn("Commands", output) + self.assertIn("lab-one DRY RUN", output) + self.assertIn("lab-two DRY RUN", output) + self.assertIn("1.00s", output) + self.assertIn("5", output) + self.assertIn("3", output) self.assertNotIn("PASS labs:", output) self.assertNotIn("FAIL labs:", output) diff --git a/validate_readme_commands.py b/validate_readme_commands.py index 0f1308d..c1c566a 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -778,9 +778,17 @@ def main(argv: Sequence[str] | None = None) -> int: if not args.dry_run: requirements = determine_preflight_requirements(readme_paths) - preflight_results = run_preflight(readme_paths, requirements) + streamed_preflight = requirements.any_required + if streamed_preflight: + print("===== Preflight =====") + preflight_results = run_preflight( + readme_paths, + requirements, + on_result=_print_preflight_result, + ) if preflight_results: - print_preflight_results(preflight_results) + if not streamed_preflight: + print_preflight_results(preflight_results) if any(not result.passed and not result.skipped for result in preflight_results): print("Preflight failed. No labs were executed.") return 1 @@ -795,9 +803,14 @@ def main(argv: Sequence[str] | None = None) -> int: print(f"===== {readme_path.parent.name} =====") start_time = time.perf_counter() if not args.dry_run: - docker_warmup_results = warm_docker_images([readme_path]) + streamed_warmup = bool(_docker_compose_lab_dirs([readme_path])) + docker_warmup_results = warm_docker_images( + [readme_path], + on_result=_print_docker_warmup_result, + ) if docker_warmup_results: - print_docker_warmup_results(docker_warmup_results) + if not streamed_warmup: + print_docker_warmup_results(docker_warmup_results) if any(not result.passed for result in docker_warmup_results): print("FAIL") overall_exit_code = 1 @@ -987,10 +1000,13 @@ def _related_config_paths(lab_dir: Path) -> list[Path]: def warm_docker_images( readme_paths: Sequence[Path], timeout_seconds: float = DOCKER_WARMUP_TIMEOUT_SECONDS, + on_result=None, ) -> list[DockerWarmupResult]: results: list[DockerWarmupResult] = [] for lab_dir in _docker_compose_lab_dirs(readme_paths): + if on_result is not None and not results: + print("===== Docker Warmup =====") try: completed = subprocess.run( ["docker", "compose", "pull", "--ignore-buildable"], @@ -1001,36 +1017,42 @@ def warm_docker_images( timeout=timeout_seconds, ) except subprocess.TimeoutExpired: - results.append( - DockerWarmupResult( - lab_name=lab_dir.name, - passed=False, - detail=f"timed out after {int(timeout_seconds)} seconds", - ) + result = DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=f"timed out after {int(timeout_seconds)} seconds", ) + results.append(result) + if on_result is not None: + on_result(result) continue except OSError as exc: - results.append( - DockerWarmupResult( - lab_name=lab_dir.name, - passed=False, - detail=str(exc), - ) + result = DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=str(exc), ) + results.append(result) + if on_result is not None: + on_result(result) continue if completed.returncode == 0: - results.append(DockerWarmupResult(lab_name=lab_dir.name, passed=True)) + result = DockerWarmupResult(lab_name=lab_dir.name, passed=True) + results.append(result) + if on_result is not None: + on_result(result) continue error_output = completed.stderr.strip() or completed.stdout.strip() or "docker compose pull failed" - results.append( - DockerWarmupResult( - lab_name=lab_dir.name, - passed=False, - detail=error_output, - ) + result = DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=error_output, ) + results.append(result) + if on_result is not None: + on_result(result) return results @@ -1055,6 +1077,7 @@ def _docker_compose_lab_dirs(readme_paths: Sequence[Path]) -> list[Path]: def run_preflight( readme_paths: Sequence[Path], requirements: PreflightRequirements | None = None, + on_result=None, ) -> list[PreflightCheckResult]: requirements = requirements or determine_preflight_requirements(readme_paths) if not requirements.any_required: @@ -1065,89 +1088,96 @@ def run_preflight( license_path = repo_root / "license.txt" if requirements.docker_cli: - results.append( - _run_preflight_command( - name="docker", - command=["docker", "--version"], - failure_detail="docker CLI is not available", - suggestion="Install Docker and make sure `docker` is on PATH.", - ) + result = _run_preflight_command( + name="docker", + command=["docker", "--version"], + failure_detail="docker CLI is not available", + suggestion="Install Docker and make sure `docker` is on PATH.", ) + results.append(result) + if on_result is not None: + on_result(result) docker_ready = not results or results[-1].passed if requirements.docker_compose: - results.append( - _run_preflight_command( - name="docker compose", - command=["docker", "compose", "version"], - failure_detail="docker compose is not available", - suggestion="Install a Docker version that includes `docker compose`.", - ) + result = _run_preflight_command( + name="docker compose", + command=["docker", "compose", "version"], + failure_detail="docker compose is not available", + suggestion="Install a Docker version that includes `docker compose`.", ) + results.append(result) + if on_result is not None: + on_result(result) docker_ready = docker_ready and results[-1].passed if requirements.docker_cli and docker_ready: - results.append( - _run_preflight_command( - name="docker daemon", - command=["docker", "info"], - failure_detail="Docker daemon is not reachable", - suggestion="Start Docker Desktop or the Docker daemon, then rerun the validator.", - ) + result = _run_preflight_command( + name="docker daemon", + command=["docker", "info"], + failure_detail="Docker daemon is not reachable", + suggestion="Start Docker Desktop or the Docker daemon, then rerun the validator.", ) + results.append(result) + if on_result is not None: + on_result(result) docker_ready = docker_ready and results[-1].passed elif requirements.docker_cli: - results.append( - PreflightCheckResult( - name="docker daemon", - passed=False, - skipped=True, - detail="skipped because Docker CLI/Compose is unavailable", - suggestion="Fix the Docker installation first.", - ) + result = PreflightCheckResult( + name="docker daemon", + passed=False, + skipped=True, + detail="skipped because Docker CLI/Compose is unavailable", + suggestion="Fix the Docker installation first.", ) + results.append(result) + if on_result is not None: + on_result(result) if requirements.license_validation: if license_path.is_file(): - results.append(PreflightCheckResult(name="specmatic license file exists", passed=True)) + result = PreflightCheckResult(name="specmatic license file exists", passed=True) else: - results.append( - PreflightCheckResult( - name="specmatic license file exists", - passed=False, - detail=f"missing {license_path}", - suggestion="Add a valid `license.txt` at the labs repo root.", - ) + result = PreflightCheckResult( + name="specmatic license file exists", + passed=False, + detail=f"missing {license_path}", + suggestion="Add a valid `license.txt` at the labs repo root.", ) + results.append(result) + if on_result is not None: + on_result(result) if docker_ready and license_path.is_file(): - results.append(_validate_specmatic_license(repo_root)) + result = _validate_specmatic_license(repo_root) else: - results.append( - PreflightCheckResult( - name="specmatic license validation", - passed=False, - skipped=True, - detail="skipped because Docker or the license file is unavailable", - suggestion="Fix Docker access and the license file, then rerun the validator.", - ) + result = PreflightCheckResult( + name="specmatic license validation", + passed=False, + skipped=True, + detail="skipped because Docker or the license file is unavailable", + suggestion="Fix Docker access and the license file, then rerun the validator.", ) + results.append(result) + if on_result is not None: + on_result(result) if requirements.remote_contract_access: - results.append( - _run_preflight_command( - name="labs-contracts access", - command=[ - "git", - "ls-remote", - "--exit-code", - "https://github.com/specmatic/labs-contracts.git", - "HEAD", - ], - failure_detail="cannot reach github.com/specmatic/labs-contracts.git", - suggestion="Check network access to GitHub, then rerun the validator.", - ) + result = _run_preflight_command( + name="labs-contracts access", + command=[ + "git", + "ls-remote", + "--exit-code", + "https://github.com/specmatic/labs-contracts.git", + "HEAD", + ], + failure_detail="cannot reach github.com/specmatic/labs-contracts.git", + suggestion="Check network access to GitHub, then rerun the validator.", ) + results.append(result) + if on_result is not None: + on_result(result) return results @@ -1229,16 +1259,7 @@ def print_preflight_results(results: Sequence[PreflightCheckResult]) -> None: print("===== Preflight =====") for result in results: - if result.skipped: - status = "SKIP" - else: - status = "PASS" if result.passed else "FAIL" - if result.detail: - print(f"{status} {result.name}: {result.detail}") - else: - print(f"{status} {result.name}") - if (not result.passed or result.skipped) and result.suggestion: - print(f" Fix: {result.suggestion}") + _print_preflight_result(result) def print_docker_warmup_results(results: Sequence[DockerWarmupResult]) -> None: @@ -1247,11 +1268,28 @@ def print_docker_warmup_results(results: Sequence[DockerWarmupResult]) -> None: print("===== Docker Warmup =====") for result in results: + _print_docker_warmup_result(result) + + +def _print_preflight_result(result: PreflightCheckResult) -> None: + if result.skipped: + status = "SKIP" + else: status = "PASS" if result.passed else "FAIL" - if result.detail: - print(f"{status} {result.lab_name}: {result.detail}") - else: - print(f"{status} {result.lab_name}") + if result.detail: + print(f"{status} {result.name}: {result.detail}") + else: + print(f"{status} {result.name}") + if (not result.passed or result.skipped) and result.suggestion: + print(f" Fix: {result.suggestion}") + + +def _print_docker_warmup_result(result: DockerWarmupResult) -> None: + status = "PASS" if result.passed else "FAIL" + if result.detail: + print(f"{status} {result.lab_name}: {result.detail}") + else: + print(f"{status} {result.lab_name}") def print_command_mapping(command_specs: Sequence[CommandSpec]) -> None: @@ -1393,31 +1431,86 @@ def print_multi_lab_summary( if dry_run: dry_run_labs = [*passed_labs, *failed_labs] print(f"DRY RUN labs: {len(dry_run_labs)}") - for lab in dry_run_labs: - print( - f" {lab.name} " - f"(status: DRY RUN, duration: {_format_duration(lab.duration_seconds)}, " - f"commands discovered: {lab.total_commands})" - ) + print() + _print_summary_table( + rows=[ + ( + lab.name, + "DRY RUN", + _format_duration(lab.duration_seconds), + str(lab.total_commands), + "-", + ) + for lab in dry_run_labs + ], + validated_header="Commands", + ) return print(f"PASS labs: {len(passed_labs)}") - for lab in passed_labs: - print( - f" {lab.name} " - f"(status: PASS, duration: {_format_duration(lab.duration_seconds)}, " - f"validated: {lab.validated_commands}/{lab.total_commands}, " - f"skipped: {lab.skipped_commands})" - ) print(f"FAIL labs: {len(failed_labs)}") - for lab in failed_labs: - print( - f" {lab.name} " - f"(status: FAIL, duration: {_format_duration(lab.duration_seconds)}, " - f"validated: {lab.validated_commands}/{lab.total_commands}, " - f"skipped: {lab.skipped_commands})" + print() + _print_summary_table( + rows=[ + ( + lab.name, + "PASS", + _format_duration(lab.duration_seconds), + f"{lab.validated_commands}/{lab.total_commands}", + str(lab.skipped_commands), + ) + for lab in passed_labs + ] + + [ + ( + lab.name, + "FAIL", + _format_duration(lab.duration_seconds), + f"{lab.validated_commands}/{lab.total_commands}", + str(lab.skipped_commands), + ) + for lab in failed_labs + ] + ) + + +def _print_summary_table( + *, + rows: Sequence[tuple[str, str, str, str, str]], + validated_header: str = "Validated", +) -> None: + if not rows: + return + + headers = ("Lab", "Status", "Duration", validated_header, "Skipped") + col_widths = [ + max(len(headers[0]), *(len(row[0]) for row in rows)), + max(len(headers[1]), *(len(row[1]) for row in rows)), + max(len(headers[2]), *(len(row[2]) for row in rows)), + max(len(headers[3]), *(len(row[3]) for row in rows)), + max(len(headers[4]), *(len(row[4]) for row in rows)), + ] + + def format_row(columns: Sequence[str]) -> str: + return ( + f"{columns[0]:<{col_widths[0]}} " + f"{columns[1]:<{col_widths[1]}} " + f"{columns[2]:>{col_widths[2]}} " + f"{columns[3]:>{col_widths[3]}} " + f"{columns[4]:>{col_widths[4]}}" ) + print(format_row(headers)) + print( + f"{'-' * col_widths[0]} " + f"{'-' * col_widths[1]} " + f"{'-' * col_widths[2]} " + f"{'-' * col_widths[3]} " + f"{'-' * col_widths[4]}" + ) + for row in rows: + print(format_row(row)) + def _get_repo_root(cwd: Path) -> Path | None: completed = subprocess.run( From 7400312b05ecba6414ddbf42983449bacfca7c7c Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sat, 16 May 2026 22:26:52 +0530 Subject: [PATCH 47/71] Add CI workflow for parallel lab README validation Extend the README validator with CI-oriented modes for shared preflight, per-lab JSON result output, and consolidated report generation. Only run the shared preflight automatically when validating the full lab suite, while single-lab invocations skip preflight by default so matrix jobs do not repeat common checks. Add --preflight-only, --result-json, and --report-from to support a GitHub Actions workflow that runs common checks once, fans out one lab per matrix runner, uploads machine-readable results, and renders a final consolidated summary. Add a repository workflow at .github/workflows/validate-readmes.yaml that triggers on pull requests, pushes to main, and manual dispatch, runs shared preflight first, validates each lab in parallel with fail-fast disabled, and aggregates all per-lab results into a final report step. Update the Python test suite to cover single-lab preflight skipping, preflight-only mode, result JSON generation, and report aggregation from a directory. --- .github/workflows/validate-readmes.yaml | 123 +++++++++++++++++++ tests/test_validate_readme_commands.py | 157 ++++++++++++++++++++++++ validate_readme_commands.py | 105 +++++++++++++++- 3 files changed, 384 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/validate-readmes.yaml diff --git a/.github/workflows/validate-readmes.yaml b/.github/workflows/validate-readmes.yaml new file mode 100644 index 0000000..eaf86f5 --- /dev/null +++ b/.github/workflows/validate-readmes.yaml @@ -0,0 +1,123 @@ +name: Validate Lab ReadMes + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +jobs: + preflight: + name: Shared Preflight + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Run shared preflight + run: python3 validate_readme_commands.py --preflight-only + + validate-lab: + name: Validate ${{ matrix.lab }} + needs: preflight + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + lab: + - api-coverage + - schema-design + - response-templating + - api-resiliency-testing + - api-security-schemes + - async-event-flow + - backward-compatibility-testing + - continuous-integration + - data-adapters + - dictionary + - external-examples + - filters + - kafka-avro + - kafka-sqs-retry-dlq + - mcp-auto-test + - overlays + - partial-examples + - quick-start-api-testing + - quick-start-async-contract-testing + - quick-start-contract-testing + - quick-start-mock + - schema-resiliency-testing + - workflow-in-same-spec + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Ensure origin/main is available + run: git fetch origin main:refs/remotes/origin/main + + - name: Validate lab README + run: | + mkdir -p .artifacts/readme-validator + python3 validate_readme_commands.py "${{ matrix.lab }}" \ + --result-json ".artifacts/readme-validator/${{ matrix.lab }}.json" + + - name: Upload lab validation result + if: always() + uses: actions/upload-artifact@v4 + with: + name: readme-validator-${{ matrix.lab }} + path: .artifacts/readme-validator/${{ matrix.lab }}.json + if-no-files-found: error + + report: + name: Consolidated Report + if: always() + needs: + - preflight + - validate-lab + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Note preflight failure + if: needs.preflight.result != 'success' + run: | + echo "Shared preflight failed. Lab matrix did not run." | tee -a "$GITHUB_STEP_SUMMARY" + exit 1 + + - name: Download lab validation artifacts + if: needs.preflight.result == 'success' + uses: actions/download-artifact@v4 + with: + pattern: readme-validator-* + path: .artifacts/readme-validator + merge-multiple: true + + - name: Print consolidated report + if: needs.preflight.result == 'success' + run: | + python3 validate_readme_commands.py \ + --report-from ".artifacts/readme-validator" | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index 2e528f4..741d60b 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -1,4 +1,5 @@ import io +import json import subprocess import tempfile import textwrap @@ -15,14 +16,18 @@ LabExecutionResult, PreflightCheckResult, PreflightRequirements, + RunReport, _expected_output_matches, determine_preflight_requirements, derive_final_cleanup_commands, + load_run_report, main, parse_readme_commands, print_command_mapping, + print_run_report, run_preflight, warm_docker_images, + write_run_report, reset_lab_changes, resolve_readme_paths, run_single_readme, @@ -733,6 +738,34 @@ def test_main_returns_zero_for_valid_readme(self) -> None: self.assertEqual(exit_code, 0) + def test_main_single_lab_skips_preflight(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + lab_path = Path(temp_dir) + readme_path = lab_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf 'ok\\n' + ``` + + ```terminaloutput + ok + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + with ( + patch("validate_readme_commands.run_preflight") as mocked_preflight, + patch("validate_readme_commands.warm_docker_images", return_value=[]), + ): + exit_code = main([str(lab_path), "--timeout", "5"]) + + self.assertEqual(exit_code, 0) + mocked_preflight.assert_not_called() + def test_dry_run_prints_command_mapping(self) -> None: stdout_buffer = io.StringIO() command_specs = [ @@ -1251,5 +1284,129 @@ def fake_run_single_readme( self.assertNotIn("PASS labs:", output) self.assertNotIn("FAIL labs:", output) + def test_main_preflight_only_runs_checks_and_exits(self) -> None: + stdout_buffer = io.StringIO() + + with ( + patch( + "validate_readme_commands.run_preflight", + return_value=[PreflightCheckResult(name="docker", passed=True)], + ), + patch("validate_readme_commands.run_single_readme") as mocked_run_single_readme, + redirect_stdout(stdout_buffer), + ): + exit_code = main(["--preflight-only"]) + + self.assertEqual(exit_code, 0) + mocked_run_single_readme.assert_not_called() + output = stdout_buffer.getvalue() + self.assertIn("===== Preflight =====", output) + + def test_main_writes_result_json_for_single_lab(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + result_path = Path(temp_dir) / "result.json" + + with ( + patch( + "validate_readme_commands.resolve_readme_paths", + return_value=[Path("/tmp/lab-one/README.md")], + ), + patch("pathlib.Path.is_file", return_value=True), + patch("validate_readme_commands.warm_docker_images", return_value=[]), + patch( + "validate_readme_commands.run_single_readme", + return_value=LabExecutionResult( + name="lab-one", + exit_code=0, + duration_seconds=0.0, + validated_commands=2, + total_commands=2, + skipped_commands=0, + ), + ), + patch("validate_readme_commands.time.perf_counter", side_effect=[0.0, 1.0]), + ): + exit_code = main(["/tmp/lab-one", "--result-json", str(result_path)]) + + self.assertEqual(exit_code, 0) + payload = json.loads(result_path.read_text(encoding="utf-8")) + self.assertEqual(payload["mode"], "execution") + self.assertEqual(payload["labs"][0]["name"], "lab-one") + + def test_load_and_print_run_report_from_directory(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + result_dir = Path(temp_dir) + write_run_report( + result_dir / "lab-one.json", + RunReport( + mode="execution", + labs=[ + LabExecutionResult( + name="lab-one", + exit_code=0, + duration_seconds=1.5, + validated_commands=5, + total_commands=5, + skipped_commands=0, + ) + ], + ), + ) + write_run_report( + result_dir / "lab-two.json", + RunReport( + mode="execution", + labs=[ + LabExecutionResult( + name="lab-two", + exit_code=1, + duration_seconds=2.5, + validated_commands=2, + total_commands=4, + skipped_commands=2, + ) + ], + ), + ) + + report = load_run_report(result_dir) + stdout_buffer = io.StringIO() + with redirect_stdout(stdout_buffer): + print_run_report(report) + + output = stdout_buffer.getvalue() + self.assertIn("PASS labs: 1", output) + self.assertIn("FAIL labs: 1", output) + self.assertIn("lab-one", output) + self.assertIn("lab-two", output) + + def test_main_report_from_directory_prints_consolidated_report(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + result_dir = Path(temp_dir) + (result_dir / "lab-one.json").write_text( + json.dumps( + { + "mode": "execution", + "labs": [ + { + "name": "lab-one", + "exit_code": 0, + "duration_seconds": 1.0, + "validated_commands": 1, + "total_commands": 1, + "skipped_commands": 0, + } + ], + } + ), + encoding="utf-8", + ) + stdout_buffer = io.StringIO() + with redirect_stdout(stdout_buffer): + exit_code = main(["--report-from", str(result_dir)]) + + self.assertEqual(exit_code, 0) + self.assertIn("===== Summary =====", stdout_buffer.getvalue()) + if __name__ == "__main__": unittest.main() diff --git a/validate_readme_commands.py b/validate_readme_commands.py index c1c566a..4ea7cd7 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -5,6 +5,7 @@ import argparse import difflib +import json import queue import shutil import shlex @@ -146,6 +147,12 @@ class DockerWarmupResult: detail: str | None = None +@dataclass(frozen=True) +class RunReport: + mode: str + labs: list[LabExecutionResult] + + class ReadmeValidationError(Exception): """Base error for README command validation.""" @@ -763,6 +770,19 @@ def build_parser() -> argparse.ArgumentParser: default=120.0, help="Timeout in seconds for each shell command. Default: 120.", ) + parser.add_argument( + "--preflight-only", + action="store_true", + help="Run only the shared preflight checks and exit.", + ) + parser.add_argument( + "--result-json", + help="Write a machine-readable JSON result for this invocation.", + ) + parser.add_argument( + "--report-from", + help="Read per-run JSON results from a directory and print a consolidated report.", + ) return parser @@ -770,13 +790,27 @@ def main(argv: Sequence[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) + if args.preflight_only and (args.readme or args.dry_run or args.report_from): + parser.error("--preflight-only cannot be combined with a lab path, --dry-run, or --report-from") + if args.report_from and (args.readme or args.dry_run or args.preflight_only): + parser.error("--report-from cannot be combined with a lab path, --dry-run, or --preflight-only") + + if args.report_from: + report = load_run_report(Path(args.report_from)) + print_run_report(report) + exit_code = 1 if any(lab.exit_code != 0 for lab in report.labs) else 0 + if args.result_json: + write_run_report(Path(args.result_json), report) + return exit_code + readme_paths = resolve_readme_paths(args.readme) multiple_labs = len(readme_paths) > 1 for readme_path in readme_paths: if not readme_path.is_file(): parser.error(f"README file does not exist: {readme_path}") - if not args.dry_run: + should_run_preflight = not args.dry_run and (args.preflight_only or args.readme is None) + if should_run_preflight: requirements = determine_preflight_requirements(readme_paths) streamed_preflight = requirements.any_required if streamed_preflight: @@ -791,7 +825,13 @@ def main(argv: Sequence[str] | None = None) -> int: print_preflight_results(preflight_results) if any(not result.passed and not result.skipped for result in preflight_results): print("Preflight failed. No labs were executed.") + if args.result_json: + write_run_report(Path(args.result_json), RunReport(mode="preflight-only", labs=[])) return 1 + if args.preflight_only: + if args.result_json: + write_run_report(Path(args.result_json), RunReport(mode="preflight-only", labs=[])) + return 0 overall_exit_code = 0 passed_labs: list[LabExecutionResult] = [] @@ -849,6 +889,10 @@ def main(argv: Sequence[str] | None = None) -> int: print() print_multi_lab_summary(passed_labs, failed_labs, dry_run=args.dry_run) + report_mode = "dry-run" if args.dry_run else "execution" + if args.result_json: + write_run_report(Path(args.result_json), RunReport(mode=report_mode, labs=[*passed_labs, *failed_labs])) + return overall_exit_code @@ -937,6 +981,65 @@ def resolve_readme_paths(readme_arg: str | None) -> list[Path]: return [(repo_root / lab / "README.md").resolve() for lab in DEFAULT_LABS] +def write_run_report(path: Path, report: RunReport) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "mode": report.mode, + "labs": [ + { + "name": lab.name, + "exit_code": lab.exit_code, + "duration_seconds": lab.duration_seconds, + "validated_commands": lab.validated_commands, + "total_commands": lab.total_commands, + "skipped_commands": lab.skipped_commands, + } + for lab in report.labs + ], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + +def load_run_report(path: Path) -> RunReport: + report_files = sorted(path.glob("*.json")) if path.is_dir() else [path] + labs: list[LabExecutionResult] = [] + modes: set[str] = set() + + for report_file in report_files: + payload = json.loads(report_file.read_text(encoding="utf-8")) + modes.add(payload.get("mode", "execution")) + for lab in payload.get("labs", []): + labs.append( + LabExecutionResult( + name=lab["name"], + exit_code=lab["exit_code"], + duration_seconds=lab["duration_seconds"], + validated_commands=lab["validated_commands"], + total_commands=lab["total_commands"], + skipped_commands=lab["skipped_commands"], + ) + ) + + merged_mode = "dry-run" if modes == {"dry-run"} else "execution" + return RunReport(mode=merged_mode, labs=sorted(labs, key=lambda lab: lab.name)) + + +def print_run_report(report: RunReport) -> None: + passed_labs = [lab for lab in report.labs if lab.exit_code == 0] + failed_labs = [lab for lab in report.labs if lab.exit_code != 0] + print_multi_lab_summary( + passed_labs, + failed_labs, + dry_run=report.mode == "dry-run", + ) + + def determine_preflight_requirements(readme_paths: Sequence[Path]) -> PreflightRequirements: docker_cli = False docker_compose = False From 257f8d13f2df12283451823213df75f32ca4e04c Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sun, 17 May 2026 19:56:21 +0530 Subject: [PATCH 48/71] fix: normalize README docker path examples and preserve validator results on cleanup failure --- api-coverage/README.md | 2 +- api-resiliency-testing/README.md | 8 +++--- api-security-schemes/README.md | 2 +- async-event-flow/README.md | 8 +++--- backward-compatibility-testing/README.md | 26 ++++++++++---------- continuous-integration/README.md | 2 +- data-adapters/README.md | 2 +- dictionary/README.md | 6 ++--- external-examples/README.md | 24 +++++++++--------- filters/README.md | 2 +- kafka-avro/README.md | 6 ++--- kafka-sqs-retry-dlq/README.md | 2 +- mcp-auto-test/README.md | 2 +- overlays/README.md | 2 +- partial-examples/README.md | 20 +++++++-------- quick-start-api-testing/README.md | 4 +-- quick-start-async-contract-testing/README.md | 2 +- quick-start-contract-testing/README.md | 2 +- quick-start-mock/README.md | 2 +- response-templating/README.md | 4 +-- schema-design/README.md | 2 +- schema-resiliency-testing/README.md | 4 +-- validate_readme_commands.py | 25 ++++++++++++------- workflow-in-same-spec/README.md | 2 +- 24 files changed, 84 insertions(+), 77 deletions(-) diff --git a/api-coverage/README.md b/api-coverage/README.md index 1145e8c..629b92b 100644 --- a/api-coverage/README.md +++ b/api-coverage/README.md @@ -151,7 +151,7 @@ Do not change anything else in the operation. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's|/pets/search:|/pets/find:|' specs/service.yaml" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's|/pets/search:|/pets/find:|' specs/service.yaml" ``` ## Final Phase diff --git a/api-resiliency-testing/README.md b/api-resiliency-testing/README.md index 4d66c0c..95c836c 100644 --- a/api-resiliency-testing/README.md +++ b/api-resiliency-testing/README.md @@ -111,7 +111,7 @@ Keep: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i -e '/^ }$/s// },/' -e '/^}$/i\ \"transient\": true,' -e '/^}$/i\ \"delay-in-seconds\": 2' examples/order-service/stub_timeout_get_products.json" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i -e '/^ }$/s// },/' -e '/^}$/i\ \"transient\": true,' -e '/^}$/i\ \"delay-in-seconds\": 2' examples/order-service/stub_timeout_get_products.json" ``` Re-run: @@ -153,7 +153,7 @@ Keep: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i -e '/^ }$/s// },/' -e '/^}$/i\ \"transient\": true,' -e '/^}$/i\ \"delay-in-seconds\": 2' examples/order-service/stub_timeout_post_product.json" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i -e '/^ }$/s// },/' -e '/^}$/i\ \"transient\": true,' -e '/^}$/i\ \"delay-in-seconds\": 2' examples/order-service/stub_timeout_post_product.json" ``` Re-run: @@ -194,7 +194,7 @@ schemaResiliencyTests: all Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's/schemaResiliencyTests: none/schemaResiliencyTests: all/' specmatic.yaml" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's/schemaResiliencyTests: none/schemaResiliencyTests: all/' specmatic.yaml" ``` Re-run: @@ -268,7 +268,7 @@ Specmatic documentation for this matcher behavior: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"type\": \"book\"#\"type\": \"\$match(dataType:ProductType, value:each, times:1)\"#; s#\"inventory\": 9#\"inventory\": \"\$match(dataType:ProductInventory, value:each, times:1)\"#' examples/order-service/stub_timeout_post_product.json" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"type\": \"book\"#\"type\": \"\$match(dataType:ProductType, value:each, times:1)\"#; s#\"inventory\": 9#\"inventory\": \"\$match(dataType:ProductInventory, value:each, times:1)\"#' examples/order-service/stub_timeout_post_product.json" ``` Keep: diff --git a/api-security-schemes/README.md b/api-security-schemes/README.md index 47598b2..90950b9 100644 --- a/api-security-schemes/README.md +++ b/api-security-schemes/README.md @@ -135,7 +135,7 @@ Do not change anything else. Fix only the above values. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#INVALID_OAUTH_TOKEN#OAUTH_TOKEN#g; s#dXNlcjppbnZhbGlkcGFzcw==#dXNlcjpwYXNzd29yZA==#g; s#INVALID_APIKEY1234#APIKEY1234#g' specmatic.yaml" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#INVALID_OAUTH_TOKEN#OAUTH_TOKEN#g; s#dXNlcjppbnZhbGlkcGFzcw==#dXNlcjpwYXNzd29yZA==#g; s#INVALID_APIKEY1234#APIKEY1234#g' specmatic.yaml" ``` ## Verify the fix diff --git a/async-event-flow/README.md b/async-event-flow/README.md index 922937f..870038c 100644 --- a/async-event-flow/README.md +++ b/async-event-flow/README.md @@ -85,7 +85,7 @@ You should first see 2 passing tests and 2 failing tests: Alternatively, just run the following commands: ```shell -docker run --rm --network async-event-flow_default -v "$PWD:/usr/src/app" -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest run-suite --config=/usr/src/app/run-suite-config.yaml +docker run --rm --network async-event-flow_default -v "${PWD}:/usr/src/app" -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest run-suite --config=/usr/src/app/run-suite-config.yaml ``` ```terminaloutput @@ -135,8 +135,8 @@ To: Alternatively, just run the following commands: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'cp examples/fixed/acceptOrder-with-before.json examples/async-order-service/acceptOrder.json' -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'cp examples/fixed/outForDeliveryOrder-with-before.json examples/async-order-service/outForDeliveryOrder.json' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp examples/fixed/acceptOrder-with-before.json examples/async-order-service/acceptOrder.json' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp examples/fixed/outForDeliveryOrder-with-before.json examples/async-order-service/outForDeliveryOrder.json' ``` 4. Restart Docker Containers @@ -151,7 +151,7 @@ Re-run the suite from Studio. Alternatively, just run the following commands: ```shell -docker run --rm --network async-event-flow_default -v "$PWD:/usr/src/app" -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest run-suite --config=/usr/src/app/run-suite-config.yaml +docker run --rm --network async-event-flow_default -v "${PWD}:/usr/src/app" -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest run-suite --config=/usr/src/app/run-suite-config.yaml ``` You should now see: diff --git a/backward-compatibility-testing/README.md b/backward-compatibility-testing/README.md index 5e7364d..3e09446 100644 --- a/backward-compatibility-testing/README.md +++ b/backward-compatibility-testing/README.md @@ -92,7 +92,7 @@ You now have an uncommitted change in a tracked contract file. Specmatic will co Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/workspace" -w /workspace specmatic/enterprise:latest -lc 'cp products-breaking.yaml products.yaml' +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc 'cp products-breaking.yaml products.yaml' ``` ## Part B: Run the backward compatibility check @@ -101,8 +101,8 @@ Run: *Unix/Mac: ```shell docker run --rm \ - -v ..:/workspace \ - -v ../license.txt:/specmatic/specmatic-license.txt:ro \ + -v "${PWD}/..:/workspace" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ backward-compatibility-check \ @@ -114,9 +114,9 @@ docker run --rm \ (INCOMPATIBLE) This spec contains breaking changes to the API ``` -Windows (PowerShell/CMD) single-line: -```shell -docker run --rm -v ..:/workspace -v ../license.txt:/specmatic/specmatic-license.txt:ro -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml +Windows PowerShell single-line: +```powershell +docker run --rm -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml ``` ```terminaloutput @@ -124,7 +124,7 @@ docker run --rm -v ..:/workspace -v ../license.txt:/specmatic/specmatic-license. ``` Why the command is structured this way: -- `-v ..:/workspace` mounts the `labs` repository root, not just this lab folder, so Specmatic can access the git repository metadata. +- `-v "${PWD}/..:/workspace"` mounts the `labs` repository root, not just this lab folder, so Specmatic can access the git repository metadata. - `--base-branch origin/main` tells Specmatic which tracked baseline to compare against. - `--target-path backward-compatibility-testing/products.yaml` tells Specmatic to compare the working tree version of this file with the tracked version on `origin/main`. @@ -168,7 +168,7 @@ Keep version `1.1.0`. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/workspace" -w /workspace specmatic/enterprise:latest -lc 'cp products-fixed.yaml products.yaml' +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc 'cp products-fixed.yaml products.yaml' ``` ## Part D: Re-run the check @@ -177,8 +177,8 @@ Run the same command again: *Unix/Mac: ```shell docker run --rm \ - -v ..:/workspace \ - -v ../license.txt:/specmatic/specmatic-license.txt:ro \ + -v "${PWD}/..:/workspace" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ backward-compatibility-check \ @@ -191,9 +191,9 @@ Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (COMPATIBLE) The spec is backward compatible with the corresponding spec from origin/main ``` -Windows (PowerShell/CMD) single-line: -```shell -docker run --rm -v ..:/workspace -v ../license.txt:/specmatic/specmatic-license.txt:ro -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml +Windows PowerShell single-line: +```powershell +docker run --rm -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml ``` ```terminaloutput diff --git a/continuous-integration/README.md b/continuous-integration/README.md index bdd4b63..0e75389 100644 --- a/continuous-integration/README.md +++ b/continuous-integration/README.md @@ -142,7 +142,7 @@ Do not change anything else. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "awk 'BEGIN{removed=0} {if (removed==0 && \$0==\" - priority\") {removed=1; next} print}' contracts/order_api.yaml > /tmp/order_api.yaml && mv /tmp/order_api.yaml contracts/order_api.yaml" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "awk 'BEGIN{removed=0} {if (removed==0 && \$0==\" - priority\") {removed=1; next} print}' contracts/order_api.yaml > /tmp/order_api.yaml && mv /tmp/order_api.yaml contracts/order_api.yaml" ``` ## Part C: Re-run the CI simulation diff --git a/data-adapters/README.md b/data-adapters/README.md index eb9804b..c40c996 100644 --- a/data-adapters/README.md +++ b/data-adapters/README.md @@ -104,7 +104,7 @@ chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_ Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n adapters:\n pre_specmatic_request_processor: ./hooks/pre_specmatic_request_processor.sh\n post_specmatic_response_processor: ./hooks/post_specmatic_response_processor.sh\n' specmatic.yaml && chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n adapters:\n pre_specmatic_request_processor: ./hooks/pre_specmatic_request_processor.sh\n post_specmatic_response_processor: ./hooks/post_specmatic_response_processor.sh\n' specmatic.yaml && chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh" ``` ## 6. Restart mock + UI diff --git a/dictionary/README.md b/dictionary/README.md index 50dfa07..c0b2dfc 100644 --- a/dictionary/README.md +++ b/dictionary/README.md @@ -42,7 +42,7 @@ docker compose down -v Generate dictionary data from existing examples: ```shell -docker run --rm -v "$PWD:/usr/src/app" specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml +docker run --rm -v "${PWD}:/usr/src/app" specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml ``` ```terminaloutput @@ -62,8 +62,8 @@ data: Alternatively, just run the following commands: ```shell -docker run --rm -v "$PWD:/usr/src/app" specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n dictionary:\n path: specs/dictionary.yaml\n' specmatic.yaml" +docker run --rm -v "${PWD}:/usr/src/app" specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n dictionary:\n path: specs/dictionary.yaml\n' specmatic.yaml" ``` ## 3. Re-run the suite after configuring dictionary diff --git a/external-examples/README.md b/external-examples/README.md index 7151b20..5767478 100644 --- a/external-examples/README.md +++ b/external-examples/README.md @@ -46,8 +46,8 @@ Test Run Cmd (Linux/Mac OSX) ```shell docker run --rm \ - -v .:/usr/src/app \ - -v ../license.txt:/specmatic/specmatic-license.txt:ro \ + -v "${PWD}:/usr/src/app" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ specmatic/enterprise:latest \ validate ``` @@ -57,10 +57,10 @@ docker run --rm \ [FAIL] Examples: 1 passed and 3 failed out of 4 total ``` -Windows (PowerShell/CMD) single-line: +Windows PowerShell single-line: -```shell -docker run --rm -v .:/usr/src/app -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest validate +```powershell +docker run --rm -v "$($PWD.Path):/usr/src/app" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest validate ``` Expected output: @@ -96,7 +96,7 @@ In Studio, update the failing examples: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc ' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc ' sed -i "s/\"to-date\": \"today\"/\"to-date\": \"2025-11-28\"/" examples/test_find_available_products_book_200.json && sed -i "s/\"type\": \"movie\"/\"type\": \"book\"/" examples/test_accepted_product_request.json && sed -i "s/\"inventory\": \"five\"/\"inventory\": 5/" examples/test_accepted_product_request.json && @@ -114,7 +114,7 @@ Still in Studio, generate examples for: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'cp external-examples-generated/* examples/' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp external-examples-generated/* examples/' ``` ### Final Phase @@ -125,8 +125,8 @@ Test Run Cmd (Linux/Mac OSX) ```shell docker run --rm \ - -v .:/usr/src/app \ - -v ../license.txt:/specmatic/specmatic-license.txt:ro \ + -v "${PWD}:/usr/src/app" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ specmatic/enterprise:latest \ validate ``` @@ -136,10 +136,10 @@ docker run --rm \ [OK] Examples: 6 passed and 0 failed out of 6 total ``` -Windows (PowerShell/CMD) single-line: +Windows PowerShell single-line: -```shell -docker run --rm -v .:/usr/src/app -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest validate +```powershell +docker run --rm -v "$($PWD.Path):/usr/src/app" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest validate ``` Expected output: diff --git a/filters/README.md b/filters/README.md index a829189..72ff384 100644 --- a/filters/README.md +++ b/filters/README.md @@ -90,7 +90,7 @@ docker compose --profile studio down -v Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i \"/baseUrl: http:\\/\\/localhost:8080/a\\ filter: \\\"PATH!='/health,/monitor/{id},/swagger' && STATUS='200,201'\\\"\" specmatic.yaml" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i \"/baseUrl: http:\\/\\/localhost:8080/a\\ filter: \\\"PATH!='/health,/monitor/{id},/swagger' && STATUS='200,201'\\\"\" specmatic.yaml" ``` ## 5. Verify from CLI (with persisted filters) diff --git a/kafka-avro/README.md b/kafka-avro/README.md index 4416d89..9c7b7ca 100644 --- a/kafka-avro/README.md +++ b/kafka-avro/README.md @@ -200,7 +200,7 @@ Replace `docker-config/avro/WipOrders.avsc` with: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'cp docker-config/fixed-avro/NewOrders.fixed.avsc docker-config/avro/NewOrders.avsc && cp docker-config/fixed-avro/WipOrders.fixed.avsc docker-config/avro/WipOrders.avsc' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp docker-config/fixed-avro/NewOrders.fixed.avsc docker-config/avro/NewOrders.avsc && cp docker-config/fixed-avro/WipOrders.fixed.avsc docker-config/avro/WipOrders.avsc' ``` ### Step 2: Update the examples @@ -277,8 +277,8 @@ Replace `api-specs/order-service-async-avro-v3_0_0_examples/PLACE_MACBOOK_ORDER. Alternatively, just run the following commands: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc "sed -i 's/\"id\": 101/\"id\": 1/; s/iPhone 14 Pro Max/iPhone/; s/\"price\": 500.00/\"price\": 5000/; s/exact:101/exact:1/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_IPHONE_ORDER.json" -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc "sed -i 's/\"id\": 102/\"id\": 2/; s/Macbook Mini Pro M5/Macbook/; s/\"price\": 600.00/\"price\": 6000/; s/exact:102/exact:2/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_MACBOOK_ORDER.json" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc "sed -i 's/\"id\": 101/\"id\": 1/; s/iPhone 14 Pro Max/iPhone/; s/\"price\": 500.00/\"price\": 5000/; s/exact:101/exact:1/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_IPHONE_ORDER.json" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc "sed -i 's/\"id\": 102/\"id\": 2/; s/Macbook Mini Pro M5/Macbook/; s/\"price\": 600.00/\"price\": 6000/; s/exact:102/exact:2/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_MACBOOK_ORDER.json" ``` ## Verify the fix diff --git a/kafka-sqs-retry-dlq/README.md b/kafka-sqs-retry-dlq/README.md index eecae8b..0a4af62 100644 --- a/kafka-sqs-retry-dlq/README.md +++ b/kafka-sqs-retry-dlq/README.md @@ -101,7 +101,7 @@ Do not change the contract, examples, or Compose wiring. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's/# threading.Thread/threading.Thread/' service/app.py" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's/# threading.Thread/threading.Thread/' service/app.py" ``` ## Pass criteria diff --git a/mcp-auto-test/README.md b/mcp-auto-test/README.md index 0f57edd..9223d25 100644 --- a/mcp-auto-test/README.md +++ b/mcp-auto-test/README.md @@ -101,7 +101,7 @@ Do not change anything else. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"damage\": 0.0#\"damaged\": 0.0#; s#order\\[\"shipment\"\\]#order[\"shipmentStatus\"]#' service/order_service.py" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"damage\": 0.0#\"damaged\": 0.0#; s#order\\[\"shipment\"\\]#order[\"shipmentStatus\"]#' service/order_service.py" ``` ## Part C: Re-run tests (expected to pass) diff --git a/overlays/README.md b/overlays/README.md index 5efa44a..5615f69 100644 --- a/overlays/README.md +++ b/overlays/README.md @@ -111,7 +111,7 @@ overlayFilePath: ./overlays/path-prefix.overlay.yaml Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc 'cp path-prefix.overlay.yaml overlays/path-prefix.overlay.yaml && sed -i "s/^# overlayFilePath:/ overlayFilePath:/" specmatic.yaml' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp path-prefix.overlay.yaml overlays/path-prefix.overlay.yaml && sed -i "s/^# overlayFilePath:/ overlayFilePath:/" specmatic.yaml' ``` ## Pass verification diff --git a/partial-examples/README.md b/partial-examples/README.md index 5754c37..f04970b 100644 --- a/partial-examples/README.md +++ b/partial-examples/README.md @@ -45,8 +45,8 @@ Test Run Cmd (Linux/Mac OSX) ```shell docker run --rm \ - -v .:/usr/src/app \ - -v ../license.txt:/specmatic/specmatic-license.txt:ro \ + -v "${PWD}:/usr/src/app" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ specmatic/enterprise:latest \ validate ``` @@ -56,10 +56,10 @@ docker run --rm \ [FAIL] Examples: 0 passed and 3 failed out of 3 total ``` -Windows (PowerShell/CMD) single-line +Windows PowerShell single-line -```shell -docker run --rm -v .:/usr/src/app -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest validate +```powershell +docker run --rm -v "$($PWD.Path):/usr/src/app" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest validate ``` ```terminaloutput @@ -101,7 +101,7 @@ docker compose --profile studio down -v Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc ' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc ' sed -i "2i\\ \\\"partial\\\": {" examples/test_accepted_order_request.json && sed -i "\$i\\ }" examples/test_accepted_order_request.json && sed -i "2i\\ \\\"partial\\\": {" examples/test_accepted_product_request.json && @@ -114,12 +114,12 @@ sed -i "/\\\"type\\\": \\\"book\\\"/d" examples/test_find_available_products_boo ### Final Phase -Re-run validation with the Windows single-line command after the Studio fixes are saved. +Re-run validation with the Windows PowerShell command after the Studio fixes are saved. -Windows (PowerShell/CMD) single-line +Windows PowerShell single-line -```shell -docker run --rm -v .:/usr/src/app -v ../license.txt:/specmatic/specmatic-license.txt:ro specmatic/enterprise:latest validate +```powershell +docker run --rm -v "$($PWD.Path):/usr/src/app" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest validate ``` ```terminaloutput diff --git a/quick-start-api-testing/README.md b/quick-start-api-testing/README.md index 9001463..db34f35 100644 --- a/quick-start-api-testing/README.md +++ b/quick-start-api-testing/README.md @@ -102,7 +102,7 @@ Do not change any other fields. Alternatively, just run the following command for Task A: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#[$]match(exact: approved)#\$match(pattern: approved\|verified)#' examples/test_finance_user_11.json" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#[$]match(exact: approved)#\$match(pattern: approved\|verified)#' examples/test_finance_user_11.json" ``` Re-run: @@ -137,7 +137,7 @@ Keep `handledBy` and `decision` as exact matches. Alternatively, just run the following command for Final Phase: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#[$]match(exact: VRF-123456)#\$match(pattern: VRF-[0-9]{6})#; s#[$]match(exact: 2026-03-17)#\$match(dataType: date)#' examples/test_support_user_55.json" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#[$]match(exact: VRF-123456)#\$match(pattern: VRF-[0-9]{6})#; s#[$]match(exact: 2026-03-17)#\$match(dataType: date)#' examples/test_support_user_55.json" ``` Run: diff --git a/quick-start-async-contract-testing/README.md b/quick-start-async-contract-testing/README.md index 8b2d5b0..2f66d62 100644 --- a/quick-start-async-contract-testing/README.md +++ b/quick-start-async-contract-testing/README.md @@ -76,7 +76,7 @@ To: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"status\": \"STARTED\"#\"status\": \"INITIATED\"#' service/processor.py" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"status\": \"STARTED\"#\"status\": \"INITIATED\"#' service/processor.py" ``` ## Pass criteria diff --git a/quick-start-contract-testing/README.md b/quick-start-contract-testing/README.md index 5318d68..99b5d46 100644 --- a/quick-start-contract-testing/README.md +++ b/quick-start-contract-testing/README.md @@ -111,7 +111,7 @@ Do not change anything else. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's/"petType"/"type"/g' service/server.py" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's/"petType"/"type"/g' service/server.py" ``` ## Part C: Re-run tests (expected to pass) diff --git a/quick-start-mock/README.md b/quick-start-mock/README.md index 8db8aa9..a46cc89 100644 --- a/quick-start-mock/README.md +++ b/quick-start-mock/README.md @@ -153,7 +153,7 @@ To inspect mock traffic in Studio: Alternatively, just run the following commands: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise:latest -lc 'mkdir -p specs/service_examples && cp quick-start-mock-generated/pets_242_GET_200_1.json specs/service_examples/pets_242_GET_200_1.json' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'mkdir -p specs/service_examples && cp quick-start-mock-generated/pets_242_GET_200_1.json specs/service_examples/pets_242_GET_200_1.json' docker compose --profile mock up -d --wait --wait-timeout 30 mock docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/242 docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/242 diff --git a/response-templating/README.md b/response-templating/README.md index 154c784..1a770d3 100644 --- a/response-templating/README.md +++ b/response-templating/README.md @@ -84,7 +84,7 @@ Keep `id` as-is. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"productid\": 1#\"productid\": \"(PRODUCTID:number)\"#; s#\"count\": 100#\"count\": \"(COUNT:number)\"#; s#\"productid\": \"\$match(dataType:number)\"#\"productid\": \"\$(PRODUCTID)\"#; s#\"count\": 1#\"count\": \"\$(COUNT)\"#' examples/mock/test_accepted_order_request.json" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#\"productid\": 1#\"productid\": \"(PRODUCTID:number)\"#; s#\"count\": 100#\"count\": \"(COUNT:number)\"#; s#\"productid\": \"\$match(dataType:number)\"#\"productid\": \"\$(PRODUCTID)\"#; s#\"count\": 1#\"count\": \"\$(COUNT)\"#' examples/mock/test_accepted_order_request.json" ``` ### Checkpoint after Task A @@ -116,7 +116,7 @@ Note: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc 'cp test_find_available_products_book_200.json examples/mock/test_find_available_products_book_200.json' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp test_find_available_products_book_200.json examples/mock/test_find_available_products_book_200.json' ``` ## 4. Final verification diff --git a/schema-design/README.md b/schema-design/README.md index b51d561..95c675c 100644 --- a/schema-design/README.md +++ b/schema-design/README.md @@ -123,7 +123,7 @@ Open `specs/payment-api.yaml` and update `PaymentRequest` to this shape: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc 'cp payments-request.yaml specs/payment-api.yaml' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp payments-request.yaml specs/payment-api.yaml' ``` ## 3. Re-run contract tests diff --git a/schema-resiliency-testing/README.md b/schema-resiliency-testing/README.md index 474f2c6..0510e91 100644 --- a/schema-resiliency-testing/README.md +++ b/schema-resiliency-testing/README.md @@ -72,7 +72,7 @@ In `specmatic.yaml` change `schemaResiliencyTests: none` to `schemaResiliencyTes Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: none#schemaResiliencyTests: positiveOnly#' specmatic.yaml" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: none#schemaResiliencyTests: positiveOnly#' specmatic.yaml" ``` #### Run Positive only Tests @@ -101,7 +101,7 @@ In `specmatic.yaml` change `schemaResiliencyTests: positiveOnly` to `schemaResil Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: positiveOnly#schemaResiliencyTests: all#' specmatic.yaml" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: positiveOnly#schemaResiliencyTests: all#' specmatic.yaml" ``` #### Run all Tests diff --git a/validate_readme_commands.py b/validate_readme_commands.py index 4ea7cd7..f5b30f0 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -13,6 +13,7 @@ import sys import threading import time +import traceback from dataclasses import dataclass from pathlib import Path from typing import Sequence @@ -945,16 +946,22 @@ def run_single_readme( print(f"README: {readme_path}", file=sys.stderr) print(summary.failure_message, file=sys.stderr) - final_cleanup_results = run_final_cleanup_commands( - command_specs=command_specs, - cwd=readme_path.parent, - timeout_seconds=timeout_seconds, - ) - if final_cleanup_results: - print_final_cleanup_summary(final_cleanup_results) + try: + final_cleanup_results = run_final_cleanup_commands( + command_specs=command_specs, + cwd=readme_path.parent, + timeout_seconds=timeout_seconds, + ) + if final_cleanup_results: + print_final_cleanup_summary(final_cleanup_results) - if repo_snapshot is not None: - print_reset_summary(reset_lab_changes(readme_path.parent, repo_snapshot)) + if repo_snapshot is not None: + print_reset_summary(reset_lab_changes(readme_path.parent, repo_snapshot)) + except Exception: + exit_code = 1 + print(f"README: {readme_path}", file=sys.stderr) + print("Cleanup/reset failed:", file=sys.stderr) + traceback.print_exc(file=sys.stderr) if exit_code == 0: print(f"Validated {len(summary.results)} command(s) in {readme_path}") diff --git a/workflow-in-same-spec/README.md b/workflow-in-same-spec/README.md index 3a6ef59..9faad74 100644 --- a/workflow-in-same-spec/README.md +++ b/workflow-in-same-spec/README.md @@ -108,7 +108,7 @@ workflow: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "$PWD:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ workflow:\n ids:\n \"POST /tasks -> 200\":\n extract: \"BODY.tasks.[0].id\"\n \"GET /tasks/(task_id:string) -> 200\":\n use: \"PATH.task_id\"\n \"PUT /tasks/(task_id:string) -> 200\":\n use: \"PATH.task_id\"\n \"DELETE /tasks/(task_id:string) -> 204\":\n use: \"PATH.task_id\"\n' specmatic.yaml" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i '/^specmatic:/i\ workflow:\n ids:\n \"POST /tasks -> 200\":\n extract: \"BODY.tasks.[0].id\"\n \"GET /tasks/(task_id:string) -> 200\":\n use: \"PATH.task_id\"\n \"PUT /tasks/(task_id:string) -> 200\":\n use: \"PATH.task_id\"\n \"DELETE /tasks/(task_id:string) -> 204\":\n use: \"PATH.task_id\"\n' specmatic.yaml" ``` Re-run: From f70dc5a311fd1fca6ed8ec4c18676f2c3cc358b3 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sun, 17 May 2026 20:48:13 +0530 Subject: [PATCH 49/71] fix: stabilize README validation and split async-event-flow studio startup from CI path --- .github/workflows/validate-readmes.yaml | 23 ++++++++++++++--------- async-event-flow/README.md | 14 +++++++++++--- async-event-flow/docker-compose.yaml | 2 ++ validate_readme_commands.py | 6 +++++- 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/.github/workflows/validate-readmes.yaml b/.github/workflows/validate-readmes.yaml index eaf86f5..d295d8c 100644 --- a/.github/workflows/validate-readmes.yaml +++ b/.github/workflows/validate-readmes.yaml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" @@ -58,12 +58,12 @@ jobs: - workflow-in-same-spec steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" @@ -78,7 +78,7 @@ jobs: - name: Upload lab validation result if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: readme-validator-${{ matrix.lab }} path: .artifacts/readme-validator/${{ matrix.lab }}.json @@ -93,12 +93,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" @@ -110,7 +110,7 @@ jobs: - name: Download lab validation artifacts if: needs.preflight.result == 'success' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: pattern: readme-validator-* path: .artifacts/readme-validator @@ -120,4 +120,9 @@ jobs: if: needs.preflight.result == 'success' run: | python3 validate_readme_commands.py \ - --report-from ".artifacts/readme-validator" | tee -a "$GITHUB_STEP_SUMMARY" + --report-from ".artifacts/readme-validator" | tee .artifacts/readme-validator/summary.txt + { + echo '```text' + cat .artifacts/readme-validator/summary.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/async-event-flow/README.md b/async-event-flow/README.md index 870038c..2da6041 100644 --- a/async-event-flow/README.md +++ b/async-event-flow/README.md @@ -73,12 +73,12 @@ Together, `receive`/`send` plus `before`/`after` fixtures let you express full e ![Event flow Verification](assets/async-interaction-validation.gif) ## Run the contract tests using Specmatic Studio -1. Start Kafka, the sample service, and Specmatic Studio. +1. Start Kafka and the sample service. ```shell docker compose up -d ``` - -2. Go to [Studio](http://127.0.0.1:9000/_specmatic/studio) and open the [specmatic.yaml](specmatic.yaml) file from the left sidebar, click on "Run Suite", and use the checked-out contract under `.specmatic/repos/labs-contracts/asyncapi/async-event-flow/async-order-service.yaml` if you want to inspect the loaded AsyncAPI file in Studio. + +If you want to inspect the suite in Studio, start it separately with `docker compose --profile studio up -d studio`, then go to [Studio](http://127.0.0.1:9000/_specmatic/studio) and open the [specmatic.yaml](specmatic.yaml) file from the left sidebar. Click on "Run Suite", and use the checked-out contract under `.specmatic/repos/labs-contracts/asyncapi/async-event-flow/async-order-service.yaml` if you want to inspect the loaded AsyncAPI file in Studio. You should first see 2 passing tests and 2 failing tests: @@ -146,6 +146,13 @@ docker compose down -v docker compose up -d ``` +If Studio is already running, restart it separately: + +```shell +docker compose --profile studio stop studio +docker compose --profile studio up -d studio +``` + Re-run the suite from Studio. Alternatively, just run the following commands: @@ -166,6 +173,7 @@ Bring down the Kafka broker after the tests are done. ```shell docker compose down -v +docker compose --profile studio down -v ``` ## Troubleshooting diff --git a/async-event-flow/docker-compose.yaml b/async-event-flow/docker-compose.yaml index 61aa439..941355a 100644 --- a/async-event-flow/docker-compose.yaml +++ b/async-event-flow/docker-compose.yaml @@ -58,6 +58,8 @@ services: image: specmatic/enterprise:latest container_name: studio restart: unless-stopped + profiles: + - studio command: ["studio"] volumes: - .:/usr/src/app diff --git a/validate_readme_commands.py b/validate_readme_commands.py index f5b30f0..99c5250 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -209,7 +209,7 @@ def parse_readme_commands(readme_path: Path) -> list[CommandSpec]: expected_outputs: list[str] = [] next_shell_index = len(blocks) for later_index in range(block_index + 1, len(blocks)): - if blocks[later_index].language == "shell": + if _is_command_block_language(blocks[later_index].language): next_shell_index = later_index break @@ -227,6 +227,10 @@ def parse_readme_commands(readme_path: Path) -> list[CommandSpec]: return commands +def _is_command_block_language(language: str) -> bool: + return language in {"shell", "powershell"} + + def _parse_fenced_blocks(lines: Sequence[str]) -> list[FencedBlock]: blocks: list[FencedBlock] = [] current_language: str | None = None From ce6fff80e6e00aff22c729efc2601fe1d40d6859 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sun, 17 May 2026 21:32:13 +0530 Subject: [PATCH 50/71] fix: harden README validator for CI and stabilize lab commands --- .github/workflows/validate-readmes.yaml | 2 +- backward-compatibility-testing/README.md | 17 ++++----- kafka-avro/README.md | 9 ----- tests/test_validate_readme_commands.py | 33 ++++++++++++++++ validate_readme_commands.py | 48 ++++++++++++++++++++---- 5 files changed, 82 insertions(+), 27 deletions(-) diff --git a/.github/workflows/validate-readmes.yaml b/.github/workflows/validate-readmes.yaml index d295d8c..e6af66e 100644 --- a/.github/workflows/validate-readmes.yaml +++ b/.github/workflows/validate-readmes.yaml @@ -110,7 +110,7 @@ jobs: - name: Download lab validation artifacts if: needs.preflight.result == 'success' - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: pattern: readme-validator-* path: .artifacts/readme-validator diff --git a/backward-compatibility-testing/README.md b/backward-compatibility-testing/README.md index 3e09446..5865d71 100644 --- a/backward-compatibility-testing/README.md +++ b/backward-compatibility-testing/README.md @@ -100,14 +100,12 @@ Run: *Unix/Mac: ```shell -docker run --rm \ +docker run --rm --entrypoint sh \ -v "${PWD}/..:/workspace" \ -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ - backward-compatibility-check \ - --base-branch origin/main \ - --target-path backward-compatibility-testing/products.yaml + -lc 'git fetch https://github.com/specmatic/labs main:refs/remotes/origin/main && specmatic backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml' ``` ```terminaloutput @@ -116,7 +114,7 @@ docker run --rm \ Windows PowerShell single-line: ```powershell -docker run --rm -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml +docker run --rm --entrypoint sh -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest -lc 'git fetch https://github.com/specmatic/labs main:refs/remotes/origin/main && specmatic backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml' ``` ```terminaloutput @@ -125,6 +123,7 @@ docker run --rm -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path .. Why the command is structured this way: - `-v "${PWD}/..:/workspace"` mounts the `labs` repository root, not just this lab folder, so Specmatic can access the git repository metadata. +- `git fetch https://github.com/specmatic/labs main:refs/remotes/origin/main` refreshes the baseline ref inside the container without depending on the mounted repo's remote URL or local SSH setup. - `--base-branch origin/main` tells Specmatic which tracked baseline to compare against. - `--target-path backward-compatibility-testing/products.yaml` tells Specmatic to compare the working tree version of this file with the tracked version on `origin/main`. @@ -176,14 +175,12 @@ Run the same command again: *Unix/Mac: ```shell -docker run --rm \ +docker run --rm --entrypoint sh \ -v "${PWD}/..:/workspace" \ -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ - backward-compatibility-check \ - --base-branch origin/main \ - --target-path backward-compatibility-testing/products.yaml + -lc 'git fetch https://github.com/specmatic/labs main:refs/remotes/origin/main && specmatic backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml' ``` ```terminaloutput @@ -193,7 +190,7 @@ Verdict for spec /workspace/backward-compatibility-testing/products.yaml: Windows PowerShell single-line: ```powershell -docker run --rm -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml +docker run --rm --entrypoint sh -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest -lc 'git fetch https://github.com/specmatic/labs main:refs/remotes/origin/main && specmatic backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml' ``` ```terminaloutput diff --git a/kafka-avro/README.md b/kafka-avro/README.md index 9c7b7ca..e4e566b 100644 --- a/kafka-avro/README.md +++ b/kafka-avro/README.md @@ -76,15 +76,6 @@ docker compose up specmatic-test --abort-on-container-exit Expected failure signal: ```terminaloutput -Unsuccessful Scenarios: - "Upon receiving a message on 'new-orders' channel, should send a message on 'wip-orders' channel. Example: PLACE_MACBOOK_ORDER FAILED" - Reason: - Cannot convert value: 600.0 to an Avro Integer - - "Upon receiving a message on 'new-orders' channel, should send a message on 'wip-orders' channel. Example: PLACE_IPHONE_ORDER FAILED" - Reason: - Cannot convert value: 500.0 to an Avro Integer - Tests run: 2, Successes: 0, Failures: 2, WIP: 0, Errors: 0 ``` diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index 741d60b..2e2e266 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -35,6 +35,7 @@ run_command_specs, snapshot_repo_state, should_skip_command, + _remove_path, ) @@ -211,12 +212,44 @@ def test_line_by_line_matching_allows_prefixed_actual_lines(self) -> None: self.assertTrue(_expected_output_matches(expected_output, actual_output)) + +class RemovePathTests(unittest.TestCase): + @patch("validate_readme_commands.subprocess.run") + @patch("validate_readme_commands.Path.unlink") + def test_remove_path_uses_docker_fallback_on_permission_error( + self, + unlink_mock, + subprocess_run_mock, + ) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_root = Path(temp_dir) + target = repo_root / "quick-start-mock" / "specs" / "service_examples" / "pets.json" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("data", encoding="utf-8") + + unlink_mock.side_effect = PermissionError("permission denied") + subprocess_run_mock.return_value = subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + + _remove_path(target, repo_root) + + subprocess_run_mock.assert_called_once() + docker_command = subprocess_run_mock.call_args.args[0] + self.assertEqual(docker_command[:4], ["docker", "run", "--rm", "-v"]) + self.assertIn(f"{repo_root}:/workspace", docker_command) + self.assertIn("rm -rf '/workspace/quick-start-mock/specs/service_examples/pets.json'", docker_command) + def test_line_by_line_matching_preserves_order(self) -> None: expected_output = "first line\nsecond line\n" actual_output = "prefix second line\nprefix first line\n" self.assertFalse(_expected_output_matches(expected_output, actual_output)) + def test_line_by_line_matching_trims_expected_line_whitespace(self) -> None: + expected_output = " first line \n\tsecond line\t\n" + actual_output = "prefix first line\nprefix second line\n" + + self.assertTrue(_expected_output_matches(expected_output, actual_output)) + def test_timeout_is_reported_cleanly(self) -> None: with self.assertRaises(CommandExecutionError) as ctx: run_command_specs( diff --git a/validate_readme_commands.py b/validate_readme_commands.py index 99c5250..5b0ef6b 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -15,7 +15,7 @@ import time import traceback from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Sequence ANSI_RESET = "\033[0m" @@ -382,7 +382,7 @@ def _assert_command_result(result: CommandResult) -> None: def _expected_output_matches(expected_output: str, actual_output: str) -> bool: - expected_lines = [line for line in expected_output.splitlines() if line.strip()] + expected_lines = [line.strip() for line in expected_output.splitlines() if line.strip()] actual_lines = actual_output.splitlines() if not expected_lines: @@ -402,7 +402,7 @@ def _expected_output_matches(expected_output: str, actual_output: str) -> bool: def _describe_output_mismatch(expected_output: str, actual_output: str) -> str | None: - expected_lines = [line for line in expected_output.splitlines() if line.strip()] + expected_lines = [line.strip() for line in expected_output.splitlines() if line.strip()] actual_lines = actual_output.splitlines() if not expected_lines: @@ -1481,15 +1481,49 @@ def reset_lab_changes(cwd: Path, baseline: RepoSnapshot | None) -> ResetSummary: absolute_path = baseline.repo_root / path if not absolute_path.exists(): continue - if absolute_path.is_dir(): - shutil.rmtree(absolute_path) - else: - absolute_path.unlink() + _remove_path(absolute_path, baseline.repo_root) removed.append(path) return ResetSummary(restored=paths_to_restore, removed=removed) +def _remove_path(path: Path, repo_root: Path) -> None: + try: + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + return + except PermissionError: + pass + + relative_path = path.resolve().relative_to(repo_root.resolve()) + container_target = PurePosixPath("/workspace") / PurePosixPath(relative_path.as_posix()) + + command = [ + "docker", + "run", + "--rm", + "-v", + f"{repo_root}:/workspace", + "--entrypoint", + "sh", + "specmatic/enterprise:latest", + "-lc", + f"rm -rf '{container_target}'", + ] + + completed = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0 and path.exists(): + error_output = completed.stderr.strip() or completed.stdout.strip() or "docker rm failed" + raise PermissionError(f"failed to remove {path}: {error_output}") + + def print_reset_summary(reset_summary: ResetSummary) -> None: if not reset_summary.restored and not reset_summary.removed: return From 5d18e4bf534769e16b3d5bbb97d99bbb0671b706 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sun, 17 May 2026 22:00:46 +0530 Subject: [PATCH 51/71] fix: drop the subsequence search --- kafka-avro/README.md | 9 +++++++++ tests/test_validate_readme_commands.py | 4 ++-- validate_readme_commands.py | 16 ++-------------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/kafka-avro/README.md b/kafka-avro/README.md index e4e566b..9c7b7ca 100644 --- a/kafka-avro/README.md +++ b/kafka-avro/README.md @@ -76,6 +76,15 @@ docker compose up specmatic-test --abort-on-container-exit Expected failure signal: ```terminaloutput +Unsuccessful Scenarios: + "Upon receiving a message on 'new-orders' channel, should send a message on 'wip-orders' channel. Example: PLACE_MACBOOK_ORDER FAILED" + Reason: + Cannot convert value: 600.0 to an Avro Integer + + "Upon receiving a message on 'new-orders' channel, should send a message on 'wip-orders' channel. Example: PLACE_IPHONE_ORDER FAILED" + Reason: + Cannot convert value: 500.0 to an Avro Integer + Tests run: 2, Successes: 0, Failures: 2, WIP: 0, Errors: 0 ``` diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index 2e2e266..7423d3d 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -238,11 +238,11 @@ def test_remove_path_uses_docker_fallback_on_permission_error( self.assertIn(f"{repo_root}:/workspace", docker_command) self.assertIn("rm -rf '/workspace/quick-start-mock/specs/service_examples/pets.json'", docker_command) - def test_line_by_line_matching_preserves_order(self) -> None: + def test_line_by_line_matching_does_not_require_order(self) -> None: expected_output = "first line\nsecond line\n" actual_output = "prefix second line\nprefix first line\n" - self.assertFalse(_expected_output_matches(expected_output, actual_output)) + self.assertTrue(_expected_output_matches(expected_output, actual_output)) def test_line_by_line_matching_trims_expected_line_whitespace(self) -> None: expected_output = " first line \n\tsecond line\t\n" diff --git a/validate_readme_commands.py b/validate_readme_commands.py index 5b0ef6b..30ef6c9 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -388,14 +388,8 @@ def _expected_output_matches(expected_output: str, actual_output: str) -> bool: if not expected_lines: return True - actual_index = 0 for expected_line in expected_lines: - while actual_index < len(actual_lines): - if expected_line in actual_lines[actual_index]: - actual_index += 1 - break - actual_index += 1 - else: + if not any(expected_line in actual_line for actual_line in actual_lines): return False return True @@ -408,14 +402,8 @@ def _describe_output_mismatch(expected_output: str, actual_output: str) -> str | if not expected_lines: return None - actual_index = 0 for expected_line in expected_lines: - while actual_index < len(actual_lines): - if expected_line in actual_lines[actual_index]: - actual_index += 1 - break - actual_index += 1 - else: + if not any(expected_line in actual_line for actual_line in actual_lines): closest_line = _find_closest_line(expected_line, actual_lines) divider = _style("-" * 48, ANSI_DIM) detail_lines = [ From ad9e5ed087cf290fcd3088767d0861617212622f Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sun, 17 May 2026 22:16:30 +0530 Subject: [PATCH 52/71] fix: simplify backward compatibility lab commands by running container as host user to avoid Git dubious ownership on mounted repo --- backward-compatibility-testing/README.md | 26 +++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/backward-compatibility-testing/README.md b/backward-compatibility-testing/README.md index 5865d71..9ef077d 100644 --- a/backward-compatibility-testing/README.md +++ b/backward-compatibility-testing/README.md @@ -100,12 +100,15 @@ Run: *Unix/Mac: ```shell -docker run --rm --entrypoint sh \ +docker run --rm \ + --user "$(id -u):$(id -g)" \ -v "${PWD}/..:/workspace" \ -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ - -lc 'git fetch https://github.com/specmatic/labs main:refs/remotes/origin/main && specmatic backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml' + backward-compatibility-check \ + --base-branch origin/main \ + --target-path backward-compatibility-testing/products.yaml ``` ```terminaloutput @@ -114,7 +117,7 @@ docker run --rm --entrypoint sh \ Windows PowerShell single-line: ```powershell -docker run --rm --entrypoint sh -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest -lc 'git fetch https://github.com/specmatic/labs main:refs/remotes/origin/main && specmatic backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml' +docker run --rm --user "$(id -u):$(id -g)" -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml ``` ```terminaloutput @@ -123,7 +126,7 @@ docker run --rm --entrypoint sh -v "$((Resolve-Path ..).Path):/workspace" -v "$( Why the command is structured this way: - `-v "${PWD}/..:/workspace"` mounts the `labs` repository root, not just this lab folder, so Specmatic can access the git repository metadata. -- `git fetch https://github.com/specmatic/labs main:refs/remotes/origin/main` refreshes the baseline ref inside the container without depending on the mounted repo's remote URL or local SSH setup. +- `--user "$(id -u):$(id -g)"` runs the container as your host user, which avoids git ownership issues when the mounted repository is inspected inside the container. - `--base-branch origin/main` tells Specmatic which tracked baseline to compare against. - `--target-path backward-compatibility-testing/products.yaml` tells Specmatic to compare the working tree version of this file with the tracked version on `origin/main`. @@ -140,6 +143,12 @@ The Incompatibility Report: This is number in the new specification response but string in the old specification ``` +Expected verdict: + +```terminaloutput +(INCOMPATIBLE) This spec contains breaking changes to the API +``` + Why this fails: - Adding optional `category` is safe. - Changing `name` from `string` to `number` is a breaking change for existing consumers. @@ -175,12 +184,15 @@ Run the same command again: *Unix/Mac: ```shell -docker run --rm --entrypoint sh \ +docker run --rm \ + --user "$(id -u):$(id -g)" \ -v "${PWD}/..:/workspace" \ -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ - -lc 'git fetch https://github.com/specmatic/labs main:refs/remotes/origin/main && specmatic backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml' + backward-compatibility-check \ + --base-branch origin/main \ + --target-path backward-compatibility-testing/products.yaml ``` ```terminaloutput @@ -190,7 +202,7 @@ Verdict for spec /workspace/backward-compatibility-testing/products.yaml: Windows PowerShell single-line: ```powershell -docker run --rm --entrypoint sh -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest -lc 'git fetch https://github.com/specmatic/labs main:refs/remotes/origin/main && specmatic backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml' +docker run --rm --user "$(id -u):$(id -g)" -v "$((Resolve-Path ..).Path):/workspace" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" -w /workspace specmatic/enterprise:latest backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml ``` ```terminaloutput From a3be4d8fb176546e975e6e3f2e8a05d5dfeb49b3 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sun, 17 May 2026 22:26:02 +0530 Subject: [PATCH 53/71] fix: add curl timeouts to data-adapters verifier to prevent README CI hangs --- data-adapters/docker-compose.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data-adapters/docker-compose.yaml b/data-adapters/docker-compose.yaml index bafe655..1f08f77 100644 --- a/data-adapters/docker-compose.yaml +++ b/data-adapters/docker-compose.yaml @@ -41,6 +41,10 @@ services: condition: service_healthy entrypoint: ["curl"] command: + - --connect-timeout + - "10" + - --max-time + - "20" - -i - -X - POST From 1a1f79e822251aae6f8228a59fe9afe7938320b7 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sun, 17 May 2026 22:43:28 +0530 Subject: [PATCH 54/71] cleanup: remove the unused print_dry_run() wrapper, stopped reparsing README command blocks by introducing a single _readme_command_specs() helper and using it in preflight and Docker warmup detection, collapsed the repeated append/callback logic in both warm_docker_images() and run_preflight() with local _emit() helpers, and simplified run_single_readme() by printing the command mapping once and consolidating the skip-count handling. --- validate_readme_commands.py | 236 +++++++++++++++++------------------- 1 file changed, 112 insertions(+), 124 deletions(-) diff --git a/validate_readme_commands.py b/validate_readme_commands.py index 30ef6c9..c94ce7b 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -227,6 +227,13 @@ def parse_readme_commands(readme_path: Path) -> list[CommandSpec]: return commands +def _readme_command_specs(readme_path: Path) -> list[CommandSpec]: + try: + return parse_readme_commands(readme_path) + except (OSError, ReadmeValidationError): + return [] + + def _is_command_block_language(language: str) -> bool: return language in {"shell", "powershell"} @@ -900,8 +907,8 @@ def run_single_readme( lab_name = readme_path.parent.name try: command_specs = parse_readme_commands(readme_path) + print_command_mapping(command_specs) if dry_run: - print_command_mapping(command_specs) return LabExecutionResult( name=lab_name, exit_code=0, @@ -910,7 +917,6 @@ def run_single_readme( total_commands=len(command_specs), skipped_commands=0, ) - print_command_mapping(command_specs) summary = run_command_specs( command_specs=command_specs, cwd=readme_path.parent, @@ -960,15 +966,15 @@ def run_single_readme( print("PASS") else: print("FAIL") - skipped_commands = sum(1 for result in summary.results if result.skipped) - validated_commands = len(summary.results) - skipped_commands + executed_skips = sum(1 for result in summary.results if result.skipped) + skipped_commands = total_commands_skipped(summary, len(command_specs)) return LabExecutionResult( name=lab_name, exit_code=exit_code, duration_seconds=0.0, - validated_commands=validated_commands, + validated_commands=len(summary.results) - executed_skips, total_commands=len(command_specs), - skipped_commands=total_commands_skipped(summary, len(command_specs)), + skipped_commands=skipped_commands, ) @@ -1046,9 +1052,14 @@ def determine_preflight_requirements(readme_paths: Sequence[Path]) -> PreflightR remote_contract_access = False for readme_path in readme_paths: - if _readme_uses_docker(readme_path): + command_specs = _readme_command_specs(readme_path) + if any("docker" in command_spec.command.lower() for command_spec in command_specs): docker_cli = True - if _readme_uses_docker_compose(readme_path): + if any( + "docker compose" in command_spec.command.lower() + or "docker-compose" in command_spec.command.lower() + for command_spec in command_specs + ): docker_compose = True for config_path in _related_config_paths(readme_path.parent): @@ -1071,26 +1082,6 @@ def determine_preflight_requirements(readme_paths: Sequence[Path]) -> PreflightR ) -def _readme_uses_docker(readme_path: Path) -> bool: - try: - command_specs = parse_readme_commands(readme_path) - except (OSError, ReadmeValidationError): - return False - return any("docker" in command_spec.command.lower() for command_spec in command_specs) - - -def _readme_uses_docker_compose(readme_path: Path) -> bool: - try: - command_specs = parse_readme_commands(readme_path) - except (OSError, ReadmeValidationError): - return False - return any( - "docker compose" in command_spec.command.lower() - or "docker-compose" in command_spec.command.lower() - for command_spec in command_specs - ) - - def _related_config_paths(lab_dir: Path) -> list[Path]: return [ lab_dir / "docker-compose.yaml", @@ -1106,6 +1097,11 @@ def warm_docker_images( ) -> list[DockerWarmupResult]: results: list[DockerWarmupResult] = [] + def _emit(result: DockerWarmupResult) -> None: + results.append(result) + if on_result is not None: + on_result(result) + for lab_dir in _docker_compose_lab_dirs(readme_paths): if on_result is not None and not results: print("===== Docker Warmup =====") @@ -1119,42 +1115,36 @@ def warm_docker_images( timeout=timeout_seconds, ) except subprocess.TimeoutExpired: - result = DockerWarmupResult( - lab_name=lab_dir.name, - passed=False, - detail=f"timed out after {int(timeout_seconds)} seconds", + _emit( + DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=f"timed out after {int(timeout_seconds)} seconds", + ) ) - results.append(result) - if on_result is not None: - on_result(result) continue except OSError as exc: - result = DockerWarmupResult( - lab_name=lab_dir.name, - passed=False, - detail=str(exc), + _emit( + DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=str(exc), + ) ) - results.append(result) - if on_result is not None: - on_result(result) continue if completed.returncode == 0: - result = DockerWarmupResult(lab_name=lab_dir.name, passed=True) - results.append(result) - if on_result is not None: - on_result(result) + _emit(DockerWarmupResult(lab_name=lab_dir.name, passed=True)) continue error_output = completed.stderr.strip() or completed.stdout.strip() or "docker compose pull failed" - result = DockerWarmupResult( - lab_name=lab_dir.name, - passed=False, - detail=error_output, + _emit( + DockerWarmupResult( + lab_name=lab_dir.name, + passed=False, + detail=error_output, + ) ) - results.append(result) - if on_result is not None: - on_result(result) return results @@ -1168,7 +1158,12 @@ def _docker_compose_lab_dirs(readme_paths: Sequence[Path]) -> list[Path]: docker_compose_path = lab_dir / "docker-compose.yaml" if lab_dir in seen or not docker_compose_path.is_file(): continue - if not _readme_uses_docker_compose(readme_path): + command_specs = _readme_command_specs(readme_path) + if not any( + "docker compose" in command_spec.command.lower() + or "docker-compose" in command_spec.command.lower() + for command_spec in command_specs + ): continue seen.add(lab_dir) lab_dirs.append(lab_dir) @@ -1189,97 +1184,95 @@ def run_preflight( repo_root = Path(__file__).resolve().parent license_path = repo_root / "license.txt" - if requirements.docker_cli: - result = _run_preflight_command( - name="docker", - command=["docker", "--version"], - failure_detail="docker CLI is not available", - suggestion="Install Docker and make sure `docker` is on PATH.", - ) + def _emit(result: PreflightCheckResult) -> None: results.append(result) if on_result is not None: on_result(result) + if requirements.docker_cli: + _emit( + _run_preflight_command( + name="docker", + command=["docker", "--version"], + failure_detail="docker CLI is not available", + suggestion="Install Docker and make sure `docker` is on PATH.", + ) + ) + docker_ready = not results or results[-1].passed if requirements.docker_compose: - result = _run_preflight_command( - name="docker compose", - command=["docker", "compose", "version"], - failure_detail="docker compose is not available", - suggestion="Install a Docker version that includes `docker compose`.", + _emit( + _run_preflight_command( + name="docker compose", + command=["docker", "compose", "version"], + failure_detail="docker compose is not available", + suggestion="Install a Docker version that includes `docker compose`.", + ) ) - results.append(result) - if on_result is not None: - on_result(result) docker_ready = docker_ready and results[-1].passed if requirements.docker_cli and docker_ready: - result = _run_preflight_command( - name="docker daemon", - command=["docker", "info"], - failure_detail="Docker daemon is not reachable", - suggestion="Start Docker Desktop or the Docker daemon, then rerun the validator.", + _emit( + _run_preflight_command( + name="docker daemon", + command=["docker", "info"], + failure_detail="Docker daemon is not reachable", + suggestion="Start Docker Desktop or the Docker daemon, then rerun the validator.", + ) ) - results.append(result) - if on_result is not None: - on_result(result) docker_ready = docker_ready and results[-1].passed elif requirements.docker_cli: - result = PreflightCheckResult( - name="docker daemon", - passed=False, - skipped=True, - detail="skipped because Docker CLI/Compose is unavailable", - suggestion="Fix the Docker installation first.", + _emit( + PreflightCheckResult( + name="docker daemon", + passed=False, + skipped=True, + detail="skipped because Docker CLI/Compose is unavailable", + suggestion="Fix the Docker installation first.", + ) ) - results.append(result) - if on_result is not None: - on_result(result) if requirements.license_validation: if license_path.is_file(): - result = PreflightCheckResult(name="specmatic license file exists", passed=True) + _emit(PreflightCheckResult(name="specmatic license file exists", passed=True)) else: - result = PreflightCheckResult( - name="specmatic license file exists", - passed=False, - detail=f"missing {license_path}", - suggestion="Add a valid `license.txt` at the labs repo root.", + _emit( + PreflightCheckResult( + name="specmatic license file exists", + passed=False, + detail=f"missing {license_path}", + suggestion="Add a valid `license.txt` at the labs repo root.", + ) ) - results.append(result) - if on_result is not None: - on_result(result) if docker_ready and license_path.is_file(): - result = _validate_specmatic_license(repo_root) + _emit(_validate_specmatic_license(repo_root)) else: - result = PreflightCheckResult( - name="specmatic license validation", - passed=False, - skipped=True, - detail="skipped because Docker or the license file is unavailable", - suggestion="Fix Docker access and the license file, then rerun the validator.", + _emit( + PreflightCheckResult( + name="specmatic license validation", + passed=False, + skipped=True, + detail="skipped because Docker or the license file is unavailable", + suggestion="Fix Docker access and the license file, then rerun the validator.", + ) ) - results.append(result) - if on_result is not None: - on_result(result) if requirements.remote_contract_access: - result = _run_preflight_command( - name="labs-contracts access", - command=[ - "git", - "ls-remote", - "--exit-code", - "https://github.com/specmatic/labs-contracts.git", - "HEAD", - ], - failure_detail="cannot reach github.com/specmatic/labs-contracts.git", - suggestion="Check network access to GitHub, then rerun the validator.", + _emit( + _run_preflight_command( + name="labs-contracts access", + command=[ + "git", + "ls-remote", + "--exit-code", + "https://github.com/specmatic/labs-contracts.git", + "HEAD", + ], + failure_detail="cannot reach github.com/specmatic/labs-contracts.git", + suggestion="Check network access to GitHub, then rerun the validator.", + ) ) - results.append(result) - if on_result is not None: - on_result(result) return results @@ -1416,11 +1409,6 @@ def print_command_mapping(command_specs: Sequence[CommandSpec]) -> None: if command_specs: print(separator) - -def print_dry_run(command_specs: Sequence[CommandSpec]) -> None: - print_command_mapping(command_specs) - - def _print_indented_block(content: str) -> None: for line in content.rstrip("\n").splitlines(): print(f" {line}") From 141e3141b20922a71bb8ec90a2a4509ed9e70c3e Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Thu, 21 May 2026 13:35:37 +0530 Subject: [PATCH 55/71] cleanup --- api-coverage/README.md | 2 +- .../acceptOrder-with-before.json | 0 .../outForDeliveryOrder-with-before.json | 1 + async-event-flow/README.md | 23 +++----- async-event-flow/docker-compose.yaml | 2 - async-event-flow/run-suite-config.yaml | 58 ------------------- async-event-flow/specmatic.yaml | 6 +- validate_readme_commands.py | 1 - 8 files changed, 12 insertions(+), 81 deletions(-) rename async-event-flow/{examples/fixed => .backup}/acceptOrder-with-before.json (100%) rename async-event-flow/{examples/fixed => .backup}/outForDeliveryOrder-with-before.json (98%) delete mode 100644 async-event-flow/run-suite-config.yaml diff --git a/api-coverage/README.md b/api-coverage/README.md index 629b92b..f3e6c16 100644 --- a/api-coverage/README.md +++ b/api-coverage/README.md @@ -154,7 +154,7 @@ Alternatively, just run the following command: docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc "sed -i 's|/pets/search:|/pets/find:|' specs/service.yaml" ``` -## Final Phase +## 3. Re-run the tests and coverage check Run the same command again: ```shell diff --git a/async-event-flow/examples/fixed/acceptOrder-with-before.json b/async-event-flow/.backup/acceptOrder-with-before.json similarity index 100% rename from async-event-flow/examples/fixed/acceptOrder-with-before.json rename to async-event-flow/.backup/acceptOrder-with-before.json diff --git a/async-event-flow/examples/fixed/outForDeliveryOrder-with-before.json b/async-event-flow/.backup/outForDeliveryOrder-with-before.json similarity index 98% rename from async-event-flow/examples/fixed/outForDeliveryOrder-with-before.json rename to async-event-flow/.backup/outForDeliveryOrder-with-before.json index f3a37b7..329a697 100644 --- a/async-event-flow/examples/fixed/outForDeliveryOrder-with-before.json +++ b/async-event-flow/.backup/outForDeliveryOrder-with-before.json @@ -3,6 +3,7 @@ "name": "ORDER_OUT_FOR_DELIVERY", "receive": { "topic": "out-for-delivery-orders", + "key": 456, "payload": { "orderId": 456, "deliveryAddress": "1234 Elm Street, Springfield", diff --git a/async-event-flow/README.md b/async-event-flow/README.md index 2da6041..366cf52 100644 --- a/async-event-flow/README.md +++ b/async-event-flow/README.md @@ -73,19 +73,19 @@ Together, `receive`/`send` plus `before`/`after` fixtures let you express full e ![Event flow Verification](assets/async-interaction-validation.gif) ## Run the contract tests using Specmatic Studio -1. Start Kafka and the sample service. +1. Start Kafka, the sample service, and Specmatic Studio. ```shell docker compose up -d ``` - -If you want to inspect the suite in Studio, start it separately with `docker compose --profile studio up -d studio`, then go to [Studio](http://127.0.0.1:9000/_specmatic/studio) and open the [specmatic.yaml](specmatic.yaml) file from the left sidebar. Click on "Run Suite", and use the checked-out contract under `.specmatic/repos/labs-contracts/asyncapi/async-event-flow/async-order-service.yaml` if you want to inspect the loaded AsyncAPI file in Studio. + +2. Go to [Studio](http://127.0.0.1:9000/_specmatic/studio) and open the [specmatic.yaml](specmatic.yaml) file from the left sidebar, click on "Run Suite", and use the checked-out contract under `.specmatic/repos/labs-contracts/asyncapi/async-event-flow/async-order-service.yaml` if you want to inspect the loaded AsyncAPI file in Studio. You should first see 2 passing tests and 2 failing tests: Alternatively, just run the following commands: ```shell -docker run --rm --network async-event-flow_default -v "${PWD}:/usr/src/app" -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest run-suite --config=/usr/src/app/run-suite-config.yaml +docker compose exec -T studio specmatic run-suite ``` ```terminaloutput @@ -120,7 +120,7 @@ Tests run: 4, Successes: 2, Failures: 2, WIP: 0, Errors: 0 } ], ``` -- In `examples/async-order-service/outForDeliveryOrder.json`, add a `before` fixture that seeds order `456` before the event is published, and fix the `after` fixture by changing: +- In `examples/async-order-service/outForDeliveryOrder.json`, fix the `after` fixture by changing: ```json "tax-invoice-for-order-456": "$match(exact: 2)" @@ -135,8 +135,7 @@ To: Alternatively, just run the following commands: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp examples/fixed/acceptOrder-with-before.json examples/async-order-service/acceptOrder.json' -docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp examples/fixed/outForDeliveryOrder-with-before.json examples/async-order-service/outForDeliveryOrder.json' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp .backup/acceptOrder-with-before.json examples/async-order-service/acceptOrder.json && cp .backup/outForDeliveryOrder-with-before.json examples/async-order-service/outForDeliveryOrder.json' ``` 4. Restart Docker Containers @@ -146,19 +145,12 @@ docker compose down -v docker compose up -d ``` -If Studio is already running, restart it separately: - -```shell -docker compose --profile studio stop studio -docker compose --profile studio up -d studio -``` - Re-run the suite from Studio. Alternatively, just run the following commands: ```shell -docker run --rm --network async-event-flow_default -v "${PWD}:/usr/src/app" -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest run-suite --config=/usr/src/app/run-suite-config.yaml +docker compose exec -T studio specmatic run-suite ``` You should now see: @@ -173,7 +165,6 @@ Bring down the Kafka broker after the tests are done. ```shell docker compose down -v -docker compose --profile studio down -v ``` ## Troubleshooting diff --git a/async-event-flow/docker-compose.yaml b/async-event-flow/docker-compose.yaml index 941355a..61aa439 100644 --- a/async-event-flow/docker-compose.yaml +++ b/async-event-flow/docker-compose.yaml @@ -58,8 +58,6 @@ services: image: specmatic/enterprise:latest container_name: studio restart: unless-stopped - profiles: - - studio command: ["studio"] volumes: - .:/usr/src/app diff --git a/async-event-flow/run-suite-config.yaml b/async-event-flow/run-suite-config.yaml deleted file mode 100644 index 5fbcc74..0000000 --- a/async-event-flow/run-suite-config.yaml +++ /dev/null @@ -1,58 +0,0 @@ -version: 3 -systemUnderTest: - service: - $ref: "#/components/services/orderAsyncService" - runOptions: - $ref: "#/components/runOptions/orderAsyncServiceTest" - data: - examples: - - directories: - - examples/async-order-service - -dependencies: - services: - - service: - $ref: "#/components/services/taxService" - runOptions: - $ref: "#/components/runOptions/taxServiceMock" - data: - examples: - - directories: - - examples/tax-service - -components: - sources: - labsContracts: - git: - url: https://github.com/specmatic/labs-contracts.git - branch: main - services: - orderAsyncService: - definitions: - - definition: - source: - $ref: "#/components/sources/labsContracts" - specs: - - asyncapi/async-event-flow/async-order-service.yaml - taxService: - definitions: - - definition: - source: - $ref: "#/components/sources/labsContracts" - specs: - - openapi/async-event-flow/tax-service.yaml - runOptions: - orderAsyncServiceTest: - asyncapi: - type: test - servers: - - host: "${KAFKA_BROKER_HOST:kafka:9092}" - protocol: kafka - taxServiceMock: - openapi: - type: mock - baseUrl: "${TAX_SERVICE_URL:http://0.0.0.0:8090}" - -specmatic: - license: - path: /specmatic/specmatic-license.txt diff --git a/async-event-flow/specmatic.yaml b/async-event-flow/specmatic.yaml index aec4cc0..204d64d 100644 --- a/async-event-flow/specmatic.yaml +++ b/async-event-flow/specmatic.yaml @@ -7,7 +7,7 @@ systemUnderTest: data: examples: - directories: - - "${EX_DIR:examples}" + - "${EX_DIR:examples/async-order-service}" dependencies: services: @@ -18,7 +18,7 @@ dependencies: data: examples: - directories: - - "${EX_DIR:examples}" + - "${EX_DIR:examples/tax-service}" components: sources: @@ -46,7 +46,7 @@ components: asyncapi: type: test servers: - - host: "${KAFKA_BROKER_HOST:localhost:9092}" + - host: "${KAFKA_BROKER_HOST:kafka:9092}" protocol: kafka taxServiceMock: openapi: diff --git a/validate_readme_commands.py b/validate_readme_commands.py index c94ce7b..a94d652 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -1086,7 +1086,6 @@ def _related_config_paths(lab_dir: Path) -> list[Path]: return [ lab_dir / "docker-compose.yaml", lab_dir / "specmatic.yaml", - lab_dir / "run-suite-config.yaml", ] From af131646204bef7c3baffba1dc50079dc1506e96 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Fri, 22 May 2026 15:48:58 +0530 Subject: [PATCH 56/71] backward-compatibility-testing does not require additional openapi spec file copies we can just use sed to do the necessary changes instead of copies whole files --- async-event-flow/README.md | 2 +- backward-compatibility-testing/README.md | 26 +++++++--------- .../products-breaking.yaml | 30 ------------------- .../products-fixed.yaml | 30 ------------------- 4 files changed, 12 insertions(+), 76 deletions(-) delete mode 100644 backward-compatibility-testing/products-breaking.yaml delete mode 100644 backward-compatibility-testing/products-fixed.yaml diff --git a/async-event-flow/README.md b/async-event-flow/README.md index 366cf52..65ddc67 100644 --- a/async-event-flow/README.md +++ b/async-event-flow/README.md @@ -145,7 +145,7 @@ docker compose down -v docker compose up -d ``` -Re-run the suite from Studio. +5. Re-run the suite from Studio. Alternatively, just run the following commands: diff --git a/backward-compatibility-testing/README.md b/backward-compatibility-testing/README.md index 9ef077d..c37226f 100644 --- a/backward-compatibility-testing/README.md +++ b/backward-compatibility-testing/README.md @@ -89,10 +89,10 @@ properties: You now have an uncommitted change in a tracked contract file. Specmatic will compare it to the version on `origin/main`. -Alternatively, just run the following command: +Alternatively, just run the following commands: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc 'cp products-breaking.yaml products.yaml' +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc "sed -i 's/version: 1.0.0/version: 1.1.0/' products.yaml; sed -i '/properties:/,/sku:/s/type: string/type: number/' products.yaml; sed -i '/^ sku:$/i\\ category:\\n type: string' products.yaml" ``` ## Part B: Run the backward compatibility check @@ -112,6 +112,7 @@ docker run --rm \ ``` ```terminaloutput +Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (INCOMPATIBLE) This spec contains breaking changes to the API ``` @@ -121,15 +122,10 @@ docker run --rm --user "$(id -u):$(id -g)" -v "$((Resolve-Path ..).Path):/worksp ``` ```terminaloutput +Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (INCOMPATIBLE) This spec contains breaking changes to the API ``` -Why the command is structured this way: -- `-v "${PWD}/..:/workspace"` mounts the `labs` repository root, not just this lab folder, so Specmatic can access the git repository metadata. -- `--user "$(id -u):$(id -g)"` runs the container as your host user, which avoids git ownership issues when the mounted repository is inspected inside the container. -- `--base-branch origin/main` tells Specmatic which tracked baseline to compare against. -- `--target-path backward-compatibility-testing/products.yaml` tells Specmatic to compare the working tree version of this file with the tracked version on `origin/main`. - Expected failure highlights: ```terminaloutput @@ -143,16 +139,16 @@ The Incompatibility Report: This is number in the new specification response but string in the old specification ``` -Expected verdict: - -```terminaloutput -(INCOMPATIBLE) This spec contains breaking changes to the API -``` - Why this fails: - Adding optional `category` is safe. - Changing `name` from `string` to `number` is a breaking change for existing consumers. +Why the command is structured this way: +- `-v "${PWD}/..:/workspace"` mounts the `labs` repository root, not just this lab folder, so Specmatic can access the git repository metadata. +- `--user "$(id -u):$(id -g)"` runs the container as your host user, which avoids git ownership issues when the mounted repository is inspected inside the container. +- `--base-branch origin/main` tells Specmatic which tracked baseline to compare against. +- `--target-path backward-compatibility-testing/products.yaml` tells Specmatic to compare the working tree version of this file with the tracked version on `origin/main`. + ## Part C: Fix the contract Open `products.yaml`. @@ -176,7 +172,7 @@ Keep version `1.1.0`. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc 'cp products-fixed.yaml products.yaml' +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc 'sed -i "/properties:/,/sku:/s/type: number/type: string/" products.yaml' ``` ## Part D: Re-run the check diff --git a/backward-compatibility-testing/products-breaking.yaml b/backward-compatibility-testing/products-breaking.yaml deleted file mode 100644 index 4fc9384..0000000 --- a/backward-compatibility-testing/products-breaking.yaml +++ /dev/null @@ -1,30 +0,0 @@ -openapi: 3.0.0 -info: - title: Sample Product API - version: 1.1.0 -paths: - /products/{id}: - get: - summary: Get product by id - parameters: - - in: path - name: id - required: true - schema: - type: number - responses: - "200": - description: Product details - content: - application/json: - schema: - type: object - required: - - name - properties: - name: - type: number - sku: - type: string - category: - type: string diff --git a/backward-compatibility-testing/products-fixed.yaml b/backward-compatibility-testing/products-fixed.yaml deleted file mode 100644 index 2a45e5d..0000000 --- a/backward-compatibility-testing/products-fixed.yaml +++ /dev/null @@ -1,30 +0,0 @@ -openapi: 3.0.0 -info: - title: Sample Product API - version: 1.1.0 -paths: - /products/{id}: - get: - summary: Get product by id - parameters: - - in: path - name: id - required: true - schema: - type: number - responses: - "200": - description: Product details - content: - application/json: - schema: - type: object - required: - - name - properties: - name: - type: string - sku: - type: string - category: - type: string From 862d0bcc11767b7468dd72b44d248bb204ca2d0a Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Mon, 25 May 2026 12:55:28 +0530 Subject: [PATCH 57/71] better demarcation between the commands and their outputs --- tests/test_validate_readme_commands.py | 33 +++++++++++++++++++ validate_readme_commands.py | 44 ++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/tests/test_validate_readme_commands.py b/tests/test_validate_readme_commands.py index 7423d3d..ef6616b 100644 --- a/tests/test_validate_readme_commands.py +++ b/tests/test_validate_readme_commands.py @@ -960,6 +960,39 @@ def test_cli_invocation_prints_pass_at_end(self) -> None: self.assertIn("expected terminaloutput blocks: 1", completed.stdout) self.assertTrue(completed.stdout.rstrip().endswith("PASS")) + def test_cli_invocation_prints_command_execution_banners(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = Path(temp_dir) + self._init_git_repo(repo_path) + readme_path = repo_path / "README.md" + readme_path.write_text( + textwrap.dedent( + """ + ```shell + printf 'hello\\n' + ``` + + ```terminaloutput + hello + ``` + """ + ).lstrip(), + encoding="utf-8", + ) + + completed = subprocess.run( + ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(repo_path)], + cwd=str(repo_path), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(completed.returncode, 0) + self.assertIn("BEGIN command #1", completed.stdout) + self.assertIn("Command #1 output:", completed.stdout) + self.assertIn("END command #1 (exit 0)", completed.stdout) + def test_main_resets_lab_changed_files_by_default(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: repo_path = Path(temp_dir) diff --git a/validate_readme_commands.py b/validate_readme_commands.py index a94d652..9b87e7c 100755 --- a/validate_readme_commands.py +++ b/validate_readme_commands.py @@ -274,9 +274,16 @@ def run_command_specs( for index, command_spec in enumerate(command_specs, start=1): if should_skip_command(command_spec.command): + print_command_skip_banner(index=index, command=command_spec.command, cwd=cwd) results.append(_build_result(index=index, command_spec=command_spec, cwd=cwd, skipped=True)) continue + print_command_execution_start( + index=index, + command=command_spec.command, + cwd=cwd, + expected_outputs=command_spec.expected_outputs, + ) try: completed = _run_command( command=command_spec.command, @@ -317,6 +324,7 @@ def run_command_specs( stdout=completed.stdout, stderr=completed.stderr, ) + print_command_execution_end(result) try: _assert_command_result(result) @@ -1408,6 +1416,42 @@ def print_command_mapping(command_specs: Sequence[CommandSpec]) -> None: if command_specs: print(separator) + +def print_command_execution_start( + *, + index: int, + command: str, + cwd: Path, + expected_outputs: Sequence[str], +) -> None: + separator = "=" * 72 + print(separator) + print(f"BEGIN command #{index}") + print(f"Working directory: {cwd}") + print(f"Expected terminaloutput blocks: {len(expected_outputs)}") + print("Command:") + _print_indented_block(command) + print("-" * 72) + print(f"Command #{index} output:") + print("-" * 72) + + +def print_command_execution_end(result: CommandResult) -> None: + print("-" * 72) + print(f"END command #{result.index} (exit {result.returncode})") + print("=" * 72) + + +def print_command_skip_banner(*, index: int, command: str, cwd: Path) -> None: + separator = "=" * 72 + print(separator) + print(f"SKIP command #{index}") + print(f"Working directory: {cwd}") + print("Command:") + _print_indented_block(command) + print("=" * 72) + + def _print_indented_block(content: str) -> None: for line in content.rstrip("\n").splitlines(): print(f" {line}") From c796cd1bdc4049a95264aa8745b3218ecf7aa8ed Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Mon, 25 May 2026 13:18:08 +0530 Subject: [PATCH 58/71] added debugging logs to bcc readme --- backward-compatibility-testing/README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/backward-compatibility-testing/README.md b/backward-compatibility-testing/README.md index c37226f..d9e2322 100644 --- a/backward-compatibility-testing/README.md +++ b/backward-compatibility-testing/README.md @@ -92,7 +92,12 @@ You now have an uncommitted change in a tracked contract file. Specmatic will co Alternatively, just run the following commands: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc "sed -i 's/version: 1.0.0/version: 1.1.0/' products.yaml; sed -i '/properties:/,/sku:/s/type: string/type: number/' products.yaml; sed -i '/^ sku:$/i\\ category:\\n type: string' products.yaml" +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc "sed -i 's/version: 1.0.0/version: 1.1.0/' products.yaml; sed -i '/properties:/,/sku:/s/type: string/type: number/' products.yaml; sed -i '/^ sku:$/i\\ category:\\n type: string' products.yaml; printf '--- products.yaml after part A ---\n'; cat products.yaml" +``` + +```terminaloutput +--- products.yaml after part A --- +version: 1.1.0 ``` ## Part B: Run the backward compatibility check @@ -100,6 +105,7 @@ Run: *Unix/Mac: ```shell +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc "printf '--- products.yaml before running bcc command ---\n'; cat products.yaml" docker run --rm \ --user "$(id -u):$(id -g)" \ -v "${PWD}/..:/workspace" \ @@ -172,7 +178,12 @@ Keep version `1.1.0`. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc 'sed -i "/properties:/,/sku:/s/type: number/type: string/" products.yaml' +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc 'sed -i "/properties:/,/sku:/s/type: number/type: string/" products.yaml; printf "--- products.yaml after part C ---\n"; cat products.yaml' +``` + +```terminaloutput +--- products.yaml after part C --- +version: 1.1.0 ``` ## Part D: Re-run the check @@ -180,6 +191,7 @@ Run the same command again: *Unix/Mac: ```shell +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc "printf '--- products.yaml before running 2nd bcc command ---\n'; cat products.yaml" docker run --rm \ --user "$(id -u):$(id -g)" \ -v "${PWD}/..:/workspace" \ From 7098832fe286ddc6aeecd4b241ef0c6ab6a3e3e1 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Thu, 28 May 2026 11:59:37 +0530 Subject: [PATCH 59/71] added debugging logs to bcc readme --- backward-compatibility-testing/README.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/backward-compatibility-testing/README.md b/backward-compatibility-testing/README.md index d9e2322..14dcd6e 100644 --- a/backward-compatibility-testing/README.md +++ b/backward-compatibility-testing/README.md @@ -105,21 +105,22 @@ Run: *Unix/Mac: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc "printf '--- products.yaml before running bcc command ---\n'; cat products.yaml" -docker run --rm \ +docker run --rm --entrypoint sh \ --user "$(id -u):$(id -g)" \ -v "${PWD}/..:/workspace" \ -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ - backward-compatibility-check \ - --base-branch origin/main \ - --target-path backward-compatibility-testing/products.yaml + -lc "printf '%s\n' '--- products.yaml before running bcc command ---'; cat backward-compatibility-testing/products.yaml; java -jar /usr/local/share/enterprise/enterprise.jar backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml; printf '%s\n' '--- products.yaml after running bcc command ---'; cat backward-compatibility-testing/products.yaml" ``` ```terminaloutput +--- products.yaml before running bcc command --- +version: 1.1.0 Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (INCOMPATIBLE) This spec contains breaking changes to the API +--- products.yaml after running bcc command --- +version: 1.1.0 ``` Windows PowerShell single-line: @@ -191,21 +192,22 @@ Run the same command again: *Unix/Mac: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc "printf '--- products.yaml before running 2nd bcc command ---\n'; cat products.yaml" -docker run --rm \ +docker run --rm --entrypoint sh \ --user "$(id -u):$(id -g)" \ -v "${PWD}/..:/workspace" \ -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ - backward-compatibility-check \ - --base-branch origin/main \ - --target-path backward-compatibility-testing/products.yaml + -lc "printf '%s\n' '--- products.yaml before running 2nd bcc command ---'; cat backward-compatibility-testing/products.yaml; java -jar /usr/local/share/enterprise/enterprise.jar backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml; printf '%s\n' '--- products.yaml after running 2nd bcc command ---'; cat backward-compatibility-testing/products.yaml" ``` ```terminaloutput +--- products.yaml before running 2nd bcc command --- +version: 1.1.0 Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (COMPATIBLE) The spec is backward compatible with the corresponding spec from origin/main +--- products.yaml after running 2nd bcc command --- +version: 1.1.0 ``` Windows PowerShell single-line: From 72f362d1c892c71002be3a1fda34c9a36d88ac41 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sat, 30 May 2026 20:49:16 +0530 Subject: [PATCH 60/71] removed all the debug logs --- backward-compatibility-testing/README.md | 34 +++++++----------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/backward-compatibility-testing/README.md b/backward-compatibility-testing/README.md index 14dcd6e..c37226f 100644 --- a/backward-compatibility-testing/README.md +++ b/backward-compatibility-testing/README.md @@ -92,12 +92,7 @@ You now have an uncommitted change in a tracked contract file. Specmatic will co Alternatively, just run the following commands: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc "sed -i 's/version: 1.0.0/version: 1.1.0/' products.yaml; sed -i '/properties:/,/sku:/s/type: string/type: number/' products.yaml; sed -i '/^ sku:$/i\\ category:\\n type: string' products.yaml; printf '--- products.yaml after part A ---\n'; cat products.yaml" -``` - -```terminaloutput ---- products.yaml after part A --- -version: 1.1.0 +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc "sed -i 's/version: 1.0.0/version: 1.1.0/' products.yaml; sed -i '/properties:/,/sku:/s/type: string/type: number/' products.yaml; sed -i '/^ sku:$/i\\ category:\\n type: string' products.yaml" ``` ## Part B: Run the backward compatibility check @@ -105,22 +100,20 @@ Run: *Unix/Mac: ```shell -docker run --rm --entrypoint sh \ +docker run --rm \ --user "$(id -u):$(id -g)" \ -v "${PWD}/..:/workspace" \ -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ - -lc "printf '%s\n' '--- products.yaml before running bcc command ---'; cat backward-compatibility-testing/products.yaml; java -jar /usr/local/share/enterprise/enterprise.jar backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml; printf '%s\n' '--- products.yaml after running bcc command ---'; cat backward-compatibility-testing/products.yaml" + backward-compatibility-check \ + --base-branch origin/main \ + --target-path backward-compatibility-testing/products.yaml ``` ```terminaloutput ---- products.yaml before running bcc command --- -version: 1.1.0 Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (INCOMPATIBLE) This spec contains breaking changes to the API ---- products.yaml after running bcc command --- -version: 1.1.0 ``` Windows PowerShell single-line: @@ -179,12 +172,7 @@ Keep version `1.1.0`. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc 'sed -i "/properties:/,/sku:/s/type: number/type: string/" products.yaml; printf "--- products.yaml after part C ---\n"; cat products.yaml' -``` - -```terminaloutput ---- products.yaml after part C --- -version: 1.1.0 +docker run --rm --entrypoint sh -v "${PWD}:/workspace" -w /workspace specmatic/enterprise:latest -lc 'sed -i "/properties:/,/sku:/s/type: number/type: string/" products.yaml' ``` ## Part D: Re-run the check @@ -192,22 +180,20 @@ Run the same command again: *Unix/Mac: ```shell -docker run --rm --entrypoint sh \ +docker run --rm \ --user "$(id -u):$(id -g)" \ -v "${PWD}/..:/workspace" \ -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ -w /workspace \ specmatic/enterprise:latest \ - -lc "printf '%s\n' '--- products.yaml before running 2nd bcc command ---'; cat backward-compatibility-testing/products.yaml; java -jar /usr/local/share/enterprise/enterprise.jar backward-compatibility-check --base-branch origin/main --target-path backward-compatibility-testing/products.yaml; printf '%s\n' '--- products.yaml after running 2nd bcc command ---'; cat backward-compatibility-testing/products.yaml" + backward-compatibility-check \ + --base-branch origin/main \ + --target-path backward-compatibility-testing/products.yaml ``` ```terminaloutput ---- products.yaml before running 2nd bcc command --- -version: 1.1.0 Verdict for spec /workspace/backward-compatibility-testing/products.yaml: (COMPATIBLE) The spec is backward compatible with the corresponding spec from origin/main ---- products.yaml after running 2nd bcc command --- -version: 1.1.0 ``` Windows PowerShell single-line: From cf1396a841643ddd447d8152f239e5440a0ccfc5 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sat, 30 May 2026 21:20:24 +0530 Subject: [PATCH 61/71] cleanup --- backward-compatibility-testing/README.md | 9 +++--- ...plication_json_201_application_json_1.json | 0 ...plication_json_201_application_json_1.json | 0 external-examples/README.md | 2 +- kafka-avro/docker-config/avro/NewOrders.avsc | 28 +++++++++++++++---- .../fixed-avro/NewOrders.fixed.avsc | 10 +++++-- 6 files changed, 37 insertions(+), 12 deletions(-) rename external-examples/{external-examples-generated => .backup}/createOrder_application_json_201_application_json_1.json (100%) rename external-examples/{external-examples-generated => .backup}/createProduct_application_json_201_application_json_1.json (100%) diff --git a/backward-compatibility-testing/README.md b/backward-compatibility-testing/README.md index c37226f..68f9b57 100644 --- a/backward-compatibility-testing/README.md +++ b/backward-compatibility-testing/README.md @@ -139,16 +139,17 @@ The Incompatibility Report: This is number in the new specification response but string in the old specification ``` -Why this fails: -- Adding optional `category` is safe. -- Changing `name` from `string` to `number` is a breaking change for existing consumers. - Why the command is structured this way: - `-v "${PWD}/..:/workspace"` mounts the `labs` repository root, not just this lab folder, so Specmatic can access the git repository metadata. - `--user "$(id -u):$(id -g)"` runs the container as your host user, which avoids git ownership issues when the mounted repository is inspected inside the container. - `--base-branch origin/main` tells Specmatic which tracked baseline to compare against. - `--target-path backward-compatibility-testing/products.yaml` tells Specmatic to compare the working tree version of this file with the tracked version on `origin/main`. + +Why this fails: +- Adding optional `category` is safe. +- Changing `name` from `string` to `number` is a breaking change for existing consumers. + ## Part C: Fix the contract Open `products.yaml`. diff --git a/external-examples/external-examples-generated/createOrder_application_json_201_application_json_1.json b/external-examples/.backup/createOrder_application_json_201_application_json_1.json similarity index 100% rename from external-examples/external-examples-generated/createOrder_application_json_201_application_json_1.json rename to external-examples/.backup/createOrder_application_json_201_application_json_1.json diff --git a/external-examples/external-examples-generated/createProduct_application_json_201_application_json_1.json b/external-examples/.backup/createProduct_application_json_201_application_json_1.json similarity index 100% rename from external-examples/external-examples-generated/createProduct_application_json_201_application_json_1.json rename to external-examples/.backup/createProduct_application_json_201_application_json_1.json diff --git a/external-examples/README.md b/external-examples/README.md index 5767478..7f5a581 100644 --- a/external-examples/README.md +++ b/external-examples/README.md @@ -114,7 +114,7 @@ Still in Studio, generate examples for: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp external-examples-generated/* examples/' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp .backup/* examples/' ``` ### Final Phase diff --git a/kafka-avro/docker-config/avro/NewOrders.avsc b/kafka-avro/docker-config/avro/NewOrders.avsc index 86cbc92..99743a8 100644 --- a/kafka-avro/docker-config/avro/NewOrders.avsc +++ b/kafka-avro/docker-config/avro/NewOrders.avsc @@ -5,7 +5,9 @@ "fields": [ { "name": "id", - "type": "int" + "type": "int", + "x-minimum": 1, + "x-maximum": 100 }, { "name": "orderItems", @@ -15,10 +17,26 @@ "type": "record", "name": "Item", "fields": [ - { "name": "id", "type": "int" }, - { "name": "name", "type": "string" }, - { "name": "quantity", "type": "int" }, - { "name": "price", "type": "int" } + { + "name": "id", + "type": "int" + }, + { + "name": "name", + "type": "string", + "x-minLength": 2, + "x-maxLength": 10, + "x-regex": "^[A-Za-z]{2,10}$" + }, + { + "name": "quantity", + "type": "int" + }, + { + "name": "price", + "type": "int", + "x-minimum": 1000 + } ] } } diff --git a/kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc b/kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc index 7de9390..99743a8 100644 --- a/kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc +++ b/kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc @@ -17,7 +17,10 @@ "type": "record", "name": "Item", "fields": [ - { "name": "id", "type": "int" }, + { + "name": "id", + "type": "int" + }, { "name": "name", "type": "string", @@ -25,7 +28,10 @@ "x-maxLength": 10, "x-regex": "^[A-Za-z]{2,10}$" }, - { "name": "quantity", "type": "int" }, + { + "name": "quantity", + "type": "int" + }, { "name": "price", "type": "int", From bdb9b0a596de17c94b8aa2946dc794287dd29f8f Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sat, 30 May 2026 21:22:27 +0530 Subject: [PATCH 62/71] reverted accidental push with fixed avro schema file --- kafka-avro/docker-config/avro/NewOrders.avsc | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/kafka-avro/docker-config/avro/NewOrders.avsc b/kafka-avro/docker-config/avro/NewOrders.avsc index 99743a8..80d92f4 100644 --- a/kafka-avro/docker-config/avro/NewOrders.avsc +++ b/kafka-avro/docker-config/avro/NewOrders.avsc @@ -5,9 +5,7 @@ "fields": [ { "name": "id", - "type": "int", - "x-minimum": 1, - "x-maximum": 100 + "type": "int" }, { "name": "orderItems", @@ -23,10 +21,7 @@ }, { "name": "name", - "type": "string", - "x-minLength": 2, - "x-maxLength": 10, - "x-regex": "^[A-Za-z]{2,10}$" + "type": "string" }, { "name": "quantity", @@ -34,8 +29,7 @@ }, { "name": "price", - "type": "int", - "x-minimum": 1000 + "type": "int" } ] } From 648adc32424d340c895894add587f781c8d7c0dd Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sat, 30 May 2026 21:54:53 +0530 Subject: [PATCH 63/71] kafka-avro - instead of coping avro file content, using sed command --- kafka-avro/README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/kafka-avro/README.md b/kafka-avro/README.md index 9c7b7ca..75cd58d 100644 --- a/kafka-avro/README.md +++ b/kafka-avro/README.md @@ -200,7 +200,15 @@ Replace `docker-config/avro/WipOrders.avsc` with: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'cp docker-config/fixed-avro/NewOrders.fixed.avsc docker-config/avro/NewOrders.avsc && cp docker-config/fixed-avro/WipOrders.fixed.avsc docker-config/avro/WipOrders.avsc' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc "sed -i -e '/^ \"name\": \"id\",$/{n;s/^ \"type\": \"int\"$/&,\\ + \"x-minimum\": 1,\\ + \"x-maximum\": 100/;}' /usr/src/app/docker-config/avro/WipOrders.avsc && sed -i -e '/^ \"name\": \"id\",$/{n;s/^ \"type\": \"int\"$/&,\\ + \"x-minimum\": 1,\\ + \"x-maximum\": 100/;}' -e '/^ \"name\": \"name\",$/{n;s/^ \"type\": \"string\"$/&,\\ + \"x-minLength\": 2,\\ + \"x-maxLength\": 10,\\ + \"x-regex\": \"^[A-Za-z]{2,10}$\"/;}' -e '/^ \"name\": \"price\",$/{n;s/^ \"type\": \"int\"$/&,\\ + \"x-minimum\": 1000/;}' /usr/src/app/docker-config/avro/NewOrders.avsc" ``` ### Step 2: Update the examples @@ -277,8 +285,7 @@ Replace `api-specs/order-service-async-avro-v3_0_0_examples/PLACE_MACBOOK_ORDER. Alternatively, just run the following commands: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc "sed -i 's/\"id\": 101/\"id\": 1/; s/iPhone 14 Pro Max/iPhone/; s/\"price\": 500.00/\"price\": 5000/; s/exact:101/exact:1/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_IPHONE_ORDER.json" -docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc "sed -i 's/\"id\": 102/\"id\": 2/; s/Macbook Mini Pro M5/Macbook/; s/\"price\": 600.00/\"price\": 6000/; s/exact:102/exact:2/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_MACBOOK_ORDER.json" +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc "sed -i 's/\"id\": 101/\"id\": 1/; s/iPhone 14 Pro Max/iPhone/; s/\"price\": 500.00/\"price\": 5000/; s/exact:101/exact:1/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_IPHONE_ORDER.json && sed -i 's/\"id\": 102/\"id\": 2/; s/Macbook Mini Pro M5/Macbook/; s/\"price\": 600.00/\"price\": 6000/; s/exact:102/exact:2/; s#\"status\": \".*\"#\"status\": \"\$match(exact:PROCESSING)\"#' api-specs/order-service-async-avro-v3_0_0_examples/PLACE_MACBOOK_ORDER.json" ``` ## Verify the fix From 261f9dd4887c4dd96b53304878b316aecd7d029f Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sat, 30 May 2026 21:59:07 +0530 Subject: [PATCH 64/71] kafka-avro - instead of coping avro file content, using sed command --- .../fixed-avro/NewOrders.fixed.avsc | 45 ------------------- .../fixed-avro/WipOrders.fixed.avsc | 21 --------- 2 files changed, 66 deletions(-) delete mode 100644 kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc delete mode 100644 kafka-avro/docker-config/fixed-avro/WipOrders.fixed.avsc diff --git a/kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc b/kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc deleted file mode 100644 index 99743a8..0000000 --- a/kafka-avro/docker-config/fixed-avro/NewOrders.fixed.avsc +++ /dev/null @@ -1,45 +0,0 @@ -{ - "type": "record", - "name": "OrderRequest", - "namespace": "order", - "fields": [ - { - "name": "id", - "type": "int", - "x-minimum": 1, - "x-maximum": 100 - }, - { - "name": "orderItems", - "type": { - "type": "array", - "items": { - "type": "record", - "name": "Item", - "fields": [ - { - "name": "id", - "type": "int" - }, - { - "name": "name", - "type": "string", - "x-minLength": 2, - "x-maxLength": 10, - "x-regex": "^[A-Za-z]{2,10}$" - }, - { - "name": "quantity", - "type": "int" - }, - { - "name": "price", - "type": "int", - "x-minimum": 1000 - } - ] - } - } - } - ] -} diff --git a/kafka-avro/docker-config/fixed-avro/WipOrders.fixed.avsc b/kafka-avro/docker-config/fixed-avro/WipOrders.fixed.avsc deleted file mode 100644 index c71aadd..0000000 --- a/kafka-avro/docker-config/fixed-avro/WipOrders.fixed.avsc +++ /dev/null @@ -1,21 +0,0 @@ -{ - "type": "record", - "name": "OrderToProcess", - "namespace": "order", - "fields": [ - { - "name": "id", - "type": "int", - "x-minimum": 1, - "x-maximum": 100 - }, - { - "name": "status", - "type": { - "type": "enum", - "name": "OrderStatus", - "symbols": ["PENDING", "PROCESSING", "COMPLETED", "CANCELLED"] - } - } - ] -} From c10edda35e94224e5a4146c64c95d81ca1729747 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sat, 30 May 2026 22:17:00 +0530 Subject: [PATCH 65/71] moved updated overlay file to .backup folder --- overlays/{ => .backup}/path-prefix.overlay.yaml | 0 overlays/README.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename overlays/{ => .backup}/path-prefix.overlay.yaml (100%) diff --git a/overlays/path-prefix.overlay.yaml b/overlays/.backup/path-prefix.overlay.yaml similarity index 100% rename from overlays/path-prefix.overlay.yaml rename to overlays/.backup/path-prefix.overlay.yaml diff --git a/overlays/README.md b/overlays/README.md index 5615f69..866197b 100644 --- a/overlays/README.md +++ b/overlays/README.md @@ -111,7 +111,7 @@ overlayFilePath: ./overlays/path-prefix.overlay.yaml Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp path-prefix.overlay.yaml overlays/path-prefix.overlay.yaml && sed -i "s/^# overlayFilePath:/ overlayFilePath:/" specmatic.yaml' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp .backup/path-prefix.overlay.yaml overlays/path-prefix.overlay.yaml && sed -i "s/^# overlayFilePath:/ overlayFilePath:/" specmatic.yaml' ``` ## Pass verification From a35f5ee801b986889c57700eec4ec4b8e8e4893c Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Sat, 30 May 2026 22:24:47 +0530 Subject: [PATCH 66/71] added missing validate command for Linux/Mac --- partial-examples/README.md | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/partial-examples/README.md b/partial-examples/README.md index f04970b..b324769 100644 --- a/partial-examples/README.md +++ b/partial-examples/README.md @@ -41,7 +41,7 @@ Some APIs may have a lot of transient, but mandatory fields which does not matte Validate the original incomplete examples and observe the intentional failures. -Test Run Cmd (Linux/Mac OSX) +Validate Command (Linux/Mac OSX) ```shell docker run --rm \ @@ -56,7 +56,7 @@ docker run --rm \ [FAIL] Examples: 0 passed and 3 failed out of 3 total ``` -Windows PowerShell single-line +Validate Command (Windows PowerShell) ```powershell docker run --rm -v "$($PWD.Path):/usr/src/app" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest validate @@ -114,9 +114,24 @@ sed -i "/\\\"type\\\": \\\"book\\\"/d" examples/test_find_available_products_boo ### Final Phase -Re-run validation with the Windows PowerShell command after the Studio fixes are saved. +Re-run validation after the Studio fixes are saved. -Windows PowerShell single-line +Validate Command (Linux/Mac OSX) + +```shell +docker run --rm \ + -v "${PWD}:/usr/src/app" \ + -v "${PWD}/../license.txt:/specmatic/specmatic-license.txt:ro" \ + specmatic/enterprise:latest \ + validate +``` + +```terminaloutput +[OK] Specification product_search_bff_v6.yaml: PASSED +[OK] Examples: 3 passed and 0 failed out of 3 total +``` + +Validate Command (Windows PowerShell) ```powershell docker run --rm -v "$($PWD.Path):/usr/src/app" -v "$((Resolve-Path ..\license.txt).Path):/specmatic/specmatic-license.txt:ro" specmatic/enterprise:latest validate From 9f304b44914d83e2aa58e5c3febc06f60a238392 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Thu, 11 Jun 2026 08:38:01 +0530 Subject: [PATCH 67/71] fixed alt commands --- .../pets_242_GET_200_1.json | 0 quick-start-mock/README.md | 95 ++++++++++++++++--- quick-start-mock/specs/service.yaml | 2 +- 3 files changed, 81 insertions(+), 16 deletions(-) rename quick-start-mock/{quick-start-mock-generated => .backup}/pets_242_GET_200_1.json (100%) diff --git a/quick-start-mock/quick-start-mock-generated/pets_242_GET_200_1.json b/quick-start-mock/.backup/pets_242_GET_200_1.json similarity index 100% rename from quick-start-mock/quick-start-mock-generated/pets_242_GET_200_1.json rename to quick-start-mock/.backup/pets_242_GET_200_1.json diff --git a/quick-start-mock/README.md b/quick-start-mock/README.md index a46cc89..ddb21b9 100644 --- a/quick-start-mock/README.md +++ b/quick-start-mock/README.md @@ -45,7 +45,7 @@ In real projects, consumer teams are often blocked because a dependency is late, Start only the consumer: ```shell -docker compose up -d consumer --build +docker compose up -d --wait consumer ``` Open [http://127.0.0.1:8081](http://127.0.0.1:8081). @@ -65,15 +65,14 @@ Why this fails: Alternatively, just run the following commands: ```shell -docker compose up -d --wait --wait-timeout 30 consumer --build -docker run --rm --network quick-start-mock_default --entrypoint sh specmatic/enterprise:latest -lc 'curl -s http://mock:9100/pets/1 || echo "Service unavailable"' +docker run --rm --network quick-start-mock_default --entrypoint sh specmatic/enterprise:latest -lc 'curl --fail --silent --show-error http://127.0.0.1:9100/pets/1 >/dev/null && exit 1 || echo "Request failed as expected"' ``` ## Part B: Start contract-generated mock (consumer unblocked) With consumer still running, run: ```shell -docker compose --profile mock up -d mock +docker compose --profile mock up -d --wait mock ``` Go back to consumer UI and click **Load Pet** again. @@ -93,11 +92,53 @@ Try additional IDs: Alternatively, just run the following commands: ```shell -docker compose --profile mock up -d --wait --wait-timeout 30 mock -docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/1 -docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/2 -docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/2 -docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -i -sS http://mock:9100/pets/abc +expected='{"id":1,"name":"Scooby","type":"GoldenRetriever","status":"Adopted"}' + +actual="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/1 | tr -d '\n ' )" + +[ "$actual" = "$expected" ] || { + echo "Unexpected payload: $actual" + exit 1 +} +``` + + +```shell +first="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://mock:9100/pets/2)" +second="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://mock:9100/pets/2)" + +first_body="$(printf '%s\n' "$first" | sed '$d')" +first_status="$(printf '%s\n' "$first" | tail -n 1)" + +second_body="$(printf '%s\n' "$second" | sed '$d')" +second_status="$(printf '%s\n' "$second" | tail -n 1)" + +[ "$first_status" = "200" ] || { + echo "First call expected 200, got $first_status" + exit 1 +} + +[ "$second_status" = "200" ] || { + echo "Second call expected 200, got $second_status" + exit 1 +} + +[ "$first_body" != "$second_body" ] || { + echo "Expected different response bodies, but both were:" + printf '%s\n' "$first_body" + exit 1 +} + +echo "Both calls returned 200 and different bodies" +``` + +```shell +status="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -o /dev/null -w '%{http_code}' http://mock:9100/pets/abc)" + +[ "$status" = "400" ] || { + echo "Expected 400, got $status" + exit 1 +} ``` ## Part C: Stop only the mock and observe fallback @@ -115,14 +156,14 @@ Expected output: Alternatively, just run the following commands: ```shell -docker run --rm --network quick-start-mock_default --entrypoint sh specmatic/enterprise:latest -lc 'curl -s http://mock:9100/pets/1 || echo "Service unavailable"' +docker run --rm --network quick-start-mock_default --entrypoint sh specmatic/enterprise:latest -lc 'curl --fail --silent --show-error http://127.0.0.1:9100/pets/1 >/dev/null && exit 1 || echo "Request failed as expected"' ``` ## Part D: Run mock from Studio and inspect traffic Start Studio in a new terminal: ```shell -docker compose --profile studio up -d studio +docker compose --profile studio up -d --wait studio ``` Open [http://127.0.0.1:9000/_specmatic/studio](http://127.0.0.1:9000/_specmatic/studio). @@ -153,10 +194,34 @@ To inspect mock traffic in Studio: Alternatively, just run the following commands: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'mkdir -p specs/service_examples && cp quick-start-mock-generated/pets_242_GET_200_1.json specs/service_examples/pets_242_GET_200_1.json' -docker compose --profile mock up -d --wait --wait-timeout 30 mock -docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/242 -docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/242 +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:latest -lc 'mkdir -p specs/service_examples && cp .backup/pets_242_GET_200_1.json specs/service_examples/pets_242_GET_200_1.json' +docker compose exec -T studio sh -lc 'specmatic mock /usr/src/app/specs/service.yaml --port 9100 >/tmp/specmatic-mock.log 2>&1 &' +first="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://studio:9100/pets/242)" +second="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://studio:9100/pets/242)" + +first_body="$(printf '%s\n' "$first" | sed '$d')" +first_status="$(printf '%s\n' "$first" | tail -n 1)" + +second_body="$(printf '%s\n' "$second" | sed '$d')" +second_status="$(printf '%s\n' "$second" | tail -n 1)" + +[ "$first_status" = "200" ] || { + echo "First call expected 200, got $first_status" + exit 1 +} + +[ "$second_status" = "200" ] || { + echo "Second call expected 200, got $second_status" + exit 1 +} + +[ "$first_body" = "$second_body" ] || { + echo "Expected same response bodies, but both were different:" + printf 'First: %s\n\nSecond: %s' "$first_body" "$second_body" + exit 1 +} + +echo "Both calls returned 200 and the same response body" ``` ## Pass criteria diff --git a/quick-start-mock/specs/service.yaml b/quick-start-mock/specs/service.yaml index 808a82a..7641441 100644 --- a/quick-start-mock/specs/service.yaml +++ b/quick-start-mock/specs/service.yaml @@ -40,5 +40,5 @@ paths: value: id: 1 name: Scooby - type: Golden Retriever + type: GoldenRetriever status: Adopted \ No newline at end of file From 8b248d8a70bec3c7e14f1044aa66934160949696 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Thu, 11 Jun 2026 08:41:27 +0530 Subject: [PATCH 68/71] fixed alt commands --- quick-start-mock/README.md | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/quick-start-mock/README.md b/quick-start-mock/README.md index ddb21b9..74ce16a 100644 --- a/quick-start-mock/README.md +++ b/quick-start-mock/README.md @@ -93,48 +93,32 @@ Alternatively, just run the following commands: ```shell expected='{"id":1,"name":"Scooby","type":"GoldenRetriever","status":"Adopted"}' - actual="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS http://mock:9100/pets/1 | tr -d '\n ' )" - [ "$actual" = "$expected" ] || { echo "Unexpected payload: $actual" exit 1 } -``` - - -```shell first="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://mock:9100/pets/2)" second="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://mock:9100/pets/2)" - first_body="$(printf '%s\n' "$first" | sed '$d')" first_status="$(printf '%s\n' "$first" | tail -n 1)" - second_body="$(printf '%s\n' "$second" | sed '$d')" second_status="$(printf '%s\n' "$second" | tail -n 1)" - [ "$first_status" = "200" ] || { echo "First call expected 200, got $first_status" exit 1 } - [ "$second_status" = "200" ] || { echo "Second call expected 200, got $second_status" exit 1 } - [ "$first_body" != "$second_body" ] || { echo "Expected different response bodies, but both were:" printf '%s\n' "$first_body" exit 1 } - echo "Both calls returned 200 and different bodies" -``` - -```shell status="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -o /dev/null -w '%{http_code}' http://mock:9100/pets/abc)" - [ "$status" = "400" ] || { echo "Expected 400, got $status" exit 1 @@ -198,29 +182,23 @@ docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise:la docker compose exec -T studio sh -lc 'specmatic mock /usr/src/app/specs/service.yaml --port 9100 >/tmp/specmatic-mock.log 2>&1 &' first="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://studio:9100/pets/242)" second="$(docker run --rm --network quick-start-mock_default --entrypoint curl specmatic/enterprise:latest -sS -w '\n%{http_code}' http://studio:9100/pets/242)" - first_body="$(printf '%s\n' "$first" | sed '$d')" first_status="$(printf '%s\n' "$first" | tail -n 1)" - second_body="$(printf '%s\n' "$second" | sed '$d')" second_status="$(printf '%s\n' "$second" | tail -n 1)" - [ "$first_status" = "200" ] || { echo "First call expected 200, got $first_status" exit 1 } - [ "$second_status" = "200" ] || { echo "Second call expected 200, got $second_status" exit 1 } - [ "$first_body" = "$second_body" ] || { echo "Expected same response bodies, but both were different:" printf 'First: %s\n\nSecond: %s' "$first_body" "$second_body" exit 1 } - echo "Both calls returned 200 and the same response body" ``` From 081b2016aeda0d5b97712447eafb073ccec90461 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Thu, 11 Jun 2026 08:50:50 +0530 Subject: [PATCH 69/71] response templating lab fixed --- .../{ => .backup}/test_find_available_products_book_200.json | 0 response-templating/README.md | 5 +++-- 2 files changed, 3 insertions(+), 2 deletions(-) rename response-templating/{ => .backup}/test_find_available_products_book_200.json (100%) diff --git a/response-templating/test_find_available_products_book_200.json b/response-templating/.backup/test_find_available_products_book_200.json similarity index 100% rename from response-templating/test_find_available_products_book_200.json rename to response-templating/.backup/test_find_available_products_book_200.json diff --git a/response-templating/README.md b/response-templating/README.md index 1a770d3..92dcd61 100644 --- a/response-templating/README.md +++ b/response-templating/README.md @@ -111,12 +111,13 @@ Configure lookup logic based on request query `type` so that: - for `type=gadget` return response values matching test expectation (`id=2`, `name=iPhone`, `type=gadget`, `inventory=500`, `createdOn` as valid date) Note: -- You can keep this as one lookup-driven mock example instead of creating duplicate mock files. +- You MUST keep this as one lookup-driven mock example instead of creating duplicate mock files. +- type's data type should be ProductType not string. type is an enum which allows only specific values and not any string. Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp test_find_available_products_book_200.json examples/mock/test_find_available_products_book_200.json' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp .backup/test_find_available_products_book_200.json examples/mock/test_find_available_products_book_200.json' ``` ## 4. Final verification From 3c9cd10ad62aab98ca67b43fd13f0fed47cb3647 Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Thu, 11 Jun 2026 08:54:18 +0530 Subject: [PATCH 70/71] schema design lab fixed --- schema-design/{ => .backup}/payments-request.yaml | 0 schema-design/README.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename schema-design/{ => .backup}/payments-request.yaml (100%) diff --git a/schema-design/payments-request.yaml b/schema-design/.backup/payments-request.yaml similarity index 100% rename from schema-design/payments-request.yaml rename to schema-design/.backup/payments-request.yaml diff --git a/schema-design/README.md b/schema-design/README.md index 95c675c..82f6124 100644 --- a/schema-design/README.md +++ b/schema-design/README.md @@ -123,7 +123,7 @@ Open `specs/payment-api.yaml` and update `PaymentRequest` to this shape: Alternatively, just run the following command: ```shell -docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp payments-request.yaml specs/payment-api.yaml' +docker run --rm --entrypoint sh -v "${PWD}:/usr/src/app" specmatic/enterprise -lc 'cp .backup/payments-request.yaml specs/payment-api.yaml' ``` ## 3. Re-run contract tests From 69f237e0eb03de30c6b7a3d13756f113f3b33ebb Mon Sep 17 00:00:00 2001 From: Naresh Jain Date: Thu, 11 Jun 2026 09:06:27 +0530 Subject: [PATCH 71/71] moved validate_readme_commands.py inside .github workflow folder --- .../test_validate_readme_commands.py | 20 +++++++++++-------- .github/workflows/validate-readmes.yaml | 9 ++++++--- .../workflows/validate_readme_commands.py | 19 +++++++++++++----- 3 files changed, 32 insertions(+), 16 deletions(-) rename {tests => .github/workflows}/test_validate_readme_commands.py (98%) rename validate_readme_commands.py => .github/workflows/validate_readme_commands.py (99%) mode change 100755 => 100644 diff --git a/tests/test_validate_readme_commands.py b/.github/workflows/test_validate_readme_commands.py similarity index 98% rename from tests/test_validate_readme_commands.py rename to .github/workflows/test_validate_readme_commands.py index ef6616b..c89dc2d 100644 --- a/tests/test_validate_readme_commands.py +++ b/.github/workflows/test_validate_readme_commands.py @@ -6,8 +6,11 @@ import unittest from contextlib import redirect_stderr, redirect_stdout from pathlib import Path +import sys from unittest.mock import patch +sys.path.insert(0, str(Path(__file__).resolve().parent)) + from validate_readme_commands import ( DEFAULT_LABS, CommandExecutionError, @@ -39,7 +42,8 @@ ) -ROOT_DIR = Path(__file__).resolve().parents[1] +ROOT_DIR = Path(__file__).resolve().parents[2] +SCRIPT_PATH = ROOT_DIR / ".github" / "workflows" / "validate_readme_commands.py" class GitRepoTestCase(unittest.TestCase): @@ -842,7 +846,7 @@ def test_cli_invocation_reports_failure(self) -> None: ) completed = subprocess.run( - ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(lab_path)], + ["python3", str(SCRIPT_PATH), str(lab_path)], cwd=str(ROOT_DIR), capture_output=True, text=True, @@ -877,7 +881,7 @@ def test_cli_invocation_prints_failure_summary(self) -> None: ) completed = subprocess.run( - ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(lab_path)], + ["python3", str(SCRIPT_PATH), str(lab_path)], cwd=str(ROOT_DIR), capture_output=True, text=True, @@ -912,7 +916,7 @@ def test_dry_run_does_not_execute_commands(self) -> None: completed = subprocess.run( [ "python3", - str(ROOT_DIR / "validate_readme_commands.py"), + str(SCRIPT_PATH), "--dry-run", str(lab_path), ], @@ -948,7 +952,7 @@ def test_cli_invocation_prints_pass_at_end(self) -> None: ) completed = subprocess.run( - ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(repo_path)], + ["python3", str(SCRIPT_PATH), str(repo_path)], cwd=str(repo_path), capture_output=True, text=True, @@ -981,7 +985,7 @@ def test_cli_invocation_prints_command_execution_banners(self) -> None: ) completed = subprocess.run( - ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(repo_path)], + ["python3", str(SCRIPT_PATH), str(repo_path)], cwd=str(repo_path), capture_output=True, text=True, @@ -1014,7 +1018,7 @@ def test_main_resets_lab_changed_files_by_default(self) -> None: ) completed = subprocess.run( - ["python3", str(ROOT_DIR / "validate_readme_commands.py"), str(repo_path)], + ["python3", str(SCRIPT_PATH), str(repo_path)], cwd=str(repo_path), capture_output=True, text=True, @@ -1049,7 +1053,7 @@ def test_main_skip_reset_preserves_lab_changed_files(self) -> None: completed = subprocess.run( [ "python3", - str(ROOT_DIR / "validate_readme_commands.py"), + str(SCRIPT_PATH), "--skip-reset", str(repo_path), ], diff --git a/.github/workflows/validate-readmes.yaml b/.github/workflows/validate-readmes.yaml index e6af66e..00f3a59 100644 --- a/.github/workflows/validate-readmes.yaml +++ b/.github/workflows/validate-readmes.yaml @@ -22,8 +22,11 @@ jobs: with: python-version: "3.11" + - name: Run validator unit tests + run: python3 -m unittest discover -s .github/workflows -p 'test_validate_readme_commands.py' -q + - name: Run shared preflight - run: python3 validate_readme_commands.py --preflight-only + run: python3 .github/workflows/validate_readme_commands.py --preflight-only validate-lab: name: Validate ${{ matrix.lab }} @@ -73,7 +76,7 @@ jobs: - name: Validate lab README run: | mkdir -p .artifacts/readme-validator - python3 validate_readme_commands.py "${{ matrix.lab }}" \ + python3 .github/workflows/validate_readme_commands.py "${{ matrix.lab }}" \ --result-json ".artifacts/readme-validator/${{ matrix.lab }}.json" - name: Upload lab validation result @@ -119,7 +122,7 @@ jobs: - name: Print consolidated report if: needs.preflight.result == 'success' run: | - python3 validate_readme_commands.py \ + python3 .github/workflows/validate_readme_commands.py \ --report-from ".artifacts/readme-validator" | tee .artifacts/readme-validator/summary.txt { echo '```text' diff --git a/validate_readme_commands.py b/.github/workflows/validate_readme_commands.py old mode 100755 new mode 100644 similarity index 99% rename from validate_readme_commands.py rename to .github/workflows/validate_readme_commands.py index 9b87e7c..a52d92f --- a/validate_readme_commands.py +++ b/.github/workflows/validate_readme_commands.py @@ -54,6 +54,17 @@ ] +def _find_repo_root_from_script() -> Path: + script_path = Path(__file__).resolve() + for candidate in script_path.parents: + if (candidate / ".git").exists(): + return candidate + return script_path.parents[2] + + +REPO_ROOT = _find_repo_root_from_script() + + @dataclass(frozen=True) class ValidationRunSummary: results: list["CommandResult"] @@ -990,8 +1001,7 @@ def resolve_readme_paths(readme_arg: str | None) -> list[Path]: if readme_arg: return [(Path(readme_arg).expanduser().resolve() / "README.md")] - repo_root = Path(__file__).resolve().parent - return [(repo_root / lab / "README.md").resolve() for lab in DEFAULT_LABS] + return [(REPO_ROOT / lab / "README.md").resolve() for lab in DEFAULT_LABS] def write_run_report(path: Path, report: RunReport) -> None: @@ -1188,8 +1198,7 @@ def run_preflight( return [] results: list[PreflightCheckResult] = [] - repo_root = Path(__file__).resolve().parent - license_path = repo_root / "license.txt" + license_path = REPO_ROOT / "license.txt" def _emit(result: PreflightCheckResult) -> None: results.append(result) @@ -1253,7 +1262,7 @@ def _emit(result: PreflightCheckResult) -> None: ) if docker_ready and license_path.is_file(): - _emit(_validate_specmatic_license(repo_root)) + _emit(_validate_specmatic_license(REPO_ROOT)) else: _emit( PreflightCheckResult(